Merge current master for native recovery qualification

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 15:37:16 -05:00
commit 5ea45f8d17
322 files changed
+21745 -1361

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
// Read trees only. The routing job fetches Git objects into a bare repository;
// no pull request code, hooks, filters or package scripts run here.
export function mergeDiff(merge, prHead, cwd = process.cwd()) {
assert.match(merge ?? "", /^[a-f0-9]{40}$/);
assert.match(prHead ?? "", /^[a-f0-9]{40}$/);
const git = (...args) => execFileSync("git", args, {
cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).trim();
const header = git("--no-replace-objects", "cat-file", "-p", merge).split("\n\n", 1)[0];
const parents = [...header.matchAll(/^parent ([a-f0-9]{40})$/gm)].map((match) => match[1]);
assert.equal(parents.length, 2, "Expected a GitHub pull request merge commit");
assert.equal(parents[1], prHead, "Merge commit does not contain the expected PR head");
const base = parents[0];
const changed = execFileSync("git", ["diff", "--no-ext-diff", "--no-textconv", "--no-renames",
"--name-only", "-z", base, merge, "--"], {
cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).split("\0").filter(Boolean).sort();
return { diff: { source: "git-merge-parent", base, head: merge, prHead }, changed };
}
+10
View File
@@ -0,0 +1,10 @@
// Audited prose inputs, shared by routing, Vitest selection and task hashing.
// Do not broaden this to all Markdown: docs/api and bundled skills are runtime inputs.
export function documentationInput(file) {
if (file === "doc/DEVELOPING.md" || file === "doc/DATABASE.md") return "guidance";
if (/^doc\/plans\/(?:[^/]+\/)*[^/]+\.md$/.test(file)
&& !file.split("/").includes("..")) return "historical";
return null;
}
export const productInputs = (files) => files.filter((file) => !documentationInput(file));
+18 -6
View File
@@ -2,6 +2,8 @@ import { appendFileSync, readFileSync, realpathSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { matchesGlob } from "node:path";
import { fileURLToPath } from "node:url";
import { documentationInput, productInputs } from "./ci-inputs.mjs";
import { mergeDiff } from "./ci-diff.mjs";
const releaseNote = /^releases\/(beta\/)?v[^/]+\.md$/;
const validPath = (file) => typeof file === "string" && file.length > 0
@@ -52,28 +54,33 @@ export function planTasks({ config, lanes, paths, mode = "enabled", forceReason
const changed = [...new Set(paths)];
if (!changed.length || changed.some((file) => !validPath(file))) forceReason ??= "Missing or invalid diff";
const matches = (file, globs) => globs.some((glob) => matchesGlob(file, glob));
const globalChanges = changed.filter((file) => matches(file, config.globalDependencies));
const sourceChanges = productInputs(changed);
const globalChanges = sourceChanges.filter((file) => matches(file, config.globalDependencies));
if (globalChanges.length) forceReason ??= `Global CI input changed: ${globalChanges[0]}`;
const unknown = changed.filter((file) => !releaseNote.test(file)
const unknown = sourceChanges.filter((file) => !releaseNote.test(file)
&& !matches(file, config.globalDependencies) && ![...inputs.values()].some((globs) => matches(file, globs)));
if (unknown.length) forceReason ??= `Unclassified path: ${unknown[0]}`;
if (mode === "full") forceReason ??= "Full CI requested";
const reasons = Object.fromEntries([...inputs].map(([task, globs]) => [task,
forceReason ? [forceReason] : [...new Set([...globalChanges, ...changed.filter((file) => matches(file, globs))])],
forceReason ? [forceReason] : [...new Set([...globalChanges, ...sourceChanges.filter((file) => matches(file, globs))])],
]));
// This uncached lane owns the repository-wide CLI guidance check.
if (reasons["ci:typecheck"]) reasons["ci:typecheck"].push(...changed.filter((file) => documentationInput(file) === "guidance"));
const jobs = Object.fromEntries(Object.entries(lanes).map(([job, entries]) => {
const affected = entries.filter((entry) => reasons[entry.task].length > 0);
const selected = mode === "shadow" ? entries : affected;
return [job, { run: selected.length > 0, matrix: { include: selected.length ? selected : entries },
selected: selected.length, proposed: affected.length, total: entries.length }];
}));
return { version: 1, mode, forceReason, changed, reasons, jobs };
return { version: 1, mode, forceReason, changed, documentation: changed.filter(documentationInput), reasons, jobs };
}
export function planSummary(plan) {
const rows = Object.entries(plan.jobs).map(([job, lane]) =>
`| ${job} | ${lane.proposed}/${lane.total} | ${lane.selected}/${lane.total} |`);
return ["### Incremental CI", "", `Mode: **${plan.mode}**${plan.forceReason ? ` — ${plan.forceReason}` : ""}`,
...(plan.diff ? [`Diff: \`${plan.diff.base}\` → \`${plan.diff.head}\` (complete Git tree comparison)`, ""] : []),
`Documentation inputs: ${(plan.documentation ?? []).join(", ") || "none"}`,
"", "| Suite | Affected jobs | Scheduled jobs |", "| --- | ---: | ---: |", ...rows,
"", "<details><summary>Selection reasons</summary>", "", "```json", JSON.stringify(plan.reasons, null, 2),
"```", "</details>", ""].join("\n");
@@ -83,9 +90,13 @@ if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.me
try {
const config = JSON.parse(readFileSync(process.env.CI_TASK_CONFIG ?? "turbo.json", "utf8"));
const lanes = JSON.parse(readFileSync(process.env.CI_TASK_MATRIX ?? ".github/ci-task-matrix.json", "utf8"));
let paths = [], forceReason = process.env.CI_FORCE_REASON || null;
let paths = [], diff, forceReason = process.env.CI_FORCE_REASON || null;
try {
if (process.env.CI_COMPARISON) {
if (process.env.CI_DIFF_REPOSITORY) {
const result = mergeDiff(process.env.EVENT_MERGE_SHA, process.env.EVENT_HEAD_SHA, process.env.CI_DIFF_REPOSITORY);
paths = result.changed;
diff = result.diff;
} else if (process.env.CI_COMPARISON) {
paths = comparisonPaths(JSON.parse(readFileSync(process.env.CI_COMPARISON, "utf8")));
} else {
const [base, head = "HEAD"] = process.argv.slice(2);
@@ -95,6 +106,7 @@ if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.me
}
} catch (error) { forceReason ??= `Diff unavailable: ${error.message}`; }
const plan = planTasks({ config, lanes, paths, mode: process.env.CI_INCREMENTAL_MODE || "enabled", forceReason });
if (diff) plan.diff = diff;
if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`);
else console.log(JSON.stringify(plan, null, 2));
if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, planSummary(plan));
+85 -6
View File
@@ -5,6 +5,7 @@ import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { comparisonPaths, planTasks } from "../ci-plan.mjs";
import { mergeDiff } from "../ci-diff.mjs";
const root = new URL("../../../", import.meta.url);
const config = JSON.parse(readFileSync(new URL("turbo.json", root), "utf8"));
@@ -18,7 +19,8 @@ test("trusted workflow fallbacks preserve every declared partition and command",
const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8");
for (const [job, entries] of Object.entries(lanes)) {
const block = workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
assert.ok(block.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job);
const normalized = block.replaceAll("(needs.test_selection.outputs.plan || needs.gate.outputs.plan)", "needs.gate.outputs.plan");
assert.ok(normalized.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job);
if (!block.includes(" matrix:")) { assert.equal(entries.length, 1); continue; }
const fallback = block.match(/matrix:.*\|\| '(.*)'\) \}\}/)?.[1];
assert.ok(fallback, `${job}: missing full fallback`);
@@ -115,7 +117,7 @@ test("cache-affecting definition changes force execution across successive PR pu
}
}
const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8");
assert.equal((workflow.match(/force:.*fromJSON\(needs.gate.outputs.plan\).forceReason != null/g) ?? []).length, 3);
assert.equal((workflow.match(/force:.*\.forceReason != null/g) ?? []).length, 3);
const action = readFileSync(new URL(".github/actions/run-ci-task/action.yml", root), "utf8");
assert.match(action, /CI_FORCE_TASKS: \$\{\{ inputs.force \}\}/);
assert.equal((action.match(/if:.*inputs.force != 'true'/g) ?? []).length, 2);
@@ -169,6 +171,9 @@ test("required aggregates reject omitted, failed and cancelled selected work", (
env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(resultPlan), CI_RESULTS: JSON.stringify(results) } });
assert.equal(run().status, 0);
for (const name of names) {
// Optional analysis can fail or be skipped; the unchanged task plan
// still has to pass. Selected-plan cases are exercised separately.
if (name === "test_selection") continue;
const previous = results[name].result;
for (const bad of previous === "success" ? ["failure", "cancelled", "skipped"] : ["success"]) {
results[name].result = bad;
@@ -185,6 +190,30 @@ test("the real routing step uses pinned trusted definitions and defaults to full
const shell = block.split(" run: |\n")[1].split("\n").map((line) => line.slice(10)).join("\n");
const directory = mkdtempSync(path.join(tmpdir(), "ci-routing-plan-"));
try {
const repository = path.join(directory, "source");
mkdirSync(repository);
const gitBin = execFileSync("which", ["git"], { encoding: "utf8" }).trim();
const git = (...args) => execFileSync(gitBin, args, { cwd: repository, encoding: "utf8" }).trim();
git("init", "-q", "-b", "main"); git("config", "user.name", "CI"); git("config", "user.email", "ci@example.invalid");
writeFileSync(path.join(repository, "base"), "base"); git("add", "."); git("commit", "-qm", "base");
git("checkout", "-qb", "pr");
mkdirSync(path.join(repository, "ui/src/components"), { recursive: true });
writeFileSync(path.join(repository, "ui/src/components/Sidebar.tsx"), "change");
git("add", "."); git("commit", "-qm", "PR");
const prHead = git("rev-parse", "HEAD");
git("checkout", "-q", "main");
writeFileSync(path.join(repository, "unrelated-master"), "already merged");
git("add", "."); git("commit", "-qm", "advance base");
const base = git("rev-parse", "HEAD");
git("merge", "--no-ff", "pr", "-m", "tested merge");
const merge = git("rev-parse", "HEAD");
writeFileSync(path.join(directory, "git"), `#!/usr/bin/env node
const {spawnSync}=require('node:child_process');
if(process.env.TEST_GIT_FAILURE==='true')process.exit(1);
const args=process.argv.slice(2).map(a=>a==='https://github.com/paperclipai/paperclip.git'?process.env.TEST_GIT_REPOSITORY:a);
const r=spawnSync(${JSON.stringify(gitBin)},args,{stdio:'inherit'});process.exit(r.status??1);
`);
chmodSync(path.join(directory, "git"), 0o755);
const gh = path.join(directory, "gh");
writeFileSync(gh, `#!/usr/bin/env node
const fs = require('node:fs'), path = require('node:path');
@@ -195,27 +224,77 @@ else if (endpoint?.includes('/contents/')) {
const [file, query] = endpoint.split('/contents/')[1].split('?');
if (query !== 'ref=' + 'a'.repeat(40)) process.exit(2);
process.stdout.write(fs.readFileSync(path.join(process.env.TEST_TRUSTED_ROOT, file)));
} else if (endpoint?.includes('/compare/')) console.log(JSON.stringify({ files: [{filename:'ui/src/components/Sidebar.tsx'}] }));
}
else process.exit(3);
`);
chmodSync(gh, 0o755);
for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) {
for (const overrides of [{}, { TEST_FETCH_FAILURE: "true" }, { TEST_GIT_FAILURE: "true" }, { PROFILE_CI: "true" }, { STACK_JSON: "{}" }, { FULL_CI: "false" }]) {
const output = path.join(directory, "output");
writeFileSync(output, "");
const result = spawnSync("bash", ["-c", shell], { encoding: "utf8", env: {
...process.env, PATH: `${directory}:${process.env.PATH}`, TEST_TRUSTED_ROOT: root.pathname,
RUNNER_TEMP: directory, FULL_CI: "true", PROFILE_CI: "false", FORCE_FULL_CI: "false",
CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_MERGE_SHA: "c".repeat(40),
CI_INCREMENTAL_MODE: "enabled", STACK_JSON: "null", EVENT_BASE_SHA: "b".repeat(40), EVENT_HEAD_SHA: prHead, EVENT_MERGE_SHA: merge,
TEST_GIT_REPOSITORY: repository,
GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: "", ...overrides,
} });
assert.equal(result.status, 0, result.stderr);
const value = readFileSync(output, "utf8").trim();
if (overrides.TEST_FETCH_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan=");
if (overrides.TEST_FETCH_FAILURE || overrides.TEST_GIT_FAILURE || overrides.FULL_CI === "false") assert.equal(value, "plan=");
else {
assert.notEqual(value, "plan=", `${JSON.stringify(overrides)}: ${result.stderr}`);
const resultPlan = JSON.parse(value.slice("plan=".length));
assert.equal(resultPlan.jobs.general_tests.selected, overrides.PROFILE_CI || overrides.STACK_JSON ? 19 : 2);
assert.equal(resultPlan.diff.base, base);
assert.deepEqual(resultPlan.changed, ["ui/src/components/Sidebar.tsx"]);
}
}
} finally { rmSync(directory, { recursive: true, force: true }); }
});
test("audited prose does not widen source selection, but guidance still runs its uncached guard", () => {
const source = "ui/src/components/Sidebar.tsx";
const sourcePlan = plan([source]);
for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
const mixed = plan([source, doc]);
assert.equal(mixed.forceReason, null);
assert.deepEqual(mixed.jobs, sourcePlan.jobs);
assert.deepEqual(mixed.documentation, [doc]);
}
const guidance = plan(["doc/DEVELOPING.md"]);
assert.deepEqual([...selectedTasks(guidance)], ["ci:typecheck"]);
assert.equal(config.tasks["//#ci:typecheck"].cache, false);
assert.deepEqual([...selectedTasks(plan(["doc/plans/2026-10-30-example.md"]))], []);
for (const file of ["docs/api/issues.md", "doc/unknown.md", "doc/plans/tool.sh", "skills/paperclip/SKILL.md",
"server/src/onboarding-assets/default/AGENTS.md", "tests/fixtures/example.md"]) {
assert.ok(selectedTasks(plan([source, file])).has("ci:server"), file);
}
});
test("complete merge trees handle more than 300 files, base advancement, renames and deletions", (t) => {
const directory = mkdtempSync(path.join(tmpdir(), "ci-merge-diff-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
const write = (file, value) => { mkdirSync(path.dirname(path.join(directory, file)), { recursive: true }); writeFileSync(path.join(directory, file), value); };
const commit = () => { git("add", "."); git("commit", "-qm", "fixture"); return git("rev-parse", "HEAD"); };
git("init", "-q", "-b", "main"); git("config", "user.name", "CI"); git("config", "user.email", "ci@example.invalid");
write("ui/src/deleted.ts", "old"); write("ui/src/renamed.ts", "rename"); const staleBase = commit();
git("checkout", "-qb", "pr");
for (let index = 0; index < 350; index++) write(`ui/src/component-${index}.ts`, "new");
git("rm", "ui/src/deleted.ts");
mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
git("mv", "ui/src/renamed.ts", "doc/plans/2026-10-30-moved.md");
const prHead = commit();
git("checkout", "-q", "main"); write(".github/workflows/unrelated.yml", "already merged CI change"); const base = commit();
git("merge", "--no-ff", "pr", "-m", "tested merge"); const head = git("rev-parse", "HEAD");
const result = mergeDiff(head, prHead, directory);
assert.equal(result.diff.base, base); assert.notEqual(result.diff.base, staleBase);
assert.equal(result.changed.length, 353);
assert.ok(result.changed.includes("ui/src/deleted.ts")); assert.ok(result.changed.includes("ui/src/renamed.ts"));
assert.ok(!result.changed.includes(".github/workflows/unrelated.yml"));
assert.equal(plan(result.changed).forceReason, null);
assert.throws(() => mergeDiff(head, base, directory), /expected PR head/);
assert.throws(() => mergeDiff(prHead, prHead, directory), /merge commit/);
assert.throws(() => mergeDiff("f".repeat(40), prHead, directory));
});
+448
View File
@@ -0,0 +1,448 @@
name: Chat feature TestOps
run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}
on:
workflow_dispatch:
inputs:
operation:
type: choice
options: [evaluate, bootstrap, launch-plan]
default: evaluate
description: Fixed evaluation, zero-result registration, or one TestOps plan launch
ALLURE_JOB_RUN_ID:
type: string
required: false
description: TestOps supplies its job-run identity; no test/source selector is accepted
ALLURE_USERNAME:
type: string
required: false
description: Reserved TestOps service metadata; never used as authorization
permissions:
contents: read
concurrency:
group: chat-feature-testops
cancel-in-progress: false
jobs:
authorize:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
sha: ${{ steps.guard.outputs.sha }}
operation: ${{ steps.guard.outputs.operation }}
triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }}
runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify fixed default-source dispatch and both actor identities
id: guard
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
NODE_OPTIONS: ''
NODE_PATH: ''
run: node tests/runner-e2e/testops-chat-dispatch.ts
select:
name: Validate TestOps selection or register zero-result setup
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
outputs:
artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
sha256: ${{ steps.selection.outputs.sha256 }}
steps:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Register the public job without an evaluation or a synthetic result
if: needs.authorize.outputs.operation == 'bootstrap'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap
- name: Validate the one real TestOps selection before any build or provider access
id: selection
if: needs.authorize.outputs.operation == 'evaluate'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection
- name: Start exactly the validated TestOps plan and exit without waiting for its child
if: needs.authorize.outputs.operation == 'launch-plan'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}
CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}
CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
GH_TOKEN: ${{ github.token }}
run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan
- name: Retain only closed metadata from zero-result setup
if: always() && needs.authorize.outputs.operation == 'bootstrap'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-setup-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json
${{ runner.temp }}/chat-testops-bootstrap-state/once.json
${{ runner.temp }}/chat-testops-bootstrap-state/plan.json
${{ runner.temp }}/chat-testops-bootstrap-state/job.json
${{ runner.temp }}/chat-testops-bootstrap-state/failure.json
retention-days: 30
if-no-files-found: error
- name: Retain the one plan-launch receipt and stop state
if: always() && needs.authorize.outputs.operation == 'launch-plan'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-plan-launch-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-launch-plan-state/once.json
${{ runner.temp }}/chat-testops-launch-plan-state/launch.json
${{ runner.temp }}/chat-testops-launch-plan-state/failure.json
retention-days: 30
if-no-files-found: error
- name: Retain only the token-free exact selection receipt
id: selection_upload
if: steps.selection.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-selection-${{ github.run_id }}-1
path: testops-selection.json
retention-days: 30
if-no-files-found: error
build:
name: Build the immutable shipped native candidate without model credentials
needs: [authorize, select]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 45
outputs:
artifact_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- name: Install only the source-reviewed graph without lifecycle scripts
run: |
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
pnpm install --frozen-lockfile --ignore-scripts
- name: Build the existing TypeScript and native runner owners
run: |
pnpm --filter @paperclipai/paperclip-runner build:typescript
pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Package source-bound outputs and resolved lock
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256
- name: Retain credential-free immutable outputs
id: bundle
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1
path: |
chat-testops-build.tar.gz
chat-testops-build.tar.gz.sha256
retention-days: 1
if-no-files-found: error
chat:
name: One real native OpenCode backlog request on AWS
needs: [authorize, select, build]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 32
environment: runner-e2e-private-paid
permissions:
contents: read
id-token: write
env: &retention
PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1
TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TARGET_SHA: ${{ needs.authorize.outputs.sha }}
MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Check out the same admitted source for trusted retention
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-testops-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- name: Verify immutable outputs and token-free TestOps selection
env:
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
(cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256)
tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE"
node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json
cmp chat-testops-lock.json chat-testops-worker-lock.json
cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store")
- name: Materialize and qualify pinned OpenCode runtime and browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Prepare private evidence with shipped-main authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only the exact private cell storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify strict private custody before the model key
env: &cell_session
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the fixed original request with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain exact original grade and private evidence, including failures
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and export only reviewed fixture results
needs: [authorize, select, chat]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 15
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies only
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store"
- name: Prepare exact private report and cell-read policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain private report writer
id: report_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Obtain exact private cell reader
id: read_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the exact private original and verify all inventories
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: |
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing normalized campaign without exposing original logs
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Retain the complete merged report privately even when its grade failed
id: private_report
if: always() && steps.report_credentials.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report
AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report
- name: Export only the bound normalized result, JUnit and marked PNGs
id: export
if: always() && steps.private_report.outcome == 'success'
env:
TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts
- name: Upload only the sanitized TestOps projection
if: always() && steps.export.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
upload:
name: Report the original outcome to its exact TestOps case
needs: [authorize, select, chat, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- *allure
- name: Verify and upload only the bound safe case outcome
id: allure_upload
env: &testops_reporter
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results
run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops
- name: Verify the actual TestOps result after upload processing
id: verify_upload
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *testops_reporter
run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops
- name: Retain only closed actual TestOps verification metadata
if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-verify-upload-state/verification.json
${{ runner.temp }}/chat-testops-verify-upload-state/failure.json
retention-days: 30
if-no-files-found: error
@@ -7,6 +7,8 @@ on:
- turbo.json
- .github/ci-task-matrix.json
- .github/scripts/ci-plan.mjs
- .github/scripts/ci-diff.mjs
- .github/scripts/ci-inputs.mjs
- .github/scripts/tests/ci-plan.test.mjs
- .github/actions/run-ci-task/**
- .github/workflows/ci-routing.yml
@@ -22,6 +24,50 @@ concurrency:
cancel-in-progress: true
jobs:
qualify_test_selection:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
package-manager-cache: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86
with:
version: 9.15.4
- uses: ./.github/actions/setup-ci-dependencies
- name: Prove conservative graphs, exact partitions, manifests and failure fallbacks
run: node --test scripts/ci/test-selection*.test.mjs scripts/ci/vitest-impact-report.integration.test.mjs
- name: Qualify selection against the real Paperclip workspace
env:
NODE_ENV: test
VITEST: 'true'
run: |
node --input-type=module <<'NODE'
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { execFileSync } from 'node:child_process';
import { planTasks } from './.github/scripts/ci-plan.mjs';
import { createTestSelection } from './scripts/ci/select-tests.mjs';
const config = JSON.parse(readFileSync('turbo.json', 'utf8'));
const lanes = JSON.parse(readFileSync('.github/ci-task-matrix.json', 'utf8'));
const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
for (const [file, project, related] of [
['ui/src/pages/Inbox.tsx', '@paperclipai/ui', 'ui/src/pages/Inbox.test.tsx'],
['server/src/middleware/auth.ts', '@paperclipai/server', 'server/src/__tests__/agent-auth-middleware.test.ts'],
]) {
const selected = await createTestSelection(planTasks({ config, lanes, paths: [file, "doc/DEVELOPING.md", "doc/plans/2026-10-30-ci-proof.md"] }), head);
const graph = selected.graphs.find(graph => graph.project === project);
assert.ok(graph.related.includes(related));
assert.ok(graph.alwaysRun.length > 0, 'Filesystem contracts must remain covered');
assert.ok(graph.selected.length < graph.inventory.length, 'The real workspace must support narrowing');
console.log(file, JSON.stringify(selected.plan.testSelection.stats, null, 2));
}
NODE
qualify:
runs-on: ubuntu-latest
timeout-minutes: 10
+9 -6
View File
@@ -350,24 +350,27 @@ jobs:
FORCE_FULL_CI: ${{ contains(github.event.pull_request.labels.*.name, 'full-ci') }}
CI_INCREMENTAL_MODE: ${{ vars.CI_INCREMENTAL_MODE || 'enabled' }}
STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }}
EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }}
EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
EVENT_MERGE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
fallback() { echo 'plan=' >> "$GITHUB_OUTPUT"; exit 0; }
[[ "$FULL_CI" == 'true' ]] || fallback
[[ "$EVENT_BASE_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback
[[ "$EVENT_HEAD_SHA" =~ ^[0-9a-f]{40}$ && "$EVENT_MERGE_SHA" =~ ^[0-9a-f]{40}$ ]] || fallback
trusted="$(gh api /repos/paperclipai/paperclip/git/ref/heads/master --jq .object.sha)" || fallback
[[ "$trusted" =~ ^[0-9a-f]{40}$ ]] || fallback
directory="$(mktemp -d)"
trap 'rm -rf "$directory"' EXIT
for file in .github/scripts/ci-plan.mjs .github/ci-task-matrix.json turbo.json; do
for file in .github/scripts/ci-plan.mjs .github/scripts/ci-inputs.mjs .github/scripts/ci-diff.mjs .github/ci-task-matrix.json turbo.json; do
gh api -H 'Accept: application/vnd.github.raw+json' \
"/repos/paperclipai/paperclip/contents/$file?ref=$trusted" \
> "$directory/${file##*/}" || fallback
done
gh api "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$EVENT_MERGE_SHA" \
> "$directory/comparison.json" || fallback
# Read the exact tested merge and both parents. The event's base SHA can
# lag behind its actual first parent. A bare tree diff has no API file cap.
git init --bare "$directory/repository.git" >/dev/null || fallback
git -C "$directory/repository.git" fetch --quiet --no-tags --filter=blob:none --depth=2 \
https://github.com/paperclipai/paperclip.git "$EVENT_MERGE_SHA" || fallback
if [[ "$PROFILE_CI" == 'true' || "$FORCE_FULL_CI" == 'true' ]]; then
export CI_FORCE_REASON='Full suite requested by label'
elif [[ "$STACK_JSON" != 'null' && -n "$STACK_JSON" ]]; then
@@ -375,5 +378,5 @@ jobs:
fi
export CI_TASK_CONFIG="$directory/turbo.json"
export CI_TASK_MATRIX="$directory/ci-task-matrix.json"
export CI_COMPARISON="$directory/comparison.json"
export CI_DIFF_REPOSITORY="$directory/repository.git"
node "$directory/ci-plan.mjs" || fallback
+105 -10
View File
@@ -24,6 +24,82 @@ jobs:
gate:
uses: paperclipai/paperclip/.github/workflows/ci-routing.yml@master
test_selection:
name: Select Vitest files
needs: [gate]
if: >-
needs.gate.outputs.full_ci == 'true' && needs.gate.outputs.plan != '' &&
fromJSON(needs.gate.outputs.plan).mode == 'enabled' &&
fromJSON(needs.gate.outputs.plan).forceReason == null &&
(vars.CI_VITEST_SELECTION == '' || vars.CI_VITEST_SELECTION == 'enabled')
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 10
# An unavailable optimization must not suppress the original test matrix.
continue-on-error: true
outputs:
plan: ${{ steps.publish.outputs.plan }}
env:
CI_PLAN: ${{ needs.gate.outputs.plan }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
fetch-depth: 0
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
package-manager-cache: false
- name: Check source-diff eligibility
id: eligible
run: |
if [[ -f scripts/ci/select-tests.mjs ]]; then
node scripts/ci/select-tests.mjs --eligible
else
echo 'enabled=false' >> "$GITHUB_OUTPUT"
fi
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86
if: steps.eligible.outputs.enabled == 'true'
with:
version: 9.15.4
- name: Restore dependencies for graph analysis
if: steps.eligible.outputs.enabled == 'true'
uses: ./.github/actions/setup-ci-dependencies
- name: Select files and rebalance test lanes
id: select
if: steps.eligible.outputs.enabled == 'true'
timeout-minutes: 5
run: node scripts/ci/select-tests.mjs
- name: Upload validated selection and audit
id: upload_selection
if: steps.select.outputs.selected == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: vitest-selection-${{ github.run_attempt }}
path: |
.ci-test-selection/manifest.json
.ci-test-selection/audit.json
include-hidden-files: true
if-no-files-found: error
retention-days: 7
# Publish only after upload succeeds. Missing/failed/timed-out selectors
# produce no plan, and dependent jobs use the original gate's matrix.
- name: Publish the executable plan
id: publish
if: steps.eligible.outputs.enabled == 'true'
env:
ARTIFACT_ID: ${{ steps.upload_selection.outputs.artifact-id }}
run: |
node --input-type=module <<'NODE'
import assert from 'node:assert/strict';
import { appendFileSync, readFileSync } from 'node:fs';
const plan = JSON.parse(readFileSync('.ci-test-selection/plan.json', 'utf8'));
if (plan.testSelection) {
assert.match(process.env.ARTIFACT_ID ?? '', /^[0-9]+$/);
plan.testSelection.artifactId = process.env.ARTIFACT_ID;
}
appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`);
NODE
policy:
needs: [gate]
runs-on: ${{ needs.gate.outputs.runner }}
@@ -358,15 +434,17 @@ jobs:
general_tests:
name: General tests (${{ matrix.group_label }})
needs: [gate]
if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.general_tests.run) }}
needs: [gate, test_selection]
if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.run) }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }}
matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }}
env:
PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }}
PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }}
PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }}
PAPERCLIP_CI_PROFILE_JOB: General tests (${{ matrix.group_label }})
@@ -376,6 +454,13 @@ jobs:
with:
persist-credentials: false
- name: Download selected test files
if: matrix.selection
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }}
path: .ci-test-selection
- name: Initialize CI profile
continue-on-error: true
if: env.PAPERCLIP_CI_PROFILE == 'true'
@@ -438,7 +523,7 @@ jobs:
task: ${{ matrix.task }}
args: ${{ matrix.shard_count && format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) || '[]' }}
partition: ${{ strategy.job-index || 0 }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }}
- name: Run grouped general test suites
if: hashFiles('turbo.json') == ''
@@ -521,14 +606,14 @@ jobs:
verify:
name: verify
if: ${{ always() }}
needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run]
needs: [gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
steps:
- name: Validate every planned suite result
env:
FULL_CI: ${{ needs.gate.outputs.full_ci }}
CI_PLAN: ${{ needs.gate.outputs.plan }}
CI_PLAN: ${{ needs.test_selection.outputs.plan || needs.gate.outputs.plan }}
CI_RESULTS: ${{ toJSON(needs) }}
run: |
node --input-type=module <<'NODE'
@@ -539,6 +624,7 @@ jobs:
assert.ok(['true', 'false'].includes(process.env.FULL_CI), 'Valid scope required');
const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null;
if (plan) assert.equal(plan.version, 1, 'Supported task plan required');
if (plan?.testSelection) assert.equal(results.test_selection.result, 'success', 'Published selection must succeed');
for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","verify_serialized_server","canary_dry_run"]) {
if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job);
const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run);
@@ -1021,15 +1107,17 @@ jobs:
verify_serialized_server:
name: Verify serialized server suites (${{ matrix.shard_label }})
needs: [gate]
if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.run) }}
needs: [gate, test_selection]
if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.run) }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }}
matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }}
env:
PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }}
PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }}
PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }}
PAPERCLIP_CI_PROFILE_JOB: Verify serialized server suites (${{ matrix.shard_label }})
@@ -1039,6 +1127,13 @@ jobs:
with:
persist-credentials: false
- name: Download selected test files
if: matrix.selection
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }}
path: .ci-test-selection
- name: Initialize CI profile
continue-on-error: true
if: env.PAPERCLIP_CI_PROFILE == 'true'
@@ -1101,7 +1196,7 @@ jobs:
task: ci:serialized
args: ${{ format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) }}
partition: ${{ strategy.job-index || 0 }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }}
force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }}
- name: Run serialized server test shard
if: hashFiles('turbo.json') == ''
+1
View File
@@ -99,6 +99,7 @@ test-results/
# Opt-in CI timing artifacts
.ci-profile/
.ci-test-selection/
# Content-addressed CI task results
.turbo/
+16
View File
@@ -71,6 +71,22 @@
"packages/paperclip-runner/src/**",
"packages/paperclip-runner/runner/**"
]
},
{
"path": "rules/acpx-profile-identity.mdc",
"description": "ACPX provider identity model: provider digests attest upstream releases, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.",
"scope": [
"packages/paperclip-runner/acpx-profiles.json",
"packages/paperclip-runner/cursor-contract.json",
"packages/paperclip-runner/cursor-distributions.json",
"packages/paperclip-runner/antigravity-distributions.json",
"packages/paperclip-runner/scripts/acpx-release-attestations.mjs",
"packages/paperclip-runner/scripts/generate-acpx-profiles.mjs",
"packages/paperclip-runner/scripts/contracts/**",
"packages/paperclip-runner/src/drivers/acpx/**",
"packages/paperclip-runner/src/providers/**",
"packages/paperclip-runner/protocol/**"
]
}
]
}
+4
View File
@@ -41,6 +41,10 @@ For `server/src/services/chat-channels.ts` and everything under
For `server/src/services/heartbeat.ts`, `server/src/services/heartbeat/`,
heartbeat-prefixed sibling services, and heartbeat regression tests, read
[`rules/heartbeat-execution-boundaries.mdc`](rules/heartbeat-execution-boundaries.mdc).
For ACPX provider profiles, their release attestations and pins, the provider-pack
contract registry, and ACPX driver and provider code under
`packages/paperclip-runner/`, read
[`rules/acpx-profile-identity.mdc`](rules/acpx-profile-identity.mdc).
See [`rules/README.md`](rules/README.md) for rule authoring and Greptile setup.
## 3. Repo Map
+2
View File
@@ -84,6 +84,8 @@ Background refreshes and history expansion keep the conversation and composer
mounted. A failed history request shows an inline retry without hiding readable
content. Full-history and classic views retain the bounded 15-second initial wait.
Standing agent chats open with the latest 20 contiguous messages and hydrate only their runs. Upward scrolling requests earlier pages; a fixed-height status slot says “Loading earlier messages…” while the page is pending. Preserve the logical row anchor and mounted composer through prepends and background refresh. Retry failed history inline without automatic retry loops. Answer receipts and created-task history use the same scrolling affordance.
Only the viewer's own human comments sit on the right in blue bubbles. Agents
and other humans sit on the left on the page surface, with their own name and
avatar above the message. Human identity comes from the company directory;
+72
View File
@@ -722,6 +722,60 @@ connection, before enabling the new server and UI.
- The repo keeps only the newest 5 snapshots. `generate` runs `prune:snapshots` afterwards to delete older ones. Drizzle only reads the newest snapshot, and each snapshot is a full copy of the schema (over 1 MB each). Older snapshots are still in git history.
- `packages/db/src/migration-snapshot-drift.test.ts` is the enforcement backstop. It repeats the diff that `generate` performs and fails when the newest snapshot no longer matches `packages/db/src/schema/`.
## Managed adapter settings
Cloud-managed instances keep adapter selection preferences in the private
`instance_settings` singleton `adapter-settings:v1`. Its `general` JSON contains
`version: 1`, a monotonic `revision`, and `disabledTypes`. It is separate from
the public `default` settings row and never appears in the instance settings API.
Each update locks this row in a transaction. The update and company activity
records commit together. Discovery and new-agent admission read the database
at their asynchronous operation boundaries; no polling or mutable settings cache
is used. Disabled adapters remain available to existing agents.
`PAPERCLIP_ADAPTERS` and `PAPERCLIP_ADAPTERS_FILE` declare the effective operator
adapter list. The declaration is parsed once for a process configuration and
validated at startup. It does not rewrite stored preferences. A conflicting
adapter toggle returns 403. Removing the declaration restores saved choices.
Operator file edits require a restart. Self-hosted instances continue to use
`adapter-settings.json` for saved preferences.
The server-only adapter settings service exposes `inspectStored()` and
`seedIfAbsent({ disabledTypes }, transaction?)` for migration tooling. Seeding
uses the unique singleton key, returns `{ seeded, current }`, and never replaces
an existing record. The caller can use the same transaction for its import
receipt. Before admission, `initializeAdapterSettings` checks the durable row.
When absent, it imports a valid legacy settings file once, retaining unknown
adapter types, or seeds empty preferences if no file exists. It rejects files
larger than 64 KiB, malformed input, and read failures. An existing durable row
wins over every stale file. Ordinary Cloud reads do not read a legacy file.
The focused PostgreSQL suite can use an owned disposable PostgreSQL database via
`PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL` when embedded PostgreSQL is
unavailable. It applies migrations and inserts test data, so never point this
variable at an existing application database.
## Cloud configuration migration receipt
The private `instance_settings` singleton `cloud-configuration-migration:v1`
records atomic import completion, semantic source digests, imported revisions,
and metadata-only external adapter blockers. It composes the adapter and
startup-config owners without replacing an existing authoritative row. Receipt
locking and the import transaction handle concurrent attempts and retries; no
local completion marker participates in conflict resolution.
Qualification records current configuration revisions and selected image package
evidence before normal plugin reconciliation. Image inspection does not hold
admin write locks; configuration and registration snapshots are rechecked in a
short transaction before the report is saved. A historical successful report
never grants current eligibility. External metadata blockers survive loss of the
legacy home and require a deliberate revision-checked recheck of a present valid
original source file. Package mismatches remain recorded after maintenance
reconciliation. See the [bounded migration and qualification procedure](cloud-configuration-migration.md)
for the stopped-writer prerequisite, commands, retry semantics, integrity limits,
and the explicit exclusion of secret bytes, whole-server zero downtime, and
volume removal.
## Cloud runtime identity singleton
The private `instance_settings` row whose singleton key is
@@ -1490,3 +1544,21 @@ It uses disposable embedded PostgreSQL by default. A disposable local database
named `review_wait_benchmark...` can be supplied through
`PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version.
DataRow bytes exclude response metadata and transport overhead.
## Object-backed run evidence
Optional segmented run logs and raw provider traces store their payloads and
range indexes directly in S3-compatible storage. Existing `heartbeat_runs` and
`provider_trace_records` contain backend/reference and lifecycle metadata;
there is no new payload table or per-segment SQL publication. Object manifests
use conditional writes to fence publication. Existing structured events retain
their database append contract. See [run evidence storage](run-evidence-storage.md).
## Cloud runtime configuration singleton
The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly
validated, versioned nonsecret configuration and revision. It is separate from
the ordinary instance settings API. Seed inserts preserve an existing winner;
replacements lock the row and reject stale revisions. Database/transaction
handles permit atomic migration receipts. Connection URLs, API keys and unknown
fields are excluded. See [field ownership and startup order](cloud-runtime-configuration.md).
+91 -5
View File
@@ -171,10 +171,12 @@ does not include the lower PRs' code.
For ordinary full-scope PRs, the routing job further selects affected suites
using `turbo.json` and `.github/ci-task-matrix.json`. It downloads the planner
and both definitions from one canonical master revision, without executing PR
code. The comparison covers the tested merge commit. Both rename paths count;
failed, empty, malformed, or potentially truncated comparisons select the full
suite. Unclassified files also select full CI. Stack verification retains the
and its helpers and definitions from one canonical master revision, without executing PR
code. It fetches the exact tested merge commit and its parents into a temporary
bare Git repository, validates the PR head, and compares the merge tree against
its first parent. The event's base SHA can be stale. Git tree comparisons have
no GitHub compare-API 300-file cap. Renames count as deletion plus addition;
failed, empty, or malformed comparisons select the full suite. Unclassified files also select full CI. Stack verification retains the
existing scope policy and runs all suites whenever that policy requires them.
Changes to global inputs, including the task graph, package scripts, workflows,
and execution helpers, force full execution and disable result-cache restore and
@@ -193,6 +195,69 @@ canary dry run.
The uncached typecheck job also runs the repository-wide CLI guidance safety
guard. It still scans UI changes when unrelated server shards are omitted.
The shared policy in `.github/scripts/ci-inputs.mjs` exempts only audited prose:
`doc/DEVELOPING.md`, `doc/DATABASE.md`, and Markdown under `doc/plans/`.
The first two still select the uncached typecheck/guidance lane; historical plans
need only the always-running policy checks. These inputs do not widen product
test selection or enter product task cache hashes. All other docs, bundled
skills, fixtures and unknown paths keep their existing conservative coverage.
A source change plus exempt prose remains graph eligible. When adding a reader
of exempt prose, update this policy as well as the task inputs.
For ordinary source PRs (including the audited prose inputs above), a separate `Select Vitest files` job refines the
general and serialized test lanes before their runners are allocated. It runs
on the gate-approved test fleet, loads the real Vitest projects, and follows
imports across workspace packages. Only projects with their own source changes
are narrowed; unchanged consumer projects retain their complete coarse lane
coverage without paying for another graph traversal. Filesystem, subprocess, network, CommonJS,
dynamic-loader, unresolved, and unapproved external-package consumers are
retained conservatively, including indirect consumers through local helpers.
The selector explicitly tracks setup/configuration dependencies as well as test
imports. The small pure-dependency allowlist lives in
`scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque
until reviewed. This is deliberately broader than `vitest related` alone.
For conservative retention, explicit complete local mock factories cut the
mocked implementation's I/O edges. Partial mocks, spies, automocks, mock bypasses,
and dynamic or escaped mocking APIs do not. Edits to a mocked module still use
Vitest's unmodified related-file graph. Non-isolated projects run in full.
Literal Vite globs use Vite's expanded dependency edges, including newly added
members. Consumers of generated or untracked modules remain in the conservative
set because the import graph does not describe their build inputs.
The server's temporary-home and Supertest setup has a reviewed, content-hashed
exception. It still follows local helper imports. Any change to the reviewed
contents disables the exception until reviewed again; setup dependency changes
invalidate the whole project. Unreviewed setup I/O uses the full task plan.
Selected files are repartitioned within their existing owners using the current
per-runner budgets and server timing weights. Small selections allocate fewer
jobs; empty lanes allocate none. Serialized files remain serialized, and chat's
line shards, native Runner lanes, Rust, builds, typechecking and E2E retain their
existing selection/execution rules. The job summary reports files and jobs
selected per lane, and the `vitest-selection` artifact records the manifest and
dependency audit. Selected jobs execute exact Vitest test specifications, so
similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards.
Missing/deleted/renamed source files, non-exempt non-source inputs, unrecognized graph inputs,
setup/config changes, failed dependency installs, graph errors and timeouts use
the complete original task plan. Source mutations during dependency setup also
disable narrowing. A published manifest must match the checked-out
merge SHA and parent diff, its plan hash and the expected shard/project ownership. Missing or
invalid manifests fail selected jobs and the required aggregate; they cannot
turn reduced matrices green. Failed-job retries download the original artifact
ID saved with the plan, even when GitHub increments the run attempt. Turbo hashes the selected file assignment through
`PAPERCLIP_CI_SELECTION_HASH`, so a passing subset cannot satisfy a full run or a
different selection.
Set the repository variable `CI_VITEST_SELECTION=off` to disable file selection
while keeping the existing task-level optimization. Empty or `enabled` enables
it; other values disable it. `full-ci`, profiling, and task-level `full`/`shadow`
modes also bypass file selection. Qualification uses real Vitest graphs and real
Turbo cache entries, including a filesystem contract omitted by the raw graph:
```sh
node --test scripts/ci/test-selection*.test.mjs scripts/ci/cache.integration.test.mjs
```
General and serialized test suites and the standalone Rust suite use Turbo's
content-addressed cache. An execution adapter maps selected lanes to the existing
package scripts and their declared inputs. It selects tracked and non-ignored
@@ -1488,7 +1553,9 @@ Paperclip also persists an empty `OPENAI_API_KEY` override for those agents so a
If the `codex` CLI is not installed or not on `PATH`, `codex_local` agent runs fail at execution time with a clear adapter error. Quota polling uses a short-lived `codex app-server` subprocess: when `codex` cannot be spawned, that provider reports `ok: false` in aggregated quota results and the API server keeps running (it must not exit on a missing binary).
Local adapters require their corresponding CLI/session setup on the machine running Paperclip. External adapters are installed through the adapter/plugin flow and should not require hardcoded imports in `server/` or `ui/`.
Local adapters require their corresponding CLI/session setup on the machine running Paperclip. Self-hosted instances can install external adapters through the adapter/plugin flow without hardcoded imports in `server/` or `ui/`.
Cloud-managed instances use only adapter implementations supplied by the release image. A tenant server token or the presence of `PAPERCLIP_MANAGED_CONFIG` enforces this boundary. Startup and UI parser requests do not read external adapter registrations or packages. Install, reinstall, reload, remove, and override operations return `403` with `adapter_install_platform_managed`, including for instance admins. Legacy `adapter-plugins.json` files stay untouched and do not change the built-in registry.
### Filesystem access boundary (2026-10-20)
@@ -3826,3 +3893,22 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`,
`PAPERCLIP_DOT_RUNNER_TEST_DATABASE_URL`. The interaction-service regression suite
also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those
URLs; the sleep suite truncates company data between tests.
## S3-backed run evidence
New logs and opted-in provider traces automatically use the instance's existing
S3 storage configuration, bucket, credentials and prefix, under `run-logs/` and
`provider-traces/`. No extra storage switches are needed. Local-disk instances
remain local. See [run evidence storage](run-evidence-storage.md) for required provider
capabilities, stable keys, operating limits, automatic history migration and qualification.
Do not remove a web volume until its complete retained evidence inventory and
the independent disposable-server requirements are satisfied.
## Cloud startup configuration
Cloud stores nonsecret startup choices in a private PostgreSQL singleton.
Bootstrap connection credentials stay in the supplied environment. A bounded
one-time config-file seed preserves existing choices, then later starts use the
database without loading home `.env` files or repairing worktree config. See
[Cloud runtime configuration](cloud-runtime-configuration.md) for field ownership,
secret handling, initialization order and migration interfaces.
+148
View File
@@ -0,0 +1,148 @@
# Cloud configuration migration and package qualification
This procedure covers saved adapter preferences, the allowlisted nonsecret
startup configuration, and selected bundled plugin packages. The result is a
point-in-time report. It does not authorize a deployment, guarantee zero
downtime, establish general Railway admission, or prove that a volume can be
removed. Secret and key bytes, database credentials, artifacts, logs, local
storage, workspaces, and plugin state outside PostgreSQL need their own storage
and replacement checks. This includes the decision signing key and local
encryption keys. Their byte authority is unchanged by this procedure.
Runtime environment and secret equality, plus an actual HTTP handoff between
application processes, remain separate rollout prerequisites. The package digest
does not establish those conditions.
## Migration prerequisite
Stop every legacy file writer before the first import. This includes old
application replicas and operator commands that can write configuration or
adapter settings. One maintenance window is sufficient. The importer cannot
make an old file writer participate in its database transaction. Keep those
writers stopped after import; later file edits do not update database authority.
Keep the normal database backup and recovery procedure for the instance. Use the
new release with the same managed environment and database connection. The
server runs migration before constructing configuration-dependent services.
For an explicit source-checkout invocation, use:
```sh
pnpm exec tsx server/src/qualify-cloud-configuration.ts --migrate
```
The command uses `DATABASE_URL` and the normal Cloud environment. It does not
accept credentials as arguments. It writes the receipt to PostgreSQL and prints
a JSON report; no report file or local completion marker is required. Exit code
0 means the requested configuration/package check passed, 2 means the report has
blockers, and 1 means the operation failed. A fresh installation can report a
missing plugin registration before normal boot installation completes. Run the
check again after that installation settles.
## Database authority and retry behavior
The private `instance_settings` singleton `cloud-configuration-migration:v1`
contains the versioned receipt. The transaction locks that row and composes the
adapter and startup-config owners' atomic seed methods. Completion and both
seeds commit together. A failed write or crash before commit leaves no completed
receipt. A retry uses committed database state and never replaces a later
settings revision. Earlier independently seeded owner records remain authoritative.
The receipt records source kind, a digest of the nonsecret semantic candidate,
the imported revision, and bounded legacy external-registration evidence.
An absent startup configuration remains an authoritative `config: null`; it is
distinct from an uninitialized database record. No raw secret values are copied.
Once the receipt exists, ordinary boots do not reread legacy source files.
A replacement home can be empty or absent without changing saved authority.
All legacy readers have a 64 KiB limit. They require regular files, reject
symlinks and FIFOs, and use bounded reads. External registration inspection reads
metadata only. It never loads a registered module or follows a registered package
path. A nonempty, invalid, or unreadable registration file creates a durable
blocker. Removing the old home does not clear that blocker.
To resolve an external-registration blocker, stop its writer, deliberately
remove the unsupported registration metadata at the original source location,
and leave a present valid empty JSON array file. Read the current receipt revision
from the report, then run:
```sh
pnpm exec tsx server/src/qualify-cloud-configuration.ts --recheck-external-revision 3
```
Use the actual current revision in place of `3`. A changed revision, a different
source location, or a missing/invalid source file rejects the recheck. Reusing
the same container path with an empty home is insufficient. This is an explicit
operator resolution; qualification never clears the blocker automatically.
## Current packages and overlap comparison
Every qualification reads current settings revisions and plugin registrations.
It resolves selected packages from this image with the same bundled package
identity rules as normal loading. It verifies package bytes before importing a
trusted image manifest, then compares that manifest with the approved database
registration. Missing packages, missing evidence, incompatible versions,
unselected registrations, and integrity failures prevent qualification. Legacy
package paths and npm discovery cannot supply missing image bytes.
Image inspection happens outside database write locks. A short transaction then
locks and rereads the current configuration and registration records. A change
during inspection produces a blocker and requires a new check. The report gives
`adapterRevision`, `configRevision`, `configurationDigest`, and
`packageEvidenceDigest`. A historical completed receipt is evidence, not current
authorization. Call qualification again for a current result.
The server records this report before ordinary bundled-plugin reconciliation.
It retains `lastPackageMismatch` after later successful checks, so reconciliation
cannot erase the evidence seen before an upgrade. The report does not block
normal maintenance upgrades or replace capability approval. After an upgrade
settles, its current packages can qualify normally.
For a same-package comparison, capture the current release's
`packageEvidenceDigest` and pass it explicitly on the replacement:
```sh
pnpm exec tsx server/src/qualify-cloud-configuration.ts --expected-package-evidence sha256:REPLACE_WITH_THE_64_HEX_DIGEST
```
`configurationPackagesEligible` describes current database/configuration/package
compatibility. `samePackageOverlapEligible` is true only when an explicit expected
digest also matches. There is no permanent first-release baseline that would
prevent future maintenance releases from establishing their own evidence.
The package evidence digest covers selected packages, not container image bytes.
Distribution packages reuse and reverify their catalog digest. Builtin packages
have a bounded inventory of physical file bytes, modes, and logical symlink
targets, including standalone pnpm dependency trees. With the selected catalog
root, inventory may follow links only into that provider, the catalog SDK,
`packages/shared`, and the current image's root `node_modules`. Reached pnpm
package units include their sibling dependency links; unrelated root packages
are not scanned. Canonical directory/file deduplication bounds cycles, and
logical root labels preserve the digest when identical image bytes relocate.
Dangling or escaping links, special files, more than 50,000 entries, depth above
32, metadata above 128 KiB, or more than 256 MiB per package fail verification.
The measured cached Cloud Daytona closure has about 22,253 entries and 130 MB;
that artifact predates this change and is not a complete current release proof.
Inventory work is synchronous and linear in selected package bytes. It runs only during startup before the application is constructed
or in the explicit operator process, with no idle polling. The report includes
`packageInspectionDurationMs`. Packages must remain immutable throughout a check.
## Verification scope
The focused PostgreSQL tests use two independent clients and separate home/image
roots. They import legacy settings, change saved revisions, remove the old roots,
reconstruct from the database, and start a real plugin worker from the replacement
image fixture. They also cover concurrent migration, commit failure and retry,
database failure, stale sources, durable external blockers, package mismatch,
maintenance reconciliation, and changes during manifest inspection.
The bundled-plugin registration suite also has an actual prebuilt Daytona case.
Build its standalone package with the normal SDK linker before running that case;
it is skipped when the provider build is absent. It qualifies the real linked
SDK/shared package graph and uses the production loader and worker manager for
credential-free initialize/health calls. The runtime-path tests also invoke the
production server bundler to verify its preserved module locations.
Set `PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL` only to an owned disposable
PostgreSQL database when embedded PostgreSQL is unavailable. The suite applies
migrations and writes/deletes test records. A complete release image, hosted
replacement, overlapping production processes, secret-byte migration, and volume
removal are not established by these local fixtures.
+118
View File
@@ -0,0 +1,118 @@
# Cloud runtime configuration
Cloud-managed servers store nonsecret startup and application choices in the
private `instance_settings` singleton `cloud-runtime-config:v1`. The row contains
`version: 1`, an increasing `revision`, and a strict allowlist of configuration
fields. It is separate from the normal instance settings row and is not exposed
by the instance settings API.
The canonical managed-instance predicate applies: a nonblank Cloud tenant token
or the presence of `PAPERCLIP_MANAGED_CONFIG`, including a blank value, selects
this configuration path. Invalid managed feature documents still retain their
existing validation at application startup.
## Startup and authority
1. The hosting environment supplies `DATABASE_URL` and, if needed,
`DATABASE_MIGRATION_URL`. Cloud requires a PostgreSQL URL and never falls back
to embedded PostgreSQL. Credentials, trust keys, provider configuration and
deployment wiring needed before the database connection remain supplied by
the environment or the existing secret system. This phase parses only the
database connection record. Runtime mode, host binding and authentication
checks wait until the saved snapshot has loaded.
2. After schema migration and database connection, startup reads the private
runtime configuration row. An existing invalid row or a database error stops
startup. Neither condition selects a legacy file or an old process snapshot.
3. If no row exists, startup imports the legacy config once. It uses
`PAPERCLIP_CONFIG` when supplied, otherwise the instance home config path.
It does not search working-directory ancestors. A truly absent file saves
an authoritative `config: null` marker. This retains the existing dynamic
environment/image defaults, including authentication and secret strictness.
The marker never enables a later file fallback. A present malformed, linked, nonregular, or oversized file
stops initialization. Reads and stored snapshots are limited to 64 KiB.
4. The seed uses an insert that does not replace an existing row. Concurrent
initializers read the committed winner. Later starts do not inspect the
legacy file, even when it is absent, corrupt, or changed.
5. Startup publishes a detached configuration snapshot, restores the signed
Cloud runtime identity, then initializes telemetry, secret-provider defaults,
authentication and application services. A saved telemetry opt-out therefore
takes effect before a telemetry client can be created.
Existing operator environment overlays keep precedence over saved values.
Reading an overlaid value does not save it back to the database. Credentials and
connection URLs are never copied into this JSONB row.
Cloud does not load instance or working-directory `.env` files and does not run
legacy worktree configuration repair. The hosting system must supply those
bootstrap variables directly. In particular, a database connection kept only
in `.env` cannot bootstrap this path. Self-hosted configuration and dotenv
behavior remain unchanged.
## Field ownership
The table lists every field consumed from the existing config file and the
remaining known schema fields. Environment-only `Config` fields, such as
heartbeat maintenance flags and feedback-export credentials, retain their
existing environment sources and are not frozen into the saved snapshot.
| Config file fields | Cloud owner |
| --- | --- |
| `server.deploymentMode`, `exposure`, `bind`, `customBindHost`, `host`, `port`, `allowedHostnames`, `serveUi` | Saved runtime configuration, with existing environment overlays. |
| `auth.baseUrlMode`, `publicBaseUrl`, `disableSignUp` | Saved runtime configuration, with environment overlays and the signed Cloud identity restoration. Saved URLs cannot contain userinfo, query parameters, or fragments. |
| `database.backup.enabled`, `intervalMinutes`, `retentionDays`, `dir` | Saved runtime configuration, with existing backup environment overlays. |
| `database.mode`, `connectionString` | Cloud bootstrap requires the environment `DATABASE_URL`. The legacy URL is never saved. |
| `database.embeddedPostgresDataDir`, `embeddedPostgresPort` | Local embedded-database settings. Cloud does not use embedded PostgreSQL or these paths. |
| `storage.provider`, `localDisk.baseDir`, `s3.bucket`, `s3.region`, `s3.endpoint`, `s3.prefix`, `s3.forcePathStyle` | Saved runtime configuration, with existing environment overlays. Saved endpoints cannot contain userinfo, query parameters, or fragments. |
| `secrets.provider`, `strictMode`, `localEncrypted.keyFilePath` | Saved nonsecret provider choice and locator, with existing environment overlays. No key bytes are imported or saved. |
| `telemetry.enabled` | Saved choice, passed to the existing telemetry initialization and opt-out policy before client creation. |
| `llm.provider` | Saved provider metadata. |
| `llm.apiKey` | Never saved. The existing OpenAI model-discovery consumer uses `OPENAI_API_KEY` first. A legacy OpenAI key requires that environment credential before import, or startup stops. Claude key metadata has no server consumer. |
| `logging.mode`, `logDir`, `updates.checkEnabled` | Nonsecret legacy/CLI metadata is preserved. Server logging continues to use its existing environment configuration. |
| `$meta` | CLI file-write metadata is not application authority. The compatibility view synthesizes valid metadata. |
| Unknown extension keys | No server config consumer exists. They are excluded from the strict saved allowlist, including any unknown credential fields. |
A saved path is a locator. This change does not make local storage objects,
backup bytes, logging files, or secret-key bytes durable. Their storage owners
and existing environment/secret mechanisms retain responsibility for those
bytes. Restart tests prove configuration restoration under a different home;
they do not prove durable file, secret, or backup storage.
The decision signing service still uses `PAPERCLIP_DECISION_SIGNING_SECRET` when
supplied and otherwise reads or creates its instance key file before the database
connection. The local encrypted secret provider still uses its environment key
or existing key-file mechanism. Those byte-storage fallbacks are outside this
configuration migration. Qualification must account for them separately; this
PR does not establish file-free secret storage on Cloud.
## Internal update and migration interface
`cloudRuntimeConfigService(db)` provides `inspectStored(handle?)`,
`seedIfAbsent(config, handle?)`, and `replace(config, expectedRevision, handle?)`.
Each method accepts a database or transaction handle. A migration can compose a
config seed, related settings and a receipt in one transaction.
Replacement validates the same allowlist, locks the singleton row, checks the
expected revision, and increases it. A stale revision fails. There is no new
public configuration endpoint or settings UI. Startup consumers use a snapshot;
a saved replacement takes effect on the next initialization/restart.
`readLegacyCloudRuntimeConfig(env?)` returns the validated nonsecret candidate
without writing it. `initializeCloudRuntimeConfig` performs the standalone
one-time seed and publishes its committed snapshot. Integrated migration
receipts and deployed Cloud qualification are separate consumers of this
interface.
## Verification
`server/src/__tests__/cloud-runtime-config.test.ts` exercises real PostgreSQL
clients, concurrent writes, transaction rollback, changed homes, a fresh Node
process, environment precedence, secret exclusion, invalid data and database
failure. It uses embedded PostgreSQL by default. A dedicated disposable database
can be selected with `PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL` when the host
cannot start embedded PostgreSQL. That database must be reserved for this test.
The test applies the current schema before use.
The startup orchestration controls in
`server-startup-feedback-export.test.ts` hold configuration initialization and
verify that telemetry, authentication and app construction wait for it. These
local controls do not claim a live Cloud rollout.
+27 -16
View File
@@ -580,22 +580,33 @@ transaction or an intentional freshness/authorization discard. These retained
homes are recovery evidence, not a background replay queue; persistent database
or lock failures still require operator intervention.
Quota OAuth replacement tokens are encrypted with the instance secrets master
key and fsynced under `<instance-root>/quota-credential-recovery/<company-id>/`
before vault, grant, or activity writes. Storage and encryption are checked before
exchanging a single-use refresh token. Failed saves retry without another OAuth
exchange; persistent failures keep the encrypted record for the next quota poll,
including after a restart. New OpenAI subscription runtimes serialize credential
reads with quota exchanges and recover a matching pending replacement before
materializing an auth home. Authentication-failure handling also recovers a
matching replacement before marking the grant invalid. A failed recovery save
defers these actions and leaves both the active grant and journal intact.
The journal is removed only after database commit (or
when an authorized newer credential makes it obsolete). Replay checks the grant,
connection, secret, and original credential fingerprint and cannot reactivate a
revoked grant or overwrite a reconnect. Back up this directory and the instance
secrets key with the instance data. Loss of durable storage while receiving a
provider token can still require reconnecting the account.
Server-owned quota refresh validates and locks the current grant and secret in
Postgres before exchanging a token. A failed database prerequisite prevents the
exchange. A returned replacement is saved directly into the existing secret
store. The same logical operation keeps it in memory for at most three save
attempts. A lost commit response is reconciled against the current saved value;
reconnect, revocation, secret-version and session-epoch changes win over a delayed
save. Save retries never repeat the OAuth exchange and no new recovery file is
written for this path, in cloud or local deployments.
If all save attempts fail, the request fails. If the server process then exits,
the unsaved replacement is lost. V1 accepts signing in again after the next failed
use; it does not report successful refresh or keep a second credential vault.
Remote Codex Runner tasks retain their private credential until secret writeback
and the lease's credential handoff commit. Provider completion follows that
handoff. Failed exports or database saves leave it pending for the existing
cleanup path, including a replacement web receiver. Its authority still checks
the company, grant, secret, version selector and session epoch.
Existing encrypted records in
`<instance-root>/quota-credential-recovery/<company-id>/` remain readable for
cutover settlement. Do not delete unresolved files or remove the old volume
before inventory and settlement. Legacy replay cannot reactivate a revoked grant
or overwrite a reconnect. Local Google Antigravity retirement still uses its
existing encrypted recovery record and private-home ownership; remote and warm
Google subscription sessions remain unsupported. This local path is not a cloud
fallback. Cloud cutover must settle any observed legacy credential files first.
## Advanced provider routing (2026-10-02)
+4 -1
View File
@@ -43,7 +43,10 @@ two natural-language task scenarios into local native Codex subscription and
OpenCode Product E2E cells. They
verify saving an unstarted task and delegating a saved announcement back to chat,
using ordinary production instructions, real browser messages and independent
state/provenance checks. The broader YAML catalog remains planning inventory.
state/provenance checks. A separately selected native OpenCode/DeepSeek1017
backlog cell measures TestOps integration acceptance with the same oracle and
original bounds. Its source registration is not an AWS or TestOps pass. The
broader YAML catalog remains planning inventory.
The explicit-only [Daytona restart continuity journey](../tests/runner-e2e/README.md)
extends the existing native restart oracle to a 300-second controller outage
+13 -1
View File
@@ -254,6 +254,17 @@ It does not extend an earlier execution attempt or include time waiting for
the background worker to run. Retries have separate roots. A check that finds
no eligible work can end without a restore child.
`task.response_presentation` owns recovery of an already committed native reply.
It starts a separate linked root because deferred presentation can run after the
execution attempt and workspace finalization have ended. Worker discovery for
a specific run and worker checks include candidate reads under
`response.resolve`. Each discovery or check owns its root; direct recovery and
its worker check share one root when nested. `response.transaction`
includes the recovery transaction; `response.transaction.entry` measures entry
into its callback, including connection acquisition. These spans retain only the
same hashed run ID and never capture reply content. A successful task status is
separate from the time the reply is saved.
These spans exist before provider dispatch, including setup failures. Preparation phases have
only the existing `paperclip.task.run.run_id` attribute: the first 12 hexadecimal
characters of the run UUID's SHA-256 hash. Failed operations set error status
@@ -641,7 +652,8 @@ Without an OTLP endpoint, tracing remains silent and does not export data.
The run link in Honeycomb matches `task.provider_session` for native runs, `task.run` for legacy adapters,
`task.execution_attempt` for server preparation and settlement, and
`task.workspace_finalization` for deferred native copyback. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the four root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged.
`task.workspace_finalization` for deferred native copyback and `task.response_presentation`
for committed native reply recovery. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the five root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged.
The canonical lifecycle is:
@@ -0,0 +1,89 @@
# Chat feature TestOps smoke — 2026-10-30
## Finish line
Create one dedicated runnable TestOps plan and obtain one genuine passing
`tasks.create-backlog` result through TestOps → GitHub Actions/AWS → native
OpenCode/OpenRouter `deepseek/deepseek-v4.1-flash-1017`. The result must prove
the saved task, assignment, backlog-at-creation state, no worker execution,
agent attribution, rendered link and reload persistence. Confirm cleanup,
safe screenshot/report access and the final TestOps result link.
This is a new integration-acceptance measurement. It preserves the original
244-scenario campaign, attempts, grades and existing Codex measurements.
Delegation and the other catalog scenarios are deferred.
## Source and execution contract
- Public selector: `chat-features.runner-opencode-deepseek-1017.local.create-backlog`.
- Existing YAML query, fixture and independent backlog oracle stay unchanged.
- One human query and one accounted controller work run; 120-second turns,
420-second attempt, 300-second settled-state observation, zero retries.
- Existing 500-cent company/agent policies and unknown setup usage remain
explicit. Morgan is fixture context and must not execute this backlog task.
- Only the existing protected `OPENROUTER_API_KEY` is required. Agent
configuration uses the normal encrypted secret-reference path.
- OpenCode 1017 is selected only for this cell and remains a qualification
candidate until the real run. Other suites retain their existing models.
## Ownership and integration
Campaign owns public catalog/profile calibration and these progress notes.
Core owns the public AWS workflow/execution adapter, one coherent public PR,
and the sole actual operation. Coordinator supplies the public TestOps
selection/reporting helper and the separate private credential sealer.
The public workflow owns the dedicated plan. Source peers precede publication.
The current Cloud TestOps adapter is scoped to `qa-basics` Product journeys;
the new chat plan must have its own exact plan/case mapping. Reuse normalized
result/check/provenance and existing packaged evidence. Do not upload private
traces, provider sessions, raw logs or unreviewed media as TestOps attachments.
## Current progress
The public patch adds only the OpenCode backlog selection while preserving
the two Codex cells. A genuine parent selector control failed; all 68 focused
oracle/catalog controls and the owning type check now pass. The expected
suite-size assertion was updated for the single added cell. Independent source
review passed and the nine-file profile slice is committed as `0a996033`.
No local app, database, native SDK,
provider, model, install or build qualification was run.
The separate protected credential bootstrap seals the existing reporting token
for the source-pinned public reporting environment. Delivery is complete without
exporting plaintext. The public adapter has 53 focused authority/export/workflow
controls and 39 API/CLI/result controls passing, plus owning typechecking and
workflow lint. Unchanged controls are reused after narrow source corrections.
The independent composition source peer passed.
The public infrastructure [PR 18322](https://github.com/paperclipai/paperclip/pull/18322)
is open. Its first normal CI run rejected the new workflow in the existing paid
workflow allowlist and reported a separate heartbeat scheduling assertion; both
failures remain recorded. The policy correction preserves the older workflow
guards and rejects unsafe new-route variants. The reporting token is configured
through the reviewed encrypted bootstrap.
Review also added a one-time initial administrator launch fence. A protected
expected GitHub run number and actual own-run metadata must match before any
TestOps mutation. REST catalog reconciliation preserves exact case aliases;
job wiring uses only the reviewed existing project GitHub integration. These
provider-free helper controls pass. Actual registration IDs, tenant API behavior,
provider execution and the TestOps result remain unqualified. Dedicated plan,
AWS evaluation and final result links remain pending. Source checks alone do not
satisfy the finish line.
The final source consolidation adds actual admission-wrapper filesystem coverage,
the existing reviewed lock normalization with frozen installs and a matched
build/worker receipt, plus same-job server-result readback after Allure upload.
Focused checks cover 41 admission controls, 31 lock/owner/workflow controls and
44 API/reporter controls; unchanged profile/oracle evidence is reused. This is
source verification. The plan and actual model result are still not created.
Current-main composition preserves the four existing Codex and production-qualified
OpenCode cells introduced by #18319. The TestOps cell has its own
`runner-opencode-deepseek-1017` profile, so the existing OpenCode model remains
unchanged. It uses the shipped v2 grader, queue accounting and `final-state.png`
capture. The new profile keeps the original query, one work run, one human message,
120-second turn, 420-second attempt, 300-second observation bound and zero retries.
The existing private session and trace-asset owners admit only this additional
exact selector; the frozen 244-case candidate-approval registry remains unchanged.
Source checks do not establish a TestOps registration or model outcome.
@@ -0,0 +1,567 @@
# Durable run logs and provider traces for disposable web servers
Status: implementation and PR qualification in progress. Runtime storage,
history migration, and canonical native source replay are implemented using the
instance's existing storage selection. Production-provider qualification remains
a rollout gate.
Owner: this Codex chat, through the reviewable PR. Date: 2026-10-30.
Revision: object-storage-first, following Dotta's operational correction on 2026-10-30.
Source revision inspected: `0c70cedce9a803bdc201324528abb35e423e7cd0`.
Parent effort: near-zero-downtime Railway deployment of the main web/API server.
## Goal and boundary
After the main Paperclip web server and its local filesystem disappear, a
replacement connected to the same Postgres, object storage, and secrets can
read all acknowledged run-log output and retained provider traces, then continue
recording the same remote execution without gaps or duplicate records.
The ordinary journey is: open a task's run, inspect its output, replace the web
server while its remote agent is working, and continue reading the run. An
authorized administrator can still inspect, reveal, download, and delete a
captured provider trace. Existing consumers of run output continue working.
This is the **run logs and provider traces** slice of the parent plan. Railway
still documents that volume-backed deployments cannot overlap, which makes
removing durable filesystem dependencies necessary. This slice alone does not
remove every volume dependency or prove zero-downtime HTTP deploys.
[Railway volume limitations](https://docs.railway.com/volumes/reference#caveats).
Firm requirements:
- All retained historical logs and unexpired traces survive migration. Dotta
explicitly selected this on 2026-10-30. Do not shorten existing retention.
- A successful durable append survives abrupt process and disk loss. An orderly
shutdown flush is useful but insufficient evidence.
- Preserve company/task authorization, redaction, stable log byte offsets,
trace integrity checks, opt-in capture, existing limits, expiry, and deletion.
- Store log and raw trace payloads directly in S3-compatible object storage.
Postgres holds small ownership/lifecycle/reference records only; no new
per-token or per-frame SQL, payload tables, or DB-to-S3 payload compaction.
Local files may be caches or unacknowledged buffers. Preserve local mode.
- Keep task results, usage settlement, and optional diagnostics independent.
- Preserve bounded work and existing sleep behavior.
- Reuse existing instance storage settings and per-stack bucket credentials.
New logs and opted-in traces follow S3 file storage automatically, with their
own folders beneath the existing prefix. Add no required configuration.
Dotta approved implementation on 2026-10-30. Delivery ends at a reviewable PR
with relevant checks and review findings resolved. Deployment, volume removal,
and production history migration require the parent effort's target and cutover.
The acceptance matrix below remains the full storage-slice contract; an
implemented primitive does not establish that every row has passed.
Deferred to the parent effort: managed agent files, workspace recovery,
accounting/credential journals, workspace-operation logs, readiness/draining,
global live-event fanout, scheduler ownership, volume detach, and fleet rollout.
These remain blockers to claiming the entire web server disposable. No changes
to first-party Telemetry or operator OpenTelemetry export are proposed. Raw
provider protocol traces here are a separate, restricted diagnostic store.
## Current implementation and implications
| Owner | Current behavior | Required change |
| --- | --- | --- |
| `server/src/services/run-log-store.ts` | Only `local_file`; local NDJSON append; best-effort final/in-flight S3 mirroring; local-first reads | Add an explicit external store with shared durable append and reads |
| `server/src/services/heartbeat/run-output.ts` | Redacts/compacts before append; process-local output sequence; progress persisted on a 60-second cadence | Preserve stable source identities; batch output to S3; distinguish live delivery from durable receipts |
| `server/src/services/heartbeat-run-events.ts` | Structured events already use atomic DB sequence allocation and native source deduplication | Preserve this authority; do not turn its event table into a duplicate blob store |
| `server/src/services/provider-trace-store.ts` | DB metadata points to local native and `.rehydration` sidecars; 24-hour retention and 64 MiB capture limit | Remote payload storage, recoverable capture cursors, durable expiry/deletion obligations |
| Runner trace sinks and `runnerd-codex-transport.ts` | Rust/TypeScript capture and interpretation producers use paths; TypeScript capture is bounded and fire-and-forget | Introduce a durable capture delivery boundary and preserve channel/frame lineage |
| `doc/testing-native-remote-continuity.md` | Sparse checkpoints deliberately omit streaming output; acknowledged display output may be lost before app projection | Qualify replay/retention through web replacement without restoring per-token DB checkpoint writes |
| Run readers | Heartbeat, issues, feedback, and customer-success inspection assume `local_file` | Resolve each persisted backend, including historical records, through one reader |
Do not infer cloud durability from `RunLogStore.append()` succeeding today, or
from a trace's local `acknowledgedDebugSequence`. Neither proves remote commit.
Generic stdout/stderr and raw traces cannot be reconstructed from
`heartbeat_run_events`. The implementation investigation found a narrower
surviving source: native control ingestion already commits the exact validated
PRP event before ACK. Its canonical `paperclip.prp.event` log export can replay
that existing source with an S3 cursor, without adding payload writes. This
replaces the tentative second Runner spool below for canonical native output.
## Operational decision
The original proposal used Postgres for payload chunks and then copied them to
S3. That choice was premature. It adds payload writes, WAL/replication/backup
bytes, index churn, deletion/vacuum work, and extra database connections to the
same database serving the control plane. Batching would reduce transaction
count but would not remove the byte load. No workload measurement justified it.
Use **S3 as the durable payload store from the first persisted batch**. Local
append files remain useful for buffering and cache speed; the web server's disk
cannot be their only acknowledged durable home. Existing S3 mirroring is a useful
starting point, but repeatedly uploading the entire growing file amplifies
network and storage work. Upload each newly sealed segment once instead.
| Location | Responsibility |
| --- | --- |
| Browser live stream / bounded server memory | Immediate progress display; not a durable receipt |
| Existing native control event source / Runner outbox | Existing committed native records survive ACK; unacknowledged events remain replayable at the Runner |
| S3-compatible storage | Immutable log/trace segments and bounded manifest/index pages |
| Postgres | Existing company/run ownership, backend/reference, final status and expiry/deletion state; no per-batch writes |
| Disposable web disk | Optional bounded buffer/read cache; never the only acknowledged copy |
Existing structured `heartbeat_run_events`, task results, usage, and controller
checkpoints retain their contracts. This is not a rewrite of those data paths.
### Cost model and the first gate
Let `A` be actively producing streams, `r` output records/second/stream, `b`
bytes/second/stream, `T` maximum dirty-buffer age, and `S` target segment bytes.
For a steady stream, approximate segment rate is
`A × max(1/T, b/S)`; record rate is `A × r`. Empty streams emit no segments.
Bursts, final flushes, retries, indexes, and trace channels add work and must be
counted separately. A stream is not necessarily an agent: raw tracing can add
multiple streams to one run.
Illustrative arithmetic, **not a fleet measurement or chosen production setting**:
100 continuously active streams at 10 records/second give 1,000 records/second.
If five-second flushing dominates, they produce 20 segments/second, or 72,000
segments/hour. With one small metadata publication transaction per segment,
that is still 20 transactions/second. A 30-second interval gives about 3.3
segments/second but retains more pending output. S3 index PUTs, GETs, retries,
and transfer costs are additional. Neither option is free.
The first gate measures real output sizes/rates and concurrency, then compares
candidate size/age thresholds (initial experiment: 256 KiB–1 MiB and 5–30
seconds, not defaults). Include quiet text, bursty tool output, raw tracing,
long runs, slow S3, and the expected concurrent-run envelope. Record:
- Incremental Postgres transactions, row updates, WAL bytes, connections, and
application p95 latency; no log/trace payload bytes inserted into Postgres.
- S3 PUT/GET/HEAD/LIST counts, index overhead, object count, stored bytes,
compression CPU, transfer volume, and upload/read p95 latency.
- Pending bytes, upload lag, read recovery time, and behavior at queue limits.
- A daily/monthly cost projection using the **actual configured provider's**
rates and observed workload/duty cycle; do not substitute AWS prices for an
S3-compatible vendor or assume 24-hour production.
- Zero added recurring SQL/network work after buffers and obligations settle.
Record a concrete operating envelope, request-rate/backlog limits, and latency
budget from this evidence before enabling the feature. If the batched metadata
load is material, evaluate an object-store manifest head with conditional writes
and lifecycle-only SQL. Choose one publication scheme for the release; do not
ship two unqualified distributed state machines. Do not return to DB payload
buffering as an automatic workaround for S3 latency or failure.
## Design
### 1. External backend and batched publication
Keep local mode and add a versioned S3-backed store. Inject the existing object
provider plus narrow metadata/ownership operations. Readers dispatch from the
persisted backend/version, including historical records. Remove hardcoded
`local_file` checks in heartbeat, issues, feedback, and customer-success
inspection together. Preserve current `RUN_LOG_S3_*` mirror configuration for
legacy mode, without redirecting unrelated product file storage.
Reuse the existing instance storage configuration without additional switches.
Cloud already provisions a bucket and bucket-scoped credentials per stack.
Preserve any configured storage prefix and place journals in `run-logs/` and
`provider-traces/`, with company/run namespaces underneath. New journals must not
use a separately configured legacy mirror bucket. Configuration
errors fail explicitly; never select local-only persistence silently. Add only
small company/run-bound stream metadata, with unique identities and constraints.
There are **no proposed run-log payload-chunk or raw-trace payload tables**.
Selected publication protocol after the local operational experiment:
1. Redact/compact log output through the existing owner and enqueue exact NDJSON
bytes with stable source/attempt identities in a bounded buffer. A buffered
enqueue and a durable receipt are different results.
2. Seal on byte threshold, the first dirty buffer's actual deadline, explicit
flush, completion, or graceful handoff. Allocate sequence/byte offsets within
the current writer generation; do not write SQL per record. Seal channels
separately when their trace ordering requires it.
3. Upload a new immutable segment with digest, length, source range, and logical
byte range. Then upload the bounded immutable index/manifest update. Keep
manifest pages bounded; do not rewrite a growing full-history JSON document
every few seconds. Retain exact logical bytes even when compressing storage.
4. Publish one bounded head with S3 ETag `If-Match` (or `If-None-Match` for
creation). The head carries owner generation, committed cursor, and at most
53 immutable index roots. Check the admitted database execution authority
before claiming a new generation. No per-batch SQL or new stream table is
needed. Conditional head publication fences stale generations and tombstones.
5. Release durable receipts for the published contiguous range. Reconcile a
lost object response from the exact head before retrying. Same-source replay
must not duplicate bytes; conflicting replay fails explicitly.
Upload/index retries reuse immutable identities and verify ambiguous results.
Do not hold a DB transaction or connection during network upload. New immutable
index nodes combine completed binary subtrees; all normally published nodes
remain referenced, avoiding an obsolete index object for every flush. Metadata must
not grow with the number or size of individual text records: segment descriptors
and source indexes live in S3. Keep lifecycle/audit metadata bounded.
Use a company/agent/run prefix with a generation-fenced head. A stale owner may finish an upload,
but cannot publish it after takeover. A successor adopts the last published
manifest and replays its uncommitted tail. Per-stream prefix tracking plus scoped,
paginated enumeration must recover unpublished-object cleanup after a crash;
never expose bucket-wide listing through public APIs. Do not garbage-collect a
pending publication merely because one DB query did not find its result.
AWS supports conditional object creation and ETag-based updates, but the current
storage interface does not expose them and compatible vendors need verification.
If eliminating per-segment SQL becomes necessary, qualify conditional writes,
ownership fencing, and manifest recovery explicitly before selecting that design.
[Conditional writes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html).
### 2. Live viewing, durability, and the pending tail
Live output need not wait for an S3 PUT to appear in the browser. Preserve its
stable source identity so reconnect/reload can reconcile it with durable history.
An open viewer can see the pending tail; a different server can read the published
S3 prefix. Do not promise zero-lag cross-server tail reads until a replay/relay
path proves them. The flush threshold bounds normal durable-read lag, not a
promise that storage is always available.
The hard problem is **who retains output until publication**, not which database
column it uses. A timer uploading from disposable web memory alone cannot
preserve the pending tail after a crash. Distinguish these boundaries:
- Store durable receipt: only after the segment/index and metadata publication.
- Runner control-protocol ACK: currently may allow streaming output to leave
replay retention before log projection. Inspect and qualify this exact path.
- Live browser delivery: presentation, not permission to discard the source.
Selected after inspecting the real ACK path: reuse the native control port's
already-durable canonical event rows. The S3 head advances their export cursor
atomically with the log bytes. Takeover reads later native rows in bounded
16-row pages, verifies source identity and hash, then uses the same formatter,
redaction and compaction as live recording. The ordinary contiguous stream adds
no SQL. Startup, missing callback ranges, and interrupted settled archives use
bounded recovery reads; settled repair waits for the old native lease to expire.
This avoids a second payload copy, new per-record DB writes, and a new Runner
retention protocol. Other stdout and raw trace bytes are not inferred from this
source. A retained native control row remains readable after Runner cleanup.
The existing Runner outbox retains unacknowledged records. No second spool,
broker, always-on logging service, or full transcript checkpoint is added.
An alternative is holding the existing transport ACK until a batch is published,
but use it only if a real protocol experiment proves bounded pipelining and
prompt Stop/control handling. Serial awaited callbacks plus a five-second timer
can become a five-second wait per record; that is unacceptable. Do not rebuild
full transcript checkpoints in Postgres to sidestep this issue.
For server-only diagnostics with no surviving source, successful durable receipts
still require S3 publication. Local enqueue alone is never advertised as durable.
Missing uncommitted optional diagnostics are explicitly incomplete. Preserve the
original strict guarantee for every acknowledged durable record; any proposal
to accept a bounded crash-loss window for ordinary output is a separate product
decision, not an automatic relaxation in this revision. Host-local agent-process
survival remains outside the web-server replacement guarantee.
On S3 failure, retain bounded pending work at the source, back off, and surface
capture/persistence state without falsely completing the log. At capacity,
backpressure or stop new admission according to the existing execution owner;
do not allocate unbounded memory/disk or switch payloads into Postgres. Optional
trace overflow can become incomplete/truncated. Preserve essential control,
Stop, task results, and usage settlement. Runner cleanup cannot destroy a pending
required-output obligation; either finish export or leave a durable cleanup hold.
### 3. Reading and storage amplification
Read the published pointer, then bounded manifest/index pages and the segments
covering the requested byte range. Cache immutable contents by digest. Preserve
byte offsets, NDJSON/UTF-8 alignment, EOF, cancellation, and bounded memory.
Compressed segments need logical offset metadata and bounded decompression;
HTTP ranges over compressed bytes are not logical log offsets. Preserve exact
full-stream digest semantics when finalizing.
The first release can serve a finalized log by streaming its segments. Do not
require a full re-upload or database compaction pass at every completion. Add
optional segment coalescing only if measured read/request savings justify its
extra I/O, publication/cleanup complexity, and temporary storage. Retention and
read-path limits must account for small-object growth and indexes. Avoid one
object per token/frame and repeated whole-log mirroring.
S3 publication is not atomic with Postgres; object upload precedes pointer
publication. An orphan object is recoverable cleanup, not proof of an accepted
append. A missing/corrupt published object is a storage integrity error, not an
empty log. Verify the actual provider's PUT/GET/range/checksum behavior rather
than assuming AWS guarantees for every compatible backend.
[S3 consistency model](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#ConsistencyModel).
### 4. Provider trace boundaries
Use the same batched object-payload mechanism with trace-specific ownership.
Keep capture opt-in, existing secret/identity suppression, admin access,
company/task-read checks, no-store responses, reveal/download audit, and trusted
workspace-diff reprojection. Raw bytes never enter ordinary events, live logs,
activity payloads, or general attachments. Provider parsing remains in drivers.
Preserve native and rehydration channels, frame IDs, source/debug sequences,
interpretation links, and raw digests. Capture transport must retain pending
records outside disposable web storage until a storage receipt. Reconstruct
frame lookup on takeover. A complete trace requires all required channel ranges
and terminal records durably published and the integrity assessment passing.
Capture failure yields incomplete/truncated without restarting the provider or
blocking task/accounting settlement. Qualify all currently supported producer
families; do not silently advertise a legacy path-only producer as durable.
Encrypt raw trace objects with the existing stable-key facilities and bind them
to company/trace/channel/segment identity. Keep only metadata and the appropriate
key reference in Postgres. Preserve the existing 24-hour expiry from original
capture and 64 MiB capture limit, with separate bounds for encoded/index overhead.
Restart/import never extends expiry. Historical imports preserve exact bytes,
including native-then-rehydration download ordering and digest.
Expiry/deletion first creates a durable tombstone and fences publication, then
cleans segments, index pages, abandoned uploads, and caches. Retain unresolved
cleanup obligations through restart and concurrent uploads; no resurrection.
Preserve `{ok:true}` only after required payload cleanup. Verify object-version
behavior for the actual bucket, and document historical backup retention
separately. Restore must honor expiry/deletion receipts before exposing traces.
Audit contains content-free metadata only.
### 5. Idle behavior and recovery
A nonempty buffer schedules one actual flush deadline; successful settlement
cancels it. Use the existing coordinator/scheduler for server-owned work and the
Runner lifecycle for producer-owned work. Retries exist only for unresolved
uploads/publications/deletes. Coalesce work and bound global as well as per-run
upload concurrency; prevent many runs flushing at once from starving HTTP or
connection pools. Do not create per-agent storage clients or idle keepalives.
Start/resume/takeover reconciles active stream metadata, source retention, and
unfinished cleanup. Existing DB work signals are process-local; do not rely on
them to wake another server. Until the parent's cross-instance wake mechanism
exists, retain explicit owner/recovery/sleep obligations. Completed history alone
creates no recurring scan. Trace expiry uses its real deadline and the existing
supported wake/blocker contract.
## Delivery slices
| Slice | Useful result and owned surfaces | Evidence/dependency |
| --- | --- | --- |
| 1. Operational and durability prototype | Workload measurement, S3 segments, publication/cost model, actual Runner pending-tail/ACK experiment | Select batching and one publication scheme; demonstrate acceptable cost and control responsiveness before schema commitment |
| 2. S3-backed run logs | Store/readers, bounded buffering/indexes, small metadata schema, replay/ownership, cleanup and local compatibility | Slice 1; two-process replacement, ambiguous PUT/commit, stale-writer and backlog tests |
| 3. S3-backed provider traces | Producer sinks/rehydration, trace store/routes, expiry/deletion | Slice 2 primitives; actual capture integrity, access and cleanup races |
| 4. History migration and qualification | Resumable importer, compatibility/authority switch, documentation and staging evidence | All retained inventory accounted for; empty-disk reads and supported rollback after new writes |
These are reviewable boundaries, not a fixed PR count. Keep each usable with the
existing instance storage profile. Raw trace capture remains opt-in. Keep storage
logic in its owners, not large heartbeat/executor additions.
Generate necessary additive metadata migrations with `pnpm db:generate` and
follow snapshot-retention rules. There is no DB payload migration or compactor.
Update `doc/run-log-events.md`, `doc/DATABASE.md`, `doc/DEVELOPING.md`, and
`doc/testing-native-remote-continuity.md` with actual guarantees and operating
limits. Update affected run/inspection feature/access map entries and run
`node scripts/check-maps.mjs`. Change shared types/UI only when their contract
changes; keep incidental storage mechanics out of normal product controls.
## Migration and rollback
1. Ship a compatibility release that reads both stores, understands new formats,
and enforces the persisted minimum reader/writer capability on external runs.
Keep the volume and legacy writing enabled initially. Record this release as
the rollback floor before switching any authority; an environment flag alone
cannot stop an old binary from mishandling new rows. The deployment/rollback
preflight must reject binaries older than that floor: a pre-feature binary
cannot be expected to understand a newly introduced DB guard.
2. Inventory every retained run log and unexpired trace by company/run. Classify
local complete files, remote mirrors, active files, missing payloads, conflicting
copies, and expired/deleted traces. A mirror may be a partial prefix: verify
against authoritative bytes/metadata, never assume it is complete.
3. Backfill settled resources with bounded indexed keyset batches and resumable
receipts. Preserve original bytes, offsets, digests, attempt information,
source attribution, and trace expiry. Expired/deleted traces stay unavailable.
4. Under the parent's permitted one-time maintenance window, stop admission and
settle/fence remaining legacy writers. Copy final active tails and both trace
channels. Recheck hashes and byte totals after fencing; no copy-while-writing
race can establish completion.
5. Publish each resource's backend/reference transactionally. Record a migration
manifest of migrated, already-remote, expired/deleted, and unresolved resources.
Missing/conflicting retained bytes block the preservation gate. Report genuine
pre-existing loss; neither fabricate history nor quietly waive Dotta's choice.
6. Start B with the log/trace directories absent and old storage inaccessible.
Read historical data and append new data through normal APIs. Confirm every
in-scope inventory item has a valid disposition. Keep the old volume retained
under the parent's recovery policy; this slice does not detach/delete it.
7. Roll back to the compatibility release against the same external data and
prove it reads writes made after cutover. Do not switch authoritative reads
back to stale local files. Defer destructive schema cleanup until rollback
readers have retired.
## Acceptance matrix
| Case | Observable finish condition | Evidence |
| --- | --- | --- |
| Operational envelope | No new payload/per-token SQL; batch metadata and object traffic fit measured budgets | Real workload baseline, WAL/transactions, request/storage/transfer model and app/control latency |
| Active/completed run | B retrieves every durable receipt exactly once after A and its disk disappear | Separate processes/directories; normal authorized reader; byte/digest comparison |
| Pending tail | Web replacement retains/replays required output without duplicating provider work | Actual Runner spool/ACK experiment; kill between generation, upload and publication |
| Existing history | All retained inventory preserved under original retention | Full checksums, resumable manifest; missing/conflicting data blocks cutover |
| Ambiguous PUT/commit | Retry reconciles publication and never duplicates bytes | Fault after object PUT or DB commit but before response |
| Ownership | Stale A cannot publish/finalize after B takes over | Concurrent server and manifest-generation tests |
| Reads | Stable offsets/digests across segments, compression, caches and replacement | UTF-8/boundary/EOF/cancellation; bounded memory and GET counts |
| Storage failure | No unbounded queues or SQL-payload fallback; Stop/API remain usable | Slow/failed S3, queue capacity, recovery and source cleanup hold |
| Traces | Accepted frames survive; completeness and privacy remain truthful | All producer families, channel hashes, disabled capture and denied actors |
| Downstream consumers | Issue previews, feedback, inspection and reprojection keep working | Normal routes and current authorization checks |
| Expiry/delete | Deny after tombstone; clean payloads; no late-upload resurrection | Capture/upload/delete races, restart, actual bucket version behavior |
| Rollback/local mode | Compatible release reads post-cutover writes; local defaults unchanged | Adjacent-version and local-store tests |
| Idle | No recurring SQL/network work after settlement; real work wakes owners | Configured idle, completed history, retry/drain/resume observations |
## First useful experiment and qualification order
Use real Postgres, the existing S3 provider implementation against a disposable
S3-compatible target, and two actual service processes with separate temporary
directories. Produce representative output through the production recorder,
including repeated chunks, multibyte text, and tool bursts. Measure the candidate
batch settings and baseline API traffic. Test both published data and the pending
tail: kill A at each boundary and read/replay through B. Never pre-populate the
storage result being tested. Record revision, configuration, receipts, timing,
SQL/WAL and object-operation evidence.
Use the existing remote-continuity harness with a deterministic provider and
actual Runner to qualify retention independently of control ACKs and responsive
Stop. The decisive question is whether durable batching can coexist with ongoing
provider execution and web replacement. Do not commit to a protocol mechanism
based on a mocked callback or a successful single PUT.
Before staging, record the chosen publication scheme, batching/concurrency
limits, backlog capacity, supported recovery behavior, and measured cost/latency
operating envelope. If those fail, revise the design before a broad schema or
migration implementation. No production sizing/cost has yet been measured.
Then verify the actual staging bucket and remote capture path with a bounded
live provider run, normal UI/inspection/download, and post-replacement reads.
Make the old web filesystem inaccessible. Preserve prior failures alongside
successful retries. Full Railway HTTP handoff and volume removal remain parent
plan gates.
Run narrow storage/output/trace/continuity/auth/migration checks while iterating.
At implementation PR handoff run `pnpm -r typecheck`, `pnpm test:run`, and
`pnpm build`, plus affected Runner checks and user-journey verification. Do not
claim live-provider or fleet evidence from deterministic fixtures.
## Implementation state and remaining gates
- Configuration follow-up: shared storage resolution now serves product files,
run logs and traces. S3 instances need no extra switches or connection fields.
Cloud's existing per-stack bucket and credentials remain the isolation
boundary; existing prefixes and company/run subfolders are preserved. The
configuration regression failed before the change, then 70 affected tests
passed, including real S3-compatible bucket/prefix isolation and cleanup tests.
Four existing Cloud environment-contract tests also passed without Cloud code
or deployment changes.
- User follow-up: remove the added run/adapters map sections. Retained-history
conversion now belongs to an automatic, bounded delivery worker; the command
is optional diagnosis/repair. Existing local references remain durable work.
Startup reconstructs it, active or blocked history retries with backoff, and
completed history has no recurring SQL/network scan. Migration stays with its
owning service; index.ts only registers its lifecycle, and the shared delivery
registry adds the queue name. The source volume must survive until all retained
history is verified. Mixed versions must stop creating new local-file runs.
- Trace encryption reuses either existing vault key source, including the
configured key file, and never creates a replacement key for external traces.
Both strict missing-key and file-key round-trip regressions failed before the fix.
The configuration/storage regression run passed 91 tests. Automatic migration
and retention passed 10 tests, including a real Postgres/S3 upgrade followed by
source-directory deletion. Harness failures remain in the local evidence: a
cold import exceeded a test deadline (moved module setup into the fixture), a
sandbox denied localhost access (rerun with network permission), and the first
automatic fixture bucket name exceeded S3's length limit (shortened the name).
- Shutdown review: automatic trace import now propagates cancellation through
source reads, opening, conditional-write qualification, upload and verification.
Five real-SDK stalled-request regressions failed before the fix, then passed.
A shared capability probe survives one cancelled claimant and aborts when its
last claimant leaves. Source and remote trace hashing stream bounded chunks.
A multi-chunk native sidecar without a final newline retains exact download
ordering and digest. Existing file-key and empty-web-home app checks passed at
a142424fc9; subsequent changes affect copy cancellation and verification only.
- Base integration: 879ba0d1fd reached Greptile 5/5 and all CI checks passed,
then master advanced and produced two append-only documentation conflicts.
Preserve both documentation sections and integrate the new Cloud snapshot
startup contract. The startup regression reproduced a redundant config read;
worker registration now accepts only the already-loaded storage-provider value.
All 103 startup controls plus storage/lifecycle controls passed (112 tests),
including saved Cloud profile reuse after the legacy config file is absent.
A further combined run passed all 124 storage and real-Postgres Cloud runtime
configuration controls on the merged source.
- Further review: trace retention now registers independently of heartbeat
scheduling, and optional issue-comment attribution reads the published log
prefix after a failed best-effort native repair. Both defects reproduced red;
all 116 selected startup, retention, attribution and cancellation controls pass.
The 19f702d780 browser session-recovery check timed out; its job log is retained.
No UI behavior or browser assertion was changed to suppress that failure.
- Branch: `codex/external-run-logs-traces`; this chat owns integration and PR review.
- S3 holds payloads from the first committed batch. Conditional object heads
hold cursors and writer generations. Existing Postgres rows hold lifecycle and
references only. No schema change, per-record SQL, or DB payload compactor.
- Implemented: stored-backend reader dispatch, bounded batching and I/O,
immutable binary range indexes, exact digest/range reads, ownership fencing,
encrypted optional trace channels, durable deletion/expiry obligations, and a
bounded resumable history importer. Local defaults remain unchanged.
- Exercised locally with real Postgres and Moto 5.2.5 through the real AWS SDK:
encrypted trace replacement/deletion and history import with original source
directories removed. Missing, corrupt, and active history blocks migration.
- Abrupt process qualification: a writer commits 1,000 records, receives storage
receipts, dies by `SIGKILL`, and loses its working directory. A separate
process reads every record exactly and continues the same journal.
- App journey: create a deterministic process agent, click Run now, open its
run history, kill the web process, restart against the same external services,
and reload the viewer. All 31 NDJSON records remained byte-identical through
the normal authorized API (6,206 bytes; SHA-256
`3e8415c79379c583b1c4c37902187593fd69ebe85db78d6486d6077f205bacba`).
- Local workload qualification reports burst and eight-stream steady request,
transfer, stored-object, and timing counts. It is a synthetic envelope,
not a fleet cost forecast. The first index prototype left obsolete COW
objects; the selected immutable binary index removes that normal growth.
- Full workspace typecheck and build passed after native replay and the storage
review fixes. The pushed server revision was rebuilt for the final app check.
The native Postgres package initially lacked its postinstall-created library
symlinks; its standard hydration helper repaired that local test prerequisite.
- Native pending-tail qualification passed with the real Rust recovery fixture,
`PaperclipControlPlanePort`, production recorder, real Postgres and Moto. A
received ACK released 32 records from the Runner while S3's source cursor was
still zero. After `SIGKILL` and deletion of the old controller directory, B
exported all 64 records once while the same Runner PID continued. Snapshot and
Stop commands completed in 453 ms and 122 ms. The first attempt failed because
its new controller directory had mode 0755; the fixture now creates mode 0700.
Both failed and successful evidence remain in this chat's temporary artifacts.
This is a deterministic protocol experiment, not a live model or Railway test.
- Final app verification used a new web home on `c2cedcd116`. Retained history
remained byte-identical (31 records, 6,206 bytes), and a new UI-triggered process
run succeeded with every generated record present in its S3-backed transcript.
The browser initially reached the listener before startup completed; after
readiness, the normal page and API checks passed.
- Durable log receipts and live log messages follow S3 publication. The native
export reuses existing control rows, adds no payload writes, and repairs missing
callback ranges. Interrupted settled archives can be repaired on normal reads
after the old native lease expires. Optional raw capture and server-only
diagnostics without a surviving source can remain incomplete after a crash.
- Review fixes cover absent-head deletion fencing, shared capture ownership,
abort/drain on deletion, local expiry deadlines and multi-page cleanup, bounded
transient retries and admission backpressure, partial-startup cleanup, and
explicit trace-import cleanup markers. A source integrity failure leaves the
native archive's final size unknown instead of certifying a partial result.
- Full local Vitest could not run on host embedded Postgres because concurrent
clusters exhausted macOS SysV semaphores. Affected suites are running against
isolated Docker databases. One local full typecheck attempt also lacked sandbox
permission for tsx IPC; the authorized rerun uses the normal workspace command.
The isolated Docker run then passed 1,014 affected storage/native tests and
all 27 privacy-route tests. Subsequent targeted review regressions passed for
native source classification, native import coverage, shared-reader repair
cancellation, serving the published prefix on repair conflicts, and a failed
local-file cleanup retaining its deadline. Live and recovered native output
redact existing agent identity material; runtime log secrets are registered
once through the existing encrypted redaction registry, without copying agent
private keys into run contexts.
- Native history migration accepts finalized files whose recorded size and
digest match, preserving the existing formatter's truncated large display
chunks. Files missing either completeness field still require ordered source
coverage. Corrupt metadata blocks publication; the conditional switch checks
that the original completeness fields have not changed.
- PR: https://github.com/paperclipai/paperclip/pull/18327. Initial CI passed 51
checks with one E2E teardown lock timeout. That E2E shard passed on the next
revision; its Runner shard then failed the Antigravity authentication fixture.
These failures are retained, and fresh-head CI remains required.
- Production remains disabled. Railway Buckets are the parent fleet's current
S3 provider; their conditional-write, version/deletion, workload cost, remote
capture, and sleep/wake behavior need actual-target qualification. No fleet
environment or volume was changed.
- Next gate: finish fresh-source checks and PR review; retain actual-provider,
live remote capture, and parent cutover gates before any volume-free rollout.
- No Paperclip issue/run context was supplied. This repository plan is the state
record; no issue work product or production migration is claimed.
+36
View File
@@ -34,6 +34,42 @@ Current limitations to keep in mind:
In practice, that means the current implementation is a good fit for local development and self-hosted persistent deployments, but not yet for multi-instance cloud plugin distribution.
### Managed image packages
Managed Cloud provisions only plugins selected from the current release image
catalog. A Cloud tenant token or managed configuration keeps this image-only
policy active. Without an explicit selected package list, Cloud selects no
packages and cannot fall back to local installs. An explicit operator-managed
list also keeps this policy active outside Cloud. The database registration's
plugin key, package name, version and
approved manifest identify that package. Its legacy `packagePath` remains for
local/self-hosted compatibility; Cloud never uses that path to locate code.
Workers, plugin migrations and UI files resolve through the same current-image
package resolver. Cloud does not install npm dependencies, build packages, use
the development UI proxy, or fall back to a server home directory.
Prebuilt workers use the current release's baked CLI `tsx` loader for the SDK's
workspace TypeScript dependencies. The loader is located from the owning server
package in both regular and bundled server layouts. Missing loaders and loader
links outside the release's CLI or root `node_modules` fail activation. This
loads image-supplied code without installing dependencies or building packages.
Equivalent packages at different image roots reuse the existing registration,
configuration, approval status and company scopes without rewriting its stored
path. Concurrent first registration of the same package retains one plugin ID.
Conflicting package identities fail explicitly. Normal maintenance boot retains
the existing bundled manifest/version reconciliation and approval gates.
`createBundledPluginPackageResolver(...).inspect(registration, verifiedManifest)`
is a read-only qualification hook. Resolve the current image path, load its
manifest with the validated plugin loader, and inspect the registration before
normal boot reconciliation. Missing manifest evidence and incompatible packages
produce explicit blockers. Distribution packages retain their catalog digest
verification; built-in packages trust the immutable release image and must bump
their version when their code changes. Qualification supports identical package
sets only. This package-resolution boundary does not coordinate workers,
background jobs, or deployment overlap between servers.
## 1. Scope
This spec covers:
+112
View File
@@ -0,0 +1,112 @@
# Check existing recovery files before web-server replacement
This check covers only observed legacy files on one explicitly selected instance.
There is no verified production backlog. An empty installation needs no migration.
This check does not inspect Postgres obligations, Runner delivery, other volumes,
other instances, or the rest of web-server replacement readiness.
Use the retained volume and the instance root from the old server configuration.
The instance root is the directory that contains `accounting-receipts/` and
`quota-credential-recovery/`. It is not the parent directory of all instances.
Use canonical absolute paths. The check rejects symlink components, including
system aliases such as `/var` on macOS; use the actual `/private/var` path there.
The chosen instance directory must exist. Missing recovery directories are normal.
Run this command from a checkout that includes the check. Node is the only runtime
dependency. It needs no database connection, master key, or package installation.
```sh
node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance
```
The default command reports JSON and exits successfully even when work remains.
For an automation gate, add `--check`:
```sh
node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance --check
```
| Exit code | Meaning |
| --- | --- |
| `0` | Report produced; with `--check`, all three scans are clear |
| `1` | With `--check`, unresolved files or an unknown/incomplete scan block cutover |
| `2` | Invalid arguments or the inventory could not run |
The sandbox directory defaults to `data/sandbox-orphan-cleanup/` below the chosen
instance. If the old server uses `SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR`, supply that same
environment value when running the check. An explicit `--orphan-directory` takes
precedence. A relative or empty override is rejected; supply its canonical absolute
path for this check. An override is an additional explicitly selected root and can
be outside the instance directory. The check does not search for other overrides.
```sh
node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance \
--orphan-directory /absolute/retained/orphans --check
```
The report gives one summary for each of the three recovery kinds. It never
prints filenames, input paths, file contents, identifiers, token values, decrypted
material, or raw filesystem errors. `unverifiedFiles` includes every regular file,
including JSON, temporary, probe, truncated, corrupt, and unreadable files. The
check does not read file bytes and cannot claim that a file is valid or settled.
That decision belongs to the existing recovery owner and its format definitions.
Only the credential directory's immediate UUID company directories are traversed;
empty company directories are normal after recovery. Accounting and orphan files
use flat directories. Unexpected directories, deeper nesting, symlinks, and
special entries count as `unknownEntries` and block. The check does not follow
symlinks or inspect unexpected subtrees. Directory/stat/close failures increment
`readFailures` or leave the scan incomplete. An invalid or missing instance root
also blocks; it is not an empty installation.
The default shared budget is 10,000 observed entries across all three scans,
including company directories. `--max-entries` accepts 1 through 100,000. At the
limit the scanner reads at most one extra entry per open directory to detect
truncation. An unfinished scan reports `complete: false` and status `unknown`.
Counts are observed lower bounds, not total records or total pending operations.
A complete scan with any files reports `blocked`; any uncertainty reports
`unknown`. Only a complete empty scan reports `clear`.
For the final cutover gate, stop the old producers first and retain the old volume.
Run the check after the relevant recovery owners finish their work and producers
are stopped again. A live scan is a point-in-time inventory, not an atomic drain
certificate. Concurrent renames or writes invalidate the snapshot; the tool does
not lock writers or protect against a hostile process that changes paths during
inspection. Do not use it to certify a live writable or untrusted volume.
If files exist, keep the old volume and let the existing owners settle actual
entries with their existing permissions and database checks:
- Accounting: `run-cost-accounting.ts` uses the `usage-receipts.ts` replay and
per-run recovery paths. The receipt owner validates the accounting envelope and
handles database conflicts and settlement.
- Credentials: `ai-connections.ts` and `ai-connection-runtime.ts` use
`quota-credential-recovery.ts` during account credential access and refresh.
The owner validates encrypted material with the original instance master key
and preserves grant, company and credential version fences.
- Orphan cleanup: `environment-runtime.ts` exposes `flushDeferredOrphanCleanups`
to the existing cleanup lifecycle. `sandbox-orphan-cleanup-spool.ts` owns the
record format and removes a file only after the recovery path saves ownership.
These are service entry points, not standalone operator commands. This change
adds no import, deletion, replay, decryption, or bulk migration command. Do not
trigger new paid runs or token exchanges merely to empty the report. If an owner
cannot settle a record, preserve it and record an explicit reconciliation outcome
through the existing incident/operations process. Corrupt files need owner review;
they do not disappear from the gate because a recovery reader skipped them.
Never delete unresolved files to obtain a clear result. An approved exception
requires separate evidence; this tool continues to block while the files remain.
Retain the volume and required secrets through rollback compatibility review.
Verify the check without any real instance data:
```sh
node --test scripts/recovery-cutover.test.mjs
```
The tests use disposable synthetic directories and spawn the actual CLI. They
cover pending legacy layouts, malformed bytes, temporary files, empty directories,
symlinks, overrides, scan limits, safe diagnostics, failure injection, exit codes,
and preservation of source bytes. They establish file inventory behavior only.
They do not establish that any production volume is clear or that domain recovery
has completed.
+276
View File
@@ -0,0 +1,276 @@
# Run log and provider trace object storage
Instances configured for S3-compatible file storage automatically use the
`s3_segments` backend for new run logs and opted-in provider traces. Postgres
retains the existing run/trace references and lifecycle rows.
It adds no payload table, per-record SQL, or database-to-object compactor.
Structured `heartbeat_run_events` retain their existing database path. Native
events already committed there before a Runner ACK also provide the recovery
source for an interrupted canonical native log export. No additional event copy
or payload write is introduced.
When native object logging uses runtime observability credentials, their log
redaction values are registered once in the existing encrypted run-secret
registry. This is bounded redaction metadata, not a transcript copy or a
per-event write. Agent identity private material stays in its existing key store.
## Configuration
No additional storage configuration is required. The existing instance
`storage.provider` setting, or `PAPERCLIP_STORAGE_PROVIDER`, selects storage.
The same config-file S3 fields and `PAPERCLIP_STORAGE_S3_*` overrides select the
bucket, endpoint, region, prefix, and path style used by product files. Credentials
use the same AWS SDK chain. Local-disk instances keep `local_file` logs and traces.
The instance's existing S3 prefix is preserved. The folders underneath it are:
```text
<existing-prefix>/run-logs/segments-v1/<company>/<agent>/<run>/
<existing-prefix>/provider-traces/segments-v1/<company>/<run>/<trace>/<channel>/
<existing-prefix>/<company>/... existing product files
```
Cloud's existing provisioning supplies a separate bucket and bucket-scoped
credentials for each stack, plus the stack's encryption key. The app reuses that
boundary rather than selecting a global customer bucket. Within a stack, journal
keys remain company/run scoped. Cleanup lists only the resource's own prefix;
it cannot sweep other companies, other stacks' prefixes, or product files.
Folders are organization, not a substitute for bucket permissions or API access
checks. Shared-bucket self-hosted deployments must retain their existing distinct
instance prefixes and appropriate storage credentials.
Legacy run rows retain their recorded backend until verified migration. Their
pre-existing `RUN_LOG_S3_*` mirror settings and key paths remain supported for
legacy reads and writes. They do not override instance storage for new journals.
Use stable bucket/prefix settings across replacements and releases.
As with existing product files, changing the instance bucket or prefix is a
separate storage move and requires copying the complete namespace and preserving
its keys before changing configuration. Switching to local storage does not move
existing S3 evidence. Readers reject an unavailable external store; they never
substitute an empty local log. Restore the original stack's storage profile to
read its history. This upgrade migrates legacy local evidence into the existing
stack store; it does not introduce bucket relocation or reverse migration.
Raw capture still requires its existing explicit capture request. Its encryption
uses the existing vault key, from `PAPERCLIP_SECRETS_MASTER_KEY` or the existing
configured key file. Cloud already supplies the environment key. No separate
trace key is introduced, and a missing vault key fails rather than creating a
replacement. Disposable deployments must preserve the existing vault key along
with their other secrets.
Raw payload and index objects
use AES-GCM with a key derived for the particular company/run/trace/channel.
The bounded head holds only ownership, range, digest, and publication metadata.
Existing capture opt-in, redaction, instance-admin access, task/company access,
reveal/download auditing, and the 24-hour capture expiry remain in effect.
The provider must enforce `If-None-Match: *` and ETag `If-Match`, return ETags,
and support prefix-scoped paginated listing. Claiming the first journal performs
a conditional-write capability probe. A backend which ignores preconditions is
rejected. Before deploying this release to an S3-configured production instance,
qualify the provider's conditional writes, range/read consistency, encryption,
object versions, and cleanup. AWS API compatibility alone is insufficient evidence.
## Publication and operating limits
Each batch uploads immutable payload segments and merges equal-height index
roots into immutable binary nodes, then conditionally publishes one head object.
The head carries at most 53 ordered roots. Every normally published payload and
index node remains reachable; appends do not leave superseded index copies.
Interrupted or fenced uploads can leave unpublished objects, which remain inside
the resource prefix for scoped deletion. The head is the commit
boundary. A stale writer can upload an orphan but cannot replace the current
owner's published history. An uncertain response is reconciled against the exact
head before returning a durability receipt. Committed reads use immutable roots,
so publication cannot change the bytes behind an already selected snapshot.
An individual store `append` resolves only after publication. Heartbeat's
`onLog` diagnostic callback waits for bounded admission without serially
waiting for each network batch. At capacity it waits for space; admission beyond
the bounded waiting allowance is rejected without poisoning accepted batches.
Transient network, 409, 429 and server errors receive bounded retries. A 412
or a changed head fences the stale writer; a 409 alone does not prove ownership
loss or conditional-write support. Live log messages and progress are published
after storage confirmation. Finalization drains admitted writes before closing
and hashing the stream. Logging failure reports incomplete persistence and
leaves the final size/digest unknown rather than claiming a complete snapshot.
The original progress projection remains coarsely throttled; concurrent batch
completions share a single progress write.
Defaults and fixed safeguards:
| Limit | Value |
| --- | --- |
| Dirty batch deadline | 1 second, configurable from 1 ms to 30 seconds |
| Segment target | 1 MiB |
| Pending bytes per writer | 8 MiB |
| Pending run-log bytes per process | 64 MiB |
| Additional waiting admission | 8 MiB / 32 calls per process |
| Concurrent run-log flushes | 4 |
| Concurrent journal object operations | 8, with 32 queued operations |
| Object operation/batch cancellation deadline | 15 seconds |
| Current index fanout / maximum roots | 2 children / 53 roots |
| Journal head read bound | 64 KiB |
| Trace encoded payload bound per sidecar | 192 MiB, in addition to the existing raw capture limit |
A configured but empty log store sends no periodic request. Dirty buffers,
capture file-change events, and actual retention/retry obligations drive work.
Use request/transfer counts, byte volume, steady versus bursty producers,
concurrent runs, and control/API latency to size a deployment. The local S3
qualification is a synthetic workload, not a production cost forecast.
Finalization streams the committed bytes once to compute the existing complete
file SHA-256. It does not re-upload the entire log. Range indexes avoid loading
all segment descriptors to read a tail. Historical consumers, UTF-8/NDJSON record
alignment, issue previews, feedback, and customer-success inspection dispatch
through the persisted backend. Logical reads retain the existing authorization.
On-demand native repair closes its verified committed range with an exact byte
count and leaves an unavailable full-file SHA-256 null. It does not make a log
page request re-download the entire archive to compute that optional digest.
## Durability boundary
Committed log receipts and captured object prefixes survive web disk loss.
Output still buffered only by the current web process has no durable receipt.
The live log path waits for publication before showing that output. Raw sidecar
capture remains optional; a crash can leave an incomplete uncommitted tail, and
only durably closed channels plus valid terminal markers qualify as complete.
Native PRP events have a surviving source: the control port already commits
validated, redacted events before acknowledging them. The S3 head checkpoints
the highest exported database event cursor with the batch's bytes. A replacement
replays later native source rows in bounded 16-row pages, checks their identity
and canonical hash, and exports the same `paperclip.prp.event` representation
through normal log redaction/compaction. Late callbacks and duplicate delivery
cannot append the same source record again. This is a canonical native export;
generic stdout, stderr, and raw provider traces are not reconstructed from event
rows. The control ACK and sparse checkpoint formats remain unchanged.
Recovery runs before native execution reconnects. If the run settled before its
archive finished, the normal log API, issue preview, or feedback reader can repair
it after the native controller lease expires. Concurrent requests share recovery;
final metadata is saved only when no later native source remains. Read-only
customer-success inspection reads the currently published prefix. A failed
repair leaves final size/digest unknown and preserves the existing source.
Retained native events are not deleted by Runner cleanup. Recovery adds bounded
reads and one final metadata update, with no new per-record SQL on the normal
contiguous output path. Raw optional capture can still be incomplete after a crash.
Credential-update, retained-cleanup, and trace-reprojection records have separate
source namespaces and do not enter this log export. Corrupt canonical records
fail integrity validation. Shared on-demand repair has a 30-second lifetime;
cancellation removes a waiting reader and aborts storage work when no readers
remain. It preserves already published cursors so a later request can resume.
Live export and recovery both redact the existing agent identity and known run
secrets before publication. Recovery reads existing identity material without
provisioning a key. If best-effort repair loses ownership or fails, ordinary log
reads serve the published prefix and leave completion metadata unknown; request
cancellation still propagates.
The [remote continuity](testing-native-remote-continuity.md) protocol and the
parent deployment's other filesystem and HTTP handoff requirements still apply.
## Trace deletion, expiry, and sleep
Deletion/expiry denies reads through a database tombstone before payload cleanup.
Object journal tombstones prevent further publication. If an active writer may
have an in-flight upload, cleanup waits at least 30 seconds and remains a durable
retry obligation. A successful manual DELETE requires payload cleanup; an
unsettled cleanup returns a retryable failure, and the shared retention worker
retries it. Tombstones remain after object removal. Historical backups and
object-version retention need a separate operator policy; an ordinary DELETE
must not be mistaken for erasure of historical versions.
The shared coordinator restores expiry/cleanup work on startup and registers
actual deadlines for local and external captures. New local inserts also wake it;
expired backlogs drain in bounded pages. No completed-history polling is added. Trace expiry
is not yet part of Cloud's durable wake contract, so outstanding trace deadlines
conservatively block instance sleep. Removing that blocker requires qualification
of the corresponding external wake, not a longer polling interval.
Trace expiry and failed-deletion recovery remain registered when
`HEARTBEAT_SCHEDULER_ENABLED=false`. Disabling agent scheduling does not disable
retained-data cleanup. The worker still respects warm standby, admission drain,
actual deadlines, and shutdown.
## History migration and rollback
S3-configured servers automatically migrate retained local history in the background.
No command or extra configuration is required. The existing delivery coordinator
starts the upgrade outside request handling, independently of the heartbeat
scheduler toggle. Each batch processes at most one log and one trace, then yields
for one second. Existing references are the durable work inventory. Restarting
reconstructs unfinished work from those references; already migrated rows are
excluded. No new payload or job table is added.
Active writers, missing files, corrupt data, and storage failures leave history
unresolved. Other pages continue. The server logs each blocked resource and a
pass summary. Unresolved passes retry after five minutes, with backoff up to
30 minutes. Transport failures retry through the shared coordinator. After all
retained history is migrated, the worker stops querying and contacting storage.
Trace source hashing and remote verification stream bounded chunks instead of
loading a complete capture into server memory. The worker cancellation signal
reaches file reads, trace opening, conditional probing, uploads and verification.
Local-disk instances do not register this worker. Outstanding migration conservatively
blocks instance sleep, and shutdown aborts the batch and waits for owned work.
Deploy compatible readers and stop old binaries from starting new local runs.
The upgrade waits for existing writers to settle and their leases to expire.
Keep the source volume attached until migration completes and clean-filesystem
reads pass. Automatic copying cannot recover a file after its only copy is lost.
`server/scripts/migrate-run-evidence.ts` remains an optional diagnostic and repair
tool. Its default is a read-only inventory; `--apply` runs the same verified
importer used by the automatic worker. `--company`, `--after-run`, `--after-trace`
and `--limit` allow scoped investigation against an explicitly selected
`DATABASE_URL`. Normal upgrades do not require this tool. It does not stop agents,
select a production target, delete source files, or certify volume removal.
For logs, migration verifies exact local bytes or a mirror with proven length
and digest, compares any previously uploaded prefix, verifies remote SHA-256,
then conditionally changes the run's backend/reference. Unknown-completeness
mirrors, missing/corrupt payloads, and active writers block publication. Existing
trace bytes, download ordering, integrity state and original expiry are retained.
Neither path fabricates missing historical data or extends trace retention.
For finalized native logs, matching the recorded size and SHA-256 proves the
complete legacy file, including display chunks truncated by the existing 64 KiB
formatter. Migration preserves those exact bytes and records the settled source
cursor. If either completeness field is absent, a bounded NDJSON scan must prove
ordered coverage of the existing canonical source records. Missing ranges or
chunks whose identities cannot be verified block publication as
`native_log_source_coverage_unproven`. A database's latest event cursor alone
does not establish completeness. Publication checks that the original reference,
completeness metadata, and source cursor have not changed. Keep the original
volume and resolve blocked inventory before cutover.
An interrupted trace import keeps an `s3-import-v1:` reference: reads still use
the original local sidecars, while deletion and expiry retain responsibility for
the staged object prefix. The automatic worker resumes the verified switch after restart or retry.
Ordinary local traces have no S3 cleanup dependency.
Keep the volume until every retained item has been verified from a clean web
filesystem. Rollback must use a compatible reader against the same remote state.
An older binary or stale local volume is not a valid rollback target after new
external writes. Enforce that floor in the owning deployment procedure.
## Qualification
Run the storage, recorder, migration, privacy and retention tests. The opt-in
S3/Postgres integration fixtures use `PAPERCLIP_LOG_TEST_S3_ENDPOINT` and a dedicated
`PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL` ending in `_test`. They create
unique test buckets and schemas, and remove only those resources. The privacy
route suite also accepts `PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL` for a
dedicated, fully migrated database ending in `_test`; it clears its fixtures.
`server/scripts/qualify-object-run-logs.ts` exercises the actual AWS SDK against
a loopback test backend, confirms exact replacement reads and stale-writer
rejection, and reports requests/bytes/timing. It uses synthetic credentials and
refuses a non-loopback target. This proves its tested transport and workload;
production-provider qualification and actual Railway deployment remain separate.
`server/scripts/qualify-native-log-recovery.ts` uses the real Rust
`recovery-fixture` example, real Postgres, the control port, production recorder,
and S3 backend. It kills the controller after Runner ACKs but before the S3
cursor catches up, deletes that controller's directory, and restores sparse
control state into a fresh controller. It checks exact source order, no duplicate
records, the same surviving Runner PID, and control/Stop responsiveness. Build
the example with `cargo build --locked --manifest-path packages/paperclip-runner/runner/Cargo.toml -p paperclip-runner-core --example recovery-fixture`.
It accepts the same two explicit loopback test endpoints and creates a private
schema and bucket. This deterministic protocol experiment does not qualify a
live provider, remote execution environment, or production S3 vendor.
+10
View File
@@ -694,3 +694,13 @@ telemetry-read authorization. Ordinary PRP reads/live delivery retain run-secret
and current-user redaction. The task page uses one history read on mount/reconnect
and the existing live transcript stream, then makes a bounded plugin readiness
check. No panel discovery timer scans runs or provider state while connected.
## Payload storage
Structured events in this guide remain in `heartbeat_run_events`. When the
instance's existing file storage uses S3, new stdout/stderr NDJSON logs use the
same stack bucket and configured prefix under `run-logs/`, with existing run
references in Postgres. Local-disk instances remain local. Raw provider
traces have their own restricted, encrypted object path. See
[run evidence storage](run-evidence-storage.md) for batching, durability,
configuration, migration, retention, and qualification boundaries.
+127 -4
View File
@@ -189,10 +189,15 @@ disk outbox. No second remote journal or new polling loop is introduced.
The live event window keeps the latest identity, turn, result and usage anchors,
as well as unsettled semantic inputs, through long output streams. The 4,096-event
and 500-command limits remain. This checkpoint is intentionally not a complete
transcript: an abrupt crash can lose display output already acknowledged but not
yet projected to the existing run log. The same provider turn continues; recovery
never reruns work to reconstruct display text. Tool outcomes and terminal state
retain their durable barriers.
transcript. With local-only logs, an abrupt crash can lose a display projection
already acknowledged by the controller. The opt-in `s3_segments` log backend
checkpoints its canonical native export cursor in S3 and replays missing records
from the existing validated `heartbeat_run_events` source on takeover. This adds
no payload writes or per-record checkpoint transactions. Generic process output
and optional raw provider sidecars have separate durability boundaries. See
[run evidence storage](run-evidence-storage.md). The same provider turn continues;
recovery never reruns provider work to reconstruct display text. Tool outcomes
and terminal state retain their durable barriers.
Rollback must use a controller that understands sparse checkpoints and the signed
recovery cursor. A v2 decoder alone is insufficient. Runners admitted before this
@@ -367,3 +372,121 @@ Correctness and absolute gates must pass for acceptance.
Dropped operations invalidate paired comparisons. The report retains failures
and refuses an overall pass if p99, drain, throughput, memory, CPU or the 10%
relative regression gate fails. Do not change pool limits to conceal a failure.
## Required native usage acceptance
Built-in native remote Runner execution accepts structured accounting before
its PRP transport ACK. `usage.reported`, turn-start, and turn-terminal facts
produce immutable `run_usage_receipts` rows. The receipt stores only structured
usage and turn identity. It does not depend on output logs or raw trace capture.
The source UUID derives from the run and Runner identity. Provider-turn IDs and
turn-start source facts preserve aggregation when several provider turns share
one PRP envelope turn. Source event IDs and
sequences remain stable across receiver replacement. Repeated delivery checks
the saved fact and does not create another receipt for the same report.
A failed receipt transaction rejects acceptance. The remote Runner keeps its
outbox report and reconnects through its existing protocol lifecycle. A lost
commit response can replay the same immutable receipt. A replacement receiver
restores accepted per-turn aggregation from Postgres, then accepts pending
reports from the same producer. The existing source fence prevents a retired
producer from replacing the current source's usage. Missing or partial usage
keeps accounting pending; it does not produce a zero-cost charge.
The native session authority epoch supplies the current run's accounting owner
to reused warm transports. The usual controller checkpoint and generation checks
still control transport acknowledgement and attachment. Ledger settlement,
pricing, reservations, and budget enforcement retain their existing owners.
No accounting receipt directory is created for this supported remote path.
Local native execution, legacy adapters, and external Dot/Muse execution retain
their existing recorder behavior. This change does not establish disposable-web
accounting durability for those paths. Server-originated model calls keep their
existing admission and settlement records. No S3 accounting spool is introduced.
The independent run-log/trace storage change can consume its own persisted
sources; output acceptance is not accounting acceptance.
Run the focused PostgreSQL/Runner experiment with:
```sh
pnpm exec vitest run server/src/services/native-runtime/native-usage-delivery.integration.test.ts --maxWorkers=1
```
The provider fixture emits a real PRP usage report through the durable Rust
Runner. A PostgreSQL trigger fails receipt insertion. The test removes the first
receiver directory, replaces its accounting owner, reconnects the same Runner,
and independently reads the receipt and ledger. The provider is controlled;
no paid provider request is made. Additional cases cover lost commit responses,
replay conflicts, per-turn aggregation, partial usage, source replacement, and
budget holds. Optional log and trace callbacks are absent from the experiment.
The cold fixture build runs before opening PostgreSQL, with the existing native
integration build budget. PR CI routes this suite through the existing cached
native Runner lane. CI builds the release fixture; local execution uses debug.
The behavior test has its own bounded receiver/process deadline after compilation.
### Codex usage authority and release order
The qualified Codex producer keeps its current-run baseline and latest session
totals in its existing durable provider state. It observes historical resume
snapshots, preserves that baseline across process recovery, and resets the
baseline to the latest totals when attaching the next run. Multiple request
reports therefore emit the whole run delta, rather than the provider's last
request. Regressing or incomplete live counters cannot certify completion.
Authoritative Codex reports carry `runDeltaBasis: "run"`. This requires a Runner
artifact built from the provider baseline implementation in this change. A
version string alone does not establish support; use the qualified artifact
digest. Existing unmarked Codex reports remain incomplete and unpriced in the
new accounting path. Their budget debt remains pending for reconciliation.
No deployed Runner artifact has been qualified by this source-only change.
PRP negotiates envelope versions and the admitted Runner digest. It does not
negotiate this usage payload field. The existing event envelope validators leave
usage payloads open, but an old standalone usage-schema consumer rejects this
new field. Update receivers and strict usage-schema consumers first, then
release the matching Runner artifact before claiming this durability guarantee.
Finish or explicitly reconcile older admitted Codex runs during cutover. Do not
claim that an old producer's final-request delta is a whole-run receipt.
Heartbeat completion reads the current immutable accepted receipt, including its
saved exact rate-card amount and provenance. It does not reprice with current
rates or replace that accepted estimate with an absent provider dollar amount.
The terminal UPDATE preserves the current producer-owned accounting projection
atomically. If another report arrived after completion read its checkpoint, the
newer receipt, readiness, price, and budget exposure win; unrelated session and
completion metadata can still update. A newer partial receipt stays pending.
Non-accounting attachment control events keep their existing handoff acceptance;
an old run's accounting report still cannot acquire a successor's authority.
The provider drains its bounded settled-turn usage tail before publishing the
terminal event that closes the run. An authoritative active goal keeps its
existing continuation path unless an uncorrected completion rejection closes
the run. The
same owner predicate selects both final usage drain and terminal outcome. A
completed turn remains authoritative after a
fully drained provider exit; ordinary session-exit handling still records an
unavailable process independently. This keeps a late cumulative increase on the old run before the
next run resets its baseline. Readiness and attachment retain accounting facts
in the existing provider checkpoint instead of discarding them, including facts
consumed before a later quiescence error. An increase
that arrives after authoritative completion requires explicit reconciliation:
PostgreSQL rejects changed finalized native usage or authoritative monetary evidence, so the existing Runner
outbox retains it under the old source identity without another charge. Equal
snapshots are harmless. No recurring scan or automatic adjustment is added.
If delayed usage arrives after attachment and cannot belong to the current
provider turn, the provider retains its cumulative evidence and the matching
saved prior identity, leaves the successor baseline unchanged, and emits a
failed run terminal. The existing reconciliation lifecycle blocks further work,
including after restart. Unknown older ownership stays explicit rather than
being inferred from provider-supplied run identity. Manual reconciliation or a
fresh session is required for this rare unsupported timing case.
Prior-turn increases are checked against the saved attachment baseline, even
when a successor has already reported a higher session total. Required usage
bypasses the optional diagnostic limit. An affected successor publishes an
explicit incomplete usage report before failure; this preserves known budget
exposure without settling ambiguous usage. That required report and its receipt
retain the accounting measurements and matching saved prior identity, so an
optional diagnostic acknowledgement cannot discard reconciliation evidence.
+3
View File
@@ -4,6 +4,9 @@ summary: Local disk vs S3-compatible storage
---
Paperclip stores uploaded files (issue attachments, images) using a configurable storage provider.
New run logs and explicitly requested provider traces reuse the same settings.
S3 instances store these under `run-logs/` and `provider-traces/` within the
existing prefix. Local-disk instances keep them local.
## Local Disk (Default)
+3 -1
View File
@@ -46,7 +46,9 @@ Review: `baseline`.
API: `present` — representative operation IDs:
- `GET /api/issues/{id}/comments`.
- `GET /api/issues/{id}/recent-comments` (20 contiguous comments for a standing chat).
- `GET /api/issues/{id}/comments` (cursor pages for older history).
- `GET /api/issues/{id}/runs?runIds=...` (at most 100 requested IDs, constrained to the accessible issue and company).
- `POST /api/issues/{id}/comments`.
- `GET /api/issues/{id}/decision-receipts` (own standing-chat board read).
- `GET /api/issues/{id}/chat-handoffs` (own standing-chat board read, paginated).
+8
View File
@@ -105,6 +105,14 @@ API: `present` — representative operation IDs:
- `GET /api/companies/{companyId}/decision-queues`.
- `GET /api/companies/{companyId}/decision-queues/{key}/items`.
- `GET /api/companies/{companyId}/attention/summary` — canonical outstanding
counts without decision cards, images, plan bodies, or pagination. Board user
and company access are required. Task privacy, addressee, dismissal, snooze,
archive, and complete-set counting rules match the attention feed. This read
grants no agent or resolution authority.
- `GET /api/companies/{companyId}/attention/tasks` — the same canonical counts
and distinct accessible task IDs for Needs attention. It has the same board,
company, privacy, addressee, dismissal, snooze, and archive gates as the feed.
- `GET /api/companies/{companyId}/attention/history` — read-only canonical
interaction/Decisions-v1 projection with cursor, search, outcome, agent, project,
date, and chronological sort filters. Each source and v1 target must pass task
@@ -74,6 +74,9 @@ API: `present` — representative operation IDs:
- `PATCH /api/instance/settings`.
- `GET /api/instance/settings/experimental`.
- `PATCH /api/instance/settings/experimental`.
- `GET /api/adapters`.
- `GET /api/adapters/{type}`.
- `PATCH /api/adapters/{type}`.
Agent access: `restricted` — Board/admin instance settings; CLI configure/env; local
configuration files.
@@ -88,9 +91,38 @@ Verification: `not-run` — assert the documented agent denial with an owned tes
verify any listed agent request/result path separately. A delegated operation needs its
own policy and access review.
Cloud runtime configuration uses a separate private PostgreSQL singleton and
adds no public read or mutation endpoint. Existing instance settings APIs do not
expose it. Bootstrap database credentials and trust keys remain in the supplied
environment; the strict saved allowlist excludes API keys and connection URLs.
Internal configuration updates require a matching revision and transaction lock.
[Database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) cover
seed/restart, credential exclusion, conflicts and rollback. These are local
controls, not agent authorization or deployed Cloud qualification.
Gaps: Settings APIs cover parts of configuration; pre-start database/binding/storage
configuration is local operator tooling.
Adapter toggles require instance-admin authority and remain blocked when
`instance.adapters` is hidden. Ordinary board members can read the catalog;
agent keys cannot toggle adapters. Managed Cloud preferences are stored in a
private database singleton. The operator adapter declaration sets the effective
list and rejects conflicting toggles. [PostgreSQL regression coverage](../../server/src/services/adapter-settings-postgres.test.ts)
exercises persistence, concurrent writers, admission freshness, activity audit,
and failed writes; [authorization tests](../../server/src/__tests__/adapter-routes-authz.test.ts)
cover admin and hidden-setting gates. These are local tests, not deployed proof.
Cloud configuration migration and package qualification are operator commands,
not agent-accessible REST capabilities. They require the managed environment and
database operator access. The [command](../../server/src/qualify-cloud-configuration.ts)
composes transaction-safe owner seeds and writes a private receipt. Every
qualification rereads current revisions and image package evidence; a completed
historical receipt grants no deployment or execution authority. [PostgreSQL coverage](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts)
uses independent clients, lost-home reconstruction, a real replacement fixture
worker, rollback/retry, and durable blockers. [Reader coverage](../../server/src/__tests__/cloud-configuration-readers.test.ts)
checks bounded sources and pnpm-style link integrity. These are local fixtures;
see the [scope and operator procedure](../../doc/cloud-configuration-migration.md).
Cloud stack Admins may update experimental settings through the trusted Cloud
actor without receiving other instance-admin powers. This uses the current
stack role on each request; company membership alone does not grant it.
+8 -1
View File
@@ -55,7 +55,14 @@ Policy: Company model discovery requires company access; account-backed discover
also enforces its account/pool and target-agent rules. Instance adapter inventory
and installation retain board/admin gates. Delegated managed
AI-account login operations are enumerated under `model-accounts`; they do not
grant raw host administration or execution-session reset.
grant raw host administration or execution-session reset. On Cloud-managed
instances, external adapter install, reinstall, reload, removal and override
operations return `403` with `adapter_install_platform_managed`, including for
instance admins. Built-in metadata remains available to board users. Startup
and UI parser reads do not consume saved external registrations or package code.
[Local fixture controls](../../server/src/adapters/cloud-external-adapters.test.ts)
and [route controls](../../server/src/__tests__/adapter-routes-authz.test.ts)
cover this boundary; they do not establish live agent or provider qualification.
Evidence: `source-only` — [implementation](../../server/src/routes/agents.ts#L6608) at
the reviewed commit above.
+4
View File
@@ -27,6 +27,8 @@ Open `/chats`, choose an agent, or use `/chats/:agentRef` and its conversation s
With Coordinator enabled, the composer exposes the shared model/thinking-level picker for supported agents. The agent stays fixed in this control; switching agents remains in the chat header. Choices save as conversation overrides on send, preserving the agent's configuration. A first message creates the conversation before saving settings and submitting through the ordered comments endpoint.
Standing chats open with the latest 20 contiguous messages. Scroll upward to fetch earlier messages, answer receipts, and created tasks. The loading indicator stays in the thread; history failures offer an inline Retry while preserving the draft and reading position. Opening a chat does not fetch its lifetime comment, activity, or run history. Already observed messages survive live refresh and Back within the query-cache session.
Coordinator answer receipts from this conversation jump to the answer bubble without leaving chat. Answer updates from an execution task link to that source task and its decision context. Compact document links read “Open [document title] in the sidebar”.
### `project-handoff`
@@ -43,6 +45,8 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Enab
### `agent-conversation`
Local browser recipe: [long-chat startup and scrolling](../../tests/perf/agent-chat/agent-chat-startup.spec.ts), with fixture and command details in the [performance README](../../tests/perf/agent-chat/README.md). It covers recent-only reveal, delayed and failed pages, Back, older comment links, and live refresh. Synthetic local data is not deployed acceptance evidence.
Automated: [chat landing](../../ui/src/pages/AgentChats.test.tsx) checks recent-chat routing; [conversation service](../../server/src/__tests__/agent-conversations.test.ts) covers persisted conversation behavior.
Automated: [composer picker](../../ui/src/components/task-chat/ComposerRunSettingsPicker.test.tsx) covers fixed-agent model/effort selection on desktop and mobile; [composer](../../ui/src/components/task-chat/TaskChatComposer.test.tsx) covers Coordinator gating and settings submission without reassignment. [Task detail](../../ui/src/pages/IssueDetail.test.tsx) verifies canonical first/existing-chat IDs, settings-before-comment ordering, and no comment submission when settings fail to save. These tests mock the API and do not prove provider execution with the chosen model.
+6
View File
@@ -92,6 +92,12 @@ approvals/runs resolve. Only the stable list placeholder should appear until the
initial sources settle; a failed-run row must not jump when tasks arrive. Release
visible-row details separately and compare row/title bounds. Mine reads its own
100-row index, with parent/run context by ID; sidebar badges use the count endpoint.
Coordinator desktop and mobile navigation share `/attention/summary` for the
outstanding count. Needs You owns its complete feed; Tasks → Needs attention
owns a compact complete task-ID index. Navigation reuses either page response. Coordinator keeps the optimized Inbox even when
the instance Streamlined UI setting is off. The loopback browser checks in
[collection startup](../../tests/perf/collections/startup.spec.ts) exercise these
reads and the initial task row budget. They do not establish deployed latency.
The [Inbox component checks](../../ui/src/pages/Inbox.test.tsx) cover the source
barrier and selected-source query; [badge tests](../../server/src/__tests__/inbox-dismissals.test.ts)
cover unread counts. Browser timing and layout proof remain separate from these tests.
+30
View File
@@ -63,8 +63,38 @@ unfinished application write must still prevent sleep after its client leaves.
### `settings-backup`
Cloud startup reads nonsecret application choices from a private, versioned
PostgreSQL singleton after a bounded one-time legacy config seed. A fresh
process and different home restore the saved choices without consulting a stale
`.env` file. Invalid saved data or an unavailable database prevents startup;
saved telemetry opt-out applies before client creation. The
[database controls](../../server/src/__tests__/cloud-runtime-config.test.ts) and
[startup-order controls](../../server/src/__tests__/server-startup-feedback-export.test.ts)
cover these local boundaries. See [field ownership](../../doc/cloud-runtime-configuration.md).
These controls do not qualify a deployed Cloud rollout or durable file storage.
Automated: [experimental settings](../../ui/src/pages/InstanceExperimentalSettings.test.tsx) and [database backup routes](../../server/src/__tests__/instance-database-backups-routes.test.ts) cover their UI/API contracts.
Cloud migration: in one maintenance window, stop legacy file writers and run the
[configuration/package qualification command](../../doc/cloud-configuration-migration.md).
Inspect its database receipt, update a saved preference, then repeat against a
second owned application/image root with the first home unavailable. Use an
explicit package evidence digest for same-package comparison. Confirm stale
qualification and missing legacy files cannot clear blockers. This recipe covers
configuration/packages and does not qualify whole-server zero downtime, secret
bytes, or volume removal. [Local PostgreSQL evidence path](../../server/src/__tests__/cloud-configuration-migration-postgres.test.ts)
includes a real fixture worker; it does not establish a hosted replacement.
Managed adapters: change a visible adapter preference as an instance admin,
then use a second application process against the same disposable database to
confirm catalog and new-agent selection reflect it. Verify saved agents remain
usable. Restart with an operator adapter declaration, confirm its effective
list, and reject conflicting toggles. Remove the declaration and confirm saved
preferences return. [Database tests](../../server/src/services/adapter-settings-postgres.test.ts)
cover local persistence, concurrent writers, audit rollback, and failure responses;
[local compatibility tests](../../server/src/services/adapter-settings-local.test.ts)
cover the legacy file. No deployed walkthrough is claimed.
Manual: Change a harmless experimental setting, reload, and confirm the affected route honors it. Back up a disposable instance, inspect the reported file and metadata, and restore using the documented procedure into a second isolated target. Verify a known record after restoration.
Cloud staging: On a managed `*.staging.paperclip.app` instance, verify experiments
+8
View File
@@ -44,6 +44,14 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use
Automated: [projects UI](../../ui/src/pages/Projects.test.tsx) and [project detail](../../ui/src/pages/ProjectDetail.test.tsx) are component tests.
Automated: the explicit [`project-task-context` deployment journey](../../tests/runner-e2e/product-journeys/project-context-flow.ts)
creates a project and a self-assigned task through the production dialogs. It
reopens both, verifies the project task list and task properties after reload,
and independently checks the saved company, project, prompt, assignment and
standard work mode with zero agent runs. [Oracle tests](../../tests/runner-e2e/product-journeys/project-context-evidence.test.ts)
reject lost links and mismatched records. This describes coverage; each target
still needs its own live pass before daily promotion.
Manual: Create a project, add a task from its view, and reload. Verify the task belongs to that project, list counts are plausible, and links preserve company scope. Archive only the disposable project and inspect list/task behavior.
### `repository-settings`
+10 -1
View File
@@ -170,6 +170,15 @@ controls do not establish live provider acceptance.
### `adapter-setup`
Cloud-managed instances load adapters from the release image. Saved external
registrations cannot replace built-ins or supply UI parser code. Install,
reinstall, reload, remove, and override operations remain platform-managed.
The [fixture loader controls](../../server/src/adapters/cloud-external-adapters.test.ts)
exercise real packages, import side effects, both managed-instance signals and
self-hosted compatibility. The [route authorization controls](../../server/src/__tests__/adapter-routes-authz.test.ts)
check managed denials before package or registration access. These are local
controls, not live Cloud provider qualification.
Automated: [adapter validation routes](../../server/src/__tests__/agent-adapter-validation-routes.test.ts) and [model refresh](../../server/src/__tests__/adapter-model-refresh-routes.test.ts) cover server behavior.
Claude native setup uses the existing Anthropic API-key or subscription connection
@@ -339,7 +348,7 @@ revised-source live onboarding and follow-up proof.
`/company/settings/connections` configures decision/fast-response models,
not the general model-account library.
- The run log is instance database data; it is not the opt-out first-party Telemetry system or operator-configured OpenTelemetry tracing.
- Structured run events stay in the instance database. Optional stdout/stderr and raw trace payload storage can use S3; see [run evidence storage](../../doc/run-evidence-storage.md). Both are separate from first-party Telemetry and operator OpenTelemetry export.
- Authentication/model discovery does not establish tool, image, steering, or resumption capability parity.
- Use [recovery](./recovery.md) for interrupted runs and [execution environments](./execution-environments.md) for local/SSH/sandbox differences.
+6
View File
@@ -27,6 +27,12 @@ checks only assignment to the signed-in user and persistence after reload. Its
source is reusable across self-hosted and Cloud; Cloud live qualification is
separate. It does not cover other property operations or agent authorization.
The [Product task-search-filter journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md)
checks `tasks/organize` with owned matching and nonmatching controls, exact task
identities, saved task content, and search/status persistence after reload and
detail/back navigation. It is model-free and reusable across deployment targets.
Live Cloud qualification is separate from the journey's assertion unit tests.
## How to get to it (user POV)
Cold collection loads use a lightweight index and batched hierarchy context,
+1
View File
@@ -101,6 +101,7 @@
"agentmail": "^0.5.31",
"cross-env": "^10.1.0",
"esbuild": "^0.28.2",
"oxc-parser": "0.127.0",
"typescript": "^7.0.2",
"vitest": "^5.0.3",
"yaml": "^2.8.1",
+7
View File
@@ -552,6 +552,13 @@ pnpm --filter @paperclipai/paperclip-runner verify
The verification command requires a stable Rust toolchain with `cargo` on
`PATH`, in addition to Node.js 24.11+ and pnpm 9+.
Rust development builds use `debug = "line-tables-only"` to reduce build storage
while retaining file names and line numbers in backtraces. The default test
profile inherits this setting. Incremental compilation remains enabled, and
release profiles keep their existing settings. For full debugger support,
including local variables and types, set `CARGO_PROFILE_DEV_DEBUG=2` and
`CARGO_PROFILE_TEST_DEBUG=2` for the build or test command.
Minimal Debian/Ubuntu hosts without root access can extract the required
Playwright browser libraries into a user-owned cache and run the same acceptance
sequence with:
@@ -73,17 +73,30 @@ real transport; it does not qualify a hosted OpenCode or ACPX deployment.
## Profile identity
Every ACPX provider's `commandDigest` attests only its own upstream release.
For Copilot (profile 92), Grok (20) and Pi (37) it binds the upstream package
versions, the platform closure pins and, for Grok, the launcher bytes.
`scripts/acpx-release-attestations.mjs` computes these digests and
`generate:acpx-profiles` writes them. Runner source changes do not change them.
Every ACPX harness binds the shared ACPX layer the same way: saved inputs must
match the current ACPX version, and the provider pack's `contractDigest` binds
the Runner's ACPX patch with the other Runner contracts.
Saved inputs with any profile version from 1 through the current one decode, so
they reach recovery. Launch and recovery admission require the exact current
profile and digest, and refuse the rest. A profile identity does not establish
hosted harness qualification.
Each provider declares how in `ACPX_RELEASE_ATTESTATIONS`
(`scripts/acpx-release-attestations.mjs`):
| Kind | Providers | The digest is |
| --- | --- | --- |
| `installed-command` | Claude, Codex | the upstream package's launch command, hashed at installation |
| `launcher` | Grok | the SHA-256 of the launcher shipped verbatim |
| `closure` | Copilot, Pi | upstream versions and the complete platform closure pins |
| `declaration` | Cursor, Antigravity | a checked-in release declaration, as canonical JSON |
`generate:acpx-profiles` writes the computed digests and `check:acpx-profiles`
refuses a stale one, so a digest changes only with its upstream release. Every
ACPX harness binds the shared ACPX layer the same way: saved inputs must match
the current ACPX version, and the provider pack's `contractDigest` binds the
Runner's ACPX patch with the other Runner contracts. Runner source changes
change no provider digest.
A profile version changes only to reset every saved session for that provider
on purpose; an upstream release keeps it, and a new provider starts at 1. Saved
inputs with any version from 1 through the current one decode, so they reach
recovery. Launch and recovery admission require the exact current profile and
digest, and refuse the rest. A profile identity does not establish hosted
harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc)
summarizes these practices for contributors.
## External controller checkpoints
@@ -140,7 +140,7 @@
{
"path": "schemas/usage.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/usage.schema.json",
"sha256": "34b1478d2054f898d64b54b62a9bb738670087893bb1e5967da541539d2daf36"
"sha256": "be9609768b224459546b800236045b375448f2cb2cb47ccb337e149e0b016361"
},
{
"path": "schemas/workspace-diff.schema.json",
@@ -11,6 +11,73 @@
"providerRequestId": { "type": ["string", "null"], "maxLength": 240 },
"cumulative": { "$ref": "#/$defs/measurement" },
"runDeltaAvailable": { "type": "boolean" },
"runDeltaBasis": { "const": "run" },
"reconciliation": {
"type": "object",
"required": [
"reason",
"receivedTurnId",
"cumulative",
"priorOrigin"
],
"properties": {
"reason": {
"const": "accounting_reconciliation_required"
},
"receivedTurnId": {
"type": [
"string",
"null"
],
"maxLength": 240
},
"cumulative": {
"$ref": "#/$defs/measurement"
},
"priorOrigin": {
"type": [
"object",
"null"
],
"required": [
"runnerInstanceId",
"runId",
"normalizedSessionId",
"turnId",
"itemId"
],
"properties": {
"runnerInstanceId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"normalizedSessionId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"turnId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"itemId": {
"type": "string",
"minLength": 1,
"maxLength": 240
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"runDelta": { "$ref": "#/$defs/measurement" }
},
"additionalProperties": false,
@@ -2,6 +2,9 @@
members = ["crates/runner-core"]
resolver = "2"
[profile.dev]
debug = "line-tables-only"
[workspace.package]
version = "0.0.0"
edition = "2021"
@@ -21,6 +21,8 @@ struct Script {
retention_limit: usize,
release_file: PathBuf,
receipt_file: PathBuf,
#[serde(default)]
accounting: bool,
}
#[derive(Default)]
@@ -86,11 +88,37 @@ fn workload(script: Script, shared: Shared) {
digest = Sha256::digest(digest);
}
work = lock.lock().unwrap();
let (event_type, payload) = if script.accounting {
if step == 1 {
(
"usage.reported",
json!({"provider":"openai", "model":"test-model",
"runDeltaAvailable":true, "runDeltaBasis":"run", "runDelta":{"inputTokens":120,"outputTokens":10,"providerCostUsd":0.25}}),
)
} else if step == 2 {
("turn.completed", json!({"status":"completed"}))
} else {
(
"usage.reported",
json!({"provider":"openai","model":"test-model","runDeltaAvailable":true,"runDeltaBasis":"run",
"runDelta":{"inputTokens":150,"outputTokens":15,"providerCostUsd":0.5}}),
)
}
} else {
(
"item.delta",
json!({"text": format!("step={step} sha256={digest:x}\n")}),
)
};
work.events.push_back(PolledEvent {
executor_event_id: format!("fixture-step-{step}"),
event_type: "item.delta".to_owned(),
priority: EventPriority::P2,
payload: json!({"text": format!("step={step} sha256={digest:x}\n")}),
event_type: event_type.to_owned(),
priority: if script.accounting {
EventPriority::P0
} else {
EventPriority::P2
},
payload,
});
work.completed = step;
work.peak_retained = work.peak_retained.max(work.events.len());
@@ -333,20 +333,60 @@ fn finish_turn_with_send(
"text": "Codex completed the fake turn."
}}
}))?;
let cumulative_accounting = std::env::args().any(|arg| arg == "--cumulative-accounting-usage");
let warm_tail_accounting = std::env::args().any(|arg| arg == "--warm-tail-accounting");
let total_input = if warm_tail_accounting {
if state.next_turn == 1 {
100
} else {
150
}
} else if cumulative_accounting {
100 + state.next_turn * 100
} else {
12
};
let total_output = if warm_tail_accounting {
if state.next_turn == 1 {
10
} else {
15
}
} else if cumulative_accounting {
20 + state.next_turn * 10
} else {
3
};
if cumulative_accounting {
send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":state.thread_id,"turnId":turn_id,
"tokenUsage":{"total":{"inputTokens":total_input-40,"outputTokens":total_output-4},"last":{"inputTokens":10,"outputTokens":1}}}}),
)?;
}
send(json!({
"method": "thread/tokenUsage/updated",
"params": {
"threadId": state.thread_id,
"tokenUsage": {
"total": {"inputTokens": 12, "outputTokens": 3},
"last": {"inputTokens": 12, "outputTokens": 3, "requests": 1}
"total": {"inputTokens": total_input, "outputTokens": total_output},
"last": {"inputTokens": if cumulative_accounting {40} else {12}, "outputTokens": if cumulative_accounting {4} else {3}, "requests": 1}
}
}
}))?;
if state.next_turn > 1 && std::env::args().any(|arg| arg == "--hold-second-accounting-terminal")
{
return Ok(());
}
send(json!({
"method": "turn/completed",
"params": {"turn": {"id": turn_id, "status": status}}
}))?;
if std::env::args().any(|arg| arg == "--accounting-tail-after-terminal") {
send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":state.thread_id,"turnId":turn_id,
"tokenUsage":{"total":{"inputTokens":120,"outputTokens":12},"last":{"inputTokens":20,"outputTokens":2}}}}),
)?;
}
Ok(())
}
@@ -1285,7 +1325,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
for _ in 0..2 {
send(json!({"method": "thread/tokenUsage/updated", "params": {
"threadId": state.thread_id, "turnId": format!("provider-turn-{}", state.next_turn),
"tokenUsage": {"total": {"inputTokens": 100, "outputTokens": 20}, "last": {"inputTokens": 50, "outputTokens": 10}}
"tokenUsage": {"total": {"inputTokens": if args.iter().any(|arg| arg == "--cumulative-accounting-usage") {100 + state.next_turn * 100} else {100}, "outputTokens": if args.iter().any(|arg| arg == "--cumulative-accounting-usage") {20 + state.next_turn * 10} else {20}}, "last": {"inputTokens": 50, "outputTokens": 10}}
}}))?;
}
}
@@ -1889,6 +1929,43 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
}))?;
}
if state.next_turn == 1 {
if let Some(gate) =
argument(&args, "--late-accounting-tail-gate").map(PathBuf::from)
{
let thread_id = state.thread_id.clone();
let turn_id = provider_turn_id.clone();
let unsafe_tail =
args.iter().any(|arg| arg == "--unsafe-after-late-usage");
let duplicate_burst = args
.iter()
.any(|arg| arg == "--late-accounting-duplicate-burst");
thread::spawn(move || {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while !gate.is_file() && std::time::Instant::now() < deadline {
thread::sleep(Duration::from_millis(1));
}
if gate.is_file() {
if duplicate_burst {
for _ in 0..33 {
let _ = send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":thread_id,"turnId":turn_id,"tokenUsage":{"total":{"inputTokens":100,"outputTokens":10}}}}),
);
}
}
let _ = send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":thread_id,"turnId":turn_id,
"tokenUsage":{"total":{"inputTokens":120,"outputTokens":12},"last":{"inputTokens":20,"outputTokens":2}}}}),
);
if unsafe_tail {
let _ = send(
json!({"method":"turn/started","params":{"threadId":thread_id,"turn":{"id":"unsafe-next-turn"}}}),
);
}
}
});
}
}
if emit_post_completion_warning {
send(json!({
"method": "warning",
@@ -1900,7 +1977,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
json!({
"method": "thread/tokenUsage/updated",
"params": {"threadId": state.thread_id, "turnId": provider_turn_id,
"tokenUsage": {"total": {"inputTokens": 120, "outputTokens": 12},
"tokenUsage": {"total": {"inputTokens": if args.iter().any(|arg| arg == "--warm-tail-accounting") && state.next_turn > 1 {150} else {120}, "outputTokens": if args.iter().any(|arg| arg == "--warm-tail-accounting") && state.next_turn > 1 {15} else {12}},
"last": {"inputTokens": 20, "outputTokens": 2}}}
}),
json!({
@@ -1342,6 +1342,7 @@ impl CodexProvider {
&mut self,
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
completion_contract: Option<(&str, &[String])>,
accounting: &mut Vec<(String, Value)>,
) -> Result<bool, LocalRunnerError> {
let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();
let completion_contract =
@@ -1354,7 +1355,15 @@ impl CodexProvider {
{
return Ok(false);
}
let blockers = self.warm_run_attachment_blockers(true)?;
let (tail, result) = self.drain_completed_turn_tail_for_warm_attachment();
accounting.extend(tail);
result?;
if !accounting.is_empty() {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment requires pending accounting reconciliation",
));
}
let blockers = self.warm_run_attachment_blockers()?;
if !blockers.is_empty() {
return Err(LocalRunnerError::invalid(
format!(
@@ -1375,49 +1384,53 @@ impl CodexProvider {
Ok(true)
}
fn drain_completed_turn_tail_for_warm_attachment(&mut self) -> Result<(), LocalRunnerError> {
let Some(completed_turn_id) = self
.completed_turn_authority
.as_ref()
.map(|authority| authority.provider_turn_id.clone())
else {
return Ok(());
};
if self.active_provider_turn_id.is_some() {
return Ok(());
}
// Readiness probes run over the PRP command channel while provider
// stdout is drained by runnerd's adjacent control-loop iteration. A
// final usage/warning/item frame can therefore land after the last
// successful probe but before run.attach executes. Close that race in
// the same critical section as authority rotation. Only bounded tail
// notifications for the already-settled turn may be discarded: a new
// turn, provider request, process exit, or mismatched turn remains a
// fail-closed attachment error.
let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE;
let mut quiet_since: Option<std::time::Instant> = None;
let mut drained = 0usize;
loop {
if std::time::Instant::now() >= deadline {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail did not become quiescent",
));
pub(crate) fn drain_completed_turn_tail_for_warm_attachment(
&mut self,
) -> (Vec<(String, Value)>, Result<(), LocalRunnerError>) {
let mut accounting = Vec::new();
let result = (|| {
let Some(completed_turn_id) = self
.completed_turn_authority
.as_ref()
.map(|authority| authority.provider_turn_id.clone())
else {
return Ok(());
};
if self.active_provider_turn_id.is_some() {
return Ok(());
}
match self.poll()? {
Some(CodexProviderEvent::Notification { method, params }) => {
quiet_since = None;
drained = drained.saturating_add(1);
if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail exceeded its bounded frame limit",
));
}
let names_other_turn = notification_turn_id(&params)
.is_some_and(|turn_id| turn_id != completed_turn_id);
let safe_tail_method = matches!(
method.as_str(),
"warning"
// Readiness probes run over the PRP command channel while provider
// stdout is drained by runnerd's adjacent control-loop iteration. A
// final usage/warning/item frame can therefore land after the last
// successful probe but before run.attach executes. Close that race in
// the same critical section as authority rotation. Only bounded tail
// notifications for the already-settled turn may be discarded: a new
// turn, provider request, process exit, or mismatched turn remains a
// fail-closed attachment error.
let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE;
let mut quiet_since: Option<std::time::Instant> = None;
let mut drained = 0usize;
loop {
if std::time::Instant::now() >= deadline {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail did not become quiescent",
));
}
match self.poll()? {
Some(CodexProviderEvent::Notification { method, params }) => {
quiet_since = None;
drained = drained.saturating_add(1);
if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail exceeded its bounded frame limit",
));
}
let names_other_turn = notification_turn_id(&params)
.is_some_and(|turn_id| turn_id != completed_turn_id);
let safe_tail_method = matches!(
method.as_str(),
"warning"
| "configWarning"
| "remoteControl/status/changed"
| "mcpServer/startupStatus/updated"
@@ -1436,73 +1449,82 @@ impl CodexProvider {
| "thread/status/changed"
| "turn/diff/updated"
| "turn/plan/updated"
);
if names_other_turn || !safe_tail_method {
return Err(LocalRunnerError::invalid(format!(
);
if names_other_turn || !safe_tail_method {
return Err(LocalRunnerError::invalid(format!(
"Codex warm run attachment observed unsafe post-terminal provider method {}",
bounded_method(&method)
)));
}
if let Some(frame_id) = self.take_provider_trace_frame_id() {
self.record_provider_trace_interpretation(
}
if matches!(
method.as_str(),
"thread/tokenUsage/updated" | "paperclip/resumeUsageSnapshot"
) {
accounting.push((method, params));
continue;
}
if let Some(frame_id) = self.take_provider_trace_frame_id() {
self.record_provider_trace_interpretation(
frame_id,
"codex.warm_attachment.completed_turn_tail",
"ignored",
Vec::new(),
"Provider emitted a bounded tail notification after the prior turn terminal and before run attachment",
);
}
}
}
Some(CodexProviderEvent::ResourceLimit { .. }) => {
return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
}
Some(CodexProviderEvent::ProtocolFailure { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex protocol integrity failed during warm attachment",
));
}
Some(CodexProviderEvent::DescendantNotification { .. }) => {
// Child output cannot certify a quiescent root attachment.
return Err(LocalRunnerError::invalid(
"Codex descendant remains active during warm run attachment",
));
}
Some(CodexProviderEvent::ToolCall { .. })
| Some(CodexProviderEvent::RuntimeRequest { .. })
| Some(CodexProviderEvent::McpPermissionRequest { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment observed a post-terminal provider request",
));
}
Some(CodexProviderEvent::Exited { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex exited while quiescing for warm run attachment",
));
}
None => {
let now = std::time::Instant::now();
let quiet_start = quiet_since.get_or_insert(now);
if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW {
return Ok(());
Some(CodexProviderEvent::ResourceLimit { .. }) => {
return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
}
Some(CodexProviderEvent::ProtocolFailure { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex protocol integrity failed during warm attachment",
));
}
Some(CodexProviderEvent::DescendantNotification { .. }) => {
// Child output cannot certify a quiescent root attachment.
return Err(LocalRunnerError::invalid(
"Codex descendant remains active during warm run attachment",
));
}
Some(CodexProviderEvent::ToolCall { .. })
| Some(CodexProviderEvent::RuntimeRequest { .. })
| Some(CodexProviderEvent::McpPermissionRequest { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment observed a post-terminal provider request",
));
}
Some(CodexProviderEvent::Exited {
completed_turn_authoritative,
completion_reconciles_exit,
..
}) if completion_reconciles_exit
|| (completed_turn_authoritative && self.process.stdout_drained()) =>
{
return Ok(())
}
Some(CodexProviderEvent::Exited { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex exited while quiescing for warm run attachment",
));
}
None => {
let now = std::time::Instant::now();
let quiet_start = quiet_since.get_or_insert(now);
if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW {
return Ok(());
}
std::thread::sleep(Duration::from_millis(1));
}
std::thread::sleep(Duration::from_millis(1));
}
}
}
})();
(accounting, result)
}
pub(crate) fn warm_run_attachment_blockers(
&mut self,
quiesce_completed_tail: bool,
) -> Result<Vec<&'static str>, LocalRunnerError> {
// Only an explicit attachment-readiness probe may consume the bounded,
// already-settled provider suffix. Ordinary checkpoint snapshots run
// immediately after a terminal frame while the provider can still be
// unwinding; turning those observations into quiescence barriers can
// quarantine an otherwise reusable runner before its next turn.
if quiesce_completed_tail {
self.drain_completed_turn_tail_for_warm_attachment()?;
}
let mut blockers = Vec::new();
if self.process.try_wait()?.is_some() {
blockers.push("process_exited");
@@ -1517,7 +1539,18 @@ impl CodexProvider {
blockers.push("ambiguous_turn_start");
}
if !self.pending_messages.is_empty() {
blockers.push("pending_messages");
blockers.push(
if self.pending_messages.iter().any(|message| {
matches!(
message.value.get("method").and_then(Value::as_str),
Some("paperclip/resumeUsageSnapshot" | "thread/tokenUsage/updated")
)
}) {
"pending_accounting_reconciliation"
} else {
"pending_messages"
},
);
}
if !self.deferred_ambiguous_messages.is_empty() {
blockers.push("deferred_messages");
@@ -2919,13 +2952,11 @@ impl CodexProvider {
.and_then(Value::as_str);
if notification_turn_id.is_some()
&& notification_turn_id != self.active_provider_turn_id.as_deref()
&& notification_turn_id
&& (notification_turn_id
.is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
|| (self.active_provider_turn_id.is_none()
&& method == "thread/tokenUsage/updated"))
{
self.notification_identity_diagnostics += 1;
if self.notification_identity_diagnostics > 32 {
return Ok(None);
}
// Resume replays the cumulative usage of the last settled turn.
// Keep its identity and baseline, but never bill its `last`
// measurement to the newly attached run.
@@ -2941,6 +2972,10 @@ impl CodexProvider {
}),
}));
}
self.notification_identity_diagnostics += 1;
if self.notification_identity_diagnostics > 32 {
return Ok(None);
}
return Ok(Some(CodexProviderEvent::Notification {
method: "warning".to_owned(),
params: json!({
@@ -4392,9 +4427,58 @@ done
});
// poll() normalizes settled-turn usage to paperclip/resumeUsageSnapshot.
// That accounting fact is not new work and must not force replacement.
let result = provider.drain_completed_turn_tail_for_warm_attachment();
let (accounting, result) = provider.drain_completed_turn_tail_for_warm_attachment();
provider.shutdown().unwrap();
result.expect("historical usage must not break provider continuity");
result.expect("historical usage must remain available to its owner");
assert_eq!(accounting.len(), 1);
assert_eq!(accounting[0].1["total"]["inputTokens"], 120);
}
#[cfg(unix)]
#[test]
fn failed_warm_drain_retains_consumed_usage_before_unsafe_tail() {
for attach in [false, true] {
let mut provider = completion_tail_provider();
provider
.restore_completed_turn_authority(true, Some(1), Some("reader-tail-1"))
.unwrap();
provider.active_provider_turn_id = None;
for value in [
json!({"method":"thread/tokenUsage/updated","params":{"threadId":"reader-tail-thread","turnId":"reader-tail-1","tokenUsage":{"total":{"inputTokens":120,"outputTokens":12}}}}),
json!({"method":"turn/started","params":{"threadId":"reader-tail-thread","turn":{"id":"unsafe-next-turn"}}}),
] {
provider
.pending_messages
.push_back(BufferedProviderMessage {
value,
trace_frame_id: None,
});
}
let (accounting, result) = if attach {
let mut accounting = Vec::new();
let tools = provider.authorized_tools.clone();
let contract = provider.completion_contract.clone();
let result = provider
.attach_run_in_place(
tools,
contract.as_ref().map(|contract| {
(
contract.revision.as_str(),
contract.criterion_ids.as_slice(),
)
}),
&mut accounting,
)
.map(|_| ());
(accounting, result)
} else {
provider.drain_completed_turn_tail_for_warm_attachment()
};
provider.shutdown().unwrap();
assert!(result.is_err());
assert_eq!(accounting.len(), 1);
assert_eq!(accounting[0].1["total"]["inputTokens"], 120);
}
}
#[cfg(unix)]
@@ -0,0 +1,234 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
pub(crate) struct CodexRunUsage {
baseline: Option<BTreeMap<String, Value>>,
latest: BTreeMap<String, Value>,
}
impl CodexRunUsage {
pub(crate) fn new(requires_baseline: bool) -> Self {
Self {
baseline: (!requires_baseline).then(BTreeMap::new),
latest: BTreeMap::new(),
}
}
pub(crate) fn is_valid(&self) -> bool {
self.baseline
.iter()
.flat_map(|values| values.values())
.chain(self.latest.values())
.all(|value| {
value
.as_f64()
.is_some_and(|value| value.is_finite() && value >= 0.0)
})
}
pub(crate) fn increases_known_usage(&self, total: &Value) -> bool {
!self.latest.is_empty()
&& total.as_object().is_some_and(|total| {
total.iter().any(|(key, value)| {
value.as_f64().is_some_and(|value| {
value > self.latest.get(key).and_then(Value::as_f64).unwrap_or(0.0)
})
})
})
}
pub(crate) fn increases_run_baseline(&self, total: &Value) -> bool {
self.baseline.as_ref().is_some_and(|baseline| {
total.as_object().is_some_and(|total| {
total.iter().any(|(key, value)| {
value.as_f64().is_some_and(|value| {
value > baseline.get(key).and_then(Value::as_f64).unwrap_or(0.0)
})
})
})
})
}
pub(crate) fn has_baseline(&self) -> bool {
self.baseline.is_some()
}
pub(crate) fn begin_run(&mut self) {
if self.latest.is_empty() {
return;
}
self.baseline = (valid_token(self.latest.get("inputTokens"))
&& valid_token(self.latest.get("outputTokens")))
.then(|| self.latest.clone());
}
pub(crate) fn observe(&mut self, total: &Value, historical: bool) -> (Value, bool) {
let measurement: BTreeMap<String, Value> = total
.as_object()
.into_iter()
.flat_map(|value| value.iter())
.filter(|(_, value)| {
value
.as_f64()
.is_some_and(|value| value.is_finite() && value >= 0.0)
})
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
let complete = self.baseline.is_some()
&& (historical
|| measurement.iter().all(|(key, value)| {
!key.ends_with("Tokens")
|| value.as_u64().is_some_and(|value| {
value >= self.latest.get(key).and_then(Value::as_u64).unwrap_or(0)
&& value
>= self
.baseline
.as_ref()
.and_then(|baseline| baseline.get(key))
.and_then(Value::as_u64)
.unwrap_or(0)
})
}))
&& valid_token(measurement.get("inputTokens"))
&& valid_token(measurement.get("outputTokens"))
&& measurement
.iter()
.all(|(key, value)| !key.ends_with("Tokens") || valid_token(Some(value)))
&& self
.latest
.keys()
.all(|key| !key.ends_with("Tokens") || valid_token(measurement.get(key)));
if historical
&& self.baseline.is_none()
&& valid_token(measurement.get("inputTokens"))
&& valid_token(measurement.get("outputTokens"))
{
self.baseline = Some(measurement.clone());
}
for (key, value) in measurement {
if value.as_f64().unwrap()
>= self.latest.get(&key).and_then(Value::as_f64).unwrap_or(0.0)
{
self.latest.insert(key, value);
}
}
let delta: serde_json::Map<String, Value> = self
.latest
.iter()
.map(|(key, value)| {
let baseline = self
.baseline
.as_ref()
.and_then(|baseline| baseline.get(key));
let delta = if let Some(value) = value.as_u64() {
json!(value.saturating_sub(baseline.and_then(Value::as_u64).unwrap_or(0)))
} else {
json!((value.as_f64().unwrap()
- baseline.and_then(Value::as_f64).unwrap_or(0.0))
.max(0.0))
};
(key.clone(), delta)
})
.collect();
(Value::Object(delta), complete)
}
}
fn valid_token(value: Option<&Value>) -> bool {
value
.and_then(Value::as_u64)
.is_some_and(|value| value <= 9_007_199_254_740_991)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_resumed_baselines_across_requests_recovery_and_attachment() {
let mut usage = CodexRunUsage::new(true);
usage.observe(
&json!({"inputTokens":100,"outputTokens":10,"cachedInputTokens":20}),
true,
);
let (first, complete) = usage.observe(
&json!({"inputTokens":140,"outputTokens":16,"cachedInputTokens":25}),
false,
);
assert!(complete);
assert_eq!(first["inputTokens"], 40);
let mut restored: CodexRunUsage =
serde_json::from_value(serde_json::to_value(&usage).unwrap()).unwrap();
let (second, complete) = restored.observe(
&json!({"inputTokens":190,"outputTokens":25,"cachedInputTokens":30}),
false,
);
assert!(complete);
assert_eq!(
second,
json!({"inputTokens":90,"outputTokens":15,"cachedInputTokens":10})
);
restored.begin_run();
let (next, complete) = restored.observe(
&json!({"inputTokens":220,"outputTokens":29,"cachedInputTokens":32}),
false,
);
assert!(complete);
assert_eq!(
next,
json!({"inputTokens":30,"outputTokens":4,"cachedInputTokens":2})
);
}
#[test]
fn attachment_before_paid_work_preserves_a_proven_empty_baseline() {
let mut usage = CodexRunUsage::new(false);
usage.begin_run();
let (delta, complete) = usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false);
assert!(complete);
assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10}));
let mut unknown = CodexRunUsage::new(true);
unknown.begin_run();
assert!(
!unknown
.observe(&json!({"inputTokens":100,"outputTokens":10}), false)
.1
);
}
#[test]
fn regressing_live_totals_remain_incomplete_without_losing_known_exposure() {
let mut usage = CodexRunUsage::new(false);
usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false);
let (delta, complete) = usage.observe(&json!({"inputTokens":90,"outputTokens":9}), false);
assert!(!complete);
assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10}));
let mut resumed = CodexRunUsage::new(true);
resumed.observe(&json!({"inputTokens":100,"outputTokens":10}), true);
assert!(
!resumed
.observe(&json!({"inputTokens":90,"outputTokens":9}), false)
.1
);
}
#[test]
fn unknown_baselines_and_missing_counters_cannot_certify_usage() {
let mut usage = CodexRunUsage::new(true);
assert!(
!usage
.observe(&json!({"inputTokens":140,"outputTokens":16}), false)
.1
);
usage.observe(&json!({"inputTokens":100,"outputTokens":10}), true);
assert!(!usage.observe(&json!({"inputTokens":160}), false).1);
assert!(
usage
.observe(&json!({"inputTokens":170,"outputTokens":20}), false)
.1
);
}
}
@@ -13,6 +13,7 @@ pub mod aws_agentcore_provider;
pub mod claude_managed_provider;
pub mod codex_provider;
mod codex_startup_trust;
mod codex_usage;
mod configured_environment;
mod credential_observer;
pub mod dot_provider_backend;
@@ -17,6 +17,7 @@ use crate::codex_provider::{
ProviderStartupObservation, ProviderStartupStage, RejectedAcceptedTurn,
MAX_SETTLED_PROVIDER_TURN_IDS,
};
use crate::codex_usage::CodexRunUsage;
use crate::durable::{
create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value,
validate_semantic_tool_input, verify_private_directory, Command, CommandExecution,
@@ -93,6 +94,13 @@ struct ProviderEventIdentity {
item_id: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct CodexAccountingOrigin {
identity: ProviderEventIdentity,
provider_turn_id: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ProviderStartupCommand {
@@ -608,6 +616,105 @@ fn normalize_provider_notification(
method: &str,
params: &Value,
) -> Result<Vec<NormalizedProviderEvent>, DurableRunnerError> {
if state.config.provider == "codex"
&& ["thread/tokenUsage/updated", "paperclip/resumeUsageSnapshot"].contains(&method)
{
let historical = method == "paperclip/resumeUsageSnapshot";
let total = if historical {
params.get("total")
} else {
params.pointer("/tokenUsage/total")
}
.unwrap_or(&Value::Null);
let increases_known_usage = state
.codex_usage
.as_ref()
.is_some_and(|usage| usage.increases_known_usage(total));
let received_turn = params.get("turnId").and_then(Value::as_str);
let increases_prior_baseline = state.previous_accounting_origin.is_some()
&& received_turn != state.completed_provider_turn_id.as_deref()
&& state
.codex_usage
.as_ref()
.is_some_and(|usage| usage.increases_run_baseline(total));
if historical
&& (increases_known_usage || increases_prior_baseline)
&& params.get("turnId").and_then(Value::as_str)
!= state.active_provider_turn_id.as_deref()
{
state.lifecycle = "reconciliation_required".to_owned();
let owns_completed_turn = state.active_provider_turn_id.is_none()
&& state
.completed_provider_turn_id
.as_deref()
.is_some_and(|id| params.get("turnId").and_then(Value::as_str) == Some(id));
if !owns_completed_turn {
if state.active_provider_turn_id.is_some() {
state.settle_active_provider_turn_identity()?;
}
state.active_provider_turn_id = None;
let mut invalidation = normalize_codex_notification("thread/tokenUsage/updated", &json!({
"threadId":params.get("threadId"), "model":state.config.model, "tokenUsage":{"total":total}
})).remove(0);
invalidation.payload["runDeltaBasis"] = json!("run");
invalidation.payload["reconciliation"] = json!({"reason":"accounting_reconciliation_required",
"receivedTurnId":params.get("turnId"),"cumulative":invalidation.payload["cumulative"],
"priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| received_turn == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity)});
return Ok(vec![
NormalizedProviderEvent {
event_type: "harness.diagnostic".to_owned(),
priority: EventPriority::P0,
payload: json!({"code":"codex_usage_accounting_integrity", "classification":"accounting_reconciliation_required",
"receivedTurnId":params.get("turnId"),"cumulative":total,"priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| params.get("turnId").and_then(Value::as_str) == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity),
"message":"Late provider usage cannot be attributed to the attached run; explicit reconciliation is required"}),
},
invalidation,
NormalizedProviderEvent {
event_type: "turn.failed".to_owned(),
priority: EventPriority::P0,
payload: json!({"provider":"codex","status":"failed","code":"codex_usage_accounting_integrity","recoverable":false,
"message":"Late provider usage requires explicit accounting reconciliation"}),
},
NormalizedProviderEvent {
event_type: "run.terminal".to_owned(),
priority: EventPriority::P0,
payload: json!({"schema":"paperclip.prp.terminal.v1","provider":"codex","turnTerminalState":"failed","runTerminalState":"failed","reportedWorkDisposition":"needs_review"}),
},
]);
}
}
let had_baseline = state
.codex_usage
.as_ref()
.is_some_and(CodexRunUsage::has_baseline);
let (delta, complete) = state
.codex_usage
.get_or_insert_with(|| CodexRunUsage::new(true))
.observe(total, historical);
let mut normalized_params = params.clone();
let emit_usage = !historical || (had_baseline && increases_known_usage);
if historical && emit_usage {
normalized_params = json!({"threadId": params.get("threadId"), "turnId": params.get("turnId"),
"tokenUsage":{"total":total,"last":delta}});
} else if !historical {
normalized_params["tokenUsage"]["last"] = delta;
}
let mut events = normalize_codex_notification(
if emit_usage {
"thread/tokenUsage/updated"
} else {
method
},
&normalized_params,
);
for event in &mut events {
if event.event_type == "usage.reported" {
event.payload["runDeltaAvailable"] = json!(complete);
event.payload["runDeltaBasis"] = json!("run");
}
}
return Ok(events);
}
if method != "paperclip/runResult" {
return Ok(normalize_codex_notification(method, params)
.into_iter()
@@ -640,14 +747,21 @@ fn normalize_provider_notification(
}
}
fn terminal_completion_rejected(state: &CodexProviderState) -> bool {
state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none()
}
fn goal_continues_after_turn(state: &CodexProviderState, goal_active: bool) -> bool {
goal_active && !terminal_completion_rejected(state)
}
fn terminal_events(
state: &CodexProviderState,
event_type: &str,
goal_status: Option<&str>,
) -> Vec<NormalizedProviderEvent> {
let rejected_completion =
state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none();
if goal_status == Some("active") && !rejected_completion {
let rejected_completion = terminal_completion_rejected(state);
if goal_continues_after_turn(state, goal_status == Some("active")) {
return Vec::new();
}
let Some(contract) = state.completion_contract.as_ref() else {
@@ -1007,6 +1121,10 @@ struct CodexProviderState {
#[serde(default)]
opencode_launch_profile_digest: Option<String>,
#[serde(default)]
codex_usage: Option<CodexRunUsage>,
#[serde(default)]
previous_accounting_origin: Option<CodexAccountingOrigin>,
#[serde(default)]
completion_contract: Option<CompletionContractBinding>,
#[serde(default)]
tool_bridge: ProviderToolBridge,
@@ -1107,11 +1225,15 @@ impl CodexProviderState {
tool_bridge: ProviderToolBridge,
) -> Self {
let thread_id = config.provider_session_id.clone();
let codex_usage =
(config.provider == "codex").then(|| CodexRunUsage::new(thread_id.is_some()));
Self {
schema: PROVIDER_STATE_SCHEMA.to_owned(),
lifecycle: "prepared".to_owned(),
config,
opencode_launch_profile_digest: None,
codex_usage,
previous_accounting_origin: None,
completion_contract,
tool_bridge,
thread_id,
@@ -1154,6 +1276,13 @@ impl CodexProviderState {
self.tool_bridge.validate_recovered().map_err(|error| {
DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))
})?;
if self
.codex_usage
.as_ref()
.is_some_and(|usage| !usage.is_valid())
{
return Err(DurableRunnerError::invalid("invalid Codex usage baseline"));
}
let mut pending_event_ids = HashSet::new();
if self.descendant_thread_ids.len() > crate::codex_provider::MAX_DESCENDANT_THREAD_IDS
|| self
@@ -2501,7 +2630,43 @@ impl CodexCommandExecutor {
stable
}
fn persist_completed_accounting_tail(
&mut self,
tail: Vec<(String, Value)>,
failed: bool,
) -> Result<(), DurableRunnerError> {
if !tail.is_empty() || failed {
let state = self.state.as_mut().ok_or_else(|| {
DurableRunnerError::invalid("Codex provider has not been prepared")
})?;
for (method, params) in tail {
let events = normalize_provider_notification(state, &method, &params)?;
state.extend_events(events)?;
}
if failed {
state.lifecycle = "reconciliation_required".to_owned();
}
self.save_state()?;
}
Ok(())
}
fn retain_completed_accounting_tail(&mut self) -> Result<(), DurableRunnerError> {
let (tail, result) = self
.provider
.as_mut()
.map(CodexProvider::drain_completed_turn_tail_for_warm_attachment)
.unwrap_or_else(|| (Vec::new(), Ok(())));
self.persist_completed_accounting_tail(tail, result.is_err())?;
result.map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to retain completed Codex accounting: {error}"
))
})
}
fn attach_run(&mut self, payload: &Value) -> Result<(), DurableRunnerError> {
self.retain_completed_accounting_tail()?;
let mut next_state = self
.state
.clone()
@@ -2600,6 +2765,17 @@ impl CodexCommandExecutor {
"run.attach tool contract could not be rebound: {error}"
))
})?;
if let Some(usage) = next_state.codex_usage.as_mut() {
usage.begin_run();
}
next_state.previous_accounting_origin = self
.event_identity
.clone()
.zip(next_state.completed_provider_turn_id.clone())
.map(|(identity, provider_turn_id)| CodexAccountingOrigin {
identity,
provider_turn_id,
});
next_state.completion_contract = completion_contract;
next_state.completed_turn_authoritative = false;
next_state.completed_turn_process_generation = None;
@@ -2613,10 +2789,12 @@ impl CodexCommandExecutor {
next_state.active_provider_result_disposition = None;
next_state.active_terminal_tool_rejected = false;
next_state.last_agent_message = None;
let retained_provider = if let Some(provider) = self.provider.as_mut() {
!runtime_launch_changed
&& !upgraded_skill_config
&& provider
let mut attachment_accounting = Vec::new();
let retention = if let Some(provider) = self.provider.as_mut() {
if runtime_launch_changed || upgraded_skill_config {
Ok(false)
} else {
provider
.attach_run_in_place(
next_state.tool_bridge.authorized_tools().cloned(),
next_state.completion_contract.as_ref().map(|contract| {
@@ -2625,12 +2803,14 @@ impl CodexCommandExecutor {
contract.criterion_ids.as_slice(),
)
}),
&mut attachment_accounting,
)
.map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to retain Codex for warm run attachment: {error}"
))
})?
})
}
} else if next_state.lifecycle == "prepared"
&& next_state.provider_process_generation > 0
&& next_state.pending_events.iter().all(is_startup_audit_event)
@@ -2640,12 +2820,17 @@ impl CodexCommandExecutor {
// validated run-scoped settings here; open_session then restores
// that same thread under the new authority. Never restart during
// drain/suspend or require the stopped process to still exist.
false
Ok(false)
} else {
return Err(DurableRunnerError::invalid(
"run.attach requires the restored Codex provider process",
));
};
self.persist_completed_accounting_tail(attachment_accounting, retention.is_err())?;
let retained_provider = match retention {
Ok(retained) => retained,
Err(error) => return Err(error),
};
if !retained_provider {
if let Some(provider) = self.provider.as_mut() {
provider.shutdown().map_err(|error| {
@@ -4044,10 +4229,13 @@ impl CodexCommandExecutor {
.get("quiesceForWarmAttach")
.and_then(Value::as_bool)
.unwrap_or(false);
if quiesce_for_warm_attach {
self.retain_completed_accounting_tail()?;
}
let mut warm_attach_blockers = self
.provider
.as_mut()
.map(|provider| provider.warm_run_attachment_blockers(quiesce_for_warm_attach))
.map(|provider| provider.warm_run_attachment_blockers())
.transpose()
.map_err(|error| {
DurableRunnerError::invalid(format!(
@@ -4071,6 +4259,9 @@ impl CodexCommandExecutor {
if !state.queued_events.is_empty() {
warm_attach_blockers.push("durable_queued_events");
}
if state.lifecycle == "reconciliation_required" {
warm_attach_blockers.push("reconciliation_required");
}
let warm_attach_ready = warm_attach_blockers.is_empty();
Ok(CommandExecution::result(json!({
"status": state.lifecycle,
@@ -4281,6 +4472,22 @@ impl CodexCommandExecutor {
} else {
None
};
let goal_remains_active = match &goal_reconciliation {
Some(Ok(snapshot)) => normalize_codex_goal(snapshot, false)
.is_some_and(|goal| goal.status == "active"),
_ => self
.state
.as_ref()
.and_then(|state| state.goal.as_ref())
.is_some_and(|goal| goal.status == "active"),
};
let run_continues = self
.state
.as_ref()
.is_some_and(|state| goal_continues_after_turn(state, goal_remains_active));
if terminal_event_type.as_deref() == Some("turn.completed") && !run_continues {
self.retain_completed_accounting_tail()?;
}
let working_now = self
.provider
.as_ref()
@@ -4370,7 +4577,9 @@ impl CodexCommandExecutor {
state.receipt_limit_interrupt_attempts = 0;
state.receipt_limit_interrupt_deadline_unix_ms = None;
state.ambiguous_turn_start_pending = false;
state.lifecycle = "session_open".to_owned();
if state.lifecycle != "reconciliation_required" {
state.lifecycle = "session_open".to_owned();
}
if let Some(goal) = state.goal.as_mut() {
goal.working_now = false;
state.goal_revision = state.goal_revision.saturating_add(1);
@@ -4473,6 +4682,13 @@ impl CodexCommandExecutor {
})
.unwrap_or_default();
self.save_state()?;
if self
.state
.as_ref()
.is_some_and(|state| state.lifecycle == "reconciliation_required")
{
break;
}
if let (Some(frame_id), Some(provider)) =
(trace_frame_id, self.provider.as_mut())
{
@@ -5894,6 +6110,8 @@ mod tests {
conversation_mode: None,
},
opencode_launch_profile_digest: None,
codex_usage: Some(CodexRunUsage::new(false)),
previous_accounting_origin: None,
completion_contract: None,
tool_bridge: ProviderToolBridge::default(),
thread_id: Some("thread-1".to_owned()),
@@ -3964,6 +3964,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
"--durable-turn-ids",
"--emit-post-completion-warning",
"--emit-post-completion-passive-statuses",
"--warm-tail-accounting",
],
);
let runner_config = durable_config(&directory);
@@ -3991,6 +3992,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
))
.expect("start the first turn");
let mut old_usage = Vec::new();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
assert!(
@@ -3998,6 +4000,12 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
"the first turn must settle before attachment"
);
let events = poll_and_ack(&mut executor).expect("poll the first turn terminal");
old_usage.extend(
events
.iter()
.filter(|event| event.event_type == "usage.reported")
.cloned(),
);
if events
.iter()
.any(|event| event.event_type == "turn.completed")
@@ -4010,6 +4018,11 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
std::thread::sleep(std::time::Duration::from_millis(1));
}
assert_eq!(
old_usage.last().unwrap().payload["runDelta"]["inputTokens"],
120
);
assert_eq!(old_usage.last().unwrap().payload["runDeltaAvailable"], true);
let readiness = executor
.execute(&command(
"readiness",
@@ -4034,6 +4047,23 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
.iter()
.any(|(event_type, _, _)| event_type == "run.attached"));
executor
.execute(&command(
"next-turn",
6,
"turn.start",
json!({"text":"Complete the next paid turn."}),
))
.unwrap();
let next_events = wait_for_executor_event_batch(&mut executor, "turn.completed");
let next_usage = next_events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.unwrap();
assert_eq!(next_usage.payload["runDelta"]["inputTokens"], 30);
assert_eq!(next_usage.payload["runDelta"]["outputTokens"], 3);
assert_eq!(next_usage.payload["runDeltaAvailable"], true);
executor.shutdown().expect("stop the warm provider");
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
@@ -6256,6 +6286,9 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() {
let mut config = provider_config(&directory, &["--complete-after-tool-call"]);
if scenario == "controller-goal" {
config.args.push("--agent-created-goal-on-open".to_owned());
config
.args
.push("--accounting-tail-after-terminal".to_owned());
}
config.args.extend([
"--terminal-result-file".to_owned(),
@@ -6327,6 +6360,13 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() {
.unwrap();
assert!(failure < turn && turn < terminal);
if scenario == "controller-goal" {
let final_usage = emitted
.iter()
.rposition(|event| event.event_type == "usage.reported")
.unwrap();
assert!(final_usage < failure && final_usage < turn && final_usage < terminal);
assert_eq!(emitted[final_usage].payload["runDelta"]["inputTokens"], 120);
assert_eq!(emitted[final_usage].payload["runDeltaAvailable"], true);
let settled_goal = emitted
.iter()
.rposition(|event| {
@@ -6687,7 +6727,11 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() {
let directory = temporary_directory("resume-usage-snapshot");
let config = provider_config(
&directory,
&["--durable-turn-ids", "--resume-usage-snapshot"],
&[
"--durable-turn-ids",
"--resume-usage-snapshot",
"--cumulative-accounting-usage",
],
);
let runner_config = durable_config(&directory);
let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
@@ -6756,7 +6800,7 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() {
.collect();
assert_eq!(snapshots.len(), 2);
assert_eq!(snapshots[0].payload["receivedTurnId"], "provider-turn-1");
assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 100);
assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 200);
assert!(!events.iter().any(|event| matches!(
event.event_type.as_str(),
"provider.notice.recorded" | "usage.reported"
@@ -6988,3 +7032,361 @@ fn skill_flag_can_be_added_to_an_old_checkpoint_at_settled_run_attach() {
);
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn durable_codex_usage_keeps_run_baselines_through_requests_recovery_and_attachment() {
let directory = temporary_directory("accounting-run-baseline");
fs::write(
directory.join("fake-state.json"),
serde_json::to_vec(&json!({"threadId":"thread-1","activeTurnId":null,"nextTurn":1}))
.unwrap(),
)
.unwrap();
let mut config = provider_config(
&directory,
&[
"--durable-turn-ids",
"--resume-usage-snapshot",
"--cumulative-accounting-usage",
],
);
config.provider_session_id = Some("thread-1".to_owned());
let runner_config = durable_config(&directory);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({"provider":config,"authorizedTools":task_context_tool_set()}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
for _ in 0..20 {
if poll_and_ack(&mut executor)
.unwrap()
.iter()
.any(|event| event.payload["code"] == "codex_resume_usage_snapshot")
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(5));
}
fn run(
executor: &mut CodexCommandExecutor,
sequence: u64,
expected_input: u64,
expected_output: u64,
) {
executor
.execute(&command(
&format!("turn-{sequence}"),
sequence,
"turn.start",
json!({"text":"Complete the accounting turn."}),
))
.unwrap();
let events = wait_for_executor_event_batch(executor, "turn.completed");
let usage = events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.expect("provider emitted usage");
assert_eq!(usage.payload["runDeltaBasis"], "run");
assert_eq!(usage.payload["runDeltaAvailable"], true);
assert_eq!(usage.payload["runDelta"]["inputTokens"], expected_input);
assert_eq!(usage.payload["runDelta"]["outputTokens"], expected_output);
for _ in 0..20 {
if poll_and_ack(executor).unwrap().is_empty() {
break;
}
}
}
run(&mut executor, 3, 100, 10);
run(&mut executor, 4, 200, 20);
executor.shutdown().unwrap();
drop(executor);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
run(&mut executor, 5, 300, 30);
executor
.execute(&command(
"attach",
6,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap();
run(&mut executor, 7, 100, 10);
executor.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn late_codex_usage_requires_durable_reconciliation_before_or_after_handoff() {
for mode in ["before", "after", "after-paid", "unsafe"] {
let handoff_first = mode.starts_with("after");
let directory = temporary_directory("late-accounting-reconciliation");
let gate = directory.join("release-late-usage");
let gate_arg = gate.to_string_lossy().into_owned();
let config = provider_config(
&directory,
&[
"--durable-turn-ids",
"--warm-tail-accounting",
"--late-accounting-tail-gate",
&gate_arg,
if mode == "after-paid" {
"--hold-second-accounting-terminal"
} else {
"--ordinary-terminal"
},
if mode == "after-paid" {
"--late-accounting-duplicate-burst"
} else {
"--ordinary-tail-count"
},
if mode == "unsafe" {
"--unsafe-after-late-usage"
} else {
"--ordinary-late-usage"
},
],
);
let runner_config = durable_config(&directory);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({"provider":config,"authorizedTools":task_context_tool_set()}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
executor
.execute(&command(
"turn",
3,
"turn.start",
json!({"text":"Finish before the delayed accounting report."}),
))
.unwrap();
let events = wait_for_executor_event_batch(&mut executor, "turn.completed");
assert_eq!(
events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.unwrap()
.payload["runDelta"]["inputTokens"],
100
);
if handoff_first {
executor
.execute(&command(
"attach",
4,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap();
let mut next = runner_config.clone();
next.run_id = "run-2".to_owned();
executor.rotate_authority(&next);
}
if mode == "after-paid" {
executor
.execute(&command(
"successor-turn",
5,
"turn.start",
json!({"text":"Start successor paid work."}),
))
.unwrap();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
assert!(std::time::Instant::now() < deadline);
let events = poll_and_ack(&mut executor).unwrap();
assert!(!events
.iter()
.any(|event| event.event_type == "turn.completed"));
if let Some(usage) = events
.iter()
.find(|event| event.event_type == "usage.reported")
{
assert_eq!(usage.payload["runDelta"]["inputTokens"], 50);
break;
}
}
}
fs::write(&gate, "release").unwrap();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
let retained = loop {
assert!(std::time::Instant::now() < deadline);
let events = if mode == "unsafe" {
match executor.execute(&command(
"quiesce",
4,
"session.snapshot",
json!({"quiesceForWarmAttach":true}),
)) {
Ok(_) => {
std::thread::sleep(std::time::Duration::from_millis(1));
continue;
}
Err(error) => {
assert!(error
.to_string()
.contains("failed to retain completed Codex accounting"));
executor.retained_events().unwrap()
}
}
} else {
executor.poll_events().unwrap()
};
if events.iter().any(|event| {
event.event_type == "usage.reported"
|| event.payload["code"] == "codex_usage_accounting_integrity"
}) {
break events;
}
executor.acknowledge_events(events.len()).unwrap();
};
if handoff_first {
assert!(retained
.iter()
.filter(|event| event.event_type == "usage.reported")
.all(|event| event.payload["runDeltaAvailable"] == false));
assert!(retained
.iter()
.any(|event| event.event_type == "usage.reported"
&& event.payload["runDeltaAvailable"] == false));
assert!(retained
.iter()
.any(|event| event.event_type == "run.terminal"
&& event.payload["runTerminalState"] == "failed"));
let required = retained
.iter()
.find(|event| event.event_type == "usage.reported")
.unwrap();
assert_eq!(
required.payload["reconciliation"]["cumulative"]["inputTokens"],
120
);
assert_eq!(
required.payload["reconciliation"]["priorOrigin"]["runId"],
"run-1"
);
let evidence = retained
.iter()
.find(|event| event.payload["code"] == "codex_usage_accounting_integrity")
.unwrap();
assert_eq!(evidence.payload["priorOrigin"]["runId"], "run-1");
assert_eq!(evidence.payload["receivedTurnId"], "provider-turn-1");
assert_eq!(evidence.payload["cumulative"]["inputTokens"], 120);
} else {
assert_eq!(
retained
.iter()
.find(|event| event.event_type == "usage.reported")
.unwrap()
.payload["runDelta"]["inputTokens"],
120
);
}
let saved: Value =
serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
.unwrap();
assert_eq!(saved["lifecycle"], "reconciliation_required");
assert!(!saved["pendingEvents"].as_array().unwrap().is_empty());
let error = executor
.execute(&command(
"blocked-attach",
5,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap_err();
assert!(error
.to_string()
.contains("requires explicit reconciliation"));
executor.shutdown().unwrap();
drop(executor);
let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
let error = restored
.execute(&command(
"blocked-after-restart",
6,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap_err();
assert!(error
.to_string()
.contains("requires explicit reconciliation"));
restored.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
}
#[test]
fn durable_codex_usage_keeps_completed_turn_through_reconciled_provider_exit() {
for exit_flag in [
"--exit-after-turn-completion",
"--fail-after-turn-completion",
] {
let directory = temporary_directory("accounting-reconciled-exit");
let config = provider_config(
&directory,
&[
"--durable-turn-ids",
"--warm-tail-accounting",
"--emit-post-completion-passive-statuses",
exit_flag,
],
);
let mut executor =
CodexCommandExecutor::with_runner_config(&directory, &durable_config(&directory));
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({"provider":config,"authorizedTools":task_context_tool_set()}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
executor
.execute(&command(
"turn",
3,
"turn.start",
json!({"text":"Complete accounting before exiting."}),
))
.unwrap();
let events = wait_for_executor_event_batch(&mut executor, "turn.completed");
assert_eq!(
events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.unwrap()
.payload["runDelta"]["inputTokens"],
120
);
assert!(!events.iter().any(|event| event.event_type == "turn.failed"));
let saved: Value =
serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
.unwrap();
assert_ne!(saved["lifecycle"], "reconciliation_required");
assert_eq!(saved["completedTurnAuthoritative"], true);
executor.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
}
@@ -3,13 +3,34 @@ import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
// Copilot, Grok and Pi profile identities attest upstream releases, like Claude,
// Codex, Cursor and Antigravity. They bind upstream versions and the bytes a
// repackager copies verbatim: complete platform closures and Grok's launcher.
// Never bind Paperclip source, the shared ACPX layer, absolute paths, the
// provider-pack manifest digest or install layout. The profile's acpxVersion and
// the pack contract digest bind ACPX and Runner source for every harness alike.
export const ATTESTED_ACPX_AGENTS = Object.freeze(["copilot", "grok", "pi"]);
/**
* How each ACPX provider's `commandDigest` attests its upstream release. Every
* profile in acpx-profiles.json declares one kind here:
*
* - `installed-command`: installation hashes the upstream package's launch
* command and compares it with the digest (Claude, Codex).
* - `launcher`: the SHA-256 of a launcher shipped verbatim (Grok).
* - `closure`: upstream versions and the complete platform closure pins.
* - `declaration`: a checked-in release declaration, hashed as canonical JSON.
*
* A digest changes only when the upstream release changes. It never binds
* Paperclip source, the shared ACPX layer, absolute paths, the provider-pack
* manifest digest or install layout: the profile's acpxVersion and the pack
* contract digest bind ACPX and Runner source for every provider alike.
* See rules/acpx-profile-identity.mdc.
*/
export const ACPX_RELEASE_ATTESTATIONS = Object.freeze({
claude: { kind: "installed-command" },
codex: { kind: "installed-command" },
grok: { kind: "launcher", path: "src/providers/grok/launcher.cjs" },
copilot: { kind: "closure", pins: { closure: ["copilot-closure-pins.ts", "COPILOT_CLOSURE_SHA256"] } },
pi: { kind: "closure", pins: {
nodeVersion: ["pi-node-pins.ts", "PI_NODE_VERSION"],
closure: ["pi-closure-pins.ts", "PI_DISTRIBUTION_CLOSURE_SHA256"],
} },
cursor: { kind: "declaration", path: "cursor-contract.json", digestField: "commandDigest" },
antigravity: { kind: "declaration", path: "antigravity-distributions.json" },
});
const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
const canonicalJson = value => value && typeof value === "object"
@@ -17,28 +38,42 @@ const canonicalJson = value => value && typeof value === "object"
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
: JSON.stringify(value);
/** The release attestation for one provider, computed from a runner package root. */
/**
* The release attestation for one provider, computed from a runner package
* root, or null when installation verifies the digest instead.
*/
export async function acpxReleaseAttestation(agent, packageRoot) {
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "acpx-profiles.json"), "utf8"));
const profile = manifest.profiles[agent];
if (!ATTESTED_ACPX_AGENTS.includes(agent) || !profile) throw new Error(`No upstream release attestation for ${agent}`);
// Grok keeps its physical byte-digest contract: admission hashes the launcher.
if (agent === "grok") {
return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, "src/providers/grok/launcher.cjs")))}` };
const attestation = Object.hasOwn(ACPX_RELEASE_ATTESTATIONS, agent) ? ACPX_RELEASE_ATTESTATIONS[agent] : undefined;
if (!profile || !attestation) {
throw new Error(`${agent} needs a release attestation: declare how its commandDigest attests the upstream release `
+ "in ACPX_RELEASE_ATTESTATIONS (scripts/acpx-release-attestations.mjs). See rules/acpx-profile-identity.mdc.");
}
switch (attestation.kind) {
case "installed-command":
return null;
case "launcher":
return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, attestation.path)))}` };
case "declaration": {
const { [attestation.digestField]: _digest, ...declaration } = JSON.parse(readFileSync(resolve(packageRoot, attestation.path), "utf8"));
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
case "closure": {
const declaration = {
schema: "paperclip.acpx-release-attestation.v1", agent,
agentProfileVersion: profile.agentProfileVersion,
agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
};
for (const [field, [file, name]] of Object.entries(attestation.pins)) {
// Key each import on content so a process never reuses pins from edited bytes.
const path = resolve(packageRoot, "src/drivers/acpx", file);
declaration[field] = (await import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`))[name];
}
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
default:
throw new Error(`${agent} declares an unknown release attestation kind: ${attestation.kind}`);
}
// Key each import on content so a process never reuses pins from edited bytes.
const pins = async file => {
const path = resolve(packageRoot, "src/drivers/acpx", file);
return import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`);
};
const declaration = {
schema: "paperclip.acpx-release-attestation.v1", agent,
agentProfileVersion: profile.agentProfileVersion,
agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
...agent === "copilot"
? { closure: (await pins("copilot-closure-pins.ts")).COPILOT_CLOSURE_SHA256 }
: { nodeVersion: (await pins("pi-node-pins.ts")).PI_NODE_VERSION, closure: (await pins("pi-closure-pins.ts")).PI_DISTRIBUTION_CLOSURE_SHA256 },
};
return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
@@ -5,10 +5,12 @@ import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const manifest = JSON.parse(readFileSync(join(packageRoot, "acpx-profiles.json"), "utf8"));
// Providers whose digest the generator computes; installation verifies the rest.
const COMPUTED = Object.keys(manifest.profiles).filter(agent => ACPX_RELEASE_ATTESTATIONS[agent]?.kind !== "installed-command");
const packageFiles = [
"acpx-profiles.json", "package.json", "cursor-distributions.json", "cursor-contract.json", "antigravity-distributions.json",
"scripts/acpx-release-attestations.mjs", "scripts/generate-acpx-profiles.mjs",
@@ -33,15 +35,27 @@ function relocatedPackage(t) {
const generate = (root, ...args) => spawnSync(process.execPath, [join(root, "scripts/generate-acpx-profiles.mjs"), ...args],
{ cwd: root, env: { PATH: process.env.PATH }, encoding: "utf8", timeout: 20_000 });
test("the manifest carries each provider's current release attestation", async () => {
for (const agent of ATTESTED_ACPX_AGENTS) {
test("every profile declares how its digest attests the upstream release", async () => {
assert.deepEqual(Object.keys(ACPX_RELEASE_ATTESTATIONS).sort(), Object.keys(manifest.profiles).sort());
assert.deepEqual(COMPUTED.sort(), ["antigravity", "copilot", "cursor", "grok", "pi"]);
for (const agent of COMPUTED) {
assert.equal(manifest.profiles[agent].commandDigest, (await acpxReleaseAttestation(agent, packageRoot)).commandDigest, agent);
}
});
test("a new provider is told to declare its release attestation", async t => {
const relocated = relocatedPackage(t);
const path = join(relocated, "acpx-profiles.json"); const value = JSON.parse(readFileSync(path, "utf8"));
value.profiles.newagent = { ...value.profiles.codex };
writeFileSync(path, JSON.stringify(value));
const result = generate(relocated, "--check");
assert.notEqual(result.status, 0);
assert.match(result.stderr, /newagent needs a release attestation: declare how its commandDigest attests the upstream release/u);
});
test("a relocated copy of the provider inputs computes the same attestations", async t => {
const relocated = relocatedPackage(t);
for (const agent of ATTESTED_ACPX_AGENTS) {
for (const agent of COMPUTED) {
const here = await acpxReleaseAttestation(agent, packageRoot);
const there = await acpxReleaseAttestation(agent, relocated);
assert.equal(there.commandDigest, here.commandDigest, agent);
@@ -76,24 +90,52 @@ for (const [name, agents, edit] of [
const path = join(root, "src/providers/grok/launcher.cjs");
writeFileSync(path, `${readFileSync(path, "utf8")}\n// changed\n`);
}],
["the Cursor release declaration", ["cursor"], root => {
const path = join(root, "cursor-contract.json"); const value = JSON.parse(readFileSync(path, "utf8"));
value.nativePatch = "changed"; writeFileSync(path, JSON.stringify(value));
}],
["the Antigravity distribution declaration", ["antigravity"], root => {
const path = join(root, "antigravity-distributions.json"); const value = JSON.parse(readFileSync(path, "utf8"));
value.version = "9.9.9"; writeFileSync(path, JSON.stringify(value));
}],
]) test(`changing ${name} changes only the affected attestations`, async t => {
const relocated = relocatedPackage(t);
const before = Object.fromEntries(await Promise.all(ATTESTED_ACPX_AGENTS.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
const before = Object.fromEntries(await Promise.all(COMPUTED.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
edit(relocated);
for (const agent of ATTESTED_ACPX_AGENTS) {
for (const agent of COMPUTED) {
const after = (await acpxReleaseAttestation(agent, relocated)).commandDigest;
if (agents.includes(agent)) assert.notEqual(after, before[agent], agent);
else assert.equal(after, before[agent], agent);
}
});
test("the generator refreshes a release declaration's own digest with the profile", async t => {
const relocated = relocatedPackage(t);
const path = join(relocated, "cursor-contract.json"); const contract = JSON.parse(readFileSync(path, "utf8"));
contract.modes = [...contract.modes, "review"];
writeFileSync(path, `${JSON.stringify(contract, null, 2)}\n`);
const stale = generate(relocated, "--check");
assert.notEqual(stale.status, 0);
assert.match(stale.stderr, /cursor commandDigest no longer matches its upstream release attestation/u);
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), contract, "check mode writes nothing");
const refreshed = generate(relocated);
assert.equal(refreshed.status, 0, refreshed.stderr);
const { commandDigest } = await acpxReleaseAttestation("cursor", relocated);
assert.notEqual(commandDigest, manifest.profiles.cursor.commandDigest);
assert.equal(JSON.parse(readFileSync(path, "utf8")).commandDigest, commandDigest);
assert.equal(JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8")).profiles.cursor.commandDigest, commandDigest);
assert.match(readFileSync(join(relocated, "src/drivers/acpx/generated-profiles.ts"), "utf8"), new RegExp(commandDigest));
assert.match(readFileSync(join(relocated, "runner/crates/runner-core/src/generated_acpx_profiles.rs"), "utf8"), new RegExp(commandDigest));
assert.equal(generate(relocated, "--check").status, 0);
});
test("the generator refuses a stale attestation in check mode and refreshes it otherwise", async t => {
const relocated = relocatedPackage(t);
const launcher = join(relocated, "src/providers/grok/launcher.cjs");
writeFileSync(launcher, `${readFileSync(launcher, "utf8")}\n// changed\n`);
const stale = generate(relocated, "--check");
assert.notEqual(stale.status, 0);
assert.match(stale.stderr, /grok release attestation is stale/u);
assert.match(stale.stderr, /grok commandDigest no longer matches its upstream release attestation/u);
const refreshed = generate(relocated);
assert.equal(refreshed.status, 0, refreshed.stderr);
const written = JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8"));
@@ -1,30 +1,36 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
const root = new URL("../", import.meta.url);
const readJson = async path => JSON.parse(await readFile(new URL(path, root), "utf8"));
const manifest = await readJson("acpx-profiles.json");
const distributions = await readJson("cursor-distributions.json");
const antigravityDistributions = await readJson("antigravity-distributions.json");
const contract = await readJson("cursor-contract.json");
const pkg = await readJson("package.json");
assert.equal(manifest.schema, "paperclip.acpx-profiles.v1");
assert.equal(manifest.acpxVersion, pkg.dependencies.acpx);
// Copilot, Grok and Pi digests attest upstream releases; they change only when
// an upstream version, a platform closure or the Grok launcher changes.
// Every profile's commandDigest attests its upstream release, so it changes only
// when that release changes. Installation verifies installed-command digests.
const check = process.argv.includes("--check");
let attestationsChanged = false;
for (const agent of ATTESTED_ACPX_AGENTS) {
const { commandDigest } = await acpxReleaseAttestation(agent, fileURLToPath(root));
if (manifest.profiles[agent].commandDigest === commandDigest) continue;
assert.ok(!check, `${agent} release attestation is stale in acpx-profiles.json; run generate:acpx-profiles`);
manifest.profiles[agent].commandDigest = commandDigest;
attestationsChanged = true;
const writes = new Map();
for (const agent of Object.keys(manifest.profiles)) {
const attestation = await acpxReleaseAttestation(agent, fileURLToPath(root));
if (!attestation) continue;
// A release declaration may carry its own copy of the digest; keep it in step.
const { path, digestField } = ACPX_RELEASE_ATTESTATIONS[agent];
const declaration = digestField ? await readJson(path) : null;
if (manifest.profiles[agent].commandDigest === attestation.commandDigest
&& (!declaration || declaration[digestField] === attestation.commandDigest)) continue;
assert.ok(!check, `${agent} commandDigest no longer matches its upstream release attestation. `
+ "Run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`; an upstream change keeps the profile version.");
manifest.profiles[agent].commandDigest = attestation.commandDigest;
writes.set("acpx-profiles.json", manifest);
if (declaration) writes.set(path, { ...declaration, [digestField]: attestation.commandDigest });
}
if (attestationsChanged) await writeFile(new URL("acpx-profiles.json", root), `${JSON.stringify(manifest, null, 2)}\n`);
for (const [path, value] of writes) await writeFile(new URL(path, root), `${JSON.stringify(value, null, 2)}\n`);
const contract = await readJson("cursor-contract.json");
const entries = Object.entries(manifest.profiles);
for (const [agent, profile] of entries) {
assert.match(agent, /^[a-z]+$/);
@@ -38,21 +44,13 @@ for (const [agent, profile] of entries) {
if (name && Object.hasOwn(pkg.dependencies, name)) assert.equal(version, pkg.dependencies[name], `${agent}: installed dependency drift`);
}
}
// The Cursor contract is an immutable release attestation, not another runtime
// lookup table. Refuse generation if a release declaration disagrees with it.
// Release declarations must agree with the profiles they attest.
assert.equal(manifest.profiles.cursor.agentServerVersion, distributions.version);
assert.equal(manifest.profiles.cursor.agentServerVersion, contract.vendor);
assert.equal(manifest.profiles.cursor.agentProfileVersion, contract.profileVersion);
assert.equal(manifest.profiles.cursor.commandDigest, contract.commandDigest);
assert.equal(contract.commandDigest, manifest.profiles.cursor.commandDigest, "cursor-contract.json carries a stale commandDigest");
assert.equal(distributions.patchVersion, contract.nativePatch);
const canonicalJson = value => value && typeof value === "object"
? Array.isArray(value) ? `[${value.map(canonicalJson).join(",")}]`
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
: JSON.stringify(value);
assert.equal(manifest.profiles.antigravity.agentServerVersion, antigravityDistributions.version);
assert.equal(manifest.profiles.antigravity.commandDigest, `sha256:${createHash("sha256").update(canonicalJson(antigravityDistributions)).digest("hex")}`, "Antigravity distribution identity is stale");
const { commandDigest, ...attestation } = contract;
assert.equal(commandDigest, `sha256:${createHash("sha256").update(canonicalJson(attestation)).digest("hex")}`);
for (const distribution of Object.values(distributions.platforms)) assert.match(distribution.closureSha256, /^[a-f0-9]{64}$/);
const quote = JSON.stringify;
@@ -2,6 +2,23 @@ import { describe, expect, it } from "vitest";
import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js";
import { isSupportedAcpxProfileVersion } from "./profile-compatibility.js";
// A profile version changes only to reset saved sessions on purpose. An upstream
// release changes the commandDigest and keeps the version; a new provider starts
// at 1. Update this table, with the reason, only for a deliberate reset.
const PROFILE_VERSIONS: Readonly<Record<string, number>> = {
antigravity: 1, claude: 1, codex: 1, copilot: 92, cursor: 17, grok: 20, pi: 37,
};
describe("ACPX profile versions", () => {
it("change only for a deliberate session reset", () => {
for (const [agent, profile] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
expect(profile.agentProfileVersion, `${agent}: a profile version bump resets every saved ${agent} session. `
+ "An upstream release changes only the commandDigest. See rules/acpx-profile-identity.mdc.")
.toBe(PROFILE_VERSIONS[agent] ?? 1);
}
});
});
describe("historical ACPX profile decoding", () => {
it("decodes every revision from 1 through the current one so saved runs reach recovery", () => {
for (const [agent, current] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
@@ -1123,6 +1123,43 @@ it("refuses a reusable close checkpoint when the local provider snapshot is unre
}
}, 15_000);
it("installs required accounting acceptance before the Runner acknowledgement callback returns", async () => {
const directory = await mkdtemp(join(tmpdir(), "runnerd-required-accounting-"));
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
let callbacks!: ConstructorParameters<typeof OriginalCore>[0];
const spy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) {
callbacks = options;
return new OriginalCore(options);
});
let release!: () => void;
const accepted = new Promise<void>(resolve => { release = resolve; });
const onAccountingEvent = vi.fn(async () => accepted);
const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: directory, sourceCodexHome: "", environment: {},
runnerBinary: process.execPath, runnerArtifact: { version: "test", digest: `sha256:${"a".repeat(64)}` },
codexCommand: process.execPath, codexArgs: [], onAccountingEvent,
controlPlaneRegistration: async () => { throw new Error("stop before Runner dispatch"); },
});
try {
await expect(bundle.transport.request("thread/start", { cwd: directory })).rejects.toThrow();
const report = { eventType: "usage.reported", sourceEventId: "required-usage" } as PrpEvent;
let returned = false;
const result = callbacks.onCommittedEvent!(report).then(() => { returned = true; });
await vi.waitFor(() => expect(onAccountingEvent).toHaveBeenCalledExactlyOnceWith(report));
expect(returned).toBe(false);
release();
await result;
expect(returned).toBe(true);
const failure = new Error("receipt unavailable");
onAccountingEvent.mockRejectedValueOnce(failure);
await expect(callbacks.onCommittedEvent!(report)).rejects.toBe(failure);
} finally {
release();
await bundle.transport.close().catch(() => undefined);
spy.mockRestore();
await rm(directory, { recursive: true, force: true });
}
});
it("uses the same API result envelope for live dispatch and recovered receipts", async () => {
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-recovered-api-"));
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
@@ -1240,6 +1240,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD
provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx";
observeCredentials?: boolean;
onCredentialsUpdated?: (event: PrpEvent) => Promise<void>;
onAccountingEvent?: (event: PrpEvent) => Promise<void>;
recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"];
opencodePermissionMode?: NativeOpenCodePermissionMode;
acpxAgent?: QualifiedAcpxAgent;
@@ -5239,6 +5240,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
onSemanticToolInput: (call) => this.#handleSemanticToolInput(call),
recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call),
onCommittedEvent: async (event) => {
await this.options.onAccountingEvent?.(event);
if (event.eventType === "auth.credentials.updated") {
await this.options.onCredentialsUpdated?.(event);
}
@@ -5691,6 +5693,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
onSemanticToolInput: (call) => this.#handleSemanticToolInput(call),
recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call),
onCommittedEvent: async (event) => {
await this.options.onAccountingEvent?.(event);
if (event.eventType === "auth.credentials.updated") {
await this.options.onCredentialsUpdated?.(event);
}
@@ -3385,6 +3385,75 @@ export const usageSchema = {
"runDeltaAvailable": {
"type": "boolean"
},
"runDeltaBasis": {
"const": "run"
},
"reconciliation": {
"type": "object",
"required": [
"reason",
"receivedTurnId",
"cumulative",
"priorOrigin"
],
"properties": {
"reason": {
"const": "accounting_reconciliation_required"
},
"receivedTurnId": {
"type": [
"string",
"null"
],
"maxLength": 240
},
"cumulative": {
"$ref": "#/$defs/measurement"
},
"priorOrigin": {
"type": [
"object",
"null"
],
"required": [
"runnerInstanceId",
"runId",
"normalizedSessionId",
"turnId",
"itemId"
],
"properties": {
"runnerInstanceId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"normalizedSessionId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"turnId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"itemId": {
"type": "string",
"minLength": 1,
"maxLength": 240
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"runDelta": {
"$ref": "#/$defs/measurement"
}
@@ -1,3 +1,5 @@
import Ajv2020 from "ajv/dist/2020.js";
import { usageSchema } from "./generated/schema-bundle.js";
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
@@ -6,6 +8,7 @@ import {
negotiateProtocolVersion,
parsePrpFixtureText,
PRP_PROTOCOL_VERSION,
validatePrpEvent,
} from "./replay-contract.js";
import { reducePrpFixture } from "../reducer/session-reducer.js";
@@ -371,3 +374,24 @@ describe("PRP v1 JSON Schema contract", () => {
).toBeNull();
});
});
it("accepts marked whole-run usage through the runtime standalone event validator", async () => {
const fixture = await readFixture();
const event = (fixture.events as Array<Record<string, unknown>>)[0];
const measurement = { inputTokens: 120, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 0,
activeSeconds: 0, requests: 1, providerCostUsd: 0 };
const result = validatePrpEvent({ ...event, eventType: "usage.reported", priority: 0,
payload: { provider: "codex", model: "gpt-6.1-sol", cumulative: measurement, runDelta: measurement,
runDeltaAvailable: true, runDeltaBasis: "run" } });
expect(result).toMatchObject({ ok: true });
});
it("declares required incomplete reconciliation evidence in the strict usage contract", () => {
const validate = new Ajv2020({ strict: true, allErrors: true }).compile(usageSchema);
const measurement = { inputTokens: 120, outputTokens: 12, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 0, providerCostUsd: 0 };
const payload = { provider: "codex", model: null, cumulative: measurement, runDelta: { ...measurement, inputTokens: 0, outputTokens: 0 }, runDeltaAvailable: false, runDeltaBasis: "run",
reconciliation: { reason: "accounting_reconciliation_required", receivedTurnId: "provider-turn-1", cumulative: measurement,
priorOrigin: { runnerInstanceId: "runner-1", runId: "old-run", normalizedSessionId: "session", turnId: "turn", itemId: "item" } } };
expect(validate(payload), JSON.stringify(validate.errors)).toBe(true);
expect(validate({ ...payload, reconciliation: { ...payload.reconciliation, arbitraryProviderData: "excluded" } })).toBe(false);
});
@@ -1,6 +1,6 @@
{
"name": "@paperclipai/plugin-daytona",
"version": "0.1.1",
"version": "0.1.9",
"description": "Daytona sandbox provider plugin for Paperclip environments",
"license": "MIT",
"homepage": "https://github.com/paperclipai/paperclip",
@@ -1,6 +1,6 @@
{
"name": "@paperclipai/plugin-kubernetes",
"version": "0.1.0",
"version": "0.1.0-alpha.1",
"description": "Kubernetes sandbox provider plugin for Paperclip environments",
"license": "MIT",
"homepage": "https://github.com/paperclipai/paperclip",
+2
View File
@@ -171,6 +171,8 @@ export type {
DecisionHistoryPage,
AttentionDetailImage,
AttentionFeed,
AttentionSummary,
AttentionTaskIndex,
AttentionItem,
AttentionItemDetail,
AttentionItemDismissal,
+7
View File
@@ -275,6 +275,13 @@ export interface AttentionFeed {
countsBySourceKind: Record<AttentionSourceKind, number>;
items: AttentionItem[];
}
/** Canonical attention counts without downloading the decision cards. */
export type AttentionSummary = Omit<AttentionFeed, "items" | "nextCursor">;
/** Complete accessible task membership for Tasks → Needs attention. */
export interface AttentionTaskIndex extends AttentionSummary {
taskIds: string[];
}
/** Read-only projection of canonical task decisions; never a second write store. */
export interface DecisionHistoryItem {
id: string;
+2
View File
@@ -45,6 +45,8 @@ export type {
DecisionHistoryPage,
AttentionDetailImage,
AttentionFeed,
AttentionSummary,
AttentionTaskIndex,
AttentionItem,
AttentionItemDetail,
AttentionItemDismissal,
+1
View File
@@ -1779,6 +1779,7 @@ export interface IssueAttachment {
/** Bounded cold-open window; the complete chronological feed stays paginated. */
export interface IssueRecentComments {
latestCommentId: string | null;
oldestCommentId?: string | null;
comments: IssueComment[];
hasMore: boolean;
}
+3
View File
@@ -68,6 +68,9 @@ importers:
esbuild:
specifier: ^0.28.2
version: 0.28.2
oxc-parser:
specifier: 0.127.0
version: 0.127.0
turbo:
specifier: 2.11.10
version: 2.11.10
+72
View File
@@ -0,0 +1,72 @@
---
description: Give every ACPX provider the same identity model - its commandDigest attests the upstream release, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.
globs: "packages/paperclip-runner/acpx-profiles.json,packages/paperclip-runner/cursor-contract.json,packages/paperclip-runner/cursor-distributions.json,packages/paperclip-runner/antigravity-distributions.json,packages/paperclip-runner/scripts/acpx-release-attestations.mjs,packages/paperclip-runner/scripts/generate-acpx-profiles.mjs,packages/paperclip-runner/scripts/contracts/**,packages/paperclip-runner/src/drivers/acpx/**,packages/paperclip-runner/src/providers/**,packages/paperclip-runner/protocol/**"
alwaysApply: false
---
# ACPX profile identity
## Scope and purpose
This rule covers ACPX provider profiles, their release declarations and pins,
the profile generator, the provider-pack contract registry, and the ACPX
driver and provider code. Every ACPX harness (Claude, Codex, Copilot, Cursor,
Grok, Pi, Antigravity and any new one) uses the same identity model, so a
change behaves the same way for all of them.
## The identity model
A provider profile in `acpx-profiles.json` has a version and a `commandDigest`.
Three things decide whether saved work and installed packs stay compatible:
| What changed | What carries it | Effect |
| --- | --- | --- |
| The provider's upstream release: package versions, platform closures, a launcher shipped verbatim | The profile's `commandDigest` | Saved sessions for that provider start fresh |
| The shared ACPX layer | The profile's `acpxVersion` and the Runner's ACPX patch in the pack `contractDigest` | The same for every provider |
| A runner/pack contract: schemas, generated catalogs, Rust shapes and env keys | The provider pack's `contractDigest` | Older image packs are not reused |
Ordinary Runner source changes, refactors and comments change none of these.
## Required behavior
- **Attest the upstream release.** Declare each provider's attestation in
`ACPX_RELEASE_ATTESTATIONS` (`scripts/acpx-release-attestations.mjs`), using
the kind that fits: `installed-command`, `launcher`, `closure` or
`declaration`. The digest binds only what a repackager copies verbatim from
the upstream release, so the same release gives the same digest wherever the
pack is built or installed.
- **Keep Paperclip source and the ACPX layer out of provider digests.**
Runner/pack compatibility belongs to the `contractDigest`. When Runner code
and a pack must agree on a new field or behavior, declare it in a schema
under `protocol/` and generate the TypeScript, so the contract registry
covers it.
- **Regenerate instead of hand-editing.** After an upstream bump or a pin
change, run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`.
`check:acpx-profiles` runs in the build and refuses a stale digest.
- **Keep the profile version.** An upstream release changes the digest, not the
version. A new provider starts at version 1. Bump a version only to reset
every saved session for that provider on purpose, such as a change in saved
session semantics that exact admission cannot detect. Record the reason in
the PR and in the version table in `profile-compatibility.test.ts`.
- **Keep earlier versions decodable.** Saved inputs with any version from 1
through the current one decode, so recovery can refuse them. Launch and
recovery admission require the exact current profile and digest.
## Adding a provider
1. Add its profile at version 1 to `acpx-profiles.json`.
2. Add its attestation to `ACPX_RELEASE_ATTESTATIONS`, with its pins in
erasable TypeScript or JSON that the generator can read.
3. Run `generate:acpx-profiles`, then `node --test scripts/acpx-release-attestations.test.mjs`.
## Review criteria
- Does every provider digest change only with its upstream release?
- Is a new runner/pack contract declared where the `contractDigest` reads it?
- Does a version bump state which saved sessions it resets, and why?
- Does the change treat every ACPX harness the same way?
The canonical reference is
[`packages/paperclip-runner/docs/durable-recovery.md`](../packages/paperclip-runner/docs/durable-recovery.md#profile-identity).
The pack contract digest is described in
[`docker/daytona-runner/README.md`](../docker/daytona-runner/README.md).
+6 -2
View File
@@ -249,7 +249,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
]) {
assert.match(
jobs.get(jobId),
/^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' && /m,
/^ {4}if: \$\{\{ (?:always\(\) && !cancelled\(\) && needs\.gate\.result == 'success' && )?needs\.gate\.outputs\.full_ci == 'true' && /m,
`${jobId} must run only when the gate selects full CI`,
);
}
@@ -263,10 +263,14 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
const verify = jobs.get("verify");
assert.match(
verify,
/^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
/^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
);
assert.equal(runAggregate(verify, true).status, 0);
assert.equal(runAggregate(verify, false).status, 0);
for (const result of ["failure", "cancelled", "skipped"]) {
assert.equal(runAggregate(verify, true, { test_selection: result }).status, 0,
"an optional selector cannot block the successful original matrix");
}
for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner",
"build", "docker_context_integrity", "verify_serialized_server", "canary_dry_run"]) {
for (const result of ["failure", "cancelled", "skipped"]) {
@@ -579,6 +579,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("runner-full-stack-e2e.yml"),
readWorkflow("e2e.yml"),
readWorkflow("runner-protocol-live-evals.yml"),
readWorkflow("chat-feature-testops.yml"),
];
for (const workflow of actionPinWorkflows) {
@@ -618,6 +619,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
"runner-protocol-live-evals.yml",
"chat-feature-testops.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
const providerSecretReference =
@@ -650,6 +652,38 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
(block) => [...block.matchAll(providerSecretReference)].length > 0,
);
assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`);
if (name === "chat-feature-testops.yml") {
assert.equal(providerJobs.length, 1);
const assertBoundary = (block) => {
assert.match(block, /^ chat:\n/);
assert.match(block, /\n needs: \[authorize, select, build\]\n/);
assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
assert.match(block, /\n environment: runner-e2e-private-paid\n/);
assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/);
assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/);
assert.doesNotMatch(block, /ALLURE_TOKEN/);
assert.equal([...block.matchAll(providerSecretReference)].length, 1);
assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference));
};
const block = providerJobs[0];
assertBoundary(block);
assert.equal([...workflow.matchAll(providerSecretReference)].length, 1);
assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/);
assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/);
assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/);
for (const mutant of [
block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
block.replace(" - *guard\n", ""),
block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"),
block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"),
block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"),
]) assert.throws(() => assertBoundary(mutant));
continue;
}
for (const block of providerJobs) {
if (name === "runner-protocol-live-evals.yml") {
assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}"));
@@ -320,6 +320,7 @@ test("the real serialized shard partition is complete, non-overlapping and durat
const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts";
const nativeRunnerSuitePath =
"server/src/services/native-runtime/native-codex-runner.integration.test.ts";
const usageDeliverySuitePath = "server/src/services/native-runtime/native-usage-delivery.integration.test.ts";
const dotRunnerSuitePath = "server/src/__tests__/dot-runner.test.ts";
const restartRecoverySuitePath =
"server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts";
@@ -363,7 +364,8 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes
assert.ok(!files.includes(nativeRunnerSuitePath));
assert.ok(!files.includes(dotRunnerSuitePath));
assert.ok(!files.includes(restartRecoverySuitePath));
assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort());
assert.ok(!files.includes(usageDeliverySuitePath));
assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, usageDeliverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort());
assert.equal(new Set(files).size, files.length);
const defaultRun = dryRunJson([], prEnv);
assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);
@@ -412,6 +414,7 @@ test("the native-runner lane runs exactly the cargo-dependent suites", () => {
"server/src/services/native-runtime/native-codex-runner.integration.test.ts",
dotRunnerSuitePath,
restartRecoverySuitePath,
usageDeliverySuitePath,
]);
});
+62 -1
View File
@@ -10,6 +10,63 @@ import { executionPlan, sourceFiles } from "./execution-plan.mjs";
const repositoryRoot = new URL("../../", import.meta.url);
const turbo = process.env.PAPERCLIP_CI_TEST_TURBO ?? new URL("node_modules/.bin/turbo", repositoryRoot).pathname;
test("real Turbo keeps full runs and different selected test sets in separate cache entries", () => {
const directory = mkdtempSync(path.join(tmpdir(), "ci-selection-cache-"));
const write = (file, value) => writeFileSync(path.join(directory, file), value);
const json = (file, value) => write(file, JSON.stringify(value));
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
try {
git("init", "-q");
for (const folder of ["scripts/ci", "server/src", "packages/fixture", "bin", ".ci-test-selection"]) mkdirSync(path.join(directory, folder), { recursive: true });
for (const file of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
copyFileSync(new URL(`scripts/ci/${file}`, repositoryRoot), path.join(directory, "scripts/ci", file));
}
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\n.ci-test-selection/\nresult.txt\ncalls.txt\n");
json("package.json", { name: "selection-cache-proof", private: true, packageManager: "npm@10.0.0", workspaces: ["packages/*"], scripts: { "test:run:general": "node check.cjs" } });
json("packages/fixture/package.json", { name: "fixture", private: true });
json("package-lock.json", { name: "selection-cache-proof", lockfileVersion: 3, packages: { "": { name: "selection-cache-proof" } } });
json("turbo.json", { globalDependencies: ["doc/**", "docs/**"], globalEnv: ["CI_TASK_CACHE_SCOPE", "PAPERCLIP_*"], tasks: {
"//#ci:server": { inputs: ["check.cjs", "server/**"], outputs: ["result.txt"], cache: true },
} });
mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
mkdirSync(path.join(directory, "docs/api"), { recursive: true });
write("doc/DEVELOPING.md", "before"); write("doc/plans/example.md", "before");
write("docs/api/issues.md", "before");
write("server/src/a.test.ts", "// a"); write("server/src/b.test.ts", "// b");
write("check.cjs", `const fs=require('node:fs');fs.appendFileSync('calls.txt','call\\n');const file=process.env.PAPERCLIP_CI_TEST_SELECTION;fs.writeFileSync('result.txt',file?JSON.parse(fs.readFileSync(file)).lanes['general-server-without-chat'].shards[0]['@paperclipai/server'].join(','):'full');`);
write("bin/pnpm", `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const args=process.argv.slice(2);if(args[0]==='--version')console.log('9.15.4');else{const r=spawnSync(${JSON.stringify(turbo)},args.slice(2),{stdio:'inherit'});process.exit(r.status??1);}`);
chmodSync(path.join(directory, "bin/pnpm"), 0o755);
git("add", ".");
git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qm", "fixture");
const head = git("rev-parse", "HEAD");
const calls = () => readFileSync(path.join(directory, "calls.txt"), "utf8").trim().split("\n").length;
const run = (testFile) => {
const selection = path.join(directory, ".ci-test-selection/manifest.json");
json(".ci-test-selection/manifest.json", { version: 1, head, lanes: {
"general-server-without-chat": { shards: [{ "@paperclipai/server": [testFile ?? "server/src/a.test.ts"] }] },
} });
const result = spawnSync(process.execPath, [path.join(directory, "scripts/ci/run-task.mjs")], {
cwd: directory, encoding: "utf8", env: { ...process.env, PATH: `${directory}/bin:${process.env.PATH}`,
CI_TASK: "ci:server", CI_TASK_ARGS: "[]", CI_FORCE_TASKS: "false", CI_TASK_DRY_RUN: "false",
PAPERCLIP_CI_TEST_SELECTION: testFile ? selection : "", PAPERCLIP_CI_SELECTION_HASH: "",
TURBO_TOKEN: "", TURBO_TEAM: "", TURBO_TELEMETRY_DISABLED: "1" },
});
assert.equal(result.status, 0, result.stdout + result.stderr);
assert.equal(readFileSync(path.join(directory, "result.txt"), "utf8"), testFile ?? "full");
};
run(); run(); assert.equal(calls(), 1);
write("doc/DEVELOPING.md", "after"); write("doc/plans/example.md", "after");
run(); assert.equal(calls(), 1, "audited prose does not invalidate product test results");
run("server/src/a.test.ts"); run("server/src/a.test.ts"); assert.equal(calls(), 2);
run("server/src/b.test.ts"); assert.equal(calls(), 3);
run(); assert.equal(calls(), 3, "the original full result is reusable without accepting a subset result");
write("docs/api/issues.md", "after");
run(); assert.equal(calls(), 4, "runtime documentation must invalidate results");
} finally { rmSync(directory, { recursive: true, force: true }); }
});
test("every cached lane maps to an existing package script with complete inputs", () => {
const config = JSON.parse(readFileSync(new URL("turbo.json", repositoryRoot), "utf8"));
for (const [name, definition] of Object.entries(config.tasks).filter(([, task]) => task.cache)) {
@@ -35,6 +92,8 @@ test("real Turbo executes existing suite scripts and invalidates source, argumen
const json = (name, value) => write(name, JSON.stringify(value));
try {
execFileSync("git", ["init", "-q"], { cwd: directory });
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\ngenerated/\nresult.txt\ncalls.txt\n");
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
json("packages/fixture/package.json", { name: "fixture", private: true });
@@ -83,9 +142,11 @@ test("the task launcher bypasses cache reads and writes when execution is forced
mkdirSync(path.join(directory, "scripts/ci"), { recursive: true });
mkdirSync(path.join(directory, "bin"));
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
for (const name of ["run-task.mjs", "execution-plan.mjs"]) {
for (const name of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
copyFileSync(new URL(`scripts/ci/${name}`, repositoryRoot), path.join(directory, "scripts/ci", name));
}
mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\ncalls.txt\nresult.txt\nbin/\n");
json("packages/fixture/package.json", { name: "fixture", private: true });
json("package.json", { name: "forced-cache-proof", private: true, packageManager: "npm@10.0.0",
+2 -1
View File
@@ -1,5 +1,6 @@
import { execFileSync } from "node:child_process";
import { matchesGlob } from "node:path";
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
export function sourceFiles(cwd = process.cwd()) {
return [...new Set(execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
@@ -30,7 +31,7 @@ export function executionPlan(config, task, args, files = sourceFiles()) {
target = "@paperclipai/paperclip-runner#check:runner";
commandArgs = args;
} else throw new Error(`Missing task command: ${task}`);
const select = (globs) => files.filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
const select = (globs) => productInputs(files).filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
const selectedInputs = select(definition.inputs);
const inputs = target.startsWith("//#") ? selectedInputs
: selectedInputs.map((input) => `$TURBO_ROOT$/${input}`);
+29
View File
@@ -0,0 +1,29 @@
import assert from "node:assert/strict";
import path from "node:path";
import { createVitest, parseCLI } from "vitest/node";
// Positional Vitest CLI filters are substrings: foo.test.ts also selects
// foo.test.tsx. Run the exact discovered specifications assigned to this shard.
const { filter, options } = parseCLI(["vitest", "run", ...process.argv.slice(2)]);
assert.ok(filter.length > 0, "Selected execution requires explicit test files");
const expected = new Set(filter.map((file) => path.resolve(file)));
assert.equal(expected.size, filter.length, "Duplicate selected file");
process.env.TEST = "true";
process.env.VITEST = "true";
process.env.NODE_ENV ??= "test";
// Match the CLI's additive --exclude semantics without replacing project rules.
options.cliExclude = options.exclude;
delete options.exclude;
const ctx = await createVitest({ ...options, watch: false });
try {
await ctx.standalone();
const specifications = (await ctx.globTestSpecifications()).filter((spec) => expected.has(spec.moduleId));
assert.deepEqual(specifications.map((spec) => spec.moduleId).sort(), [...expected].sort(),
"Selected files must match exactly one discovered specification in this project");
await ctx.runTestSpecifications(specifications);
} catch (error) {
console.error(error);
process.exitCode = 1;
} finally {
await ctx.exit();
}
+8 -1
View File
@@ -3,6 +3,8 @@ import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from
import { createHash } from "node:crypto";
import { matchesGlob } from "node:path";
import { executionPlan, sourceFiles } from "./execution-plan.mjs";
import { loadSelection, selectionHash } from "./test-selection-manifest.mjs";
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
const task = process.env.CI_TASK;
if (!/^ci:[a-z-]+$/.test(task ?? "")) throw new Error("Invalid CI task");
@@ -15,13 +17,18 @@ const version = (command, argv) => {
const config = JSON.parse(readFileSync("turbo.json", "utf8"));
const files = sourceFiles();
const execution = executionPlan(config, task, args, files);
// Validate before Turbo can return a cache hit. The hash separates selected
// executions from full executions and from every different selected file set.
if (process.env.PAPERCLIP_CI_TEST_SELECTION) {
process.env.PAPERCLIP_CI_SELECTION_HASH = selectionHash(loadSelection());
}
mkdirSync(".turbo/task-configs", { recursive: true });
const runtimeConfig = `.turbo/task-configs/${task.replace(":", "-")}.json`;
writeFileSync(runtimeConfig, JSON.stringify(execution.config));
const classified = [...config.globalDependencies, ...Object.values(config.tasks).flatMap((entry) => entry.inputs)];
const unknownInputs = createHash("sha256");
let uncacheable = false;
for (const file of files) {
for (const file of productInputs(files)) {
if (classified.some((glob) => matchesGlob(file, glob))) continue;
unknownInputs.update(JSON.stringify(file));
try {
+124
View File
@@ -0,0 +1,124 @@
import { execFileSync } from "node:child_process";
import { appendFileSync, lstatSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { fileURLToPath } from "node:url";
import path from "node:path";
import assert from "node:assert/strict";
import { refineTestPlan, selectionDurations } from "./test-selection-plan.mjs";
import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
export function selectionEligibility(plan) {
if (plan?.version !== 1 || plan.mode !== "enabled" || plan.forceReason) return "Full/shadow/forced task plan";
if (!Array.isArray(plan.changed) || !plan.changed.length) return "Incomplete comparison";
const changed = productInputs(plan.changed);
if (!changed.length) return "Documentation-only diff";
for (const file of changed) {
if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file)
|| /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`;
try {
if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`;
} catch { return `Deleted or missing source: ${file}`; }
}
return null;
}
function runnerInventory(group) {
const args = group === "serialized" ? ["--mode", "serialized", "--shard-index", "0", "--shard-count", "1"]
: ["--mode", "general", "--group", group,
...(group === "general-server-without-chat" ? ["--shard-index", "0", "--shard-count", "1"] : [])];
const env = { ...process.env, GITHUB_WORKFLOW: "PR" };
delete env.PAPERCLIP_CI_TEST_SELECTION;
return JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", ...args, "--dry-run"],
{ env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }));
}
export async function createTestSelection(plan, head) {
const reason = selectionEligibility(plan);
if (reason) throw new Error(reason);
const groups = new Set();
for (const job of ["general_tests", "verify_serialized_server"]) {
if (plan.jobs[job].run) for (const entry of plan.jobs[job].matrix.include) groups.add(entry.group ?? "serialized");
}
groups.delete("general-chat"); // Keep its line sharding and DB ownership intact.
const owners = {}, projects = new Set();
for (const group of groups) {
const inventory = runnerInventory(group);
if (group === "serialized" || group === "general-server-without-chat") {
owners[group] = { "@paperclipai/server": group === "serialized"
? inventory.selectedSerializedSuites : inventory.selectedGeneralServerSuites };
projects.add("@paperclipai/server");
} else {
owners[group] = Object.fromEntries(inventory.workspaceProjects.map((project) => [project, null]));
for (const project of inventory.workspaceProjects) projects.add(project);
}
}
const { selectProjectTests } = await import("./vitest-selection-graph.mjs");
const graphs = [];
const changed = productInputs(plan.changed);
for (const project of projects) {
const graph = await selectProjectTests({ project, changed, onlyOwnChanges: true });
graphs.push(graph);
console.log(`${project}: ${graph.selected.length}/${graph.inventory.length} files, ${graph.alwaysRun.length} retained conservatively, ${graph.seconds.toFixed(1)}s`);
for (const owner of Object.values(owners)) if (owner[project] === null) owner[project] = graph.inventory;
}
const covered = new Set(graphs.flatMap((graph) => graph.coveredChanges));
for (const file of changed) assert.ok(covered.has(file), `Change outside the resolved test graphs: ${file}`);
const server = graphs.find((graph) => graph.project === "@paperclipai/server");
if (server) {
const general = runnerInventory("general-server-without-chat");
const native = runnerInventory("general-server-native-runner");
assert.deepEqual([...general.selectedGeneralServerSuites, ...general.selectedSerializedSuites,
...native.selectedGeneralServerSuites, "server/src/__tests__/chat-channels.integration.test.ts"].sort(),
server.inventory, "Server inventory must be completely covered by existing lane ownership");
}
return { ...refineTestPlan({ plan, head, owners, graphs, durations: selectionDurations() }), graphs };
}
async function main() {
const plan = JSON.parse(process.env.CI_PLAN || "null");
const output = (key, value) => {
if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`);
};
if (process.argv.includes("--eligible")) {
const reason = selectionEligibility(plan);
console.log(reason ?? "Source diff is eligible for graph selection");
output("enabled", !reason);
return;
}
const directory = ".ci-test-selection";
mkdirSync(directory, { recursive: true });
const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).trim();
let result;
try {
const head = git("rev-parse", "HEAD");
assert.equal(head, process.env.GITHUB_SHA, "Selection must describe the tested merge commit");
assert.ok(plan.diff, "Missing trusted merge diff");
const local = mergeDiff(head, plan.diff.prHead);
assert.deepEqual(local.diff, plan.diff, "Selection must use the trusted merge parents");
assert.deepEqual(local.changed, [...new Set(plan.changed)].sort(), "Local diff disagrees with the trusted routing diff");
assert.equal(git("status", "--porcelain", "--untracked-files=normal"), "", "Source inputs changed during dependency setup");
process.env.NODE_ENV = "test";
process.env.VITEST = "true";
result = await createTestSelection(plan, head);
writeFileSync(`${directory}/manifest.json`, JSON.stringify(result.manifest));
writeFileSync(`${directory}/audit.json`, JSON.stringify(result.graphs, null, 2));
output("selected", "true");
} catch (error) {
console.warn(`Using the complete task plan: ${error.message}`);
result = { plan: { ...plan, testSelectionFallback: error.message } };
output("selected", "false");
}
writeFileSync(`${directory}/plan.json`, JSON.stringify(result.plan));
if (process.env.GITHUB_STEP_SUMMARY) {
const stats = result.plan.testSelection?.stats;
appendFileSync(process.env.GITHUB_STEP_SUMMARY, stats
? ["### Vitest test selection", "", "| Lane | Test files | Jobs |", "| --- | ---: | ---: |",
...stats.map((s) => `| ${s.group} | ${s.selected}/${s.total} | ${s.jobs}/${s.beforeJobs} |`), "",
"Filesystem/process/opaque consumers are retained. Chat and native Runner lanes keep their existing ownership.", ""].join("\n")
: `### Vitest test selection\n\nFull task plan: ${result.plan.testSelectionFallback}\n`);
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((error) => { console.error(error); process.exitCode = 1; });
}
+66
View File
@@ -0,0 +1,66 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { execFileSync } from "node:child_process";
import { readFileSync, statSync } from "node:fs";
export const selectableGroups = ["general-server-without-chat", "general-workspaces-a",
"general-workspaces-b", "general-workspaces-c", "serialized"];
export function selectionHash(manifest) {
return createHash("sha256").update(JSON.stringify({ version: manifest.version, lanes: manifest.lanes })).digest("hex");
}
export function validateSelection(manifest, { head, hash } = {}) {
assert.equal(manifest.version, 1, "Unsupported test selection");
assert.match(manifest.head, /^[a-f0-9]{40}$/);
if (head) assert.equal(manifest.head, head, "Selection belongs to another checkout");
assert.ok(manifest.lanes && typeof manifest.lanes === "object" && !Array.isArray(manifest.lanes));
const seen = new Set();
for (const [group, lane] of Object.entries(manifest.lanes)) {
assert.ok(selectableGroups.includes(group), `Unknown selection group: ${group}`);
assert.ok(Array.isArray(lane.shards) && lane.shards.length > 0 && lane.shards.length <= 12);
for (const shard of lane.shards) {
assert.ok(shard && typeof shard === "object" && !Array.isArray(shard));
let count = 0;
for (const [project, files] of Object.entries(shard)) {
assert.ok(project && Array.isArray(files) && files.length > 0);
for (const file of files) {
assert.equal(typeof file, "string");
assert.ok(/^(ui|server|cli|packages)\//.test(file) && !/[\0\r\n\\:*?\[\]{}]/.test(file)
&& !file.split("/").some((part) => ["", ".", ".."].includes(part)), `Invalid selected path: ${file}`);
const key = `${project}:${file}`;
assert.ok(!seen.has(key), `Test assigned twice: ${key}`);
seen.add(key);
count++;
}
}
assert.ok(count > 0, "Empty selection shard");
}
}
if (hash) assert.equal(selectionHash(manifest), hash, "Selection artifact does not match its plan");
return manifest;
}
export function loadSelection(file = process.env.PAPERCLIP_CI_TEST_SELECTION) {
const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
const manifest = validateSelection(JSON.parse(readFileSync(file, "utf8")), {
head, hash: process.env.PAPERCLIP_CI_SELECTION_HASH,
});
for (const lane of Object.values(manifest.lanes)) {
for (const shard of lane.shards) for (const files of Object.values(shard)) {
for (const file of files) assert.ok(statSync(file).isFile(), `Selected test missing: ${file}`);
}
}
return manifest;
}
export function selectedShard(manifest, options, projects) {
const group = options.mode === "serialized" ? "serialized" : options.group;
const lane = manifest.lanes[group];
assert.ok(lane, `Missing selection for ${group}`);
assert.equal(lane.shards.length, options.shardCount ?? 1, "Selection shard count mismatch");
const shard = lane.shards[options.shardIndex ?? 0];
assert.ok(shard, "Missing selection shard");
for (const project of Object.keys(shard)) assert.ok(projects.includes(project), `Wrong project for ${group}: ${project}`);
return shard;
}
+77
View File
@@ -0,0 +1,77 @@
import assert from "node:assert/strict";
import { loadShardDurations, partitionGeneralServerSuites } from "../general-server-shard.mjs";
import { selectionHash, validateSelection } from "./test-selection-manifest.mjs";
// Preserve the existing per-job budget. Small selections use fewer runners;
// large selections retain the existing parallelism and serialized ownership.
export function partitionSelection(inventory, selected, maxShards, durations = {}) {
const totalWeight = partitionGeneralServerSuites(inventory, 1, durations)[0].totalWeight;
const selectedWeight = partitionGeneralServerSuites(selected, 1, durations)[0].totalWeight;
const budget = totalWeight / maxShards || 1;
const count = Math.min(maxShards, selected.length, Math.max(1, Math.ceil(selectedWeight / budget)));
return selected.length ? partitionGeneralServerSuites(selected, count, durations).map((shard) => shard.files) : [];
}
export function refineTestPlan({ plan, head, owners, graphs, durations = {} }) {
const result = structuredClone(plan);
const manifest = { version: 1, head, lanes: {} };
const stats = [];
const byProject = new Map(graphs.map((graph) => [graph.project, graph]));
for (const job of ["general_tests", "verify_serialized_server"]) {
if (!plan.jobs[job].run) continue;
const entries = plan.jobs[job].matrix.include;
const groups = [...new Set(entries.map((entry) => entry.group ?? "serialized"))];
const matrix = [];
for (const group of groups) {
const original = entries.filter((entry) => (entry.group ?? "serialized") === group);
if (group === "general-chat") { matrix.push(...original); continue; }
const owned = owners[group];
assert.ok(owned, `Missing inventory owner: ${group}`);
const projectByFile = new Map();
const selected = [];
for (const [project, files] of Object.entries(owned)) {
const graph = byProject.get(project);
assert.ok(graph, `Missing project graph: ${project}`);
const inventory = new Set(graph.inventory);
const affected = new Set(graph.selected);
for (const file of files) {
assert.ok(inventory.has(file), `Runner/Vitest inventory disagreement: ${file}`);
assert.ok(!projectByFile.has(file), `Ambiguous owner: ${file}`);
projectByFile.set(file, project);
if (affected.has(file)) selected.push(file);
}
}
const partitions = partitionSelection([...projectByFile.keys()], selected, original.length, durations[group]);
if (partitions.length) {
const shards = partitions.map((files) => {
const shard = {};
for (const file of files) (shard[projectByFile.get(file)] ??= []).push(file);
return shard;
});
manifest.lanes[group] = { shards };
for (let index = 0; index < shards.length; index++) {
const entry = { ...original[0], selection: true };
if (original[0].shard_count) {
Object.assign(entry, { shard_index: index, shard_count: shards.length });
if (entry.shard_label) entry.shard_label = `${index + 1}/${shards.length}`;
if (entry.group_label) entry.group_label = entry.group_label.replace(/\(\d+\/\d+\)/, `(${index + 1}/${shards.length})`);
}
matrix.push(entry);
}
}
stats.push({ group, total: projectByFile.size, selected: selected.length, beforeJobs: original.length, jobs: partitions.length });
}
result.jobs[job] = { ...result.jobs[job], run: matrix.length > 0, selected: matrix.length, proposed: matrix.length,
matrix: { include: matrix.length ? matrix : entries } };
}
validateSelection(manifest);
result.testSelection = { version: 1, head, hash: selectionHash(manifest), stats };
return { plan: result, manifest };
}
export function selectionDurations() {
return {
"general-server-without-chat": loadShardDurations("scripts/general-server-shard-durations.json"),
serialized: loadShardDurations("scripts/serialized-shard-durations.json"),
};
}
@@ -0,0 +1,56 @@
import assert from "node:assert/strict";
import { execFileSync, spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { selectProjectTests } from "./vitest-selection-graph.mjs";
import { selectionHash } from "./test-selection-manifest.mjs";
test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => {
const repo = process.cwd();
const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-")));
t.after(() => rmSync(root, { recursive: true, force: true }));
for (const directory of ["ui/src", "server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true });
symlinkSync(path.join(repo, "node_modules"), path.join(root, "node_modules"), "dir");
const write = (file, body) => writeFileSync(path.join(root, file), body);
const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
write(".gitignore", "node_modules/\nselection.json\n");
write("package.json", '{"name":"selection-regression","private":true,"type":"module"}');
write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}');
write("ui/src/leaf.mjs", "export const value = 42;\n");
write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));');
write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");');
write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));');
git("init", "-q"); git("add", ".");
const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
commit();
const run = (files) => {
const manifest = { version: 1, head: git("rev-parse", "HEAD"), lanes: {
"general-workspaces-a": { shards: [{ "@paperclipai/ui": files }] },
} };
write("selection.json", JSON.stringify(manifest));
return spawnSync(process.execPath, [path.join(repo, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1"], {
cwd: root, encoding: "utf8", timeout: 60000,
env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: path.join(root, "selection.json"),
PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) },
});
};
const baseline = run(["ui/src/import.test.mjs", "ui/src/contract.test.mjs"]);
assert.equal(baseline.status, 0, baseline.stdout + baseline.stderr);
const missing = run(["ui/src/leaf.mjs"]);
assert.equal(missing.status, 1, "an existing source file is not an executable test specification");
assert.match(missing.stdout + missing.stderr, /Selected files must match/);
write("ui/src/leaf.mjs", `${readFileSync(path.join(root, "ui/src/leaf.mjs"), "utf8")}// FORBIDDEN_MARKER\n`);
commit();
const graph = await selectProjectTests({ root, project: "@paperclipai/ui", changed: ["ui/src/leaf.mjs"] });
assert.deepEqual(graph.related, ["ui/src/import.test.mjs"]);
const raw = run(graph.related);
assert.equal(raw.status, 0, raw.stdout + raw.stderr);
assert.ok(graph.selected.includes("ui/src/contract.test.mjs"));
const guarded = run(graph.selected);
assert.equal(guarded.status, 1, guarded.stdout + guarded.stderr);
assert.match(guarded.stdout + guarded.stderr, /filesystem contract/);
assert.match(guarded.stdout + guarded.stderr, /FORBIDDEN_MARKER/);
});
+100
View File
@@ -0,0 +1,100 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import vm from "node:vm";
import test from "node:test";
import { planTasks } from "../../.github/scripts/ci-plan.mjs";
const workflow = readFileSync(".github/workflows/pr-trusted.yml", "utf8");
const block = (job) => workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
const plan = planTasks({ config: JSON.parse(readFileSync("turbo.json", "utf8")),
lanes: JSON.parse(readFileSync(".github/ci-task-matrix.json", "utf8")), paths: ["ui/src/pages/Inbox.tsx"] });
const evaluate = (expression, needs, cancelled = false) => vm.runInNewContext(expression, {
needs, fromJSON: JSON.parse, toJSON: JSON.stringify, always: () => true, cancelled: () => cancelled,
});
test("failed, timed-out, skipped and unavailable selectors leave the original matrix runnable", () => {
for (const result of ["failure", "cancelled", "skipped", "success"]) {
const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
test_selection: { result, outputs: { plan: "" } } };
const job = block("general_tests");
const condition = job.match(/if: \$\{\{ (.*) \}\}/)[1];
const matrix = job.match(/matrix: \$\{\{ (.*) \}\}/)[1];
assert.equal(evaluate(condition, needs), true, result);
assert.deepEqual(JSON.parse(JSON.stringify(evaluate(matrix, needs))), plan.jobs.general_tests.matrix);
assert.equal(evaluate(condition, needs, true), false, "cancellation must not start more runners");
}
});
test("a published selection controls the matrix and the required aggregate", () => {
const selected = structuredClone(plan);
selected.testSelection = { version: 1, hash: "test" };
selected.jobs.general_tests.matrix.include = [selected.jobs.general_tests.matrix.include[0]];
selected.jobs.general_tests.selected = 1;
const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
test_selection: { result: "success", outputs: { plan: JSON.stringify(selected) } } };
const matrix = block("general_tests").match(/matrix: \$\{\{ (.*) \}\}/)[1];
assert.equal(evaluate(matrix, needs).include.length, 1);
const aggregate = block("verify");
const names = aggregate.match(/needs: \[([^\]]+)\]/)[1].split(", ");
const code = aggregate.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
.split("\n").map((line) => line.slice(10)).join("\n");
const results = Object.fromEntries(names.map((name) => [name, { result:
["gate", "policy", "test_selection"].includes(name) || selected.jobs[name]?.run ? "success" : "skipped" }]));
const run = () => spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8",
env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(selected), CI_RESULTS: JSON.stringify(results) } });
assert.equal(run().status, 0);
for (const job of ["general_tests", "test_selection"]) {
for (const result of ["failure", "cancelled", "skipped"]) {
results[job].result = result;
assert.notEqual(run().status, 0, `${job}: ${result}`);
}
results[job].result = "success";
}
selected.jobs.general_tests.run = false;
results.general_tests.result = "skipped";
assert.equal(run().status, 0, "an explicitly empty selected lane may be skipped");
assert.ok(aggregate.includes("needs.test_selection.outputs.plan || needs.gate.outputs.plan"));
});
test("publication follows successful upload and manifests are downloaded without ignoring failures", () => {
const selection = block("test_selection");
assert.ok(selection.indexOf("Upload validated selection") < selection.indexOf("Publish the executable plan"));
assert.match(selection, /timeout-minutes: 5/);
assert.match(selection, /vars.CI_VITEST_SELECTION == '' \|\| vars.CI_VITEST_SELECTION == 'enabled'/);
for (const job of ["general_tests", "verify_serialized_server"]) {
assert.match(block(job), /PAPERCLIP_CI_SELECTION_HASH:.*testSelection.hash/);
assert.match(block(job), /Download selected test files\n if: matrix.selection\n uses: actions\/download-artifact@/);
assert.doesNotMatch(block(job), /continue-on-error: true\n.*download-artifact/);
}
});
test("failed-job retries use the original selector artifact, not the retry attempt", (t) => {
const directory = mkdtempSync(path.join(tmpdir(), "ci-retry-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
mkdirSync(path.join(directory, ".ci-test-selection"));
const output = path.join(directory, "output");
const publication = block("test_selection").split(" - name: Publish the executable plan")[1];
assert.match(publication, /ARTIFACT_ID: \$\{\{ steps.upload_selection.outputs.artifact-id \}\}/);
const code = publication.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
.split("\n").map((line) => line.slice(10)).join("\n");
const selected = { ...plan, testSelection: { version: 1, hash: "test" } };
const run = (value, artifactId) => {
writeFileSync(path.join(directory, ".ci-test-selection/plan.json"), JSON.stringify(value));
writeFileSync(output, "");
return spawnSync(process.execPath, ["--input-type=module", "-e", code], {
cwd: directory, encoding: "utf8", env: { ...process.env, ARTIFACT_ID: artifactId, GITHUB_OUTPUT: output },
});
};
assert.equal(run(selected, "12345").status, 0);
const savedPlan = readFileSync(output, "utf8").trim().slice("plan=".length);
for (const job of ["general_tests", "verify_serialized_server"]) {
const expression = block(job).match(/artifact-ids: \$\{\{ (.*) \}\}/)[1];
assert.equal(vm.runInNewContext(expression, { fromJSON: JSON.parse, github: { run_attempt: 2 },
needs: { test_selection: { outputs: { plan: savedPlan } } } }), "12345");
}
assert.notEqual(run(selected, "").status, 0, "a selection cannot be published without its uploaded artifact");
assert.equal(run(plan, "").status, 0, "fallback plans need no selection artifact");
});
@@ -0,0 +1,198 @@
import assert from "node:assert/strict";
import { execFileSync, spawnSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { selectProjectTests } from "./vitest-selection-graph.mjs";
import { partitionSelection, refineTestPlan } from "./test-selection-plan.mjs";
import { selectionHash, selectedShard, validateSelection } from "./test-selection-manifest.mjs";
import { selectionEligibility } from "./select-tests.mjs";
import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
const root = process.cwd();
const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
test("real graph retains direct and indirect filesystem, process and opaque dependencies", async (t) => {
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vg-")));
t.after(() => rmSync(directory, { recursive: true, force: true }));
mkdirSync(path.join(directory, "ui/src"), { recursive: true });
mkdirSync(path.join(directory, "ui/src/glob"));
symlinkSync(path.join(root, "node_modules"), path.join(directory, "node_modules"), "dir");
const write = (file, text) => writeFileSync(path.join(directory, file), text);
write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
write("ui/vitest.config.mjs", `export default {resolve:{alias:{"@leaf":${JSON.stringify(path.join(directory, "ui/src/leaf.ts"))}}},test:{name:"fixture",include:["src/**/*.test.ts"],setupFiles:["./setup.ts"]}}`);
write("ui/setup.ts", 'import "./src/environment.ts";');
write("ui/src/environment.ts", "export const environment = true;");
write("ui/src/leaf.ts", "export const value = 42;");
write("ui/src/bridge.ts", 'export {value} from "@leaf";');
write("ui/src/helper.ts", 'import {readFileSync} from "node:fs"; export const read = (file: string) => readFileSync(file, "utf8");');
write("ui/src/import.test.ts", 'import {value} from "./bridge"; console.log(value); throw new Error("selection must not execute tests");');
write("ui/src/reader.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
write("ui/src/indirect.test.ts", 'import {read} from "./helper"; read("input"); throw new Error("selection must not execute tests");');
write("ui/src/process.test.ts", 'import {execFileSync} from "node:child_process"; execFileSync("must-not-execute");');
write("ui/src/dynamic.test.ts", 'const load = (file: string) => import(file);');
write("ui/src/cjs.test.ts", 'const load = require;');
write("ui/src/builtin-loader.test.ts", 'process.getBuiltinModule("fs").readFileSync("input");');
write("ui/src/opaque.test.ts", 'import {parse} from "yaml"; console.log(parse("value: true"));');
write("ui/src/independent.test.ts", 'export const independent = true;');
write("ui/src/mocked.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); import {read} from "./helper"; read("input");');
write("ui/src/partial.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",async (original)=>await original()); import {read} from "./helper"; read("input");');
write("ui/src/automock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper"); import {read} from "./helper"; read("input");');
write("ui/src/bypass.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); await vi.importActual("./helper");');
write("ui/src/spy.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",{spy:true}); import {read} from "./helper"; read("input");');
write("ui/src/remock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); vi.doMock("./helper",async original=>await original());');
write("ui/src/arguments.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",function(){return arguments[0]();});');
write("ui/src/glob/one.ts", "export const value = 1;");
write("ui/src/globber.ts", 'export const modules = import.meta.glob("./glob/*.ts", {eager:true});');
write("ui/src/glob.test.ts", 'import {modules} from "./globber"; console.log(modules);');
execFileSync("git", ["init", "-q"], { cwd: directory });
execFileSync("git", ["add", "."], { cwd: directory });
const select = (changed) => selectProjectTests({ root: directory, project: "fixture", changed });
const result = await select(["ui/src/leaf.ts"]);
assert.deepEqual(result.related, ["ui/src/import.test.ts"]);
for (const file of ["reader", "indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) {
assert.ok(result.alwaysRun.includes(`ui/src/${file}.test.ts`), `${file} must never be dropped`);
}
assert.ok(!result.selected.includes("ui/src/independent.test.ts"));
assert.ok(!result.alwaysRun.includes("ui/src/mocked.test.ts"), "a completely replaced local dependency performs no real IO");
assert.ok(!result.alwaysRun.includes("ui/src/glob.test.ts"), "literal Vite globs use the real dependency graph");
assert.ok((await select(["ui/src/glob/one.ts"])).related.includes("ui/src/glob.test.ts"));
write("ui/src/glob/two.ts", "export const value = 2;");
execFileSync("git", ["add", "."], { cwd: directory });
assert.ok((await select(["ui/src/glob/two.ts"])).related.includes("ui/src/glob.test.ts"), "new glob members cannot be omitted");
const setup = await select(["ui/src/environment.ts"]);
assert.equal(setup.setupChanged, true);
assert.deepEqual(setup.selected, setup.inventory, "transitive setup edits affect every test");
write("ui/src/broken.test.ts", 'import "./missing";');
const broken = await select(["ui/src/leaf.ts"]);
assert.ok(broken.selected.includes("ui/src/broken.test.ts"), "an unresolved dependency must retain its test");
write("ui/src/generated.ts", "export const generated = 42;");
write("ui/src/generated.test.ts", 'import {generated} from "./generated"; console.log(generated);');
execFileSync("git", ["add", "ui/src/generated.test.ts"], { cwd: directory });
assert.ok((await select(["ui/src/leaf.ts"])).alwaysRun.includes("ui/src/generated.test.ts"),
"generated inputs have no trustworthy source-to-output graph and retain their consumers");
write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph");
mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true });
for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) {
write(file, readFileSync(path.join(root, file), "utf8"));
}
write("ui/setup.ts", 'import "../server/src/__tests__/setup-supertest.ts";');
const reviewed = await select(["ui/src/leaf.ts"]);
assert.ok(!reviewed.selected.includes("ui/src/independent.test.ts"), "reviewed isolated setup permits file selection");
const helper = "server/src/__tests__/helpers/isolated-test-home.ts";
const changedSetup = await select([helper]);
assert.deepEqual(changedSetup.selected, changedSetup.inventory, "reviewed helpers still invalidate all tests when changed");
write(helper, `${readFileSync(path.join(directory, helper), "utf8")}\n// setup edited since review\n`);
await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup/, "a stale review cannot authorize new setup effects");
});
test("partitioning reduces runners and preserves an exact, non-overlapping file cover", () => {
const all = Array.from({ length: 120 }, (_, index) => `server/src/${index}.test.ts`);
const selected = all.slice(0, 15);
const shards = partitionSelection(all, selected, 12);
assert.equal(shards.length, 2);
assert.deepEqual(shards.flat().sort(), selected.sort());
assert.equal(new Set(shards.flat()).size, selected.length);
assert.deepEqual(partitionSelection(all, [], 12), []);
assert.equal(partitionSelection(all, all, 12).length, 12);
const durations = Object.fromEntries(all.map((file, index) => [file, index === 0 ? 1000 : 1]));
assert.equal(partitionSelection(all, [all[0]], 12, durations).length, 1);
assert.equal(partitionSelection(all, [all[0]], 12, Object.fromEntries(all.map((file) => [file, 0]))).length, 1);
});
function fixturePlan() {
const files = ["ui/src/pages/Inbox.test.tsx", "ui/src/pages/apps/chat/chat-ui-contract.test.ts"];
const entries = [0, 1].map((shard_index) => ({ group: "general-workspaces-a", group_label: `workspaces-a (${shard_index + 1}/2)`, shard_index, shard_count: 2, task: "ci:workspaces-a" }));
const plan = { version: 1, mode: "enabled", forceReason: null, changed: ["ui/src/pages/Inbox.tsx"], jobs: {
general_tests: { run: true, matrix: { include: entries }, selected: 2, total: 2 },
verify_serialized_server: { run: false, matrix: { include: [{ task: "ci:serialized" }] } },
} };
const owners = { "general-workspaces-a": { "@paperclipai/ui": files } };
const graphs = [{ project: "@paperclipai/ui", inventory: files, selected: [files[1]] }];
return { files, ...refineTestPlan({ plan, head, owners, graphs }) };
}
test("runner consumes the exact manifest, and rejects stale, missing, or mismatched manifests", (t) => {
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vm-")));
t.after(() => rmSync(directory, { recursive: true, force: true }));
const { manifest, plan } = fixturePlan();
assert.equal(plan.jobs.general_tests.selected, 1);
const file = path.join(directory, "selection.json");
const run = (selection = manifest, hash = selectionHash(manifest)) => {
writeFileSync(file, JSON.stringify(selection));
return spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: hash },
});
};
const result = run();
assert.equal(result.status, 0, result.stderr);
assert.deepEqual(JSON.parse(result.stdout).selection, manifest.lanes["general-workspaces-a"].shards[0]);
assert.notEqual(run({ ...manifest, head: "0".repeat(40) }).status, 0);
assert.notEqual(run(manifest, "bad-hash").status, 0);
assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 2 }, ["@paperclipai/ui"]));
assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 1 }, ["@paperclipai/server"]));
const duplicate = structuredClone(manifest);
duplicate.lanes["general-workspaces-a"].shards.push(duplicate.lanes["general-workspaces-a"].shards[0]);
assert.throws(() => validateSelection(duplicate), /assigned twice/);
const wrongLane = { version: 1, head, lanes: { "general-server-without-chat": {
shards: [{ "@paperclipai/server": ["server/src/__tests__/issues-service.test.ts"] }],
} } };
writeFileSync(file, JSON.stringify(wrongLane));
const misplaced = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(wrongLane) },
});
assert.notEqual(misplaced.status, 0);
assert.match(misplaced.stderr, /belongs to another server lane/);
});
test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow CI", () => {
const plan = fixturePlan().plan;
assert.equal(selectionEligibility(plan), null);
for (const changed of [["ui/vitest.setup.ts"], ["ui/vite.config.ts"], ["package.json"], ["pnpm-lock.yaml"],
["docs/api/issues.md"], ["ui/src/missing-deleted.ts"], ["server/fixtures/input.json"], []]) {
assert.ok(selectionEligibility({ ...plan, changed }), JSON.stringify(changed));
}
for (const mode of ["full", "shadow"]) assert.ok(selectionEligibility({ ...plan, mode }));
assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" }));
});
test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => {
const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-")));
t.after(() => rmSync(directory, { recursive: true, force: true }));
mkdirSync(path.join(directory, "ui/src"), { recursive: true });
const file = path.join(directory, "ui/src/input.ts");
const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
git("init", "-q"); writeFileSync(file, "export const value = 1;"); git("add", "."); commit();
const base = git("rev-parse", "HEAD");
git("checkout", "-qb", "pr");
writeFileSync(file, "export const value = 2;"); commit();
const prHead = git("rev-parse", "HEAD");
git("checkout", "--detach", base); git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "merge", "--no-ff", "pr", "-m", "merge");
const merge = git("rev-parse", "HEAD");
writeFileSync(file, "export const value = 3; // generated during setup");
const plan = { version: 1, mode: "enabled", forceReason: null, ...mergeDiff(merge, prHead, directory),
jobs: { general_tests: { run: true, selected: 2, matrix: { include: [{ shard_index: 0 }, { shard_index: 1 }] } } } };
const result = spawnSync(process.execPath, [path.join(root, "scripts/ci/select-tests.mjs")], {
cwd: directory, encoding: "utf8", env: { ...process.env, CI_PLAN: JSON.stringify(plan), BASE_SHA: base,
GITHUB_SHA: merge, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" },
});
assert.equal(result.status, 0, result.stderr);
const fallback = JSON.parse(readFileSync(path.join(directory, ".ci-test-selection/plan.json"), "utf8"));
assert.match(fallback.testSelectionFallback, /Source inputs changed/);
assert.deepEqual(fallback.jobs, plan.jobs);
assert.equal(fallback.testSelection, undefined);
});
test("mixed audited documentation stays graph eligible and large complete source diffs are allowed", () => {
const plan = fixturePlan().plan;
for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
assert.equal(selectionEligibility({ ...plan, changed: [...plan.changed, doc] }), null);
}
assert.equal(selectionEligibility({ ...plan, changed: Array(350).fill(plan.changed[0]) }), null);
assert.ok(selectionEligibility({ ...plan, changed: ["doc/DEVELOPING.md"] }));
assert.ok(selectionEligibility({ ...plan, changed: [...plan.changed, "docs/api/issues.md"] }));
});
@@ -0,0 +1,38 @@
import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { measureVitestImpact } from "./vitest-impact-report.mjs";
test("real Vitest graph follows aliases/transitive imports but misses filesystem contracts", async (t) => {
const root = mkdtempSync(path.join(tmpdir(), "vitest-impact-"));
t.after(() => rmSync(root, { recursive: true, force: true }));
mkdirSync(path.join(root, "unit"));
const write = (file, text) => writeFileSync(path.join(root, file), text);
write("vitest.config.mjs", 'export default { test: { projects: ["unit"] } };');
write("unit/vitest.config.mjs", `export default {
resolve: { alias: { "@leaf": ${JSON.stringify(path.join(root, "unit/leaf.mjs"))} } },
test: { name: "fixture", include: ["**/*.test.mjs"] }
};`);
write("unit/leaf.mjs", "export const value = 42;");
write("unit/bridge.mjs", 'export { value } from "@leaf";');
// The graph must be collected without executing the test modules.
write("unit/related.test.mjs", 'import { value } from "./bridge.mjs"; throw new Error("must not execute");');
write("unit/dynamic.test.mjs", 'const load = () => import("./leaf.mjs"); throw new Error("must not execute");');
write("unit/independent.test.mjs", 'throw new Error("must not execute");');
write("unit/reader.test.mjs", 'import { readFileSync } from "node:fs"; const body = readFileSync(new URL("./leaf.mjs", import.meta.url), "utf8");');
const report = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/leaf.mjs"] });
assert.equal(report.mode, "report-only");
assert.equal(report.projects[0].totalFiles, 4);
assert.deepEqual(report.projects[0].selectedTests, ["unit/dynamic.test.mjs", "unit/related.test.mjs"]);
assert.deepEqual(report.projects[0].omittedTests, ["unit/independent.test.mjs", "unit/reader.test.mjs"]);
assert.ok(report.limitations.some((message) => message.includes("filesystem")));
const direct = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/independent.test.mjs"] });
assert.deepEqual(direct.projects[0].selectedTests, ["unit/independent.test.mjs"]);
const unmapped = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unmapped.md"] });
assert.equal(unmapped.projects[0].selectedFiles, 0);
assert.equal(unmapped.mode, "report-only", "an empty graph is not permission to skip CI");
});
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import { realpathSync, writeFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
function repoPath(root, file) {
const relative = path.relative(root, path.resolve(root, file));
if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
throw new Error(`Expected a file inside the checkout: ${file}`);
}
return relative.split(path.sep).join("/");
}
// This is an experiment report, deliberately not a CI execution plan. Import
// graphs cannot describe filesystem reads, subprocesses, or external services.
export async function measureVitestImpact({ root = process.cwd(), projects, changedFiles }) {
root = realpathSync(root);
if (!projects?.length || !changedFiles?.length) {
throw new Error("Specify at least one project and one changed file");
}
const files = [...new Set(changedFiles.map((file) => repoPath(root, file)))].sort();
const { createVitest } = await import("vitest/node");
const results = [];
// Each project gets its own graph to bound memory usage. Cross-package imports
// are still followed, but projects omitted by the caller are NOT assessed.
for (const project of new Set(projects)) {
const started = performance.now();
const ctx = await createVitest({
root,
watch: false,
project: [project],
exclude: ["**/dist/**"],
reporters: [],
related: files.map((file) => path.resolve(root, file)),
});
try {
if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) {
throw new Error(`Expected exactly one project named ${project}`);
}
const inventory = await ctx.globTestSpecifications();
if (!inventory.length) throw new Error(`No test files found for ${project}`);
const ready = performance.now();
const selected = await ctx.getRelevantTestSpecifications();
const finished = performance.now();
const selectedTests = selected.map((spec) => repoPath(root, spec.moduleId)).sort();
const selectedSet = new Set(selectedTests);
results.push({
project,
totalFiles: inventory.length,
selectedFiles: selectedTests.length,
startupSeconds: (ready - started) / 1000,
selectionSeconds: (finished - ready) / 1000,
selectedTests,
omittedTests: inventory.map((spec) => repoPath(root, spec.moduleId))
.filter((file) => !selectedSet.has(file)).sort(),
});
} finally {
await ctx.close();
}
}
return {
mode: "report-only",
changedFiles: files,
limitations: [
"Only the explicitly requested projects were assessed.",
"Selection follows imports, not filesystem reads, subprocesses, or external services.",
"Zero selected files is not proof that a change needs no tests.",
"This report must not be used to omit required CI jobs.",
],
projects: results,
};
}
async function main() {
const { values } = parseArgs({ options: {
project: { type: "string", multiple: true },
file: { type: "string", multiple: true },
base: { type: "string" },
output: { type: "string" },
} });
if (!values.project?.length || (!!values.base === !!values.file?.length)) {
throw new Error("Usage: node scripts/ci/vitest-impact-report.mjs --project NAME [--project NAME] (--file PATH [--file PATH] | --base REF) [--output FILE]");
}
const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
const head = git("rev-parse", "--verify", "HEAD^{commit}").trim();
let comparison = { kind: "hypothetical", head };
let changedFiles = values.file;
if (values.base) {
// Vitest reads the working tree, so do not label dirty sources as HEAD.
if (git("status", "--porcelain", "--untracked-files=no").trim()) {
throw new Error("--base requires a clean tracked working tree; use --file for a hypothetical change");
}
const base = git("rev-parse", "--verify", "--end-of-options", `${values.base}^{commit}`).trim();
// Compare the actual trees, including both sides of a rename. The caller
// supplies the event base; HEAD should be the tested PR merge commit in CI.
changedFiles = git("diff", "--name-only", "--no-renames", "-z", base, head, "--")
.split("\0").filter(Boolean);
comparison = { kind: "git-diff", base, head };
}
process.env.NODE_ENV = "test";
process.env.VITEST = "true";
// Config/plugins are executable code. Run this only in a test job, never in
// the trusted metadata-only CI routing gate. Bound a failed graph traversal.
const timeout = setTimeout(() => {
console.error("Vitest impact analysis exceeded five minutes; report unavailable");
process.exit(1);
}, 300_000);
try {
const report = await measureVitestImpact({ projects: values.project, changedFiles });
const json = `${JSON.stringify({ ...report, comparison }, null, 2)}\n`;
if (values.output) writeFileSync(values.output, json);
else process.stdout.write(json);
for (const result of report.projects) {
console.error(`${result.project}: ${result.selectedFiles}/${result.totalFiles} files; `
+ `${result.selectionSeconds.toFixed(2)}s graph + ${result.startupSeconds.toFixed(2)}s startup`);
}
} finally {
clearTimeout(timeout);
}
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
main().catch((error) => { console.error(error); process.exitCode = 1; });
}
+113
View File
@@ -0,0 +1,113 @@
# Vitest dependency selection experiment
The production selector is documented in `doc/DEVELOPING.md`. This probe remains
a diagnostic for the raw import graph, without the production retention rules.
This probe measures Vitest's real import graph against the current checkout.
It does not change the CI matrix or authorize skipping tests. It loads Vitest
configurations and Vite plugins, so run it in a normal test environment, never
in the trusted metadata-only routing job.
Install the workspace dependencies and prepare the same shared build outputs
used by CI:
```sh
pnpm install --frozen-lockfile
node scripts/ensure-plugin-build-deps.mjs
node --test scripts/ci/vitest-impact-report.integration.test.mjs
```
Probe a hypothetical change (the file need not actually be modified):
```sh
node scripts/ci/vitest-impact-report.mjs \
--project @paperclipai/ui --file ui/src/pages/Inbox.tsx \
--output /tmp/inbox-impact.json
```
Or compare a base revision with the checked-out HEAD. This requires a clean
tracked working tree. For a PR, check out the tested merge commit and supply
the event's base SHA, as the existing CI planner does:
```sh
node scripts/ci/vitest-impact-report.mjs \
--project @paperclipai/ui --project @paperclipai/server \
--base BASE_SHA --output /tmp/pr-impact.json
```
Every requested project is analyzed separately to bound memory usage. Imports
across package boundaries are followed within each graph, but unrequested
projects are **not assessed**. Runner's separate Vitest configurations, browser
E2E, Rust, typechecking, and builds are outside this experiment. The JSON records
selected and omitted test files, project inventory, startup and graph times,
and the comparison revisions. A failed or timed-out probe is not an empty plan.
## Initial observations
Measured on `5c807d11fa` with Vitest 5.0.3, macOS, and Node 26.4.0. These are
hypothetical single-file changes against that revision, not replays of historical
PR checkouts. CI uses Node 24 and different hardware.
| Changed file | Assessed project | Selected / total test files |
| --- | --- | ---: |
| `ui/src/pages/Inbox.tsx` | UI | 6 / 844 |
| `ui/src/components/MarkdownBody.tsx` | UI | 135 / 844 |
| `server/src/middleware/auth.ts` | Server | 45 / 1,428 |
| `server/src/services/run-failure-report.ts` | Server | 787 / 1,428 |
| `packages/shared/src/issue-references.ts` | Shared only | 5 / 96 |
| `docs/api/issues.md` | Server | 0 / 1,428 — known false negative |
Cold graph construction took 27.43 seconds for UI, 25.68 seconds for server,
and 2.79 seconds for shared. Subsequent queries on the same graph took about
0.05 seconds; fresh CI jobs must budget for cold construction. The six Inbox
files ran 81 assertions successfully in 20.92 seconds with four workers.
These figures exclude dependency installation and shared package builds.
For a local execution comparison, the complete UI project passed all 10,043
assertions in 844 files in 361.50 seconds with the same four-worker setting.
The initial Inbox graph plus selected execution took about 48 seconds, versus
six minutes for the full project. This is a local experiment, not a measured
change in GitHub CI latency. Later runs benefit from filesystem/transform caches.
The CLI's Git-diff path was also exercised against the actual `HEAD^..HEAD`
change for PR #18321. Across its eleven changed files, the server graph selected
1,138 / 1,428 files and the shared graph selected 3 / 96. Other projects were
not assessed. Broad shared/service imports can therefore preserve most of a
suite; the Inbox reduction should not be generalized to every PR.
## Why the raw graph is not yet a safe execution plan
There are concrete missing edges in this repository:
- `ui/src/pages/apps/chat/chat-ui-contract.test.ts` reads `Inbox.tsx` through
`readFileSync`. It is absent from the six-file selection even though changes
to Inbox can break its assertions.
- `server/src/__tests__/paperclip-skill-utils.test.ts` reads `docs/api/issues.md`
through `fs.readFile`. Changing that document selects zero server tests.
A deliberate negative control confirmed the first missing edge: temporarily
append a comment containing `reason: "retry_failed_run"` to `Inbox.tsx`, which
violates the contract test's source assertion without changing runtime behavior.
The fresh graph still selected six files. Running those six plus the omitted
contract produced six passing files and exactly one failing assertion, in
`chat-ui-contract.test.ts` (92 assertions passed, one failed). The original source
was restored immediately afterward. The full baseline had passed this contract.
The integration test uses actual Vitest graph traversal to verify transitive
imports, aliases, literal dynamic imports, direct test edits, and the missing
filesystem edge. Test modules deliberately throw if executed: graph discovery
must not execute the tests. Passing this test documents the limitation; it does
not prove that skipped repository tests are irrelevant.
A production rollout needs explicit dependencies or an always-run lane for
filesystem/subprocess contract tests; broad fallbacks for configuration, setup,
lockfiles, deleted files, unknown inputs, and selector failures; and comparison
against full-suite failures. Source-only changes also need those protections:
the Inbox example is not a documentation-only edge case. Once qualified, compute
the selected test list before constructing the matrix and partition that list,
preserving the existing serialized, chat, and native-runner ownership rules.
Verification for this experiment covers the real-graph integration test, CLI
probes, the complete UI project, and the deliberate negative control. Repository
wide typechecking, build, server execution, and the complete `test:run` suite
were not run; this is a report-only experiment, not a production CI rollout.
+264
View File
@@ -0,0 +1,264 @@
import { readFileSync, realpathSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import path from "node:path";
import { isBuiltin } from "node:module";
import { parseSync } from "oxc-parser";
import { createVitest } from "vitest/node";
// Everything else is opaque by default. Do not add IO clients, module loaders,
// filesystem helpers or process launchers here to improve selection counts.
const purePackages = new Set([
"vitest", "@vitest/expect", "@testing-library/react", "@testing-library/dom",
"@testing-library/user-event", "@testing-library/jest-dom", "react", "react-dom",
"react-router", "react-router-dom", "@tanstack/react-query", "@tanstack/query-core",
"zod", "lodash", "lodash-es", "date-fns", "clsx", "tailwind-merge", "class-variance-authority",
"lucide-react", "immer", "nanoid", "uuid", "semver",
"radix-ui", "motion", "@dnd-kit/core", "@dnd-kit/sortable", "@dnd-kit/utilities",
"i18next", "react-i18next",
]);
const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", "path/win32",
"url", "util", "util/types", "buffer", "events", "string_decoder", "crypto"]);
// Reviewed setup performs only temporary-home IO and Supertest socket setup,
// not reads of arbitrary repository inputs. Pin contents, not just paths: any
// edit disables this exception until its effects are reviewed again. Continue
// following local imports even for these files; helper changes affect all tests.
const reviewedSetup = new Map([
["server/src/__tests__/setup-supertest.ts", {
hash: "99d7193927468e98b41255ea9a5e37b1a681f02b259c304f3af1a52304eed2bb",
external: ["supertest/lib/test.js"],
}],
["server/src/__tests__/helpers/isolated-test-home.ts", {
hash: "bd5645dc590087f26b473beab06d5c3ddd7956a79949f0b3e9713f11897a0c1b",
external: [],
}],
]);
const packageName = (specifier) => specifier.startsWith("@")
? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
const inside = (root, file) => file.startsWith(`${root}${path.sep}`) && !file.includes(`${path.sep}node_modules${path.sep}`);
// Parse source, including imports erased by mocks/transforms. Unknown loaders
// retain their consumers. This is intentionally conservative, not a regex-based
// promise that arbitrary JavaScript has no effects.
export function sourceDependencies(file, source) {
const ast = parseSync(file, source);
const imports = new Set();
const reasons = new Set();
const mockBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
.flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)).map((s) => s.local.name)));
const hoistedBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
.flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)
&& s.local.name === s.imported.name).map((s) => s.local.name)));
const mocks = [];
if (ast.errors.length) reasons.add("unparsed source");
const literal = (node) => node?.type === "Literal" && typeof node.value === "string";
const importSource = (node) => {
if (literal(node)) imports.add(node.value);
else reasons.add("dynamic module name");
};
for (const statement of ast.program.body) {
const call = statement.type === "ExpressionStatement" ? statement.expression : null;
const callee = call?.type === "CallExpression" ? call.callee : null;
const factory = call?.arguments?.[1];
if (callee?.type === "MemberExpression" && !callee.computed && callee.property.name === "mock"
&& hoistedBindings.has(callee.object.name) && literal(call.arguments[0])
&& ["ArrowFunctionExpression", "FunctionExpression"].includes(factory?.type) && factory.params.length === 0) {
mocks.push(call.arguments[0].value);
}
}
function visit(node, parent = null) {
if (!node || typeof node !== "object") return;
if ((node.type === "ImportDeclaration" && node.importKind !== "type")
|| (["ExportNamedDeclaration", "ExportAllDeclaration"].includes(node.type) && node.exportKind !== "type")) {
if (node.source) importSource(node.source);
}
if (node.type === "ImportExpression") importSource(node.source);
if (node.type === "TSImportEqualsDeclaration") reasons.add("CommonJS import");
if (node.type === "Identifier" && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) {
reasons.add("external input or dynamic code");
}
if ((node.type === "Identifier" && node.name === "getBuiltinModule")
|| (literal(node) && node.value === "getBuiltinModule")) reasons.add("dynamic builtin loader");
if ((node.type === "Identifier" && /^(importActual|unmock|doUnmock|doMock|arguments)$/.test(node.name))
|| (literal(node) && /^(importActual|unmock|doUnmock|doMock)$/.test(node.value))) reasons.add("mock bypass");
if (node.type === "Identifier" && mockBindings.has(node.name) && parent
&& parent.type !== "ImportSpecifier"
&& !(parent.type === "MemberExpression" && parent.object === node && !parent.computed)) {
reasons.add("escaped or dynamic mocking API");
}
if (node.type === "CallExpression") {
const name = source.slice(node.callee.start, node.callee.end);
if (/^(?:vi|vitest)\.(?:mock|doMock|importActual|importMock)$/.test(name)) {
importSource(node.arguments[0]);
}
if (/\brequire\b/.test(name)) {
reasons.add("CommonJS loader");
importSource(node.arguments[0]);
}
const viteGlob = name === "import.meta.glob" && (literal(node.arguments[0])
|| (node.arguments[0]?.type === "ArrayExpression" && node.arguments[0].elements.every(literal)));
// Vite expands literal globs into dependency edges. New files appear in
// that expansion; deleted/renamed and non-source inputs force full lanes.
if (!viteGlob && /\b(?:fetch|eval|Function|glob|globSync)\b/.test(name)) reasons.add("external input or dynamic code");
}
for (const value of Object.values(node)) {
if (Array.isArray(value)) value.forEach((child) => visit(child, node));
else if (value && typeof value === "object" && value.type) visit(value, node);
}
}
visit(ast.program);
return { imports: [...imports], reasons: [...reasons], mocks };
}
export async function selectProjectTests({ root = process.cwd(), project, changed, onlyOwnChanges = false }) {
root = realpathSync(root);
const started = performance.now();
const absoluteChanges = changed.map((file) => path.resolve(root, file));
const ctx = await createVitest({ root, watch: false, project: [project],
exclude: ["**/dist/**"], reporters: [], related: absoluteChanges });
try {
if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) throw new Error(`Unknown project: ${project}`);
const owner = ctx.projects[0];
const environment = owner.vite.environments.ssr;
const inventory = await ctx.globTestSpecifications();
if (!inventory.length) throw new Error(`Empty inventory: ${project}`);
const relative = (spec) => path.relative(root, spec.moduleId).split(path.sep).join("/");
if (onlyOwnChanges && !absoluteChanges.some((file) => file.startsWith(`${owner.config.root}${path.sep}`))) {
// Keep coarse dependency coverage, but avoid spending a graph traversal
// on an unchanged consumer project merely to retain most of its IO tests.
const files = inventory.map(relative).sort();
return { project, inventory: files, selected: files, related: [], alwaysRun: files, coveredChanges: [],
opaqueModules: {}, retainedReason: "Unchanged project retains complete lane coverage", seconds: (performance.now() - started) / 1000 };
}
const tracked = new Set(execFileSync("git", ["ls-files", "--cached", "-z"], {
cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
}).split("\0").filter(Boolean));
const audit = new Map();
const parsed = new Map();
const parse = (file) => {
if (!parsed.has(file)) parsed.set(file, sourceDependencies(file, readFileSync(file, "utf8")));
return parsed.get(file);
};
const resolve = async (specifier, importer) => {
try {
const resolved = await environment.pluginContainer.resolveId(specifier, importer);
if (!resolved || resolved.external || !path.isAbsolute(resolved.id) || resolved.id.includes("?")) return null;
return realpathSync(resolved.id);
} catch { return null; } // A conditional/require-only export is opaque.
};
// Setup dependencies are not test-import edges. A change anywhere in their
// local import closure invalidates the entire project, including helpers.
const setup = new Set();
async function setupClosure(file) {
if (setup.has(file)) return;
setup.add(file);
const source = parse(file);
const review = reviewedSetup.get(path.relative(root, file).split(path.sep).join("/"));
const reviewed = review?.hash === createHash("sha256").update(readFileSync(file)).digest("hex");
if (!reviewed && source.reasons.length) throw new Error(`Opaque setup dependency: ${file}`);
for (const specifier of source.imports) {
if (isBuiltin(specifier)) {
if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`);
continue;
}
if (reviewed && review.external.includes(specifier)) continue;
const dependency = await resolve(specifier, file);
if (dependency && inside(root, dependency)) await setupClosure(dependency);
else if (!dependency) throw new Error(`Unresolved setup dependency: ${specifier}`);
else if (!purePackages.has(packageName(specifier))) throw new Error(`Opaque setup package: ${specifier}`);
}
}
for (const file of [...(owner.config.setupFiles ?? []), ...(owner.config.globalSetup ?? []),
...(owner.config.snapshotSerializers ?? [])]) {
const resolved = path.isAbsolute(file) ? file : await resolve(file, owner.vite.config.configFile);
if (resolved && inside(root, resolved)) await setupClosure(resolved);
else if (!resolved) throw new Error(`Unresolved setup module: ${file}`);
}
for (const file of owner.vite.config.configFileDependencies ?? []) setup.add(realpathSync(file));
const setupChanged = absoluteChanges.some((file) => setup.has(file));
const raw = await ctx.getRelevantTestSpecifications();
const modules = environment.moduleGraph.idToModuleMap;
const resolvedDependencies = new Map();
const localImports = new Map();
const dependencyPath = (dep) => {
if (!resolvedDependencies.has(dep)) {
const file = dep.startsWith("/@fs/") ? dep.slice(process.platform === "win32" ? 5 : 4)
: path.join(owner.config.root, dep);
try { resolvedDependencies.set(dep, realpathSync(file)); }
catch { resolvedDependencies.set(dep, null); }
}
return resolvedDependencies.get(dep);
};
for (const [id, module] of modules) {
if (!inside(root, id) || !/\.[cm]?[jt]sx?$/.test(id)) continue;
const source = parse(id);
const reasons = [...source.reasons];
if (!tracked.has(path.relative(root, id).split(path.sep).join("/"))) reasons.push("generated or untracked source");
const local = [];
localImports.set(id, local);
// Use the same resolved runtime edges as Vitest, including literal dynamic
// imports. Resolving the original syntax again uses the wrong conditions
// for require-only exports and invents edges for erased type imports.
const transformed = module.transformResult;
if (!Array.isArray(transformed?.deps)) reasons.push("unknown transform shape");
for (const dependencyId of [...(transformed?.deps ?? []), ...(transformed?.dynamicDeps ?? [])]) {
if (isBuiltin(dependencyId)) {
if (!pureBuiltins.has(dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`);
continue;
}
const dependency = dependencyPath(dependencyId);
if (dependency && inside(root, dependency)) {
local.push(dependency);
// Vitest intentionally skips some edges (e.g. CJS, externalized
// workspaces). Never regard an untracked local edge as independent.
if (!modules.get(dependency)?.transformResult) reasons.push(`untracked dependency ${path.relative(root, dependency)}`);
} else {
const external = dependency?.split(`${path.sep}node_modules${path.sep}`).at(-1).split(path.sep).join("/");
if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) || !purePackages.has(packageName(external))) {
reasons.push(`opaque dependency ${external ? packageName(external) : dependencyId}`);
}
}
}
if (!module.transformResult) reasons.push("untransformed module");
if (reasons.length) audit.set(id, [...new Set(reasons)]);
}
const retained = [];
for (const spec of inventory) {
const mocked = new Set();
// Honor only unconditional, explicitly supplied mock factories without an
// importOriginal parameter. Automocks, spies, unmock/importActual, dynamic
// mocking and escaped vi bindings stay conservative. The related-file
// graph remains uncut: edits to a mocked module can still select its tests.
if (owner.config.isolate !== false) {
for (const name of parse(spec.moduleId).mocks) {
const target = await resolve(name, spec.moduleId);
if (target && inside(root, target)) mocked.add(target);
}
}
const queue = [spec.moduleId], visited = new Set();
let opaque = setupChanged || owner.config.isolate === false || !localImports.has(spec.moduleId);
while (queue.length && !opaque) {
const file = queue.pop();
if (visited.has(file)) continue;
visited.add(file);
if (audit.has(file)) { opaque = true; break; }
for (const dependency of localImports.get(file) ?? []) if (!mocked.has(dependency)) queue.push(dependency);
}
if (opaque) retained.push(spec);
}
const selected = [...new Set([...raw, ...retained].map(relative))].sort();
return {
project,
inventory: inventory.map(relative).sort(),
selected,
related: raw.map(relative).sort(),
alwaysRun: retained.map(relative).sort(),
coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index])),
opaqueModules: Object.fromEntries([...audit].map(([file, reasons]) => [path.relative(root, file), reasons])),
setupChanged,
seconds: (performance.now() - started) / 1000,
};
} finally {
await ctx.close();
}
}
@@ -0,0 +1,99 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import { classifyRecoveryEntry, cutoverStatus, recoveryKinds } from "./recovery-cutover-policy.mjs";
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
// Inspect every ancestor: lstat on only the final component would follow an
// intermediate symlink. No directory/file names or OS errors enter the report.
async function directoryState(directory, io) {
const parsed = path.parse(directory);
let current = parsed.root;
for (const part of directory.slice(parsed.root.length).split(path.sep).filter(Boolean)) {
current = path.join(current, part);
try {
const stat = await io.lstat(current);
if (!stat.isDirectory() || stat.isSymbolicLink()) return "unsafe";
} catch (error) {
return error?.code === "ENOENT" ? "absent" : "unreadable";
}
}
return "directory";
}
/** Read-only, bounded entry inventory. Run with producers stopped on the retained
* volume. This is not an atomic snapshot and does not establish DB settlement.
* io exists for deterministic fault tests; production uses the actual filesystem. */
export async function inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries = 10_000 }, io = fs) {
if (!path.isAbsolute(instanceRoot) || (orphanDirectory !== undefined && !path.isAbsolute(orphanDirectory))) {
throw new Error("Recovery inventory requires absolute directories");
}
if (!Number.isSafeInteger(maxEntries) || maxEntries < 1 || maxEntries > 100_000) {
throw new Error("Entry limit must be between 1 and 100000");
}
const root = path.resolve(instanceRoot);
const rootState = await directoryState(root, io);
const directories = [path.join(root, "accounting-receipts"), path.join(root, "quota-credential-recovery"),
orphanDirectory === undefined ? path.join(root, "data", "sandbox-orphan-cleanup") : path.resolve(orphanDirectory)];
let remaining = maxEntries;
const summaries = [];
for (const [index, directory] of directories.entries()) {
const summary = { kind: recoveryKinds[index], presence: "unknown", observedEntries: 0,
unverifiedFiles: 0, unknownEntries: 0, readFailures: 0, complete: true, status: "unknown" };
if (rootState !== "directory") {
summary.complete = false;
} else {
const state = await directoryState(directory, io);
if (state === "absent") summary.presence = "absent";
else if (state !== "directory") {
summary.unknownEntries++;
summary.complete = false;
if (state === "unreadable") summary.readFailures++;
} else {
summary.presence = "present";
async function scan(target, depth) {
let handle;
try {
handle = await io.opendir(target);
while (true) {
// Read one sentinel entry to distinguish exactly-at-limit from
// truncated scans. The sentinel is not included in observed counts.
const entry = await handle.read();
if (entry === null) break;
if (remaining === 0) { summary.complete = false; break; }
remaining--;
summary.observedEntries++;
const entryKind = entry.isFile() ? "file"
: entry.isDirectory() && uuid.test(entry.name) ? "company-directory" : "other";
const classification = classifyRecoveryEntry(summary.kind, depth, entryKind);
if (classification === "descend") {
const child = path.join(target, entry.name);
// Recheck the entry before traversing; never intentionally follow
// links. Operators must quiesce writers for the snapshot to hold.
if (await directoryState(child, io) !== "directory") {
summary.unknownEntries++;
summary.complete = false;
} else await scan(child, depth + 1);
} else summary[classification]++;
if (!summary.complete && remaining === 0) break;
}
} catch {
summary.readFailures++;
summary.complete = false;
} finally {
if (handle) {
try { await handle.close(); }
catch { summary.readFailures++; summary.complete = false; }
}
}
}
await scan(directory, 0);
}
}
summary.status = cutoverStatus(summary);
summaries.push(summary);
}
return { schema: "paperclip/recovery-cutover-inventory/v1", status:
summaries.some(s => s.status === "unknown") ? "unknown" : summaries.some(s => s.status === "blocked") ? "blocked" : "clear",
instanceStatus: rootState, contentInspection: "none", counts: "observed-lower-bounds", maxEntries, directories: summaries };
}
+14
View File
@@ -0,0 +1,14 @@
/** File contents belong to each recovery owner. Inventory never parses them:
* even an apparently valid file cannot establish that its obligation settled. */
export const recoveryKinds = ["accounting-receipts", "quota-credential-recovery", "sandbox-orphan-cleanup"];
export function classifyRecoveryEntry(kind, depth, entry) {
if (entry === "file") return "unverifiedFiles";
if (entry === "company-directory" && kind === "quota-credential-recovery" && depth === 0) return "descend";
return "unknownEntries";
}
export function cutoverStatus(summary) {
if (!summary.complete || summary.unknownEntries > 0 || summary.readFailures > 0) return "unknown";
return summary.unverifiedFiles > 0 ? "blocked" : "clear";
}
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env node
import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs";
const usage = "Usage: node scripts/recovery-cutover.mjs --instance-root /absolute/instance [--orphan-directory /absolute/override] [--max-entries 10000] [--check]";
const args = process.argv.slice(2);
try {
let instanceRoot;
let orphanDirectory = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
let maxEntries = 10_000;
let check = false;
const seen = new Set();
for (let i = 0; i < args.length; i++) {
const argument = args[i];
if (seen.has(argument)) throw new Error("duplicate");
seen.add(argument);
if (argument === "--check") check = true;
else if (["--instance-root", "--orphan-directory", "--max-entries"].includes(argument)) {
const value = args[++i];
if (value === undefined || value.startsWith("--")) throw new Error("missing");
if (argument === "--instance-root") instanceRoot = value;
else if (argument === "--orphan-directory") orphanDirectory = value;
else maxEntries = Number(value);
} else throw new Error("unknown");
}
if (!instanceRoot) throw new Error("missing");
const report = await inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries });
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
if (check && report.status !== "clear") process.exitCode = 1;
} catch {
// Never echo args, filesystem errors or paths: they may contain secrets.
process.stderr.write(`Recovery inventory could not run. ${usage}\n`);
process.exitCode = 2;
}
+163
View File
@@ -0,0 +1,163 @@
import { afterEach, test } from "node:test";
import assert from "node:assert/strict";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs";
import { classifyRecoveryEntry, cutoverStatus } from "./lib/recovery-cutover-policy.mjs";
const cli = fileURLToPath(new URL("./recovery-cutover.mjs", import.meta.url));
const roots = [];
const company = "00000000-0000-4000-8000-000000000001";
const grant = "00000000-0000-4000-8000-000000000002";
async function fixture() {
// Canonicalize macOS /var -> /private/var, since the check rejects links.
const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "cutover-")));
roots.push(root);
return root;
}
async function write(root, relative, content = "secret-token-must-never-appear") {
const file = path.join(root, relative);
await fs.mkdir(path.dirname(file), { recursive: true });
await fs.writeFile(file, content);
return file;
}
function run(root, args = [], env = {}) {
return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], {
encoding: "utf8", env: { ...process.env, SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: "", ...env },
});
}
// An unset override differs from an invalid empty override.
function cleanRun(root, args = []) {
const env = { ...process.env };
delete env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], { encoding: "utf8", env });
}
afterEach(async () => { await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); });
test("pure policy never clears files or unknown entries", () => {
assert.equal(classifyRecoveryEntry("accounting-receipts", 0, "file"), "unverifiedFiles");
assert.equal(classifyRecoveryEntry("quota-credential-recovery", 0, "company-directory"), "descend");
assert.equal(classifyRecoveryEntry("quota-credential-recovery", 1, "company-directory"), "unknownEntries");
assert.equal(cutoverStatus({ complete: false, unknownEntries: 0, readFailures: 0, unverifiedFiles: 0 }), "unknown");
});
test("CLI: absent and empty legacy layouts clear without migration", async () => {
const root = await fixture();
let result = cleanRun(root, ["--check"]);
assert.equal(result.status, 0);
assert.equal(JSON.parse(result.stdout).status, "clear");
for (const directory of ["accounting-receipts", `quota-credential-recovery/${company}`, "data/sandbox-orphan-cleanup"]) {
await fs.mkdir(path.join(root, directory), { recursive: true });
}
result = cleanRun(root, ["--check"]);
assert.equal(result.status, 0);
assert.equal(JSON.parse(result.stdout).status, "clear");
});
test("CLI: real pending legacy files block, preserve bytes and reveal no secrets or names", async () => {
const root = await fixture();
const receipt = JSON.stringify({ schema: "paperclip/accounting-receipt/v1", id: grant, companyId: company,
runId: company, sourceId: company, sequence: 1, receivedAt: "2026-10-30T00:00:00.000Z", adapterType: "codex_local",
receipt: { complete: true, usage: { inputTokens: 1, outputTokens: 1 } } });
const files = await Promise.all([
write(root, `accounting-receipts/${grant}.json`, receipt),
write(root, `quota-credential-recovery/${company}/${grant}.json`, '{"ciphertext":"secret-token-must-never-appear"}'),
write(root, "data/sandbox-orphan-cleanup/secret-filename.json", JSON.stringify({ companyId: company,
environmentId: company, executionWorkspaceId: null, issueId: null, heartbeatRunId: null, provider: "daytona",
providerLeaseId: "private-resource", metadata: {} })),
]);
const before = await Promise.all(files.map(file => fs.readFile(file)));
const result = cleanRun(root, ["--check"]);
assert.equal(result.status, 1);
const report = JSON.parse(result.stdout);
assert.equal(report.status, "blocked");
assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 1]);
assert.doesNotMatch(result.stdout + result.stderr, /secret-token|secret-filename|private-resource|ciphertext|00000000-/);
assert.deepEqual(await Promise.all(files.map(file => fs.readFile(file))), before);
assert.equal(cleanRun(root).status, 0, "report-only mode does not fail on obligations");
});
test("truncated, temporary, and unreadable files are unverified, never clear", async () => {
const root = await fixture();
await write(root, "accounting-receipts/truncated.json", '{"schema":');
const file = await write(root, `quota-credential-recovery/${company}/${grant}.json.probe`);
await fs.chmod(file, 0);
const report = await inventoryRecoveryFiles({ instanceRoot: root });
assert.equal(report.status, "blocked");
assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 0]);
});
test("unknown nesting and symlinks block without traversing outside the chosen roots", async () => {
const root = await fixture();
const outside = await fixture();
await write(outside, "do-not-read.json");
await fs.mkdir(path.join(root, "accounting-receipts"));
await fs.symlink(outside, path.join(root, "accounting-receipts", "link"));
await fs.mkdir(path.join(root, "quota-credential-recovery", company, "unexpected"), { recursive: true });
const report = await inventoryRecoveryFiles({ instanceRoot: root });
assert.equal(report.status, "unknown");
assert.deepEqual(report.directories.map(s => s.unknownEntries), [1, 1, 0]);
assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [0, 0, 0]);
await fs.symlink(outside, path.join(root, "data"));
assert.equal((await inventoryRecoveryFiles({ instanceRoot: root })).directories[2].status, "unknown");
await fs.symlink(outside, path.join(root, "instance-link"));
assert.equal((await inventoryRecoveryFiles({ instanceRoot: path.join(root, "instance-link") })).status, "unknown");
});
test("entry budget gives incomplete lower-bound counts and never clears omitted work", async () => {
const root = await fixture();
for (let i = 0; i < 4; i++) await write(root, `accounting-receipts/${i}.json`);
const report = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 2 });
assert.equal(report.status, "unknown");
assert.equal(report.directories[0].observedEntries, 2);
assert.equal(report.directories[0].complete, false);
assert.equal(report.directories[0].unverifiedFiles, 2);
const exact = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 4 });
assert.equal(exact.directories[0].complete, true);
});
test("CLI respects configured orphan override and explicit override precedence", async () => {
const root = await fixture();
const override = await fixture();
await write(override, "pending.json");
let result = run(root, ["--check"], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override });
assert.equal(result.status, 1);
assert.equal(JSON.parse(result.stdout).directories[2].unverifiedFiles, 1);
const emptyOverride = await fixture();
result = run(root, ["--check", "--orphan-directory", emptyOverride], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override });
assert.equal(result.status, 0);
assert.equal(JSON.parse(result.stdout).status, "clear");
});
test("fault boundary reports stat, directory read and close failures without error data", async () => {
const root = await fixture();
await fs.mkdir(path.join(root, "accounting-receipts"));
for (const operation of ["lstat", "opendir", "read", "close"]) {
const sensitiveError = Object.assign(new Error("secret-token-must-never-appear"), { code: "EACCES" });
const io = { ...fs };
if (operation === "lstat" || operation === "opendir") io[operation] = async () => { throw sensitiveError; };
else io.opendir = async () => ({
read: async () => { if (operation === "read") throw sensitiveError; return null; },
close: async () => { if (operation === "close") throw sensitiveError; },
});
const report = await inventoryRecoveryFiles({ instanceRoot: root }, io);
assert.equal(report.status, "unknown");
if (operation === "lstat") assert.equal(report.instanceStatus, "unreadable");
else assert.ok(report.directories[0].readFailures > 0);
assert.doesNotMatch(JSON.stringify(report), /secret-token|EACCES/);
}
});
test("CLI fails closed for missing instance, unsafe roots and bad arguments", async () => {
const root = await fixture();
assert.equal(cleanRun(path.join(root, "absent"), ["--check"]).status, 1);
for (const args of [["--max-entries", "0"], ["--max-entries", "100001"], ["--max-entries", "NaN"],
["--orphan-directory", "relative-secret"], ["--bad-secret"], ["--check", "--check"]]) {
const result = cleanRun(root, args);
assert.equal(result.status, 2);
assert.doesNotMatch(result.stdout + result.stderr, /relative-secret|bad-secret/);
}
});
+35 -4
View File
@@ -8,6 +8,7 @@ import { loadShardDurations, selectGeneralServerShard } from "./general-server-s
import { invocationEnv } from "./ci-profile/record.mjs";
import { assertSelectedTests, partitionTestLines } from "./test-line-shard.mjs";
import { loadSelection, selectedShard } from "./ci/test-selection-manifest.mjs";
const repoRoot = process.cwd();
const scriptsDir = path.dirname(fileURLToPath(import.meta.url));
@@ -91,6 +92,7 @@ const nativeRunnerSuites = [
"server/src/__tests__/dot-runner.test.ts",
// These process cases otherwise skip in server shards without Runner binaries.
"server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts",
"server/src/services/native-runtime/native-usage-delivery.integration.test.ts",
];
// In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable
// workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip
@@ -378,7 +380,7 @@ function selectSerializedSuites(routeTests, shardIndex, shardCount) {
return shardFiles.map((file) => byRepoPath.get(file));
}
async function runVitest(args, label, testShard = null) {
async function runVitest(args, label, testShard = null, exactFiles = false) {
console.log(`\n[test:run] ${label}`);
invocationIndex += 1;
const tempRootParent = process.platform === "win32" ? os.tmpdir() : "/tmp";
@@ -436,7 +438,9 @@ async function runVitest(args, label, testShard = null) {
args = [...args.filter((arg) => arg !== chatSuite), ...runFilters, "--allowOnly=false"];
}
await new Promise((resolve, reject) => {
const child = spawn("pnpm", ["exec", "vitest", "run", ...sourceOnlyVitestArgs, ...args], {
const command = exactFiles ? process.execPath : "pnpm";
const prefix = exactFiles ? [path.join(scriptsDir, "ci/run-selected-vitest.mjs")] : ["exec", "vitest", "run"];
const child = spawn(command, [...prefix, ...sourceOnlyVitestArgs, ...args], {
cwd: repoRoot,
env: invocationEnv("vitest", env),
stdio: "inherit",
@@ -615,6 +619,24 @@ const generalServerTestFiles = [
.sort((a, b) => a.localeCompare(b));
const options = parseCliOptions(process.argv.slice(2));
const selectionProjects = options.mode === serializedModeName || generalServerGroups.includes(options.group)
? ["@paperclipai/server"]
: options.group === generalWorkspacesAGroupName ? generalWorkspacesAProjects
: options.group === generalWorkspacesBGroupName ? generalWorkspacesBProjects
: options.group === generalWorkspacesCGroupName ? generalWorkspacesCProjects : [];
const selection = process.env.PAPERCLIP_CI_TEST_SELECTION
? selectedShard(loadSelection(), options, selectionProjects) : null;
if (selection?.["@paperclipai/server"]) {
const owned = new Set(options.mode === serializedModeName ? routeTests.map((test) => test.repoPath)
: generalServerTestFiles.filter((file) => !excludedServerSuites(options.group).includes(file)));
for (const file of selection["@paperclipai/server"]) {
if (!owned.has(file)) fail(`Selected test belongs to another server lane: ${file}`);
}
}
if (selection && options.dryRun) {
writeFileSync(1, JSON.stringify({ selection }));
process.exit(0);
}
if (options.dryRun) {
const serializedSuites =
options.mode === serializedModeName
@@ -670,7 +692,16 @@ if (options.dryRun) {
}
try {
if (options.mode === generalModeName || options.mode === allModeName) {
if (selection) {
for (const [project, files] of Object.entries(selection)) {
const serverArgs = project !== "@paperclipai/server" ? []
: options.mode === serializedModeName || process.env.PAPERCLIP_TEST_DATABASE_URL
? ["--pool=forks", "--isolate", ...serializedServerVitestArgs]
: ["--pool=forks", "--isolate", "--fileParallelism", "--maxWorkers=2"];
await runVitest(["--project", project, ...serverArgs, ...files],
`selected ${project}: ${files.length} files`, null, true);
}
} else if (options.mode === generalModeName || options.mode === allModeName) {
if (options.group) {
await runGeneralGroup(routeTests, options.group, options.shardIndex, options.shardCount);
} else {
@@ -678,7 +709,7 @@ try {
}
}
if (options.mode === serializedModeName || options.mode === allModeName) {
if (!selection && (options.mode === serializedModeName || options.mode === allModeName)) {
await runSerializedSuites(routeTests, options.shardIndex ?? 0, options.shardCount ?? 1);
}
} catch (error) {
+32
View File
@@ -0,0 +1,32 @@
import { createDb } from "@paperclipai/db";
import { migrateRunEvidence } from "../src/services/run-evidence-migration.js";
if (!process.env.DATABASE_URL)
throw new Error("Set DATABASE_URL to the intended instance; run with legacy writers drained");
const args = process.argv.slice(2);
const flags = new Set(["--apply", "--company", "--after-run", "--after-trace", "--limit"]);
for (let index = 0; index < args.length; index++) {
const flag = args[index]!;
if (!flags.has(flag)) throw new Error(`Unknown migration option: ${flag}`);
if (flag !== "--apply" && (!args[++index] || args[index]!.startsWith("--"))) {
throw new Error(`Missing value for ${flag}`);
}
}
const value = (flag: string) => {
const i = args.indexOf(flag);
return i < 0 ? undefined : args[i + 1];
};
const db = createDb(process.env.DATABASE_URL);
try {
const result = await migrateRunEvidence(db, {
apply: args.includes("--apply"),
companyId: value("--company"),
afterRunId: value("--after-run"),
afterTraceId: value("--after-trace"),
limit: Number(value("--limit") || 50),
});
console.log(JSON.stringify(result, null, 2));
if (result.items.some((item) => item.state === "blocked")) process.exitCode = 1;
} finally {
await db.$client.end();
}
@@ -0,0 +1,445 @@
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { spawn, type ChildProcess } from "node:child_process";
import { once } from "node:events";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { setTimeout as delay } from "node:timers/promises";
import { eq, sql } from "drizzle-orm";
import { createDb, heartbeatRuns } from "@paperclipai/db";
import {
S3Client,
CreateBucketCommand,
DeleteBucketCommand,
ListObjectsV2Command,
DeleteObjectsCommand,
} from "@aws-sdk/client-s3";
import {
DurablePrpControlPlane,
decodeDurablePrpCheckpoint,
} from "../src/vendor/paperclip-runner/index.js";
import { createS3StorageProvider } from "../src/storage/s3-provider.js";
import { createObjectRunLogStore } from "../src/services/object-run-log-store.js";
import { createObjectByteJournal } from "../src/services/object-byte-journal.js";
import { createHeartbeatRunOutput } from "../src/services/heartbeat/run-output.js";
import { createAgentIdentityRedactor } from "../src/services/agent-identity-redaction.js";
import { appendHeartbeatRunEvent } from "../src/services/heartbeat-run-events.js";
import { PaperclipControlPlanePort } from "../src/services/native-runtime/paperclip-control-plane-port.js";
import { formatNativeRunLogEvent } from "../src/services/native-runtime/native-run-log-export.js";
import { createNativeRunLogTestTables } from "../src/__tests__/helpers/native-run-log-database.js";
const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
const databaseUrl = process.env.PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL;
if (
!endpoint ||
!databaseUrl ||
[endpoint, databaseUrl].some((url) => !["127.0.0.1", "localhost"].includes(new URL(url).hostname))
)
throw new Error("Qualification requires explicit disposable loopback S3 and Postgres endpoints");
const childMode = process.argv[2] === "--controller";
type Config = {
schema: string;
bucket: string;
checkpoint: string;
companyId: string;
agentId: string;
runId: string;
sessionId: string;
runnerInstanceId: string;
environmentLeaseId: string;
turnId: string;
itemId: string;
};
const config: Config = childMode
? JSON.parse(await fs.readFile(process.argv[3]!, "utf8"))
: {
schema: `native_log_proof_${randomUUID().replaceAll("-", "")}`,
bucket: `pc-native-proof-${randomUUID()}`,
checkpoint: "",
companyId: randomUUID(),
agentId: randomUUID(),
runId: randomUUID(),
sessionId: randomUUID(),
runnerInstanceId: randomUUID(),
environmentLeaseId: randomUUID(),
turnId: randomUUID(),
itemId: randomUUID(),
};
const db = createDb(databaseUrl, { maxConnections: 1, idleTimeoutSeconds: 0 });
await db.execute(sql`set client_min_messages=warning`);
const provider = createS3StorageProvider({
endpoint,
bucket: config.bucket,
region: "us-east-1",
forcePathStyle: true,
});
const logStore = () =>
createObjectRunLogStore({
provider,
keyPrefix: "logs",
flushMs: 30_000,
assertWriterAuthority: async (input) => {
await input.assertWriterAuthority?.();
},
});
const identity = {
runnerInstanceId: config.runnerInstanceId,
environmentLeaseId: config.environmentLeaseId,
runId: config.runId,
normalizedSessionId: config.sessionId,
turnId: config.turnId,
itemId: config.itemId,
};
const expectedRunnerVersion = "0.3.0",
expectedRunnerDigest = `sha256:${"c".repeat(64)}`;
async function waitFor<T>(read: () => Promise<T>, done: (value: T) => boolean, timeout = 30_000) {
const until = Date.now() + timeout;
while (Date.now() < until) {
const value = await read();
if (done(value)) return value;
await delay(40);
}
throw new Error("Native log qualification deadline exceeded");
}
if (childMode) {
const home = process.argv[4]!,
portNumber = Number(process.argv[5] || 0);
await db.execute(sql.raw(`set search_path=${config.schema}`));
if (portNumber) {
const records = new Map<string, string>(
JSON.parse(await fs.readFile(config.checkpoint, "utf8")),
);
decodeDurablePrpCheckpoint(records).restoreSessionDirectory(home);
}
const store = logStore();
const [row] = await db
.select({
id: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
status: heartbeatRuns.status,
runtimeMode: heartbeatRuns.runtimeMode,
logStore: heartbeatRuns.logStore,
logRef: heartbeatRuns.logRef,
logBytes: heartbeatRuns.logBytes,
lastOutputSeq: heartbeatRuns.lastOutputSeq,
lastOutputAt: heartbeatRuns.lastOutputAt,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, config.runId));
type Run = typeof heartbeatRuns.$inferSelect;
const output = createHeartbeatRunOutput(db, {
run: row as Run,
runId: config.runId,
issueId: null,
runLogStore: store,
identityRedactor: createAgentIdentityRedactor(),
secretKeys: new Set(),
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
appendRunEvent: async (_run, event) => {
await appendHeartbeatRunEvent(db, {
companyId: config.companyId,
agentId: config.agentId,
runId: config.runId,
...event,
});
},
assertWriterAuthority: async () => {
const [current] = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, config.runId));
if (current?.status !== "running") throw new Error("fixture_writer_revoked");
},
});
const handlers = await output.begin(row as Run);
const record = async (event: Parameters<typeof formatNativeRunLogEvent>[0], cursor: number) => {
await handlers.onPersistedNativeLog!(cursor, formatNativeRunLogEvent(event));
};
const persistence = new PaperclipControlPlanePort(
db,
{
...config,
issueId: randomUUID(),
sourceInstanceId: config.runnerInstanceId,
controlPlaneSourceInstanceId: `control-${config.runId}`,
completionContractId: randomUUID(),
completionContractSha256: "qualification",
},
{ onCommittedEvent: record, onDuplicateEvent: record },
);
const core = new DurablePrpControlPlane({
stateDirectory: path.join(home, "control-plane"),
identity,
expectedRunnerVersion,
expectedRunnerDigest,
onCommittedEvent: async (event) => {
await persistence.appendEvent(event);
},
});
core.setCheckpointPersistence(async (checkpoint) => {
const temporary = `${config.checkpoint}.pending`;
await fs.writeFile(temporary, JSON.stringify([...checkpoint.records]));
await fs.rename(temporary, config.checkpoint);
});
await core.start(portNumber || undefined);
const ticket = core.issueBootstrapTicket();
await core.flushCheckpoint();
process.on("message", async (message: { id: string; type: string }) => {
try {
if (message.type === "checkpoint") await core.flushCheckpoint();
else if (message.type === "finish-log") {
const result = await store.finalize(output.handle!);
await output.flushOutputProgress({ force: true });
if (result.bytes === null) throw new Error("qualification_log_incomplete");
} else {
core.queueCommand(message.type, {}, message.id, true);
await waitFor(
async () => core.getCommand(message.id)?.status,
(status) => status === "completed",
);
}
process.send!({ id: message.id, ok: true });
} catch (error) {
process.send!({ id: message.id, error: String(error) });
}
});
process.send!({ ready: true, connectUrl: core.connectUrl, ticket });
} else {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "native-log-qualification-"));
config.checkpoint = path.join(root, "checkpoint.json");
const configFile = path.join(root, "config.json"),
receiptFile = path.join(root, "receipt.json"),
releaseFile = path.join(root, "release"),
scriptFile = path.join(root, "script.json");
const children: ChildProcess[] = [];
const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
let bucketCreated = false;
async function controller(home: string, portNumber = 0) {
await fs.mkdir(home, { mode: 0o700 });
const child = spawn(
process.execPath,
[
"--import",
path.resolve("server/node_modules/tsx/dist/loader.mjs"),
fileURLToPath(import.meta.url),
"--controller",
configFile,
home,
String(portNumber),
],
{
cwd: process.cwd(),
env: { ...process.env, PAPERCLIP_HOME: home },
stdio: ["ignore", "ignore", "pipe", "ipc"],
},
);
children.push(child);
let diagnostics = "";
child.stderr!.on("data", (bytes) => {
diagnostics = (diagnostics + String(bytes)).slice(-8000);
});
const ready = await new Promise<{ connectUrl: string; ticket: string }>((resolve, reject) => {
const timeout = setTimeout(
() => reject(new Error(`Controller startup timeout: ${diagnostics}`)),
45_000,
);
child.on("message", (message: { ready?: boolean; connectUrl: string; ticket: string }) => {
if (message.ready) {
clearTimeout(timeout);
resolve(message);
}
});
child.once("exit", () => {
clearTimeout(timeout);
reject(new Error(`Controller startup failed: ${diagnostics}`));
});
});
async function command(type: string) {
const id = randomUUID();
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
child.off("message", onMessage);
reject(new Error(`Command timeout: ${type}: ${diagnostics}`));
}, 35_000);
function onMessage(message: { id: string; error?: string }) {
if (message.id !== id) return;
clearTimeout(timer);
child.off("message", onMessage);
message.error ? reject(new Error(message.error)) : resolve();
}
child.on("message", onMessage);
child.send({ id, type });
});
}
return { child, ready, command };
}
try {
await db.execute(sql.raw(`create schema ${config.schema}`));
await db.execute(sql.raw(`set search_path=${config.schema}`));
await createNativeRunLogTestTables(db);
await db.execute(
sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${config.runId},${config.companyId},${config.agentId},'running')`,
);
await client.send(new CreateBucketCommand({ Bucket: config.bucket }));
bucketCreated = true;
await fs.writeFile(configFile, JSON.stringify(config));
await fs.writeFile(
scriptFile,
JSON.stringify({ steps: 64, waitAfter: 32, retentionLimit: 8, receiptFile, releaseFile }),
);
const homeA = path.join(root, "web-a"),
a = await controller(homeA);
const runner = spawn(
path.resolve("packages/paperclip-runner/runner/target/debug/examples/recovery-fixture"),
[
"--fixture-script",
scriptFile,
"--connect-url",
a.ready.connectUrl,
"--state-dir",
path.join(root, "runner"),
"--runner-id",
config.runnerInstanceId,
"--environment-lease-id",
config.environmentLeaseId,
"--run-id",
config.runId,
"--session-id",
config.sessionId,
"--turn-id",
config.turnId,
"--item-id",
config.itemId,
"--runner-version",
expectedRunnerVersion,
"--runner-digest",
expectedRunnerDigest,
"--max-outbox-bytes",
"65536",
"--p0-reserve-bytes",
"16384",
"--reconnect-grace-ms",
"60000",
],
{
env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: a.ready.ticket },
stdio: ["ignore", "ignore", "pipe"],
},
);
children.push(runner);
let runnerDiagnostics = "";
runner.stderr!.on("data", (bytes) => {
runnerDiagnostics = (runnerDiagnostics + String(bytes)).slice(-4000);
});
const receipt = async () =>
JSON.parse(await fs.readFile(receiptFile, "utf8").catch(() => "{}")) as {
pid: number;
acknowledgedEvents: number;
completedSteps: number;
retainedEvents: number;
peakRetainedEvents: number;
};
await a.command("turn.start");
await fs.writeFile(releaseFile, "go");
const acknowledged = await waitFor(
receipt,
(value) => value.acknowledgedEvents === 32 && value.retainedEvents === 0,
);
const journal = createObjectByteJournal({
provider,
prefix: `logs/segments-v1/${config.companyId}/${config.agentId}/${config.runId}`,
});
const before = (await journal.head())!.head;
assert(
before.sourceCursor < 32,
"Experiment must kill a controller with acknowledged output still pending in S3",
);
const controlStarted = performance.now();
await a.command("session.snapshot");
const controlMs = performance.now() - controlStarted;
await a.command("checkpoint");
const exitA = once(a.child, "exit");
a.child.kill("SIGKILL");
await exitA;
await fs.rm(homeA, { recursive: true });
const b = await controller(path.join(root, "web-b"), Number(new URL(a.ready.connectUrl).port));
await b.command("turn.steer");
const completed = await waitFor(
receipt,
(value) => value.acknowledgedEvents === 64 && value.retainedEvents === 0,
);
assert.equal(completed.pid, acknowledged.pid);
assert.equal(runner.exitCode, null, runnerDiagnostics);
const stopStarted = performance.now();
await b.command("turn.stop");
const stopMs = performance.now() - stopStarted;
await b.command("finish-log");
const store = logStore(),
content = (
await store.read({
store: "s3_segments",
logRef: `${config.companyId}/${config.agentId}/${config.runId}`,
})
).content;
const events = content
.trim()
.split("\n")
.map((line) => JSON.parse(JSON.parse(line).chunk).event);
assert.deepEqual(
events.map((event) => Number(event.payload.text.match(/^step=(\d+)/)[1])),
Array.from({ length: 64 }, (_, index) => index + 1),
);
assert.equal(new Set(events.map((event) => event.sourceEventId)).size, 64);
const [source] = await db.execute(
sql`select count(*)::int as count from heartbeat_run_events where run_id=${config.runId}`,
);
assert.equal(source!.count, 64);
const shutdown = once(runner, "exit");
await b.command("runner.shutdown");
assert.equal((await shutdown)[0], 0);
process.stdout.write(
`${JSON.stringify({
result: "passed",
source: "real Rust recovery fixture and PaperclipControlPlanePort",
acknowledgedBeforeKill: 32,
objectSourceCursorBeforeKill: before.sourceCursor,
controllerLoss: "SIGKILL",
oldDirectoryRemoved: true,
sameRunnerProcess: true,
recoveredRecords: events.length,
duplicates: 0,
existingControlRows: source!.count,
controlCommandMs: Math.round(controlMs),
stopMs: Math.round(stopMs),
peakRunnerRetained: completed.peakRetainedEvents,
})}\n`,
);
} finally {
for (const child of children)
if (child.exitCode === null && child.signalCode === null) {
const exit = once(child, "exit");
child.kill("SIGKILL");
await exit;
}
if (bucketCreated) {
for (;;) {
const page = await client.send(new ListObjectsV2Command({ Bucket: config.bucket }));
if (!page.Contents?.length) break;
await client.send(
new DeleteObjectsCommand({
Bucket: config.bucket,
Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) },
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: config.bucket }));
}
client.destroy();
await db.execute(sql.raw(`drop schema if exists ${config.schema} cascade`));
await db.$client.end();
await fs.rm(root, { recursive: true, force: true });
}
}
+213
View File
@@ -0,0 +1,213 @@
import { randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { once } from "node:events";
import { promises as fs } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import {
S3Client,
CreateBucketCommand,
DeleteBucketCommand,
ListObjectsV2Command,
DeleteObjectsCommand,
} from "@aws-sdk/client-s3";
import { createS3StorageProvider } from "../src/storage/s3-provider.js";
import { createObjectRunLogStore } from "../src/services/object-run-log-store.js";
const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
if (!endpoint || !["127.0.0.1", "localhost"].includes(new URL(endpoint).hostname)) {
throw new Error(
"Qualification requires PAPERCLIP_LOG_TEST_S3_ENDPOINT on a disposable loopback backend",
);
}
const childMode = process.argv[2] === "--child-writer";
const bucket = childMode ? process.argv[3]! : `paperclip-logs-test-${randomUUID()}`;
const raw = createS3StorageProvider({
endpoint,
bucket,
region: "us-east-1",
forcePathStyle: true,
});
const counts = { get: 0, put: 0, uploadedBytes: 0 };
const latencies: number[] = [];
const provider = {
...raw,
async getObject(...args: Parameters<typeof raw.getObject>) {
counts.get++;
return raw.getObject(...args);
},
async putObjectConditional(...args: Parameters<NonNullable<typeof raw.putObjectConditional>>) {
counts.put++;
counts.uploadedBytes += args[0].contentLength;
const start = performance.now();
try {
return await raw.putObjectConditional!(...args);
} finally {
latencies.push(performance.now() - start);
}
},
};
const input = { companyId: "company", agentId: "agent", runId: "crash" };
const options = {
provider,
keyPrefix: "qualification",
flushMs: 1000,
assertWriterAuthority: async () => undefined,
};
const output = (seq: number) => ({
stream: "stdout" as const,
seq,
chunk: `${seq}: 再開😀 ${"x".repeat(900)}`,
ts: "2026-10-30T00:00:00Z",
});
if (childMode) {
const store = createObjectRunLogStore(options),
handle = await store.begin(input);
const before = counts.put;
await Promise.all(Array.from({ length: 1000 }, (_, i) => store.append(handle, output(i + 1))));
process.stdout.write(
JSON.stringify({ receipt: true, batchPuts: counts.put - before, counts }) + "\n",
);
setTimeout(() => process.exit(2), 30_000);
} else {
const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
await client.send(new CreateBucketCommand({ Bucket: bucket }));
const directory = await fs.mkdtemp(path.join(tmpdir(), "run-log-crash-"));
let child: ReturnType<typeof spawn> | undefined;
try {
const started = performance.now();
child = spawn(
process.execPath,
[
"--import",
path.resolve("server/node_modules/tsx/dist/loader.mjs"),
import.meta.filename,
"--child-writer",
bucket,
],
{ cwd: directory, env: process.env, stdio: ["ignore", "pipe", "pipe"] },
);
const exited = once(child, "exit");
let childError = "";
child.stderr!.on("data", (bytes) => {
childError = (childError + bytes).slice(-8192);
});
const receipt = await new Promise<{ batchPuts: number; counts: typeof counts }>(
(resolve, reject) => {
let text = "";
const timeout = setTimeout(
() => reject(new Error(`Writer receipt timed out: ${childError}`)),
30_000,
);
child!.stdout!.on("data", (bytes) => {
text += bytes;
if (!text.includes("\n")) return;
clearTimeout(timeout);
try {
resolve(JSON.parse(text.split("\n")[0]!));
} catch (error) {
reject(error);
}
});
child!.on("error", (error) => {
clearTimeout(timeout);
reject(error);
});
void exited.then(() => {
clearTimeout(timeout);
if (!text) reject(new Error(`Writer exited before its receipt: ${childError}`));
});
},
);
child.kill("SIGKILL");
const [, signal] = await exited;
if (signal !== "SIGKILL") throw new Error("Writer did not die by SIGKILL");
await fs.rm(directory, { recursive: true, force: true });
const successorStore = createObjectRunLogStore(options);
const successor = await successorStore.begin(input);
let content = "",
offset = 0;
for (;;) {
const page = await successorStore.read(successor, { offset, limitBytes: 131072 });
content += page.content;
if (page.nextOffset === undefined) break;
offset = page.nextOffset;
}
const lines = content
.trim()
.split("\n")
.map((line) => JSON.parse(line));
if (
lines.length !== 1000 ||
!lines.every((line, i) => line.seq === i + 1 && line.chunk === output(i + 1).chunk)
)
throw new Error("Replacement bytes did not match");
await successorStore.append(successor, output(1001));
const summary = await successorStore.finalize(successor);
const steadyBefore = { ...counts },
steadyStarted = performance.now();
const steady = createObjectRunLogStore(options);
const handles = await Promise.all(
Array.from({ length: 8 }, (_, i) => steady.begin({ ...input, runId: `steady-${i}` })),
);
const pending: Promise<number>[] = [];
for (let tick = 0; tick < 30; tick++) {
for (const handle of handles) pending.push(steady.append(handle, output(tick + 1)));
await delay(100);
}
await Promise.all(pending);
await Promise.all(handles.map((handle) => steady.finalize(handle)));
const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
if (objects.IsTruncated) throw new Error("Qualification inventory exceeded one bounded page");
latencies.sort((a, b) => a - b);
console.log(
JSON.stringify(
{
backend: "loopback S3-compatible test service",
flushMs: options.flushMs,
burst: {
records: lines.length,
batchPuts: receipt.batchPuts,
writerCounts: receipt.counts,
},
steady: {
streams: handles.length,
records: pending.length,
elapsedMs: Math.round(performance.now() - steadyStarted),
putRequests: counts.put - steadyBefore.put,
getRequests: counts.get - steadyBefore.get,
uploadedBytes: counts.uploadedBytes - steadyBefore.uploadedBytes,
},
readerCounts: counts,
committedCrashBytes: summary.bytes,
objects: objects.KeyCount,
storedBytes: objects.Contents?.reduce((sum, object) => sum + (object.Size ?? 0), 0),
putP95Ms: Math.round(latencies[Math.floor(latencies.length * 0.95)] ?? 0),
elapsedMs: Math.round(performance.now() - started),
abruptProcessLoss: signal,
oldDirectoryRemoved: true,
replacement: "passed",
},
null,
2,
),
);
} finally {
child?.kill("SIGKILL");
await fs.rm(directory, { recursive: true, force: true });
for (;;) {
const page = await client.send(new ListObjectsV2Command({ Bucket: bucket, MaxKeys: 1000 }));
if (!page.Contents?.length) break;
await client.send(
new DeleteObjectsCommand({
Bucket: bucket,
Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) },
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
client.destroy();
}
}
@@ -333,6 +333,20 @@ describe("activity routes", () => {
expect(res.body).toEqual([{ runId: "run-1", adapterType: "codex_local" }]);
});
it("validates bounded run IDs before using the authorized issue projection", async () => {
const runId = "10000000-0000-4000-8000-000000000001";
mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" });
mockActivityService.runsForIssue.mockResolvedValue([{ runId }]);
const app = await createApp();
const response = await requestApp(app, base => request(base).get(`/api/issues/PAP-1/runs?runIds=${runId}`));
expect(response.status).toBe(200);
expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1", "user-1", undefined, [runId]);
for (const ids of ["", "not-a-uuid", Array.from({ length: 101 }, () => runId).join(",")]) {
const denied = await requestApp(app, base => request(base).get("/api/issues/PAP-1/runs").query({ runIds: ids }));
expect(denied.status).toBe(400);
}
});
it("requires company access before creating activity events", async () => {
const app = await createApp();
const res = await requestApp(app, (baseUrl) => request(baseUrl)
@@ -300,7 +300,7 @@ describe("adapter management route authorization", () => {
mocks.reloadExternalAdapter.mockImplementation(async (type: string) =>
createAdapter(type),
);
}, 20_000);
}, 60_000);
afterEach(() => {
unregisterServerAdapter(EXTERNAL_ADAPTER_TYPE);
@@ -373,35 +373,31 @@ describe("adapter management route authorization", () => {
},
);
describe("cloud-managed adapter code install floor", () => {
beforeEach(() => {
process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-server-token";
describe("cloud-managed adapter code floor", () => {
afterEach(() => vi.unstubAllEnvs());
it.each(
["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"].flatMap((signal) =>
["install", "reinstall", "reload", "override", "delete"].map((routeName) => [signal, routeName]),
),
)("blocks %s adapter %s before package or registration access", async (signal, routeName) => {
seedInstalledExternalAdapter();
vi.clearAllMocks();
vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-server-token");
const app = createApp(instanceAdmin);
const res = await sendMutatingRequest(app, routeName);
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(403);
expect(res.body.details).toMatchObject({ code: "adapter_install_platform_managed" });
expect(mocks.execFile).not.toHaveBeenCalled();
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
expect(mocks.reloadExternalAdapter).not.toHaveBeenCalled();
expect(mocks.getAdapterPluginByType).not.toHaveBeenCalled();
expect(mocks.addAdapterPlugin).not.toHaveBeenCalled();
expect(mocks.removeAdapterPlugin).not.toHaveBeenCalled();
expect(mocks.externalRecords.get(EXTERNAL_ADAPTER_TYPE)).toEqual(installedRecord());
});
afterEach(() => {
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
});
it.each(["install", "reinstall"] as const)(
"floors adapter %s off for instance admins on cloud-managed instances",
async (routeName) => {
resetInstalledExternalAdapterState();
if (routeName !== "install") {
seedInstalledExternalAdapter();
}
const app = createApp(instanceAdmin);
const res = await sendMutatingRequest(app, routeName);
expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(
403,
);
expect(res.body.details).toMatchObject({
code: "adapter_install_platform_managed",
});
expect(mocks.execFile).not.toHaveBeenCalled();
expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
},
);
});
describe("operator-hidden adapter management floor", () => {
+7 -2
View File
@@ -10,6 +10,7 @@ import { eq } from "drizzle-orm";
import { agentIdentityKeys, agents, companies, createDb, closeRegisteredClients, ensurePostgresDatabase, runDatabaseBackup, runDatabaseRestore, type Db } from "@paperclipai/db";
import { resolveWorktreeSeedPlan } from "../../../cli/src/commands/worktree-lib.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { agentIdentityService, supportsManagedAgentIdentity } from "../services/agent-identity.js";
import { agentService } from "../services/agents.js";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
@@ -21,9 +22,10 @@ import { actorMiddleware } from "../middleware/auth.js";
import { errorHandler } from "../middleware/error-handler.js";
import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
const externalIdentityDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_IDENTITY_TEST_DATABASE_URL");
describe("agent cryptographic identity", () => {
let db: Db;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let database: Pick<Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, "connectionString" | "cleanup">;
let directory: string;
let companyId: string;
@@ -32,7 +34,9 @@ describe("agent cryptographic identity", () => {
vi.stubEnv("PAPERCLIP_HOME", directory);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key"));
database = await startEmbeddedPostgresTestDatabase("paperclip-identity-");
database = externalIdentityDatabase ? { connectionString: externalIdentityDatabase,
cleanup: () => closeRegisteredClients(externalIdentityDatabase) }
: await startEmbeddedPostgresTestDatabase("paperclip-identity-");
db = createDb(database.connectionString);
companyId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Identity test", issuePrefix: "KEY", defaultResponsibleUserId: "identity-test-owner" });
@@ -51,6 +55,7 @@ describe("agent cryptographic identity", () => {
it("migration and public reads leave existing agents unprovisioned", async () => {
const agent = await oldAgent();
expect(await agentIdentityService(db).getPublicIdentity(companyId, agent.id)).toBeNull();
expect((await agentIdentityService(db).createExistingLogRedactor(companyId, agent.id)).values).toEqual([]);
expect(await db.select().from(agentIdentityKeys).where(eq(agentIdentityKeys.agentId, agent.id))).toHaveLength(0);
for (const type of ["http", "openclaw_gateway", "hermes_gateway"]) expect(supportsManagedAgentIdentity(type)).toBe(false);
for (const provider of ["claude_managed", "aws_agentcore"]) expect(supportsManagedAgentIdentity("paperclip_runner", { provider })).toBe(false);
@@ -103,6 +103,11 @@ vi.mock("../services/instance-settings.js", () => ({
instanceSettingsService: () => mockInstanceSettingsService,
}));
vi.mock("../services/adapter-settings.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../services/adapter-settings.js")>()),
adapterSettingsService: () => ({ getDisabledTypes: async () => [] }),
}));
vi.mock("../services/dot-runner-broker.js", () => ({
dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }),
}));
+154 -34
View File
@@ -9,7 +9,8 @@ vi.mock("../services/subscription-refresh.js", () => ({ refreshSubscriptionConne
import * as localCredentials from "../services/local-ai-credentials.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { createHash, randomUUID } from "node:crypto";
import { spawnSync } from "node:child_process";
import { spawn, spawnSync } from "node:child_process";
import { createRequire } from "node:module";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import { promises as fs } from "node:fs";
import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:fs/promises";
@@ -38,6 +39,8 @@ import { secretService } from "../services/secrets.js";
import { resolveExecutionRunAdapterConfig } from "../services/heartbeat.js";
import { aiConnectionBindingSchema, connectionPurposeTransportSchema, isAiConnectionCompatible } from "@paperclipai/shared";
import { openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution";
import { createServer } from "node:http";
import { once } from "node:events";
import express from "express";
import request from "supertest";
import { aiConnectionRoutes, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "../routes/ai-connections.js";
@@ -385,8 +388,8 @@ describe("managed AI connections", () => {
await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable");
});
it("persists first-time recovery directories before consuming a single-use token", async () => {
const owner = "quota-durable-path";
it("commits one exchanged token without writable credential recovery storage", async () => {
const owner = "quota-database-first";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { account_id: owner, access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
@@ -394,32 +397,32 @@ describe("managed AI connections", () => {
const file = quotaCredentialRecoveryPath(companyId, row.grant.id);
const recoveryRoot = path.dirname(path.dirname(file));
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
const synced: string[] = [];
let rejectRoot = true;
const realOpen = fs.open.bind(fs);
const writes: string[] = [];
const rejectRecoveryWrite = (target: unknown) => {
if (String(target).startsWith(recoveryRoot)) {
writes.push(String(target));
throw Object.assign(new Error("Fixture recovery storage is read-only"), { code: "EROFS" });
}
};
const realOpen = fs.open.bind(fs), realMkdir = fs.mkdir.bind(fs);
const open = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const handle = await realOpen(...args);
const sync = handle.sync.bind(handle);
vi.spyOn(handle, "sync").mockImplementation(async () => {
if (rejectRoot && args[0] === recoveryRoot) throw new Error("Injected recovery directory flush failure");
await sync(); synced.push(String(args[0]));
});
return handle;
rejectRecoveryWrite(args[0]);
return realOpen(...args);
});
const request = vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
expect(synced).toContain(recoveryRoot);
expect(synced).toContain(path.dirname(recoveryRoot));
return Response.json({ access_token: "fresh-durable", refresh_token: "replacement-durable" });
const mkdir = vi.spyOn(fs, "mkdir").mockImplementation(async (...args: Parameters<typeof fs.mkdir>) => {
rejectRecoveryWrite(args[0]);
return realMkdir(...args);
});
const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "fresh-direct", refresh_token: "replacement-direct" }));
try {
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow("Injected recovery directory flush failure");
expect(request).not.toHaveBeenCalled();
expect(await service.credential(row)).toBe(original);
rejectRoot = false;
await service.refreshQuotaCredential(row, original, new AbortController().signal);
const refreshed = await service.refreshQuotaCredential(row, original, new AbortController().signal);
expect(JSON.parse(refreshed)).toMatchObject({ tokens: { access_token: "fresh-direct", refresh_token: "replacement-direct" } });
expect(await service.credential(row)).toBe(refreshed);
expect(request).toHaveBeenCalledTimes(1);
expect(JSON.parse(await service.credential(row))).toMatchObject({ tokens: { access_token: "fresh-durable", refresh_token: "replacement-durable" } });
} finally { open.mockRestore(); request.mockRestore(); }
expect(writes).toEqual([]);
await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
} finally { open.mockRestore(); mkdir.mockRestore(); request.mockRestore(); }
});
it.each(["timeout", "unexpected"] as const)("classifies a manual probe's %s lock failure before starting the provider", async (kind) => {
@@ -703,7 +706,7 @@ describe("managed AI connections", () => {
} finally { request.mockRestore(); }
});
it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("recovers exchanged OAuth tokens after %s persistence failure without another exchange", async (failure) => {
it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("fails truthfully after %s persistence failure and permits sign-in recovery without a journal", async (failure) => {
const owner = `quota-rollback-${failure}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: `expired-${failure}`, refresh_token: "single-use", account_id: owner } });
@@ -723,17 +726,25 @@ describe("managed AI connections", () => {
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
expect(request).toHaveBeenCalledTimes(1);
expect(await service.credential(row)).toBe(original);
const encrypted = await readFile(file, "utf8");
expect(encrypted).not.toContain(`fresh-${failure}`);
expect(encrypted).not.toContain(`replacement-${failure}`);
await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
await db.execute(sql.raw(`drop trigger quota_save_failure on ${table}; drop function quota_save_failure();`));
triggerDropped = true;
// A new service instance recovers from disk, without an in-memory cache.
const recovered = aiConnectionService(db);
await recovered.refreshQuotaCredential(row, original, new AbortController().signal);
expect(JSON.parse(await recovered.credential(row))).toMatchObject({ tokens: { access_token: `fresh-${failure}`, refresh_token: `replacement-${failure}` } });
// A new receiver has no replacement to recover. A failed use routes to the
// existing sign-in flow, rather than claiming the failed save succeeded.
const receiver = aiConnectionService(db);
const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
} });
expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
const signedIn = JSON.stringify({ tokens: { access_token: "signed-in", refresh_token: "new-login" } });
await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription",
ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
const [current] = await receiver.quotaAccounts(companyId, owner);
expect(current.grant.status).toBe("active");
expect(await receiver.runtimeCredential(current)).toBe(signedIn);
expect(request).toHaveBeenCalledTimes(1);
await expect(access(file)).rejects.toThrow();
expect((await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, ref.secretId))).map(version => version.version).sort()).toEqual([1, 2]);
} finally {
request.mockRestore();
@@ -819,6 +830,112 @@ describe("managed AI connections", () => {
} finally { request.mockRestore(); await oldRun.cleanup(); await nextRun?.cleanup(); await journal.clear(); }
});
it("requires sign-in after the receiver process dies with an exchanged but unsaved token", async () => {
const owner = "quota-receiver-process-loss";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: "expired-process", refresh_token: "single-use-process" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
const [row] = await service.quotaAccounts(companyId, owner);
const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
const held = deferredSignal(), release = deferredSignal();
const lockKey = 7192031;
const holder = db.transaction(async tx => {
await tx.execute(sql`select pg_advisory_xact_lock(${lockKey})`);
held.resolve(); await release.promise;
});
await held.promise;
await db.execute(sql.raw(`create function hold_receiver_save() returns trigger language plpgsql as $$ begin
if new.secret_id = '${ref.secretId}'::uuid then perform pg_advisory_xact_lock(${lockKey}); end if; return new; end $$;
create trigger hold_receiver_save before insert on company_secret_versions for each row execute function hold_receiver_save();`));
let exchanges = 0;
const provider = createServer((req, res) => {
exchanges++; req.resume();
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ access_token: "unsaved-process", refresh_token: "replacement-process" }));
});
provider.listen(0, "127.0.0.1"); await once(provider, "listening");
const address = provider.address() as { port: number };
const child = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), new URL("./helpers/quota-credential-process.ts", import.meta.url).pathname], {
env: { ...process.env, CREDENTIAL_TEST_DATABASE_URL: database.connectionString,
CREDENTIAL_TEST_PROVIDER_URL: `http://127.0.0.1:${address.port}`, CREDENTIAL_TEST_COMPANY_ID: companyId, CREDENTIAL_TEST_OWNER: owner },
stdio: "ignore",
});
const exited = once(child, "exit");
try {
let blocked = false;
const deadline = Date.now() + 25_000;
while (!blocked && Date.now() < deadline && child.exitCode === null) {
const rows = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and wait_event_type = 'Lock' and wait_event = 'advisory'`);
blocked = exchanges === 1 && rows.length > 0;
if (!blocked) await new Promise(resolve => setTimeout(resolve, 25));
}
expect(blocked).toBe(true);
child.kill("SIGKILL"); await exited;
} finally {
if (child.exitCode === null && child.signalCode === null) { child.kill("SIGKILL"); await exited; }
release.resolve(); await holder;
await db.execute(sql`drop trigger hold_receiver_save on company_secret_versions; drop function hold_receiver_save()`);
await new Promise<void>(resolve => provider.close(() => resolve()));
}
expect(exchanges).toBe(1);
expect(await service.credential(row)).toBe(original);
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
const receiver = aiConnectionService(db);
const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
} });
expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
const signedIn = JSON.stringify({ tokens: { access_token: "process-sign-in", refresh_token: "process-new-login" } });
await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
const [current] = await receiver.quotaAccounts(companyId, owner);
expect(await receiver.runtimeCredential(current)).toBe(signedIn);
}, 40_000);
it("does not exchange a token when a required database transaction fails", async () => {
const owner = "quota-db-prerequisite";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
const [row] = await service.quotaAccounts(companyId, owner);
const request = vi.spyOn(globalThis, "fetch");
const unavailableUrl = new URL(database.connectionString);
unavailableUrl.port = "1";
const unavailable = createDb(unavailableUrl.toString());
try {
await expect(aiConnectionService(unavailable).refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
expect(request).not.toHaveBeenCalled();
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
} finally { await unavailable.$client.end(); request.mockRestore(); }
});
it.each(["reconnect", "same_value_new_epoch", "revoked"] as const)("lets %s win after a returned token's save rolls back", async change => {
const owner = `quota-delayed-save-${change}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
const [row] = await service.quotaAccounts(companyId, owner);
const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
const transaction = db.transaction.bind(db);
const interrupted = vi.spyOn(db, "transaction").mockImplementationOnce(async (work, config) => {
await transaction(async tx => { await work(tx); throw new Error("fixture rollback"); }, config).catch(() => {});
if (change === "revoked") await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, row.grant.id));
else await secretService(db).rotate(ref.secretId, { value: change === "reconnect" ? "new-login" : original });
throw new Error("fixture lost response");
});
const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "late", refresh_token: "late-refresh" }));
try {
const refresh = service.refreshQuotaCredential(row, original, new AbortController().signal);
if (change === "revoked") await expect(refresh).rejects.toThrow("credentials_unavailable");
else expect(await refresh).toBe(change === "reconnect" ? "new-login" : original);
expect(await service.credential(row)).toBe(change === "reconnect" ? "new-login" : original);
expect(request).toHaveBeenCalledTimes(1);
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
} finally { interrupted.mockRestore(); request.mockRestore(); }
});
it.each(["rollback", "lost_commit_reply"] as const)("automatically saves one exchanged token across %s", async (failure) => {
const owner = `quota-once-${failure}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
@@ -863,7 +980,10 @@ describe("managed AI connections", () => {
triggerInstalled = true;
await expect(service.refreshQuotaCredential(row, current, new AbortController().signal)).rejects.toThrow();
expect(await service.credential(row)).toBe(current);
expect(JSON.parse((await journal.read())!.value).tokens.access_token).toBe("C");
expect(await journal.read()).toBeNull();
// Only observed records from the earlier release use the legacy reader.
await journal.write({ companyId, grantId: row.grant.id, connectionId: row.connection.id, secretId: ref.secretId,
baseHash: quotaCredentialHash(current), value: JSON.stringify({ tokens: { access_token: "C", refresh_token: "refresh-C" } }) });
await db.execute(sql`drop trigger delayed_quota_commit_failure on company_secret_versions; drop function delayed_quota_commit_failure()`);
triggerInstalled = false;
const recovered = await service.refreshQuotaCredential(row, original, new AbortController().signal);
@@ -1406,6 +1406,22 @@ describeEmbeddedPostgres("attention service", () => {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
try {
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
expect(summary.totalCount).toBe(feed.totalCount);
const index = await attentionService(db).taskIndex(companyId, { userId: "board-user" });
expect(index.taskIds).toEqual([...new Set(feed.items.flatMap(item => item.relatedIssue ? [item.relatedIssue.id] : item.subject.kind === "issue" ? [item.subject.id] : []))]);
expect(index.totalCount).toBe(feed.totalCount);
} finally { summaryCapture.flush(); }
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
expect(cardReads).toEqual([]);
const planItem = detailsByKind.get("plan_approval");
expect(planItem?.subject.title).toBe("Plan approval - Approve launch plan");
@@ -1527,6 +1543,9 @@ describeEmbeddedPostgres("attention service", () => {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
const interactions = feed.items.filter(item => item.sourceKind === "issue_thread_interaction");
expect(interactions.map(item => item.subject.id).sort()).toEqual([ids[0], ids[1], ids[3]].sort());
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
// Resolving one output removes only that decision, leaving the other output
// and the ordinary confirmation independently actionable.
await db.update(issueThreadInteractions).set({ status: "accepted", resolvedAt: new Date() }).where(eq(issueThreadInteractions.id, ids[0]));
@@ -1858,6 +1877,7 @@ describeEmbeddedPostgres("attention service", () => {
await expect(attentionService(db).list(companyId, { userId: "board-user" }))
.resolves.toMatchObject({ totalCount: 1 }); // agent_error_alert from seed
await expect(attentionService(db).summary(companyId, { userId: "board-user" })).resolves.toMatchObject({ totalCount: 1 });
const hiddenFeed = await attentionService(db).list(companyId, { userId: "board-user", includeDismissed: true });
expect(hiddenFeed.items.find((item) => item.dedupKey === `approval:${approvalId}`)?.dismissal)
.toMatchObject({ kind: "snooze", isActive: true, snoozedUntil: "2099-01-02T00:00:00.000Z" });
@@ -1871,6 +1891,9 @@ describeEmbeddedPostgres("attention service", () => {
const visibleApproval = visibleFeed.items.find((item) => item.dedupKey === `approval:${approvalId}`);
expect(visibleApproval?.dismissal).toMatchObject({ kind: "snooze", isActive: false });
expect(visibleApproval).toBeTruthy();
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
expect(summary.totalCount).toBe(visibleFeed.totalCount);
});
it("enriches, filters, paginates, snoozes, and ranks the decide-now feed", async () => {
@@ -2113,6 +2136,13 @@ describeEmbeddedPostgres("attention service", () => {
expect(approvalRows.every((approval) => internalSnapshotIds.has(approval.id))).toBe(true);
expect(internalSnapshot.items.length).toBeGreaterThan(100);
expect(internalSnapshot.nextCursor).toBeNull();
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
expect(summary).toEqual({
companyId, generatedAt: expect.any(String), totalCount: internalSnapshot.totalCount,
deskBadgeCount: internalSnapshot.deskBadgeCount, countsBySourceKind: internalSnapshot.countsBySourceKind,
});
expect(JSON.stringify(summary).length).toBeLessThan(JSON.stringify(internalSnapshot).length / 10);
await expect(attentionService(db).list(companyId, { userId: "board-user", all: true }))
.rejects.toThrow("all requires a queue filter");
@@ -2166,6 +2196,10 @@ describeEmbeddedPostgres("attention service", () => {
allowUnscopedAll: true,
});
expect(complete.items.filter((item) => item.sourceKind === "decision")).toHaveLength(2);
const summary = await svc.summary(companyId, { userId: "board-user" });
expect(summary.totalCount).toBe(complete.totalCount);
expect(summary.countsBySourceKind).toEqual(complete.countsBySourceKind);
expect(complete.nextCursor).toBeNull();
});
@@ -2281,6 +2315,18 @@ describeEmbeddedPostgres("attention service", () => {
.get(`/api/companies/${companyId}/attention?includeDismissed=true&all=true`)
.expect(200);
expect(completeFeed.body.nextCursor).toBeNull();
const summary = await request(app(board)).get(`/api/companies/${companyId}/attention/summary`).expect(200);
expect(summary.body).not.toHaveProperty("items");
expect(summary.body.totalCount).toBe(completeFeed.body.totalCount);
const taskIndex = await request(app(board)).get(`/api/companies/${companyId}/attention/tasks`).expect(200);
expect(taskIndex.body).not.toHaveProperty("items");
expect(taskIndex.body.taskIds).toEqual([]);
await request(app(agent)).get(`/api/companies/${companyId}/attention/tasks`).expect(403);
await request(app({ ...board, source: "session", companyIds: [] })).get(`/api/companies/${companyId}/attention/tasks`).expect(403);
await request(app(agent)).get(`/api/companies/${companyId}/attention/summary`).expect(403);
await request(app({ ...board, source: "session", companyIds: [] })).get(`/api/companies/${companyId}/attention/summary`).expect(403);
await request(app(board))
.get(`/api/companies/${companyId}/attention?activitySince=yesterday`)
.expect(400, { error: "activitySince must be an ISO timestamp" });
@@ -2317,6 +2363,10 @@ describeEmbeddedPostgres("attention service", () => {
expect(feed.items.map((item) => item.subject.title)).not.toContain("Confidential review");
expect(JSON.stringify(feed)).not.toContain(privateIssueId);
const summary = await attentionService(db).summary(companyId, { userId: actor.userId, actor });
expect(summary.totalCount).toBe(feed.totalCount);
expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
});
it("computes the aging shelf uniformly across approval, interaction, and review sources", async () => {
@@ -0,0 +1,86 @@
import { rmSync, writeFileSync, symlinkSync, mkdirSync, readFileSync } from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
import { readDistributionPluginCatalog } from "../services/distribution-plugin-catalog.js";
import boxd from "../../../packages/plugins/sandbox-providers/boxd/src/manifest.js";
import createos from "../../../packages/plugins/sandbox-providers/createos/src/manifest.js";
import cloudflare from "../../../packages/plugins/sandbox-providers/cloudflare/src/manifest.js";
import daytona from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
import e2b from "../../../packages/plugins/sandbox-providers/e2b/src/manifest.js";
import exe_dev from "../../../packages/plugins/sandbox-providers/exe-dev/src/manifest.js";
import kubernetes from "../../../packages/plugins/sandbox-providers/kubernetes/src/manifest.js";
import modal from "../../../packages/plugins/sandbox-providers/modal/src/manifest.js";
import novita from "../../../packages/plugins/sandbox-providers/novita/src/manifest.js";
const manifests = { boxd, createos, cloudflare, daytona, e2b, "exe-dev": exe_dev, kubernetes, modal, novita };
const roots: string[] = [];
function fixture() { const result = bundledImageFixture(); roots.push(result.root); return result; }
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
describe("bundled image package identity", () => {
it("resolves independent image roots from the same registration without any previous path", () => {
const first = fixture();
const second = fixture();
const registration = { ...first.registration, packagePath: "/lost/old/home/plugins/fixture" };
expect(first.resolver.resolve(registration)).toBe(first.packageRoot);
expect(second.resolver.resolve(registration)).toBe(second.packageRoot);
expect(first.resolver.inspect(registration, first.manifest)).toEqual(second.resolver.inspect(registration, second.manifest));
expect(registration.packagePath).toBe("/lost/old/home/plugins/fixture");
});
it("requires verified manifest evidence and blocks same-version manifest drift", () => {
const { resolver, registration, manifest } = fixture();
expect(resolver.inspect(registration)).toMatchObject({ compatible: false, code: "missing_manifest_evidence" });
expect(resolver.inspect(registration, { ...manifest, description: "changed" })).toMatchObject({ compatible: false, code: "incompatible_bundled_package", reason: expect.stringContaining("Approved plugin manifest") });
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true, identity: { kind: "bundled_image", catalogKey: "fixture", pluginKey: manifest.id, version: "1.0.0" } });
});
it("rejects mismatched package names, versions, identity and escaping entrypoints", () => {
const { resolver, registration } = fixture();
for (const update of [
{ packageName: "@other/plugin" }, { version: "2.0.0" }, { pluginKey: "other.plugin" },
{ manifestJson: { ...registration.manifestJson, version: "2.0.0" } },
{ manifestJson: { ...registration.manifestJson, entrypoints: { worker: "../worker.js" } } },
]) expect(() => resolver.resolve({ ...registration, ...update })).toThrow();
});
it("fails explicitly for missing or deselected bundles even when a stale npm package exists", () => {
const { root, packageRoot, resolver, registration } = fixture();
const fallback = path.join(root, "home", "node_modules", registration.packageName);
mkdirSync(fallback, { recursive: true });
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('unsafe fallback');");
rmSync(packageRoot, { recursive: true });
expect(() => resolver.resolve(registration)).toThrow(/ENOENT/);
expect(() => createBundledPluginPackageResolver([]).resolve(registration)).toThrow(/absent or not selected/);
});
it("rejects tampering before manifest import and symlink escapes before worker resolution", () => {
const { root, packageRoot, registration, resolver } = fixture();
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), "throw new Error('must never import');");
expect(() => readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG)).toThrow(/digest mismatch/);
rmSync(path.join(packageRoot, "dist", "worker.js"));
symlinkSync(path.join(root, "distribution", "catalog.json"), path.join(packageRoot, "dist", "worker.js"));
expect(() => resolver.resolve(registration)).toThrow(/escapes/);
});
it.each(BUNDLED_PLUGIN_CATALOG)("accepts the real shipped $key package and manifest contract", (entry) => {
const manifest = manifests[entry.key as keyof typeof manifests];
const pkg = JSON.parse(readFileSync(new URL(`../../../packages/plugins/${entry.relativePath}/package.json`, import.meta.url), "utf8"));
const { root } = fixture();
const packageRoot = path.join(root, entry.relativePath);
mkdirSync(path.join(packageRoot, "dist"), { recursive: true });
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(pkg));
writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
writeFileSync(path.join(packageRoot, "dist", "worker.js"), "export default {};");
const installs = resolveBundledPluginInstalls([entry.key], { catalogRoot: root, enforceCatalogRoot: true, distributionPlugins: [], env: {} });
const resolver = createBundledPluginPackageResolver(installs);
const registration = { pluginKey: entry.pluginKey, packageName: pkg.name, version: manifest.version, manifestJson: manifest };
expect(pkg.version).toBe(manifest.version);
expect(resolver.resolve(registration)).toBe(packageRoot);
expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true });
});
});
@@ -0,0 +1,287 @@
import express from "express";
import request from "supertest";
import { execFileSync } from "node:child_process";
import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs";
import path from "node:path";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db";
import { eq } from "drizzle-orm";
import type { PluginRecord } from "@paperclipai/shared";
import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs";
import { fileURLToPath } from "node:url";
import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
import { inspectCloudPackages } from "../services/cloud-package-qualification.js";
import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
import { pluginRegistryService } from "../services/plugin-registry.js";
import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js";
import { errorHandler } from "../middleware/error-handler.js";
import { derivePluginDatabaseNamespace } from "../services/plugin-database.js";
import { ensureBundledPlugins } from "../services/bundled-plugins.js";
import daytonaManifest from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const externalDatabaseUrl = process.env.PAPERCLIP_BUNDLED_PLUGIN_TEST_DATABASE_URL?.trim();
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describePostgres = support.supported ? describe : describe.skip;
describePostgres("bundled image registration in PostgreSQL", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let peer: ReturnType<typeof createDb>;
const roots: string[] = [];
const fixture = () => { const result = bundledImageFixture(); roots.push(result.root); return result; };
beforeAll(async () => {
if (externalDatabaseUrl) {
await applyPendingMigrations(externalDatabaseUrl);
database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
} else {
database = await startEmbeddedPostgresTestDatabase("bundled-image-registration-");
}
db = createDb(database.connectionString);
peer = createDb(database.connectionString);
}, 90_000);
afterEach(async () => {
await db.delete(plugins);
await db.delete(companies);
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
afterAll(async () => { await database?.cleanup(); });
function runtime() {
const output: string[] = [];
const services = {
workerManager: {
startWorker: vi.fn(async (_id: string, options: { entrypointPath: string }) => { output.push(execFileSync(process.execPath, [options.entrypointPath, "--source"], { encoding: "utf8" })); }),
call: vi.fn(async () => ({})),
},
lifecycleManager: { markError: vi.fn() },
jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" },
};
return { services, output };
}
function loaderFor(image: ReturnType<typeof fixture>, databaseClient = db, services?: PluginRuntimeServices) {
return pluginLoader(databaseClient, {
localPluginDir: path.join(image.root, "lost-home"), prebuiltOnly: true,
resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, services);
}
it("starts current-image workers after losing the old home and retains each company's config", async () => {
const old = fixture();
const first = fixture();
const second = fixture();
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
const registry = pluginRegistryService(db);
let row = (await registry.getByKey(old.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
const [companyA, companyB, unconfigured] = await db.insert(companies).values([
{ name: "Company A", issuePrefix: "CFA" }, { name: "Company B", issuePrefix: "CFB" }, { name: "Unconfigured", issuePrefix: "CFC" },
]).returning();
await registry.upsertConfig(row.id, companyA!.id, { configJson: { token: "only-A" } });
await registry.upsertConfig(row.id, companyB!.id, { configJson: { token: "only-B" } });
row = (await registry.getById(row.id))!;
rmSync(old.root, { recursive: true });
const original = { ...row };
for (const image of [first, second]) {
const current = runtime();
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
const result = await loader.loadSingle(row.id);
expect(result.success).toBe(true);
expect(current.output).toEqual([expect.stringContaining(image.packageRoot)]);
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({
entrypointPath: path.join(image.packageRoot, "dist", "worker.js"), config: {}, proactiveCompanyScopes: [companyA!.id, companyB!.id].sort(),
}));
expect(current.services.workerManager.call.mock.calls).toEqual(expect.arrayContaining([
[row.id, "configChanged", { companyId: companyA!.id, config: { token: "only-A" } }],
[row.id, "configChanged", { companyId: companyB!.id, config: { token: "only-B" } }],
]));
expect(current.services.workerManager.call).toHaveBeenCalledTimes(2);
expect(await registry.getConfig(row.id, unconfigured!.id)).toBeNull();
const app = express();
app.use((req, _res, next) => {
req.actor = { type: "board", userId: "fixture-user", source: "session", isInstanceAdmin: false, companyIds: [companyA!.id] };
next();
});
app.use(pluginUiStaticRoutes(db, { localPluginDir: path.join(image.root, "lost-home"), resolvePackageRoot: image.resolver.resolve }));
app.use(errorHandler);
const served = await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyA!.id });
expect(served.status).toBe(200);
expect(served.text).toContain("current-image");
expect((await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyB!.id })).status).toBe(403);
}
expect(await registry.getById(row.id)).toEqual(original);
});
it("completes the real worker initialize and health protocol from a relocated image", async () => {
const old = fixture(); const image = fixture();
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(old.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
rmSync(old.root, { recursive: true });
const manager = createPluginWorkerManager();
try {
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
const result = await loaderFor(image, db, services).loadSingle(row.id);
expect(result.success).toBe(true);
expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(image.packageRoot), configs: {} } });
} finally { await manager.stopAll(); }
});
it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => {
const image = fixture();
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-"));
roots.push(provider);
cpSync(image.packageRoot, provider, { recursive: true });
linkSdkInto(provider);
writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);');
const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }];
const resolver = createBundledPluginPackageResolver(installs);
const manager = createPluginWorkerManager();
const loader = pluginLoader(db, {
prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices);
const registry = pluginRegistryService(db);
try {
await loader.installPlugin({ localPath: provider });
const installed = (await registry.getByKey(image.manifest.id))!;
await registry.updateStatus(installed.id, { status: "ready" });
const registration = (await registry.getById(installed.id))!;
expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]);
expect((await loader.loadSingle(installed.id)).success).toBe(true);
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" });
} finally { await manager.stopAll(); }
}, 90_000);
it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => {
const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] });
const resolver = createBundledPluginPackageResolver(installs);
const manager = createPluginWorkerManager();
const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
const loader = pluginLoader(db, {
localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true,
resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
enableLocalFilesystem: false, enableNpmDiscovery: false,
}, services);
const registry = pluginRegistryService(db);
vi.stubEnv("DAYTONA_API_KEY", "");
try {
await loader.installPlugin({ localPath: installs[0]!.localPath });
const installed = (await registry.getByKey(daytonaManifest.id))!;
await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" });
await registry.updateStatus(installed.id, { status: "ready" });
const registration = (await registry.getById(installed.id))!;
const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) });
expect(inspected.blockers).toEqual([]);
expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/);
expect((await loader.loadSingle(installed.id)).success).toBe(true);
expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" });
} finally { await manager.stopAll(); vi.unstubAllEnvs(); }
}, 90_000);
it("registers the same package concurrently from two roots without replacing the winner", async () => {
const first = fixture(); const second = fixture();
await Promise.all([
loaderFor(first).installPlugin({ localPath: first.packageRoot }),
loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }),
]);
const rows = await pluginRegistryService(db).listInstalled();
expect(rows).toHaveLength(1);
const original = rows[0]!;
expect([first.packageRoot, second.packageRoot]).toContain(original.packagePath);
await loaderFor(second, peer).installPlugin({ localPath: second.packageRoot });
expect(await pluginRegistryService(db).getById(original.id)).toEqual(original);
await expect(pluginRegistryService(peer).install({ packageName: original.packageName }, { ...first.manifest, description: "different package metadata" }, { idempotent: true })).rejects.toThrow(/already installed/);
});
it("preserves disabled registrations and avoids path writes during managed boot", async () => {
const old = fixture(); const image = fixture();
await loaderFor(old).installPlugin({ localPath: old.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(old.manifest.id))!;
await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, row.id));
const original = await registry.getById(row.id);
await ensureBundledPlugins(image.installs, { registry, loader: loaderFor(image), lifecycle: { load: vi.fn() }, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }, { reinstallUninstalled: true, preservePackagePath: true });
expect(await registry.getById(row.id)).toEqual(original);
});
it("fails before worker startup for missing image bytes without using a home fallback or building", async () => {
const image = fixture();
await loaderFor(image).installPlugin({ localPath: image.packageRoot });
const registry = pluginRegistryService(db);
const row = (await registry.getByKey(image.manifest.id))!;
await registry.updateStatus(row.id, { status: "ready" });
const fallback = path.join(image.root, "lost-home", "node_modules", row.packageName);
mkdirSync(fallback, { recursive: true });
writeFileSync(path.join(fallback, "package.json"), JSON.stringify({ name: row.packageName }));
writeFileSync(path.join(fallback, "worker.js"), "throw new Error('must not start');");
rmSync(path.join(image.packageRoot, "dist", "worker.js"));
const current = runtime();
const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
const result = await loader.loadSingle(row.id);
expect(result.success).toBe(false);
expect(result.error).toContain("ENOENT");
expect(current.services.workerManager.startWorker).not.toHaveBeenCalled();
await expect(loader.installPlugin({ packageName: row.packageName })).rejects.toThrow(/current image catalog/);
await loader.cleanupInstallArtifacts(row as PluginRecord);
expect(image.resolver.inspect(row, image.manifest).compatible).toBe(false);
});
it("applies plugin migrations from the current image after the legacy package disappears", async () => {
const image = bundledImageFixture({
capabilities: ["database.namespace.migrate", "database.namespace.read", "database.namespace.write"],
database: { migrationsDir: "migrations" },
});
roots.push(image.root);
const [row] = await db.insert(plugins).values({
...image.registration, packagePath: "/lost/legacy/home/plugin", apiVersion: 1, categories: image.manifest.categories, status: "ready",
}).returning();
const current = runtime();
const result = await loaderFor(image, db, current.services as unknown as PluginRuntimeServices).loadSingle(row!.id);
expect(result.success).toBe(true);
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row!.id, expect.objectContaining({ databaseNamespace: derivePluginDatabaseNamespace(image.manifest.id) }));
await db.execute((await import("drizzle-orm")).sql.raw(`DROP SCHEMA "${derivePluginDatabaseNamespace(image.manifest.id)}" CASCADE`));
});
it("retains registry-install provider credentials for an ordinary npm package", async () => {
const npmManifest = { ...daytonaManifest, id: "paperclip.image-npm-fixture" };
const image = bundledImageFixture(npmManifest);
roots.push(image.root);
const localPluginDir = path.join(image.root, "npm-home");
const packageRoot = path.join(localPluginDir, "node_modules", "@paperclipai", "plugin-daytona");
mkdirSync(path.dirname(packageRoot), { recursive: true });
cpSync(image.packageRoot, packageRoot, { recursive: true });
const metadata = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8"));
metadata.name = "@paperclipai/plugin-daytona";
writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(metadata));
const registry = pluginRegistryService(db);
const row = (await registry.install({ packageName: metadata.name }, npmManifest))!;
await registry.updateStatus(row.id, { status: "ready" });
vi.stubEnv("DAYTONA_API_KEY", "synthetic-provider-key");
try {
const current = runtime();
const loader = pluginLoader(db, { localPluginDir, enableNpmDiscovery: false, enableLocalFilesystem: false }, current.services as unknown as PluginRuntimeServices);
expect((await loader.loadSingle(row.id)).success).toBe(true);
expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ env: expect.objectContaining({ DAYTONA_API_KEY: "synthetic-provider-key" }) }));
} finally { vi.unstubAllEnvs(); }
});
it("keeps self-hosted local-path loading available without a managed resolver", async () => {
const image = fixture();
const loader = pluginLoader(db, { localPluginDir: path.join(image.root, "selfhost"), enableNpmDiscovery: false, enableLocalFilesystem: false });
await loader.installPlugin({ localPath: image.packageRoot });
const row = (await pluginRegistryService(db).getByKey(image.manifest.id))!;
expect(row.packagePath).toBe(image.packageRoot);
await expect(loader.installPlugin({ localPath: image.packageRoot })).rejects.toThrow(/already installed/);
});
});
@@ -0,0 +1,72 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { build, transform } from "esbuild";
import { bundleRuntime } from "../../scripts/bundle-runtime.mjs";
import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js";
const roots: string[] = [];
function fixture() {
const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-")));
roots.push(root);
fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true });
fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" }));
const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs");
fs.mkdirSync(path.dirname(loader), { recursive: true });
fs.writeFileSync(loader, "export {};\n");
return { root, loader };
}
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
describe("current image plugin runtime loader", () => {
it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => {
const { root, loader } = fixture();
const moduleDirectory = path.join(root, "server", location);
fs.mkdirSync(moduleDirectory, { recursive: true });
expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader);
});
it("resolves a bundled entrypoint without preserved source module locations", async () => {
const { root, loader } = fixture();
const runtime = path.join(root, "server", "dist-runtime", "index.mjs");
await build({
stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() },
outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent",
});
expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader);
});
it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => {
const { root, loader } = fixture();
const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js");
fs.mkdirSync(path.dirname(service), { recursive: true });
const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8");
fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code);
const entry = path.join(root, "server", "dist", "probe.js");
fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());');
const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") });
expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader);
});
it("accepts the baked pnpm target inside the current image node_modules", () => {
const { root, loader } = fixture();
const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs");
fs.mkdirSync(path.dirname(physical), { recursive: true });
fs.writeFileSync(physical, "export {};\n");
fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader);
expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical);
});
it("fails explicitly for missing or legacy-home loaders", () => {
const { root, loader } = fixture();
const moduleDirectory = path.join(root, "server", "dist-runtime");
fs.rmSync(loader);
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image");
const outside = fixture().loader;
fs.symlinkSync(outside, loader);
expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image");
});
});
Loaded 100 of 322 files, more files were not shown because too many files have changed in this diff. Show more