diff --git a/.github/scripts/ci-diff.mjs b/.github/scripts/ci-diff.mjs
new file mode 100644
index 00000000000..324b87a18c5
--- /dev/null
+++ b/.github/scripts/ci-diff.mjs
@@ -0,0 +1,22 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+
+// Read trees only. The routing job fetches Git objects into a bare repository;
+// no pull request code, hooks, filters or package scripts run here.
+export function mergeDiff(merge, prHead, cwd = process.cwd()) {
+ assert.match(merge ?? "", /^[a-f0-9]{40}$/);
+ assert.match(prHead ?? "", /^[a-f0-9]{40}$/);
+ const git = (...args) => execFileSync("git", args, {
+ cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
+ }).trim();
+ const header = git("--no-replace-objects", "cat-file", "-p", merge).split("\n\n", 1)[0];
+ const parents = [...header.matchAll(/^parent ([a-f0-9]{40})$/gm)].map((match) => match[1]);
+ assert.equal(parents.length, 2, "Expected a GitHub pull request merge commit");
+ assert.equal(parents[1], prHead, "Merge commit does not contain the expected PR head");
+ const base = parents[0];
+ const changed = execFileSync("git", ["diff", "--no-ext-diff", "--no-textconv", "--no-renames",
+ "--name-only", "-z", base, merge, "--"], {
+ cwd, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
+ }).split("\0").filter(Boolean).sort();
+ return { diff: { source: "git-merge-parent", base, head: merge, prHead }, changed };
+}
diff --git a/.github/scripts/ci-inputs.mjs b/.github/scripts/ci-inputs.mjs
new file mode 100644
index 00000000000..ad17a8acdc5
--- /dev/null
+++ b/.github/scripts/ci-inputs.mjs
@@ -0,0 +1,10 @@
+// Audited prose inputs, shared by routing, Vitest selection and task hashing.
+// Do not broaden this to all Markdown: docs/api and bundled skills are runtime inputs.
+export function documentationInput(file) {
+ if (file === "doc/DEVELOPING.md" || file === "doc/DATABASE.md") return "guidance";
+ if (/^doc\/plans\/(?:[^/]+\/)*[^/]+\.md$/.test(file)
+ && !file.split("/").includes("..")) return "historical";
+ return null;
+}
+
+export const productInputs = (files) => files.filter((file) => !documentationInput(file));
diff --git a/.github/scripts/ci-plan.mjs b/.github/scripts/ci-plan.mjs
index 3019b936ecf..f58665218a2 100644
--- a/.github/scripts/ci-plan.mjs
+++ b/.github/scripts/ci-plan.mjs
@@ -2,6 +2,8 @@ import { appendFileSync, readFileSync, realpathSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { matchesGlob } from "node:path";
import { fileURLToPath } from "node:url";
+import { documentationInput, productInputs } from "./ci-inputs.mjs";
+import { mergeDiff } from "./ci-diff.mjs";
const releaseNote = /^releases\/(beta\/)?v[^/]+\.md$/;
const validPath = (file) => typeof file === "string" && file.length > 0
@@ -52,28 +54,33 @@ export function planTasks({ config, lanes, paths, mode = "enabled", forceReason
const changed = [...new Set(paths)];
if (!changed.length || changed.some((file) => !validPath(file))) forceReason ??= "Missing or invalid diff";
const matches = (file, globs) => globs.some((glob) => matchesGlob(file, glob));
- const globalChanges = changed.filter((file) => matches(file, config.globalDependencies));
+ const sourceChanges = productInputs(changed);
+ const globalChanges = sourceChanges.filter((file) => matches(file, config.globalDependencies));
if (globalChanges.length) forceReason ??= `Global CI input changed: ${globalChanges[0]}`;
- const unknown = changed.filter((file) => !releaseNote.test(file)
+ const unknown = sourceChanges.filter((file) => !releaseNote.test(file)
&& !matches(file, config.globalDependencies) && ![...inputs.values()].some((globs) => matches(file, globs)));
if (unknown.length) forceReason ??= `Unclassified path: ${unknown[0]}`;
if (mode === "full") forceReason ??= "Full CI requested";
const reasons = Object.fromEntries([...inputs].map(([task, globs]) => [task,
- forceReason ? [forceReason] : [...new Set([...globalChanges, ...changed.filter((file) => matches(file, globs))])],
+ forceReason ? [forceReason] : [...new Set([...globalChanges, ...sourceChanges.filter((file) => matches(file, globs))])],
]));
+ // This uncached lane owns the repository-wide CLI guidance check.
+ if (reasons["ci:typecheck"]) reasons["ci:typecheck"].push(...changed.filter((file) => documentationInput(file) === "guidance"));
const jobs = Object.fromEntries(Object.entries(lanes).map(([job, entries]) => {
const affected = entries.filter((entry) => reasons[entry.task].length > 0);
const selected = mode === "shadow" ? entries : affected;
return [job, { run: selected.length > 0, matrix: { include: selected.length ? selected : entries },
selected: selected.length, proposed: affected.length, total: entries.length }];
}));
- return { version: 1, mode, forceReason, changed, reasons, jobs };
+ return { version: 1, mode, forceReason, changed, documentation: changed.filter(documentationInput), reasons, jobs };
}
export function planSummary(plan) {
const rows = Object.entries(plan.jobs).map(([job, lane]) =>
`| ${job} | ${lane.proposed}/${lane.total} | ${lane.selected}/${lane.total} |`);
return ["### Incremental CI", "", `Mode: **${plan.mode}**${plan.forceReason ? ` — ${plan.forceReason}` : ""}`,
+ ...(plan.diff ? [`Diff: \`${plan.diff.base}\` → \`${plan.diff.head}\` (complete Git tree comparison)`, ""] : []),
+ `Documentation inputs: ${(plan.documentation ?? []).join(", ") || "none"}`,
"", "| Suite | Affected jobs | Scheduled jobs |", "| --- | ---: | ---: |", ...rows,
"", "Selection reasons
", "", "```json", JSON.stringify(plan.reasons, null, 2),
"```", "- ,
completion_contract: Option<(&str, &[String])>,
+ accounting: &mut Vec<(String, Value)>,
) -> Result {
let authorized_tools = authorized_tools.into_iter().collect::>();
let completion_contract =
@@ -1354,7 +1355,15 @@ impl CodexProvider {
{
return Ok(false);
}
- let blockers = self.warm_run_attachment_blockers(true)?;
+ let (tail, result) = self.drain_completed_turn_tail_for_warm_attachment();
+ accounting.extend(tail);
+ result?;
+ if !accounting.is_empty() {
+ return Err(LocalRunnerError::invalid(
+ "Codex warm run attachment requires pending accounting reconciliation",
+ ));
+ }
+ let blockers = self.warm_run_attachment_blockers()?;
if !blockers.is_empty() {
return Err(LocalRunnerError::invalid(
format!(
@@ -1375,49 +1384,53 @@ impl CodexProvider {
Ok(true)
}
- fn drain_completed_turn_tail_for_warm_attachment(&mut self) -> Result<(), LocalRunnerError> {
- let Some(completed_turn_id) = self
- .completed_turn_authority
- .as_ref()
- .map(|authority| authority.provider_turn_id.clone())
- else {
- return Ok(());
- };
- if self.active_provider_turn_id.is_some() {
- return Ok(());
- }
-
- // Readiness probes run over the PRP command channel while provider
- // stdout is drained by runnerd's adjacent control-loop iteration. A
- // final usage/warning/item frame can therefore land after the last
- // successful probe but before run.attach executes. Close that race in
- // the same critical section as authority rotation. Only bounded tail
- // notifications for the already-settled turn may be discarded: a new
- // turn, provider request, process exit, or mismatched turn remains a
- // fail-closed attachment error.
- let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE;
- let mut quiet_since: Option = None;
- let mut drained = 0usize;
- loop {
- if std::time::Instant::now() >= deadline {
- return Err(LocalRunnerError::invalid(
- "Codex warm run attachment tail did not become quiescent",
- ));
+ pub(crate) fn drain_completed_turn_tail_for_warm_attachment(
+ &mut self,
+ ) -> (Vec<(String, Value)>, Result<(), LocalRunnerError>) {
+ let mut accounting = Vec::new();
+ let result = (|| {
+ let Some(completed_turn_id) = self
+ .completed_turn_authority
+ .as_ref()
+ .map(|authority| authority.provider_turn_id.clone())
+ else {
+ return Ok(());
+ };
+ if self.active_provider_turn_id.is_some() {
+ return Ok(());
}
- match self.poll()? {
- Some(CodexProviderEvent::Notification { method, params }) => {
- quiet_since = None;
- drained = drained.saturating_add(1);
- if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT {
- return Err(LocalRunnerError::invalid(
- "Codex warm run attachment tail exceeded its bounded frame limit",
- ));
- }
- let names_other_turn = notification_turn_id(¶ms)
- .is_some_and(|turn_id| turn_id != completed_turn_id);
- let safe_tail_method = matches!(
- method.as_str(),
- "warning"
+
+ // Readiness probes run over the PRP command channel while provider
+ // stdout is drained by runnerd's adjacent control-loop iteration. A
+ // final usage/warning/item frame can therefore land after the last
+ // successful probe but before run.attach executes. Close that race in
+ // the same critical section as authority rotation. Only bounded tail
+ // notifications for the already-settled turn may be discarded: a new
+ // turn, provider request, process exit, or mismatched turn remains a
+ // fail-closed attachment error.
+ let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE;
+ let mut quiet_since: Option = None;
+ let mut drained = 0usize;
+ loop {
+ if std::time::Instant::now() >= deadline {
+ return Err(LocalRunnerError::invalid(
+ "Codex warm run attachment tail did not become quiescent",
+ ));
+ }
+ match self.poll()? {
+ Some(CodexProviderEvent::Notification { method, params }) => {
+ quiet_since = None;
+ drained = drained.saturating_add(1);
+ if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT {
+ return Err(LocalRunnerError::invalid(
+ "Codex warm run attachment tail exceeded its bounded frame limit",
+ ));
+ }
+ let names_other_turn = notification_turn_id(¶ms)
+ .is_some_and(|turn_id| turn_id != completed_turn_id);
+ let safe_tail_method = matches!(
+ method.as_str(),
+ "warning"
| "configWarning"
| "remoteControl/status/changed"
| "mcpServer/startupStatus/updated"
@@ -1436,73 +1449,82 @@ impl CodexProvider {
| "thread/status/changed"
| "turn/diff/updated"
| "turn/plan/updated"
- );
- if names_other_turn || !safe_tail_method {
- return Err(LocalRunnerError::invalid(format!(
+ );
+ if names_other_turn || !safe_tail_method {
+ return Err(LocalRunnerError::invalid(format!(
"Codex warm run attachment observed unsafe post-terminal provider method {}",
bounded_method(&method)
)));
- }
- if let Some(frame_id) = self.take_provider_trace_frame_id() {
- self.record_provider_trace_interpretation(
+ }
+ if matches!(
+ method.as_str(),
+ "thread/tokenUsage/updated" | "paperclip/resumeUsageSnapshot"
+ ) {
+ accounting.push((method, params));
+ continue;
+ }
+ if let Some(frame_id) = self.take_provider_trace_frame_id() {
+ self.record_provider_trace_interpretation(
frame_id,
"codex.warm_attachment.completed_turn_tail",
"ignored",
Vec::new(),
"Provider emitted a bounded tail notification after the prior turn terminal and before run attachment",
);
+ }
}
- }
- Some(CodexProviderEvent::ResourceLimit { .. }) => {
- return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
- }
- Some(CodexProviderEvent::ProtocolFailure { .. }) => {
- return Err(LocalRunnerError::invalid(
- "Codex protocol integrity failed during warm attachment",
- ));
- }
- Some(CodexProviderEvent::DescendantNotification { .. }) => {
- // Child output cannot certify a quiescent root attachment.
- return Err(LocalRunnerError::invalid(
- "Codex descendant remains active during warm run attachment",
- ));
- }
- Some(CodexProviderEvent::ToolCall { .. })
- | Some(CodexProviderEvent::RuntimeRequest { .. })
- | Some(CodexProviderEvent::McpPermissionRequest { .. }) => {
- return Err(LocalRunnerError::invalid(
- "Codex warm run attachment observed a post-terminal provider request",
- ));
- }
- Some(CodexProviderEvent::Exited { .. }) => {
- return Err(LocalRunnerError::invalid(
- "Codex exited while quiescing for warm run attachment",
- ));
- }
- None => {
- let now = std::time::Instant::now();
- let quiet_start = quiet_since.get_or_insert(now);
- if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW {
- return Ok(());
+ Some(CodexProviderEvent::ResourceLimit { .. }) => {
+ return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
+ }
+ Some(CodexProviderEvent::ProtocolFailure { .. }) => {
+ return Err(LocalRunnerError::invalid(
+ "Codex protocol integrity failed during warm attachment",
+ ));
+ }
+ Some(CodexProviderEvent::DescendantNotification { .. }) => {
+ // Child output cannot certify a quiescent root attachment.
+ return Err(LocalRunnerError::invalid(
+ "Codex descendant remains active during warm run attachment",
+ ));
+ }
+ Some(CodexProviderEvent::ToolCall { .. })
+ | Some(CodexProviderEvent::RuntimeRequest { .. })
+ | Some(CodexProviderEvent::McpPermissionRequest { .. }) => {
+ return Err(LocalRunnerError::invalid(
+ "Codex warm run attachment observed a post-terminal provider request",
+ ));
+ }
+ Some(CodexProviderEvent::Exited {
+ completed_turn_authoritative,
+ completion_reconciles_exit,
+ ..
+ }) if completion_reconciles_exit
+ || (completed_turn_authoritative && self.process.stdout_drained()) =>
+ {
+ return Ok(())
+ }
+ Some(CodexProviderEvent::Exited { .. }) => {
+ return Err(LocalRunnerError::invalid(
+ "Codex exited while quiescing for warm run attachment",
+ ));
+ }
+ None => {
+ let now = std::time::Instant::now();
+ let quiet_start = quiet_since.get_or_insert(now);
+ if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW {
+ return Ok(());
+ }
+ std::thread::sleep(Duration::from_millis(1));
}
- std::thread::sleep(Duration::from_millis(1));
}
}
- }
+ })();
+ (accounting, result)
}
pub(crate) fn warm_run_attachment_blockers(
&mut self,
- quiesce_completed_tail: bool,
) -> Result, LocalRunnerError> {
- // Only an explicit attachment-readiness probe may consume the bounded,
- // already-settled provider suffix. Ordinary checkpoint snapshots run
- // immediately after a terminal frame while the provider can still be
- // unwinding; turning those observations into quiescence barriers can
- // quarantine an otherwise reusable runner before its next turn.
- if quiesce_completed_tail {
- self.drain_completed_turn_tail_for_warm_attachment()?;
- }
let mut blockers = Vec::new();
if self.process.try_wait()?.is_some() {
blockers.push("process_exited");
@@ -1517,7 +1539,18 @@ impl CodexProvider {
blockers.push("ambiguous_turn_start");
}
if !self.pending_messages.is_empty() {
- blockers.push("pending_messages");
+ blockers.push(
+ if self.pending_messages.iter().any(|message| {
+ matches!(
+ message.value.get("method").and_then(Value::as_str),
+ Some("paperclip/resumeUsageSnapshot" | "thread/tokenUsage/updated")
+ )
+ }) {
+ "pending_accounting_reconciliation"
+ } else {
+ "pending_messages"
+ },
+ );
}
if !self.deferred_ambiguous_messages.is_empty() {
blockers.push("deferred_messages");
@@ -2919,13 +2952,11 @@ impl CodexProvider {
.and_then(Value::as_str);
if notification_turn_id.is_some()
&& notification_turn_id != self.active_provider_turn_id.as_deref()
- && notification_turn_id
+ && (notification_turn_id
.is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
+ || (self.active_provider_turn_id.is_none()
+ && method == "thread/tokenUsage/updated"))
{
- self.notification_identity_diagnostics += 1;
- if self.notification_identity_diagnostics > 32 {
- return Ok(None);
- }
// Resume replays the cumulative usage of the last settled turn.
// Keep its identity and baseline, but never bill its `last`
// measurement to the newly attached run.
@@ -2941,6 +2972,10 @@ impl CodexProvider {
}),
}));
}
+ self.notification_identity_diagnostics += 1;
+ if self.notification_identity_diagnostics > 32 {
+ return Ok(None);
+ }
return Ok(Some(CodexProviderEvent::Notification {
method: "warning".to_owned(),
params: json!({
@@ -4392,9 +4427,58 @@ done
});
// poll() normalizes settled-turn usage to paperclip/resumeUsageSnapshot.
// That accounting fact is not new work and must not force replacement.
- let result = provider.drain_completed_turn_tail_for_warm_attachment();
+ let (accounting, result) = provider.drain_completed_turn_tail_for_warm_attachment();
provider.shutdown().unwrap();
- result.expect("historical usage must not break provider continuity");
+ result.expect("historical usage must remain available to its owner");
+ assert_eq!(accounting.len(), 1);
+ assert_eq!(accounting[0].1["total"]["inputTokens"], 120);
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn failed_warm_drain_retains_consumed_usage_before_unsafe_tail() {
+ for attach in [false, true] {
+ let mut provider = completion_tail_provider();
+ provider
+ .restore_completed_turn_authority(true, Some(1), Some("reader-tail-1"))
+ .unwrap();
+ provider.active_provider_turn_id = None;
+ for value in [
+ json!({"method":"thread/tokenUsage/updated","params":{"threadId":"reader-tail-thread","turnId":"reader-tail-1","tokenUsage":{"total":{"inputTokens":120,"outputTokens":12}}}}),
+ json!({"method":"turn/started","params":{"threadId":"reader-tail-thread","turn":{"id":"unsafe-next-turn"}}}),
+ ] {
+ provider
+ .pending_messages
+ .push_back(BufferedProviderMessage {
+ value,
+ trace_frame_id: None,
+ });
+ }
+ let (accounting, result) = if attach {
+ let mut accounting = Vec::new();
+ let tools = provider.authorized_tools.clone();
+ let contract = provider.completion_contract.clone();
+ let result = provider
+ .attach_run_in_place(
+ tools,
+ contract.as_ref().map(|contract| {
+ (
+ contract.revision.as_str(),
+ contract.criterion_ids.as_slice(),
+ )
+ }),
+ &mut accounting,
+ )
+ .map(|_| ());
+ (accounting, result)
+ } else {
+ provider.drain_completed_turn_tail_for_warm_attachment()
+ };
+ provider.shutdown().unwrap();
+ assert!(result.is_err());
+ assert_eq!(accounting.len(), 1);
+ assert_eq!(accounting[0].1["total"]["inputTokens"], 120);
+ }
}
#[cfg(unix)]
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs
new file mode 100644
index 00000000000..41e282ffe97
--- /dev/null
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs
@@ -0,0 +1,234 @@
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+pub(crate) struct CodexRunUsage {
+ baseline: Option>,
+ latest: BTreeMap,
+}
+
+impl CodexRunUsage {
+ pub(crate) fn new(requires_baseline: bool) -> Self {
+ Self {
+ baseline: (!requires_baseline).then(BTreeMap::new),
+ latest: BTreeMap::new(),
+ }
+ }
+
+ pub(crate) fn is_valid(&self) -> bool {
+ self.baseline
+ .iter()
+ .flat_map(|values| values.values())
+ .chain(self.latest.values())
+ .all(|value| {
+ value
+ .as_f64()
+ .is_some_and(|value| value.is_finite() && value >= 0.0)
+ })
+ }
+
+ pub(crate) fn increases_known_usage(&self, total: &Value) -> bool {
+ !self.latest.is_empty()
+ && total.as_object().is_some_and(|total| {
+ total.iter().any(|(key, value)| {
+ value.as_f64().is_some_and(|value| {
+ value > self.latest.get(key).and_then(Value::as_f64).unwrap_or(0.0)
+ })
+ })
+ })
+ }
+
+ pub(crate) fn increases_run_baseline(&self, total: &Value) -> bool {
+ self.baseline.as_ref().is_some_and(|baseline| {
+ total.as_object().is_some_and(|total| {
+ total.iter().any(|(key, value)| {
+ value.as_f64().is_some_and(|value| {
+ value > baseline.get(key).and_then(Value::as_f64).unwrap_or(0.0)
+ })
+ })
+ })
+ })
+ }
+
+ pub(crate) fn has_baseline(&self) -> bool {
+ self.baseline.is_some()
+ }
+
+ pub(crate) fn begin_run(&mut self) {
+ if self.latest.is_empty() {
+ return;
+ }
+ self.baseline = (valid_token(self.latest.get("inputTokens"))
+ && valid_token(self.latest.get("outputTokens")))
+ .then(|| self.latest.clone());
+ }
+
+ pub(crate) fn observe(&mut self, total: &Value, historical: bool) -> (Value, bool) {
+ let measurement: BTreeMap = total
+ .as_object()
+ .into_iter()
+ .flat_map(|value| value.iter())
+ .filter(|(_, value)| {
+ value
+ .as_f64()
+ .is_some_and(|value| value.is_finite() && value >= 0.0)
+ })
+ .map(|(key, value)| (key.clone(), value.clone()))
+ .collect();
+ let complete = self.baseline.is_some()
+ && (historical
+ || measurement.iter().all(|(key, value)| {
+ !key.ends_with("Tokens")
+ || value.as_u64().is_some_and(|value| {
+ value >= self.latest.get(key).and_then(Value::as_u64).unwrap_or(0)
+ && value
+ >= self
+ .baseline
+ .as_ref()
+ .and_then(|baseline| baseline.get(key))
+ .and_then(Value::as_u64)
+ .unwrap_or(0)
+ })
+ }))
+ && valid_token(measurement.get("inputTokens"))
+ && valid_token(measurement.get("outputTokens"))
+ && measurement
+ .iter()
+ .all(|(key, value)| !key.ends_with("Tokens") || valid_token(Some(value)))
+ && self
+ .latest
+ .keys()
+ .all(|key| !key.ends_with("Tokens") || valid_token(measurement.get(key)));
+ if historical
+ && self.baseline.is_none()
+ && valid_token(measurement.get("inputTokens"))
+ && valid_token(measurement.get("outputTokens"))
+ {
+ self.baseline = Some(measurement.clone());
+ }
+ for (key, value) in measurement {
+ if value.as_f64().unwrap()
+ >= self.latest.get(&key).and_then(Value::as_f64).unwrap_or(0.0)
+ {
+ self.latest.insert(key, value);
+ }
+ }
+ let delta: serde_json::Map = self
+ .latest
+ .iter()
+ .map(|(key, value)| {
+ let baseline = self
+ .baseline
+ .as_ref()
+ .and_then(|baseline| baseline.get(key));
+ let delta = if let Some(value) = value.as_u64() {
+ json!(value.saturating_sub(baseline.and_then(Value::as_u64).unwrap_or(0)))
+ } else {
+ json!((value.as_f64().unwrap()
+ - baseline.and_then(Value::as_f64).unwrap_or(0.0))
+ .max(0.0))
+ };
+ (key.clone(), delta)
+ })
+ .collect();
+ (Value::Object(delta), complete)
+ }
+}
+
+fn valid_token(value: Option<&Value>) -> bool {
+ value
+ .and_then(Value::as_u64)
+ .is_some_and(|value| value <= 9_007_199_254_740_991)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn preserves_resumed_baselines_across_requests_recovery_and_attachment() {
+ let mut usage = CodexRunUsage::new(true);
+ usage.observe(
+ &json!({"inputTokens":100,"outputTokens":10,"cachedInputTokens":20}),
+ true,
+ );
+ let (first, complete) = usage.observe(
+ &json!({"inputTokens":140,"outputTokens":16,"cachedInputTokens":25}),
+ false,
+ );
+ assert!(complete);
+ assert_eq!(first["inputTokens"], 40);
+ let mut restored: CodexRunUsage =
+ serde_json::from_value(serde_json::to_value(&usage).unwrap()).unwrap();
+ let (second, complete) = restored.observe(
+ &json!({"inputTokens":190,"outputTokens":25,"cachedInputTokens":30}),
+ false,
+ );
+ assert!(complete);
+ assert_eq!(
+ second,
+ json!({"inputTokens":90,"outputTokens":15,"cachedInputTokens":10})
+ );
+ restored.begin_run();
+ let (next, complete) = restored.observe(
+ &json!({"inputTokens":220,"outputTokens":29,"cachedInputTokens":32}),
+ false,
+ );
+ assert!(complete);
+ assert_eq!(
+ next,
+ json!({"inputTokens":30,"outputTokens":4,"cachedInputTokens":2})
+ );
+ }
+
+ #[test]
+ fn attachment_before_paid_work_preserves_a_proven_empty_baseline() {
+ let mut usage = CodexRunUsage::new(false);
+ usage.begin_run();
+ let (delta, complete) = usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false);
+ assert!(complete);
+ assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10}));
+ let mut unknown = CodexRunUsage::new(true);
+ unknown.begin_run();
+ assert!(
+ !unknown
+ .observe(&json!({"inputTokens":100,"outputTokens":10}), false)
+ .1
+ );
+ }
+
+ #[test]
+ fn regressing_live_totals_remain_incomplete_without_losing_known_exposure() {
+ let mut usage = CodexRunUsage::new(false);
+ usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false);
+ let (delta, complete) = usage.observe(&json!({"inputTokens":90,"outputTokens":9}), false);
+ assert!(!complete);
+ assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10}));
+ let mut resumed = CodexRunUsage::new(true);
+ resumed.observe(&json!({"inputTokens":100,"outputTokens":10}), true);
+ assert!(
+ !resumed
+ .observe(&json!({"inputTokens":90,"outputTokens":9}), false)
+ .1
+ );
+ }
+
+ #[test]
+ fn unknown_baselines_and_missing_counters_cannot_certify_usage() {
+ let mut usage = CodexRunUsage::new(true);
+ assert!(
+ !usage
+ .observe(&json!({"inputTokens":140,"outputTokens":16}), false)
+ .1
+ );
+ usage.observe(&json!({"inputTokens":100,"outputTokens":10}), true);
+ assert!(!usage.observe(&json!({"inputTokens":160}), false).1);
+ assert!(
+ usage
+ .observe(&json!({"inputTokens":170,"outputTokens":20}), false)
+ .1
+ );
+ }
+}
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs
index 5c1d58d335f..ffbc524010e 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs
@@ -13,6 +13,7 @@ pub mod aws_agentcore_provider;
pub mod claude_managed_provider;
pub mod codex_provider;
mod codex_startup_trust;
+mod codex_usage;
mod configured_environment;
mod credential_observer;
pub mod dot_provider_backend;
diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
index c84f57773e8..74a14c53002 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
@@ -17,6 +17,7 @@ use crate::codex_provider::{
ProviderStartupObservation, ProviderStartupStage, RejectedAcceptedTurn,
MAX_SETTLED_PROVIDER_TURN_IDS,
};
+use crate::codex_usage::CodexRunUsage;
use crate::durable::{
create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value,
validate_semantic_tool_input, verify_private_directory, Command, CommandExecution,
@@ -93,6 +94,13 @@ struct ProviderEventIdentity {
item_id: String,
}
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+struct CodexAccountingOrigin {
+ identity: ProviderEventIdentity,
+ provider_turn_id: String,
+}
+
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ProviderStartupCommand {
@@ -608,6 +616,105 @@ fn normalize_provider_notification(
method: &str,
params: &Value,
) -> Result, DurableRunnerError> {
+ if state.config.provider == "codex"
+ && ["thread/tokenUsage/updated", "paperclip/resumeUsageSnapshot"].contains(&method)
+ {
+ let historical = method == "paperclip/resumeUsageSnapshot";
+ let total = if historical {
+ params.get("total")
+ } else {
+ params.pointer("/tokenUsage/total")
+ }
+ .unwrap_or(&Value::Null);
+ let increases_known_usage = state
+ .codex_usage
+ .as_ref()
+ .is_some_and(|usage| usage.increases_known_usage(total));
+ let received_turn = params.get("turnId").and_then(Value::as_str);
+ let increases_prior_baseline = state.previous_accounting_origin.is_some()
+ && received_turn != state.completed_provider_turn_id.as_deref()
+ && state
+ .codex_usage
+ .as_ref()
+ .is_some_and(|usage| usage.increases_run_baseline(total));
+ if historical
+ && (increases_known_usage || increases_prior_baseline)
+ && params.get("turnId").and_then(Value::as_str)
+ != state.active_provider_turn_id.as_deref()
+ {
+ state.lifecycle = "reconciliation_required".to_owned();
+ let owns_completed_turn = state.active_provider_turn_id.is_none()
+ && state
+ .completed_provider_turn_id
+ .as_deref()
+ .is_some_and(|id| params.get("turnId").and_then(Value::as_str) == Some(id));
+ if !owns_completed_turn {
+ if state.active_provider_turn_id.is_some() {
+ state.settle_active_provider_turn_identity()?;
+ }
+ state.active_provider_turn_id = None;
+ let mut invalidation = normalize_codex_notification("thread/tokenUsage/updated", &json!({
+ "threadId":params.get("threadId"), "model":state.config.model, "tokenUsage":{"total":total}
+ })).remove(0);
+ invalidation.payload["runDeltaBasis"] = json!("run");
+ invalidation.payload["reconciliation"] = json!({"reason":"accounting_reconciliation_required",
+ "receivedTurnId":params.get("turnId"),"cumulative":invalidation.payload["cumulative"],
+ "priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| received_turn == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity)});
+ return Ok(vec![
+ NormalizedProviderEvent {
+ event_type: "harness.diagnostic".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({"code":"codex_usage_accounting_integrity", "classification":"accounting_reconciliation_required",
+ "receivedTurnId":params.get("turnId"),"cumulative":total,"priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| params.get("turnId").and_then(Value::as_str) == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity),
+ "message":"Late provider usage cannot be attributed to the attached run; explicit reconciliation is required"}),
+ },
+ invalidation,
+ NormalizedProviderEvent {
+ event_type: "turn.failed".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({"provider":"codex","status":"failed","code":"codex_usage_accounting_integrity","recoverable":false,
+ "message":"Late provider usage requires explicit accounting reconciliation"}),
+ },
+ NormalizedProviderEvent {
+ event_type: "run.terminal".to_owned(),
+ priority: EventPriority::P0,
+ payload: json!({"schema":"paperclip.prp.terminal.v1","provider":"codex","turnTerminalState":"failed","runTerminalState":"failed","reportedWorkDisposition":"needs_review"}),
+ },
+ ]);
+ }
+ }
+ let had_baseline = state
+ .codex_usage
+ .as_ref()
+ .is_some_and(CodexRunUsage::has_baseline);
+ let (delta, complete) = state
+ .codex_usage
+ .get_or_insert_with(|| CodexRunUsage::new(true))
+ .observe(total, historical);
+ let mut normalized_params = params.clone();
+ let emit_usage = !historical || (had_baseline && increases_known_usage);
+ if historical && emit_usage {
+ normalized_params = json!({"threadId": params.get("threadId"), "turnId": params.get("turnId"),
+ "tokenUsage":{"total":total,"last":delta}});
+ } else if !historical {
+ normalized_params["tokenUsage"]["last"] = delta;
+ }
+ let mut events = normalize_codex_notification(
+ if emit_usage {
+ "thread/tokenUsage/updated"
+ } else {
+ method
+ },
+ &normalized_params,
+ );
+ for event in &mut events {
+ if event.event_type == "usage.reported" {
+ event.payload["runDeltaAvailable"] = json!(complete);
+ event.payload["runDeltaBasis"] = json!("run");
+ }
+ }
+ return Ok(events);
+ }
if method != "paperclip/runResult" {
return Ok(normalize_codex_notification(method, params)
.into_iter()
@@ -640,14 +747,21 @@ fn normalize_provider_notification(
}
}
+fn terminal_completion_rejected(state: &CodexProviderState) -> bool {
+ state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none()
+}
+
+fn goal_continues_after_turn(state: &CodexProviderState, goal_active: bool) -> bool {
+ goal_active && !terminal_completion_rejected(state)
+}
+
fn terminal_events(
state: &CodexProviderState,
event_type: &str,
goal_status: Option<&str>,
) -> Vec {
- let rejected_completion =
- state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none();
- if goal_status == Some("active") && !rejected_completion {
+ let rejected_completion = terminal_completion_rejected(state);
+ if goal_continues_after_turn(state, goal_status == Some("active")) {
return Vec::new();
}
let Some(contract) = state.completion_contract.as_ref() else {
@@ -1007,6 +1121,10 @@ struct CodexProviderState {
#[serde(default)]
opencode_launch_profile_digest: Option,
#[serde(default)]
+ codex_usage: Option,
+ #[serde(default)]
+ previous_accounting_origin: Option,
+ #[serde(default)]
completion_contract: Option,
#[serde(default)]
tool_bridge: ProviderToolBridge,
@@ -1107,11 +1225,15 @@ impl CodexProviderState {
tool_bridge: ProviderToolBridge,
) -> Self {
let thread_id = config.provider_session_id.clone();
+ let codex_usage =
+ (config.provider == "codex").then(|| CodexRunUsage::new(thread_id.is_some()));
Self {
schema: PROVIDER_STATE_SCHEMA.to_owned(),
lifecycle: "prepared".to_owned(),
config,
opencode_launch_profile_digest: None,
+ codex_usage,
+ previous_accounting_origin: None,
completion_contract,
tool_bridge,
thread_id,
@@ -1154,6 +1276,13 @@ impl CodexProviderState {
self.tool_bridge.validate_recovered().map_err(|error| {
DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))
})?;
+ if self
+ .codex_usage
+ .as_ref()
+ .is_some_and(|usage| !usage.is_valid())
+ {
+ return Err(DurableRunnerError::invalid("invalid Codex usage baseline"));
+ }
let mut pending_event_ids = HashSet::new();
if self.descendant_thread_ids.len() > crate::codex_provider::MAX_DESCENDANT_THREAD_IDS
|| self
@@ -2501,7 +2630,43 @@ impl CodexCommandExecutor {
stable
}
+ fn persist_completed_accounting_tail(
+ &mut self,
+ tail: Vec<(String, Value)>,
+ failed: bool,
+ ) -> Result<(), DurableRunnerError> {
+ if !tail.is_empty() || failed {
+ let state = self.state.as_mut().ok_or_else(|| {
+ DurableRunnerError::invalid("Codex provider has not been prepared")
+ })?;
+ for (method, params) in tail {
+ let events = normalize_provider_notification(state, &method, ¶ms)?;
+ state.extend_events(events)?;
+ }
+ if failed {
+ state.lifecycle = "reconciliation_required".to_owned();
+ }
+ self.save_state()?;
+ }
+ Ok(())
+ }
+
+ fn retain_completed_accounting_tail(&mut self) -> Result<(), DurableRunnerError> {
+ let (tail, result) = self
+ .provider
+ .as_mut()
+ .map(CodexProvider::drain_completed_turn_tail_for_warm_attachment)
+ .unwrap_or_else(|| (Vec::new(), Ok(())));
+ self.persist_completed_accounting_tail(tail, result.is_err())?;
+ result.map_err(|error| {
+ DurableRunnerError::invalid(format!(
+ "failed to retain completed Codex accounting: {error}"
+ ))
+ })
+ }
+
fn attach_run(&mut self, payload: &Value) -> Result<(), DurableRunnerError> {
+ self.retain_completed_accounting_tail()?;
let mut next_state = self
.state
.clone()
@@ -2600,6 +2765,17 @@ impl CodexCommandExecutor {
"run.attach tool contract could not be rebound: {error}"
))
})?;
+ if let Some(usage) = next_state.codex_usage.as_mut() {
+ usage.begin_run();
+ }
+ next_state.previous_accounting_origin = self
+ .event_identity
+ .clone()
+ .zip(next_state.completed_provider_turn_id.clone())
+ .map(|(identity, provider_turn_id)| CodexAccountingOrigin {
+ identity,
+ provider_turn_id,
+ });
next_state.completion_contract = completion_contract;
next_state.completed_turn_authoritative = false;
next_state.completed_turn_process_generation = None;
@@ -2613,10 +2789,12 @@ impl CodexCommandExecutor {
next_state.active_provider_result_disposition = None;
next_state.active_terminal_tool_rejected = false;
next_state.last_agent_message = None;
- let retained_provider = if let Some(provider) = self.provider.as_mut() {
- !runtime_launch_changed
- && !upgraded_skill_config
- && provider
+ let mut attachment_accounting = Vec::new();
+ let retention = if let Some(provider) = self.provider.as_mut() {
+ if runtime_launch_changed || upgraded_skill_config {
+ Ok(false)
+ } else {
+ provider
.attach_run_in_place(
next_state.tool_bridge.authorized_tools().cloned(),
next_state.completion_contract.as_ref().map(|contract| {
@@ -2625,12 +2803,14 @@ impl CodexCommandExecutor {
contract.criterion_ids.as_slice(),
)
}),
+ &mut attachment_accounting,
)
.map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to retain Codex for warm run attachment: {error}"
))
- })?
+ })
+ }
} else if next_state.lifecycle == "prepared"
&& next_state.provider_process_generation > 0
&& next_state.pending_events.iter().all(is_startup_audit_event)
@@ -2640,12 +2820,17 @@ impl CodexCommandExecutor {
// validated run-scoped settings here; open_session then restores
// that same thread under the new authority. Never restart during
// drain/suspend or require the stopped process to still exist.
- false
+ Ok(false)
} else {
return Err(DurableRunnerError::invalid(
"run.attach requires the restored Codex provider process",
));
};
+ self.persist_completed_accounting_tail(attachment_accounting, retention.is_err())?;
+ let retained_provider = match retention {
+ Ok(retained) => retained,
+ Err(error) => return Err(error),
+ };
if !retained_provider {
if let Some(provider) = self.provider.as_mut() {
provider.shutdown().map_err(|error| {
@@ -4044,10 +4229,13 @@ impl CodexCommandExecutor {
.get("quiesceForWarmAttach")
.and_then(Value::as_bool)
.unwrap_or(false);
+ if quiesce_for_warm_attach {
+ self.retain_completed_accounting_tail()?;
+ }
let mut warm_attach_blockers = self
.provider
.as_mut()
- .map(|provider| provider.warm_run_attachment_blockers(quiesce_for_warm_attach))
+ .map(|provider| provider.warm_run_attachment_blockers())
.transpose()
.map_err(|error| {
DurableRunnerError::invalid(format!(
@@ -4071,6 +4259,9 @@ impl CodexCommandExecutor {
if !state.queued_events.is_empty() {
warm_attach_blockers.push("durable_queued_events");
}
+ if state.lifecycle == "reconciliation_required" {
+ warm_attach_blockers.push("reconciliation_required");
+ }
let warm_attach_ready = warm_attach_blockers.is_empty();
Ok(CommandExecution::result(json!({
"status": state.lifecycle,
@@ -4281,6 +4472,22 @@ impl CodexCommandExecutor {
} else {
None
};
+ let goal_remains_active = match &goal_reconciliation {
+ Some(Ok(snapshot)) => normalize_codex_goal(snapshot, false)
+ .is_some_and(|goal| goal.status == "active"),
+ _ => self
+ .state
+ .as_ref()
+ .and_then(|state| state.goal.as_ref())
+ .is_some_and(|goal| goal.status == "active"),
+ };
+ let run_continues = self
+ .state
+ .as_ref()
+ .is_some_and(|state| goal_continues_after_turn(state, goal_remains_active));
+ if terminal_event_type.as_deref() == Some("turn.completed") && !run_continues {
+ self.retain_completed_accounting_tail()?;
+ }
let working_now = self
.provider
.as_ref()
@@ -4370,7 +4577,9 @@ impl CodexCommandExecutor {
state.receipt_limit_interrupt_attempts = 0;
state.receipt_limit_interrupt_deadline_unix_ms = None;
state.ambiguous_turn_start_pending = false;
- state.lifecycle = "session_open".to_owned();
+ if state.lifecycle != "reconciliation_required" {
+ state.lifecycle = "session_open".to_owned();
+ }
if let Some(goal) = state.goal.as_mut() {
goal.working_now = false;
state.goal_revision = state.goal_revision.saturating_add(1);
@@ -4473,6 +4682,13 @@ impl CodexCommandExecutor {
})
.unwrap_or_default();
self.save_state()?;
+ if self
+ .state
+ .as_ref()
+ .is_some_and(|state| state.lifecycle == "reconciliation_required")
+ {
+ break;
+ }
if let (Some(frame_id), Some(provider)) =
(trace_frame_id, self.provider.as_mut())
{
@@ -5894,6 +6110,8 @@ mod tests {
conversation_mode: None,
},
opencode_launch_profile_digest: None,
+ codex_usage: Some(CodexRunUsage::new(false)),
+ previous_accounting_origin: None,
completion_contract: None,
tool_bridge: ProviderToolBridge::default(),
thread_id: Some("thread-1".to_owned()),
diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
index ead00ee0f33..09368feb54a 100644
--- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
+++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
@@ -3964,6 +3964,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
"--durable-turn-ids",
"--emit-post-completion-warning",
"--emit-post-completion-passive-statuses",
+ "--warm-tail-accounting",
],
);
let runner_config = durable_config(&directory);
@@ -3991,6 +3992,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
))
.expect("start the first turn");
+ let mut old_usage = Vec::new();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
assert!(
@@ -3998,6 +4000,12 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
"the first turn must settle before attachment"
);
let events = poll_and_ack(&mut executor).expect("poll the first turn terminal");
+ old_usage.extend(
+ events
+ .iter()
+ .filter(|event| event.event_type == "usage.reported")
+ .cloned(),
+ );
if events
.iter()
.any(|event| event.event_type == "turn.completed")
@@ -4010,6 +4018,11 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
std::thread::sleep(std::time::Duration::from_millis(1));
}
+ assert_eq!(
+ old_usage.last().unwrap().payload["runDelta"]["inputTokens"],
+ 120
+ );
+ assert_eq!(old_usage.last().unwrap().payload["runDeltaAvailable"], true);
let readiness = executor
.execute(&command(
"readiness",
@@ -4034,6 +4047,23 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
.iter()
.any(|(event_type, _, _)| event_type == "run.attached"));
+ executor
+ .execute(&command(
+ "next-turn",
+ 6,
+ "turn.start",
+ json!({"text":"Complete the next paid turn."}),
+ ))
+ .unwrap();
+ let next_events = wait_for_executor_event_batch(&mut executor, "turn.completed");
+ let next_usage = next_events
+ .iter()
+ .filter(|event| event.event_type == "usage.reported")
+ .last()
+ .unwrap();
+ assert_eq!(next_usage.payload["runDelta"]["inputTokens"], 30);
+ assert_eq!(next_usage.payload["runDelta"]["outputTokens"], 3);
+ assert_eq!(next_usage.payload["runDeltaAvailable"], true);
executor.shutdown().expect("stop the warm provider");
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
@@ -6256,6 +6286,9 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() {
let mut config = provider_config(&directory, &["--complete-after-tool-call"]);
if scenario == "controller-goal" {
config.args.push("--agent-created-goal-on-open".to_owned());
+ config
+ .args
+ .push("--accounting-tail-after-terminal".to_owned());
}
config.args.extend([
"--terminal-result-file".to_owned(),
@@ -6327,6 +6360,13 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() {
.unwrap();
assert!(failure < turn && turn < terminal);
if scenario == "controller-goal" {
+ let final_usage = emitted
+ .iter()
+ .rposition(|event| event.event_type == "usage.reported")
+ .unwrap();
+ assert!(final_usage < failure && final_usage < turn && final_usage < terminal);
+ assert_eq!(emitted[final_usage].payload["runDelta"]["inputTokens"], 120);
+ assert_eq!(emitted[final_usage].payload["runDeltaAvailable"], true);
let settled_goal = emitted
.iter()
.rposition(|event| {
@@ -6687,7 +6727,11 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() {
let directory = temporary_directory("resume-usage-snapshot");
let config = provider_config(
&directory,
- &["--durable-turn-ids", "--resume-usage-snapshot"],
+ &[
+ "--durable-turn-ids",
+ "--resume-usage-snapshot",
+ "--cumulative-accounting-usage",
+ ],
);
let runner_config = durable_config(&directory);
let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
@@ -6756,7 +6800,7 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() {
.collect();
assert_eq!(snapshots.len(), 2);
assert_eq!(snapshots[0].payload["receivedTurnId"], "provider-turn-1");
- assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 100);
+ assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 200);
assert!(!events.iter().any(|event| matches!(
event.event_type.as_str(),
"provider.notice.recorded" | "usage.reported"
@@ -6988,3 +7032,361 @@ fn skill_flag_can_be_added_to_an_old_checkpoint_at_settled_run_attach() {
);
fs::remove_dir_all(directory).unwrap();
}
+
+#[test]
+fn durable_codex_usage_keeps_run_baselines_through_requests_recovery_and_attachment() {
+ let directory = temporary_directory("accounting-run-baseline");
+ fs::write(
+ directory.join("fake-state.json"),
+ serde_json::to_vec(&json!({"threadId":"thread-1","activeTurnId":null,"nextTurn":1}))
+ .unwrap(),
+ )
+ .unwrap();
+ let mut config = provider_config(
+ &directory,
+ &[
+ "--durable-turn-ids",
+ "--resume-usage-snapshot",
+ "--cumulative-accounting-usage",
+ ],
+ );
+ config.provider_session_id = Some("thread-1".to_owned());
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider":config,"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap();
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .unwrap();
+ for _ in 0..20 {
+ if poll_and_ack(&mut executor)
+ .unwrap()
+ .iter()
+ .any(|event| event.payload["code"] == "codex_resume_usage_snapshot")
+ {
+ break;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(5));
+ }
+ fn run(
+ executor: &mut CodexCommandExecutor,
+ sequence: u64,
+ expected_input: u64,
+ expected_output: u64,
+ ) {
+ executor
+ .execute(&command(
+ &format!("turn-{sequence}"),
+ sequence,
+ "turn.start",
+ json!({"text":"Complete the accounting turn."}),
+ ))
+ .unwrap();
+ let events = wait_for_executor_event_batch(executor, "turn.completed");
+ let usage = events
+ .iter()
+ .filter(|event| event.event_type == "usage.reported")
+ .last()
+ .expect("provider emitted usage");
+ assert_eq!(usage.payload["runDeltaBasis"], "run");
+ assert_eq!(usage.payload["runDeltaAvailable"], true);
+ assert_eq!(usage.payload["runDelta"]["inputTokens"], expected_input);
+ assert_eq!(usage.payload["runDelta"]["outputTokens"], expected_output);
+ for _ in 0..20 {
+ if poll_and_ack(executor).unwrap().is_empty() {
+ break;
+ }
+ }
+ }
+ run(&mut executor, 3, 100, 10);
+ run(&mut executor, 4, 200, 20);
+ executor.shutdown().unwrap();
+ drop(executor);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ run(&mut executor, 5, 300, 30);
+ executor
+ .execute(&command(
+ "attach",
+ 6,
+ "run.attach",
+ json!({"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap();
+ run(&mut executor, 7, 100, 10);
+ executor.shutdown().unwrap();
+ fs::remove_dir_all(directory).unwrap();
+}
+
+#[test]
+fn late_codex_usage_requires_durable_reconciliation_before_or_after_handoff() {
+ for mode in ["before", "after", "after-paid", "unsafe"] {
+ let handoff_first = mode.starts_with("after");
+ let directory = temporary_directory("late-accounting-reconciliation");
+ let gate = directory.join("release-late-usage");
+ let gate_arg = gate.to_string_lossy().into_owned();
+ let config = provider_config(
+ &directory,
+ &[
+ "--durable-turn-ids",
+ "--warm-tail-accounting",
+ "--late-accounting-tail-gate",
+ &gate_arg,
+ if mode == "after-paid" {
+ "--hold-second-accounting-terminal"
+ } else {
+ "--ordinary-terminal"
+ },
+ if mode == "after-paid" {
+ "--late-accounting-duplicate-burst"
+ } else {
+ "--ordinary-tail-count"
+ },
+ if mode == "unsafe" {
+ "--unsafe-after-late-usage"
+ } else {
+ "--ordinary-late-usage"
+ },
+ ],
+ );
+ let runner_config = durable_config(&directory);
+ let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider":config,"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap();
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .unwrap();
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text":"Finish before the delayed accounting report."}),
+ ))
+ .unwrap();
+ let events = wait_for_executor_event_batch(&mut executor, "turn.completed");
+ assert_eq!(
+ events
+ .iter()
+ .filter(|event| event.event_type == "usage.reported")
+ .last()
+ .unwrap()
+ .payload["runDelta"]["inputTokens"],
+ 100
+ );
+ if handoff_first {
+ executor
+ .execute(&command(
+ "attach",
+ 4,
+ "run.attach",
+ json!({"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap();
+ let mut next = runner_config.clone();
+ next.run_id = "run-2".to_owned();
+ executor.rotate_authority(&next);
+ }
+ if mode == "after-paid" {
+ executor
+ .execute(&command(
+ "successor-turn",
+ 5,
+ "turn.start",
+ json!({"text":"Start successor paid work."}),
+ ))
+ .unwrap();
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ loop {
+ assert!(std::time::Instant::now() < deadline);
+ let events = poll_and_ack(&mut executor).unwrap();
+ assert!(!events
+ .iter()
+ .any(|event| event.event_type == "turn.completed"));
+ if let Some(usage) = events
+ .iter()
+ .find(|event| event.event_type == "usage.reported")
+ {
+ assert_eq!(usage.payload["runDelta"]["inputTokens"], 50);
+ break;
+ }
+ }
+ }
+ fs::write(&gate, "release").unwrap();
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ let retained = loop {
+ assert!(std::time::Instant::now() < deadline);
+ let events = if mode == "unsafe" {
+ match executor.execute(&command(
+ "quiesce",
+ 4,
+ "session.snapshot",
+ json!({"quiesceForWarmAttach":true}),
+ )) {
+ Ok(_) => {
+ std::thread::sleep(std::time::Duration::from_millis(1));
+ continue;
+ }
+ Err(error) => {
+ assert!(error
+ .to_string()
+ .contains("failed to retain completed Codex accounting"));
+ executor.retained_events().unwrap()
+ }
+ }
+ } else {
+ executor.poll_events().unwrap()
+ };
+ if events.iter().any(|event| {
+ event.event_type == "usage.reported"
+ || event.payload["code"] == "codex_usage_accounting_integrity"
+ }) {
+ break events;
+ }
+ executor.acknowledge_events(events.len()).unwrap();
+ };
+ if handoff_first {
+ assert!(retained
+ .iter()
+ .filter(|event| event.event_type == "usage.reported")
+ .all(|event| event.payload["runDeltaAvailable"] == false));
+ assert!(retained
+ .iter()
+ .any(|event| event.event_type == "usage.reported"
+ && event.payload["runDeltaAvailable"] == false));
+ assert!(retained
+ .iter()
+ .any(|event| event.event_type == "run.terminal"
+ && event.payload["runTerminalState"] == "failed"));
+ let required = retained
+ .iter()
+ .find(|event| event.event_type == "usage.reported")
+ .unwrap();
+ assert_eq!(
+ required.payload["reconciliation"]["cumulative"]["inputTokens"],
+ 120
+ );
+ assert_eq!(
+ required.payload["reconciliation"]["priorOrigin"]["runId"],
+ "run-1"
+ );
+ let evidence = retained
+ .iter()
+ .find(|event| event.payload["code"] == "codex_usage_accounting_integrity")
+ .unwrap();
+ assert_eq!(evidence.payload["priorOrigin"]["runId"], "run-1");
+ assert_eq!(evidence.payload["receivedTurnId"], "provider-turn-1");
+ assert_eq!(evidence.payload["cumulative"]["inputTokens"], 120);
+ } else {
+ assert_eq!(
+ retained
+ .iter()
+ .find(|event| event.event_type == "usage.reported")
+ .unwrap()
+ .payload["runDelta"]["inputTokens"],
+ 120
+ );
+ }
+ let saved: Value =
+ serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
+ .unwrap();
+ assert_eq!(saved["lifecycle"], "reconciliation_required");
+ assert!(!saved["pendingEvents"].as_array().unwrap().is_empty());
+ let error = executor
+ .execute(&command(
+ "blocked-attach",
+ 5,
+ "run.attach",
+ json!({"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("requires explicit reconciliation"));
+ executor.shutdown().unwrap();
+ drop(executor);
+ let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
+ let error = restored
+ .execute(&command(
+ "blocked-after-restart",
+ 6,
+ "run.attach",
+ json!({"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("requires explicit reconciliation"));
+ restored.shutdown().unwrap();
+ fs::remove_dir_all(directory).unwrap();
+ }
+}
+
+#[test]
+fn durable_codex_usage_keeps_completed_turn_through_reconciled_provider_exit() {
+ for exit_flag in [
+ "--exit-after-turn-completion",
+ "--fail-after-turn-completion",
+ ] {
+ let directory = temporary_directory("accounting-reconciled-exit");
+ let config = provider_config(
+ &directory,
+ &[
+ "--durable-turn-ids",
+ "--warm-tail-accounting",
+ "--emit-post-completion-passive-statuses",
+ exit_flag,
+ ],
+ );
+ let mut executor =
+ CodexCommandExecutor::with_runner_config(&directory, &durable_config(&directory));
+ executor
+ .execute(&command(
+ "prepare",
+ 1,
+ "run.prepare",
+ json!({"provider":config,"authorizedTools":task_context_tool_set()}),
+ ))
+ .unwrap();
+ executor
+ .execute(&command("open", 2, "session.open", json!({})))
+ .unwrap();
+ executor
+ .execute(&command(
+ "turn",
+ 3,
+ "turn.start",
+ json!({"text":"Complete accounting before exiting."}),
+ ))
+ .unwrap();
+ let events = wait_for_executor_event_batch(&mut executor, "turn.completed");
+ assert_eq!(
+ events
+ .iter()
+ .filter(|event| event.event_type == "usage.reported")
+ .last()
+ .unwrap()
+ .payload["runDelta"]["inputTokens"],
+ 120
+ );
+ assert!(!events.iter().any(|event| event.event_type == "turn.failed"));
+ let saved: Value =
+ serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
+ .unwrap();
+ assert_ne!(saved["lifecycle"], "reconciliation_required");
+ assert_eq!(saved["completedTurnAuthoritative"], true);
+ executor.shutdown().unwrap();
+ fs::remove_dir_all(directory).unwrap();
+ }
+}
diff --git a/packages/paperclip-runner/scripts/acpx-release-attestations.mjs b/packages/paperclip-runner/scripts/acpx-release-attestations.mjs
index fc55dd9ff0a..dc58149e0e7 100644
--- a/packages/paperclip-runner/scripts/acpx-release-attestations.mjs
+++ b/packages/paperclip-runner/scripts/acpx-release-attestations.mjs
@@ -3,13 +3,34 @@ import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
-// Copilot, Grok and Pi profile identities attest upstream releases, like Claude,
-// Codex, Cursor and Antigravity. They bind upstream versions and the bytes a
-// repackager copies verbatim: complete platform closures and Grok's launcher.
-// Never bind Paperclip source, the shared ACPX layer, absolute paths, the
-// provider-pack manifest digest or install layout. The profile's acpxVersion and
-// the pack contract digest bind ACPX and Runner source for every harness alike.
-export const ATTESTED_ACPX_AGENTS = Object.freeze(["copilot", "grok", "pi"]);
+/**
+ * How each ACPX provider's `commandDigest` attests its upstream release. Every
+ * profile in acpx-profiles.json declares one kind here:
+ *
+ * - `installed-command`: installation hashes the upstream package's launch
+ * command and compares it with the digest (Claude, Codex).
+ * - `launcher`: the SHA-256 of a launcher shipped verbatim (Grok).
+ * - `closure`: upstream versions and the complete platform closure pins.
+ * - `declaration`: a checked-in release declaration, hashed as canonical JSON.
+ *
+ * A digest changes only when the upstream release changes. It never binds
+ * Paperclip source, the shared ACPX layer, absolute paths, the provider-pack
+ * manifest digest or install layout: the profile's acpxVersion and the pack
+ * contract digest bind ACPX and Runner source for every provider alike.
+ * See rules/acpx-profile-identity.mdc.
+ */
+export const ACPX_RELEASE_ATTESTATIONS = Object.freeze({
+ claude: { kind: "installed-command" },
+ codex: { kind: "installed-command" },
+ grok: { kind: "launcher", path: "src/providers/grok/launcher.cjs" },
+ copilot: { kind: "closure", pins: { closure: ["copilot-closure-pins.ts", "COPILOT_CLOSURE_SHA256"] } },
+ pi: { kind: "closure", pins: {
+ nodeVersion: ["pi-node-pins.ts", "PI_NODE_VERSION"],
+ closure: ["pi-closure-pins.ts", "PI_DISTRIBUTION_CLOSURE_SHA256"],
+ } },
+ cursor: { kind: "declaration", path: "cursor-contract.json", digestField: "commandDigest" },
+ antigravity: { kind: "declaration", path: "antigravity-distributions.json" },
+});
const sha256 = bytes => createHash("sha256").update(bytes).digest("hex");
const canonicalJson = value => value && typeof value === "object"
@@ -17,28 +38,42 @@ const canonicalJson = value => value && typeof value === "object"
: `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
: JSON.stringify(value);
-/** The release attestation for one provider, computed from a runner package root. */
+/**
+ * The release attestation for one provider, computed from a runner package
+ * root, or null when installation verifies the digest instead.
+ */
export async function acpxReleaseAttestation(agent, packageRoot) {
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "acpx-profiles.json"), "utf8"));
const profile = manifest.profiles[agent];
- if (!ATTESTED_ACPX_AGENTS.includes(agent) || !profile) throw new Error(`No upstream release attestation for ${agent}`);
- // Grok keeps its physical byte-digest contract: admission hashes the launcher.
- if (agent === "grok") {
- return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, "src/providers/grok/launcher.cjs")))}` };
+ const attestation = Object.hasOwn(ACPX_RELEASE_ATTESTATIONS, agent) ? ACPX_RELEASE_ATTESTATIONS[agent] : undefined;
+ if (!profile || !attestation) {
+ throw new Error(`${agent} needs a release attestation: declare how its commandDigest attests the upstream release `
+ + "in ACPX_RELEASE_ATTESTATIONS (scripts/acpx-release-attestations.mjs). See rules/acpx-profile-identity.mdc.");
+ }
+ switch (attestation.kind) {
+ case "installed-command":
+ return null;
+ case "launcher":
+ return { commandDigest: `sha256:${sha256(readFileSync(resolve(packageRoot, attestation.path)))}` };
+ case "declaration": {
+ const { [attestation.digestField]: _digest, ...declaration } = JSON.parse(readFileSync(resolve(packageRoot, attestation.path), "utf8"));
+ return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
+ }
+ case "closure": {
+ const declaration = {
+ schema: "paperclip.acpx-release-attestation.v1", agent,
+ agentProfileVersion: profile.agentProfileVersion,
+ agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
+ agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
+ };
+ for (const [field, [file, name]] of Object.entries(attestation.pins)) {
+ // Key each import on content so a process never reuses pins from edited bytes.
+ const path = resolve(packageRoot, "src/drivers/acpx", file);
+ declaration[field] = (await import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`))[name];
+ }
+ return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
+ }
+ default:
+ throw new Error(`${agent} declares an unknown release attestation kind: ${attestation.kind}`);
}
- // Key each import on content so a process never reuses pins from edited bytes.
- const pins = async file => {
- const path = resolve(packageRoot, "src/drivers/acpx", file);
- return import(`${pathToFileURL(path).href}?sha256=${sha256(readFileSync(path))}`);
- };
- const declaration = {
- schema: "paperclip.acpx-release-attestation.v1", agent,
- agentProfileVersion: profile.agentProfileVersion,
- agentServerPackage: profile.agentServerPackage, agentServerVersion: profile.agentServerVersion,
- agentRuntimePackage: profile.agentRuntimePackage, agentRuntimeVersion: profile.agentRuntimeVersion,
- ...agent === "copilot"
- ? { closure: (await pins("copilot-closure-pins.ts")).COPILOT_CLOSURE_SHA256 }
- : { nodeVersion: (await pins("pi-node-pins.ts")).PI_NODE_VERSION, closure: (await pins("pi-closure-pins.ts")).PI_DISTRIBUTION_CLOSURE_SHA256 },
- };
- return { commandDigest: `sha256:${sha256(canonicalJson(declaration))}`, declaration };
}
diff --git a/packages/paperclip-runner/scripts/acpx-release-attestations.test.mjs b/packages/paperclip-runner/scripts/acpx-release-attestations.test.mjs
index c79aa0fb540..c95cabe00de 100644
--- a/packages/paperclip-runner/scripts/acpx-release-attestations.test.mjs
+++ b/packages/paperclip-runner/scripts/acpx-release-attestations.test.mjs
@@ -5,10 +5,12 @@ import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
-import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
+import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const manifest = JSON.parse(readFileSync(join(packageRoot, "acpx-profiles.json"), "utf8"));
+// Providers whose digest the generator computes; installation verifies the rest.
+const COMPUTED = Object.keys(manifest.profiles).filter(agent => ACPX_RELEASE_ATTESTATIONS[agent]?.kind !== "installed-command");
const packageFiles = [
"acpx-profiles.json", "package.json", "cursor-distributions.json", "cursor-contract.json", "antigravity-distributions.json",
"scripts/acpx-release-attestations.mjs", "scripts/generate-acpx-profiles.mjs",
@@ -33,15 +35,27 @@ function relocatedPackage(t) {
const generate = (root, ...args) => spawnSync(process.execPath, [join(root, "scripts/generate-acpx-profiles.mjs"), ...args],
{ cwd: root, env: { PATH: process.env.PATH }, encoding: "utf8", timeout: 20_000 });
-test("the manifest carries each provider's current release attestation", async () => {
- for (const agent of ATTESTED_ACPX_AGENTS) {
+test("every profile declares how its digest attests the upstream release", async () => {
+ assert.deepEqual(Object.keys(ACPX_RELEASE_ATTESTATIONS).sort(), Object.keys(manifest.profiles).sort());
+ assert.deepEqual(COMPUTED.sort(), ["antigravity", "copilot", "cursor", "grok", "pi"]);
+ for (const agent of COMPUTED) {
assert.equal(manifest.profiles[agent].commandDigest, (await acpxReleaseAttestation(agent, packageRoot)).commandDigest, agent);
}
});
+test("a new provider is told to declare its release attestation", async t => {
+ const relocated = relocatedPackage(t);
+ const path = join(relocated, "acpx-profiles.json"); const value = JSON.parse(readFileSync(path, "utf8"));
+ value.profiles.newagent = { ...value.profiles.codex };
+ writeFileSync(path, JSON.stringify(value));
+ const result = generate(relocated, "--check");
+ assert.notEqual(result.status, 0);
+ assert.match(result.stderr, /newagent needs a release attestation: declare how its commandDigest attests the upstream release/u);
+});
+
test("a relocated copy of the provider inputs computes the same attestations", async t => {
const relocated = relocatedPackage(t);
- for (const agent of ATTESTED_ACPX_AGENTS) {
+ for (const agent of COMPUTED) {
const here = await acpxReleaseAttestation(agent, packageRoot);
const there = await acpxReleaseAttestation(agent, relocated);
assert.equal(there.commandDigest, here.commandDigest, agent);
@@ -76,24 +90,52 @@ for (const [name, agents, edit] of [
const path = join(root, "src/providers/grok/launcher.cjs");
writeFileSync(path, `${readFileSync(path, "utf8")}\n// changed\n`);
}],
+ ["the Cursor release declaration", ["cursor"], root => {
+ const path = join(root, "cursor-contract.json"); const value = JSON.parse(readFileSync(path, "utf8"));
+ value.nativePatch = "changed"; writeFileSync(path, JSON.stringify(value));
+ }],
+ ["the Antigravity distribution declaration", ["antigravity"], root => {
+ const path = join(root, "antigravity-distributions.json"); const value = JSON.parse(readFileSync(path, "utf8"));
+ value.version = "9.9.9"; writeFileSync(path, JSON.stringify(value));
+ }],
]) test(`changing ${name} changes only the affected attestations`, async t => {
const relocated = relocatedPackage(t);
- const before = Object.fromEntries(await Promise.all(ATTESTED_ACPX_AGENTS.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
+ const before = Object.fromEntries(await Promise.all(COMPUTED.map(async agent => [agent, (await acpxReleaseAttestation(agent, relocated)).commandDigest])));
edit(relocated);
- for (const agent of ATTESTED_ACPX_AGENTS) {
+ for (const agent of COMPUTED) {
const after = (await acpxReleaseAttestation(agent, relocated)).commandDigest;
if (agents.includes(agent)) assert.notEqual(after, before[agent], agent);
else assert.equal(after, before[agent], agent);
}
});
+test("the generator refreshes a release declaration's own digest with the profile", async t => {
+ const relocated = relocatedPackage(t);
+ const path = join(relocated, "cursor-contract.json"); const contract = JSON.parse(readFileSync(path, "utf8"));
+ contract.modes = [...contract.modes, "review"];
+ writeFileSync(path, `${JSON.stringify(contract, null, 2)}\n`);
+ const stale = generate(relocated, "--check");
+ assert.notEqual(stale.status, 0);
+ assert.match(stale.stderr, /cursor commandDigest no longer matches its upstream release attestation/u);
+ assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), contract, "check mode writes nothing");
+ const refreshed = generate(relocated);
+ assert.equal(refreshed.status, 0, refreshed.stderr);
+ const { commandDigest } = await acpxReleaseAttestation("cursor", relocated);
+ assert.notEqual(commandDigest, manifest.profiles.cursor.commandDigest);
+ assert.equal(JSON.parse(readFileSync(path, "utf8")).commandDigest, commandDigest);
+ assert.equal(JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8")).profiles.cursor.commandDigest, commandDigest);
+ assert.match(readFileSync(join(relocated, "src/drivers/acpx/generated-profiles.ts"), "utf8"), new RegExp(commandDigest));
+ assert.match(readFileSync(join(relocated, "runner/crates/runner-core/src/generated_acpx_profiles.rs"), "utf8"), new RegExp(commandDigest));
+ assert.equal(generate(relocated, "--check").status, 0);
+});
+
test("the generator refuses a stale attestation in check mode and refreshes it otherwise", async t => {
const relocated = relocatedPackage(t);
const launcher = join(relocated, "src/providers/grok/launcher.cjs");
writeFileSync(launcher, `${readFileSync(launcher, "utf8")}\n// changed\n`);
const stale = generate(relocated, "--check");
assert.notEqual(stale.status, 0);
- assert.match(stale.stderr, /grok release attestation is stale/u);
+ assert.match(stale.stderr, /grok commandDigest no longer matches its upstream release attestation/u);
const refreshed = generate(relocated);
assert.equal(refreshed.status, 0, refreshed.stderr);
const written = JSON.parse(readFileSync(join(relocated, "acpx-profiles.json"), "utf8"));
diff --git a/packages/paperclip-runner/scripts/generate-acpx-profiles.mjs b/packages/paperclip-runner/scripts/generate-acpx-profiles.mjs
index 4ec70598b78..7b8d5fb59ad 100644
--- a/packages/paperclip-runner/scripts/generate-acpx-profiles.mjs
+++ b/packages/paperclip-runner/scripts/generate-acpx-profiles.mjs
@@ -1,30 +1,36 @@
import assert from "node:assert/strict";
-import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
-import { acpxReleaseAttestation, ATTESTED_ACPX_AGENTS } from "./acpx-release-attestations.mjs";
+import { ACPX_RELEASE_ATTESTATIONS, acpxReleaseAttestation } from "./acpx-release-attestations.mjs";
const root = new URL("../", import.meta.url);
const readJson = async path => JSON.parse(await readFile(new URL(path, root), "utf8"));
const manifest = await readJson("acpx-profiles.json");
const distributions = await readJson("cursor-distributions.json");
const antigravityDistributions = await readJson("antigravity-distributions.json");
-const contract = await readJson("cursor-contract.json");
const pkg = await readJson("package.json");
assert.equal(manifest.schema, "paperclip.acpx-profiles.v1");
assert.equal(manifest.acpxVersion, pkg.dependencies.acpx);
-// Copilot, Grok and Pi digests attest upstream releases; they change only when
-// an upstream version, a platform closure or the Grok launcher changes.
+// Every profile's commandDigest attests its upstream release, so it changes only
+// when that release changes. Installation verifies installed-command digests.
const check = process.argv.includes("--check");
-let attestationsChanged = false;
-for (const agent of ATTESTED_ACPX_AGENTS) {
- const { commandDigest } = await acpxReleaseAttestation(agent, fileURLToPath(root));
- if (manifest.profiles[agent].commandDigest === commandDigest) continue;
- assert.ok(!check, `${agent} release attestation is stale in acpx-profiles.json; run generate:acpx-profiles`);
- manifest.profiles[agent].commandDigest = commandDigest;
- attestationsChanged = true;
+const writes = new Map();
+for (const agent of Object.keys(manifest.profiles)) {
+ const attestation = await acpxReleaseAttestation(agent, fileURLToPath(root));
+ if (!attestation) continue;
+ // A release declaration may carry its own copy of the digest; keep it in step.
+ const { path, digestField } = ACPX_RELEASE_ATTESTATIONS[agent];
+ const declaration = digestField ? await readJson(path) : null;
+ if (manifest.profiles[agent].commandDigest === attestation.commandDigest
+ && (!declaration || declaration[digestField] === attestation.commandDigest)) continue;
+ assert.ok(!check, `${agent} commandDigest no longer matches its upstream release attestation. `
+ + "Run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`; an upstream change keeps the profile version.");
+ manifest.profiles[agent].commandDigest = attestation.commandDigest;
+ writes.set("acpx-profiles.json", manifest);
+ if (declaration) writes.set(path, { ...declaration, [digestField]: attestation.commandDigest });
}
-if (attestationsChanged) await writeFile(new URL("acpx-profiles.json", root), `${JSON.stringify(manifest, null, 2)}\n`);
+for (const [path, value] of writes) await writeFile(new URL(path, root), `${JSON.stringify(value, null, 2)}\n`);
+const contract = await readJson("cursor-contract.json");
const entries = Object.entries(manifest.profiles);
for (const [agent, profile] of entries) {
assert.match(agent, /^[a-z]+$/);
@@ -38,21 +44,13 @@ for (const [agent, profile] of entries) {
if (name && Object.hasOwn(pkg.dependencies, name)) assert.equal(version, pkg.dependencies[name], `${agent}: installed dependency drift`);
}
}
-// The Cursor contract is an immutable release attestation, not another runtime
-// lookup table. Refuse generation if a release declaration disagrees with it.
+// Release declarations must agree with the profiles they attest.
assert.equal(manifest.profiles.cursor.agentServerVersion, distributions.version);
assert.equal(manifest.profiles.cursor.agentServerVersion, contract.vendor);
assert.equal(manifest.profiles.cursor.agentProfileVersion, contract.profileVersion);
-assert.equal(manifest.profiles.cursor.commandDigest, contract.commandDigest);
+assert.equal(contract.commandDigest, manifest.profiles.cursor.commandDigest, "cursor-contract.json carries a stale commandDigest");
assert.equal(distributions.patchVersion, contract.nativePatch);
-const canonicalJson = value => value && typeof value === "object"
- ? Array.isArray(value) ? `[${value.map(canonicalJson).join(",")}]`
- : `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`
- : JSON.stringify(value);
assert.equal(manifest.profiles.antigravity.agentServerVersion, antigravityDistributions.version);
-assert.equal(manifest.profiles.antigravity.commandDigest, `sha256:${createHash("sha256").update(canonicalJson(antigravityDistributions)).digest("hex")}`, "Antigravity distribution identity is stale");
-const { commandDigest, ...attestation } = contract;
-assert.equal(commandDigest, `sha256:${createHash("sha256").update(canonicalJson(attestation)).digest("hex")}`);
for (const distribution of Object.values(distributions.platforms)) assert.match(distribution.closureSha256, /^[a-f0-9]{64}$/);
const quote = JSON.stringify;
diff --git a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.test.ts b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.test.ts
index eb2848c2c6e..d122058e4de 100644
--- a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.test.ts
+++ b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.test.ts
@@ -2,6 +2,23 @@ import { describe, expect, it } from "vitest";
import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js";
import { isSupportedAcpxProfileVersion } from "./profile-compatibility.js";
+// A profile version changes only to reset saved sessions on purpose. An upstream
+// release changes the commandDigest and keeps the version; a new provider starts
+// at 1. Update this table, with the reason, only for a deliberate reset.
+const PROFILE_VERSIONS: Readonly> = {
+ antigravity: 1, claude: 1, codex: 1, copilot: 92, cursor: 17, grok: 20, pi: 37,
+};
+
+describe("ACPX profile versions", () => {
+ it("change only for a deliberate session reset", () => {
+ for (const [agent, profile] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
+ expect(profile.agentProfileVersion, `${agent}: a profile version bump resets every saved ${agent} session. `
+ + "An upstream release changes only the commandDigest. See rules/acpx-profile-identity.mdc.")
+ .toBe(PROFILE_VERSIONS[agent] ?? 1);
+ }
+ });
+});
+
describe("historical ACPX profile decoding", () => {
it("decodes every revision from 1 through the current one so saved runs reach recovery", () => {
for (const [agent, current] of Object.entries(QUALIFIED_ACPX_PROFILE_DATA)) {
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
index c09a2c50ea0..4b351a656f3 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
@@ -1123,6 +1123,43 @@ it("refuses a reusable close checkpoint when the local provider snapshot is unre
}
}, 15_000);
+it("installs required accounting acceptance before the Runner acknowledgement callback returns", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "runnerd-required-accounting-"));
+ const OriginalCore = durableControlPlane.DurablePrpControlPlane;
+ let callbacks!: ConstructorParameters[0];
+ const spy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) {
+ callbacks = options;
+ return new OriginalCore(options);
+ });
+ let release!: () => void;
+ const accepted = new Promise(resolve => { release = resolve; });
+ const onAccountingEvent = vi.fn(async () => accepted);
+ const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: directory, sourceCodexHome: "", environment: {},
+ runnerBinary: process.execPath, runnerArtifact: { version: "test", digest: `sha256:${"a".repeat(64)}` },
+ codexCommand: process.execPath, codexArgs: [], onAccountingEvent,
+ controlPlaneRegistration: async () => { throw new Error("stop before Runner dispatch"); },
+ });
+ try {
+ await expect(bundle.transport.request("thread/start", { cwd: directory })).rejects.toThrow();
+ const report = { eventType: "usage.reported", sourceEventId: "required-usage" } as PrpEvent;
+ let returned = false;
+ const result = callbacks.onCommittedEvent!(report).then(() => { returned = true; });
+ await vi.waitFor(() => expect(onAccountingEvent).toHaveBeenCalledExactlyOnceWith(report));
+ expect(returned).toBe(false);
+ release();
+ await result;
+ expect(returned).toBe(true);
+ const failure = new Error("receipt unavailable");
+ onAccountingEvent.mockRejectedValueOnce(failure);
+ await expect(callbacks.onCommittedEvent!(report)).rejects.toBe(failure);
+ } finally {
+ release();
+ await bundle.transport.close().catch(() => undefined);
+ spy.mockRestore();
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
it("uses the same API result envelope for live dispatch and recovered receipts", async () => {
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-recovered-api-"));
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
index d994f6466de..23ab1f1721e 100644
--- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
+++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts
@@ -1240,6 +1240,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD
provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx";
observeCredentials?: boolean;
onCredentialsUpdated?: (event: PrpEvent) => Promise;
+ onAccountingEvent?: (event: PrpEvent) => Promise;
recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"];
opencodePermissionMode?: NativeOpenCodePermissionMode;
acpxAgent?: QualifiedAcpxAgent;
@@ -5239,6 +5240,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
onSemanticToolInput: (call) => this.#handleSemanticToolInput(call),
recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call),
onCommittedEvent: async (event) => {
+ await this.options.onAccountingEvent?.(event);
if (event.eventType === "auth.credentials.updated") {
await this.options.onCredentialsUpdated?.(event);
}
@@ -5691,6 +5693,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
onSemanticToolInput: (call) => this.#handleSemanticToolInput(call),
recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call),
onCommittedEvent: async (event) => {
+ await this.options.onAccountingEvent?.(event);
if (event.eventType === "auth.credentials.updated") {
await this.options.onCredentialsUpdated?.(event);
}
diff --git a/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts b/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts
index 2720e4a5326..92905627855 100644
--- a/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts
+++ b/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts
@@ -3385,6 +3385,75 @@ export const usageSchema = {
"runDeltaAvailable": {
"type": "boolean"
},
+ "runDeltaBasis": {
+ "const": "run"
+ },
+ "reconciliation": {
+ "type": "object",
+ "required": [
+ "reason",
+ "receivedTurnId",
+ "cumulative",
+ "priorOrigin"
+ ],
+ "properties": {
+ "reason": {
+ "const": "accounting_reconciliation_required"
+ },
+ "receivedTurnId": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "maxLength": 240
+ },
+ "cumulative": {
+ "$ref": "#/$defs/measurement"
+ },
+ "priorOrigin": {
+ "type": [
+ "object",
+ "null"
+ ],
+ "required": [
+ "runnerInstanceId",
+ "runId",
+ "normalizedSessionId",
+ "turnId",
+ "itemId"
+ ],
+ "properties": {
+ "runnerInstanceId": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 240
+ },
+ "runId": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 240
+ },
+ "normalizedSessionId": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 240
+ },
+ "turnId": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 240
+ },
+ "itemId": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 240
+ }
+ },
+ "additionalProperties": false
+ }
+ },
+ "additionalProperties": false
+ },
"runDelta": {
"$ref": "#/$defs/measurement"
}
diff --git a/packages/paperclip-runner/src/protocol/replay-contract.test.ts b/packages/paperclip-runner/src/protocol/replay-contract.test.ts
index 3f17ecf2b62..bad4d2536ee 100644
--- a/packages/paperclip-runner/src/protocol/replay-contract.test.ts
+++ b/packages/paperclip-runner/src/protocol/replay-contract.test.ts
@@ -1,3 +1,5 @@
+import Ajv2020 from "ajv/dist/2020.js";
+import { usageSchema } from "./generated/schema-bundle.js";
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
@@ -6,6 +8,7 @@ import {
negotiateProtocolVersion,
parsePrpFixtureText,
PRP_PROTOCOL_VERSION,
+ validatePrpEvent,
} from "./replay-contract.js";
import { reducePrpFixture } from "../reducer/session-reducer.js";
@@ -371,3 +374,24 @@ describe("PRP v1 JSON Schema contract", () => {
).toBeNull();
});
});
+
+it("accepts marked whole-run usage through the runtime standalone event validator", async () => {
+ const fixture = await readFixture();
+ const event = (fixture.events as Array>)[0];
+ const measurement = { inputTokens: 120, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 0,
+ activeSeconds: 0, requests: 1, providerCostUsd: 0 };
+ const result = validatePrpEvent({ ...event, eventType: "usage.reported", priority: 0,
+ payload: { provider: "codex", model: "gpt-6.1-sol", cumulative: measurement, runDelta: measurement,
+ runDeltaAvailable: true, runDeltaBasis: "run" } });
+ expect(result).toMatchObject({ ok: true });
+});
+
+it("declares required incomplete reconciliation evidence in the strict usage contract", () => {
+ const validate = new Ajv2020({ strict: true, allErrors: true }).compile(usageSchema);
+ const measurement = { inputTokens: 120, outputTokens: 12, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 0, providerCostUsd: 0 };
+ const payload = { provider: "codex", model: null, cumulative: measurement, runDelta: { ...measurement, inputTokens: 0, outputTokens: 0 }, runDeltaAvailable: false, runDeltaBasis: "run",
+ reconciliation: { reason: "accounting_reconciliation_required", receivedTurnId: "provider-turn-1", cumulative: measurement,
+ priorOrigin: { runnerInstanceId: "runner-1", runId: "old-run", normalizedSessionId: "session", turnId: "turn", itemId: "item" } } };
+ expect(validate(payload), JSON.stringify(validate.errors)).toBe(true);
+ expect(validate({ ...payload, reconciliation: { ...payload.reconciliation, arbitraryProviderData: "excluded" } })).toBe(false);
+});
diff --git a/packages/plugins/sandbox-providers/daytona/package.json b/packages/plugins/sandbox-providers/daytona/package.json
index ddc5acd934f..84007584468 100644
--- a/packages/plugins/sandbox-providers/daytona/package.json
+++ b/packages/plugins/sandbox-providers/daytona/package.json
@@ -1,6 +1,6 @@
{
"name": "@paperclipai/plugin-daytona",
- "version": "0.1.1",
+ "version": "0.1.9",
"description": "Daytona sandbox provider plugin for Paperclip environments",
"license": "MIT",
"homepage": "https://github.com/paperclipai/paperclip",
diff --git a/packages/plugins/sandbox-providers/kubernetes/package.json b/packages/plugins/sandbox-providers/kubernetes/package.json
index d1238aa7073..057f6685b8b 100644
--- a/packages/plugins/sandbox-providers/kubernetes/package.json
+++ b/packages/plugins/sandbox-providers/kubernetes/package.json
@@ -1,6 +1,6 @@
{
"name": "@paperclipai/plugin-kubernetes",
- "version": "0.1.0",
+ "version": "0.1.0-alpha.1",
"description": "Kubernetes sandbox provider plugin for Paperclip environments",
"license": "MIT",
"homepage": "https://github.com/paperclipai/paperclip",
diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts
index 77c4479d15c..160e0a887f3 100644
--- a/packages/shared/src/index.ts
+++ b/packages/shared/src/index.ts
@@ -171,6 +171,8 @@ export type {
DecisionHistoryPage,
AttentionDetailImage,
AttentionFeed,
+ AttentionSummary,
+ AttentionTaskIndex,
AttentionItem,
AttentionItemDetail,
AttentionItemDismissal,
diff --git a/packages/shared/src/types/attention.ts b/packages/shared/src/types/attention.ts
index 6b290998ea3..4978a893caa 100644
--- a/packages/shared/src/types/attention.ts
+++ b/packages/shared/src/types/attention.ts
@@ -275,6 +275,13 @@ export interface AttentionFeed {
countsBySourceKind: Record;
items: AttentionItem[];
}
+
+/** Canonical attention counts without downloading the decision cards. */
+export type AttentionSummary = Omit;
+/** Complete accessible task membership for Tasks → Needs attention. */
+export interface AttentionTaskIndex extends AttentionSummary {
+ taskIds: string[];
+}
/** Read-only projection of canonical task decisions; never a second write store. */
export interface DecisionHistoryItem {
id: string;
diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts
index 5feb5cdaad2..f5edbfca2c3 100644
--- a/packages/shared/src/types/index.ts
+++ b/packages/shared/src/types/index.ts
@@ -45,6 +45,8 @@ export type {
DecisionHistoryPage,
AttentionDetailImage,
AttentionFeed,
+ AttentionSummary,
+ AttentionTaskIndex,
AttentionItem,
AttentionItemDetail,
AttentionItemDismissal,
diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts
index ada9938813b..6c4c07c36cd 100644
--- a/packages/shared/src/types/issue.ts
+++ b/packages/shared/src/types/issue.ts
@@ -1779,6 +1779,7 @@ export interface IssueAttachment {
/** Bounded cold-open window; the complete chronological feed stays paginated. */
export interface IssueRecentComments {
latestCommentId: string | null;
+ oldestCommentId?: string | null;
comments: IssueComment[];
hasMore: boolean;
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index dc06e807394..c55bccca790 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -68,6 +68,9 @@ importers:
esbuild:
specifier: ^0.28.2
version: 0.28.2
+ oxc-parser:
+ specifier: 0.127.0
+ version: 0.127.0
turbo:
specifier: 2.11.10
version: 2.11.10
diff --git a/rules/acpx-profile-identity.mdc b/rules/acpx-profile-identity.mdc
new file mode 100644
index 00000000000..b27c138a771
--- /dev/null
+++ b/rules/acpx-profile-identity.mdc
@@ -0,0 +1,72 @@
+---
+description: Give every ACPX provider the same identity model - its commandDigest attests the upstream release, the pack contract digest covers Runner source and the shared ACPX layer, and profile versions change only for deliberate resets.
+globs: "packages/paperclip-runner/acpx-profiles.json,packages/paperclip-runner/cursor-contract.json,packages/paperclip-runner/cursor-distributions.json,packages/paperclip-runner/antigravity-distributions.json,packages/paperclip-runner/scripts/acpx-release-attestations.mjs,packages/paperclip-runner/scripts/generate-acpx-profiles.mjs,packages/paperclip-runner/scripts/contracts/**,packages/paperclip-runner/src/drivers/acpx/**,packages/paperclip-runner/src/providers/**,packages/paperclip-runner/protocol/**"
+alwaysApply: false
+---
+
+# ACPX profile identity
+
+## Scope and purpose
+
+This rule covers ACPX provider profiles, their release declarations and pins,
+the profile generator, the provider-pack contract registry, and the ACPX
+driver and provider code. Every ACPX harness (Claude, Codex, Copilot, Cursor,
+Grok, Pi, Antigravity and any new one) uses the same identity model, so a
+change behaves the same way for all of them.
+
+## The identity model
+
+A provider profile in `acpx-profiles.json` has a version and a `commandDigest`.
+Three things decide whether saved work and installed packs stay compatible:
+
+| What changed | What carries it | Effect |
+| --- | --- | --- |
+| The provider's upstream release: package versions, platform closures, a launcher shipped verbatim | The profile's `commandDigest` | Saved sessions for that provider start fresh |
+| The shared ACPX layer | The profile's `acpxVersion` and the Runner's ACPX patch in the pack `contractDigest` | The same for every provider |
+| A runner/pack contract: schemas, generated catalogs, Rust shapes and env keys | The provider pack's `contractDigest` | Older image packs are not reused |
+
+Ordinary Runner source changes, refactors and comments change none of these.
+
+## Required behavior
+
+- **Attest the upstream release.** Declare each provider's attestation in
+ `ACPX_RELEASE_ATTESTATIONS` (`scripts/acpx-release-attestations.mjs`), using
+ the kind that fits: `installed-command`, `launcher`, `closure` or
+ `declaration`. The digest binds only what a repackager copies verbatim from
+ the upstream release, so the same release gives the same digest wherever the
+ pack is built or installed.
+- **Keep Paperclip source and the ACPX layer out of provider digests.**
+ Runner/pack compatibility belongs to the `contractDigest`. When Runner code
+ and a pack must agree on a new field or behavior, declare it in a schema
+ under `protocol/` and generate the TypeScript, so the contract registry
+ covers it.
+- **Regenerate instead of hand-editing.** After an upstream bump or a pin
+ change, run `pnpm --filter @paperclipai/paperclip-runner generate:acpx-profiles`.
+ `check:acpx-profiles` runs in the build and refuses a stale digest.
+- **Keep the profile version.** An upstream release changes the digest, not the
+ version. A new provider starts at version 1. Bump a version only to reset
+ every saved session for that provider on purpose, such as a change in saved
+ session semantics that exact admission cannot detect. Record the reason in
+ the PR and in the version table in `profile-compatibility.test.ts`.
+- **Keep earlier versions decodable.** Saved inputs with any version from 1
+ through the current one decode, so recovery can refuse them. Launch and
+ recovery admission require the exact current profile and digest.
+
+## Adding a provider
+
+1. Add its profile at version 1 to `acpx-profiles.json`.
+2. Add its attestation to `ACPX_RELEASE_ATTESTATIONS`, with its pins in
+ erasable TypeScript or JSON that the generator can read.
+3. Run `generate:acpx-profiles`, then `node --test scripts/acpx-release-attestations.test.mjs`.
+
+## Review criteria
+
+- Does every provider digest change only with its upstream release?
+- Is a new runner/pack contract declared where the `contractDigest` reads it?
+- Does a version bump state which saved sessions it resets, and why?
+- Does the change treat every ACPX harness the same way?
+
+The canonical reference is
+[`packages/paperclip-runner/docs/durable-recovery.md`](../packages/paperclip-runner/docs/durable-recovery.md#profile-identity).
+The pack contract digest is described in
+[`docker/daytona-runner/README.md`](../docker/daytona-runner/README.md).
diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs
index 9f16cb86763..4d2296345bd 100644
--- a/scripts/__tests__/e2e-shard.test.mjs
+++ b/scripts/__tests__/e2e-shard.test.mjs
@@ -249,7 +249,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
]) {
assert.match(
jobs.get(jobId),
- /^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' && /m,
+ /^ {4}if: \$\{\{ (?:always\(\) && !cancelled\(\) && needs\.gate\.result == 'success' && )?needs\.gate\.outputs\.full_ci == 'true' && /m,
`${jobId} must run only when the gate selects full CI`,
);
}
@@ -263,10 +263,14 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
const verify = jobs.get("verify");
assert.match(
verify,
- /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
+ /^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m,
);
assert.equal(runAggregate(verify, true).status, 0);
assert.equal(runAggregate(verify, false).status, 0);
+ for (const result of ["failure", "cancelled", "skipped"]) {
+ assert.equal(runAggregate(verify, true, { test_selection: result }).status, 0,
+ "an optional selector cannot block the successful original matrix");
+ }
for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner",
"build", "docker_context_integrity", "verify_serialized_server", "canary_dry_run"]) {
for (const result of ["failure", "cancelled", "skipped"]) {
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index 3d4ddd3fd5f..8efc8264bed 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -579,6 +579,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("runner-full-stack-e2e.yml"),
readWorkflow("e2e.yml"),
readWorkflow("runner-protocol-live-evals.yml"),
+ readWorkflow("chat-feature-testops.yml"),
];
for (const workflow of actionPinWorkflows) {
@@ -618,6 +619,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
"runner-protocol-live-evals.yml",
+ "chat-feature-testops.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
const providerSecretReference =
@@ -650,6 +652,38 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
(block) => [...block.matchAll(providerSecretReference)].length > 0,
);
assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`);
+ if (name === "chat-feature-testops.yml") {
+ assert.equal(providerJobs.length, 1);
+ const assertBoundary = (block) => {
+ assert.match(block, /^ chat:\n/);
+ assert.match(block, /\n needs: \[authorize, select, build\]\n/);
+ assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
+ assert.match(block, /\n environment: runner-e2e-private-paid\n/);
+ assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
+ assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/);
+ assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
+ assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/);
+ assert.doesNotMatch(block, /ALLURE_TOKEN/);
+ assert.equal([...block.matchAll(providerSecretReference)].length, 1);
+ assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
+ assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference));
+ };
+ const block = providerJobs[0];
+ assertBoundary(block);
+ assert.equal([...workflow.matchAll(providerSecretReference)].length, 1);
+ assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/);
+ assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/);
+ assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/);
+ for (const mutant of [
+ block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
+ block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
+ block.replace(" - *guard\n", ""),
+ block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"),
+ block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"),
+ block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"),
+ ]) assert.throws(() => assertBoundary(mutant));
+ continue;
+ }
for (const block of providerJobs) {
if (name === "runner-protocol-live-evals.yml") {
assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}"));
diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs
index c4dd568a72f..52209944424 100644
--- a/scripts/__tests__/run-vitest-stable-shard.test.mjs
+++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs
@@ -320,6 +320,7 @@ test("the real serialized shard partition is complete, non-overlapping and durat
const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts";
const nativeRunnerSuitePath =
"server/src/services/native-runtime/native-codex-runner.integration.test.ts";
+const usageDeliverySuitePath = "server/src/services/native-runtime/native-usage-delivery.integration.test.ts";
const dotRunnerSuitePath = "server/src/__tests__/dot-runner.test.ts";
const restartRecoverySuitePath =
"server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts";
@@ -363,7 +364,8 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes
assert.ok(!files.includes(nativeRunnerSuitePath));
assert.ok(!files.includes(dotRunnerSuitePath));
assert.ok(!files.includes(restartRecoverySuitePath));
- assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort());
+ assert.ok(!files.includes(usageDeliverySuitePath));
+ assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, usageDeliverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort());
assert.equal(new Set(files).size, files.length);
const defaultRun = dryRunJson([], prEnv);
assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);
@@ -412,6 +414,7 @@ test("the native-runner lane runs exactly the cargo-dependent suites", () => {
"server/src/services/native-runtime/native-codex-runner.integration.test.ts",
dotRunnerSuitePath,
restartRecoverySuitePath,
+ usageDeliverySuitePath,
]);
});
diff --git a/scripts/ci/cache.integration.test.mjs b/scripts/ci/cache.integration.test.mjs
index c7fddcad772..8aebc6d92e3 100644
--- a/scripts/ci/cache.integration.test.mjs
+++ b/scripts/ci/cache.integration.test.mjs
@@ -10,6 +10,63 @@ import { executionPlan, sourceFiles } from "./execution-plan.mjs";
const repositoryRoot = new URL("../../", import.meta.url);
const turbo = process.env.PAPERCLIP_CI_TEST_TURBO ?? new URL("node_modules/.bin/turbo", repositoryRoot).pathname;
+test("real Turbo keeps full runs and different selected test sets in separate cache entries", () => {
+ const directory = mkdtempSync(path.join(tmpdir(), "ci-selection-cache-"));
+ const write = (file, value) => writeFileSync(path.join(directory, file), value);
+ const json = (file, value) => write(file, JSON.stringify(value));
+ const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
+ try {
+ git("init", "-q");
+ for (const folder of ["scripts/ci", "server/src", "packages/fixture", "bin", ".ci-test-selection"]) mkdirSync(path.join(directory, folder), { recursive: true });
+ for (const file of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
+ copyFileSync(new URL(`scripts/ci/${file}`, repositoryRoot), path.join(directory, "scripts/ci", file));
+ }
+ mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
+ copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
+ write(".gitignore", ".turbo/\n.ci-test-selection/\nresult.txt\ncalls.txt\n");
+ json("package.json", { name: "selection-cache-proof", private: true, packageManager: "npm@10.0.0", workspaces: ["packages/*"], scripts: { "test:run:general": "node check.cjs" } });
+ json("packages/fixture/package.json", { name: "fixture", private: true });
+ json("package-lock.json", { name: "selection-cache-proof", lockfileVersion: 3, packages: { "": { name: "selection-cache-proof" } } });
+ json("turbo.json", { globalDependencies: ["doc/**", "docs/**"], globalEnv: ["CI_TASK_CACHE_SCOPE", "PAPERCLIP_*"], tasks: {
+ "//#ci:server": { inputs: ["check.cjs", "server/**"], outputs: ["result.txt"], cache: true },
+ } });
+ mkdirSync(path.join(directory, "doc/plans"), { recursive: true });
+ mkdirSync(path.join(directory, "docs/api"), { recursive: true });
+ write("doc/DEVELOPING.md", "before"); write("doc/plans/example.md", "before");
+ write("docs/api/issues.md", "before");
+ write("server/src/a.test.ts", "// a"); write("server/src/b.test.ts", "// b");
+ write("check.cjs", `const fs=require('node:fs');fs.appendFileSync('calls.txt','call\\n');const file=process.env.PAPERCLIP_CI_TEST_SELECTION;fs.writeFileSync('result.txt',file?JSON.parse(fs.readFileSync(file)).lanes['general-server-without-chat'].shards[0]['@paperclipai/server'].join(','):'full');`);
+ write("bin/pnpm", `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const args=process.argv.slice(2);if(args[0]==='--version')console.log('9.15.4');else{const r=spawnSync(${JSON.stringify(turbo)},args.slice(2),{stdio:'inherit'});process.exit(r.status??1);}`);
+ chmodSync(path.join(directory, "bin/pnpm"), 0o755);
+ git("add", ".");
+ git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qm", "fixture");
+ const head = git("rev-parse", "HEAD");
+ const calls = () => readFileSync(path.join(directory, "calls.txt"), "utf8").trim().split("\n").length;
+ const run = (testFile) => {
+ const selection = path.join(directory, ".ci-test-selection/manifest.json");
+ json(".ci-test-selection/manifest.json", { version: 1, head, lanes: {
+ "general-server-without-chat": { shards: [{ "@paperclipai/server": [testFile ?? "server/src/a.test.ts"] }] },
+ } });
+ const result = spawnSync(process.execPath, [path.join(directory, "scripts/ci/run-task.mjs")], {
+ cwd: directory, encoding: "utf8", env: { ...process.env, PATH: `${directory}/bin:${process.env.PATH}`,
+ CI_TASK: "ci:server", CI_TASK_ARGS: "[]", CI_FORCE_TASKS: "false", CI_TASK_DRY_RUN: "false",
+ PAPERCLIP_CI_TEST_SELECTION: testFile ? selection : "", PAPERCLIP_CI_SELECTION_HASH: "",
+ TURBO_TOKEN: "", TURBO_TEAM: "", TURBO_TELEMETRY_DISABLED: "1" },
+ });
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ assert.equal(readFileSync(path.join(directory, "result.txt"), "utf8"), testFile ?? "full");
+ };
+ run(); run(); assert.equal(calls(), 1);
+ write("doc/DEVELOPING.md", "after"); write("doc/plans/example.md", "after");
+ run(); assert.equal(calls(), 1, "audited prose does not invalidate product test results");
+ run("server/src/a.test.ts"); run("server/src/a.test.ts"); assert.equal(calls(), 2);
+ run("server/src/b.test.ts"); assert.equal(calls(), 3);
+ run(); assert.equal(calls(), 3, "the original full result is reusable without accepting a subset result");
+ write("docs/api/issues.md", "after");
+ run(); assert.equal(calls(), 4, "runtime documentation must invalidate results");
+ } finally { rmSync(directory, { recursive: true, force: true }); }
+});
+
test("every cached lane maps to an existing package script with complete inputs", () => {
const config = JSON.parse(readFileSync(new URL("turbo.json", repositoryRoot), "utf8"));
for (const [name, definition] of Object.entries(config.tasks).filter(([, task]) => task.cache)) {
@@ -35,6 +92,8 @@ test("real Turbo executes existing suite scripts and invalidates source, argumen
const json = (name, value) => write(name, JSON.stringify(value));
try {
execFileSync("git", ["init", "-q"], { cwd: directory });
+ mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
+ copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\ngenerated/\nresult.txt\ncalls.txt\n");
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
json("packages/fixture/package.json", { name: "fixture", private: true });
@@ -83,9 +142,11 @@ test("the task launcher bypasses cache reads and writes when execution is forced
mkdirSync(path.join(directory, "scripts/ci"), { recursive: true });
mkdirSync(path.join(directory, "bin"));
mkdirSync(path.join(directory, "packages/fixture"), { recursive: true });
- for (const name of ["run-task.mjs", "execution-plan.mjs"]) {
+ for (const name of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) {
copyFileSync(new URL(`scripts/ci/${name}`, repositoryRoot), path.join(directory, "scripts/ci", name));
}
+ mkdirSync(path.join(directory, ".github/scripts"), { recursive: true });
+ copyFileSync(new URL(".github/scripts/ci-inputs.mjs", repositoryRoot), path.join(directory, ".github/scripts/ci-inputs.mjs"));
write(".gitignore", ".turbo/\ncalls.txt\nresult.txt\nbin/\n");
json("packages/fixture/package.json", { name: "fixture", private: true });
json("package.json", { name: "forced-cache-proof", private: true, packageManager: "npm@10.0.0",
diff --git a/scripts/ci/execution-plan.mjs b/scripts/ci/execution-plan.mjs
index 4d1357d8c8a..1e764294c87 100644
--- a/scripts/ci/execution-plan.mjs
+++ b/scripts/ci/execution-plan.mjs
@@ -1,5 +1,6 @@
import { execFileSync } from "node:child_process";
import { matchesGlob } from "node:path";
+import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
export function sourceFiles(cwd = process.cwd()) {
return [...new Set(execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
@@ -30,7 +31,7 @@ export function executionPlan(config, task, args, files = sourceFiles()) {
target = "@paperclipai/paperclip-runner#check:runner";
commandArgs = args;
} else throw new Error(`Missing task command: ${task}`);
- const select = (globs) => files.filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
+ const select = (globs) => productInputs(files).filter((file) => globs.some((glob) => matchesGlob(file, glob))).map(literalGlob);
const selectedInputs = select(definition.inputs);
const inputs = target.startsWith("//#") ? selectedInputs
: selectedInputs.map((input) => `$TURBO_ROOT$/${input}`);
diff --git a/scripts/ci/run-selected-vitest.mjs b/scripts/ci/run-selected-vitest.mjs
new file mode 100644
index 00000000000..11a47b5cb2c
--- /dev/null
+++ b/scripts/ci/run-selected-vitest.mjs
@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import path from "node:path";
+import { createVitest, parseCLI } from "vitest/node";
+
+// Positional Vitest CLI filters are substrings: foo.test.ts also selects
+// foo.test.tsx. Run the exact discovered specifications assigned to this shard.
+const { filter, options } = parseCLI(["vitest", "run", ...process.argv.slice(2)]);
+assert.ok(filter.length > 0, "Selected execution requires explicit test files");
+const expected = new Set(filter.map((file) => path.resolve(file)));
+assert.equal(expected.size, filter.length, "Duplicate selected file");
+process.env.TEST = "true";
+process.env.VITEST = "true";
+process.env.NODE_ENV ??= "test";
+// Match the CLI's additive --exclude semantics without replacing project rules.
+options.cliExclude = options.exclude;
+delete options.exclude;
+const ctx = await createVitest({ ...options, watch: false });
+try {
+ await ctx.standalone();
+ const specifications = (await ctx.globTestSpecifications()).filter((spec) => expected.has(spec.moduleId));
+ assert.deepEqual(specifications.map((spec) => spec.moduleId).sort(), [...expected].sort(),
+ "Selected files must match exactly one discovered specification in this project");
+ await ctx.runTestSpecifications(specifications);
+} catch (error) {
+ console.error(error);
+ process.exitCode = 1;
+} finally {
+ await ctx.exit();
+}
diff --git a/scripts/ci/run-task.mjs b/scripts/ci/run-task.mjs
index 73e8a23985a..b6a683f688d 100644
--- a/scripts/ci/run-task.mjs
+++ b/scripts/ci/run-task.mjs
@@ -3,6 +3,8 @@ import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from
import { createHash } from "node:crypto";
import { matchesGlob } from "node:path";
import { executionPlan, sourceFiles } from "./execution-plan.mjs";
+import { loadSelection, selectionHash } from "./test-selection-manifest.mjs";
+import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
const task = process.env.CI_TASK;
if (!/^ci:[a-z-]+$/.test(task ?? "")) throw new Error("Invalid CI task");
@@ -15,13 +17,18 @@ const version = (command, argv) => {
const config = JSON.parse(readFileSync("turbo.json", "utf8"));
const files = sourceFiles();
const execution = executionPlan(config, task, args, files);
+// Validate before Turbo can return a cache hit. The hash separates selected
+// executions from full executions and from every different selected file set.
+if (process.env.PAPERCLIP_CI_TEST_SELECTION) {
+ process.env.PAPERCLIP_CI_SELECTION_HASH = selectionHash(loadSelection());
+}
mkdirSync(".turbo/task-configs", { recursive: true });
const runtimeConfig = `.turbo/task-configs/${task.replace(":", "-")}.json`;
writeFileSync(runtimeConfig, JSON.stringify(execution.config));
const classified = [...config.globalDependencies, ...Object.values(config.tasks).flatMap((entry) => entry.inputs)];
const unknownInputs = createHash("sha256");
let uncacheable = false;
-for (const file of files) {
+for (const file of productInputs(files)) {
if (classified.some((glob) => matchesGlob(file, glob))) continue;
unknownInputs.update(JSON.stringify(file));
try {
diff --git a/scripts/ci/select-tests.mjs b/scripts/ci/select-tests.mjs
new file mode 100644
index 00000000000..5ad76a10e24
--- /dev/null
+++ b/scripts/ci/select-tests.mjs
@@ -0,0 +1,124 @@
+import { execFileSync } from "node:child_process";
+import { appendFileSync, lstatSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+import assert from "node:assert/strict";
+import { refineTestPlan, selectionDurations } from "./test-selection-plan.mjs";
+import { productInputs } from "../../.github/scripts/ci-inputs.mjs";
+import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
+
+export function selectionEligibility(plan) {
+ if (plan?.version !== 1 || plan.mode !== "enabled" || plan.forceReason) return "Full/shadow/forced task plan";
+ if (!Array.isArray(plan.changed) || !plan.changed.length) return "Incomplete comparison";
+ const changed = productInputs(plan.changed);
+ if (!changed.length) return "Documentation-only diff";
+ for (const file of changed) {
+ if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file)
+ || /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`;
+ try {
+ if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`;
+ } catch { return `Deleted or missing source: ${file}`; }
+ }
+ return null;
+}
+
+function runnerInventory(group) {
+ const args = group === "serialized" ? ["--mode", "serialized", "--shard-index", "0", "--shard-count", "1"]
+ : ["--mode", "general", "--group", group,
+ ...(group === "general-server-without-chat" ? ["--shard-index", "0", "--shard-count", "1"] : [])];
+ const env = { ...process.env, GITHUB_WORKFLOW: "PR" };
+ delete env.PAPERCLIP_CI_TEST_SELECTION;
+ return JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", ...args, "--dry-run"],
+ { env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }));
+}
+
+export async function createTestSelection(plan, head) {
+ const reason = selectionEligibility(plan);
+ if (reason) throw new Error(reason);
+ const groups = new Set();
+ for (const job of ["general_tests", "verify_serialized_server"]) {
+ if (plan.jobs[job].run) for (const entry of plan.jobs[job].matrix.include) groups.add(entry.group ?? "serialized");
+ }
+ groups.delete("general-chat"); // Keep its line sharding and DB ownership intact.
+ const owners = {}, projects = new Set();
+ for (const group of groups) {
+ const inventory = runnerInventory(group);
+ if (group === "serialized" || group === "general-server-without-chat") {
+ owners[group] = { "@paperclipai/server": group === "serialized"
+ ? inventory.selectedSerializedSuites : inventory.selectedGeneralServerSuites };
+ projects.add("@paperclipai/server");
+ } else {
+ owners[group] = Object.fromEntries(inventory.workspaceProjects.map((project) => [project, null]));
+ for (const project of inventory.workspaceProjects) projects.add(project);
+ }
+ }
+ const { selectProjectTests } = await import("./vitest-selection-graph.mjs");
+ const graphs = [];
+ const changed = productInputs(plan.changed);
+ for (const project of projects) {
+ const graph = await selectProjectTests({ project, changed, onlyOwnChanges: true });
+ graphs.push(graph);
+ console.log(`${project}: ${graph.selected.length}/${graph.inventory.length} files, ${graph.alwaysRun.length} retained conservatively, ${graph.seconds.toFixed(1)}s`);
+ for (const owner of Object.values(owners)) if (owner[project] === null) owner[project] = graph.inventory;
+ }
+ const covered = new Set(graphs.flatMap((graph) => graph.coveredChanges));
+ for (const file of changed) assert.ok(covered.has(file), `Change outside the resolved test graphs: ${file}`);
+ const server = graphs.find((graph) => graph.project === "@paperclipai/server");
+ if (server) {
+ const general = runnerInventory("general-server-without-chat");
+ const native = runnerInventory("general-server-native-runner");
+ assert.deepEqual([...general.selectedGeneralServerSuites, ...general.selectedSerializedSuites,
+ ...native.selectedGeneralServerSuites, "server/src/__tests__/chat-channels.integration.test.ts"].sort(),
+ server.inventory, "Server inventory must be completely covered by existing lane ownership");
+ }
+ return { ...refineTestPlan({ plan, head, owners, graphs, durations: selectionDurations() }), graphs };
+}
+
+async function main() {
+ const plan = JSON.parse(process.env.CI_PLAN || "null");
+ const output = (key, value) => {
+ if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`);
+ };
+ if (process.argv.includes("--eligible")) {
+ const reason = selectionEligibility(plan);
+ console.log(reason ?? "Source diff is eligible for graph selection");
+ output("enabled", !reason);
+ return;
+ }
+ const directory = ".ci-test-selection";
+ mkdirSync(directory, { recursive: true });
+ const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).trim();
+ let result;
+ try {
+ const head = git("rev-parse", "HEAD");
+ assert.equal(head, process.env.GITHUB_SHA, "Selection must describe the tested merge commit");
+ assert.ok(plan.diff, "Missing trusted merge diff");
+ const local = mergeDiff(head, plan.diff.prHead);
+ assert.deepEqual(local.diff, plan.diff, "Selection must use the trusted merge parents");
+ assert.deepEqual(local.changed, [...new Set(plan.changed)].sort(), "Local diff disagrees with the trusted routing diff");
+ assert.equal(git("status", "--porcelain", "--untracked-files=normal"), "", "Source inputs changed during dependency setup");
+ process.env.NODE_ENV = "test";
+ process.env.VITEST = "true";
+ result = await createTestSelection(plan, head);
+ writeFileSync(`${directory}/manifest.json`, JSON.stringify(result.manifest));
+ writeFileSync(`${directory}/audit.json`, JSON.stringify(result.graphs, null, 2));
+ output("selected", "true");
+ } catch (error) {
+ console.warn(`Using the complete task plan: ${error.message}`);
+ result = { plan: { ...plan, testSelectionFallback: error.message } };
+ output("selected", "false");
+ }
+ writeFileSync(`${directory}/plan.json`, JSON.stringify(result.plan));
+ if (process.env.GITHUB_STEP_SUMMARY) {
+ const stats = result.plan.testSelection?.stats;
+ appendFileSync(process.env.GITHUB_STEP_SUMMARY, stats
+ ? ["### Vitest test selection", "", "| Lane | Test files | Jobs |", "| --- | ---: | ---: |",
+ ...stats.map((s) => `| ${s.group} | ${s.selected}/${s.total} | ${s.jobs}/${s.beforeJobs} |`), "",
+ "Filesystem/process/opaque consumers are retained. Chat and native Runner lanes keep their existing ownership.", ""].join("\n")
+ : `### Vitest test selection\n\nFull task plan: ${result.plan.testSelectionFallback}\n`);
+ }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ main().catch((error) => { console.error(error); process.exitCode = 1; });
+}
diff --git a/scripts/ci/test-selection-manifest.mjs b/scripts/ci/test-selection-manifest.mjs
new file mode 100644
index 00000000000..50b46995d78
--- /dev/null
+++ b/scripts/ci/test-selection-manifest.mjs
@@ -0,0 +1,66 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { readFileSync, statSync } from "node:fs";
+
+export const selectableGroups = ["general-server-without-chat", "general-workspaces-a",
+ "general-workspaces-b", "general-workspaces-c", "serialized"];
+
+export function selectionHash(manifest) {
+ return createHash("sha256").update(JSON.stringify({ version: manifest.version, lanes: manifest.lanes })).digest("hex");
+}
+
+export function validateSelection(manifest, { head, hash } = {}) {
+ assert.equal(manifest.version, 1, "Unsupported test selection");
+ assert.match(manifest.head, /^[a-f0-9]{40}$/);
+ if (head) assert.equal(manifest.head, head, "Selection belongs to another checkout");
+ assert.ok(manifest.lanes && typeof manifest.lanes === "object" && !Array.isArray(manifest.lanes));
+ const seen = new Set();
+ for (const [group, lane] of Object.entries(manifest.lanes)) {
+ assert.ok(selectableGroups.includes(group), `Unknown selection group: ${group}`);
+ assert.ok(Array.isArray(lane.shards) && lane.shards.length > 0 && lane.shards.length <= 12);
+ for (const shard of lane.shards) {
+ assert.ok(shard && typeof shard === "object" && !Array.isArray(shard));
+ let count = 0;
+ for (const [project, files] of Object.entries(shard)) {
+ assert.ok(project && Array.isArray(files) && files.length > 0);
+ for (const file of files) {
+ assert.equal(typeof file, "string");
+ assert.ok(/^(ui|server|cli|packages)\//.test(file) && !/[\0\r\n\\:*?\[\]{}]/.test(file)
+ && !file.split("/").some((part) => ["", ".", ".."].includes(part)), `Invalid selected path: ${file}`);
+ const key = `${project}:${file}`;
+ assert.ok(!seen.has(key), `Test assigned twice: ${key}`);
+ seen.add(key);
+ count++;
+ }
+ }
+ assert.ok(count > 0, "Empty selection shard");
+ }
+ }
+ if (hash) assert.equal(selectionHash(manifest), hash, "Selection artifact does not match its plan");
+ return manifest;
+}
+
+export function loadSelection(file = process.env.PAPERCLIP_CI_TEST_SELECTION) {
+ const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+ const manifest = validateSelection(JSON.parse(readFileSync(file, "utf8")), {
+ head, hash: process.env.PAPERCLIP_CI_SELECTION_HASH,
+ });
+ for (const lane of Object.values(manifest.lanes)) {
+ for (const shard of lane.shards) for (const files of Object.values(shard)) {
+ for (const file of files) assert.ok(statSync(file).isFile(), `Selected test missing: ${file}`);
+ }
+ }
+ return manifest;
+}
+
+export function selectedShard(manifest, options, projects) {
+ const group = options.mode === "serialized" ? "serialized" : options.group;
+ const lane = manifest.lanes[group];
+ assert.ok(lane, `Missing selection for ${group}`);
+ assert.equal(lane.shards.length, options.shardCount ?? 1, "Selection shard count mismatch");
+ const shard = lane.shards[options.shardIndex ?? 0];
+ assert.ok(shard, "Missing selection shard");
+ for (const project of Object.keys(shard)) assert.ok(projects.includes(project), `Wrong project for ${group}: ${project}`);
+ return shard;
+}
diff --git a/scripts/ci/test-selection-plan.mjs b/scripts/ci/test-selection-plan.mjs
new file mode 100644
index 00000000000..309299c41fd
--- /dev/null
+++ b/scripts/ci/test-selection-plan.mjs
@@ -0,0 +1,77 @@
+import assert from "node:assert/strict";
+import { loadShardDurations, partitionGeneralServerSuites } from "../general-server-shard.mjs";
+import { selectionHash, validateSelection } from "./test-selection-manifest.mjs";
+
+// Preserve the existing per-job budget. Small selections use fewer runners;
+// large selections retain the existing parallelism and serialized ownership.
+export function partitionSelection(inventory, selected, maxShards, durations = {}) {
+ const totalWeight = partitionGeneralServerSuites(inventory, 1, durations)[0].totalWeight;
+ const selectedWeight = partitionGeneralServerSuites(selected, 1, durations)[0].totalWeight;
+ const budget = totalWeight / maxShards || 1;
+ const count = Math.min(maxShards, selected.length, Math.max(1, Math.ceil(selectedWeight / budget)));
+ return selected.length ? partitionGeneralServerSuites(selected, count, durations).map((shard) => shard.files) : [];
+}
+
+export function refineTestPlan({ plan, head, owners, graphs, durations = {} }) {
+ const result = structuredClone(plan);
+ const manifest = { version: 1, head, lanes: {} };
+ const stats = [];
+ const byProject = new Map(graphs.map((graph) => [graph.project, graph]));
+ for (const job of ["general_tests", "verify_serialized_server"]) {
+ if (!plan.jobs[job].run) continue;
+ const entries = plan.jobs[job].matrix.include;
+ const groups = [...new Set(entries.map((entry) => entry.group ?? "serialized"))];
+ const matrix = [];
+ for (const group of groups) {
+ const original = entries.filter((entry) => (entry.group ?? "serialized") === group);
+ if (group === "general-chat") { matrix.push(...original); continue; }
+ const owned = owners[group];
+ assert.ok(owned, `Missing inventory owner: ${group}`);
+ const projectByFile = new Map();
+ const selected = [];
+ for (const [project, files] of Object.entries(owned)) {
+ const graph = byProject.get(project);
+ assert.ok(graph, `Missing project graph: ${project}`);
+ const inventory = new Set(graph.inventory);
+ const affected = new Set(graph.selected);
+ for (const file of files) {
+ assert.ok(inventory.has(file), `Runner/Vitest inventory disagreement: ${file}`);
+ assert.ok(!projectByFile.has(file), `Ambiguous owner: ${file}`);
+ projectByFile.set(file, project);
+ if (affected.has(file)) selected.push(file);
+ }
+ }
+ const partitions = partitionSelection([...projectByFile.keys()], selected, original.length, durations[group]);
+ if (partitions.length) {
+ const shards = partitions.map((files) => {
+ const shard = {};
+ for (const file of files) (shard[projectByFile.get(file)] ??= []).push(file);
+ return shard;
+ });
+ manifest.lanes[group] = { shards };
+ for (let index = 0; index < shards.length; index++) {
+ const entry = { ...original[0], selection: true };
+ if (original[0].shard_count) {
+ Object.assign(entry, { shard_index: index, shard_count: shards.length });
+ if (entry.shard_label) entry.shard_label = `${index + 1}/${shards.length}`;
+ if (entry.group_label) entry.group_label = entry.group_label.replace(/\(\d+\/\d+\)/, `(${index + 1}/${shards.length})`);
+ }
+ matrix.push(entry);
+ }
+ }
+ stats.push({ group, total: projectByFile.size, selected: selected.length, beforeJobs: original.length, jobs: partitions.length });
+ }
+ result.jobs[job] = { ...result.jobs[job], run: matrix.length > 0, selected: matrix.length, proposed: matrix.length,
+ matrix: { include: matrix.length ? matrix : entries } };
+ }
+ validateSelection(manifest);
+ result.testSelection = { version: 1, head, hash: selectionHash(manifest), stats };
+ return { plan: result, manifest };
+}
+
+export function selectionDurations() {
+ return {
+ "general-server-without-chat": loadShardDurations("scripts/general-server-shard-durations.json"),
+ serialized: loadShardDurations("scripts/serialized-shard-durations.json"),
+ };
+}
diff --git a/scripts/ci/test-selection-regression.integration.test.mjs b/scripts/ci/test-selection-regression.integration.test.mjs
new file mode 100644
index 00000000000..a18f0f019a7
--- /dev/null
+++ b/scripts/ci/test-selection-regression.integration.test.mjs
@@ -0,0 +1,56 @@
+import assert from "node:assert/strict";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { selectProjectTests } from "./vitest-selection-graph.mjs";
+import { selectionHash } from "./test-selection-manifest.mjs";
+
+test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => {
+ const repo = process.cwd();
+ const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-")));
+ t.after(() => rmSync(root, { recursive: true, force: true }));
+ for (const directory of ["ui/src", "server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true });
+ symlinkSync(path.join(repo, "node_modules"), path.join(root, "node_modules"), "dir");
+ const write = (file, body) => writeFileSync(path.join(root, file), body);
+ const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim();
+ write(".gitignore", "node_modules/\nselection.json\n");
+ write("package.json", '{"name":"selection-regression","private":true,"type":"module"}');
+ write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
+ write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}');
+ write("ui/src/leaf.mjs", "export const value = 42;\n");
+ write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));');
+ write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");');
+ write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));');
+ git("init", "-q"); git("add", ".");
+ const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
+ commit();
+ const run = (files) => {
+ const manifest = { version: 1, head: git("rev-parse", "HEAD"), lanes: {
+ "general-workspaces-a": { shards: [{ "@paperclipai/ui": files }] },
+ } };
+ write("selection.json", JSON.stringify(manifest));
+ return spawnSync(process.execPath, [path.join(repo, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1"], {
+ cwd: root, encoding: "utf8", timeout: 60000,
+ env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: path.join(root, "selection.json"),
+ PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) },
+ });
+ };
+ const baseline = run(["ui/src/import.test.mjs", "ui/src/contract.test.mjs"]);
+ assert.equal(baseline.status, 0, baseline.stdout + baseline.stderr);
+ const missing = run(["ui/src/leaf.mjs"]);
+ assert.equal(missing.status, 1, "an existing source file is not an executable test specification");
+ assert.match(missing.stdout + missing.stderr, /Selected files must match/);
+ write("ui/src/leaf.mjs", `${readFileSync(path.join(root, "ui/src/leaf.mjs"), "utf8")}// FORBIDDEN_MARKER\n`);
+ commit();
+ const graph = await selectProjectTests({ root, project: "@paperclipai/ui", changed: ["ui/src/leaf.mjs"] });
+ assert.deepEqual(graph.related, ["ui/src/import.test.mjs"]);
+ const raw = run(graph.related);
+ assert.equal(raw.status, 0, raw.stdout + raw.stderr);
+ assert.ok(graph.selected.includes("ui/src/contract.test.mjs"));
+ const guarded = run(graph.selected);
+ assert.equal(guarded.status, 1, guarded.stdout + guarded.stderr);
+ assert.match(guarded.stdout + guarded.stderr, /filesystem contract/);
+ assert.match(guarded.stdout + guarded.stderr, /FORBIDDEN_MARKER/);
+});
diff --git a/scripts/ci/test-selection-workflow.test.mjs b/scripts/ci/test-selection-workflow.test.mjs
new file mode 100644
index 00000000000..f0ee415a55a
--- /dev/null
+++ b/scripts/ci/test-selection-workflow.test.mjs
@@ -0,0 +1,100 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import vm from "node:vm";
+import test from "node:test";
+import { planTasks } from "../../.github/scripts/ci-plan.mjs";
+
+const workflow = readFileSync(".github/workflows/pr-trusted.yml", "utf8");
+const block = (job) => workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0];
+const plan = planTasks({ config: JSON.parse(readFileSync("turbo.json", "utf8")),
+ lanes: JSON.parse(readFileSync(".github/ci-task-matrix.json", "utf8")), paths: ["ui/src/pages/Inbox.tsx"] });
+const evaluate = (expression, needs, cancelled = false) => vm.runInNewContext(expression, {
+ needs, fromJSON: JSON.parse, toJSON: JSON.stringify, always: () => true, cancelled: () => cancelled,
+});
+
+test("failed, timed-out, skipped and unavailable selectors leave the original matrix runnable", () => {
+ for (const result of ["failure", "cancelled", "skipped", "success"]) {
+ const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
+ test_selection: { result, outputs: { plan: "" } } };
+ const job = block("general_tests");
+ const condition = job.match(/if: \$\{\{ (.*) \}\}/)[1];
+ const matrix = job.match(/matrix: \$\{\{ (.*) \}\}/)[1];
+ assert.equal(evaluate(condition, needs), true, result);
+ assert.deepEqual(JSON.parse(JSON.stringify(evaluate(matrix, needs))), plan.jobs.general_tests.matrix);
+ assert.equal(evaluate(condition, needs, true), false, "cancellation must not start more runners");
+ }
+});
+
+test("a published selection controls the matrix and the required aggregate", () => {
+ const selected = structuredClone(plan);
+ selected.testSelection = { version: 1, hash: "test" };
+ selected.jobs.general_tests.matrix.include = [selected.jobs.general_tests.matrix.include[0]];
+ selected.jobs.general_tests.selected = 1;
+ const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } },
+ test_selection: { result: "success", outputs: { plan: JSON.stringify(selected) } } };
+ const matrix = block("general_tests").match(/matrix: \$\{\{ (.*) \}\}/)[1];
+ assert.equal(evaluate(matrix, needs).include.length, 1);
+ const aggregate = block("verify");
+ const names = aggregate.match(/needs: \[([^\]]+)\]/)[1].split(", ");
+ const code = aggregate.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
+ .split("\n").map((line) => line.slice(10)).join("\n");
+ const results = Object.fromEntries(names.map((name) => [name, { result:
+ ["gate", "policy", "test_selection"].includes(name) || selected.jobs[name]?.run ? "success" : "skipped" }]));
+ const run = () => spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8",
+ env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(selected), CI_RESULTS: JSON.stringify(results) } });
+ assert.equal(run().status, 0);
+ for (const job of ["general_tests", "test_selection"]) {
+ for (const result of ["failure", "cancelled", "skipped"]) {
+ results[job].result = result;
+ assert.notEqual(run().status, 0, `${job}: ${result}`);
+ }
+ results[job].result = "success";
+ }
+ selected.jobs.general_tests.run = false;
+ results.general_tests.result = "skipped";
+ assert.equal(run().status, 0, "an explicitly empty selected lane may be skipped");
+ assert.ok(aggregate.includes("needs.test_selection.outputs.plan || needs.gate.outputs.plan"));
+});
+
+test("publication follows successful upload and manifests are downloaded without ignoring failures", () => {
+ const selection = block("test_selection");
+ assert.ok(selection.indexOf("Upload validated selection") < selection.indexOf("Publish the executable plan"));
+ assert.match(selection, /timeout-minutes: 5/);
+ assert.match(selection, /vars.CI_VITEST_SELECTION == '' \|\| vars.CI_VITEST_SELECTION == 'enabled'/);
+ for (const job of ["general_tests", "verify_serialized_server"]) {
+ assert.match(block(job), /PAPERCLIP_CI_SELECTION_HASH:.*testSelection.hash/);
+ assert.match(block(job), /Download selected test files\n if: matrix.selection\n uses: actions\/download-artifact@/);
+ assert.doesNotMatch(block(job), /continue-on-error: true\n.*download-artifact/);
+ }
+});
+
+test("failed-job retries use the original selector artifact, not the retry attempt", (t) => {
+ const directory = mkdtempSync(path.join(tmpdir(), "ci-retry-"));
+ t.after(() => rmSync(directory, { recursive: true, force: true }));
+ mkdirSync(path.join(directory, ".ci-test-selection"));
+ const output = path.join(directory, "output");
+ const publication = block("test_selection").split(" - name: Publish the executable plan")[1];
+ assert.match(publication, /ARTIFACT_ID: \$\{\{ steps.upload_selection.outputs.artifact-id \}\}/);
+ const code = publication.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0]
+ .split("\n").map((line) => line.slice(10)).join("\n");
+ const selected = { ...plan, testSelection: { version: 1, hash: "test" } };
+ const run = (value, artifactId) => {
+ writeFileSync(path.join(directory, ".ci-test-selection/plan.json"), JSON.stringify(value));
+ writeFileSync(output, "");
+ return spawnSync(process.execPath, ["--input-type=module", "-e", code], {
+ cwd: directory, encoding: "utf8", env: { ...process.env, ARTIFACT_ID: artifactId, GITHUB_OUTPUT: output },
+ });
+ };
+ assert.equal(run(selected, "12345").status, 0);
+ const savedPlan = readFileSync(output, "utf8").trim().slice("plan=".length);
+ for (const job of ["general_tests", "verify_serialized_server"]) {
+ const expression = block(job).match(/artifact-ids: \$\{\{ (.*) \}\}/)[1];
+ assert.equal(vm.runInNewContext(expression, { fromJSON: JSON.parse, github: { run_attempt: 2 },
+ needs: { test_selection: { outputs: { plan: savedPlan } } } }), "12345");
+ }
+ assert.notEqual(run(selected, "").status, 0, "a selection cannot be published without its uploaded artifact");
+ assert.equal(run(plan, "").status, 0, "fallback plans need no selection artifact");
+});
diff --git a/scripts/ci/test-selection.integration.test.mjs b/scripts/ci/test-selection.integration.test.mjs
new file mode 100644
index 00000000000..15442fcd097
--- /dev/null
+++ b/scripts/ci/test-selection.integration.test.mjs
@@ -0,0 +1,198 @@
+import assert from "node:assert/strict";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { selectProjectTests } from "./vitest-selection-graph.mjs";
+import { partitionSelection, refineTestPlan } from "./test-selection-plan.mjs";
+import { selectionHash, selectedShard, validateSelection } from "./test-selection-manifest.mjs";
+import { selectionEligibility } from "./select-tests.mjs";
+import { mergeDiff } from "../../.github/scripts/ci-diff.mjs";
+
+const root = process.cwd();
+const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+
+test("real graph retains direct and indirect filesystem, process and opaque dependencies", async (t) => {
+ const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vg-")));
+ t.after(() => rmSync(directory, { recursive: true, force: true }));
+ mkdirSync(path.join(directory, "ui/src"), { recursive: true });
+ mkdirSync(path.join(directory, "ui/src/glob"));
+ symlinkSync(path.join(root, "node_modules"), path.join(directory, "node_modules"), "dir");
+ const write = (file, text) => writeFileSync(path.join(directory, file), text);
+ write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}');
+ write("ui/vitest.config.mjs", `export default {resolve:{alias:{"@leaf":${JSON.stringify(path.join(directory, "ui/src/leaf.ts"))}}},test:{name:"fixture",include:["src/**/*.test.ts"],setupFiles:["./setup.ts"]}}`);
+ write("ui/setup.ts", 'import "./src/environment.ts";');
+ write("ui/src/environment.ts", "export const environment = true;");
+ write("ui/src/leaf.ts", "export const value = 42;");
+ write("ui/src/bridge.ts", 'export {value} from "@leaf";');
+ write("ui/src/helper.ts", 'import {readFileSync} from "node:fs"; export const read = (file: string) => readFileSync(file, "utf8");');
+ write("ui/src/import.test.ts", 'import {value} from "./bridge"; console.log(value); throw new Error("selection must not execute tests");');
+ write("ui/src/reader.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
+ write("ui/src/indirect.test.ts", 'import {read} from "./helper"; read("input"); throw new Error("selection must not execute tests");');
+ write("ui/src/process.test.ts", 'import {execFileSync} from "node:child_process"; execFileSync("must-not-execute");');
+ write("ui/src/dynamic.test.ts", 'const load = (file: string) => import(file);');
+ write("ui/src/cjs.test.ts", 'const load = require;');
+ write("ui/src/builtin-loader.test.ts", 'process.getBuiltinModule("fs").readFileSync("input");');
+ write("ui/src/opaque.test.ts", 'import {parse} from "yaml"; console.log(parse("value: true"));');
+ write("ui/src/independent.test.ts", 'export const independent = true;');
+ write("ui/src/mocked.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); import {read} from "./helper"; read("input");');
+ write("ui/src/partial.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",async (original)=>await original()); import {read} from "./helper"; read("input");');
+ write("ui/src/automock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper"); import {read} from "./helper"; read("input");');
+ write("ui/src/bypass.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); await vi.importActual("./helper");');
+ write("ui/src/spy.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",{spy:true}); import {read} from "./helper"; read("input");');
+ write("ui/src/remock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); vi.doMock("./helper",async original=>await original());');
+ write("ui/src/arguments.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",function(){return arguments[0]();});');
+ write("ui/src/glob/one.ts", "export const value = 1;");
+ write("ui/src/globber.ts", 'export const modules = import.meta.glob("./glob/*.ts", {eager:true});');
+ write("ui/src/glob.test.ts", 'import {modules} from "./globber"; console.log(modules);');
+ execFileSync("git", ["init", "-q"], { cwd: directory });
+ execFileSync("git", ["add", "."], { cwd: directory });
+ const select = (changed) => selectProjectTests({ root: directory, project: "fixture", changed });
+ const result = await select(["ui/src/leaf.ts"]);
+ assert.deepEqual(result.related, ["ui/src/import.test.ts"]);
+ for (const file of ["reader", "indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) {
+ assert.ok(result.alwaysRun.includes(`ui/src/${file}.test.ts`), `${file} must never be dropped`);
+ }
+ assert.ok(!result.selected.includes("ui/src/independent.test.ts"));
+ assert.ok(!result.alwaysRun.includes("ui/src/mocked.test.ts"), "a completely replaced local dependency performs no real IO");
+ assert.ok(!result.alwaysRun.includes("ui/src/glob.test.ts"), "literal Vite globs use the real dependency graph");
+ assert.ok((await select(["ui/src/glob/one.ts"])).related.includes("ui/src/glob.test.ts"));
+ write("ui/src/glob/two.ts", "export const value = 2;");
+ execFileSync("git", ["add", "."], { cwd: directory });
+ assert.ok((await select(["ui/src/glob/two.ts"])).related.includes("ui/src/glob.test.ts"), "new glob members cannot be omitted");
+ const setup = await select(["ui/src/environment.ts"]);
+ assert.equal(setup.setupChanged, true);
+ assert.deepEqual(setup.selected, setup.inventory, "transitive setup edits affect every test");
+ write("ui/src/broken.test.ts", 'import "./missing";');
+ const broken = await select(["ui/src/leaf.ts"]);
+ assert.ok(broken.selected.includes("ui/src/broken.test.ts"), "an unresolved dependency must retain its test");
+ write("ui/src/generated.ts", "export const generated = 42;");
+ write("ui/src/generated.test.ts", 'import {generated} from "./generated"; console.log(generated);');
+ execFileSync("git", ["add", "ui/src/generated.test.ts"], { cwd: directory });
+ assert.ok((await select(["ui/src/leaf.ts"])).alwaysRun.includes("ui/src/generated.test.ts"),
+ "generated inputs have no trustworthy source-to-output graph and retain their consumers");
+ write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));');
+ await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph");
+ mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true });
+ for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) {
+ write(file, readFileSync(path.join(root, file), "utf8"));
+ }
+ write("ui/setup.ts", 'import "../server/src/__tests__/setup-supertest.ts";');
+ const reviewed = await select(["ui/src/leaf.ts"]);
+ assert.ok(!reviewed.selected.includes("ui/src/independent.test.ts"), "reviewed isolated setup permits file selection");
+ const helper = "server/src/__tests__/helpers/isolated-test-home.ts";
+ const changedSetup = await select([helper]);
+ assert.deepEqual(changedSetup.selected, changedSetup.inventory, "reviewed helpers still invalidate all tests when changed");
+ write(helper, `${readFileSync(path.join(directory, helper), "utf8")}\n// setup edited since review\n`);
+ await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup/, "a stale review cannot authorize new setup effects");
+});
+
+test("partitioning reduces runners and preserves an exact, non-overlapping file cover", () => {
+ const all = Array.from({ length: 120 }, (_, index) => `server/src/${index}.test.ts`);
+ const selected = all.slice(0, 15);
+ const shards = partitionSelection(all, selected, 12);
+ assert.equal(shards.length, 2);
+ assert.deepEqual(shards.flat().sort(), selected.sort());
+ assert.equal(new Set(shards.flat()).size, selected.length);
+ assert.deepEqual(partitionSelection(all, [], 12), []);
+ assert.equal(partitionSelection(all, all, 12).length, 12);
+ const durations = Object.fromEntries(all.map((file, index) => [file, index === 0 ? 1000 : 1]));
+ assert.equal(partitionSelection(all, [all[0]], 12, durations).length, 1);
+ assert.equal(partitionSelection(all, [all[0]], 12, Object.fromEntries(all.map((file) => [file, 0]))).length, 1);
+});
+
+function fixturePlan() {
+ const files = ["ui/src/pages/Inbox.test.tsx", "ui/src/pages/apps/chat/chat-ui-contract.test.ts"];
+ const entries = [0, 1].map((shard_index) => ({ group: "general-workspaces-a", group_label: `workspaces-a (${shard_index + 1}/2)`, shard_index, shard_count: 2, task: "ci:workspaces-a" }));
+ const plan = { version: 1, mode: "enabled", forceReason: null, changed: ["ui/src/pages/Inbox.tsx"], jobs: {
+ general_tests: { run: true, matrix: { include: entries }, selected: 2, total: 2 },
+ verify_serialized_server: { run: false, matrix: { include: [{ task: "ci:serialized" }] } },
+ } };
+ const owners = { "general-workspaces-a": { "@paperclipai/ui": files } };
+ const graphs = [{ project: "@paperclipai/ui", inventory: files, selected: [files[1]] }];
+ return { files, ...refineTestPlan({ plan, head, owners, graphs }) };
+}
+
+test("runner consumes the exact manifest, and rejects stale, missing, or mismatched manifests", (t) => {
+ const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vm-")));
+ t.after(() => rmSync(directory, { recursive: true, force: true }));
+ const { manifest, plan } = fixturePlan();
+ assert.equal(plan.jobs.general_tests.selected, 1);
+ const file = path.join(directory, "selection.json");
+ const run = (selection = manifest, hash = selectionHash(manifest)) => {
+ writeFileSync(file, JSON.stringify(selection));
+ return spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
+ cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: hash },
+ });
+ };
+ const result = run();
+ assert.equal(result.status, 0, result.stderr);
+ assert.deepEqual(JSON.parse(result.stdout).selection, manifest.lanes["general-workspaces-a"].shards[0]);
+ assert.notEqual(run({ ...manifest, head: "0".repeat(40) }).status, 0);
+ assert.notEqual(run(manifest, "bad-hash").status, 0);
+ assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 2 }, ["@paperclipai/ui"]));
+ assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 1 }, ["@paperclipai/server"]));
+ const duplicate = structuredClone(manifest);
+ duplicate.lanes["general-workspaces-a"].shards.push(duplicate.lanes["general-workspaces-a"].shards[0]);
+ assert.throws(() => validateSelection(duplicate), /assigned twice/);
+ const wrongLane = { version: 1, head, lanes: { "general-server-without-chat": {
+ shards: [{ "@paperclipai/server": ["server/src/__tests__/issues-service.test.ts"] }],
+ } } };
+ writeFileSync(file, JSON.stringify(wrongLane));
+ const misplaced = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1", "--dry-run"], {
+ cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(wrongLane) },
+ });
+ assert.notEqual(misplaced.status, 0);
+ assert.match(misplaced.stderr, /belongs to another server lane/);
+});
+
+test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow CI", () => {
+ const plan = fixturePlan().plan;
+ assert.equal(selectionEligibility(plan), null);
+ for (const changed of [["ui/vitest.setup.ts"], ["ui/vite.config.ts"], ["package.json"], ["pnpm-lock.yaml"],
+ ["docs/api/issues.md"], ["ui/src/missing-deleted.ts"], ["server/fixtures/input.json"], []]) {
+ assert.ok(selectionEligibility({ ...plan, changed }), JSON.stringify(changed));
+ }
+ for (const mode of ["full", "shadow"]) assert.ok(selectionEligibility({ ...plan, mode }));
+ assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" }));
+});
+
+test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => {
+ const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-")));
+ t.after(() => rmSync(directory, { recursive: true, force: true }));
+ mkdirSync(path.join(directory, "ui/src"), { recursive: true });
+ const file = path.join(directory, "ui/src/input.ts");
+ const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim();
+ const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture");
+ git("init", "-q"); writeFileSync(file, "export const value = 1;"); git("add", "."); commit();
+ const base = git("rev-parse", "HEAD");
+ git("checkout", "-qb", "pr");
+ writeFileSync(file, "export const value = 2;"); commit();
+ const prHead = git("rev-parse", "HEAD");
+ git("checkout", "--detach", base); git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "merge", "--no-ff", "pr", "-m", "merge");
+ const merge = git("rev-parse", "HEAD");
+ writeFileSync(file, "export const value = 3; // generated during setup");
+ const plan = { version: 1, mode: "enabled", forceReason: null, ...mergeDiff(merge, prHead, directory),
+ jobs: { general_tests: { run: true, selected: 2, matrix: { include: [{ shard_index: 0 }, { shard_index: 1 }] } } } };
+ const result = spawnSync(process.execPath, [path.join(root, "scripts/ci/select-tests.mjs")], {
+ cwd: directory, encoding: "utf8", env: { ...process.env, CI_PLAN: JSON.stringify(plan), BASE_SHA: base,
+ GITHUB_SHA: merge, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" },
+ });
+ assert.equal(result.status, 0, result.stderr);
+ const fallback = JSON.parse(readFileSync(path.join(directory, ".ci-test-selection/plan.json"), "utf8"));
+ assert.match(fallback.testSelectionFallback, /Source inputs changed/);
+ assert.deepEqual(fallback.jobs, plan.jobs);
+ assert.equal(fallback.testSelection, undefined);
+});
+
+
+test("mixed audited documentation stays graph eligible and large complete source diffs are allowed", () => {
+ const plan = fixturePlan().plan;
+ for (const doc of ["doc/DEVELOPING.md", "doc/DATABASE.md", "doc/plans/2026-10-30-example.md"]) {
+ assert.equal(selectionEligibility({ ...plan, changed: [...plan.changed, doc] }), null);
+ }
+ assert.equal(selectionEligibility({ ...plan, changed: Array(350).fill(plan.changed[0]) }), null);
+ assert.ok(selectionEligibility({ ...plan, changed: ["doc/DEVELOPING.md"] }));
+ assert.ok(selectionEligibility({ ...plan, changed: [...plan.changed, "docs/api/issues.md"] }));
+});
diff --git a/scripts/ci/vitest-impact-report.integration.test.mjs b/scripts/ci/vitest-impact-report.integration.test.mjs
new file mode 100644
index 00000000000..0ac09e37cf5
--- /dev/null
+++ b/scripts/ci/vitest-impact-report.integration.test.mjs
@@ -0,0 +1,38 @@
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { measureVitestImpact } from "./vitest-impact-report.mjs";
+
+test("real Vitest graph follows aliases/transitive imports but misses filesystem contracts", async (t) => {
+ const root = mkdtempSync(path.join(tmpdir(), "vitest-impact-"));
+ t.after(() => rmSync(root, { recursive: true, force: true }));
+ mkdirSync(path.join(root, "unit"));
+ const write = (file, text) => writeFileSync(path.join(root, file), text);
+ write("vitest.config.mjs", 'export default { test: { projects: ["unit"] } };');
+ write("unit/vitest.config.mjs", `export default {
+ resolve: { alias: { "@leaf": ${JSON.stringify(path.join(root, "unit/leaf.mjs"))} } },
+ test: { name: "fixture", include: ["**/*.test.mjs"] }
+ };`);
+ write("unit/leaf.mjs", "export const value = 42;");
+ write("unit/bridge.mjs", 'export { value } from "@leaf";');
+ // The graph must be collected without executing the test modules.
+ write("unit/related.test.mjs", 'import { value } from "./bridge.mjs"; throw new Error("must not execute");');
+ write("unit/dynamic.test.mjs", 'const load = () => import("./leaf.mjs"); throw new Error("must not execute");');
+ write("unit/independent.test.mjs", 'throw new Error("must not execute");');
+ write("unit/reader.test.mjs", 'import { readFileSync } from "node:fs"; const body = readFileSync(new URL("./leaf.mjs", import.meta.url), "utf8");');
+
+ const report = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/leaf.mjs"] });
+ assert.equal(report.mode, "report-only");
+ assert.equal(report.projects[0].totalFiles, 4);
+ assert.deepEqual(report.projects[0].selectedTests, ["unit/dynamic.test.mjs", "unit/related.test.mjs"]);
+ assert.deepEqual(report.projects[0].omittedTests, ["unit/independent.test.mjs", "unit/reader.test.mjs"]);
+ assert.ok(report.limitations.some((message) => message.includes("filesystem")));
+
+ const direct = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/independent.test.mjs"] });
+ assert.deepEqual(direct.projects[0].selectedTests, ["unit/independent.test.mjs"]);
+ const unmapped = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unmapped.md"] });
+ assert.equal(unmapped.projects[0].selectedFiles, 0);
+ assert.equal(unmapped.mode, "report-only", "an empty graph is not permission to skip CI");
+});
diff --git a/scripts/ci/vitest-impact-report.mjs b/scripts/ci/vitest-impact-report.mjs
new file mode 100644
index 00000000000..4502516249a
--- /dev/null
+++ b/scripts/ci/vitest-impact-report.mjs
@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+import { execFileSync } from "node:child_process";
+import { realpathSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+function repoPath(root, file) {
+ const relative = path.relative(root, path.resolve(root, file));
+ if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+ throw new Error(`Expected a file inside the checkout: ${file}`);
+ }
+ return relative.split(path.sep).join("/");
+}
+
+// This is an experiment report, deliberately not a CI execution plan. Import
+// graphs cannot describe filesystem reads, subprocesses, or external services.
+export async function measureVitestImpact({ root = process.cwd(), projects, changedFiles }) {
+ root = realpathSync(root);
+ if (!projects?.length || !changedFiles?.length) {
+ throw new Error("Specify at least one project and one changed file");
+ }
+ const files = [...new Set(changedFiles.map((file) => repoPath(root, file)))].sort();
+ const { createVitest } = await import("vitest/node");
+ const results = [];
+ // Each project gets its own graph to bound memory usage. Cross-package imports
+ // are still followed, but projects omitted by the caller are NOT assessed.
+ for (const project of new Set(projects)) {
+ const started = performance.now();
+ const ctx = await createVitest({
+ root,
+ watch: false,
+ project: [project],
+ exclude: ["**/dist/**"],
+ reporters: [],
+ related: files.map((file) => path.resolve(root, file)),
+ });
+ try {
+ if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) {
+ throw new Error(`Expected exactly one project named ${project}`);
+ }
+ const inventory = await ctx.globTestSpecifications();
+ if (!inventory.length) throw new Error(`No test files found for ${project}`);
+ const ready = performance.now();
+ const selected = await ctx.getRelevantTestSpecifications();
+ const finished = performance.now();
+ const selectedTests = selected.map((spec) => repoPath(root, spec.moduleId)).sort();
+ const selectedSet = new Set(selectedTests);
+ results.push({
+ project,
+ totalFiles: inventory.length,
+ selectedFiles: selectedTests.length,
+ startupSeconds: (ready - started) / 1000,
+ selectionSeconds: (finished - ready) / 1000,
+ selectedTests,
+ omittedTests: inventory.map((spec) => repoPath(root, spec.moduleId))
+ .filter((file) => !selectedSet.has(file)).sort(),
+ });
+ } finally {
+ await ctx.close();
+ }
+ }
+ return {
+ mode: "report-only",
+ changedFiles: files,
+ limitations: [
+ "Only the explicitly requested projects were assessed.",
+ "Selection follows imports, not filesystem reads, subprocesses, or external services.",
+ "Zero selected files is not proof that a change needs no tests.",
+ "This report must not be used to omit required CI jobs.",
+ ],
+ projects: results,
+ };
+}
+
+async function main() {
+ const { values } = parseArgs({ options: {
+ project: { type: "string", multiple: true },
+ file: { type: "string", multiple: true },
+ base: { type: "string" },
+ output: { type: "string" },
+ } });
+ if (!values.project?.length || (!!values.base === !!values.file?.length)) {
+ throw new Error("Usage: node scripts/ci/vitest-impact-report.mjs --project NAME [--project NAME] (--file PATH [--file PATH] | --base REF) [--output FILE]");
+ }
+ const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
+ const head = git("rev-parse", "--verify", "HEAD^{commit}").trim();
+ let comparison = { kind: "hypothetical", head };
+ let changedFiles = values.file;
+ if (values.base) {
+ // Vitest reads the working tree, so do not label dirty sources as HEAD.
+ if (git("status", "--porcelain", "--untracked-files=no").trim()) {
+ throw new Error("--base requires a clean tracked working tree; use --file for a hypothetical change");
+ }
+ const base = git("rev-parse", "--verify", "--end-of-options", `${values.base}^{commit}`).trim();
+ // Compare the actual trees, including both sides of a rename. The caller
+ // supplies the event base; HEAD should be the tested PR merge commit in CI.
+ changedFiles = git("diff", "--name-only", "--no-renames", "-z", base, head, "--")
+ .split("\0").filter(Boolean);
+ comparison = { kind: "git-diff", base, head };
+ }
+ process.env.NODE_ENV = "test";
+ process.env.VITEST = "true";
+ // Config/plugins are executable code. Run this only in a test job, never in
+ // the trusted metadata-only CI routing gate. Bound a failed graph traversal.
+ const timeout = setTimeout(() => {
+ console.error("Vitest impact analysis exceeded five minutes; report unavailable");
+ process.exit(1);
+ }, 300_000);
+ try {
+ const report = await measureVitestImpact({ projects: values.project, changedFiles });
+ const json = `${JSON.stringify({ ...report, comparison }, null, 2)}\n`;
+ if (values.output) writeFileSync(values.output, json);
+ else process.stdout.write(json);
+ for (const result of report.projects) {
+ console.error(`${result.project}: ${result.selectedFiles}/${result.totalFiles} files; `
+ + `${result.selectionSeconds.toFixed(2)}s graph + ${result.startupSeconds.toFixed(2)}s startup`);
+ }
+ } finally {
+ clearTimeout(timeout);
+ }
+}
+
+if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+ main().catch((error) => { console.error(error); process.exitCode = 1; });
+}
diff --git a/scripts/ci/vitest-impact.md b/scripts/ci/vitest-impact.md
new file mode 100644
index 00000000000..7ee9c2d8930
--- /dev/null
+++ b/scripts/ci/vitest-impact.md
@@ -0,0 +1,113 @@
+# Vitest dependency selection experiment
+
+The production selector is documented in `doc/DEVELOPING.md`. This probe remains
+a diagnostic for the raw import graph, without the production retention rules.
+
+This probe measures Vitest's real import graph against the current checkout.
+It does not change the CI matrix or authorize skipping tests. It loads Vitest
+configurations and Vite plugins, so run it in a normal test environment, never
+in the trusted metadata-only routing job.
+
+Install the workspace dependencies and prepare the same shared build outputs
+used by CI:
+
+```sh
+pnpm install --frozen-lockfile
+node scripts/ensure-plugin-build-deps.mjs
+node --test scripts/ci/vitest-impact-report.integration.test.mjs
+```
+
+Probe a hypothetical change (the file need not actually be modified):
+
+```sh
+node scripts/ci/vitest-impact-report.mjs \
+ --project @paperclipai/ui --file ui/src/pages/Inbox.tsx \
+ --output /tmp/inbox-impact.json
+```
+
+Or compare a base revision with the checked-out HEAD. This requires a clean
+tracked working tree. For a PR, check out the tested merge commit and supply
+the event's base SHA, as the existing CI planner does:
+
+```sh
+node scripts/ci/vitest-impact-report.mjs \
+ --project @paperclipai/ui --project @paperclipai/server \
+ --base BASE_SHA --output /tmp/pr-impact.json
+```
+
+Every requested project is analyzed separately to bound memory usage. Imports
+across package boundaries are followed within each graph, but unrequested
+projects are **not assessed**. Runner's separate Vitest configurations, browser
+E2E, Rust, typechecking, and builds are outside this experiment. The JSON records
+selected and omitted test files, project inventory, startup and graph times,
+and the comparison revisions. A failed or timed-out probe is not an empty plan.
+
+## Initial observations
+
+Measured on `5c807d11fa` with Vitest 5.0.3, macOS, and Node 26.4.0. These are
+hypothetical single-file changes against that revision, not replays of historical
+PR checkouts. CI uses Node 24 and different hardware.
+
+| Changed file | Assessed project | Selected / total test files |
+| --- | --- | ---: |
+| `ui/src/pages/Inbox.tsx` | UI | 6 / 844 |
+| `ui/src/components/MarkdownBody.tsx` | UI | 135 / 844 |
+| `server/src/middleware/auth.ts` | Server | 45 / 1,428 |
+| `server/src/services/run-failure-report.ts` | Server | 787 / 1,428 |
+| `packages/shared/src/issue-references.ts` | Shared only | 5 / 96 |
+| `docs/api/issues.md` | Server | 0 / 1,428 — known false negative |
+
+Cold graph construction took 27.43 seconds for UI, 25.68 seconds for server,
+and 2.79 seconds for shared. Subsequent queries on the same graph took about
+0.05 seconds; fresh CI jobs must budget for cold construction. The six Inbox
+files ran 81 assertions successfully in 20.92 seconds with four workers.
+These figures exclude dependency installation and shared package builds.
+
+For a local execution comparison, the complete UI project passed all 10,043
+assertions in 844 files in 361.50 seconds with the same four-worker setting.
+The initial Inbox graph plus selected execution took about 48 seconds, versus
+six minutes for the full project. This is a local experiment, not a measured
+change in GitHub CI latency. Later runs benefit from filesystem/transform caches.
+
+The CLI's Git-diff path was also exercised against the actual `HEAD^..HEAD`
+change for PR #18321. Across its eleven changed files, the server graph selected
+1,138 / 1,428 files and the shared graph selected 3 / 96. Other projects were
+not assessed. Broad shared/service imports can therefore preserve most of a
+suite; the Inbox reduction should not be generalized to every PR.
+
+## Why the raw graph is not yet a safe execution plan
+
+There are concrete missing edges in this repository:
+
+- `ui/src/pages/apps/chat/chat-ui-contract.test.ts` reads `Inbox.tsx` through
+ `readFileSync`. It is absent from the six-file selection even though changes
+ to Inbox can break its assertions.
+- `server/src/__tests__/paperclip-skill-utils.test.ts` reads `docs/api/issues.md`
+ through `fs.readFile`. Changing that document selects zero server tests.
+
+A deliberate negative control confirmed the first missing edge: temporarily
+append a comment containing `reason: "retry_failed_run"` to `Inbox.tsx`, which
+violates the contract test's source assertion without changing runtime behavior.
+The fresh graph still selected six files. Running those six plus the omitted
+contract produced six passing files and exactly one failing assertion, in
+`chat-ui-contract.test.ts` (92 assertions passed, one failed). The original source
+was restored immediately afterward. The full baseline had passed this contract.
+
+The integration test uses actual Vitest graph traversal to verify transitive
+imports, aliases, literal dynamic imports, direct test edits, and the missing
+filesystem edge. Test modules deliberately throw if executed: graph discovery
+must not execute the tests. Passing this test documents the limitation; it does
+not prove that skipped repository tests are irrelevant.
+
+A production rollout needs explicit dependencies or an always-run lane for
+filesystem/subprocess contract tests; broad fallbacks for configuration, setup,
+lockfiles, deleted files, unknown inputs, and selector failures; and comparison
+against full-suite failures. Source-only changes also need those protections:
+the Inbox example is not a documentation-only edge case. Once qualified, compute
+the selected test list before constructing the matrix and partition that list,
+preserving the existing serialized, chat, and native-runner ownership rules.
+
+Verification for this experiment covers the real-graph integration test, CLI
+probes, the complete UI project, and the deliberate negative control. Repository
+wide typechecking, build, server execution, and the complete `test:run` suite
+were not run; this is a report-only experiment, not a production CI rollout.
diff --git a/scripts/ci/vitest-selection-graph.mjs b/scripts/ci/vitest-selection-graph.mjs
new file mode 100644
index 00000000000..24c42a2df80
--- /dev/null
+++ b/scripts/ci/vitest-selection-graph.mjs
@@ -0,0 +1,264 @@
+import { readFileSync, realpathSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { isBuiltin } from "node:module";
+import { parseSync } from "oxc-parser";
+import { createVitest } from "vitest/node";
+
+// Everything else is opaque by default. Do not add IO clients, module loaders,
+// filesystem helpers or process launchers here to improve selection counts.
+const purePackages = new Set([
+ "vitest", "@vitest/expect", "@testing-library/react", "@testing-library/dom",
+ "@testing-library/user-event", "@testing-library/jest-dom", "react", "react-dom",
+ "react-router", "react-router-dom", "@tanstack/react-query", "@tanstack/query-core",
+ "zod", "lodash", "lodash-es", "date-fns", "clsx", "tailwind-merge", "class-variance-authority",
+ "lucide-react", "immer", "nanoid", "uuid", "semver",
+ "radix-ui", "motion", "@dnd-kit/core", "@dnd-kit/sortable", "@dnd-kit/utilities",
+ "i18next", "react-i18next",
+]);
+const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", "path/win32",
+ "url", "util", "util/types", "buffer", "events", "string_decoder", "crypto"]);
+// Reviewed setup performs only temporary-home IO and Supertest socket setup,
+// not reads of arbitrary repository inputs. Pin contents, not just paths: any
+// edit disables this exception until its effects are reviewed again. Continue
+// following local imports even for these files; helper changes affect all tests.
+const reviewedSetup = new Map([
+ ["server/src/__tests__/setup-supertest.ts", {
+ hash: "99d7193927468e98b41255ea9a5e37b1a681f02b259c304f3af1a52304eed2bb",
+ external: ["supertest/lib/test.js"],
+ }],
+ ["server/src/__tests__/helpers/isolated-test-home.ts", {
+ hash: "bd5645dc590087f26b473beab06d5c3ddd7956a79949f0b3e9713f11897a0c1b",
+ external: [],
+ }],
+]);
+const packageName = (specifier) => specifier.startsWith("@")
+ ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0];
+const inside = (root, file) => file.startsWith(`${root}${path.sep}`) && !file.includes(`${path.sep}node_modules${path.sep}`);
+
+// Parse source, including imports erased by mocks/transforms. Unknown loaders
+// retain their consumers. This is intentionally conservative, not a regex-based
+// promise that arbitrary JavaScript has no effects.
+export function sourceDependencies(file, source) {
+ const ast = parseSync(file, source);
+ const imports = new Set();
+ const reasons = new Set();
+ const mockBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
+ .flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)).map((s) => s.local.name)));
+ const hoistedBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest")
+ .flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)
+ && s.local.name === s.imported.name).map((s) => s.local.name)));
+ const mocks = [];
+ if (ast.errors.length) reasons.add("unparsed source");
+ const literal = (node) => node?.type === "Literal" && typeof node.value === "string";
+ const importSource = (node) => {
+ if (literal(node)) imports.add(node.value);
+ else reasons.add("dynamic module name");
+ };
+ for (const statement of ast.program.body) {
+ const call = statement.type === "ExpressionStatement" ? statement.expression : null;
+ const callee = call?.type === "CallExpression" ? call.callee : null;
+ const factory = call?.arguments?.[1];
+ if (callee?.type === "MemberExpression" && !callee.computed && callee.property.name === "mock"
+ && hoistedBindings.has(callee.object.name) && literal(call.arguments[0])
+ && ["ArrowFunctionExpression", "FunctionExpression"].includes(factory?.type) && factory.params.length === 0) {
+ mocks.push(call.arguments[0].value);
+ }
+ }
+ function visit(node, parent = null) {
+ if (!node || typeof node !== "object") return;
+ if ((node.type === "ImportDeclaration" && node.importKind !== "type")
+ || (["ExportNamedDeclaration", "ExportAllDeclaration"].includes(node.type) && node.exportKind !== "type")) {
+ if (node.source) importSource(node.source);
+ }
+ if (node.type === "ImportExpression") importSource(node.source);
+ if (node.type === "TSImportEqualsDeclaration") reasons.add("CommonJS import");
+ if (node.type === "Identifier" && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) {
+ reasons.add("external input or dynamic code");
+ }
+ if ((node.type === "Identifier" && node.name === "getBuiltinModule")
+ || (literal(node) && node.value === "getBuiltinModule")) reasons.add("dynamic builtin loader");
+ if ((node.type === "Identifier" && /^(importActual|unmock|doUnmock|doMock|arguments)$/.test(node.name))
+ || (literal(node) && /^(importActual|unmock|doUnmock|doMock)$/.test(node.value))) reasons.add("mock bypass");
+ if (node.type === "Identifier" && mockBindings.has(node.name) && parent
+ && parent.type !== "ImportSpecifier"
+ && !(parent.type === "MemberExpression" && parent.object === node && !parent.computed)) {
+ reasons.add("escaped or dynamic mocking API");
+ }
+ if (node.type === "CallExpression") {
+ const name = source.slice(node.callee.start, node.callee.end);
+ if (/^(?:vi|vitest)\.(?:mock|doMock|importActual|importMock)$/.test(name)) {
+ importSource(node.arguments[0]);
+ }
+ if (/\brequire\b/.test(name)) {
+ reasons.add("CommonJS loader");
+ importSource(node.arguments[0]);
+ }
+ const viteGlob = name === "import.meta.glob" && (literal(node.arguments[0])
+ || (node.arguments[0]?.type === "ArrayExpression" && node.arguments[0].elements.every(literal)));
+ // Vite expands literal globs into dependency edges. New files appear in
+ // that expansion; deleted/renamed and non-source inputs force full lanes.
+ if (!viteGlob && /\b(?:fetch|eval|Function|glob|globSync)\b/.test(name)) reasons.add("external input or dynamic code");
+ }
+ for (const value of Object.values(node)) {
+ if (Array.isArray(value)) value.forEach((child) => visit(child, node));
+ else if (value && typeof value === "object" && value.type) visit(value, node);
+ }
+ }
+ visit(ast.program);
+ return { imports: [...imports], reasons: [...reasons], mocks };
+}
+
+export async function selectProjectTests({ root = process.cwd(), project, changed, onlyOwnChanges = false }) {
+ root = realpathSync(root);
+ const started = performance.now();
+ const absoluteChanges = changed.map((file) => path.resolve(root, file));
+ const ctx = await createVitest({ root, watch: false, project: [project],
+ exclude: ["**/dist/**"], reporters: [], related: absoluteChanges });
+ try {
+ if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) throw new Error(`Unknown project: ${project}`);
+ const owner = ctx.projects[0];
+ const environment = owner.vite.environments.ssr;
+ const inventory = await ctx.globTestSpecifications();
+ if (!inventory.length) throw new Error(`Empty inventory: ${project}`);
+ const relative = (spec) => path.relative(root, spec.moduleId).split(path.sep).join("/");
+ if (onlyOwnChanges && !absoluteChanges.some((file) => file.startsWith(`${owner.config.root}${path.sep}`))) {
+ // Keep coarse dependency coverage, but avoid spending a graph traversal
+ // on an unchanged consumer project merely to retain most of its IO tests.
+ const files = inventory.map(relative).sort();
+ return { project, inventory: files, selected: files, related: [], alwaysRun: files, coveredChanges: [],
+ opaqueModules: {}, retainedReason: "Unchanged project retains complete lane coverage", seconds: (performance.now() - started) / 1000 };
+ }
+ const tracked = new Set(execFileSync("git", ["ls-files", "--cached", "-z"], {
+ cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024,
+ }).split("\0").filter(Boolean));
+ const audit = new Map();
+ const parsed = new Map();
+ const parse = (file) => {
+ if (!parsed.has(file)) parsed.set(file, sourceDependencies(file, readFileSync(file, "utf8")));
+ return parsed.get(file);
+ };
+ const resolve = async (specifier, importer) => {
+ try {
+ const resolved = await environment.pluginContainer.resolveId(specifier, importer);
+ if (!resolved || resolved.external || !path.isAbsolute(resolved.id) || resolved.id.includes("?")) return null;
+ return realpathSync(resolved.id);
+ } catch { return null; } // A conditional/require-only export is opaque.
+ };
+ // Setup dependencies are not test-import edges. A change anywhere in their
+ // local import closure invalidates the entire project, including helpers.
+ const setup = new Set();
+ async function setupClosure(file) {
+ if (setup.has(file)) return;
+ setup.add(file);
+ const source = parse(file);
+ const review = reviewedSetup.get(path.relative(root, file).split(path.sep).join("/"));
+ const reviewed = review?.hash === createHash("sha256").update(readFileSync(file)).digest("hex");
+ if (!reviewed && source.reasons.length) throw new Error(`Opaque setup dependency: ${file}`);
+ for (const specifier of source.imports) {
+ if (isBuiltin(specifier)) {
+ if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`);
+ continue;
+ }
+ if (reviewed && review.external.includes(specifier)) continue;
+ const dependency = await resolve(specifier, file);
+ if (dependency && inside(root, dependency)) await setupClosure(dependency);
+ else if (!dependency) throw new Error(`Unresolved setup dependency: ${specifier}`);
+ else if (!purePackages.has(packageName(specifier))) throw new Error(`Opaque setup package: ${specifier}`);
+ }
+ }
+ for (const file of [...(owner.config.setupFiles ?? []), ...(owner.config.globalSetup ?? []),
+ ...(owner.config.snapshotSerializers ?? [])]) {
+ const resolved = path.isAbsolute(file) ? file : await resolve(file, owner.vite.config.configFile);
+ if (resolved && inside(root, resolved)) await setupClosure(resolved);
+ else if (!resolved) throw new Error(`Unresolved setup module: ${file}`);
+ }
+ for (const file of owner.vite.config.configFileDependencies ?? []) setup.add(realpathSync(file));
+ const setupChanged = absoluteChanges.some((file) => setup.has(file));
+ const raw = await ctx.getRelevantTestSpecifications();
+ const modules = environment.moduleGraph.idToModuleMap;
+ const resolvedDependencies = new Map();
+ const localImports = new Map();
+ const dependencyPath = (dep) => {
+ if (!resolvedDependencies.has(dep)) {
+ const file = dep.startsWith("/@fs/") ? dep.slice(process.platform === "win32" ? 5 : 4)
+ : path.join(owner.config.root, dep);
+ try { resolvedDependencies.set(dep, realpathSync(file)); }
+ catch { resolvedDependencies.set(dep, null); }
+ }
+ return resolvedDependencies.get(dep);
+ };
+ for (const [id, module] of modules) {
+ if (!inside(root, id) || !/\.[cm]?[jt]sx?$/.test(id)) continue;
+ const source = parse(id);
+ const reasons = [...source.reasons];
+ if (!tracked.has(path.relative(root, id).split(path.sep).join("/"))) reasons.push("generated or untracked source");
+ const local = [];
+ localImports.set(id, local);
+ // Use the same resolved runtime edges as Vitest, including literal dynamic
+ // imports. Resolving the original syntax again uses the wrong conditions
+ // for require-only exports and invents edges for erased type imports.
+ const transformed = module.transformResult;
+ if (!Array.isArray(transformed?.deps)) reasons.push("unknown transform shape");
+ for (const dependencyId of [...(transformed?.deps ?? []), ...(transformed?.dynamicDeps ?? [])]) {
+ if (isBuiltin(dependencyId)) {
+ if (!pureBuiltins.has(dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`);
+ continue;
+ }
+ const dependency = dependencyPath(dependencyId);
+ if (dependency && inside(root, dependency)) {
+ local.push(dependency);
+ // Vitest intentionally skips some edges (e.g. CJS, externalized
+ // workspaces). Never regard an untracked local edge as independent.
+ if (!modules.get(dependency)?.transformResult) reasons.push(`untracked dependency ${path.relative(root, dependency)}`);
+ } else {
+ const external = dependency?.split(`${path.sep}node_modules${path.sep}`).at(-1).split(path.sep).join("/");
+ if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) || !purePackages.has(packageName(external))) {
+ reasons.push(`opaque dependency ${external ? packageName(external) : dependencyId}`);
+ }
+ }
+ }
+ if (!module.transformResult) reasons.push("untransformed module");
+ if (reasons.length) audit.set(id, [...new Set(reasons)]);
+ }
+ const retained = [];
+ for (const spec of inventory) {
+ const mocked = new Set();
+ // Honor only unconditional, explicitly supplied mock factories without an
+ // importOriginal parameter. Automocks, spies, unmock/importActual, dynamic
+ // mocking and escaped vi bindings stay conservative. The related-file
+ // graph remains uncut: edits to a mocked module can still select its tests.
+ if (owner.config.isolate !== false) {
+ for (const name of parse(spec.moduleId).mocks) {
+ const target = await resolve(name, spec.moduleId);
+ if (target && inside(root, target)) mocked.add(target);
+ }
+ }
+ const queue = [spec.moduleId], visited = new Set();
+ let opaque = setupChanged || owner.config.isolate === false || !localImports.has(spec.moduleId);
+ while (queue.length && !opaque) {
+ const file = queue.pop();
+ if (visited.has(file)) continue;
+ visited.add(file);
+ if (audit.has(file)) { opaque = true; break; }
+ for (const dependency of localImports.get(file) ?? []) if (!mocked.has(dependency)) queue.push(dependency);
+ }
+ if (opaque) retained.push(spec);
+ }
+ const selected = [...new Set([...raw, ...retained].map(relative))].sort();
+ return {
+ project,
+ inventory: inventory.map(relative).sort(),
+ selected,
+ related: raw.map(relative).sort(),
+ alwaysRun: retained.map(relative).sort(),
+ coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index])),
+ opaqueModules: Object.fromEntries([...audit].map(([file, reasons]) => [path.relative(root, file), reasons])),
+ setupChanged,
+ seconds: (performance.now() - started) / 1000,
+ };
+ } finally {
+ await ctx.close();
+ }
+}
diff --git a/scripts/lib/recovery-cutover-inventory.mjs b/scripts/lib/recovery-cutover-inventory.mjs
new file mode 100644
index 00000000000..6c9d74d2b17
--- /dev/null
+++ b/scripts/lib/recovery-cutover-inventory.mjs
@@ -0,0 +1,99 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { classifyRecoveryEntry, cutoverStatus, recoveryKinds } from "./recovery-cutover-policy.mjs";
+
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+// Inspect every ancestor: lstat on only the final component would follow an
+// intermediate symlink. No directory/file names or OS errors enter the report.
+async function directoryState(directory, io) {
+ const parsed = path.parse(directory);
+ let current = parsed.root;
+ for (const part of directory.slice(parsed.root.length).split(path.sep).filter(Boolean)) {
+ current = path.join(current, part);
+ try {
+ const stat = await io.lstat(current);
+ if (!stat.isDirectory() || stat.isSymbolicLink()) return "unsafe";
+ } catch (error) {
+ return error?.code === "ENOENT" ? "absent" : "unreadable";
+ }
+ }
+ return "directory";
+}
+
+/** Read-only, bounded entry inventory. Run with producers stopped on the retained
+ * volume. This is not an atomic snapshot and does not establish DB settlement.
+ * io exists for deterministic fault tests; production uses the actual filesystem. */
+export async function inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries = 10_000 }, io = fs) {
+ if (!path.isAbsolute(instanceRoot) || (orphanDirectory !== undefined && !path.isAbsolute(orphanDirectory))) {
+ throw new Error("Recovery inventory requires absolute directories");
+ }
+ if (!Number.isSafeInteger(maxEntries) || maxEntries < 1 || maxEntries > 100_000) {
+ throw new Error("Entry limit must be between 1 and 100000");
+ }
+ const root = path.resolve(instanceRoot);
+ const rootState = await directoryState(root, io);
+ const directories = [path.join(root, "accounting-receipts"), path.join(root, "quota-credential-recovery"),
+ orphanDirectory === undefined ? path.join(root, "data", "sandbox-orphan-cleanup") : path.resolve(orphanDirectory)];
+ let remaining = maxEntries;
+ const summaries = [];
+ for (const [index, directory] of directories.entries()) {
+ const summary = { kind: recoveryKinds[index], presence: "unknown", observedEntries: 0,
+ unverifiedFiles: 0, unknownEntries: 0, readFailures: 0, complete: true, status: "unknown" };
+ if (rootState !== "directory") {
+ summary.complete = false;
+ } else {
+ const state = await directoryState(directory, io);
+ if (state === "absent") summary.presence = "absent";
+ else if (state !== "directory") {
+ summary.unknownEntries++;
+ summary.complete = false;
+ if (state === "unreadable") summary.readFailures++;
+ } else {
+ summary.presence = "present";
+ async function scan(target, depth) {
+ let handle;
+ try {
+ handle = await io.opendir(target);
+ while (true) {
+ // Read one sentinel entry to distinguish exactly-at-limit from
+ // truncated scans. The sentinel is not included in observed counts.
+ const entry = await handle.read();
+ if (entry === null) break;
+ if (remaining === 0) { summary.complete = false; break; }
+ remaining--;
+ summary.observedEntries++;
+ const entryKind = entry.isFile() ? "file"
+ : entry.isDirectory() && uuid.test(entry.name) ? "company-directory" : "other";
+ const classification = classifyRecoveryEntry(summary.kind, depth, entryKind);
+ if (classification === "descend") {
+ const child = path.join(target, entry.name);
+ // Recheck the entry before traversing; never intentionally follow
+ // links. Operators must quiesce writers for the snapshot to hold.
+ if (await directoryState(child, io) !== "directory") {
+ summary.unknownEntries++;
+ summary.complete = false;
+ } else await scan(child, depth + 1);
+ } else summary[classification]++;
+ if (!summary.complete && remaining === 0) break;
+ }
+ } catch {
+ summary.readFailures++;
+ summary.complete = false;
+ } finally {
+ if (handle) {
+ try { await handle.close(); }
+ catch { summary.readFailures++; summary.complete = false; }
+ }
+ }
+ }
+ await scan(directory, 0);
+ }
+ }
+ summary.status = cutoverStatus(summary);
+ summaries.push(summary);
+ }
+ return { schema: "paperclip/recovery-cutover-inventory/v1", status:
+ summaries.some(s => s.status === "unknown") ? "unknown" : summaries.some(s => s.status === "blocked") ? "blocked" : "clear",
+ instanceStatus: rootState, contentInspection: "none", counts: "observed-lower-bounds", maxEntries, directories: summaries };
+}
diff --git a/scripts/lib/recovery-cutover-policy.mjs b/scripts/lib/recovery-cutover-policy.mjs
new file mode 100644
index 00000000000..19f5a7568fe
--- /dev/null
+++ b/scripts/lib/recovery-cutover-policy.mjs
@@ -0,0 +1,14 @@
+/** File contents belong to each recovery owner. Inventory never parses them:
+ * even an apparently valid file cannot establish that its obligation settled. */
+export const recoveryKinds = ["accounting-receipts", "quota-credential-recovery", "sandbox-orphan-cleanup"];
+
+export function classifyRecoveryEntry(kind, depth, entry) {
+ if (entry === "file") return "unverifiedFiles";
+ if (entry === "company-directory" && kind === "quota-credential-recovery" && depth === 0) return "descend";
+ return "unknownEntries";
+}
+
+export function cutoverStatus(summary) {
+ if (!summary.complete || summary.unknownEntries > 0 || summary.readFailures > 0) return "unknown";
+ return summary.unverifiedFiles > 0 ? "blocked" : "clear";
+}
diff --git a/scripts/recovery-cutover.mjs b/scripts/recovery-cutover.mjs
new file mode 100644
index 00000000000..138a5f17947
--- /dev/null
+++ b/scripts/recovery-cutover.mjs
@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs";
+
+const usage = "Usage: node scripts/recovery-cutover.mjs --instance-root /absolute/instance [--orphan-directory /absolute/override] [--max-entries 10000] [--check]";
+const args = process.argv.slice(2);
+try {
+ let instanceRoot;
+ let orphanDirectory = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
+ let maxEntries = 10_000;
+ let check = false;
+ const seen = new Set();
+ for (let i = 0; i < args.length; i++) {
+ const argument = args[i];
+ if (seen.has(argument)) throw new Error("duplicate");
+ seen.add(argument);
+ if (argument === "--check") check = true;
+ else if (["--instance-root", "--orphan-directory", "--max-entries"].includes(argument)) {
+ const value = args[++i];
+ if (value === undefined || value.startsWith("--")) throw new Error("missing");
+ if (argument === "--instance-root") instanceRoot = value;
+ else if (argument === "--orphan-directory") orphanDirectory = value;
+ else maxEntries = Number(value);
+ } else throw new Error("unknown");
+ }
+ if (!instanceRoot) throw new Error("missing");
+ const report = await inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries });
+ process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+ if (check && report.status !== "clear") process.exitCode = 1;
+} catch {
+ // Never echo args, filesystem errors or paths: they may contain secrets.
+ process.stderr.write(`Recovery inventory could not run. ${usage}\n`);
+ process.exitCode = 2;
+}
diff --git a/scripts/recovery-cutover.test.mjs b/scripts/recovery-cutover.test.mjs
new file mode 100644
index 00000000000..56bd6a0998d
--- /dev/null
+++ b/scripts/recovery-cutover.test.mjs
@@ -0,0 +1,163 @@
+import { afterEach, test } from "node:test";
+import assert from "node:assert/strict";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs";
+import { classifyRecoveryEntry, cutoverStatus } from "./lib/recovery-cutover-policy.mjs";
+
+const cli = fileURLToPath(new URL("./recovery-cutover.mjs", import.meta.url));
+const roots = [];
+const company = "00000000-0000-4000-8000-000000000001";
+const grant = "00000000-0000-4000-8000-000000000002";
+async function fixture() {
+ // Canonicalize macOS /var -> /private/var, since the check rejects links.
+ const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "cutover-")));
+ roots.push(root);
+ return root;
+}
+async function write(root, relative, content = "secret-token-must-never-appear") {
+ const file = path.join(root, relative);
+ await fs.mkdir(path.dirname(file), { recursive: true });
+ await fs.writeFile(file, content);
+ return file;
+}
+function run(root, args = [], env = {}) {
+ return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], {
+ encoding: "utf8", env: { ...process.env, SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: "", ...env },
+ });
+}
+// An unset override differs from an invalid empty override.
+function cleanRun(root, args = []) {
+ const env = { ...process.env };
+ delete env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
+ return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], { encoding: "utf8", env });
+}
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); });
+
+test("pure policy never clears files or unknown entries", () => {
+ assert.equal(classifyRecoveryEntry("accounting-receipts", 0, "file"), "unverifiedFiles");
+ assert.equal(classifyRecoveryEntry("quota-credential-recovery", 0, "company-directory"), "descend");
+ assert.equal(classifyRecoveryEntry("quota-credential-recovery", 1, "company-directory"), "unknownEntries");
+ assert.equal(cutoverStatus({ complete: false, unknownEntries: 0, readFailures: 0, unverifiedFiles: 0 }), "unknown");
+});
+
+test("CLI: absent and empty legacy layouts clear without migration", async () => {
+ const root = await fixture();
+ let result = cleanRun(root, ["--check"]);
+ assert.equal(result.status, 0);
+ assert.equal(JSON.parse(result.stdout).status, "clear");
+ for (const directory of ["accounting-receipts", `quota-credential-recovery/${company}`, "data/sandbox-orphan-cleanup"]) {
+ await fs.mkdir(path.join(root, directory), { recursive: true });
+ }
+ result = cleanRun(root, ["--check"]);
+ assert.equal(result.status, 0);
+ assert.equal(JSON.parse(result.stdout).status, "clear");
+});
+
+test("CLI: real pending legacy files block, preserve bytes and reveal no secrets or names", async () => {
+ const root = await fixture();
+ const receipt = JSON.stringify({ schema: "paperclip/accounting-receipt/v1", id: grant, companyId: company,
+ runId: company, sourceId: company, sequence: 1, receivedAt: "2026-10-30T00:00:00.000Z", adapterType: "codex_local",
+ receipt: { complete: true, usage: { inputTokens: 1, outputTokens: 1 } } });
+ const files = await Promise.all([
+ write(root, `accounting-receipts/${grant}.json`, receipt),
+ write(root, `quota-credential-recovery/${company}/${grant}.json`, '{"ciphertext":"secret-token-must-never-appear"}'),
+ write(root, "data/sandbox-orphan-cleanup/secret-filename.json", JSON.stringify({ companyId: company,
+ environmentId: company, executionWorkspaceId: null, issueId: null, heartbeatRunId: null, provider: "daytona",
+ providerLeaseId: "private-resource", metadata: {} })),
+ ]);
+ const before = await Promise.all(files.map(file => fs.readFile(file)));
+ const result = cleanRun(root, ["--check"]);
+ assert.equal(result.status, 1);
+ const report = JSON.parse(result.stdout);
+ assert.equal(report.status, "blocked");
+ assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 1]);
+ assert.doesNotMatch(result.stdout + result.stderr, /secret-token|secret-filename|private-resource|ciphertext|00000000-/);
+ assert.deepEqual(await Promise.all(files.map(file => fs.readFile(file))), before);
+ assert.equal(cleanRun(root).status, 0, "report-only mode does not fail on obligations");
+});
+
+test("truncated, temporary, and unreadable files are unverified, never clear", async () => {
+ const root = await fixture();
+ await write(root, "accounting-receipts/truncated.json", '{"schema":');
+ const file = await write(root, `quota-credential-recovery/${company}/${grant}.json.probe`);
+ await fs.chmod(file, 0);
+ const report = await inventoryRecoveryFiles({ instanceRoot: root });
+ assert.equal(report.status, "blocked");
+ assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 0]);
+});
+
+test("unknown nesting and symlinks block without traversing outside the chosen roots", async () => {
+ const root = await fixture();
+ const outside = await fixture();
+ await write(outside, "do-not-read.json");
+ await fs.mkdir(path.join(root, "accounting-receipts"));
+ await fs.symlink(outside, path.join(root, "accounting-receipts", "link"));
+ await fs.mkdir(path.join(root, "quota-credential-recovery", company, "unexpected"), { recursive: true });
+ const report = await inventoryRecoveryFiles({ instanceRoot: root });
+ assert.equal(report.status, "unknown");
+ assert.deepEqual(report.directories.map(s => s.unknownEntries), [1, 1, 0]);
+ assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [0, 0, 0]);
+ await fs.symlink(outside, path.join(root, "data"));
+ assert.equal((await inventoryRecoveryFiles({ instanceRoot: root })).directories[2].status, "unknown");
+ await fs.symlink(outside, path.join(root, "instance-link"));
+ assert.equal((await inventoryRecoveryFiles({ instanceRoot: path.join(root, "instance-link") })).status, "unknown");
+});
+
+test("entry budget gives incomplete lower-bound counts and never clears omitted work", async () => {
+ const root = await fixture();
+ for (let i = 0; i < 4; i++) await write(root, `accounting-receipts/${i}.json`);
+ const report = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 2 });
+ assert.equal(report.status, "unknown");
+ assert.equal(report.directories[0].observedEntries, 2);
+ assert.equal(report.directories[0].complete, false);
+ assert.equal(report.directories[0].unverifiedFiles, 2);
+ const exact = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 4 });
+ assert.equal(exact.directories[0].complete, true);
+});
+
+test("CLI respects configured orphan override and explicit override precedence", async () => {
+ const root = await fixture();
+ const override = await fixture();
+ await write(override, "pending.json");
+ let result = run(root, ["--check"], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override });
+ assert.equal(result.status, 1);
+ assert.equal(JSON.parse(result.stdout).directories[2].unverifiedFiles, 1);
+ const emptyOverride = await fixture();
+ result = run(root, ["--check", "--orphan-directory", emptyOverride], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override });
+ assert.equal(result.status, 0);
+ assert.equal(JSON.parse(result.stdout).status, "clear");
+});
+
+test("fault boundary reports stat, directory read and close failures without error data", async () => {
+ const root = await fixture();
+ await fs.mkdir(path.join(root, "accounting-receipts"));
+ for (const operation of ["lstat", "opendir", "read", "close"]) {
+ const sensitiveError = Object.assign(new Error("secret-token-must-never-appear"), { code: "EACCES" });
+ const io = { ...fs };
+ if (operation === "lstat" || operation === "opendir") io[operation] = async () => { throw sensitiveError; };
+ else io.opendir = async () => ({
+ read: async () => { if (operation === "read") throw sensitiveError; return null; },
+ close: async () => { if (operation === "close") throw sensitiveError; },
+ });
+ const report = await inventoryRecoveryFiles({ instanceRoot: root }, io);
+ assert.equal(report.status, "unknown");
+ if (operation === "lstat") assert.equal(report.instanceStatus, "unreadable");
+ else assert.ok(report.directories[0].readFailures > 0);
+ assert.doesNotMatch(JSON.stringify(report), /secret-token|EACCES/);
+ }
+});
+
+test("CLI fails closed for missing instance, unsafe roots and bad arguments", async () => {
+ const root = await fixture();
+ assert.equal(cleanRun(path.join(root, "absent"), ["--check"]).status, 1);
+ for (const args of [["--max-entries", "0"], ["--max-entries", "100001"], ["--max-entries", "NaN"],
+ ["--orphan-directory", "relative-secret"], ["--bad-secret"], ["--check", "--check"]]) {
+ const result = cleanRun(root, args);
+ assert.equal(result.status, 2);
+ assert.doesNotMatch(result.stdout + result.stderr, /relative-secret|bad-secret/);
+ }
+});
diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs
index 197b1b5945c..d72bd532e57 100644
--- a/scripts/run-vitest-stable.mjs
+++ b/scripts/run-vitest-stable.mjs
@@ -8,6 +8,7 @@ import { loadShardDurations, selectGeneralServerShard } from "./general-server-s
import { invocationEnv } from "./ci-profile/record.mjs";
import { assertSelectedTests, partitionTestLines } from "./test-line-shard.mjs";
+import { loadSelection, selectedShard } from "./ci/test-selection-manifest.mjs";
const repoRoot = process.cwd();
const scriptsDir = path.dirname(fileURLToPath(import.meta.url));
@@ -91,6 +92,7 @@ const nativeRunnerSuites = [
"server/src/__tests__/dot-runner.test.ts",
// These process cases otherwise skip in server shards without Runner binaries.
"server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts",
+ "server/src/services/native-runtime/native-usage-delivery.integration.test.ts",
];
// In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable
// workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip
@@ -378,7 +380,7 @@ function selectSerializedSuites(routeTests, shardIndex, shardCount) {
return shardFiles.map((file) => byRepoPath.get(file));
}
-async function runVitest(args, label, testShard = null) {
+async function runVitest(args, label, testShard = null, exactFiles = false) {
console.log(`\n[test:run] ${label}`);
invocationIndex += 1;
const tempRootParent = process.platform === "win32" ? os.tmpdir() : "/tmp";
@@ -436,7 +438,9 @@ async function runVitest(args, label, testShard = null) {
args = [...args.filter((arg) => arg !== chatSuite), ...runFilters, "--allowOnly=false"];
}
await new Promise((resolve, reject) => {
- const child = spawn("pnpm", ["exec", "vitest", "run", ...sourceOnlyVitestArgs, ...args], {
+ const command = exactFiles ? process.execPath : "pnpm";
+ const prefix = exactFiles ? [path.join(scriptsDir, "ci/run-selected-vitest.mjs")] : ["exec", "vitest", "run"];
+ const child = spawn(command, [...prefix, ...sourceOnlyVitestArgs, ...args], {
cwd: repoRoot,
env: invocationEnv("vitest", env),
stdio: "inherit",
@@ -615,6 +619,24 @@ const generalServerTestFiles = [
.sort((a, b) => a.localeCompare(b));
const options = parseCliOptions(process.argv.slice(2));
+const selectionProjects = options.mode === serializedModeName || generalServerGroups.includes(options.group)
+ ? ["@paperclipai/server"]
+ : options.group === generalWorkspacesAGroupName ? generalWorkspacesAProjects
+ : options.group === generalWorkspacesBGroupName ? generalWorkspacesBProjects
+ : options.group === generalWorkspacesCGroupName ? generalWorkspacesCProjects : [];
+const selection = process.env.PAPERCLIP_CI_TEST_SELECTION
+ ? selectedShard(loadSelection(), options, selectionProjects) : null;
+if (selection?.["@paperclipai/server"]) {
+ const owned = new Set(options.mode === serializedModeName ? routeTests.map((test) => test.repoPath)
+ : generalServerTestFiles.filter((file) => !excludedServerSuites(options.group).includes(file)));
+ for (const file of selection["@paperclipai/server"]) {
+ if (!owned.has(file)) fail(`Selected test belongs to another server lane: ${file}`);
+ }
+}
+if (selection && options.dryRun) {
+ writeFileSync(1, JSON.stringify({ selection }));
+ process.exit(0);
+}
if (options.dryRun) {
const serializedSuites =
options.mode === serializedModeName
@@ -670,7 +692,16 @@ if (options.dryRun) {
}
try {
- if (options.mode === generalModeName || options.mode === allModeName) {
+ if (selection) {
+ for (const [project, files] of Object.entries(selection)) {
+ const serverArgs = project !== "@paperclipai/server" ? []
+ : options.mode === serializedModeName || process.env.PAPERCLIP_TEST_DATABASE_URL
+ ? ["--pool=forks", "--isolate", ...serializedServerVitestArgs]
+ : ["--pool=forks", "--isolate", "--fileParallelism", "--maxWorkers=2"];
+ await runVitest(["--project", project, ...serverArgs, ...files],
+ `selected ${project}: ${files.length} files`, null, true);
+ }
+ } else if (options.mode === generalModeName || options.mode === allModeName) {
if (options.group) {
await runGeneralGroup(routeTests, options.group, options.shardIndex, options.shardCount);
} else {
@@ -678,7 +709,7 @@ try {
}
}
- if (options.mode === serializedModeName || options.mode === allModeName) {
+ if (!selection && (options.mode === serializedModeName || options.mode === allModeName)) {
await runSerializedSuites(routeTests, options.shardIndex ?? 0, options.shardCount ?? 1);
}
} catch (error) {
diff --git a/server/scripts/migrate-run-evidence.ts b/server/scripts/migrate-run-evidence.ts
new file mode 100644
index 00000000000..71454359053
--- /dev/null
+++ b/server/scripts/migrate-run-evidence.ts
@@ -0,0 +1,32 @@
+import { createDb } from "@paperclipai/db";
+import { migrateRunEvidence } from "../src/services/run-evidence-migration.js";
+
+if (!process.env.DATABASE_URL)
+ throw new Error("Set DATABASE_URL to the intended instance; run with legacy writers drained");
+const args = process.argv.slice(2);
+const flags = new Set(["--apply", "--company", "--after-run", "--after-trace", "--limit"]);
+for (let index = 0; index < args.length; index++) {
+ const flag = args[index]!;
+ if (!flags.has(flag)) throw new Error(`Unknown migration option: ${flag}`);
+ if (flag !== "--apply" && (!args[++index] || args[index]!.startsWith("--"))) {
+ throw new Error(`Missing value for ${flag}`);
+ }
+}
+const value = (flag: string) => {
+ const i = args.indexOf(flag);
+ return i < 0 ? undefined : args[i + 1];
+};
+const db = createDb(process.env.DATABASE_URL);
+try {
+ const result = await migrateRunEvidence(db, {
+ apply: args.includes("--apply"),
+ companyId: value("--company"),
+ afterRunId: value("--after-run"),
+ afterTraceId: value("--after-trace"),
+ limit: Number(value("--limit") || 50),
+ });
+ console.log(JSON.stringify(result, null, 2));
+ if (result.items.some((item) => item.state === "blocked")) process.exitCode = 1;
+} finally {
+ await db.$client.end();
+}
diff --git a/server/scripts/qualify-native-log-recovery.ts b/server/scripts/qualify-native-log-recovery.ts
new file mode 100644
index 00000000000..8f726f59af6
--- /dev/null
+++ b/server/scripts/qualify-native-log-recovery.ts
@@ -0,0 +1,445 @@
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { spawn, type ChildProcess } from "node:child_process";
+import { once } from "node:events";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { setTimeout as delay } from "node:timers/promises";
+import { eq, sql } from "drizzle-orm";
+import { createDb, heartbeatRuns } from "@paperclipai/db";
+import {
+ S3Client,
+ CreateBucketCommand,
+ DeleteBucketCommand,
+ ListObjectsV2Command,
+ DeleteObjectsCommand,
+} from "@aws-sdk/client-s3";
+import {
+ DurablePrpControlPlane,
+ decodeDurablePrpCheckpoint,
+} from "../src/vendor/paperclip-runner/index.js";
+import { createS3StorageProvider } from "../src/storage/s3-provider.js";
+import { createObjectRunLogStore } from "../src/services/object-run-log-store.js";
+import { createObjectByteJournal } from "../src/services/object-byte-journal.js";
+import { createHeartbeatRunOutput } from "../src/services/heartbeat/run-output.js";
+import { createAgentIdentityRedactor } from "../src/services/agent-identity-redaction.js";
+import { appendHeartbeatRunEvent } from "../src/services/heartbeat-run-events.js";
+import { PaperclipControlPlanePort } from "../src/services/native-runtime/paperclip-control-plane-port.js";
+import { formatNativeRunLogEvent } from "../src/services/native-runtime/native-run-log-export.js";
+import { createNativeRunLogTestTables } from "../src/__tests__/helpers/native-run-log-database.js";
+
+const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
+const databaseUrl = process.env.PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL;
+if (
+ !endpoint ||
+ !databaseUrl ||
+ [endpoint, databaseUrl].some((url) => !["127.0.0.1", "localhost"].includes(new URL(url).hostname))
+)
+ throw new Error("Qualification requires explicit disposable loopback S3 and Postgres endpoints");
+const childMode = process.argv[2] === "--controller";
+type Config = {
+ schema: string;
+ bucket: string;
+ checkpoint: string;
+ companyId: string;
+ agentId: string;
+ runId: string;
+ sessionId: string;
+ runnerInstanceId: string;
+ environmentLeaseId: string;
+ turnId: string;
+ itemId: string;
+};
+const config: Config = childMode
+ ? JSON.parse(await fs.readFile(process.argv[3]!, "utf8"))
+ : {
+ schema: `native_log_proof_${randomUUID().replaceAll("-", "")}`,
+ bucket: `pc-native-proof-${randomUUID()}`,
+ checkpoint: "",
+ companyId: randomUUID(),
+ agentId: randomUUID(),
+ runId: randomUUID(),
+ sessionId: randomUUID(),
+ runnerInstanceId: randomUUID(),
+ environmentLeaseId: randomUUID(),
+ turnId: randomUUID(),
+ itemId: randomUUID(),
+ };
+const db = createDb(databaseUrl, { maxConnections: 1, idleTimeoutSeconds: 0 });
+await db.execute(sql`set client_min_messages=warning`);
+const provider = createS3StorageProvider({
+ endpoint,
+ bucket: config.bucket,
+ region: "us-east-1",
+ forcePathStyle: true,
+});
+const logStore = () =>
+ createObjectRunLogStore({
+ provider,
+ keyPrefix: "logs",
+ flushMs: 30_000,
+ assertWriterAuthority: async (input) => {
+ await input.assertWriterAuthority?.();
+ },
+ });
+const identity = {
+ runnerInstanceId: config.runnerInstanceId,
+ environmentLeaseId: config.environmentLeaseId,
+ runId: config.runId,
+ normalizedSessionId: config.sessionId,
+ turnId: config.turnId,
+ itemId: config.itemId,
+};
+const expectedRunnerVersion = "0.3.0",
+ expectedRunnerDigest = `sha256:${"c".repeat(64)}`;
+async function waitFor(read: () => Promise, done: (value: T) => boolean, timeout = 30_000) {
+ const until = Date.now() + timeout;
+ while (Date.now() < until) {
+ const value = await read();
+ if (done(value)) return value;
+ await delay(40);
+ }
+ throw new Error("Native log qualification deadline exceeded");
+}
+if (childMode) {
+ const home = process.argv[4]!,
+ portNumber = Number(process.argv[5] || 0);
+ await db.execute(sql.raw(`set search_path=${config.schema}`));
+ if (portNumber) {
+ const records = new Map(
+ JSON.parse(await fs.readFile(config.checkpoint, "utf8")),
+ );
+ decodeDurablePrpCheckpoint(records).restoreSessionDirectory(home);
+ }
+ const store = logStore();
+ const [row] = await db
+ .select({
+ id: heartbeatRuns.id,
+ companyId: heartbeatRuns.companyId,
+ agentId: heartbeatRuns.agentId,
+ status: heartbeatRuns.status,
+ runtimeMode: heartbeatRuns.runtimeMode,
+ logStore: heartbeatRuns.logStore,
+ logRef: heartbeatRuns.logRef,
+ logBytes: heartbeatRuns.logBytes,
+ lastOutputSeq: heartbeatRuns.lastOutputSeq,
+ lastOutputAt: heartbeatRuns.lastOutputAt,
+ })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, config.runId));
+ type Run = typeof heartbeatRuns.$inferSelect;
+ const output = createHeartbeatRunOutput(db, {
+ run: row as Run,
+ runId: config.runId,
+ issueId: null,
+ runLogStore: store,
+ identityRedactor: createAgentIdentityRedactor(),
+ secretKeys: new Set(),
+ getCurrentUserRedactionOptions: async () => ({ enabled: false }),
+ appendRunEvent: async (_run, event) => {
+ await appendHeartbeatRunEvent(db, {
+ companyId: config.companyId,
+ agentId: config.agentId,
+ runId: config.runId,
+ ...event,
+ });
+ },
+ assertWriterAuthority: async () => {
+ const [current] = await db
+ .select({ status: heartbeatRuns.status })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, config.runId));
+ if (current?.status !== "running") throw new Error("fixture_writer_revoked");
+ },
+ });
+ const handlers = await output.begin(row as Run);
+ const record = async (event: Parameters[0], cursor: number) => {
+ await handlers.onPersistedNativeLog!(cursor, formatNativeRunLogEvent(event));
+ };
+ const persistence = new PaperclipControlPlanePort(
+ db,
+ {
+ ...config,
+ issueId: randomUUID(),
+ sourceInstanceId: config.runnerInstanceId,
+ controlPlaneSourceInstanceId: `control-${config.runId}`,
+ completionContractId: randomUUID(),
+ completionContractSha256: "qualification",
+ },
+ { onCommittedEvent: record, onDuplicateEvent: record },
+ );
+ const core = new DurablePrpControlPlane({
+ stateDirectory: path.join(home, "control-plane"),
+ identity,
+ expectedRunnerVersion,
+ expectedRunnerDigest,
+ onCommittedEvent: async (event) => {
+ await persistence.appendEvent(event);
+ },
+ });
+ core.setCheckpointPersistence(async (checkpoint) => {
+ const temporary = `${config.checkpoint}.pending`;
+ await fs.writeFile(temporary, JSON.stringify([...checkpoint.records]));
+ await fs.rename(temporary, config.checkpoint);
+ });
+ await core.start(portNumber || undefined);
+ const ticket = core.issueBootstrapTicket();
+ await core.flushCheckpoint();
+ process.on("message", async (message: { id: string; type: string }) => {
+ try {
+ if (message.type === "checkpoint") await core.flushCheckpoint();
+ else if (message.type === "finish-log") {
+ const result = await store.finalize(output.handle!);
+ await output.flushOutputProgress({ force: true });
+ if (result.bytes === null) throw new Error("qualification_log_incomplete");
+ } else {
+ core.queueCommand(message.type, {}, message.id, true);
+ await waitFor(
+ async () => core.getCommand(message.id)?.status,
+ (status) => status === "completed",
+ );
+ }
+ process.send!({ id: message.id, ok: true });
+ } catch (error) {
+ process.send!({ id: message.id, error: String(error) });
+ }
+ });
+ process.send!({ ready: true, connectUrl: core.connectUrl, ticket });
+} else {
+ const root = await fs.mkdtemp(path.join(os.tmpdir(), "native-log-qualification-"));
+ config.checkpoint = path.join(root, "checkpoint.json");
+ const configFile = path.join(root, "config.json"),
+ receiptFile = path.join(root, "receipt.json"),
+ releaseFile = path.join(root, "release"),
+ scriptFile = path.join(root, "script.json");
+ const children: ChildProcess[] = [];
+ const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
+ let bucketCreated = false;
+ async function controller(home: string, portNumber = 0) {
+ await fs.mkdir(home, { mode: 0o700 });
+ const child = spawn(
+ process.execPath,
+ [
+ "--import",
+ path.resolve("server/node_modules/tsx/dist/loader.mjs"),
+ fileURLToPath(import.meta.url),
+ "--controller",
+ configFile,
+ home,
+ String(portNumber),
+ ],
+ {
+ cwd: process.cwd(),
+ env: { ...process.env, PAPERCLIP_HOME: home },
+ stdio: ["ignore", "ignore", "pipe", "ipc"],
+ },
+ );
+ children.push(child);
+ let diagnostics = "";
+ child.stderr!.on("data", (bytes) => {
+ diagnostics = (diagnostics + String(bytes)).slice(-8000);
+ });
+ const ready = await new Promise<{ connectUrl: string; ticket: string }>((resolve, reject) => {
+ const timeout = setTimeout(
+ () => reject(new Error(`Controller startup timeout: ${diagnostics}`)),
+ 45_000,
+ );
+ child.on("message", (message: { ready?: boolean; connectUrl: string; ticket: string }) => {
+ if (message.ready) {
+ clearTimeout(timeout);
+ resolve(message);
+ }
+ });
+ child.once("exit", () => {
+ clearTimeout(timeout);
+ reject(new Error(`Controller startup failed: ${diagnostics}`));
+ });
+ });
+ async function command(type: string) {
+ const id = randomUUID();
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ child.off("message", onMessage);
+ reject(new Error(`Command timeout: ${type}: ${diagnostics}`));
+ }, 35_000);
+ function onMessage(message: { id: string; error?: string }) {
+ if (message.id !== id) return;
+ clearTimeout(timer);
+ child.off("message", onMessage);
+ message.error ? reject(new Error(message.error)) : resolve();
+ }
+ child.on("message", onMessage);
+ child.send({ id, type });
+ });
+ }
+ return { child, ready, command };
+ }
+ try {
+ await db.execute(sql.raw(`create schema ${config.schema}`));
+ await db.execute(sql.raw(`set search_path=${config.schema}`));
+ await createNativeRunLogTestTables(db);
+ await db.execute(
+ sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${config.runId},${config.companyId},${config.agentId},'running')`,
+ );
+ await client.send(new CreateBucketCommand({ Bucket: config.bucket }));
+ bucketCreated = true;
+ await fs.writeFile(configFile, JSON.stringify(config));
+ await fs.writeFile(
+ scriptFile,
+ JSON.stringify({ steps: 64, waitAfter: 32, retentionLimit: 8, receiptFile, releaseFile }),
+ );
+ const homeA = path.join(root, "web-a"),
+ a = await controller(homeA);
+ const runner = spawn(
+ path.resolve("packages/paperclip-runner/runner/target/debug/examples/recovery-fixture"),
+ [
+ "--fixture-script",
+ scriptFile,
+ "--connect-url",
+ a.ready.connectUrl,
+ "--state-dir",
+ path.join(root, "runner"),
+ "--runner-id",
+ config.runnerInstanceId,
+ "--environment-lease-id",
+ config.environmentLeaseId,
+ "--run-id",
+ config.runId,
+ "--session-id",
+ config.sessionId,
+ "--turn-id",
+ config.turnId,
+ "--item-id",
+ config.itemId,
+ "--runner-version",
+ expectedRunnerVersion,
+ "--runner-digest",
+ expectedRunnerDigest,
+ "--max-outbox-bytes",
+ "65536",
+ "--p0-reserve-bytes",
+ "16384",
+ "--reconnect-grace-ms",
+ "60000",
+ ],
+ {
+ env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: a.ready.ticket },
+ stdio: ["ignore", "ignore", "pipe"],
+ },
+ );
+ children.push(runner);
+ let runnerDiagnostics = "";
+ runner.stderr!.on("data", (bytes) => {
+ runnerDiagnostics = (runnerDiagnostics + String(bytes)).slice(-4000);
+ });
+ const receipt = async () =>
+ JSON.parse(await fs.readFile(receiptFile, "utf8").catch(() => "{}")) as {
+ pid: number;
+ acknowledgedEvents: number;
+ completedSteps: number;
+ retainedEvents: number;
+ peakRetainedEvents: number;
+ };
+ await a.command("turn.start");
+ await fs.writeFile(releaseFile, "go");
+ const acknowledged = await waitFor(
+ receipt,
+ (value) => value.acknowledgedEvents === 32 && value.retainedEvents === 0,
+ );
+ const journal = createObjectByteJournal({
+ provider,
+ prefix: `logs/segments-v1/${config.companyId}/${config.agentId}/${config.runId}`,
+ });
+ const before = (await journal.head())!.head;
+ assert(
+ before.sourceCursor < 32,
+ "Experiment must kill a controller with acknowledged output still pending in S3",
+ );
+ const controlStarted = performance.now();
+ await a.command("session.snapshot");
+ const controlMs = performance.now() - controlStarted;
+ await a.command("checkpoint");
+ const exitA = once(a.child, "exit");
+ a.child.kill("SIGKILL");
+ await exitA;
+ await fs.rm(homeA, { recursive: true });
+ const b = await controller(path.join(root, "web-b"), Number(new URL(a.ready.connectUrl).port));
+ await b.command("turn.steer");
+ const completed = await waitFor(
+ receipt,
+ (value) => value.acknowledgedEvents === 64 && value.retainedEvents === 0,
+ );
+ assert.equal(completed.pid, acknowledged.pid);
+ assert.equal(runner.exitCode, null, runnerDiagnostics);
+ const stopStarted = performance.now();
+ await b.command("turn.stop");
+ const stopMs = performance.now() - stopStarted;
+ await b.command("finish-log");
+ const store = logStore(),
+ content = (
+ await store.read({
+ store: "s3_segments",
+ logRef: `${config.companyId}/${config.agentId}/${config.runId}`,
+ })
+ ).content;
+ const events = content
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(JSON.parse(line).chunk).event);
+ assert.deepEqual(
+ events.map((event) => Number(event.payload.text.match(/^step=(\d+)/)[1])),
+ Array.from({ length: 64 }, (_, index) => index + 1),
+ );
+ assert.equal(new Set(events.map((event) => event.sourceEventId)).size, 64);
+ const [source] = await db.execute(
+ sql`select count(*)::int as count from heartbeat_run_events where run_id=${config.runId}`,
+ );
+ assert.equal(source!.count, 64);
+ const shutdown = once(runner, "exit");
+ await b.command("runner.shutdown");
+ assert.equal((await shutdown)[0], 0);
+ process.stdout.write(
+ `${JSON.stringify({
+ result: "passed",
+ source: "real Rust recovery fixture and PaperclipControlPlanePort",
+ acknowledgedBeforeKill: 32,
+ objectSourceCursorBeforeKill: before.sourceCursor,
+ controllerLoss: "SIGKILL",
+ oldDirectoryRemoved: true,
+ sameRunnerProcess: true,
+ recoveredRecords: events.length,
+ duplicates: 0,
+ existingControlRows: source!.count,
+ controlCommandMs: Math.round(controlMs),
+ stopMs: Math.round(stopMs),
+ peakRunnerRetained: completed.peakRetainedEvents,
+ })}\n`,
+ );
+ } finally {
+ for (const child of children)
+ if (child.exitCode === null && child.signalCode === null) {
+ const exit = once(child, "exit");
+ child.kill("SIGKILL");
+ await exit;
+ }
+ if (bucketCreated) {
+ for (;;) {
+ const page = await client.send(new ListObjectsV2Command({ Bucket: config.bucket }));
+ if (!page.Contents?.length) break;
+ await client.send(
+ new DeleteObjectsCommand({
+ Bucket: config.bucket,
+ Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) },
+ }),
+ );
+ }
+ await client.send(new DeleteBucketCommand({ Bucket: config.bucket }));
+ }
+ client.destroy();
+ await db.execute(sql.raw(`drop schema if exists ${config.schema} cascade`));
+ await db.$client.end();
+ await fs.rm(root, { recursive: true, force: true });
+ }
+}
diff --git a/server/scripts/qualify-object-run-logs.ts b/server/scripts/qualify-object-run-logs.ts
new file mode 100644
index 00000000000..2890894c999
--- /dev/null
+++ b/server/scripts/qualify-object-run-logs.ts
@@ -0,0 +1,213 @@
+import { randomUUID } from "node:crypto";
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { promises as fs } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import {
+ S3Client,
+ CreateBucketCommand,
+ DeleteBucketCommand,
+ ListObjectsV2Command,
+ DeleteObjectsCommand,
+} from "@aws-sdk/client-s3";
+import { createS3StorageProvider } from "../src/storage/s3-provider.js";
+import { createObjectRunLogStore } from "../src/services/object-run-log-store.js";
+
+const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
+if (!endpoint || !["127.0.0.1", "localhost"].includes(new URL(endpoint).hostname)) {
+ throw new Error(
+ "Qualification requires PAPERCLIP_LOG_TEST_S3_ENDPOINT on a disposable loopback backend",
+ );
+}
+const childMode = process.argv[2] === "--child-writer";
+const bucket = childMode ? process.argv[3]! : `paperclip-logs-test-${randomUUID()}`;
+const raw = createS3StorageProvider({
+ endpoint,
+ bucket,
+ region: "us-east-1",
+ forcePathStyle: true,
+});
+const counts = { get: 0, put: 0, uploadedBytes: 0 };
+const latencies: number[] = [];
+const provider = {
+ ...raw,
+ async getObject(...args: Parameters) {
+ counts.get++;
+ return raw.getObject(...args);
+ },
+ async putObjectConditional(...args: Parameters>) {
+ counts.put++;
+ counts.uploadedBytes += args[0].contentLength;
+ const start = performance.now();
+ try {
+ return await raw.putObjectConditional!(...args);
+ } finally {
+ latencies.push(performance.now() - start);
+ }
+ },
+};
+const input = { companyId: "company", agentId: "agent", runId: "crash" };
+const options = {
+ provider,
+ keyPrefix: "qualification",
+ flushMs: 1000,
+ assertWriterAuthority: async () => undefined,
+};
+const output = (seq: number) => ({
+ stream: "stdout" as const,
+ seq,
+ chunk: `${seq}: 再開😀 ${"x".repeat(900)}`,
+ ts: "2026-10-30T00:00:00Z",
+});
+if (childMode) {
+ const store = createObjectRunLogStore(options),
+ handle = await store.begin(input);
+ const before = counts.put;
+ await Promise.all(Array.from({ length: 1000 }, (_, i) => store.append(handle, output(i + 1))));
+ process.stdout.write(
+ JSON.stringify({ receipt: true, batchPuts: counts.put - before, counts }) + "\n",
+ );
+ setTimeout(() => process.exit(2), 30_000);
+} else {
+ const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
+ await client.send(new CreateBucketCommand({ Bucket: bucket }));
+ const directory = await fs.mkdtemp(path.join(tmpdir(), "run-log-crash-"));
+ let child: ReturnType | undefined;
+ try {
+ const started = performance.now();
+ child = spawn(
+ process.execPath,
+ [
+ "--import",
+ path.resolve("server/node_modules/tsx/dist/loader.mjs"),
+ import.meta.filename,
+ "--child-writer",
+ bucket,
+ ],
+ { cwd: directory, env: process.env, stdio: ["ignore", "pipe", "pipe"] },
+ );
+ const exited = once(child, "exit");
+ let childError = "";
+ child.stderr!.on("data", (bytes) => {
+ childError = (childError + bytes).slice(-8192);
+ });
+ const receipt = await new Promise<{ batchPuts: number; counts: typeof counts }>(
+ (resolve, reject) => {
+ let text = "";
+ const timeout = setTimeout(
+ () => reject(new Error(`Writer receipt timed out: ${childError}`)),
+ 30_000,
+ );
+ child!.stdout!.on("data", (bytes) => {
+ text += bytes;
+ if (!text.includes("\n")) return;
+ clearTimeout(timeout);
+ try {
+ resolve(JSON.parse(text.split("\n")[0]!));
+ } catch (error) {
+ reject(error);
+ }
+ });
+ child!.on("error", (error) => {
+ clearTimeout(timeout);
+ reject(error);
+ });
+ void exited.then(() => {
+ clearTimeout(timeout);
+ if (!text) reject(new Error(`Writer exited before its receipt: ${childError}`));
+ });
+ },
+ );
+ child.kill("SIGKILL");
+ const [, signal] = await exited;
+ if (signal !== "SIGKILL") throw new Error("Writer did not die by SIGKILL");
+ await fs.rm(directory, { recursive: true, force: true });
+ const successorStore = createObjectRunLogStore(options);
+ const successor = await successorStore.begin(input);
+ let content = "",
+ offset = 0;
+ for (;;) {
+ const page = await successorStore.read(successor, { offset, limitBytes: 131072 });
+ content += page.content;
+ if (page.nextOffset === undefined) break;
+ offset = page.nextOffset;
+ }
+ const lines = content
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
+ if (
+ lines.length !== 1000 ||
+ !lines.every((line, i) => line.seq === i + 1 && line.chunk === output(i + 1).chunk)
+ )
+ throw new Error("Replacement bytes did not match");
+ await successorStore.append(successor, output(1001));
+ const summary = await successorStore.finalize(successor);
+
+ const steadyBefore = { ...counts },
+ steadyStarted = performance.now();
+ const steady = createObjectRunLogStore(options);
+ const handles = await Promise.all(
+ Array.from({ length: 8 }, (_, i) => steady.begin({ ...input, runId: `steady-${i}` })),
+ );
+ const pending: Promise[] = [];
+ for (let tick = 0; tick < 30; tick++) {
+ for (const handle of handles) pending.push(steady.append(handle, output(tick + 1)));
+ await delay(100);
+ }
+ await Promise.all(pending);
+ await Promise.all(handles.map((handle) => steady.finalize(handle)));
+ const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
+ if (objects.IsTruncated) throw new Error("Qualification inventory exceeded one bounded page");
+ latencies.sort((a, b) => a - b);
+ console.log(
+ JSON.stringify(
+ {
+ backend: "loopback S3-compatible test service",
+ flushMs: options.flushMs,
+ burst: {
+ records: lines.length,
+ batchPuts: receipt.batchPuts,
+ writerCounts: receipt.counts,
+ },
+ steady: {
+ streams: handles.length,
+ records: pending.length,
+ elapsedMs: Math.round(performance.now() - steadyStarted),
+ putRequests: counts.put - steadyBefore.put,
+ getRequests: counts.get - steadyBefore.get,
+ uploadedBytes: counts.uploadedBytes - steadyBefore.uploadedBytes,
+ },
+ readerCounts: counts,
+ committedCrashBytes: summary.bytes,
+ objects: objects.KeyCount,
+ storedBytes: objects.Contents?.reduce((sum, object) => sum + (object.Size ?? 0), 0),
+ putP95Ms: Math.round(latencies[Math.floor(latencies.length * 0.95)] ?? 0),
+ elapsedMs: Math.round(performance.now() - started),
+ abruptProcessLoss: signal,
+ oldDirectoryRemoved: true,
+ replacement: "passed",
+ },
+ null,
+ 2,
+ ),
+ );
+ } finally {
+ child?.kill("SIGKILL");
+ await fs.rm(directory, { recursive: true, force: true });
+ for (;;) {
+ const page = await client.send(new ListObjectsV2Command({ Bucket: bucket, MaxKeys: 1000 }));
+ if (!page.Contents?.length) break;
+ await client.send(
+ new DeleteObjectsCommand({
+ Bucket: bucket,
+ Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) },
+ }),
+ );
+ }
+ await client.send(new DeleteBucketCommand({ Bucket: bucket }));
+ client.destroy();
+ }
+}
diff --git a/server/src/__tests__/activity-routes.test.ts b/server/src/__tests__/activity-routes.test.ts
index e8f7d67d258..0c507b0a157 100644
--- a/server/src/__tests__/activity-routes.test.ts
+++ b/server/src/__tests__/activity-routes.test.ts
@@ -333,6 +333,20 @@ describe("activity routes", () => {
expect(res.body).toEqual([{ runId: "run-1", adapterType: "codex_local" }]);
});
+ it("validates bounded run IDs before using the authorized issue projection", async () => {
+ const runId = "10000000-0000-4000-8000-000000000001";
+ mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" });
+ mockActivityService.runsForIssue.mockResolvedValue([{ runId }]);
+ const app = await createApp();
+ const response = await requestApp(app, base => request(base).get(`/api/issues/PAP-1/runs?runIds=${runId}`));
+ expect(response.status).toBe(200);
+ expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1", "user-1", undefined, [runId]);
+ for (const ids of ["", "not-a-uuid", Array.from({ length: 101 }, () => runId).join(",")]) {
+ const denied = await requestApp(app, base => request(base).get("/api/issues/PAP-1/runs").query({ runIds: ids }));
+ expect(denied.status).toBe(400);
+ }
+ });
+
it("requires company access before creating activity events", async () => {
const app = await createApp();
const res = await requestApp(app, (baseUrl) => request(baseUrl)
diff --git a/server/src/__tests__/adapter-routes-authz.test.ts b/server/src/__tests__/adapter-routes-authz.test.ts
index 22aa7b2c268..10adbc978e6 100644
--- a/server/src/__tests__/adapter-routes-authz.test.ts
+++ b/server/src/__tests__/adapter-routes-authz.test.ts
@@ -300,7 +300,7 @@ describe("adapter management route authorization", () => {
mocks.reloadExternalAdapter.mockImplementation(async (type: string) =>
createAdapter(type),
);
- }, 20_000);
+ }, 60_000);
afterEach(() => {
unregisterServerAdapter(EXTERNAL_ADAPTER_TYPE);
@@ -373,35 +373,31 @@ describe("adapter management route authorization", () => {
},
);
- describe("cloud-managed adapter code install floor", () => {
- beforeEach(() => {
- process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "test-server-token";
+ describe("cloud-managed adapter code floor", () => {
+ afterEach(() => vi.unstubAllEnvs());
+
+ it.each(
+ ["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"].flatMap((signal) =>
+ ["install", "reinstall", "reload", "override", "delete"].map((routeName) => [signal, routeName]),
+ ),
+ )("blocks %s adapter %s before package or registration access", async (signal, routeName) => {
+ seedInstalledExternalAdapter();
+ vi.clearAllMocks();
+ vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-server-token");
+ const app = createApp(instanceAdmin);
+
+ const res = await sendMutatingRequest(app, routeName);
+
+ expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(403);
+ expect(res.body.details).toMatchObject({ code: "adapter_install_platform_managed" });
+ expect(mocks.execFile).not.toHaveBeenCalled();
+ expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
+ expect(mocks.reloadExternalAdapter).not.toHaveBeenCalled();
+ expect(mocks.getAdapterPluginByType).not.toHaveBeenCalled();
+ expect(mocks.addAdapterPlugin).not.toHaveBeenCalled();
+ expect(mocks.removeAdapterPlugin).not.toHaveBeenCalled();
+ expect(mocks.externalRecords.get(EXTERNAL_ADAPTER_TYPE)).toEqual(installedRecord());
});
- afterEach(() => {
- delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
- });
-
- it.each(["install", "reinstall"] as const)(
- "floors adapter %s off for instance admins on cloud-managed instances",
- async (routeName) => {
- resetInstalledExternalAdapterState();
- if (routeName !== "install") {
- seedInstalledExternalAdapter();
- }
- const app = createApp(instanceAdmin);
-
- const res = await sendMutatingRequest(app, routeName);
-
- expect(res.status, `${routeName}: ${JSON.stringify(res.body)}`).toBe(
- 403,
- );
- expect(res.body.details).toMatchObject({
- code: "adapter_install_platform_managed",
- });
- expect(mocks.execFile).not.toHaveBeenCalled();
- expect(mocks.loadExternalAdapterPackage).not.toHaveBeenCalled();
- },
- );
});
describe("operator-hidden adapter management floor", () => {
diff --git a/server/src/__tests__/agent-identity.test.ts b/server/src/__tests__/agent-identity.test.ts
index d0e7dcbe246..69a29a40049 100644
--- a/server/src/__tests__/agent-identity.test.ts
+++ b/server/src/__tests__/agent-identity.test.ts
@@ -10,6 +10,7 @@ import { eq } from "drizzle-orm";
import { agentIdentityKeys, agents, companies, createDb, closeRegisteredClients, ensurePostgresDatabase, runDatabaseBackup, runDatabaseRestore, type Db } from "@paperclipai/db";
import { resolveWorktreeSeedPlan } from "../../../cli/src/commands/worktree-lib.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { agentIdentityService, supportsManagedAgentIdentity } from "../services/agent-identity.js";
import { agentService } from "../services/agents.js";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
@@ -21,9 +22,10 @@ import { actorMiddleware } from "../middleware/auth.js";
import { errorHandler } from "../middleware/error-handler.js";
import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
+const externalIdentityDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_IDENTITY_TEST_DATABASE_URL");
describe("agent cryptographic identity", () => {
let db: Db;
- let database: Awaited>;
+ let database: Pick>, "connectionString" | "cleanup">;
let directory: string;
let companyId: string;
@@ -32,7 +34,9 @@ describe("agent cryptographic identity", () => {
vi.stubEnv("PAPERCLIP_HOME", directory);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key"));
- database = await startEmbeddedPostgresTestDatabase("paperclip-identity-");
+ database = externalIdentityDatabase ? { connectionString: externalIdentityDatabase,
+ cleanup: () => closeRegisteredClients(externalIdentityDatabase) }
+ : await startEmbeddedPostgresTestDatabase("paperclip-identity-");
db = createDb(database.connectionString);
companyId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Identity test", issuePrefix: "KEY", defaultResponsibleUserId: "identity-test-owner" });
@@ -51,6 +55,7 @@ describe("agent cryptographic identity", () => {
it("migration and public reads leave existing agents unprovisioned", async () => {
const agent = await oldAgent();
expect(await agentIdentityService(db).getPublicIdentity(companyId, agent.id)).toBeNull();
+ expect((await agentIdentityService(db).createExistingLogRedactor(companyId, agent.id)).values).toEqual([]);
expect(await db.select().from(agentIdentityKeys).where(eq(agentIdentityKeys.agentId, agent.id))).toHaveLength(0);
for (const type of ["http", "openclaw_gateway", "hermes_gateway"]) expect(supportsManagedAgentIdentity(type)).toBe(false);
for (const provider of ["claude_managed", "aws_agentcore"]) expect(supportsManagedAgentIdentity("paperclip_runner", { provider })).toBe(false);
diff --git a/server/src/__tests__/agent-test-environment-routes.test.ts b/server/src/__tests__/agent-test-environment-routes.test.ts
index 4b5cf7b39d4..7a9088665ed 100644
--- a/server/src/__tests__/agent-test-environment-routes.test.ts
+++ b/server/src/__tests__/agent-test-environment-routes.test.ts
@@ -103,6 +103,11 @@ vi.mock("../services/instance-settings.js", () => ({
instanceSettingsService: () => mockInstanceSettingsService,
}));
+vi.mock("../services/adapter-settings.js", async (importOriginal) => ({
+ ...(await importOriginal()),
+ adapterSettingsService: () => ({ getDisabledTypes: async () => [] }),
+}));
+
vi.mock("../services/dot-runner-broker.js", () => ({
dotRunnerBroker: () => ({ enabled: async () => true, bindingForAgent: async () => null }),
}));
diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts
index 463a6463c25..e408200aa50 100644
--- a/server/src/__tests__/ai-connections.test.ts
+++ b/server/src/__tests__/ai-connections.test.ts
@@ -9,7 +9,8 @@ vi.mock("../services/subscription-refresh.js", () => ({ refreshSubscriptionConne
import * as localCredentials from "../services/local-ai-credentials.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { createHash, randomUUID } from "node:crypto";
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
+import { createRequire } from "node:module";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import { promises as fs } from "node:fs";
import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:fs/promises";
@@ -38,6 +39,8 @@ import { secretService } from "../services/secrets.js";
import { resolveExecutionRunAdapterConfig } from "../services/heartbeat.js";
import { aiConnectionBindingSchema, connectionPurposeTransportSchema, isAiConnectionCompatible } from "@paperclipai/shared";
import { openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution";
+import { createServer } from "node:http";
+import { once } from "node:events";
import express from "express";
import request from "supertest";
import { aiConnectionRoutes, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "../routes/ai-connections.js";
@@ -385,8 +388,8 @@ describe("managed AI connections", () => {
await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable");
});
- it("persists first-time recovery directories before consuming a single-use token", async () => {
- const owner = "quota-durable-path";
+ it("commits one exchanged token without writable credential recovery storage", async () => {
+ const owner = "quota-database-first";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { account_id: owner, access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
@@ -394,32 +397,32 @@ describe("managed AI connections", () => {
const file = quotaCredentialRecoveryPath(companyId, row.grant.id);
const recoveryRoot = path.dirname(path.dirname(file));
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
- const synced: string[] = [];
- let rejectRoot = true;
- const realOpen = fs.open.bind(fs);
+ const writes: string[] = [];
+ const rejectRecoveryWrite = (target: unknown) => {
+ if (String(target).startsWith(recoveryRoot)) {
+ writes.push(String(target));
+ throw Object.assign(new Error("Fixture recovery storage is read-only"), { code: "EROFS" });
+ }
+ };
+ const realOpen = fs.open.bind(fs), realMkdir = fs.mkdir.bind(fs);
const open = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters) => {
- const handle = await realOpen(...args);
- const sync = handle.sync.bind(handle);
- vi.spyOn(handle, "sync").mockImplementation(async () => {
- if (rejectRoot && args[0] === recoveryRoot) throw new Error("Injected recovery directory flush failure");
- await sync(); synced.push(String(args[0]));
- });
- return handle;
+ rejectRecoveryWrite(args[0]);
+ return realOpen(...args);
});
- const request = vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
- expect(synced).toContain(recoveryRoot);
- expect(synced).toContain(path.dirname(recoveryRoot));
- return Response.json({ access_token: "fresh-durable", refresh_token: "replacement-durable" });
+ const mkdir = vi.spyOn(fs, "mkdir").mockImplementation(async (...args: Parameters) => {
+ rejectRecoveryWrite(args[0]);
+ return realMkdir(...args);
});
+ const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "fresh-direct", refresh_token: "replacement-direct" }));
try {
- await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow("Injected recovery directory flush failure");
- expect(request).not.toHaveBeenCalled();
- expect(await service.credential(row)).toBe(original);
- rejectRoot = false;
- await service.refreshQuotaCredential(row, original, new AbortController().signal);
+ const refreshed = await service.refreshQuotaCredential(row, original, new AbortController().signal);
+ expect(JSON.parse(refreshed)).toMatchObject({ tokens: { access_token: "fresh-direct", refresh_token: "replacement-direct" } });
+ expect(await service.credential(row)).toBe(refreshed);
expect(request).toHaveBeenCalledTimes(1);
- expect(JSON.parse(await service.credential(row))).toMatchObject({ tokens: { access_token: "fresh-durable", refresh_token: "replacement-durable" } });
- } finally { open.mockRestore(); request.mockRestore(); }
+ expect(writes).toEqual([]);
+ await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
+ await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
+ } finally { open.mockRestore(); mkdir.mockRestore(); request.mockRestore(); }
});
it.each(["timeout", "unexpected"] as const)("classifies a manual probe's %s lock failure before starting the provider", async (kind) => {
@@ -703,7 +706,7 @@ describe("managed AI connections", () => {
} finally { request.mockRestore(); }
});
- it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("recovers exchanged OAuth tokens after %s persistence failure without another exchange", async (failure) => {
+ it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("fails truthfully after %s persistence failure and permits sign-in recovery without a journal", async (failure) => {
const owner = `quota-rollback-${failure}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: `expired-${failure}`, refresh_token: "single-use", account_id: owner } });
@@ -723,17 +726,25 @@ describe("managed AI connections", () => {
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
expect(request).toHaveBeenCalledTimes(1);
expect(await service.credential(row)).toBe(original);
- const encrypted = await readFile(file, "utf8");
- expect(encrypted).not.toContain(`fresh-${failure}`);
- expect(encrypted).not.toContain(`replacement-${failure}`);
+ await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
await db.execute(sql.raw(`drop trigger quota_save_failure on ${table}; drop function quota_save_failure();`));
triggerDropped = true;
- // A new service instance recovers from disk, without an in-memory cache.
- const recovered = aiConnectionService(db);
- await recovered.refreshQuotaCredential(row, original, new AbortController().signal);
- expect(JSON.parse(await recovered.credential(row))).toMatchObject({ tokens: { access_token: `fresh-${failure}`, refresh_token: `replacement-${failure}` } });
+ // A new receiver has no replacement to recover. A failed use routes to the
+ // existing sign-in flow, rather than claiming the failed save succeeded.
+ const receiver = aiConnectionService(db);
+ const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
+ await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
+ provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
+ mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
+ } });
+ expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
+ const signedIn = JSON.stringify({ tokens: { access_token: "signed-in", refresh_token: "new-login" } });
+ await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription",
+ ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
+ const [current] = await receiver.quotaAccounts(companyId, owner);
+ expect(current.grant.status).toBe("active");
+ expect(await receiver.runtimeCredential(current)).toBe(signedIn);
expect(request).toHaveBeenCalledTimes(1);
- await expect(access(file)).rejects.toThrow();
expect((await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, ref.secretId))).map(version => version.version).sort()).toEqual([1, 2]);
} finally {
request.mockRestore();
@@ -819,6 +830,112 @@ describe("managed AI connections", () => {
} finally { request.mockRestore(); await oldRun.cleanup(); await nextRun?.cleanup(); await journal.clear(); }
});
+ it("requires sign-in after the receiver process dies with an exchanged but unsaved token", async () => {
+ const owner = "quota-receiver-process-loss";
+ await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
+ const original = JSON.stringify({ tokens: { access_token: "expired-process", refresh_token: "single-use-process" } });
+ await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
+ const [row] = await service.quotaAccounts(companyId, owner);
+ const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
+ const held = deferredSignal(), release = deferredSignal();
+ const lockKey = 7192031;
+ const holder = db.transaction(async tx => {
+ await tx.execute(sql`select pg_advisory_xact_lock(${lockKey})`);
+ held.resolve(); await release.promise;
+ });
+ await held.promise;
+ await db.execute(sql.raw(`create function hold_receiver_save() returns trigger language plpgsql as $$ begin
+ if new.secret_id = '${ref.secretId}'::uuid then perform pg_advisory_xact_lock(${lockKey}); end if; return new; end $$;
+ create trigger hold_receiver_save before insert on company_secret_versions for each row execute function hold_receiver_save();`));
+ let exchanges = 0;
+ const provider = createServer((req, res) => {
+ exchanges++; req.resume();
+ res.setHeader("Content-Type", "application/json");
+ res.end(JSON.stringify({ access_token: "unsaved-process", refresh_token: "replacement-process" }));
+ });
+ provider.listen(0, "127.0.0.1"); await once(provider, "listening");
+ const address = provider.address() as { port: number };
+ const child = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), new URL("./helpers/quota-credential-process.ts", import.meta.url).pathname], {
+ env: { ...process.env, CREDENTIAL_TEST_DATABASE_URL: database.connectionString,
+ CREDENTIAL_TEST_PROVIDER_URL: `http://127.0.0.1:${address.port}`, CREDENTIAL_TEST_COMPANY_ID: companyId, CREDENTIAL_TEST_OWNER: owner },
+ stdio: "ignore",
+ });
+ const exited = once(child, "exit");
+ try {
+ let blocked = false;
+ const deadline = Date.now() + 25_000;
+ while (!blocked && Date.now() < deadline && child.exitCode === null) {
+ const rows = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
+ and wait_event_type = 'Lock' and wait_event = 'advisory'`);
+ blocked = exchanges === 1 && rows.length > 0;
+ if (!blocked) await new Promise(resolve => setTimeout(resolve, 25));
+ }
+ expect(blocked).toBe(true);
+ child.kill("SIGKILL"); await exited;
+ } finally {
+ if (child.exitCode === null && child.signalCode === null) { child.kill("SIGKILL"); await exited; }
+ release.resolve(); await holder;
+ await db.execute(sql`drop trigger hold_receiver_save on company_secret_versions; drop function hold_receiver_save()`);
+ await new Promise(resolve => provider.close(() => resolve()));
+ }
+ expect(exchanges).toBe(1);
+ expect(await service.credential(row)).toBe(original);
+ await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
+ const receiver = aiConnectionService(db);
+ const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
+ await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
+ provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
+ mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
+ } });
+ expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
+ const signedIn = JSON.stringify({ tokens: { access_token: "process-sign-in", refresh_token: "process-new-login" } });
+ await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
+ const [current] = await receiver.quotaAccounts(companyId, owner);
+ expect(await receiver.runtimeCredential(current)).toBe(signedIn);
+ }, 40_000);
+
+ it("does not exchange a token when a required database transaction fails", async () => {
+ const owner = "quota-db-prerequisite";
+ await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
+ const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
+ await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
+ const [row] = await service.quotaAccounts(companyId, owner);
+ const request = vi.spyOn(globalThis, "fetch");
+ const unavailableUrl = new URL(database.connectionString);
+ unavailableUrl.port = "1";
+ const unavailable = createDb(unavailableUrl.toString());
+ try {
+ await expect(aiConnectionService(unavailable).refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
+ expect(request).not.toHaveBeenCalled();
+ await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
+ } finally { await unavailable.$client.end(); request.mockRestore(); }
+ });
+
+ it.each(["reconnect", "same_value_new_epoch", "revoked"] as const)("lets %s win after a returned token's save rolls back", async change => {
+ const owner = `quota-delayed-save-${change}`;
+ await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
+ const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
+ await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
+ const [row] = await service.quotaAccounts(companyId, owner);
+ const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
+ const transaction = db.transaction.bind(db);
+ const interrupted = vi.spyOn(db, "transaction").mockImplementationOnce(async (work, config) => {
+ await transaction(async tx => { await work(tx); throw new Error("fixture rollback"); }, config).catch(() => {});
+ if (change === "revoked") await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, row.grant.id));
+ else await secretService(db).rotate(ref.secretId, { value: change === "reconnect" ? "new-login" : original });
+ throw new Error("fixture lost response");
+ });
+ const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "late", refresh_token: "late-refresh" }));
+ try {
+ const refresh = service.refreshQuotaCredential(row, original, new AbortController().signal);
+ if (change === "revoked") await expect(refresh).rejects.toThrow("credentials_unavailable");
+ else expect(await refresh).toBe(change === "reconnect" ? "new-login" : original);
+ expect(await service.credential(row)).toBe(change === "reconnect" ? "new-login" : original);
+ expect(request).toHaveBeenCalledTimes(1);
+ await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
+ } finally { interrupted.mockRestore(); request.mockRestore(); }
+ });
+
it.each(["rollback", "lost_commit_reply"] as const)("automatically saves one exchanged token across %s", async (failure) => {
const owner = `quota-once-${failure}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
@@ -863,7 +980,10 @@ describe("managed AI connections", () => {
triggerInstalled = true;
await expect(service.refreshQuotaCredential(row, current, new AbortController().signal)).rejects.toThrow();
expect(await service.credential(row)).toBe(current);
- expect(JSON.parse((await journal.read())!.value).tokens.access_token).toBe("C");
+ expect(await journal.read()).toBeNull();
+ // Only observed records from the earlier release use the legacy reader.
+ await journal.write({ companyId, grantId: row.grant.id, connectionId: row.connection.id, secretId: ref.secretId,
+ baseHash: quotaCredentialHash(current), value: JSON.stringify({ tokens: { access_token: "C", refresh_token: "refresh-C" } }) });
await db.execute(sql`drop trigger delayed_quota_commit_failure on company_secret_versions; drop function delayed_quota_commit_failure()`);
triggerInstalled = false;
const recovered = await service.refreshQuotaCredential(row, original, new AbortController().signal);
diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts
index 8cb9bf90788..a024ed421d7 100644
--- a/server/src/__tests__/attention-service.test.ts
+++ b/server/src/__tests__/attention-service.test.ts
@@ -1406,6 +1406,22 @@ describeEmbeddedPostgres("attention service", () => {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
+ const summaryEvents: Array> = [], captureNow = Date.now();
+ const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
+ installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
+ try {
+ const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
+ expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
+ expect(summary.totalCount).toBe(feed.totalCount);
+ const index = await attentionService(db).taskIndex(companyId, { userId: "board-user" });
+ expect(index.taskIds).toEqual([...new Set(feed.items.flatMap(item => item.relatedIssue ? [item.relatedIssue.id] : item.subject.kind === "issue" ? [item.subject.id] : []))]);
+ expect(index.totalCount).toBe(feed.totalCount);
+
+ } finally { summaryCapture.flush(); }
+ const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
+ && ["documents", "assets", "issue_attachments"].includes(String(event.table)));
+ expect(cardReads).toEqual([]);
+
const planItem = detailsByKind.get("plan_approval");
expect(planItem?.subject.title).toBe("Plan approval - Approve launch plan");
@@ -1527,6 +1543,9 @@ describeEmbeddedPostgres("attention service", () => {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
const interactions = feed.items.filter(item => item.sourceKind === "issue_thread_interaction");
expect(interactions.map(item => item.subject.id).sort()).toEqual([ids[0], ids[1], ids[3]].sort());
+ const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
+ expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
+
// Resolving one output removes only that decision, leaving the other output
// and the ordinary confirmation independently actionable.
await db.update(issueThreadInteractions).set({ status: "accepted", resolvedAt: new Date() }).where(eq(issueThreadInteractions.id, ids[0]));
@@ -1858,6 +1877,7 @@ describeEmbeddedPostgres("attention service", () => {
await expect(attentionService(db).list(companyId, { userId: "board-user" }))
.resolves.toMatchObject({ totalCount: 1 }); // agent_error_alert from seed
+ await expect(attentionService(db).summary(companyId, { userId: "board-user" })).resolves.toMatchObject({ totalCount: 1 });
const hiddenFeed = await attentionService(db).list(companyId, { userId: "board-user", includeDismissed: true });
expect(hiddenFeed.items.find((item) => item.dedupKey === `approval:${approvalId}`)?.dismissal)
.toMatchObject({ kind: "snooze", isActive: true, snoozedUntil: "2099-01-02T00:00:00.000Z" });
@@ -1871,6 +1891,9 @@ describeEmbeddedPostgres("attention service", () => {
const visibleApproval = visibleFeed.items.find((item) => item.dedupKey === `approval:${approvalId}`);
expect(visibleApproval?.dismissal).toMatchObject({ kind: "snooze", isActive: false });
expect(visibleApproval).toBeTruthy();
+ const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
+ expect(summary.totalCount).toBe(visibleFeed.totalCount);
+
});
it("enriches, filters, paginates, snoozes, and ranks the decide-now feed", async () => {
@@ -2113,6 +2136,13 @@ describeEmbeddedPostgres("attention service", () => {
expect(approvalRows.every((approval) => internalSnapshotIds.has(approval.id))).toBe(true);
expect(internalSnapshot.items.length).toBeGreaterThan(100);
expect(internalSnapshot.nextCursor).toBeNull();
+ const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
+ expect(summary).toEqual({
+ companyId, generatedAt: expect.any(String), totalCount: internalSnapshot.totalCount,
+ deskBadgeCount: internalSnapshot.deskBadgeCount, countsBySourceKind: internalSnapshot.countsBySourceKind,
+ });
+ expect(JSON.stringify(summary).length).toBeLessThan(JSON.stringify(internalSnapshot).length / 10);
+
await expect(attentionService(db).list(companyId, { userId: "board-user", all: true }))
.rejects.toThrow("all requires a queue filter");
@@ -2166,6 +2196,10 @@ describeEmbeddedPostgres("attention service", () => {
allowUnscopedAll: true,
});
expect(complete.items.filter((item) => item.sourceKind === "decision")).toHaveLength(2);
+ const summary = await svc.summary(companyId, { userId: "board-user" });
+ expect(summary.totalCount).toBe(complete.totalCount);
+ expect(summary.countsBySourceKind).toEqual(complete.countsBySourceKind);
+
expect(complete.nextCursor).toBeNull();
});
@@ -2281,6 +2315,18 @@ describeEmbeddedPostgres("attention service", () => {
.get(`/api/companies/${companyId}/attention?includeDismissed=true&all=true`)
.expect(200);
expect(completeFeed.body.nextCursor).toBeNull();
+ const summary = await request(app(board)).get(`/api/companies/${companyId}/attention/summary`).expect(200);
+ expect(summary.body).not.toHaveProperty("items");
+ expect(summary.body.totalCount).toBe(completeFeed.body.totalCount);
+ const taskIndex = await request(app(board)).get(`/api/companies/${companyId}/attention/tasks`).expect(200);
+ expect(taskIndex.body).not.toHaveProperty("items");
+ expect(taskIndex.body.taskIds).toEqual([]);
+ await request(app(agent)).get(`/api/companies/${companyId}/attention/tasks`).expect(403);
+ await request(app({ ...board, source: "session", companyIds: [] })).get(`/api/companies/${companyId}/attention/tasks`).expect(403);
+
+ await request(app(agent)).get(`/api/companies/${companyId}/attention/summary`).expect(403);
+ await request(app({ ...board, source: "session", companyIds: [] })).get(`/api/companies/${companyId}/attention/summary`).expect(403);
+
await request(app(board))
.get(`/api/companies/${companyId}/attention?activitySince=yesterday`)
.expect(400, { error: "activitySince must be an ISO timestamp" });
@@ -2317,6 +2363,10 @@ describeEmbeddedPostgres("attention service", () => {
expect(feed.items.map((item) => item.subject.title)).not.toContain("Confidential review");
expect(JSON.stringify(feed)).not.toContain(privateIssueId);
+ const summary = await attentionService(db).summary(companyId, { userId: actor.userId, actor });
+ expect(summary.totalCount).toBe(feed.totalCount);
+ expect(summary.countsBySourceKind).toEqual(feed.countsBySourceKind);
+
});
it("computes the aging shelf uniformly across approval, interaction, and review sources", async () => {
diff --git a/server/src/__tests__/bundled-plugin-package.test.ts b/server/src/__tests__/bundled-plugin-package.test.ts
new file mode 100644
index 00000000000..367006a2c65
--- /dev/null
+++ b/server/src/__tests__/bundled-plugin-package.test.ts
@@ -0,0 +1,86 @@
+import { rmSync, writeFileSync, symlinkSync, mkdirSync, readFileSync } from "node:fs";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
+import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
+import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
+import { readDistributionPluginCatalog } from "../services/distribution-plugin-catalog.js";
+
+import boxd from "../../../packages/plugins/sandbox-providers/boxd/src/manifest.js";
+import createos from "../../../packages/plugins/sandbox-providers/createos/src/manifest.js";
+import cloudflare from "../../../packages/plugins/sandbox-providers/cloudflare/src/manifest.js";
+import daytona from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
+import e2b from "../../../packages/plugins/sandbox-providers/e2b/src/manifest.js";
+import exe_dev from "../../../packages/plugins/sandbox-providers/exe-dev/src/manifest.js";
+import kubernetes from "../../../packages/plugins/sandbox-providers/kubernetes/src/manifest.js";
+import modal from "../../../packages/plugins/sandbox-providers/modal/src/manifest.js";
+import novita from "../../../packages/plugins/sandbox-providers/novita/src/manifest.js";
+
+const manifests = { boxd, createos, cloudflare, daytona, e2b, "exe-dev": exe_dev, kubernetes, modal, novita };
+const roots: string[] = [];
+function fixture() { const result = bundledImageFixture(); roots.push(result.root); return result; }
+afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
+
+describe("bundled image package identity", () => {
+ it("resolves independent image roots from the same registration without any previous path", () => {
+ const first = fixture();
+ const second = fixture();
+ const registration = { ...first.registration, packagePath: "/lost/old/home/plugins/fixture" };
+ expect(first.resolver.resolve(registration)).toBe(first.packageRoot);
+ expect(second.resolver.resolve(registration)).toBe(second.packageRoot);
+ expect(first.resolver.inspect(registration, first.manifest)).toEqual(second.resolver.inspect(registration, second.manifest));
+ expect(registration.packagePath).toBe("/lost/old/home/plugins/fixture");
+ });
+
+ it("requires verified manifest evidence and blocks same-version manifest drift", () => {
+ const { resolver, registration, manifest } = fixture();
+ expect(resolver.inspect(registration)).toMatchObject({ compatible: false, code: "missing_manifest_evidence" });
+ expect(resolver.inspect(registration, { ...manifest, description: "changed" })).toMatchObject({ compatible: false, code: "incompatible_bundled_package", reason: expect.stringContaining("Approved plugin manifest") });
+ expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true, identity: { kind: "bundled_image", catalogKey: "fixture", pluginKey: manifest.id, version: "1.0.0" } });
+ });
+
+ it("rejects mismatched package names, versions, identity and escaping entrypoints", () => {
+ const { resolver, registration } = fixture();
+ for (const update of [
+ { packageName: "@other/plugin" }, { version: "2.0.0" }, { pluginKey: "other.plugin" },
+ { manifestJson: { ...registration.manifestJson, version: "2.0.0" } },
+ { manifestJson: { ...registration.manifestJson, entrypoints: { worker: "../worker.js" } } },
+ ]) expect(() => resolver.resolve({ ...registration, ...update })).toThrow();
+ });
+
+ it("fails explicitly for missing or deselected bundles even when a stale npm package exists", () => {
+ const { root, packageRoot, resolver, registration } = fixture();
+ const fallback = path.join(root, "home", "node_modules", registration.packageName);
+ mkdirSync(fallback, { recursive: true });
+ writeFileSync(path.join(fallback, "worker.js"), "throw new Error('unsafe fallback');");
+ rmSync(packageRoot, { recursive: true });
+ expect(() => resolver.resolve(registration)).toThrow(/ENOENT/);
+ expect(() => createBundledPluginPackageResolver([]).resolve(registration)).toThrow(/absent or not selected/);
+ });
+
+ it("rejects tampering before manifest import and symlink escapes before worker resolution", () => {
+ const { root, packageRoot, registration, resolver } = fixture();
+ writeFileSync(path.join(packageRoot, "dist", "manifest.js"), "throw new Error('must never import');");
+ expect(() => readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG)).toThrow(/digest mismatch/);
+ rmSync(path.join(packageRoot, "dist", "worker.js"));
+ symlinkSync(path.join(root, "distribution", "catalog.json"), path.join(packageRoot, "dist", "worker.js"));
+ expect(() => resolver.resolve(registration)).toThrow(/escapes/);
+ });
+
+ it.each(BUNDLED_PLUGIN_CATALOG)("accepts the real shipped $key package and manifest contract", (entry) => {
+ const manifest = manifests[entry.key as keyof typeof manifests];
+ const pkg = JSON.parse(readFileSync(new URL(`../../../packages/plugins/${entry.relativePath}/package.json`, import.meta.url), "utf8"));
+ const { root } = fixture();
+ const packageRoot = path.join(root, entry.relativePath);
+ mkdirSync(path.join(packageRoot, "dist"), { recursive: true });
+ writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(pkg));
+ writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
+ writeFileSync(path.join(packageRoot, "dist", "worker.js"), "export default {};");
+ const installs = resolveBundledPluginInstalls([entry.key], { catalogRoot: root, enforceCatalogRoot: true, distributionPlugins: [], env: {} });
+ const resolver = createBundledPluginPackageResolver(installs);
+ const registration = { pluginKey: entry.pluginKey, packageName: pkg.name, version: manifest.version, manifestJson: manifest };
+ expect(pkg.version).toBe(manifest.version);
+ expect(resolver.resolve(registration)).toBe(packageRoot);
+ expect(resolver.inspect(registration, manifest)).toMatchObject({ compatible: true });
+ });
+});
diff --git a/server/src/__tests__/bundled-plugin-registration.test.ts b/server/src/__tests__/bundled-plugin-registration.test.ts
new file mode 100644
index 00000000000..87b495dcf1f
--- /dev/null
+++ b/server/src/__tests__/bundled-plugin-registration.test.ts
@@ -0,0 +1,287 @@
+import express from "express";
+import request from "supertest";
+import { execFileSync } from "node:child_process";
+import { rmSync, mkdirSync, writeFileSync, cpSync, readFileSync, existsSync, mkdtempSync } from "node:fs";
+import path from "node:path";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { applyPendingMigrations, closeRegisteredClients, companies, createDb, plugins } from "@paperclipai/db";
+import { eq } from "drizzle-orm";
+import type { PluginRecord } from "@paperclipai/shared";
+import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
+import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
+import { linkSdkInto } from "../../../scripts/link-plugin-dev-sdk.mjs";
+import { fileURLToPath } from "node:url";
+import { createBundledPluginPackageResolver } from "../services/bundled-plugin-package.js";
+import { inspectCloudPackages } from "../services/cloud-package-qualification.js";
+import { resolveBundledPluginInstalls } from "../services/bundled-plugins.js";
+import { pluginRegistryService } from "../services/plugin-registry.js";
+import { pluginUiStaticRoutes } from "../routes/plugin-ui-static.js";
+import { errorHandler } from "../middleware/error-handler.js";
+import { derivePluginDatabaseNamespace } from "../services/plugin-database.js";
+import { ensureBundledPlugins } from "../services/bundled-plugins.js";
+import daytonaManifest from "../../../packages/plugins/sandbox-providers/daytona/src/manifest.js";
+import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+
+const externalDatabaseUrl = process.env.PAPERCLIP_BUNDLED_PLUGIN_TEST_DATABASE_URL?.trim();
+const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
+const describePostgres = support.supported ? describe : describe.skip;
+
+describePostgres("bundled image registration in PostgreSQL", () => {
+ let database: Awaited>;
+ let db: ReturnType;
+ let peer: ReturnType;
+ const roots: string[] = [];
+ const fixture = () => { const result = bundledImageFixture(); roots.push(result.root); return result; };
+ beforeAll(async () => {
+ if (externalDatabaseUrl) {
+ await applyPendingMigrations(externalDatabaseUrl);
+ database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
+ } else {
+ database = await startEmbeddedPostgresTestDatabase("bundled-image-registration-");
+ }
+ db = createDb(database.connectionString);
+ peer = createDb(database.connectionString);
+ }, 90_000);
+ afterEach(async () => {
+ await db.delete(plugins);
+ await db.delete(companies);
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+ });
+ afterAll(async () => { await database?.cleanup(); });
+
+ function runtime() {
+ const output: string[] = [];
+ const services = {
+ workerManager: {
+ startWorker: vi.fn(async (_id: string, options: { entrypointPath: string }) => { output.push(execFileSync(process.execPath, [options.entrypointPath, "--source"], { encoding: "utf8" })); }),
+ call: vi.fn(async () => ({})),
+ },
+ lifecycleManager: { markError: vi.fn() },
+ jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
+ toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
+ buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "fixture", hostVersion: "1.0.0" },
+ };
+ return { services, output };
+ }
+ function loaderFor(image: ReturnType, databaseClient = db, services?: PluginRuntimeServices) {
+ return pluginLoader(databaseClient, {
+ localPluginDir: path.join(image.root, "lost-home"), prebuiltOnly: true,
+ resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
+ enableLocalFilesystem: false, enableNpmDiscovery: false,
+ }, services);
+ }
+
+ it("starts current-image workers after losing the old home and retains each company's config", async () => {
+ const old = fixture();
+ const first = fixture();
+ const second = fixture();
+ await loaderFor(old).installPlugin({ localPath: old.packageRoot });
+ const registry = pluginRegistryService(db);
+ let row = (await registry.getByKey(old.manifest.id))!;
+ await registry.updateStatus(row.id, { status: "ready" });
+ const [companyA, companyB, unconfigured] = await db.insert(companies).values([
+ { name: "Company A", issuePrefix: "CFA" }, { name: "Company B", issuePrefix: "CFB" }, { name: "Unconfigured", issuePrefix: "CFC" },
+ ]).returning();
+ await registry.upsertConfig(row.id, companyA!.id, { configJson: { token: "only-A" } });
+ await registry.upsertConfig(row.id, companyB!.id, { configJson: { token: "only-B" } });
+ row = (await registry.getById(row.id))!;
+ rmSync(old.root, { recursive: true });
+ const original = { ...row };
+ for (const image of [first, second]) {
+ const current = runtime();
+ const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
+ const result = await loader.loadSingle(row.id);
+ expect(result.success).toBe(true);
+ expect(current.output).toEqual([expect.stringContaining(image.packageRoot)]);
+ expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({
+ entrypointPath: path.join(image.packageRoot, "dist", "worker.js"), config: {}, proactiveCompanyScopes: [companyA!.id, companyB!.id].sort(),
+ }));
+ expect(current.services.workerManager.call.mock.calls).toEqual(expect.arrayContaining([
+ [row.id, "configChanged", { companyId: companyA!.id, config: { token: "only-A" } }],
+ [row.id, "configChanged", { companyId: companyB!.id, config: { token: "only-B" } }],
+ ]));
+ expect(current.services.workerManager.call).toHaveBeenCalledTimes(2);
+ expect(await registry.getConfig(row.id, unconfigured!.id)).toBeNull();
+ const app = express();
+ app.use((req, _res, next) => {
+ req.actor = { type: "board", userId: "fixture-user", source: "session", isInstanceAdmin: false, companyIds: [companyA!.id] };
+ next();
+ });
+ app.use(pluginUiStaticRoutes(db, { localPluginDir: path.join(image.root, "lost-home"), resolvePackageRoot: image.resolver.resolve }));
+ app.use(errorHandler);
+ const served = await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyA!.id });
+ expect(served.status).toBe(200);
+ expect(served.text).toContain("current-image");
+ expect((await request(app).get(`/_plugins/${row.id}/ui/index.js`).query({ companyId: companyB!.id })).status).toBe(403);
+
+ }
+ expect(await registry.getById(row.id)).toEqual(original);
+ });
+
+ it("completes the real worker initialize and health protocol from a relocated image", async () => {
+ const old = fixture(); const image = fixture();
+ await loaderFor(old).installPlugin({ localPath: old.packageRoot });
+ const registry = pluginRegistryService(db);
+ const row = (await registry.getByKey(old.manifest.id))!;
+ await registry.updateStatus(row.id, { status: "ready" });
+ rmSync(old.root, { recursive: true });
+ const manager = createPluginWorkerManager();
+ try {
+ const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
+ const result = await loaderFor(image, db, services).loadSingle(row.id);
+ expect(result.success).toBe(true);
+ expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(image.packageRoot), configs: {} } });
+ } finally { await manager.stopAll(); }
+ });
+
+ it("qualifies and initializes the current release SDK/shared exports through the baked loader", async () => {
+ const image = fixture();
+ const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
+ const provider = mkdtempSync(path.join(catalogRoot, "sandbox-providers", "image-sdk-proof-"));
+ roots.push(provider);
+ cpSync(image.packageRoot, provider, { recursive: true });
+ linkSdkInto(provider);
+ writeFileSync(path.join(provider, "dist", "worker.js"), 'import { definePlugin, runWorker } from "@paperclipai/plugin-sdk"; runWorker(definePlugin({ async setup() {}, async onHealth() { return { status: "ok", message: "Current image SDK/shared loaded" }; } }), import.meta.url);');
+ const installs = [{ key: "sdk-proof", pluginKey: image.manifest.id, localPath: provider }];
+ const resolver = createBundledPluginPackageResolver(installs);
+ const manager = createPluginWorkerManager();
+ const loader = pluginLoader(db, {
+ prebuiltOnly: true, resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
+ enableLocalFilesystem: false, enableNpmDiscovery: false,
+ }, { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices);
+ const registry = pluginRegistryService(db);
+ try {
+ await loader.installPlugin({ localPath: provider });
+ const installed = (await registry.getByKey(image.manifest.id))!;
+ await registry.updateStatus(installed.id, { status: "ready" });
+ const registration = (await registry.getById(installed.id))!;
+ expect((await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) })).blockers).toEqual([]);
+ expect((await loader.loadSingle(installed.id)).success).toBe(true);
+ expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Current image SDK/shared loaded" });
+ } finally { await manager.stopAll(); }
+ }, 90_000);
+
+ it.skipIf(!existsSync(fileURLToPath(new URL("../../../packages/plugins/sandbox-providers/daytona/dist/worker.js", import.meta.url))))("qualifies and initializes the actual prebuilt Daytona release package", async () => {
+ const catalogRoot = fileURLToPath(new URL("../../../packages/plugins", import.meta.url));
+ const installs = resolveBundledPluginInstalls(["daytona"], { catalogRoot, env: {}, enforceCatalogRoot: true, distributionPlugins: [] });
+ const resolver = createBundledPluginPackageResolver(installs);
+ const manager = createPluginWorkerManager();
+ const services = { ...runtime().services, workerManager: manager } as unknown as PluginRuntimeServices;
+ const loader = pluginLoader(db, {
+ localPluginDir: "/lost/release/home/plugins", prebuiltOnly: true,
+ resolvePackageRoot: resolver.resolve, assertPackageActivation: resolver.assertActivation,
+ enableLocalFilesystem: false, enableNpmDiscovery: false,
+ }, services);
+ const registry = pluginRegistryService(db);
+ vi.stubEnv("DAYTONA_API_KEY", "");
+ try {
+ await loader.installPlugin({ localPath: installs[0]!.localPath });
+ const installed = (await registry.getByKey(daytonaManifest.id))!;
+ await registry.update(installed.id, { packagePath: "/lost/release/home/daytona" });
+ await registry.updateStatus(installed.id, { status: "ready" });
+ const registration = (await registry.getById(installed.id))!;
+ const inspected = await inspectCloudPackages({ installs, catalogRoot, registrations: [registration], loadManifest: (root) => loader.loadManifest(root) });
+ expect(inspected.blockers).toEqual([]);
+ expect(inspected.packages[0]?.packageDigest).toMatch(/^sha256:/);
+ expect((await loader.loadSingle(installed.id)).success).toBe(true);
+ expect(await manager.call(installed.id, "health", {})).toMatchObject({ status: "ok", message: "Daytona sandbox provider plugin healthy" });
+ } finally { await manager.stopAll(); vi.unstubAllEnvs(); }
+ }, 90_000);
+
+ it("registers the same package concurrently from two roots without replacing the winner", async () => {
+ const first = fixture(); const second = fixture();
+ await Promise.all([
+ loaderFor(first).installPlugin({ localPath: first.packageRoot }),
+ loaderFor(second, peer).installPlugin({ localPath: second.packageRoot }),
+ ]);
+ const rows = await pluginRegistryService(db).listInstalled();
+ expect(rows).toHaveLength(1);
+ const original = rows[0]!;
+ expect([first.packageRoot, second.packageRoot]).toContain(original.packagePath);
+ await loaderFor(second, peer).installPlugin({ localPath: second.packageRoot });
+ expect(await pluginRegistryService(db).getById(original.id)).toEqual(original);
+ await expect(pluginRegistryService(peer).install({ packageName: original.packageName }, { ...first.manifest, description: "different package metadata" }, { idempotent: true })).rejects.toThrow(/already installed/);
+ });
+
+ it("preserves disabled registrations and avoids path writes during managed boot", async () => {
+ const old = fixture(); const image = fixture();
+ await loaderFor(old).installPlugin({ localPath: old.packageRoot });
+ const registry = pluginRegistryService(db);
+ const row = (await registry.getByKey(old.manifest.id))!;
+ await db.update(plugins).set({ status: "disabled" }).where(eq(plugins.id, row.id));
+ const original = await registry.getById(row.id);
+ await ensureBundledPlugins(image.installs, { registry, loader: loaderFor(image), lifecycle: { load: vi.fn() }, logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }, { reinstallUninstalled: true, preservePackagePath: true });
+ expect(await registry.getById(row.id)).toEqual(original);
+ });
+
+ it("fails before worker startup for missing image bytes without using a home fallback or building", async () => {
+ const image = fixture();
+ await loaderFor(image).installPlugin({ localPath: image.packageRoot });
+ const registry = pluginRegistryService(db);
+ const row = (await registry.getByKey(image.manifest.id))!;
+ await registry.updateStatus(row.id, { status: "ready" });
+ const fallback = path.join(image.root, "lost-home", "node_modules", row.packageName);
+ mkdirSync(fallback, { recursive: true });
+ writeFileSync(path.join(fallback, "package.json"), JSON.stringify({ name: row.packageName }));
+ writeFileSync(path.join(fallback, "worker.js"), "throw new Error('must not start');");
+ rmSync(path.join(image.packageRoot, "dist", "worker.js"));
+ const current = runtime();
+ const loader = loaderFor(image, db, current.services as unknown as PluginRuntimeServices);
+ const result = await loader.loadSingle(row.id);
+ expect(result.success).toBe(false);
+ expect(result.error).toContain("ENOENT");
+ expect(current.services.workerManager.startWorker).not.toHaveBeenCalled();
+ await expect(loader.installPlugin({ packageName: row.packageName })).rejects.toThrow(/current image catalog/);
+ await loader.cleanupInstallArtifacts(row as PluginRecord);
+ expect(image.resolver.inspect(row, image.manifest).compatible).toBe(false);
+ });
+
+ it("applies plugin migrations from the current image after the legacy package disappears", async () => {
+ const image = bundledImageFixture({
+ capabilities: ["database.namespace.migrate", "database.namespace.read", "database.namespace.write"],
+ database: { migrationsDir: "migrations" },
+ });
+ roots.push(image.root);
+ const [row] = await db.insert(plugins).values({
+ ...image.registration, packagePath: "/lost/legacy/home/plugin", apiVersion: 1, categories: image.manifest.categories, status: "ready",
+ }).returning();
+ const current = runtime();
+ const result = await loaderFor(image, db, current.services as unknown as PluginRuntimeServices).loadSingle(row!.id);
+ expect(result.success).toBe(true);
+ expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row!.id, expect.objectContaining({ databaseNamespace: derivePluginDatabaseNamespace(image.manifest.id) }));
+ await db.execute((await import("drizzle-orm")).sql.raw(`DROP SCHEMA "${derivePluginDatabaseNamespace(image.manifest.id)}" CASCADE`));
+ });
+
+ it("retains registry-install provider credentials for an ordinary npm package", async () => {
+ const npmManifest = { ...daytonaManifest, id: "paperclip.image-npm-fixture" };
+ const image = bundledImageFixture(npmManifest);
+ roots.push(image.root);
+ const localPluginDir = path.join(image.root, "npm-home");
+ const packageRoot = path.join(localPluginDir, "node_modules", "@paperclipai", "plugin-daytona");
+ mkdirSync(path.dirname(packageRoot), { recursive: true });
+ cpSync(image.packageRoot, packageRoot, { recursive: true });
+ const metadata = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8"));
+ metadata.name = "@paperclipai/plugin-daytona";
+ writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify(metadata));
+ const registry = pluginRegistryService(db);
+ const row = (await registry.install({ packageName: metadata.name }, npmManifest))!;
+ await registry.updateStatus(row.id, { status: "ready" });
+ vi.stubEnv("DAYTONA_API_KEY", "synthetic-provider-key");
+ try {
+ const current = runtime();
+ const loader = pluginLoader(db, { localPluginDir, enableNpmDiscovery: false, enableLocalFilesystem: false }, current.services as unknown as PluginRuntimeServices);
+ expect((await loader.loadSingle(row.id)).success).toBe(true);
+ expect(current.services.workerManager.startWorker).toHaveBeenCalledWith(row.id, expect.objectContaining({ env: expect.objectContaining({ DAYTONA_API_KEY: "synthetic-provider-key" }) }));
+ } finally { vi.unstubAllEnvs(); }
+ });
+
+ it("keeps self-hosted local-path loading available without a managed resolver", async () => {
+ const image = fixture();
+ const loader = pluginLoader(db, { localPluginDir: path.join(image.root, "selfhost"), enableNpmDiscovery: false, enableLocalFilesystem: false });
+ await loader.installPlugin({ localPath: image.packageRoot });
+ const row = (await pluginRegistryService(db).getByKey(image.manifest.id))!;
+ expect(row.packagePath).toBe(image.packageRoot);
+ await expect(loader.installPlugin({ localPath: image.packageRoot })).rejects.toThrow(/already installed/);
+ });
+});
diff --git a/server/src/__tests__/bundled-plugin-runtime.test.ts b/server/src/__tests__/bundled-plugin-runtime.test.ts
new file mode 100644
index 00000000000..936687a91be
--- /dev/null
+++ b/server/src/__tests__/bundled-plugin-runtime.test.ts
@@ -0,0 +1,72 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { afterEach, describe, expect, it } from "vitest";
+import { build, transform } from "esbuild";
+import { bundleRuntime } from "../../scripts/bundle-runtime.mjs";
+import { resolveBundledPluginRuntimeLoader } from "../services/bundled-plugin-runtime.js";
+
+const roots: string[] = [];
+function fixture() {
+ const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-plugin-runtime-")));
+ roots.push(root);
+ fs.mkdirSync(path.join(root, "server", "dist-runtime"), { recursive: true });
+ fs.writeFileSync(path.join(root, "server", "package.json"), JSON.stringify({ name: "@paperclipai/server", type: "module" }));
+ const loader = path.join(root, "cli", "node_modules", "tsx", "dist", "loader.mjs");
+ fs.mkdirSync(path.dirname(loader), { recursive: true });
+ fs.writeFileSync(loader, "export {};\n");
+ return { root, loader };
+}
+afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
+
+describe("current image plugin runtime loader", () => {
+ it.each(["src/services", "dist/services", "dist-runtime"])("resolves the owning release from server/%s", (location) => {
+ const { root, loader } = fixture();
+ const moduleDirectory = path.join(root, "server", location);
+ fs.mkdirSync(moduleDirectory, { recursive: true });
+ expect(resolveBundledPluginRuntimeLoader(moduleDirectory)).toBe(loader);
+ });
+
+ it("resolves a bundled entrypoint without preserved source module locations", async () => {
+ const { root, loader } = fixture();
+ const runtime = path.join(root, "server", "dist-runtime", "index.mjs");
+ await build({
+ stdin: { contents: `import { resolveBundledPluginRuntimeLoader } from ${JSON.stringify(fileURLToPath(new URL("../services/bundled-plugin-runtime.ts", import.meta.url)))}; process.stdout.write(resolveBundledPluginRuntimeLoader());`, resolveDir: process.cwd() },
+ outfile: runtime, bundle: true, platform: "node", format: "esm", target: "node24", logLevel: "silent",
+ });
+ expect(execFileSync(process.execPath, [runtime], { encoding: "utf8" })).toBe(loader);
+ });
+
+ it("uses the production bundler's preserved module location at the dist-runtime entrypoint", async () => {
+ const { root, loader } = fixture();
+ const service = path.join(root, "server", "dist", "services", "bundled-plugin-runtime.js");
+ fs.mkdirSync(path.dirname(service), { recursive: true });
+ const source = fs.readFileSync(new URL("../services/bundled-plugin-runtime.ts", import.meta.url), "utf8");
+ fs.writeFileSync(service, (await transform(source, { loader: "ts", format: "esm", target: "node24" })).code);
+ const entry = path.join(root, "server", "dist", "probe.js");
+ fs.writeFileSync(entry, 'import { resolveBundledPluginRuntimeLoader } from "./services/bundled-plugin-runtime.js"; process.stdout.write(resolveBundledPluginRuntimeLoader());');
+ const bundled = await bundleRuntime({ entry, outdir: path.join(root, "server", "dist-runtime") });
+ expect(execFileSync(process.execPath, [bundled.outfile], { encoding: "utf8" })).toBe(loader);
+ });
+
+ it("accepts the baked pnpm target inside the current image node_modules", () => {
+ const { root, loader } = fixture();
+ const physical = path.join(root, "node_modules", ".pnpm", "tsx@1", "node_modules", "tsx", "dist", "loader.mjs");
+ fs.mkdirSync(path.dirname(physical), { recursive: true });
+ fs.writeFileSync(physical, "export {};\n");
+ fs.rmSync(loader); fs.symlinkSync(path.relative(path.dirname(loader), physical), loader);
+ expect(resolveBundledPluginRuntimeLoader(path.join(root, "server", "dist-runtime"))).toBe(physical);
+ });
+
+ it("fails explicitly for missing or legacy-home loaders", () => {
+ const { root, loader } = fixture();
+ const moduleDirectory = path.join(root, "server", "dist-runtime");
+ fs.rmSync(loader);
+ expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("missing from the current image");
+ const outside = fixture().loader;
+ fs.symlinkSync(outside, loader);
+ expect(() => resolveBundledPluginRuntimeLoader(moduleDirectory)).toThrow("escapes the current image");
+ });
+});
diff --git a/server/src/__tests__/bundled-plugins.test.ts b/server/src/__tests__/bundled-plugins.test.ts
index 5dfd5e8f90e..d66b0d849da 100644
--- a/server/src/__tests__/bundled-plugins.test.ts
+++ b/server/src/__tests__/bundled-plugins.test.ts
@@ -9,6 +9,7 @@ import {
ensureBundledPlugins,
resolveBundledCatalogRoot,
resolveBundledPluginInstalls,
+ resolveManagedBundledPluginKeys,
type BundledPluginProvisionerDeps,
type ResolvedBundledPlugin,
} from "../services/bundled-plugins.js";
@@ -30,6 +31,21 @@ afterAll(() => {
// Catalog-root resolution (fail-to-start allowlist)
// ---------------------------------------------------------------------------
+describe("managed bundled plugin selection", () => {
+ it.each([
+ { env: {}, selected: undefined, expected: null },
+ { env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: undefined, expected: [] },
+ { env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: " " }, selected: undefined, expected: null },
+ { env: { PAPERCLIP_MANAGED_CONFIG: "" }, selected: undefined, expected: [] },
+ { env: { PAPERCLIP_MANAGED_CONFIG: "{}" }, selected: null, expected: [] },
+ { env: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-fixture" }, selected: ["daytona"], expected: ["daytona"] },
+ { env: {}, selected: ["modal"], expected: ["modal"] },
+ { env: {}, selected: [], expected: [] },
+ ])("uses the canonical Cloud floor and preserves explicit selection: $env / $selected", ({ env, selected, expected }) => {
+ expect(resolveManagedBundledPluginKeys(selected, env)).toEqual(expected);
+ });
+});
+
describe("resolveBundledPluginInstalls", () => {
it("resolves the optional CreateOS sandbox provider without making it a self-hosted default", () => {
expect(resolveBundledPluginInstalls(["createos"], {
diff --git a/server/src/__tests__/cloud-bundled-package-digest.test.ts b/server/src/__tests__/cloud-bundled-package-digest.test.ts
new file mode 100644
index 00000000000..7978c46795e
--- /dev/null
+++ b/server/src/__tests__/cloud-bundled-package-digest.test.ts
@@ -0,0 +1,93 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
+
+const roots: string[] = [];
+function file(name: string, content = "export const value = 1;") {
+ fs.mkdirSync(path.dirname(name), { recursive: true }); fs.writeFileSync(name, content);
+}
+function link(target: string, name: string) {
+ fs.mkdirSync(path.dirname(name), { recursive: true }); fs.symlinkSync(path.relative(path.dirname(name), target), name);
+}
+function fixture() {
+ const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "image-package-graph-"))); roots.push(root);
+ const catalogRoot = path.join(root, "packages", "plugins");
+ const provider = path.join(catalogRoot, "sandbox-providers", "daytona");
+ const sdk = path.join(catalogRoot, "sdk");
+ const shared = path.join(root, "packages", "shared");
+ const dependency = path.join(root, "node_modules", ".pnpm", "dependency@1", "node_modules", "dependency");
+ const transitive = path.join(root, "node_modules", ".pnpm", "transitive@1", "node_modules", "transitive");
+ for (const directory of [provider, sdk, shared, dependency, transitive]) file(path.join(directory, "index.js"));
+ link(sdk, path.join(provider, "node_modules", "@paperclipai", "plugin-sdk"));
+ link(shared, path.join(sdk, "node_modules", "@paperclipai", "shared"));
+ link(dependency, path.join(shared, "node_modules", "dependency"));
+ link(transitive, path.join(path.dirname(dependency), "transitive"));
+ return { root, catalogRoot, provider, sdk, shared, dependency, transitive };
+}
+afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
+
+describe("bounded current-image package dependency inventory", () => {
+ it("hashes external SDK, shared, and pnpm sibling dependency bytes", () => {
+ const image = fixture();
+ const digest = () => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
+ let previous = digest();
+ for (const directory of [image.sdk, image.shared, image.dependency, image.transitive]) {
+ file(path.join(directory, "index.js"), "export const value = 2;");
+ const next = digest(); expect(next).not.toBe(previous); previous = next;
+ }
+ });
+
+ it("keeps the digest stable when an identical release moves to a new root", () => {
+ const original = fixture(); const replacement = fixture();
+ expect(cloudBundledPackageDigest(original.provider, { catalogRoot: original.catalogRoot }))
+ .toBe(cloudBundledPackageDigest(replacement.provider, { catalogRoot: replacement.catalogRoot }));
+ });
+
+ it("rejects legacy-home links and preserves package-contained defaults", () => {
+ const image = fixture();
+ expect(() => cloudBundledPackageDigest(image.provider)).toThrow("escapes its package");
+ const outside = fixture();
+ link(path.join(outside.sdk, "index.js"), path.join(image.shared, "legacy-home.js"));
+ expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("authorized image roots");
+ });
+
+ it("deduplicates cycles without scanning unrelated image node_modules", () => {
+ const image = fixture();
+ link(image.sdk, path.join(image.shared, "node_modules", "cycle"));
+ const unrelated = path.join(image.root, "node_modules", "unrelated", "special");
+ file(unrelated);
+ const first = cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot });
+ file(unrelated, "changed unrelated code");
+ expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toBe(first);
+ });
+
+ it.each(["", ".pnpm", ".pnpm/node_modules"])("rejects aggregate image dependency targets: %s", (aggregate) => {
+ const image = fixture();
+ const target = path.join(image.root, "node_modules", aggregate);
+ fs.mkdirSync(target, { recursive: true });
+ link(target, path.join(image.sdk, "node_modules", "aggregate"));
+ expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("select an individual package");
+ });
+
+ it("accepts the shipped provider's observed entry class", () => {
+ const image = fixture();
+ const directory = path.join(image.provider, "many"); fs.mkdirSync(directory);
+ for (let index = 0; index < 16_000; index++) fs.writeFileSync(path.join(directory, `${index}.js`), "");
+ expect(cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toMatch(/^sha256:/);
+ });
+
+ it("rejects oversized metadata and excessive depth", () => {
+ const image = fixture();
+ file(path.join(image.sdk, "package.json"), " ".repeat(128 * 1024 + 1));
+ expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("metadata");
+ fs.rmSync(path.join(image.sdk, "package.json"));
+ const large = path.join(image.shared, "large.js");
+ fs.closeSync(fs.openSync(large, "w")); fs.truncateSync(large, 256 * 1024 * 1024 + 1);
+ expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("byte limit");
+ fs.rmSync(large);
+ file(path.join(image.sdk, ...Array.from({ length: 33 }, () => "deep"), "file.js"));
+ expect(() => cloudBundledPackageDigest(image.provider, { catalogRoot: image.catalogRoot })).toThrow("depth limit");
+ });
+});
diff --git a/server/src/__tests__/cloud-configuration-migration-postgres.test.ts b/server/src/__tests__/cloud-configuration-migration-postgres.test.ts
new file mode 100644
index 00000000000..dd747d06ac6
--- /dev/null
+++ b/server/src/__tests__/cloud-configuration-migration-postgres.test.ts
@@ -0,0 +1,325 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, plugins, type Db } from "@paperclipai/db";
+import { eq, inArray, sql } from "drizzle-orm";
+import { paperclipConfigSchema } from "@paperclipai/shared";
+import { adapterSettingsService, initializeAdapterSettings } from "../services/adapter-settings.js";
+import { cloudRuntimeConfigService, initializeCloudRuntimeConfig } from "../services/cloud-runtime-config.js";
+import { cloudConfigurationMigrationService } from "../services/cloud-configuration-migration.js";
+import { qualifyCloudConfigurationForCurrentImage } from "../services/cloud-configuration-qualification.js";
+import { runCloudConfigurationQualification } from "../qualify-cloud-configuration.js";
+import { pluginLoader, type PluginRuntimeServices } from "../services/plugin-loader.js";
+import { pluginRegistryService } from "../services/plugin-registry.js";
+import { createPluginWorkerManager } from "../services/plugin-worker-manager.js";
+import { ensureBundledPlugins } from "../services/bundled-plugins.js";
+import { bundledImageFixture } from "./helpers/bundled-image-fixture.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+
+const externalDatabaseUrl = process.env.PAPERCLIP_CLOUD_CONFIGURATION_TEST_DATABASE_URL?.trim();
+const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
+const keys = ["adapter-settings:v1", "cloud-runtime-config:v1", "cloud-configuration-migration:v1"];
+(support.supported ? describe : describe.skip)("Cloud configuration migration and current-image qualification", () => {
+ let database: Awaited>;
+ let db: Db;
+ let peer: Db;
+ let homeA: string;
+ let homeB: string;
+ const roots: string[] = [];
+ const fixture = (version = "1.0.0") => { const image = bundledImageFixture({ version }); roots.push(image.root); return image; };
+ const env = (home = homeA) => ({ PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "test-cloud-token", PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json") });
+ const migration = (client = db, home = homeA) => cloudConfigurationMigrationService(client, { runtimeEnv: env(home) });
+ beforeAll(async () => {
+ if (externalDatabaseUrl) {
+ await applyPendingMigrations(externalDatabaseUrl);
+ database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl) };
+ } else database = await startEmbeddedPostgresTestDatabase("cloud-configuration-migration-");
+ db = createDb(database.connectionString);
+ peer = createDb(database.connectionString);
+ }, 90_000);
+ beforeEach(async () => {
+ homeA = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-legacy-A-"));
+ homeB = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-replacement-B-"));
+ roots.push(homeA, homeB);
+ await db.delete(instanceSettings).where(inArray(instanceSettings.singletonKey, keys));
+ await db.delete(plugins);
+ });
+ afterEach(async () => {
+ vi.restoreAllMocks();
+ for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true });
+ });
+ afterAll(async () => { await database?.cleanup(); });
+
+ async function legacy() {
+ const config = paperclipConfigSchema.parse({
+ $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
+ database: { mode: "postgres" }, logging: { mode: "cloud" },
+ server: { port: 3199 }, telemetry: { enabled: false },
+ });
+ await fs.writeFile(path.join(homeA, "config.json"), JSON.stringify(config));
+ await fs.writeFile(path.join(homeA, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local", "future_adapter"] }));
+ await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
+ return config;
+ }
+
+ function loaderFor(image: ReturnType, client = db, services?: PluginRuntimeServices) {
+ return pluginLoader(client, {
+ localPluginDir: path.join(homeB, "plugins"), prebuiltOnly: true,
+ resolvePackageRoot: image.resolver.resolve, assertPackageActivation: image.resolver.assertActivation,
+ enableLocalFilesystem: false, enableNpmDiscovery: false,
+ }, services);
+ }
+ async function install(image: ReturnType) {
+ await loaderFor(image).installPlugin({ localPath: image.packageRoot });
+ const registry = pluginRegistryService(db);
+ const row = (await registry.getByKey(image.manifest.id))!;
+ await registry.updateStatus(row.id, { status: "ready" });
+ return row;
+ }
+ function qualify(image: ReturnType, client = db, home = homeA, expectedPackageEvidenceDigest?: string) {
+ const loader = loaderFor(image, client);
+ return migration(client, home).qualify({ installs: image.installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest });
+ }
+
+ it("imports A atomically, retains later DB revisions, and starts B's real image worker after A disappears", async () => {
+ await legacy();
+ const oldImage = fixture();
+ const newImage = fixture();
+ const row = await install(oldImage);
+ const first = await migration().migrate();
+ expect(first.migration.adapter.authority).toBe("legacy_file");
+ expect(first.migration.config.authority).toBe("legacy_file");
+ const original = await qualify(oldImage);
+ expect(original.configurationPackagesEligible).toBe(true);
+ expect(original.samePackageOverlapEligible).toBe(false);
+ await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
+ const config = (await cloudRuntimeConfigService(peer).inspectStored())!;
+ expect(config.config).not.toBeNull();
+ await cloudRuntimeConfigService(peer).replace({ ...config.config!, server: { ...config.config!.server, port: 3299 } }, config.revision);
+ await fs.rm(homeA, { recursive: true });
+ await fs.rm(oldImage.root, { recursive: true });
+ expect((await migration(peer, homeB).migrate()).migration).toEqual(first.migration);
+ await initializeAdapterSettings(peer, { runtimeEnv: env(homeB) });
+ const restored = await initializeCloudRuntimeConfig(peer, { runtimeEnv: env(homeB) });
+ expect(restored?.current.config?.server.port).toBe(3299);
+ expect(await adapterSettingsService(peer, { runtimeEnv: env(homeB) }).getDisabledTypes()).toEqual(["claude_local", "codex_local", "future_adapter"]);
+ const replacement = await qualify(newImage, peer, homeB, original.packageEvidenceDigest);
+ expect(replacement).toMatchObject({ adapterRevision: 1, configRevision: 1, configurationPackagesEligible: true, samePackageOverlapEligible: true });
+ expect(replacement.configurationDigest).not.toBe(original.configurationDigest);
+ const manager = createPluginWorkerManager();
+ try {
+ const services = {
+ workerManager: manager, lifecycleManager: { markError: vi.fn() },
+ jobStore: { syncJobDeclarations: vi.fn() }, jobScheduler: { registerPlugin: vi.fn() },
+ toolDispatcher: {}, eventBus: { forPlugin: vi.fn(), subscriptionCount: vi.fn(() => 0) },
+ buildHostHandlers: vi.fn(() => ({})), instanceInfo: { instanceId: "qualification-fixture", hostVersion: "1.0.0" },
+ } as unknown as PluginRuntimeServices;
+ expect((await loaderFor(newImage, peer, services).loadSingle(row.id)).success).toBe(true);
+ expect(await manager.call(row.id, "health", {})).toMatchObject({ status: "ok", details: { source: expect.stringContaining(newImage.packageRoot) } });
+ } finally { await manager.stopAll(); }
+ });
+
+ it("serializes concurrent attempts and never imports stale inputs over later DB changes", async () => {
+ await legacy();
+ const [first, second] = await Promise.all([migration().migrate(), migration(peer).migrate()]);
+ expect(first).toEqual(second);
+ await adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("claude_local", false);
+ await fs.writeFile(path.join(homeA, "config.json"), "invalid stale file");
+ await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid stale file");
+ expect(await migration().migrate()).toEqual(first);
+ expect(await adapterSettingsService(db, { runtimeEnv: env() }).getDisabledTypes()).toEqual(["future_adapter"]);
+ });
+
+ it("rolls back both seeds and the receipt if completion fails, then retries after the crash boundary", async () => {
+ await legacy();
+ await db.execute(sql`CREATE FUNCTION reject_cloud_migration_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.singleton_key = 'cloud-configuration-migration:v1' THEN RAISE EXCEPTION 'test receipt failure'; END IF; RETURN NEW; END $$`);
+ await db.execute(sql`CREATE TRIGGER reject_cloud_migration_receipt BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_cloud_migration_receipt()`);
+ try {
+ await expect(migration().migrate()).rejects.toMatchObject({ cause: { message: expect.stringContaining("test receipt failure") } });
+ expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
+ } finally {
+ await db.execute(sql`DROP TRIGGER reject_cloud_migration_receipt ON instance_settings`);
+ await db.execute(sql`DROP FUNCTION reject_cloud_migration_receipt()`);
+ }
+ expect((await migration(peer).migrate()).revision).toBe(0);
+ expect((await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).length).toBe(3);
+ });
+
+ it("fails without partial authority for malformed legacy config and preserves already-authoritative rows", async () => {
+ await legacy();
+ await fs.writeFile(path.join(homeA, "config.json"), "invalid");
+ await expect(migration().migrate()).rejects.toThrow("Legacy Cloud runtime configuration");
+ expect(await peer.select().from(instanceSettings).where(inArray(instanceSettings.singletonKey, keys))).toEqual([]);
+ await adapterSettingsService(db, { runtimeEnv: env() }).seedIfAbsent({ disabledTypes: ["codex_local"] });
+ await cloudRuntimeConfigService(db).seedIfAbsent(null);
+ await fs.writeFile(path.join(homeA, "adapter-settings.json"), "invalid");
+ const receipt = await migration(peer).migrate();
+ expect(receipt.migration.adapter.authority).toBe("database");
+ expect(receipt.migration.config.authority).toBe("database");
+ expect((await cloudRuntimeConfigService(peer).inspectStored())?.config).toBeNull();
+ });
+
+ it("returns database errors instead of falling back to files or a completed receipt", async () => {
+ await legacy();
+ await migration().migrate();
+ const broken = createDb(database.connectionString);
+ await broken.$client.end({ timeout: 0 });
+ await expect(migration(broken).migrate()).rejects.toThrow();
+ const image = fixture();
+ await expect(qualify(image, broken)).rejects.toThrow();
+ });
+
+ it("keeps external registration blockers durable when B has no legacy home and requires deliberate original-source resolution", async () => {
+ await legacy();
+ const sentinel = path.join(homeA, "external-code-executed");
+ await fs.writeFile(path.join(homeA, "external.js"), `require('node:fs').writeFileSync(${JSON.stringify(sentinel)}, 'executed');`);
+ await fs.writeFile(path.join(homeA, "adapter-plugins.json"), JSON.stringify([{ type: "legacy_external", packageName: "legacy-adapter", localPath: path.join(homeA, "external.js") }]));
+ const image = fixture();
+ await install(image);
+ await migration().migrate();
+ expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "unsupported_external_adapters" }));
+ await expect(fs.stat(sentinel)).rejects.toMatchObject({ code: "ENOENT" });
+ await migration(peer, homeB).migrate();
+ const replacement = await qualify(image, peer, homeB);
+ expect(replacement.configurationPackagesEligible).toBe(false);
+ await expect(migration(peer, homeB).recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("original legacy source");
+ await fs.unlink(path.join(homeA, "adapter-plugins.json"));
+ await expect(migration().recheckLegacyExternalRegistrations(replacement.receiptRevision)).rejects.toThrow("present, valid registration file");
+ expect((await migration().inspectStored())?.migration.externalRegistrations.blocker).toBe("unsupported_external_adapters");
+ await fs.writeFile(path.join(homeA, "adapter-plugins.json"), "[]");
+ await expect(migration().recheckLegacyExternalRegistrations(0)).rejects.toThrow("revision conflict");
+ await migration().recheckLegacyExternalRegistrations(replacement.receiptRevision);
+ expect((await qualify(image)).configurationPackagesEligible).toBe(true);
+ });
+
+ it("retains a metadata blocker for malformed or oversized registration files", async () => {
+ await legacy();
+ await fs.writeFile(path.join(homeA, "adapter-plugins.json"), Buffer.alloc(65_537));
+ const receipt = await migration().migrate();
+ expect(receipt.migration.externalRegistrations.blocker).toBe("invalid_external_registration_metadata");
+ await fs.rm(homeA, { recursive: true });
+ expect((await migration(peer, homeB).migrate()).migration.externalRegistrations).toEqual(receipt.migration.externalRegistrations);
+ });
+
+ it("records image mismatches before reconciliation, permits maintenance upgrades, and requires explicit fresh overlap evidence", async () => {
+ await legacy();
+ const oldImage = fixture();
+ const nextImage = fixture("1.1.0");
+ await install(oldImage);
+ await migration().migrate();
+ const old = await qualify(oldImage);
+ const mismatch = await qualify(nextImage, peer, homeB, old.packageEvidenceDigest);
+ expect(mismatch.configurationPackagesEligible).toBe(false);
+ expect(mismatch.samePackageOverlapEligible).toBe(false);
+ expect(mismatch.blockers).toContainEqual(expect.objectContaining({ code: "incompatible_bundled_package", registeredVersion: "1.0.0", imageVersion: "1.1.0" }));
+ const registry = pluginRegistryService(db);
+ await ensureBundledPlugins(nextImage.installs, {
+ registry, loader: loaderFor(nextImage), lifecycle: { load: vi.fn() },
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+ }, { reinstallUninstalled: true, preservePackagePath: true });
+ const current = await qualify(nextImage);
+ expect(current.configurationPackagesEligible).toBe(true);
+ expect(current.samePackageOverlapEligible).toBe(false);
+ expect((await migration().inspectStored())?.lastPackageMismatch?.packageEvidenceDigest).toBe(mismatch.packageEvidenceDigest);
+ expect((await qualify(nextImage, peer, homeB, current.packageEvidenceDigest)).samePackageOverlapEligible).toBe(true);
+ });
+
+ it("fails qualification for missing package or manifest bytes without loading a home fallback", async () => {
+ await legacy();
+ const image = fixture();
+ await install(image);
+ await migration().migrate();
+ const good = await qualify(image);
+ await fs.rm(path.join(image.packageRoot, "dist", "manifest.js"));
+ const broken = await qualify(image, peer, homeB, good.packageEvidenceDigest);
+ expect(broken.configurationPackagesEligible).toBe(false);
+ expect(broken.samePackageOverlapEligible).toBe(false);
+ expect(broken.blockers.length).toBeGreaterThan(0);
+ await fs.rm(image.packageRoot, { recursive: true });
+ expect((await qualify(image)).blockers).toContainEqual(expect.objectContaining({ code: "missing_bundled_package" }));
+ });
+
+ it("rejects changed image bytes before manifest import and never uses stale completed qualification", async () => {
+ await legacy();
+ const image = fixture();
+ await install(image);
+ await migration().migrate();
+ const original = await qualify(image);
+ await fs.appendFile(path.join(image.packageRoot, "dist", "worker.js"), "\nprocess.exit(99);\n");
+ const loadManifest = vi.fn();
+ const current = await migration(peer, homeB).qualify({ installs: image.installs, loadManifest, expectedPackageEvidenceDigest: original.packageEvidenceDigest });
+ expect(current.configurationPackagesEligible).toBe(false);
+ expect(current.samePackageOverlapEligible).toBe(false);
+ expect(current.blockers).toContainEqual(expect.objectContaining({ code: "package_integrity_mismatch" }));
+ expect(loadManifest).not.toHaveBeenCalled();
+ });
+
+ it("allows an administrator update while manifest inspection waits and refuses to certify the stale snapshot", async () => {
+ await legacy();
+ const image = fixture();
+ await install(image);
+ await migration().migrate();
+ let started!: () => void;
+ let resume!: () => void;
+ const entered = new Promise((resolve) => { started = resolve; });
+ const release = new Promise((resolve) => { resume = resolve; });
+ const loader = loaderFor(image);
+ const pending = migration().qualify({ installs: image.installs, loadManifest: async (root) => { started(); await release; return loader.loadManifest(root); } });
+ await entered;
+ try {
+ const change = adapterSettingsService(peer, { runtimeEnv: env() }).setDisabled("codex_local", true);
+ let timer: ReturnType | undefined;
+ try {
+ expect(await Promise.race([change, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(new Error("Admin update was blocked by image inspection")), 2_000); })])).toBe(true);
+ } finally { clearTimeout(timer); }
+ } finally { resume(); }
+ const result = await pending;
+ expect(result).toMatchObject({ configurationPackagesEligible: false, adapterRevision: 1 });
+ expect(result.blockers).toContainEqual(expect.objectContaining({ code: "configuration_changed_during_inspection" }));
+ expect((await qualify(image)).configurationPackagesEligible).toBe(true);
+ });
+
+ it("uses the current managed catalog through the operator command and durably records catalog failure", async () => {
+ await legacy();
+ const image = fixture();
+ await install(image);
+ await migration().migrate();
+ const runtimeEnv = {
+ ...env(homeB), DATABASE_URL: database.connectionString, PAPERCLIP_BUNDLED_PLUGIN_ROOT: image.root,
+ PAPERCLIP_MANAGED_CONFIG: JSON.stringify({ v: 1, mode: "cloud", catalogVersion: "fixture", features: {}, plugins: { autoInstall: ["fixture"] } }),
+ };
+ const initial = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv });
+ expect(initial?.configurationPackagesEligible).toBe(true);
+ const stdout = vi.spyOn(process.stdout, "write").mockImplementation(() => true);
+ try {
+ expect(await runCloudConfigurationQualification(["--expected-package-evidence", initial!.packageEvidenceDigest], runtimeEnv)).toBe(0);
+ const report = stdout.mock.calls.map((call) => String(call[0])).find((text) => text.startsWith('{\n "receiptRevision"'));
+ expect(report && JSON.parse(report).samePackageOverlapEligible).toBe(true);
+ } finally { stdout.mockRestore(); }
+ const missing = await qualifyCloudConfigurationForCurrentImage(peer, { runtimeEnv: { ...runtimeEnv, PAPERCLIP_MANAGED_CONFIG: runtimeEnv.PAPERCLIP_MANAGED_CONFIG.replace('"fixture"]', '"absent"]') } });
+ expect(missing?.configurationPackagesEligible).toBe(false);
+ expect(missing?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
+ expect((await migration().inspectStored())?.lastPackageMismatch?.blockers).toContainEqual(expect.objectContaining({ code: "current_image_catalog_unavailable" }));
+ });
+
+ it.skipIf(process.platform === "win32")("detects changed pnpm dependency bytes in a builtin package overlap comparison", async () => {
+ await legacy();
+ const image = fixture();
+ await install(image);
+ await migration().migrate();
+ const dependency = path.join(image.packageRoot, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
+ await fs.mkdir(dependency, { recursive: true });
+ await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
+ await fs.symlink(path.relative(path.join(image.packageRoot, "node_modules"), dependency), path.join(image.packageRoot, "node_modules", "dep"));
+ const installs = image.installs.map(({ distribution: _distribution, ...install }) => install);
+ const loader = loaderFor(image);
+ const original = await migration().qualify({ installs, loadManifest: (root) => loader.loadManifest(root) });
+ expect(original.configurationPackagesEligible).toBe(true);
+ await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
+ const changed = await migration(peer, homeB).qualify({ installs, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: original.packageEvidenceDigest });
+ expect(changed.configurationPackagesEligible).toBe(true);
+ expect(changed.samePackageOverlapEligible).toBe(false);
+ expect(changed.overlapBlockers).toContainEqual(expect.objectContaining({ code: "package_evidence_changed" }));
+ });
+});
diff --git a/server/src/__tests__/cloud-configuration-readers.test.ts b/server/src/__tests__/cloud-configuration-readers.test.ts
new file mode 100644
index 00000000000..9abfda97b7d
--- /dev/null
+++ b/server/src/__tests__/cloud-configuration-readers.test.ts
@@ -0,0 +1,48 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { readLegacyAdapterSettings } from "../services/adapter-settings.js";
+import { inspectLegacyExternalRegistrations } from "../services/cloud-configuration-evidence.js";
+import { cloudBundledPackageDigest } from "../services/cloud-bundled-package-digest.js";
+
+vi.mock("../adapters/plugin-loader.js", () => ({ buildExternalAdapters: async () => [] }));
+
+describe("bounded Cloud legacy readers and image inventory", () => {
+ let root: string;
+ beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "cloud-config-reader-")); });
+ afterEach(async () => { await fs.rm(root, { recursive: true, force: true }); });
+
+ it("distinguishes an absent adapter file from a saved empty preference", async () => {
+ expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toBeNull();
+ await fs.writeFile(path.join(root, "adapter-settings.json"), '{"disabledTypes":[]}');
+ expect(await readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).toEqual({ disabledTypes: [] });
+ });
+
+ it.skipIf(process.platform === "win32")("rejects symlinks and FIFOs without waiting for a writer", async () => {
+ const target = path.join(root, "target.json");
+ const settings = path.join(root, "adapter-settings.json");
+ await fs.writeFile(target, '{"disabledTypes":[]}');
+ await fs.symlink(target, settings);
+ await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toMatchObject({ code: "ELOOP" });
+ expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("unreadable_external_registration_metadata");
+ await fs.unlink(settings);
+ execFileSync("mkfifo", [settings]);
+ await expect(readLegacyAdapterSettings({ PAPERCLIP_HOME: root })).rejects.toThrow("regular file");
+ expect((await inspectLegacyExternalRegistrations(settings)).blocker).toBe("invalid_external_registration_metadata");
+ });
+
+ it.skipIf(process.platform === "win32")("hashes pnpm-style contained links and the physical dependency bytes", async () => {
+ const dependency = path.join(root, "node_modules", ".pnpm", "dep@1.0.0", "node_modules", "dep");
+ await fs.mkdir(dependency, { recursive: true });
+ await fs.writeFile(path.join(dependency, "index.js"), "export const value = 1;");
+ await fs.symlink(path.relative(path.join(root, "node_modules"), dependency), path.join(root, "node_modules", "dep"));
+ const first = cloudBundledPackageDigest(root);
+ await fs.writeFile(path.join(dependency, "index.js"), "export const value = 2;");
+ expect(cloudBundledPackageDigest(root)).not.toBe(first);
+ await fs.unlink(path.join(root, "node_modules", "dep"));
+ await fs.symlink("/", path.join(root, "node_modules", "dep"));
+ expect(() => cloudBundledPackageDigest(root)).toThrow("escapes its package");
+ });
+});
diff --git a/server/src/__tests__/cloud-runtime-config.test.ts b/server/src/__tests__/cloud-runtime-config.test.ts
new file mode 100644
index 00000000000..c7078f7cddc
--- /dev/null
+++ b/server/src/__tests__/cloud-runtime-config.test.ts
@@ -0,0 +1,302 @@
+import fs from "node:fs";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import os from "node:os";
+import path from "node:path";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { applyPendingMigrations, closeRegisteredClients, createDb, instanceSettings, type Db } from "@paperclipai/db";
+import { eq } from "drizzle-orm";
+import { paperclipConfigSchema } from "@paperclipai/shared";
+import { loadCloudBootstrapConfig, loadConfig } from "../config.js";
+import { readConfigFile } from "../config-file.js";
+import {
+ CLOUD_RUNTIME_CONFIG_KEY,
+ cloudRuntimeConfigService,
+ initializeCloudRuntimeConfig,
+ readLegacyCloudRuntimeConfig,
+} from "../services/cloud-runtime-config.js";
+import {
+ getCloudRuntimeConfigSnapshot,
+ projectCloudRuntimeConfig,
+ setCloudRuntimeConfigSnapshot,
+} from "../services/cloud-runtime-config-snapshot.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+
+const externalDatabaseUrl = process.env.PAPERCLIP_RUNTIME_CONFIG_TEST_DATABASE_URL;
+const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
+const describePostgres = support.supported ? describe : describe.skip;
+let home: string;
+let configPath: string;
+
+function legacyConfig() {
+ return {
+ $meta: { version: 1, updatedAt: "2026-10-30T00:00:00.000Z", source: "configure" },
+ llm: { provider: "openai", apiKey: "legacy-openai-secret" },
+ database: {
+ mode: "postgres",
+ connectionString: "postgres://legacy:database-secret@example.test/paperclip",
+ backup: { enabled: false, intervalMinutes: 73, retentionDays: 11, dir: "/tmp/legacy-backups" },
+ },
+ logging: { mode: "cloud", logDir: "/tmp/legacy-logs" },
+ server: { deploymentMode: "authenticated", exposure: "private", port: 3210, serveUi: false, allowedHostnames: ["saved.example.test"] },
+ auth: { baseUrlMode: "explicit", publicBaseUrl: "https://saved.example.test", disableSignUp: true },
+ storage: { provider: "s3", s3: { bucket: "saved-bucket", region: "eu-west-1", prefix: "saved/", forcePathStyle: true } },
+ secrets: { provider: "local_encrypted", strictMode: true, localEncrypted: { keyFilePath: "/tmp/legacy-master.key" } },
+ telemetry: { enabled: false },
+ updates: { checkEnabled: false },
+ unsupportedSecret: "unknown-secret-value",
+ };
+}
+
+beforeEach(() => {
+ home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-runtime-config-"));
+ configPath = path.join(home, "config.json");
+ vi.stubEnv("PAPERCLIP_HOME", home);
+ vi.stubEnv("PAPERCLIP_CONFIG", configPath);
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "test-token");
+ vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
+ vi.stubEnv("DATABASE_MIGRATION_URL", undefined);
+ vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
+ vi.stubEnv("PAPERCLIP_AUTH_PUBLIC_BASE_URL", undefined);
+ vi.stubEnv("PAPERCLIP_AUTH_BASE_URL_MODE", undefined);
+ vi.stubEnv("PAPERCLIP_PUBLIC_URL", undefined);
+ vi.stubEnv("BETTER_AUTH_URL", undefined);
+ vi.stubEnv("BETTER_AUTH_BASE_URL", undefined);
+ vi.stubEnv("PAPERCLIP_ALLOWED_HOSTNAMES", undefined);
+ vi.stubEnv("PAPERCLIP_TAILNET_BIND_HOST", "100.100.0.1");
+ vi.stubEnv("OPENAI_API_KEY", "environment-api-secret");
+ setCloudRuntimeConfigSnapshot(null);
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ setCloudRuntimeConfigSnapshot(null);
+ fs.rmSync(home, { recursive: true, force: true });
+});
+
+describe("Cloud configuration file boundary", () => {
+ it.each(["null", "{", JSON.stringify({ unexpected: true })])("rejects malformed present legacy data %s", async (contents) => {
+ fs.writeFileSync(configPath, contents);
+ await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/Legacy Cloud runtime configuration/);
+ });
+
+ it("rejects oversized and linked legacy sources", async () => {
+ fs.writeFileSync(configPath, " ".repeat(65_537));
+ await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
+ fs.rmSync(configPath);
+ const target = path.join(home, "target.json");
+ fs.writeFileSync(target, JSON.stringify(legacyConfig()));
+ fs.symlinkSync(target, configPath);
+ await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/safely/);
+ });
+
+ it("requires an environment credential for the legacy OpenAI consumer", async () => {
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ vi.stubEnv("OPENAI_API_KEY", undefined);
+ await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/OPENAI_API_KEY/);
+ expect(() => readConfigFile()).toThrow(/not been initialized/);
+ });
+
+ it.each(["https://user:secret@saved.example.test", "https://saved.example.test?key=secret", "https://saved.example.test#secret"])("rejects secret-bearing URL %s", async (url) => {
+ const config = legacyConfig();
+ config.auth.publicBaseUrl = url;
+ fs.writeFileSync(configPath, JSON.stringify(config));
+ await expect(readLegacyCloudRuntimeConfig()).rejects.toThrow(/configuration is invalid/);
+ });
+
+ it("loads the minimal Cloud bootstrap from environment only", () => {
+ fs.writeFileSync(configPath, "malformed");
+ vi.stubEnv("DATABASE_URL", "postgres://bootstrap:bootstrap-secret@127.0.0.1:5432/runtime");
+ expect(loadCloudBootstrapConfig()).toMatchObject({ databaseMode: "postgres", databaseUrl: process.env.DATABASE_URL });
+ vi.stubEnv("DATABASE_URL", undefined);
+ expect(() => loadCloudBootstrapConfig()).toThrow(/requires DATABASE_URL/);
+ vi.stubEnv("DATABASE_URL", "sqlite:///tmp/db");
+ expect(() => loadCloudBootstrapConfig()).toThrow(/must use PostgreSQL/);
+ });
+
+ it.each(["", " "])("treats an empty migration URL as absent (%j)", (migrationUrl) => {
+ vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
+ vi.stubEnv("DATABASE_MIGRATION_URL", migrationUrl);
+ expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBeUndefined();
+ });
+
+ it("normalizes and validates an explicit migration URL", () => {
+ vi.stubEnv("DATABASE_URL", "postgres://bootstrap:password@127.0.0.1/runtime");
+ vi.stubEnv("DATABASE_MIGRATION_URL", " postgres://migration:password@127.0.0.1/runtime ");
+ expect(loadCloudBootstrapConfig().databaseMigrationUrl).toBe("postgres://migration:password@127.0.0.1/runtime");
+ vi.stubEnv("DATABASE_MIGRATION_URL", "https://wrong.example.test");
+ expect(() => loadCloudBootstrapConfig()).toThrow("DATABASE_MIGRATION_URL must use PostgreSQL");
+ });
+
+ it("preserves local config file behavior without managed signals", () => {
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ expect(readConfigFile()?.llm?.apiKey).toBe("legacy-openai-secret");
+ expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
+ });
+});
+
+describePostgres("Cloud runtime configuration in PostgreSQL", () => {
+ let database: Awaited>;
+ let first: Db;
+ let second: Db;
+
+ beforeAll(async () => {
+ if (externalDatabaseUrl) {
+ await applyPendingMigrations(externalDatabaseUrl);
+ database = { connectionString: externalDatabaseUrl, cleanup: () => closeRegisteredClients(externalDatabaseUrl, { scope: "database" }) };
+ } else {
+ database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-config-db-");
+ }
+ first = createDb(database.connectionString);
+ second = createDb(database.connectionString);
+ }, 90_000);
+
+ beforeEach(async () => {
+ await first.delete(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
+ vi.stubEnv("DATABASE_URL", database.connectionString);
+ });
+
+ afterAll(async () => { await database?.cleanup(); });
+
+ it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])("seeds a fresh database with %s", async (signal) => {
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
+ vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
+ const result = await initializeCloudRuntimeConfig(first);
+ expect(result).toMatchObject({ seeded: true, current: { version: 1, revision: 0, config: null } });
+ expect(await cloudRuntimeConfigService(second).inspectStored()).toEqual(result?.current);
+ expect(fs.existsSync(configPath)).toBe(false);
+ vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", "authenticated");
+ vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://environment.example.test");
+ expect(loadConfig()).toMatchObject({ databaseMode: "postgres", authBaseUrlMode: "explicit", secretsStrictMode: true, telemetryEnabled: true });
+ });
+
+ it("loads the saved authenticated mode before validating an environment host bind", async () => {
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ vi.stubEnv("HOST", "0.0.0.0");
+ vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
+ expect(loadCloudBootstrapConfig()).toEqual({
+ databaseMode: "postgres",
+ databaseUrl: database.connectionString,
+ databaseMigrationUrl: process.env.DATABASE_MIGRATION_URL,
+ });
+ await initializeCloudRuntimeConfig(first);
+ expect(loadConfig()).toMatchObject({ deploymentMode: "authenticated", host: "0.0.0.0", port: 3210 });
+ });
+
+ it("imports existing choices without credentials and restores them from a second client and new home", async () => {
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ const initial = await initializeCloudRuntimeConfig(first);
+ expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, databaseBackupEnabled: false, storageS3Bucket: "saved-bucket", authDisableSignUp: true });
+ const [row] = await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY));
+ const persisted = JSON.stringify(row.general);
+ for (const secret of ["legacy-openai-secret", "database-secret", "unknown-secret-value", "environment-api-secret", database.connectionString]) {
+ expect(persisted).not.toContain(secret);
+ }
+ const replacementHome = fs.mkdtempSync(path.join(home, "replacement-"));
+ const replacementConfigDir = path.join(replacementHome, ".paperclip");
+ fs.mkdirSync(replacementConfigDir);
+ const envPath = path.join(replacementConfigDir, ".env");
+ const envContents = "PORT=9999\nPAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL=should-not-load\n";
+ fs.writeFileSync(envPath, envContents);
+ vi.stubEnv("PAPERCLIP_HOME", replacementHome);
+ vi.stubEnv("PAPERCLIP_CONFIG", path.join(replacementConfigDir, "config.json"));
+ setCloudRuntimeConfigSnapshot(null);
+ const files = vi.spyOn(fs, "readFileSync");
+ expect(await initializeCloudRuntimeConfig(second)).toEqual({ seeded: false, current: initial?.current });
+ expect(loadConfig()).toMatchObject({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket" });
+ expect(files.mock.calls.map(([file]) => String(file)).filter(file => file.startsWith(replacementHome))).toEqual([]);
+ files.mockRestore();
+ const root = fileURLToPath(new URL("../../../", import.meta.url));
+ const childScript = path.join(home, "restore.mts");
+ fs.writeFileSync(childScript, `
+ import { createDb } from ${JSON.stringify(pathToFileURL(path.join(root, "packages/db/src/index.ts")).href)};
+ import { initializeCloudRuntimeConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/services/cloud-runtime-config.ts")).href)};
+ import { loadConfig } from ${JSON.stringify(pathToFileURL(path.join(root, "server/src/config.ts")).href)};
+ const db = createDb(process.env.DATABASE_URL);
+ await initializeCloudRuntimeConfig(db);
+ const config = loadConfig();
+ console.log(JSON.stringify({ port: config.port, telemetryEnabled: config.telemetryEnabled, storageS3Bucket: config.storageS3Bucket,
+ envSentinel: process.env.PAPERCLIP_RUNTIME_CONFIG_ENV_SENTINEL ?? null }));
+ await db.$client.end({ timeout: 1 });
+ `);
+ const child = await promisify(execFile)(process.execPath, [path.join(root, "server/node_modules/tsx/dist/cli.mjs"), childScript], {
+ cwd: replacementHome,
+ env: { ...process.env },
+ timeout: 30_000,
+ });
+ expect(JSON.parse(child.stdout.trim())).toEqual({ port: 3210, telemetryEnabled: false, storageS3Bucket: "saved-bucket", envSentinel: null });
+ expect(fs.readFileSync(envPath, "utf8")).toBe(envContents);
+ }, 45_000);
+
+ it("keeps environment overlays out of saved settings and ignores corrupt old files after seed", async () => {
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ await initializeCloudRuntimeConfig(first);
+ vi.stubEnv("PORT", "4567");
+ vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", "operator-bucket");
+ vi.stubEnv("PAPERCLIP_AUTH_DISABLE_SIGN_UP", "false");
+ fs.writeFileSync(configPath, "malformed legacy contents");
+ await initializeCloudRuntimeConfig(second);
+ expect(loadConfig()).toMatchObject({ port: 4567, storageS3Bucket: "operator-bucket", authDisableSignUp: false, telemetryEnabled: false });
+ expect((await cloudRuntimeConfigService(first).inspectStored())?.config).toMatchObject({ server: { port: 3210 }, storage: { s3: { bucket: "saved-bucket" } }, auth: { disableSignUp: true } });
+ });
+
+ it("keeps one committed seed when two clients propose different initial values", async () => {
+ const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
+ const results = await Promise.all([
+ cloudRuntimeConfigService(first).seedIfAbsent({ ...config, telemetry: { enabled: true } }),
+ cloudRuntimeConfigService(second).seedIfAbsent({ ...config, telemetry: { enabled: false } }),
+ ]);
+ expect(results.filter(result => result.seeded)).toHaveLength(1);
+ expect(results[0].current).toEqual(results[1].current);
+ expect(await cloudRuntimeConfigService(first).inspectStored()).toEqual(results[0].current);
+ });
+
+ it("rejects unknown or credential fields on the direct mutation path", async () => {
+ const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
+ await cloudRuntimeConfigService(first).seedIfAbsent(config);
+ await expect(cloudRuntimeConfigService(first).replace({ ...config, llm: { provider: "openai", apiKey: "must-not-store" } } as never, 0)).rejects.toThrow(/configuration is invalid/);
+ expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(0);
+ });
+
+ it("serializes revisions and rejects competing stale writes from two database clients", async () => {
+ const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
+ await cloudRuntimeConfigService(first).seedIfAbsent(config);
+ const results = await Promise.allSettled([
+ cloudRuntimeConfigService(first).replace({ ...config, telemetry: { enabled: true } }, 0),
+ cloudRuntimeConfigService(second).replace({ ...config, telemetry: { enabled: false } }, 0),
+ ]);
+ expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1);
+ expect(results.filter(result => result.status === "rejected")).toHaveLength(1);
+ expect((await cloudRuntimeConfigService(second).inspectStored())?.revision).toBe(1);
+ });
+
+ it("composes a seed with the caller transaction and rolls back both on failure", async () => {
+ const config = projectCloudRuntimeConfig(paperclipConfigSchema.parse(legacyConfig()));
+ await expect(first.transaction(async tx => {
+ await cloudRuntimeConfigService(first).seedIfAbsent(config, tx);
+ await tx.insert(instanceSettings).values({ singletonKey: "runtime-config-test-receipt", general: {} });
+ throw new Error("injected migration failure");
+ })).rejects.toThrow(/injected migration failure/);
+ expect(await cloudRuntimeConfigService(second).inspectStored()).toBeNull();
+ expect(await second.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, "runtime-config-test-receipt"))).toEqual([]);
+ });
+
+ it("fails closed on invalid database settings without consulting a valid legacy file", async () => {
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ await first.insert(instanceSettings).values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: { version: 99, config: { secret: "hidden" } } });
+ await expect(initializeCloudRuntimeConfig(second)).rejects.toThrow(/Persisted Cloud runtime configuration is invalid/);
+ expect(() => getCloudRuntimeConfigSnapshot()).toThrow(/not been initialized/);
+ });
+
+ it("does not fall back to a file or old snapshot when the database fails", async () => {
+ fs.writeFileSync(configPath, JSON.stringify(legacyConfig()));
+ await initializeCloudRuntimeConfig(first);
+ const unavailable = createDb(database.connectionString);
+ await unavailable.$client.end({ timeout: 1 });
+ await expect(initializeCloudRuntimeConfig(unavailable)).rejects.toThrow();
+ expect(() => readConfigFile()).toThrow(/not been initialized/);
+ });
+});
diff --git a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts
index 572e721996a..c610d17ded9 100644
--- a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts
+++ b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts
@@ -32,8 +32,10 @@ import { activityRoutes } from "../routes/activity.js";
import { agentRoutes } from "../routes/agents.js";
import { authorizationService } from "../services/authorization.js";
import { canActorReadWorkspaceOperation, getHeartbeatRunReadDecisions } from "../services/heartbeat-run-privacy.js";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
-const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL");
+const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, () => {
@@ -42,8 +44,8 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
const previousPrivacyMode = process.env.PAPERCLIP_ISSUE_PRIVACY_MODE;
beforeAll(async () => {
- tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-");
- db = createDb(tempDb.connectionString);
+ if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-");
+ db = createDb(externalDatabaseUrl ?? tempDb!.connectionString);
process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce";
}, 120_000);
@@ -66,6 +68,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
if (previousPrivacyMode === undefined) delete process.env.PAPERCLIP_ISSUE_PRIVACY_MODE;
else process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = previousPrivacyMode;
await tempDb?.cleanup();
+ if (externalDatabaseUrl) await db.$client.end({ timeout: 1 });
});
async function seedFixture() {
@@ -264,7 +267,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
runnerProfileJson: { instructions: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
}).where(eq(heartbeatRuns.id, f.privateRunId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
- const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now();
+ const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
@@ -281,8 +284,9 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
} finally { capture.flush(); await observed.$client.end(); }
});
- it("returns 404 for direct private-run content and metadata-only list stubs to a non-member", async () => {
+ it.each(["local_file", "s3_segments"])("returns 404 for private %s run content and metadata-only list stubs", async (logStore) => {
const fixture = await seedFixture();
+ await db.update(heartbeatRuns).set({ logStore }).where(eq(heartbeatRuns.id, fixture.privateRunId));
const app = createApp(fixture.companyId, fixture.otherAgentId);
for (const path of [
diff --git a/server/src/__tests__/helpers/bundled-image-fixture.ts b/server/src/__tests__/helpers/bundled-image-fixture.ts
new file mode 100644
index 00000000000..dcb958a5235
--- /dev/null
+++ b/server/src/__tests__/helpers/bundled-image-fixture.ts
@@ -0,0 +1,50 @@
+import { mkdtempSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
+import { BUNDLED_PLUGIN_CATALOG, resolveBundledPluginInstalls } from "../../services/bundled-plugins.js";
+import { distributionBundleDigest, readDistributionPluginCatalog } from "../../services/distribution-plugin-catalog.js";
+import { derivePluginDatabaseNamespace } from "../../services/plugin-database.js";
+import { createBundledPluginPackageResolver } from "../../services/bundled-plugin-package.js";
+
+export function bundledImageFixture(manifestOverrides: Partial = {}) {
+ const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "bundled-image-")));
+ const packageRoot = path.join(root, "distribution", "fixture");
+ const manifest: PaperclipPluginManifestV1 = {
+ id: "paperclip.image-fixture", apiVersion: 1, version: "1.0.0", displayName: "Image Fixture",
+ description: "Image package relocation fixture", author: "Paperclip", categories: ["automation"], capabilities: ["issues.read"],
+ entrypoints: { worker: "dist/worker.js", ui: "dist/ui" }, ...manifestOverrides,
+ };
+ mkdirSync(path.join(packageRoot, "dist", "ui"), { recursive: true });
+ writeFileSync(path.join(packageRoot, "package.json"), JSON.stringify({
+ name: "@paperclipai/plugin-image-fixture", version: manifest.version, type: "module",
+ paperclipPlugin: { manifest: "dist/manifest.js", ...manifest.entrypoints },
+ }));
+ writeFileSync(path.join(packageRoot, "dist", "manifest.js"), `export default ${JSON.stringify(manifest)};`);
+ writeFileSync(path.join(packageRoot, "dist", "worker.js"), `import readline from "node:readline";
+if (process.argv.includes("--source")) process.stdout.write(import.meta.url);
+else {
+ const configs = {};
+ readline.createInterface({ input: process.stdin }).on("line", line => {
+ const request = JSON.parse(line);
+ let result = {};
+ if (request.method === "initialize") result = { ok: true, supportedMethods: ["configChanged", "health"] };
+ if (request.method === "configChanged") configs[request.params.companyId] = request.params.config;
+ if (request.method === "health") result = { status: "ok", details: { source: import.meta.url, configs } };
+ process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result }) + "\\n");
+ if (request.method === "shutdown") setImmediate(() => process.exit(0));
+ });
+}`);
+ writeFileSync(path.join(packageRoot, "dist", "ui", "index.js"), "export const source = 'current-image';");
+ if (manifest.database) {
+ mkdirSync(path.join(packageRoot, manifest.database.migrationsDir), { recursive: true });
+ writeFileSync(path.join(packageRoot, manifest.database.migrationsDir, "001_fixture.sql"), `CREATE TABLE ${derivePluginDatabaseNamespace(manifest.id)}.image_rows (id uuid PRIMARY KEY);`);
+ }
+ const entry = { key: "fixture", pluginKey: manifest.id, version: manifest.version, directory: "fixture", digest: distributionBundleDigest(packageRoot) };
+ writeFileSync(path.join(root, "distribution", "catalog.json"), JSON.stringify({ schemaVersion: 1, plugins: [entry] }));
+ const catalog = readDistributionPluginCatalog(root, BUNDLED_PLUGIN_CATALOG);
+ const installs = resolveBundledPluginInstalls(["fixture"], { catalogRoot: root, enforceCatalogRoot: true, env: {}, distributionPlugins: catalog });
+ const resolver = createBundledPluginPackageResolver(installs);
+ const registration = { pluginKey: manifest.id, packageName: "@paperclipai/plugin-image-fixture", version: manifest.version, manifestJson: manifest };
+ return { root, packageRoot, manifest, catalog, installs, resolver, registration };
+}
diff --git a/server/src/__tests__/helpers/native-run-log-database.ts b/server/src/__tests__/helpers/native-run-log-database.ts
new file mode 100644
index 00000000000..a19b5c2b5ae
--- /dev/null
+++ b/server/src/__tests__/helpers/native-run-log-database.ts
@@ -0,0 +1,22 @@
+import { sql } from "drizzle-orm";
+import type { Db } from "@paperclipai/db";
+
+export async function createNativeRunLogTestTables(db: Db) {
+ await db.execute(sql`create table heartbeat_runs(id uuid primary key, company_id uuid not null, agent_id uuid not null,
+ status text not null, runtime_mode text not null default 'native', native_session_id text, runner_instance_id text,
+ native_issue_id uuid, issue_id uuid, context_snapshot jsonb,
+ log_store text, log_ref text, log_bytes bigint,
+ log_sha256 text, last_output_seq bigint not null default 0, last_output_at timestamptz, last_output_stream text,
+ last_output_bytes bigint, next_event_seq bigint not null default 1, controller_lease_expires_at timestamptz,
+ created_at timestamptz not null default now(), updated_at timestamptz not null default now())`);
+ await db.execute(sql`create table heartbeat_run_events(id bigserial primary key, company_id uuid not null,
+ run_id uuid not null, agent_id uuid not null, seq bigint not null, event_type text not null, stream text,
+ level text, color text, message text, payload jsonb, source_instance_id text, source_event_id text,
+ source_seq bigint, source_payload_sha256 text, protocol_schema_version int, created_at timestamptz not null default now(),
+ unique(run_id,seq), unique(run_id,source_event_id), unique(run_id,source_instance_id,source_seq))`);
+ await db.execute(
+ sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`,
+ );
+ await db.execute(sql`create table agent_identity_keys(agent_id uuid primary key,company_id uuid,algorithm text,key_id text,
+ public_key_pem text,private_key_material jsonb,created_at timestamptz default now())`);
+}
diff --git a/server/src/__tests__/helpers/quota-credential-process.ts b/server/src/__tests__/helpers/quota-credential-process.ts
new file mode 100644
index 00000000000..3914dcfe7e5
--- /dev/null
+++ b/server/src/__tests__/helpers/quota-credential-process.ts
@@ -0,0 +1,19 @@
+// A real receiver process for the credential-loss boundary test. No provider
+// credentials leave this process through logs, IPC, or diagnostic exceptions.
+import { createDb } from "@paperclipai/db";
+import { aiConnectionService } from "../../services/ai-connections.js";
+
+const db = createDb(process.env.CREDENTIAL_TEST_DATABASE_URL!);
+const provider = process.env.CREDENTIAL_TEST_PROVIDER_URL!;
+const request = globalThis.fetch;
+globalThis.fetch = (url, init) => {
+ if (String(url) !== "https://auth.openai.com/oauth/token") throw new Error("Unexpected fixture network request");
+ return request(provider, init);
+};
+try {
+ const service = aiConnectionService(db);
+ const [row] = await service.quotaAccounts(process.env.CREDENTIAL_TEST_COMPANY_ID!, process.env.CREDENTIAL_TEST_OWNER!);
+ await service.refreshQuotaCredential(row, await service.credential(row), new AbortController().signal);
+ process.exitCode = 0;
+} catch { process.exitCode = 1; }
+finally { await db.$client.end(); }
diff --git a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts
index 826fe78ec31..68f1c5eda6f 100644
--- a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts
+++ b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts
@@ -1090,6 +1090,7 @@ describe("agent issue mutation checkout ownership", () => {
]);
expect(mockAccessService.decide).toHaveBeenCalledWith(expect.objectContaining({ action: "issue:read" }));
expect(mockIssueService.listComments).toHaveBeenCalledWith(issueId, {
+ companyId,
afterCommentId: null,
order: "desc",
limit: null,
diff --git a/server/src/__tests__/issue-scheduled-retry-routes.test.ts b/server/src/__tests__/issue-scheduled-retry-routes.test.ts
index 81168a07195..2f196d89289 100644
--- a/server/src/__tests__/issue-scheduled-retry-routes.test.ts
+++ b/server/src/__tests__/issue-scheduled-retry-routes.test.ts
@@ -580,16 +580,33 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => {
.where(inArray(heartbeatRuns.id, work.map(row => row.id)));
clock();
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
+ const register = coordinator.register.bind(coordinator);
+ let initial: Awaited> | undefined;
+ let releaseFirstPass!: () => void;
+ const firstPassHeld = new Promise(resolve => { releaseFirstPass = resolve; });
+ vi.spyOn(coordinator, "register").mockImplementation((queue, task) => register(queue, {
+ ...task,
+ run: async signal => {
+ const result = await task.run(signal);
+ if (initial) return result;
+ const cancelled = (await states()).filter(row => row.status === "cancelled");
+ if (cancelled.length === 0) return result;
+ initial = cancelled;
+ await firstPassHeld;
+ return result;
+ },
+ }));
try {
await heartbeatService(db, { runtimeEnv: {} }).registerScheduledRetryCancellationWork(coordinator, () => true).ready;
expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(0);
const earlierId = "00000000-0000-4000-8000-000000000000";
- await writeIntent({ companyId: a.companyId, retryRunId: earlierId });
work.push({ id: earlierId });
+ await writeIntent({ companyId: a.companyId, retryRunId: earlierId });
await vi.advanceTimersByTimeAsync(1000);
- await expect.poll(async () => (await states()).filter(row => row.status === "cancelled").length).toBe(20);
- const initial = (await states()).filter(row => row.status === "cancelled");
- expect(new Set(initial.map(row => row.companyId)).size).toBe(2);
+ await expect.poll(() => initial).toBeDefined();
+ expect(initial).toHaveLength(20);
+ expect(new Set(initial!.map(row => row.companyId)).size).toBe(2);
+ releaseFirstPass();
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(29);
@@ -600,7 +617,7 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => {
const reads = vi.spyOn(db, "select");
await vi.advanceTimersByTimeAsync(180_000);
expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
- } finally { await coordinator.stop(); }
+ } finally { releaseFirstPass(); await coordinator.stop(); }
});
});
diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts
index bbf94717ff3..fb13ff43e82 100644
--- a/server/src/__tests__/issues-service.test.ts
+++ b/server/src/__tests__/issues-service.test.ts
@@ -48,6 +48,7 @@ import {
readIssueCommentRunLogText,
} from "../services/issues.ts";
import { getRunLogStore } from "../services/run-log-store.js";
+import * as nativeRunLogRecovery from "../services/native-runtime/native-run-log-recovery.js";
import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js";
import {
@@ -192,6 +193,35 @@ describeEmbeddedPostgres("issueService run attachment artifacts", () => {
});
describe("readIssueCommentRunLogText", () => {
+ it("does not read after a timed-out native repair", async () => {
+ const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockImplementationOnce((_db, _store, _ref, _redaction, signal) => new Promise((_resolve, reject) => {
+ signal!.addEventListener("abort", () => reject(signal!.reason), { once: true });
+ }));
+ const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValue({ content: "too late" });
+ vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
+ const pending = readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType);
+ try {
+ await vi.advanceTimersByTimeAsync(3_000);
+ await expect(pending).resolves.toBe("");
+ expect(read).not.toHaveBeenCalled();
+ } finally {
+ await pending;
+ vi.useRealTimers();
+ repair.mockRestore();
+ read.mockRestore();
+ }
+ });
+ it("keeps published evidence when a concurrent native repair wins ownership", async () => {
+ const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockRejectedValueOnce(new Error("The output journal is owned by another writer"));
+ const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValueOnce({ content: "published evidence" });
+ try {
+ await expect(readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType)).resolves.toBe("published evidence");
+ expect(read).toHaveBeenCalledOnce();
+ } finally {
+ repair.mockRestore();
+ read.mockRestore();
+ }
+ });
it("cancels timed-out storage reads so later listings can recover", async () => {
let active = 0;
const cleanups: Array<() => void> = [];
diff --git a/server/src/__tests__/local-encrypted-cipher.test.ts b/server/src/__tests__/local-encrypted-cipher.test.ts
index 06e2b0de36f..88f04740e67 100644
--- a/server/src/__tests__/local-encrypted-cipher.test.ts
+++ b/server/src/__tests__/local-encrypted-cipher.test.ts
@@ -1,9 +1,9 @@
import { randomBytes } from "node:crypto";
-import { mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { createLocalEncryptedCipher, localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
+import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret, localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
describe("operation-scoped local encryption", () => {
let root: string;
@@ -60,4 +60,16 @@ describe("operation-scoped local encryption", () => {
code: "provider_unavailable", operation: "resolveVersion",
}));
});
+ it("derives the same instance secret from an existing key file or environment without creating missing keys", () => {
+ expect(() => deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false }))
+ .toThrow("Secrets master key file is missing");
+ expect(existsSync(keyFile)).toBe(false);
+ const key = randomBytes(32).toString("hex");
+ writeFileSync(keyFile, key, { mode: 0o600 });
+ const fromFile = deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false });
+ unlinkSync(keyFile);
+ vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", key);
+ expect(deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false })).toBe(fromFile);
+ expect(existsSync(keyFile)).toBe(false);
+ });
});
diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts
index cc519e63d2f..cd242bd6392 100644
--- a/server/src/__tests__/native-harness-backup-expiration.test.ts
+++ b/server/src/__tests__/native-harness-backup-expiration.test.ts
@@ -654,8 +654,13 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
const available = await seedArchive(healthy.root, { name: `${dueAt - 7 * DAY_MS - 1}-${randomUUID()}` });
await startEvidence({ isSessionBusy: scope => scope === busy.scope }); await settled(dueAt);
await vi.advanceTimersByTimeAsync(dueAt - Date.now());
- await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(await exists(available)).toBe(false); });
- expect(await exists(held)).toBe(true); expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000);
+ await vi.waitFor(async () => {
+ await vi.advanceTimersByTimeAsync(100);
+ expect(await exists(available)).toBe(false);
+ expect(idleWorkSnapshot().active).toBe(0);
+ expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000);
+ });
+ expect(await exists(held)).toBe(true);
await quiet(60_000);
});
diff --git a/server/src/__tests__/openapi-routes.test.ts b/server/src/__tests__/openapi-routes.test.ts
index f1523060fe4..0a1a220b8fd 100644
--- a/server/src/__tests__/openapi-routes.test.ts
+++ b/server/src/__tests__/openapi-routes.test.ts
@@ -315,6 +315,21 @@ function loadSpecRoutes() {
}
describe("openapi routes", () => {
+ it.each(["summary", "tasks"])("documents the board-only compact attention %s contract", projection => {
+ const spec = buildOpenApiSpec() as any;
+ const operation = spec.paths[`/api/companies/{companyId}/attention/${projection}`].get;
+ expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]);
+ expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
+ expect(operation.parameters.filter((parameter: { in: string }) => parameter.in === "query")).toEqual([]);
+ const response = operation.responses["200"].content["application/json"].schema;
+ expect(response.properties.totalCount).toMatchObject({ type: "integer", minimum: 0 });
+ expect(response.properties.generatedAt).toMatchObject({ format: "date-time" });
+ expect(response.properties.items).toBeUndefined();
+ expect(response.properties.nextCursor).toBeUndefined();
+ if (projection === "tasks") expect(response.properties.taskIds).toMatchObject({ type: "array", items: { type: "string", format: "uuid" } });
+ for (const code of ["200", "401", "403"]) expect(operation.responses[code]).toBeDefined();
+ });
+
it("documents the owned pending-question read as canonical attention items without client scope overrides", () => {
const spec = buildOpenApiSpec() as any;
const operation = spec.paths["/api/issues/{id}/pending-questions"].get;
diff --git a/server/src/__tests__/plugin-task-credential-writeback.test.ts b/server/src/__tests__/plugin-task-credential-writeback.test.ts
index 8ef383854c9..0603d09f59a 100644
--- a/server/src/__tests__/plugin-task-credential-writeback.test.ts
+++ b/server/src/__tests__/plugin-task-credential-writeback.test.ts
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import { mkdtemp, realpath, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { and, eq } from "drizzle-orm";
+import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -90,6 +90,26 @@ suite("durable plugin task credential handoff", () => {
expect(await f.credential()).toBe(auth("fixture-refreshed", 2021));
expect(f.calls.at(-1)).toBe("complete");
});
+ it("retains the Runner credential after a database save failure until a fresh receiver commits", async () => {
+ const f = await fixture();
+ await db.execute(sql.raw(`create function reject_runner_credential_save() returns trigger language plpgsql as $$ begin
+ if new.secret_id = '${f.managed.credentialWriteback!.secretId}'::uuid then raise exception 'fixture receiver save unavailable'; end if; return new; end $$;
+ create constraint trigger reject_runner_credential_save after insert on company_secret_versions
+ deferrable initially deferred for each row execute function reject_runner_credential_save();`));
+ try {
+ await expect(f.settle()).rejects.toThrow();
+ expect(f.calls).toContain("credentials");
+ expect(f.calls).not.toContain("complete");
+ expect(await f.credential()).toBe(auth("fixture-original"));
+ expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "pending" });
+ } finally { await db.execute(sql`drop trigger reject_runner_credential_save on company_secret_versions; drop function reject_runner_credential_save()`); }
+ const receiver = createDb(database.connectionString);
+ expect(await settleDispatchedEnvironmentTask(receiver, f.workers, { companyId: f.companyId, id: f.leaseId })).toBe(true);
+ expect(await f.credential()).toBe(auth("fixture-refreshed", 2021));
+ expect(f.calls.filter(call => call === "credentials")).toHaveLength(2);
+ expect(f.calls.at(-1)).toBe("complete");
+ expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "complete" });
+ });
it("does not overwrite an explicit account replacement", async () => {
const f = await fixture();
await secretService(db).rotate(f.managed.credentialWriteback!.secretId, { value: auth("fixture-reconnected", 2022) }, { userId: f.owner });
diff --git a/server/src/__tests__/provider-trace-retention-work.test.ts b/server/src/__tests__/provider-trace-retention-work.test.ts
new file mode 100644
index 00000000000..2c2d2fcc3ac
--- /dev/null
+++ b/server/src/__tests__/provider-trace-retention-work.test.ts
@@ -0,0 +1,66 @@
+import type { Db } from "@paperclipai/db";
+import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
+import { afterEach, expect, it, vi } from "vitest";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { registerProviderTraceRetentionWork } from "../services/provider-trace-retention-work.js";
+
+const state = vi.hoisted(() => ({
+ cleanup: vi.fn(async () => []),
+ next: vi.fn<() => Promise>(),
+ probe: undefined as undefined | (() => string),
+ unregister: vi.fn(),
+}));
+vi.mock("../services/provider-trace-store.js", () => ({
+ providerTraceStore: () => ({ cleanupExpired: state.cleanup, nextRetentionDeadline: state.next }),
+}));
+vi.mock("../services/idle-local-work.js", () => ({
+ registerIdleLocalWorkProbe: (probe: () => string) => {
+ state.probe = probe;
+ return state.unregister;
+ },
+}));
+let coordinator: ReturnType | undefined;
+afterEach(async () => {
+ await coordinator?.stop();
+ vi.useRealTimers();
+ vi.clearAllMocks();
+});
+
+it("reconstructs trace deadlines, stays quiet between obligations, and discharges the sleep blocker", async () => {
+ vi.useFakeTimers();
+ const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
+ const onError = vi.fn();
+ coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError });
+ state.next.mockResolvedValue(null);
+ const worker = registerProviderTraceRetentionWork(
+ owner as unknown as Db,
+ coordinator,
+ () => true,
+ );
+ expect(state.probe!()).toBe("unknown");
+ await worker.ready;
+ expect(state.probe!()).toBe("none");
+ expect(coordinator.nextWakeAt()).toBeNull();
+ const initial = state.cleanup.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(300_000);
+ expect(state.cleanup).toHaveBeenCalledTimes(initial);
+ expect(state.next).toHaveBeenCalledOnce();
+
+ const deadline = Date.now() + 60_000;
+ state.next.mockResolvedValue(deadline);
+ worker.wake();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(state.probe!()).toBe("present");
+ expect(coordinator.nextWakeAt()).toBe(deadline);
+ const scheduled = state.cleanup.mock.calls.length;
+ await vi.advanceTimersByTimeAsync(59_999);
+ expect(state.cleanup).toHaveBeenCalledTimes(scheduled);
+ state.next.mockResolvedValue(null);
+ await vi.advanceTimersByTimeAsync(1);
+ expect(state.cleanup).toHaveBeenCalledTimes(scheduled + 1);
+ expect(state.probe!()).toBe("none");
+ expect(coordinator.nextWakeAt()).toBeNull();
+ await worker.stop();
+ expect(state.unregister).toHaveBeenCalledOnce();
+ expect(onError).not.toHaveBeenCalled();
+});
diff --git a/server/src/__tests__/recovery-cutover.test.ts b/server/src/__tests__/recovery-cutover.test.ts
new file mode 100644
index 00000000000..ee9aca31a51
--- /dev/null
+++ b/server/src/__tests__/recovery-cutover.test.ts
@@ -0,0 +1,15 @@
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { it } from "vitest";
+
+const run = promisify(execFile);
+
+// Keep the dependency-free, actual-CLI acceptance suite usable on a retained
+// volume while also exercising the same cases in the normal CI test lane.
+it("passes the read-only recovery cutover CLI acceptance suite", async () => {
+ await run(process.execPath, ["--test", fileURLToPath(new URL("../../../scripts/recovery-cutover.test.mjs", import.meta.url))], {
+ timeout: 45_000,
+ maxBuffer: 1024 * 1024,
+ });
+}, 60_000);
diff --git a/server/src/__tests__/run-evidence-migration-work.test.ts b/server/src/__tests__/run-evidence-migration-work.test.ts
new file mode 100644
index 00000000000..d14da6c3018
--- /dev/null
+++ b/server/src/__tests__/run-evidence-migration-work.test.ts
@@ -0,0 +1,168 @@
+import type { Db } from "@paperclipai/db";
+import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
+import { afterEach, expect, it, vi } from "vitest";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { registerRunEvidenceMigrationWork } from "../services/run-evidence-migration-work.js";
+
+const state = vi.hoisted(() => ({
+ provider: "s3",
+ migrate: vi.fn(),
+ probe: undefined as undefined | (() => string),
+ unregister: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+}));
+vi.mock("../storage/config.js", () => ({
+ loadStorageConfig: () => ({ storageProvider: state.provider }),
+}));
+vi.mock("../services/run-evidence-migration.js", () => ({ migrateRunEvidence: state.migrate }));
+vi.mock("../middleware/logger.js", () => ({ logger: { info: state.info, warn: state.warn } }));
+vi.mock("../services/idle-local-work.js", () => ({
+ registerIdleLocalWorkProbe: (probe: () => string) => {
+ state.probe = probe;
+ return state.unregister;
+ },
+}));
+let coordinator: ReturnType | undefined;
+afterEach(async () => {
+ await coordinator?.stop();
+ coordinator = undefined;
+ state.provider = "s3";
+ state.probe = undefined;
+ vi.resetAllMocks();
+ vi.useRealTimers();
+});
+function setup(canRun = () => true) {
+ vi.useFakeTimers();
+ const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
+ const onError = vi.fn();
+ coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile: () => true, onError });
+ const worker = registerRunEvidenceMigrationWork(owner as unknown as Db, coordinator, canRun);
+ return { worker, onError };
+}
+it("does no migration work for local storage", async () => {
+ state.provider = "local_disk";
+ const { worker } = setup();
+ expect(worker).toBeNull();
+ await vi.advanceTimersByTimeAsync(3_600_000);
+ expect(state.migrate).not.toHaveBeenCalled();
+ expect(state.probe).toBeUndefined();
+ expect(coordinator!.nextWakeAt()).toBeNull();
+});
+it("automatically pages both resources, then leaves completed history quiet", async () => {
+ state.migrate
+ .mockResolvedValueOnce({
+ items: [{ kind: "log", id: "one", state: "migrated" }],
+ nextRunId: "one",
+ })
+ .mockResolvedValueOnce({
+ items: [{ kind: "log", id: "two", state: "migrated" }],
+ nextRunId: "two",
+ })
+ .mockResolvedValue({ items: [] });
+ const { worker, onError } = setup();
+ expect(state.probe!()).toBe("unknown");
+ await worker!.ready;
+ expect(state.migrate).toHaveBeenNthCalledWith(
+ 1,
+ expect.anything(),
+ expect.objectContaining({
+ apply: true,
+ limit: 1,
+ kinds: ["log", "trace"],
+ }),
+ );
+ expect(state.probe!()).toBe("present");
+ await vi.advanceTimersByTimeAsync(2_000);
+ expect(state.migrate).toHaveBeenNthCalledWith(
+ 2,
+ expect.anything(),
+ expect.objectContaining({
+ afterRunId: "one",
+ kinds: ["log"],
+ }),
+ );
+ expect(state.migrate).toHaveBeenNthCalledWith(
+ 3,
+ expect.anything(),
+ expect.objectContaining({
+ afterRunId: "two",
+ kinds: ["log"],
+ }),
+ );
+ expect(state.probe!()).toBe("none");
+ expect(coordinator!.nextWakeAt()).toBeNull();
+ await vi.advanceTimersByTimeAsync(3_600_000);
+ expect(state.migrate).toHaveBeenCalledTimes(3);
+ expect(onError).not.toHaveBeenCalled();
+ await worker!.stop();
+ expect(state.unregister).toHaveBeenCalledOnce();
+});
+it("keeps blocked history visible and retries it without stranding later pages", async () => {
+ state.migrate
+ .mockResolvedValueOnce({
+ items: [{ kind: "log", id: "active", state: "blocked", reason: "legacy_writer_not_settled" }],
+ nextRunId: "active",
+ })
+ .mockResolvedValueOnce({ items: [{ kind: "log", id: "later", state: "migrated" }] })
+ .mockResolvedValue({ items: [{ kind: "log", id: "active", state: "migrated" }] });
+ const { worker } = setup();
+ await worker!.ready;
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(state.warn).toHaveBeenCalledWith(
+ expect.objectContaining({ id: "active", reason: "legacy_writer_not_settled" }),
+ expect.any(String),
+ );
+ expect(state.probe!()).toBe("present");
+ await vi.advanceTimersByTimeAsync(299_999);
+ expect(state.migrate).toHaveBeenCalledTimes(2);
+ await vi.advanceTimersByTimeAsync(1);
+ expect(state.migrate).toHaveBeenNthCalledWith(
+ 3,
+ expect.anything(),
+ expect.objectContaining({ afterRunId: undefined, kinds: ["log", "trace"] }),
+ );
+ expect(state.probe!()).toBe("none");
+ expect(coordinator!.nextWakeAt()).toBeNull();
+});
+it("retains startup work during drain and retries a failed sweep before becoming quiet", async () => {
+ let admitted = false;
+ state.migrate
+ .mockRejectedValueOnce(new Error("storage unavailable"))
+ .mockResolvedValue({ items: [] });
+ const { worker, onError } = setup(() => admitted);
+ await worker!.ready;
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(state.migrate).not.toHaveBeenCalled();
+ expect(state.probe!()).toBe("unknown");
+ admitted = true;
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(onError).toHaveBeenCalledOnce();
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(state.migrate).toHaveBeenCalledTimes(2);
+ expect(state.probe!()).toBe("none");
+ expect(coordinator!.nextWakeAt()).toBeNull();
+});
+it("aborts owned work on shutdown and reconstructs outstanding history on restart", async () => {
+ let started: AbortSignal | undefined;
+ state.migrate.mockImplementationOnce(
+ (_db, options) =>
+ new Promise((_resolve, reject) => {
+ started = options.signal;
+ started!.addEventListener("abort", () => reject(started!.reason), { once: true });
+ }),
+ );
+ const { worker } = setup();
+ await vi.advanceTimersByTimeAsync(0);
+ expect(started?.aborted).toBe(false);
+ await worker!.stop();
+ await worker!.ready;
+ expect(started?.aborted).toBe(true);
+ expect(coordinator!.nextWakeAt()).toBeNull();
+ await coordinator!.stop();
+ state.migrate.mockResolvedValue({ items: [] });
+ const restarted = setup();
+ await restarted.worker!.ready;
+ expect(state.migrate).toHaveBeenCalledTimes(2);
+ expect(state.probe!()).toBe("none");
+});
diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts
index 420a18bbc86..b645d7437c8 100644
--- a/server/src/__tests__/server-startup-feedback-export.test.ts
+++ b/server/src/__tests__/server-startup-feedback-export.test.ts
@@ -20,6 +20,22 @@ import { gzipSync } from "node:zlib";
import { tmpdir } from "node:os";
import path from "node:path";
+const cloudRuntimeStartupMocks = vi.hoisted(() => ({
+ initialize: vi.fn(async () => null),
+ bootstrap: vi.fn(),
+ telemetry: vi.fn(),
+}));
+
+vi.mock("../services/adapter-settings.js", async () => ({
+ ...await vi.importActual("../services/adapter-settings.js"),
+ initializeAdapterSettings: vi.fn(async () => null),
+}));
+vi.mock("../services/cloud-runtime-config.js", () => ({ initializeCloudRuntimeConfig: cloudRuntimeStartupMocks.initialize }));
+const cloudMigrationMocks = vi.hoisted(() => ({ migrate: vi.fn(async () => null), qualify: vi.fn(async () => null) }));
+vi.mock("../services/cloud-configuration-migration.js", () => ({ migrateCloudConfiguration: cloudMigrationMocks.migrate }));
+vi.mock("../services/cloud-configuration-qualification.js", () => ({ qualifyCloudConfigurationForCurrentImage: cloudMigrationMocks.qualify }));
+vi.mock("../telemetry.js", () => ({ initTelemetry: cloudRuntimeStartupMocks.telemetry, getTelemetryClient: vi.fn(() => null) }));
+
const ORIGINAL_PAPERCLIP_API_URL = process.env.PAPERCLIP_API_URL;
const ORIGINAL_PAPERCLIP_RUNTIME_API_URL = process.env.PAPERCLIP_RUNTIME_API_URL;
const ORIGINAL_PAPERCLIP_RUNTIME_API_CANDIDATES_JSON = process.env.PAPERCLIP_RUNTIME_API_CANDIDATES_JSON;
@@ -343,6 +359,7 @@ vi.mock("../services/native-runtime/native-safe-replacement.js", () => ({
vi.mock("../config.js", () => ({
loadConfig: loadConfigMock,
+ loadCloudBootstrapConfig: cloudRuntimeStartupMocks.bootstrap,
}));
vi.mock("../middleware/logger.js", () => ({
@@ -586,6 +603,8 @@ describe("startServer feedback export wiring", () => {
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
process.env.PAPERCLIP_AGENT_JWT_SECRET = "0123456789abcdef0123456789abcdef";
loadConfigMock.mockReturnValue(buildTestConfig());
+ cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
+ cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
resolveHeartbeatSchedulingSuppressionMock.mockReturnValue({
suppressed: false,
reason: null,
@@ -595,6 +614,64 @@ describe("startServer feedback export wiring", () => {
process.env.BETTER_AUTH_SECRET = "test-secret";
});
+ it("awaits saved Cloud configuration before telemetry, authentication and app construction", async () => {
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
+ vi.stubEnv("HOST", "0.0.0.0");
+ vi.stubEnv("PAPERCLIP_DEPLOYMENT_MODE", undefined);
+ cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
+ let release: (() => void) | undefined;
+ cloudRuntimeStartupMocks.initialize.mockImplementationOnce(() => new Promise(resolve => {
+ release = () => {
+ loadConfigMock.mockReturnValue(buildTestConfig({ telemetryEnabled: false, authDisableSignUp: true, port: 4321, host: "0.0.0.0", bind: "all" }));
+ resolve(null);
+ };
+ }));
+ try {
+ const starting = startServer();
+ for (let index = 0; index < 30 && !release; index += 1) await new Promise(resolve => setImmediate(resolve));
+ expect(release).toBeTypeOf("function");
+ expect(cloudMigrationMocks.migrate.mock.invocationCallOrder[0]).toBeLessThan(cloudRuntimeStartupMocks.initialize.mock.invocationCallOrder[0]!);
+ expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
+ expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
+ expect(createAppMock).not.toHaveBeenCalled();
+ expect(loadConfigMock).not.toHaveBeenCalled();
+ release!();
+ const started = await starting;
+ await settleStartupHousekeeping();
+ expect(cloudRuntimeStartupMocks.telemetry).toHaveBeenCalledExactlyOnceWith({ enabled: false });
+ expect(createBetterAuthInstanceMock).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ authDisableSignUp: true, port: 4321 }), expect.anything());
+ expect(started.listenPort).toBe(4321);
+ expect(cloudMigrationMocks.qualify.mock.invocationCallOrder[0]).toBeLessThan(createAppMock.mock.invocationCallOrder[0]!);
+ } finally {
+ release?.();
+ vi.unstubAllEnvs();
+ }
+ });
+
+ it("does not create telemetry or the app when saved Cloud configuration fails", async () => {
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
+ cloudRuntimeStartupMocks.initialize.mockRejectedValueOnce(new Error("persisted config unavailable"));
+ try {
+ await expect(startServer()).rejects.toThrow("persisted config unavailable");
+ expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
+ expect(createBetterAuthInstanceMock).not.toHaveBeenCalled();
+ expect(createAppMock).not.toHaveBeenCalled();
+ } finally {
+ vi.unstubAllEnvs();
+ }
+ });
+
+ it("stops before configuration consumers when the atomic Cloud migration fails", async () => {
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
+ cloudMigrationMocks.migrate.mockRejectedValueOnce(new Error("migration did not commit"));
+ try {
+ await expect(startServer()).rejects.toThrow("migration did not commit");
+ expect(cloudRuntimeStartupMocks.initialize).not.toHaveBeenCalled();
+ expect(cloudRuntimeStartupMocks.telemetry).not.toHaveBeenCalled();
+ expect(createAppMock).not.toHaveBeenCalled();
+ } finally { vi.unstubAllEnvs(); }
+ });
+
it.each([false, true])("registers runner evidence independently from the backup scan (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
@@ -620,6 +697,12 @@ describe("startServer feedback export wiring", () => {
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toEqual([]);
});
+ it.each([false, true])("registers trace retention independently of heartbeat scheduling (%s)", async enabled => {
+ loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
+ await startServer(); await settleStartupHousekeeping();
+ expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.providerTraceRetention)).toHaveLength(1);
+ });
+
it.each([false, true])("registers skill cache retention only when enabled (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ skillVersionRetentionEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
@@ -1509,6 +1592,8 @@ describe("startServer authenticated auth origin setup", () => {
vi.clearAllMocks();
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
loadConfigMock.mockReturnValue(buildTestConfig());
+ cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
+ cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
createBetterAuthInstanceMock.mockReturnValue({});
deriveAuthTrustedOriginsMock.mockReturnValue([]);
process.env.BETTER_AUTH_SECRET = "test-secret";
@@ -1570,6 +1655,8 @@ describe("startServer PAPERCLIP_API_URL handling", () => {
vi.clearAllMocks();
process.env.PAPERCLIP_DECISION_SIGNING_SECRET = "fedcba9876543210fedcba9876543210";
loadConfigMock.mockReturnValue(buildTestConfig());
+ cloudRuntimeStartupMocks.initialize.mockReset().mockResolvedValue(null);
+ cloudRuntimeStartupMocks.bootstrap.mockReturnValue({ databaseMode: "postgres", databaseUrl: buildTestConfig().databaseUrl, databaseMigrationUrl: undefined });
process.env.BETTER_AUTH_SECRET = "test-secret";
delete process.env.PAPERCLIP_API_URL;
});
diff --git a/server/src/__tests__/task-cold-open.test.ts b/server/src/__tests__/task-cold-open.test.ts
index 15fc364a644..3d6aac70b08 100644
--- a/server/src/__tests__/task-cold-open.test.ts
+++ b/server/src/__tests__/task-cold-open.test.ts
@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { activityLog, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db";
+import { activityLog, agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { issueRoutes } from "../routes/issues.js";
import { errorHandler } from "../middleware/index.js";
@@ -11,16 +11,25 @@ import { createHeartbeatRunState } from "../services/heartbeat/run-state.js";
import { activityService } from "../services/activity.js";
import { documentService } from "../services/documents.js";
-const support = await getEmbeddedPostgresTestSupport();
+const externalDatabase = process.env.PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL;
+if (externalDatabase) {
+ const target = new URL(externalDatabase);
+ if (!["localhost", "127.0.0.1", "[::1]"].includes(target.hostname) || !target.pathname.endsWith("_test"))
+ throw new Error("PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL must name a disposable loopback database ending in _test");
+}
+const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("task cold-open reads", () => {
- let fixture: Awaited>;
+ let fixture: Pick>, "connectionString" | "cleanup">;
let db: ReturnType;
const companyId = randomUUID();
const issueId = randomUUID();
const agentId = randomUUID();
const runId = randomUUID();
beforeAll(async () => {
- fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
+ if (externalDatabase) {
+ await applyPendingMigrations(externalDatabase);
+ fixture = { connectionString: externalDatabase, cleanup: () => closeRegisteredClients(externalDatabase) };
+ } else fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
db = createDb(fixture.connectionString);
await db.insert(companies).values({ id: companyId, name: "Cold open", issuePrefix: `T${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Writer", role: "engineer", adapterType: "codex_local" });
@@ -79,6 +88,39 @@ const support = await getEmbeddedPostgresTestSupport();
expect(response.body.latestCommentId).toBe(response.body.comments[1].id);
expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${issueId}/recent-comments`)).status);
});
+ it("opens long chats with a contiguous recent window and pages every older message", async () => {
+ const chatId = randomUUID();
+ await db.insert(issues).values({ id: chatId, companyId, title: "Persistent chat", conversationAgentId: agentId, conversationUserId: "local-board", assigneeAgentId: agentId, conversationState: "waiting" });
+ const rows = Array.from({ length: 123 }, (_, i) => ({ id: randomUUID(), companyId, issueId: chatId,
+ authorType: "user" as const, authorUserId: "local-board", body: `Chat ${i}`, createdAt: new Date(1000 + Math.floor(i / 3)) }));
+ await db.insert(issueComments).values(rows);
+ const ordered = [...rows].sort((a, b) => +b.createdAt - +a.createdAt || b.id.localeCompare(a.id));
+ const response = await request(app()).get(`/api/issues/${chatId}/recent-comments`);
+ expect(response.status).toBe(200);
+ expect(response.body.comments.map((row: { id: string }) => row.id)).toEqual(ordered.slice(0, 20).map(row => row.id));
+ expect(response.body.oldestCommentId).toBe(ordered[19].id);
+ expect(response.body.hasMore).toBe(true);
+ const seen = response.body.comments.map((row: { id: string }) => row.id);
+ let cursor = response.body.oldestCommentId;
+ while (cursor) {
+ const page = await request(app()).get(`/api/issues/${chatId}/comments`).query({ order: "desc", limit: 50, after: cursor });
+ expect(page.status).toBe(200);
+ seen.push(...page.body.map((row: { id: string }) => row.id));
+ cursor = page.body.at(-1)?.id;
+ }
+ expect(seen).toEqual(ordered.map(row => row.id));
+ expect(new Set(seen).size).toBe(123);
+ expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${chatId}/recent-comments`)).status);
+ });
+ it("hydrates only requested runs linked to the company and issue", async () => {
+ const otherIssue = randomUUID(), otherRun = randomUUID();
+ await db.insert(issues).values({ id: otherIssue, companyId, title: "Other task" });
+ await db.insert(heartbeatRuns).values({ id: otherRun, companyId, agentId, issueId: otherIssue, status: "succeeded" });
+ const activity = activityService(db);
+ expect((await activity.runsForIssue(companyId, issueId, null, undefined, [runId, otherRun])).map(run => run.runId)).toEqual([runId]);
+ expect(await activity.runsForIssue(randomUUID(), issueId, null, undefined, [runId])).toEqual([]);
+ expect(await activity.runsForIssue(companyId, issueId, null, undefined, [])).toEqual([]);
+ });
it.each(["user", null] as const)("preserves initial and latest human instructions without a description (authorType=%s)", async (authorType) => {
const bareIssueId = randomUUID();
await db.insert(issues).values({ id: bareIssueId, companyId, title: "No description" });
diff --git a/server/src/adapters/cloud-external-adapters.test.ts b/server/src/adapters/cloud-external-adapters.test.ts
new file mode 100644
index 00000000000..21da3bccbeb
--- /dev/null
+++ b/server/src/adapters/cloud-external-adapters.test.ts
@@ -0,0 +1,136 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+let home: string;
+let localPackage: string;
+let npmPackage: string;
+let storePath: string;
+let sentinel: string;
+const parserSource = "export function parseStdoutLine() { return { message: 'external parser' }; }";
+const managedError = { status: 403, details: { code: "adapter_install_platform_managed" } };
+
+function writePackage(packageDir: string, type: string): void {
+ fs.mkdirSync(packageDir, { recursive: true });
+ fs.writeFileSync(path.join(packageDir, "package.json"), JSON.stringify({
+ name: `fixture-${type}`,
+ type: "module",
+ exports: { ".": "./index.mjs", "./ui-parser": "./ui-parser.mjs" },
+ paperclip: { adapterUiParser: "1.0.0" },
+ }));
+ fs.writeFileSync(path.join(packageDir, "index.mjs"), `
+ import fs from "node:fs";
+ fs.appendFileSync(${JSON.stringify(sentinel)}, ${JSON.stringify(type + "\n")});
+ export function createServerAdapter() {
+ return {
+ type: ${JSON.stringify(type)},
+ agentConfigurationDoc: "External fixture",
+ models: [],
+ execute: async () => ({ exitCode: 41, signal: null, timedOut: false }),
+ testEnvironment: async () => ({ status: "pass", checks: [] }),
+ };
+ }
+ `);
+ fs.writeFileSync(path.join(packageDir, "ui-parser.mjs"), parserSource);
+}
+
+beforeEach(() => {
+ vi.resetModules();
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
+ vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
+ home = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-cloud-adapters-"));
+ vi.stubEnv("PAPERCLIP_HOME", home);
+ sentinel = path.join(home, "external-code-executed");
+ localPackage = path.join(home, "local-package");
+ npmPackage = path.join(home, "adapter-plugins", "node_modules", "fixture-npm");
+ storePath = path.join(home, "adapter-plugins.json");
+ writePackage(localPackage, "codex_local");
+ writePackage(npmPackage, "fixture_npm");
+ fs.writeFileSync(storePath, JSON.stringify([
+ { packageName: "fixture-local", localPath: localPackage, type: "codex_local", installedAt: "2026-01-01" },
+ { packageName: "fixture-npm", type: "fixture_npm", installedAt: "2026-01-01" },
+ ]));
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ fs.rmSync(home, { recursive: true, force: true });
+});
+
+describe("Cloud image adapter boundary", () => {
+ it.each(["PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "PAPERCLIP_MANAGED_CONFIG"])(
+ "cold boots built-ins without reading or executing persisted packages with %s",
+ async (signal) => {
+ vi.stubEnv(signal, signal === "PAPERCLIP_MANAGED_CONFIG" ? "" : "test-token");
+ const registrationBefore = fs.readFileSync(storePath, "utf8");
+ const reads = vi.spyOn(fs, "readFileSync");
+ const registry = await import("./registry.js");
+ await registry.waitForExternalAdapters();
+ const loader = await import("./plugin-loader.js");
+ const store = await import("../services/adapter-plugin-store.js");
+
+ expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
+ expect(registry.requireServerAdapter("process").execute).toBeTypeOf("function");
+ expect(registry.findServerAdapter("fixture_npm")).toBeNull();
+ expect(registry.hasActiveAdapterOverride("codex_local")).toBe(false);
+ expect(await loader.buildExternalAdapters()).toEqual([]);
+ expect(store.listAdapterPlugins()).toEqual([]);
+ expect(store.getAdapterPluginByType("codex_local")).toBeUndefined();
+ expect(loader.getUiParserSource("codex_local")).toBeUndefined();
+ expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
+ await expect(loader.loadExternalAdapterPackage("fixture-local", localPackage)).rejects.toMatchObject(managedError);
+ await expect(loader.loadExternalAdapterPackage("fixture-npm")).rejects.toMatchObject(managedError);
+ await expect(loader.reloadExternalAdapter("codex_local")).rejects.toMatchObject(managedError);
+ expect(() => registry.registerServerAdapter(registry.requireServerAdapter("codex_local"))).toThrow();
+ expect(() => registry.unregisterServerAdapter("codex_local")).toThrow();
+ expect(() => registry.setOverridePaused("codex_local", false)).toThrow();
+ expect(() => store.addAdapterPlugin({ packageName: "fixture", type: "fixture", installedAt: "2026-01-01" })).toThrow();
+ expect(() => store.removeAdapterPlugin("codex_local")).toThrow();
+ expect(() => store.getAdapterPluginsDir()).toThrow();
+ expect(store.getAdapterPluginStorePath()).toBe(storePath);
+ expect(reads.mock.calls.map(([file]) => String(file)).filter(file =>
+ file === storePath || file.startsWith(localPackage + path.sep) || file.startsWith(npmPackage + path.sep),
+ )).toEqual([]);
+ reads.mockRestore();
+ expect(fs.existsSync(sentinel)).toBe(false);
+ expect(fs.readFileSync(storePath, "utf8")).toBe(registrationBefore);
+ },
+ );
+
+ it("preserves local and npm package loading, overrides, reload, and lazy UI parsing on self-hosted instances", async () => {
+ const loader = await import("./plugin-loader.js");
+ expect(loader.getOrExtractUiParserSource("codex_local")).toBe(parserSource);
+ const registry = await import("./registry.js");
+ await registry.waitForExternalAdapters();
+
+ const override = registry.requireServerAdapter("codex_local");
+ expect(override.agentConfigurationDoc).toBe("External fixture");
+ expect(registry.hasActiveAdapterOverride("codex_local")).toBe(true);
+ expect(registry.requireServerAdapter("fixture_npm").agentConfigurationDoc).toBe("External fixture");
+ expect(await override.execute({} as never)).toMatchObject({ exitCode: 41 });
+ expect(fs.readFileSync(sentinel, "utf8").trim().split("\n")).toEqual(["codex_local", "fixture_npm"]);
+ expect(loader.getUiParserSource("fixture_npm")).toBe(parserSource);
+ const reloaded = await loader.reloadExternalAdapter("codex_local");
+ expect(reloaded?.type).toBe("codex_local");
+ expect(await reloaded?.execute({} as never)).toMatchObject({ exitCode: 41 });
+ expect(registry.setOverridePaused("codex_local", true)).toBe(true);
+ expect(registry.findActiveServerAdapter("codex_local")?.agentConfigurationDoc).not.toBe("External fixture");
+ expect(registry.setOverridePaused("codex_local", false)).toBe(true);
+ registry.unregisterServerAdapter("codex_local");
+ expect(registry.requireServerAdapter("codex_local").agentConfigurationDoc).not.toBe("External fixture");
+ });
+
+ it("does not expose a cached registration or parser after a managed signal becomes present", async () => {
+ const loader = await import("./plugin-loader.js");
+ const store = await import("../services/adapter-plugin-store.js");
+ await loader.loadExternalAdapterPackage("fixture-local", localPackage);
+ expect(loader.getUiParserSource("codex_local")).toBe(parserSource);
+ expect(store.listAdapterPlugins()).toHaveLength(2);
+ vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", "{}");
+ expect(store.listAdapterPlugins()).toEqual([]);
+ expect(loader.getUiParserSource("codex_local")).toBeUndefined();
+ expect(loader.getOrExtractUiParserSource("codex_local")).toBeUndefined();
+ });
+});
diff --git a/server/src/adapters/plugin-loader.ts b/server/src/adapters/plugin-loader.ts
index 236e8ddd50f..2777d813af1 100644
--- a/server/src/adapters/plugin-loader.ts
+++ b/server/src/adapters/plugin-loader.ts
@@ -15,6 +15,8 @@ import { pathToFileURL } from "node:url";
import type { ServerAdapterModule } from "./types.js";
import { validateAdapterLoginCapability } from "@paperclipai/adapter-utils";
import { logger } from "../middleware/logger.js";
+import { isCloudManagedInstance } from "../services/cloud-instance.js";
+import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
import {
listAdapterPlugins,
@@ -30,6 +32,7 @@ import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
const uiParserCache = new Map();
export function getUiParserSource(adapterType: string): string | undefined {
+ if (isCloudManagedInstance()) return undefined;
return uiParserCache.get(adapterType);
}
@@ -38,6 +41,7 @@ export function getUiParserSource(adapterType: string): string | undefined {
* Makes the ui-parser.js endpoint self-healing.
*/
export function getOrExtractUiParserSource(adapterType: string): string | undefined {
+ if (isCloudManagedInstance()) return undefined;
const cached = uiParserCache.get(adapterType);
if (cached) return cached;
@@ -182,6 +186,7 @@ export async function loadExternalAdapterPackage(
packageName: string,
localPath?: string,
): Promise {
+ assertExternalAdapterCodeAllowed();
const packageDir = localPath
? path.resolve(localPath)
: path.resolve(getAdapterPluginsDir(), "node_modules", packageName);
@@ -221,6 +226,7 @@ async function loadFromRecord(record: AdapterPluginRecord): Promise {
+ assertExternalAdapterCodeAllowed();
const record = getAdapterPluginByType(type);
if (!record) return null;
@@ -271,6 +277,7 @@ export async function reloadExternalAdapter(
* Build all external adapter modules from the plugin store.
*/
export async function buildExternalAdapters(): Promise {
+ if (isCloudManagedInstance()) return [];
const results: ServerAdapterModule[] = [];
const storeRecords = listAdapterPlugins();
diff --git a/server/src/adapters/registry.ts b/server/src/adapters/registry.ts
index 2a271a5d720..e4c6dddf53d 100644
--- a/server/src/adapters/registry.ts
+++ b/server/src/adapters/registry.ts
@@ -136,7 +136,8 @@ import {
import { agentConfigurationDoc as piAgentConfigurationDoc } from "@paperclipai/adapter-pi-local";
import { BUILTIN_ADAPTER_TYPES } from "./builtin-adapter-types.js";
import { buildExternalAdapters } from "./plugin-loader.js";
-import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
+import { isCloudManagedInstance } from "../services/cloud-instance.js";
+import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
import { NativeEnvironmentProbeCleanupError } from "../vendor/paperclip-runner/index.js";
import { processAdapter } from "./process/index.js";
import { httpAdapter } from "./http/index.js";
@@ -997,11 +998,6 @@ registerBuiltInAdapters();
// applies the registry fallback for externals overriding a built-in type.
// ---------------------------------------------------------------------------
-/** Cached sync wrapper — the store is a simple JSON file read, safe to call frequently. */
-function getDisabledAdapterTypesFromStore(): string[] {
- return getDisabledAdapterTypes();
-}
-
/**
* Merge an external adapter module with host-provided session management.
*
@@ -1034,6 +1030,7 @@ export function resolveExternalAdapterRegistration(
* and avoid racing against the loading window.
*/
const externalAdaptersReady: Promise = (async () => {
+ if (isCloudManagedInstance()) return;
try {
const externalAdapters = await buildExternalAdapters();
for (const externalAdapter of externalAdapters) {
@@ -1069,6 +1066,7 @@ export function waitForExternalAdapters(): Promise {
}
export function registerServerAdapter(adapter: ServerAdapterModule): void {
+ assertExternalAdapterCodeAllowed();
if (BUILTIN_ADAPTER_TYPES.has(adapter.type) && !builtinFallbacks.has(adapter.type)) {
const existing = adaptersByType.get(adapter.type);
if (existing) {
@@ -1079,6 +1077,7 @@ export function registerServerAdapter(adapter: ServerAdapterModule): void {
}
export function unregisterServerAdapter(type: string): void {
+ assertExternalAdapterCodeAllowed();
if (type === processAdapter.type || type === httpAdapter.type) return;
if (builtinFallbacks.has(type)) {
pausedOverrides.delete(type);
@@ -1186,8 +1185,7 @@ export function listServerAdapters(): ServerAdapterModule[] {
* Used for menus and agent creation flows — disabled adapters remain
* functional for existing agents but hidden from selection.
*/
-export function listEnabledServerAdapters(): ServerAdapterModule[] {
- const disabled = getDisabledAdapterTypesFromStore();
+export function listEnabledServerAdapters(disabled: string[]): ServerAdapterModule[] {
const disabledSet = disabled.length > 0 ? new Set(disabled) : null;
return disabledSet
? Array.from(adaptersByType.values()).filter((a) => !disabledSet.has(a.type))
@@ -1227,6 +1225,7 @@ export async function detectAdapterModel(
* an override or was already in the requested state.
*/
export function setOverridePaused(type: string, paused: boolean): boolean {
+ assertExternalAdapterCodeAllowed();
if (!builtinFallbacks.has(type)) return false;
const wasPaused = pausedOverrides.has(type);
if (paused && !wasPaused) {
diff --git a/server/src/app.ts b/server/src/app.ts
index dcb51306a6b..130bfed7b60 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -183,7 +183,9 @@ import {
ensureBundledPlugins,
resolveBundledCatalogRoot,
resolveBundledPluginInstalls,
+ resolveManagedBundledPluginKeys,
} from "./services/bundled-plugins.js";
+import { createBundledPluginPackageResolver } from "./services/bundled-plugin-package.js";
import { readDistributionPluginCatalog, distributionPluginActivationGuard } from "./services/distribution-plugin-catalog.js";
import {
createPluginWorkerManager,
@@ -623,7 +625,7 @@ export async function createApp(
res.set("Cache-Control", "no-store"); res.set("Referrer-Policy", "no-referrer");
res.redirect(303, await chatChannels.githubWizard.identityCallback(String(req.query.state ?? ""), String(req.query.code ?? ""), req.actor));
});
- const managedAutoInstallKeys = opts.managedPluginAutoInstall ?? null;
+ const managedAutoInstallKeys = resolveManagedBundledPluginKeys(opts.managedPluginAutoInstall);
const bundledCatalogRoot =
opts.bundledPluginCatalogRoot ?? resolveBundledCatalogRoot(process.env);
const distributionPlugins = readDistributionPluginCatalog(bundledCatalogRoot, BUNDLED_PLUGIN_CATALOG);
@@ -636,6 +638,7 @@ export async function createApp(
distributionPlugins,
},
);
+ const bundledPackageResolver = managedAutoInstallKeys === null ? undefined : createBundledPluginPackageResolver(bundledPluginInstalls);
const managedBundledPluginKeys =
managedAutoInstallKeys !== null
? bundledPluginInstalls.map((install) => install.pluginKey)
@@ -906,7 +909,11 @@ export async function createApp(
{
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
migrationDb: opts.pluginMigrationDb,
- assertPackageActivation: distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
+ assertPackageActivation: bundledPackageResolver?.assertActivation ?? distributionPluginActivationGuard(bundledCatalogRoot, distributionPlugins, managedAutoInstallKeys),
+ resolvePackageRoot: bundledPackageResolver?.resolve,
+ prebuiltOnly: managedAutoInstallKeys !== null,
+ enableLocalFilesystem: managedAutoInstallKeys === null,
+ enableNpmDiscovery: managedAutoInstallKeys === null,
},
{
workerManager,
@@ -965,6 +972,7 @@ export async function createApp(
);
api.use(
adapterRoutes({
+ db,
resolveRunnerTarget: (companyId, environmentId) => resolveAgentRunnerTargetForCompany(db, companyId, environmentId),
getNativeRunnerEnabled: async () =>
(await instanceSettingsService(db).getExperimental())
@@ -1014,6 +1022,7 @@ export async function createApp(
app.use(
pluginUiStaticRoutes(db, {
localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
+ resolvePackageRoot: bundledPackageResolver?.resolve,
}),
);
@@ -1253,7 +1262,7 @@ export async function createApp(
void toolDispatcher.initialize().catch((err) => {
logger.error({ err }, "Failed to initialize plugin tool dispatcher");
});
- const devWatcher = createPluginDevWatcher(
+ const devWatcher = managedAutoInstallKeys !== null ? null : createPluginDevWatcher(
lifecycle,
async (pluginId) =>
(await pluginRegistry.getById(pluginId))?.packagePath ?? null,
@@ -1299,7 +1308,7 @@ export async function createApp(
// owns provisioning); self-hosted leaves an operator's uninstall alone.
// Disabled plugins never start automatically. Added distribution permissions
// still enter upgrade_pending so enabling them requires an operator decision.
- { reinstallUninstalled: managedAutoInstallKeys !== null },
+ { reinstallUninstalled: managedAutoInstallKeys !== null, preservePackagePath: managedAutoInstallKeys !== null },
)
.then(() => loader.loadAll())
.then((result) => {
diff --git a/server/src/config-file.ts b/server/src/config-file.ts
index 957e8d05881..d750793c204 100644
--- a/server/src/config-file.ts
+++ b/server/src/config-file.ts
@@ -8,6 +8,8 @@ import {
} from "@paperclipai/shared";
import { ZodError } from "zod";
import { resolvePaperclipConfigPath } from "./paths.js";
+import { isCloudManagedInstance } from "./services/cloud-instance.js";
+import { getCloudRuntimeConfigSnapshot, toPaperclipConfig } from "./services/cloud-runtime-config-snapshot.js";
function formatConfigValidationError(error: ZodError): string {
return error.issues
@@ -19,6 +21,7 @@ function formatConfigValidationError(error: ZodError): string {
}
export function readConfigFile(): PaperclipConfig | null {
+ if (isCloudManagedInstance()) return toPaperclipConfig(getCloudRuntimeConfigSnapshot().config);
const configPath = resolvePaperclipConfigPath();
if (!fs.existsSync(configPath)) return null;
diff --git a/server/src/config.ts b/server/src/config.ts
index dac9dba2142..c1156068fa8 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -1,4 +1,7 @@
import { readConfigFile, resolveDeploymentMode } from "./config-file.js";
+import { isCloudManagedInstance } from "./services/cloud-instance.js";
+import { StartupRefusalError } from "./startup-refusals.js";
+import type { PaperclipConfig } from "@paperclipai/shared";
import { parseChatWebhookPublicBaseUrl } from "./chat-webhook-public-url.js";
import { execFileSync } from "node:child_process";
import { existsSync, realpathSync } from "node:fs";
@@ -13,13 +16,11 @@ import {
BIND_MODES,
DEPLOYMENT_EXPOSURES,
SECRET_PROVIDERS,
- STORAGE_PROVIDERS,
type BindMode,
type AuthBaseUrlMode,
type DeploymentExposure,
type DeploymentMode,
type SecretProvider,
- type StorageProvider,
inferBindModeFromHost,
resolveRuntimeBind,
validateConfiguredBindMode,
@@ -28,37 +29,39 @@ import {
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir,
resolveDefaultSecretsKeyFilePath,
- resolveDefaultStorageDir,
resolveHomeAwarePath,
} from "./home-paths.js";
+import { loadStorageConfig, type StorageConfig } from "./storage/config.js";
-const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
-if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
- loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
-}
+if (!isCloudManagedInstance()) {
+ const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
+ if (existsSync(PAPERCLIP_ENV_FILE_PATH)) {
+ loadDotenv({ path: PAPERCLIP_ENV_FILE_PATH, override: false, quiet: true });
+ }
-const CWD_ENV_PATH = resolve(process.cwd(), ".env");
-const cwdEnvExists = existsSync(CWD_ENV_PATH);
-const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
- ? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
- : CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
-if (shouldLoadWorkingDirectoryEnv({
- cwdEnvExists,
- isPaperclipEnvFile: isSameFile,
-})) {
- loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
-}
+ const CWD_ENV_PATH = resolve(process.cwd(), ".env");
+ const cwdEnvExists = existsSync(CWD_ENV_PATH);
+ const isSameFile = cwdEnvExists && existsSync(PAPERCLIP_ENV_FILE_PATH)
+ ? realpathSync(CWD_ENV_PATH) === realpathSync(PAPERCLIP_ENV_FILE_PATH)
+ : CWD_ENV_PATH === PAPERCLIP_ENV_FILE_PATH;
+ if (shouldLoadWorkingDirectoryEnv({
+ cwdEnvExists,
+ isPaperclipEnvFile: isSameFile,
+ })) {
+ loadDotenv({ path: CWD_ENV_PATH, override: false, quiet: true });
+ }
-if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
- applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
+ if (basename(dirname(PAPERCLIP_ENV_FILE_PATH)) === ".paperclip") {
+ applyEmptyWorktreeSigningSecrets(dirname(dirname(PAPERCLIP_ENV_FILE_PATH)));
+ }
+ maybeRepairLegacyWorktreeConfigAndEnvFiles();
}
-maybeRepairLegacyWorktreeConfigAndEnvFiles();
const TAILSCALE_DETECT_TIMEOUT_MS = 3000;
type DatabaseMode = "embedded-postgres" | "postgres";
-export interface Config {
+export interface Config extends StorageConfig {
deploymentMode: DeploymentMode;
deploymentExposure: DeploymentExposure;
bind: BindMode;
@@ -111,13 +114,6 @@ export interface Config {
secretsProvider: SecretProvider;
secretsStrictMode: boolean;
secretsMasterKeyFilePath: string;
- storageProvider: StorageProvider;
- storageLocalDiskBaseDir: string;
- storageS3Bucket: string;
- storageS3Region: string;
- storageS3Endpoint: string | undefined;
- storageS3Prefix: string;
- storageS3ForcePathStyle: boolean;
feedbackExportBackendUrl: string | undefined;
feedbackExportBackendToken: string | undefined;
heartbeatSchedulerEnabled: boolean;
@@ -147,18 +143,16 @@ function detectTailnetBindHost(): string | undefined {
}
}
-export function loadConfig(): Config {
- const fileConfig = readConfigFile();
+export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()): Config {
const fileDatabaseMode =
- (fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
+ (isCloudManagedInstance() || fileConfig?.database.mode === "postgres" ? "postgres" : "embedded-postgres") as DatabaseMode;
const fileDbUrl =
- fileDatabaseMode === "postgres"
+ !isCloudManagedInstance() && fileDatabaseMode === "postgres"
? fileConfig?.database.connectionString
: undefined;
const fileDatabaseBackup = fileConfig?.database.backup;
const fileSecrets = fileConfig?.secrets;
- const fileStorage = fileConfig?.storage;
const providerFromEnvRaw = process.env.PAPERCLIP_SECRETS_PROVIDER;
const providerFromEnv =
@@ -168,25 +162,7 @@ export function loadConfig(): Config {
const providerFromFile = fileSecrets?.provider;
const secretsProvider: SecretProvider = providerFromEnv ?? providerFromFile ?? "local_encrypted";
- const storageProviderFromEnvRaw = process.env.PAPERCLIP_STORAGE_PROVIDER;
- const storageProviderFromEnv =
- storageProviderFromEnvRaw && STORAGE_PROVIDERS.includes(storageProviderFromEnvRaw as StorageProvider)
- ? (storageProviderFromEnvRaw as StorageProvider)
- : null;
- const storageProvider: StorageProvider = storageProviderFromEnv ?? fileStorage?.provider ?? "local_disk";
- const storageLocalDiskBaseDir = resolveHomeAwarePath(
- process.env.PAPERCLIP_STORAGE_LOCAL_DIR ??
- fileStorage?.localDisk?.baseDir ??
- resolveDefaultStorageDir(),
- );
- const storageS3Bucket = process.env.PAPERCLIP_STORAGE_S3_BUCKET ?? fileStorage?.s3?.bucket ?? "paperclip";
- const storageS3Region = process.env.PAPERCLIP_STORAGE_S3_REGION ?? fileStorage?.s3?.region ?? "us-east-1";
- const storageS3Endpoint = process.env.PAPERCLIP_STORAGE_S3_ENDPOINT ?? fileStorage?.s3?.endpoint ?? undefined;
- const storageS3Prefix = process.env.PAPERCLIP_STORAGE_S3_PREFIX ?? fileStorage?.s3?.prefix ?? "";
- const storageS3ForcePathStyle =
- process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE !== undefined
- ? process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE === "true"
- : (fileStorage?.s3?.forcePathStyle ?? false);
+ const storage = loadStorageConfig(fileConfig);
const feedbackExportBackendUrl =
process.env.PAPERCLIP_FEEDBACK_EXPORT_BACKEND_URL?.trim() ||
process.env.PAPERCLIP_TELEMETRY_BACKEND_URL?.trim() ||
@@ -443,13 +419,7 @@ export function loadConfig(): Config {
fileSecrets?.localEncrypted.keyFilePath ??
resolveDefaultSecretsKeyFilePath(),
),
- storageProvider,
- storageLocalDiskBaseDir,
- storageS3Bucket,
- storageS3Region,
- storageS3Endpoint,
- storageS3Prefix,
- storageS3ForcePathStyle,
+ ...storage,
feedbackExportBackendUrl,
feedbackExportBackendToken,
heartbeatSchedulerEnabled: process.env.HEARTBEAT_SCHEDULER_ENABLED !== "false",
@@ -460,3 +430,23 @@ export function loadConfig(): Config {
announcementsFeedUrl: process.env.PAPERCLIP_ANNOUNCEMENTS_FEED_URL?.trim() || "https://pages.paperclip.ing/announcements/v1/current.json",
};
}
+
+export type CloudBootstrapConfig = {
+ databaseMode: "postgres";
+ databaseUrl: string;
+ databaseMigrationUrl: string | undefined;
+};
+
+export function loadCloudBootstrapConfig(): CloudBootstrapConfig {
+ if (!isCloudManagedInstance()) throw new Error("Cloud bootstrap configuration requires a managed instance");
+ const url = process.env.DATABASE_URL?.trim();
+ if (!url) throw new StartupRefusalError("database-contract-unmet", "Cloud bootstrap requires DATABASE_URL from the environment");
+ const migrationUrl = process.env.DATABASE_MIGRATION_URL?.trim() || undefined;
+ for (const [name, value] of [["DATABASE_URL", url], ["DATABASE_MIGRATION_URL", migrationUrl]]) {
+ if (!value) continue;
+ let parsed: URL;
+ try { parsed = new URL(value); } catch { throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`); }
+ if (!["postgres:", "postgresql:"].includes(parsed.protocol)) throw new Error(`Cloud bootstrap ${name} must use PostgreSQL`);
+ }
+ return { databaseMode: "postgres", databaseUrl: url, databaseMigrationUrl: migrationUrl };
+}
diff --git a/server/src/index.ts b/server/src/index.ts
index 424580c2a69..ae229f03518 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -1,3 +1,5 @@
+import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js";
+import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js";
import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js";
import { registerDatabaseBackupWork } from "./services/database-backup-work.js";
import { registerToolConnectionHealthWork } from "./services/tool-connection-health-work.js";
@@ -104,7 +106,8 @@ import {
} from "@paperclipai/db";
import detectPort from "detect-port";
import { createApp } from "./app.js";
-import { loadConfig } from "./config.js";
+import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
+import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
import { logger } from "./middleware/logger.js";
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
@@ -186,6 +189,9 @@ import {
loadWithoutCoordinatedShutdownSignalHooks,
} from "./shutdown.js";
import { initializeCloudRuntimeIdentity } from "./services/cloud-runtime-identity.js";
+import { initializeAdapterSettings } from "./services/adapter-settings.js";
+import { migrateCloudConfiguration } from "./services/cloud-configuration-migration.js";
+import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
import { systemdNotify } from "./services/systemd-notify.js";
import { flushInFlightRunLogMirrors } from "./services/run-log-store.js";
import {
@@ -251,6 +257,19 @@ async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promi
await sql.end({ timeout: timeoutSeconds });
}
+function initializeRuntimeConfigConsumers(config: ReturnType): void {
+ initTelemetry({ enabled: config.telemetryEnabled });
+ if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
+ process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
+ }
+ if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
+ process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
+ }
+ if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
+ process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
+ }
+}
+
export async function startServer(): Promise {
const startupDatabase: StartupDatabaseTeardown = { close: null };
try {
@@ -278,17 +297,8 @@ async function startServerWithDatabaseTeardown(
// sentry.ts.
await sentryReady;
ensureDecisionSigningSecret();
- let config = loadConfig();
- initTelemetry({ enabled: config.telemetryEnabled });
- if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
- process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
- }
- if (process.env.PAPERCLIP_SECRETS_STRICT_MODE === undefined) {
- process.env.PAPERCLIP_SECRETS_STRICT_MODE = config.secretsStrictMode ? "true" : "false";
- }
- if (process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE === undefined) {
- process.env.PAPERCLIP_SECRETS_MASTER_KEY_FILE = config.secretsMasterKeyFilePath;
- }
+ const startupConfig = isCloudManagedInstance() ? loadCloudBootstrapConfig() : loadConfig();
+ if ("deploymentMode" in startupConfig) initializeRuntimeConfigConsumers(startupConfig);
type MigrationSummary =
| "skipped"
@@ -390,7 +400,7 @@ async function startServerWithDatabaseTeardown(
}
}
- function assertCloudDatabaseContract(): void {
+ function assertCloudDatabaseContract(config: Config): void {
if (config.deploymentMode !== "authenticated" || config.deploymentExposure !== "public") {
return;
}
@@ -483,17 +493,19 @@ async function startServerWithDatabaseTeardown(
let startupDbInfo:
| { mode: "external-postgres"; connectionString: string }
| { mode: "embedded-postgres"; dataDir: string; port: number };
- assertCloudDatabaseContract();
- if (config.databaseUrl) {
- const migrationUrl = config.databaseMigrationUrl ?? config.databaseUrl;
+ if ("deploymentMode" in startupConfig) assertCloudDatabaseContract(startupConfig);
+ if (startupConfig.databaseUrl) {
+ const migrationUrl = startupConfig.databaseMigrationUrl ?? startupConfig.databaseUrl;
migrationSummary = await ensureMigrations(migrationUrl, "PostgreSQL");
-
- db = createDb(config.databaseUrl);
- pluginMigrationDb = config.databaseMigrationUrl ? createDb(config.databaseMigrationUrl) : db;
+
+ db = createDb(startupConfig.databaseUrl);
+ pluginMigrationDb = startupConfig.databaseMigrationUrl ? createDb(startupConfig.databaseMigrationUrl) : db;
logger.info("Using external PostgreSQL via DATABASE_URL/config");
- activeDatabaseConnectionString = config.databaseUrl;
- startupDbInfo = { mode: "external-postgres", connectionString: config.databaseUrl };
+ activeDatabaseConnectionString = startupConfig.databaseUrl;
+ startupDbInfo = { mode: "external-postgres", connectionString: startupConfig.databaseUrl };
} else {
+ if (!("deploymentMode" in startupConfig)) throw new Error("Cloud bootstrap requires an external PostgreSQL connection");
+ const config = startupConfig;
const moduleName = "embedded-postgres";
let EmbeddedPostgres: EmbeddedPostgresCtor;
try {
@@ -711,12 +723,19 @@ async function startServerWithDatabaseTeardown(
};
startupDatabase.close = closeDatabaseClients;
setSentryRunLinkResolver(createSentryRunLinkResolver(db));
+ await migrateCloudConfiguration(db as any);
+ await initializeAdapterSettings(db as any);
+ await initializeCloudRuntimeConfig(db as any);
+ let config = "deploymentMode" in startupConfig ? startupConfig : loadConfig();
+
// A claimed warm-pool stack may restart while its provider environment still
// names the pool host. Restore the signed, durable identity before Better
// Auth, routes, or child-runtime configuration capture any public URL.
const restoredCloudRuntimeIdentity = await initializeCloudRuntimeIdentity(db as any);
if (restoredCloudRuntimeIdentity) config = loadConfig();
+ if (isCloudManagedInstance()) initializeRuntimeConfigConsumers(config);
+
const isWarmStandby = await createCloudWarmStandby(db as any);
if (config.deploymentMode === "local_trusted" && !isLoopbackHost(config.host)) {
@@ -963,6 +982,10 @@ async function startServerWithDatabaseTeardown(
// document parsed fail-closed above (`plugins.autoInstall`). Absent env means
// self-hosted: createApp falls back to its built-in kubernetes-only default.
const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null;
+ const cloudQualification = await qualifyCloudConfigurationForCurrentImage(db as any);
+ if (cloudQualification && !cloudQualification.configurationPackagesEligible) {
+ logger.warn({ scope: cloudQualification.scope, blockerCodes: cloudQualification.blockers.map((blocker) => blocker.code) }, "Cloud configuration and package qualification has blockers");
+ }
const app = await createApp(db as any, {
prepareIdleDatabaseBackup: idleDatabaseBackupEnabled
? async () => (await runServerDatabaseBackup("idle")) !== null
@@ -1344,6 +1367,13 @@ async function startServerWithDatabaseTeardown(
}));
});
server.on("close", () => { void taskDrainExpiryWork.stop(); });
+ const runEvidenceMigrationWork = registerRunEvidenceMigrationWork(db, deliveryWork,
+ () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), { storageProvider: config.storageProvider });
+ if (runEvidenceMigrationWork) trackHeartbeatSchedulerWork(runEvidenceMigrationWork.ready);
+ const providerTraceRetentionWork = registerProviderTraceRetentionWork(db, deliveryWork,
+ () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby());
+ trackHeartbeatSchedulerWork(providerTraceRetentionWork.ready);
+ server.on("close", providerTraceRetentionWork.stop);
const executionSchedule = executionControlSchedule(db);
const workspaceRecoveryRuntime = environmentRuntimeService(db, { pluginWorkerManager });
const nativeRunnerEvidenceRetentionWorker = config.nativeHarnessBackupExpirationEnabled
@@ -2215,6 +2245,8 @@ async function startServerWithDatabaseTeardown(
unsubscribeNativeWorkspaceFinalization?.();
unsubscribeRetainedNativeSessionCleanup?.();
await deliveryWork.stop();
+ await runEvidenceMigrationWork?.stop();
+ await providerTraceRetentionWork.stop();
await skillVersionRetentionWorker?.dispose();
await workspaceRecoveryWork.stop();
unregisterTerminalWorkspaceProbe?.();
diff --git a/server/src/paths.ts b/server/src/paths.ts
index 21856a6946e..5ed3971d4fd 100644
--- a/server/src/paths.ts
+++ b/server/src/paths.ts
@@ -1,6 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import { resolveDefaultConfigPath } from "./home-paths.js";
+import { isCloudManagedInstance } from "./services/cloud-instance.js";
const PAPERCLIP_CONFIG_BASENAME = "config.json";
const PAPERCLIP_ENV_FILENAME = ".env";
@@ -26,6 +27,7 @@ function findConfigFileFromAncestors(startDir: string): string | null {
export function resolvePaperclipConfigPath(overridePath?: string): string {
if (overridePath) return path.resolve(overridePath);
if (process.env.PAPERCLIP_CONFIG) return path.resolve(process.env.PAPERCLIP_CONFIG);
+ if (isCloudManagedInstance()) return resolveDefaultConfigPath();
return findConfigFileFromAncestors(process.cwd()) ?? resolveDefaultConfigPath();
}
diff --git a/server/src/qualify-cloud-configuration.ts b/server/src/qualify-cloud-configuration.ts
new file mode 100644
index 00000000000..f2e50830a3f
--- /dev/null
+++ b/server/src/qualify-cloud-configuration.ts
@@ -0,0 +1,47 @@
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { createDb } from "@paperclipai/db";
+import { cloudConfigurationMigrationService } from "./services/cloud-configuration-migration.js";
+import { qualifyCloudConfigurationForCurrentImage } from "./services/cloud-configuration-qualification.js";
+
+export async function runCloudConfigurationQualification(args: string[], env: NodeJS.ProcessEnv = process.env): Promise {
+ const allowed = new Set(["--migrate", "--expected-package-evidence", "--recheck-external-revision"]);
+ let migrate = false;
+ let expectedPackageEvidenceDigest: string | undefined;
+ let recheckRevision: number | undefined;
+ for (let index = 0; index < args.length; index += 1) {
+ const argument = args[index];
+ if (!allowed.has(argument)) throw new Error(`Unknown qualification argument: ${argument}`);
+ if (argument === "--migrate") migrate = true;
+ if (argument === "--expected-package-evidence") {
+ expectedPackageEvidenceDigest = args[++index];
+ if (!expectedPackageEvidenceDigest || !/^sha256:[a-f0-9]{64}$/.test(expectedPackageEvidenceDigest)) throw new Error("A SHA-256 package evidence digest is required");
+ }
+ if (argument === "--recheck-external-revision") {
+ const value = args[++index];
+ if (!value || !/^\d+$/.test(value) || !Number.isSafeInteger(Number(value))) throw new Error("A nonnegative receipt revision is required");
+ recheckRevision = Number(value);
+ }
+ }
+ const url = env.DATABASE_URL?.trim();
+ if (!url) throw new Error("DATABASE_URL is required for Cloud configuration qualification");
+ const db = createDb(url);
+ try {
+ const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
+ if (migrate) await service.migrate();
+ if (recheckRevision !== undefined) await service.recheckLegacyExternalRegistrations(recheckRevision);
+ const report = await qualifyCloudConfigurationForCurrentImage(db, { runtimeEnv: env, expectedPackageEvidenceDigest });
+ if (!report) throw new Error("Cloud configuration qualification requires a managed instance");
+ process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+ return report.configurationPackagesEligible && (!expectedPackageEvidenceDigest || report.samePackageOverlapEligible) ? 0 : 2;
+ } finally {
+ await db.$client.end({ timeout: 5 });
+ }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
+ runCloudConfigurationQualification(process.argv.slice(2)).then((code) => { process.exitCode = code; }).catch((error) => {
+ process.stderr.write(`${error instanceof Error ? error.message : "Cloud configuration qualification failed"}\n`);
+ process.exitCode = 1;
+ });
+}
diff --git a/server/src/routes/activity.ts b/server/src/routes/activity.ts
index fd46f3d1208..7a9b187dbab 100644
--- a/server/src/routes/activity.ts
+++ b/server/src/routes/activity.ts
@@ -373,6 +373,14 @@ export function activityRoutes(db: Db) {
if (!issue) return;
if (!(await assertIssueReadAllowed(req, res, issue))) return;
const viewerUserId = req.actor.type === "board" ? getActorInfo(req).actorId : null;
+ if (req.query.runIds !== undefined) {
+ const parsed = z.array(z.string().uuid()).min(1).max(100).safeParse(
+ typeof req.query.runIds === "string" ? req.query.runIds.split(",") : null,
+ );
+ if (!parsed.success) throw badRequest("runIds must contain 1 to 100 UUIDs");
+ res.json(await svc.runsForIssue(issue.companyId, issue.id, viewerUserId, undefined, parsed.data));
+ return;
+ }
const result = await (req.query.view === "recent"
? svc.runsForIssue(issue.companyId, issue.id, viewerUserId, 2)
: svc.runsForIssue(issue.companyId, issue.id, viewerUserId));
diff --git a/server/src/routes/adapters.ts b/server/src/routes/adapters.ts
index 8e0d41e1d64..7a02043644b 100644
--- a/server/src/routes/adapters.ts
+++ b/server/src/routes/adapters.ts
@@ -19,11 +19,13 @@ import { readFile } from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { Router } from "express";
+import { companies, type Db } from "@paperclipai/db";
+import { adapterSettingsService } from "../services/adapter-settings.js";
+import { logActivity, publishActivity, type ActivityPublication } from "../services/activity-log.js";
import {
listServerAdapters,
findServerAdapter,
findActiveServerAdapter,
- listEnabledServerAdapters,
registerServerAdapter,
resolveExternalAdapterRegistration,
unregisterServerAdapter,
@@ -37,8 +39,6 @@ import {
removeAdapterPlugin,
getAdapterPluginByType,
getAdapterPluginsDir,
- getDisabledAdapterTypes,
- setAdapterDisabled,
} from "../services/adapter-plugin-store.js";
import type { AdapterPluginRecord } from "../services/adapter-plugin-store.js";
import type { ServerAdapterModule, AdapterConfigSchema } from "../adapters/types.js";
@@ -49,9 +49,9 @@ import type {
import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js";
import { logger } from "../middleware/logger.js";
import { badRequest, forbidden } from "../errors.js";
-import { isCloudManagedInstance } from "../services/cloud-instance.js";
+import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
import { getHiddenSettings } from "../services/settings-visibility.js";
-import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin } from "./authz.js";
+import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js";
import { BUILTIN_ADAPTER_TYPES } from "../adapters/builtin-adapter-types.js";
import { agentRunnerAvailability, type RunnerTarget } from "../services/agent-runner-selection.js";
import type { AgentRunnerAvailability } from "@paperclipai/shared";
@@ -60,22 +60,6 @@ import { resolveAntigravityCandidateAdmission } from "../services/native-runtime
const execFileAsync = promisify(execFile);
let warnedMalformedAntigravityAdmission = false;
-/**
- * Floor: on cloud-managed instances adapter code is bundled into the platform
- * image; fetching and loading external adapter packages at runtime stays off
- * for every actor, including instance admins. Adapter code executes in the
- * server process, so a runtime install would let an instance admin read the
- * platform trust anchors from the process environment (mirrors the
- * bundled-only plugin install floor in plugin-install-guard.ts).
- */
-function assertAdapterCodeInstallAllowed() {
- if (isCloudManagedInstance()) {
- throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
- code: "adapter_install_platform_managed",
- });
- }
-}
-
/**
* Floor: when the hosting operator hides the Adapters settings surface
* (`instance.adapters` in PAPERCLIP_HIDDEN_SETTINGS), adapter management
@@ -209,7 +193,7 @@ function buildAdapterInfo(adapter: ServerAdapterModule, externalRecord: AdapterP
loaded: true, // If it's in the registry, it's loaded
disabled: disabledSet.has(adapter.type),
capabilities: buildAdapterCapabilities(adapter),
- ...agentRunnerAvailability(adapter.type, target),
+ ...agentRunnerAvailability(adapter.type, target, [...disabledSet]),
...(adapter.acp ? { acp: adapter.acp } : {}),
overriddenBuiltin: externalRecord ? BUILTIN_ADAPTER_TYPES.has(adapter.type) : undefined,
overridePaused: BUILTIN_ADAPTER_TYPES.has(adapter.type) ? isOverridePaused(adapter.type) : undefined,
@@ -265,11 +249,13 @@ function registerWithSessionManagement(adapter: ServerAdapterModule): void {
// ---------------------------------------------------------------------------
export function adapterRoutes(options: {
+ db?: Db;
getNativeRunnerEnabled?: () => Promise;
getOpenAiDotEnabled?: () => Promise;
resolveRunnerTarget?: (companyId: string, environmentId?: string) => Promise;
} = {}) {
const router = Router();
+ const settings = adapterSettingsService(options.db);
/**
* GET /api/adapters
@@ -303,7 +289,7 @@ export function adapterRoutes(options: {
const externalRecords = new Map(
listAdapterPlugins().map((r) => [r.type, r]),
);
- const disabledSet = new Set(getDisabledAdapterTypes());
+ const disabledSet = new Set(await settings.getDisabledTypes());
// One shared implementation, with independent provider rollouts. Explicit
// adapter-admin disabling still applies to both choices.
@@ -343,7 +329,7 @@ export function adapterRoutes(options: {
*/
router.post("/adapters/install", async (req, res) => {
assertInstanceAdmin(req);
- assertAdapterCodeInstallAllowed();
+ assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const { packageName, isLocalPath = false, version } = req.body as AdapterInstallRequest;
@@ -477,7 +463,7 @@ export function adapterRoutes(options: {
}
const externalRecord = getAdapterPluginByType(adapterType);
- const disabledSet = new Set(getDisabledAdapterTypes());
+ const disabledSet = new Set(await settings.getDisabledTypes());
res.json(buildAdapterInfo(adapter, externalRecord, disabledSet));
});
@@ -509,7 +495,28 @@ export function adapterRoutes(options: {
return;
}
- const changed = setAdapterDisabled(adapterType, disabled);
+ const publications: ActivityPublication[] = [];
+ const changed = await settings.setDisabled(adapterType, disabled, async (tx, current) => {
+ const actor = getActorInfo(req);
+ const companyIds = await tx.select({ id: companies.id }).from(companies);
+ for (const company of companyIds) {
+ await logActivity(tx as Db, {
+ companyId: company.id,
+ ...actor,
+ action: "instance.adapter.updated",
+ entityType: "instance_settings",
+ entityId: "adapter-settings:v1",
+ details: { adapterType, disabled, revision: current.revision },
+ }, publications);
+ }
+ });
+ for (const publication of publications) {
+ try {
+ publishActivity(publication);
+ } catch (error) {
+ logger.error({ err: error, companyId: publication.companyId }, "failed to publish adapter settings activity");
+ }
+ }
if (changed) {
logger.info({ type: adapterType, disabled }, "Adapter enabled/disabled");
@@ -528,6 +535,7 @@ export function adapterRoutes(options: {
*/
router.patch("/adapters/:type/override", async (req, res) => {
assertInstanceAdmin(req);
+ assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
@@ -558,6 +566,7 @@ export function adapterRoutes(options: {
*/
router.delete("/adapters/:type", async (req, res) => {
assertInstanceAdmin(req);
+ assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const adapterType = req.params.type;
@@ -635,6 +644,7 @@ export function adapterRoutes(options: {
*/
router.post("/adapters/:type/reload", async (req, res) => {
assertInstanceAdmin(req);
+ assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const type = req.params.type;
@@ -688,7 +698,7 @@ export function adapterRoutes(options: {
// package name, but without the risk of losing the store record.
router.post("/adapters/:type/reinstall", async (req, res) => {
assertInstanceAdmin(req);
- assertAdapterCodeInstallAllowed();
+ assertExternalAdapterCodeAllowed();
assertAdapterManagementVisible();
const type = req.params.type;
diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts
index 8b6cd0128f9..7f4b0990f48 100644
--- a/server/src/routes/agents.ts
+++ b/server/src/routes/agents.ts
@@ -154,7 +154,7 @@ import type {
} from "@paperclipai/adapter-utils";
import { evaluateCodexCredentialReadiness } from "@paperclipai/adapter-codex-local/server";
import type { AdapterAuthSignal, AdapterAuthSignalResponse, CodexAccountBindingClaim } from "@paperclipai/shared";
-import { getDisabledAdapterTypes } from "../services/adapter-plugin-store.js";
+import { adapterSettingsService } from "../services/adapter-settings.js";
import { skillVersionSelectionMap } from "../services/runtime-skill-selections.js";
import { isFixedClaudeOAuthBinding, secretService } from "../services/secrets.js";
import { authorizationDeniedDetails } from "../services/authorization.js";
@@ -1859,15 +1859,6 @@ export function agentRoutes(
* Adapter validation for the paths that CHOOSE a harness for a new agent
* (hire + create), as opposed to the paths that operate on an existing one.
*
- * A disabled adapter is one this instance cannot run — most often because a
- * declarative registry (PAPERCLIP_ADAPTERS) curated it out, which
- * reconcileAdapterAvailability turns into a disabled type at boot. Registered
- * but disabled still passes assertKnownAdapterType, so an agent could be
- * created on it and then fail EVERY run at lease time with
- * `Adapter "..." is not in the configured adapter registry` — an error that
- * arrives minutes later, in a run log, with no way back to the choice that
- * caused it. Refuse at selection time instead, and name what can be chosen.
- *
* Existing agents on a now-disabled adapter are deliberately untouched
* (listEnabledServerAdapters documents the same rule: hidden from selection,
* still functional for agents that already use them).
@@ -1902,7 +1893,7 @@ export function agentRoutes(
);
}
}
- const disabled = new Set(getDisabledAdapterTypes());
+ const disabled = new Set(await adapterSettingsService(db).getDisabledTypes());
if (!disabled.has(adapterType)) return adapterType;
const available = listServerAdapters()
.map((a) => a.type)
diff --git a/server/src/routes/attention.ts b/server/src/routes/attention.ts
index d1c7a7c2a92..9f1671885d7 100644
--- a/server/src/routes/attention.ts
+++ b/server/src/routes/attention.ts
@@ -17,6 +17,22 @@ export function attentionRoutes(db: Db) {
const router = Router();
const svc = attentionService(db);
+ router.get("/companies/:companyId/attention/summary", async (req, res) => {
+ const companyId = req.params.companyId as string;
+ assertCompanyAccess(req, companyId);
+ assertBoard(req);
+ if (!req.actor.userId) { res.status(403).json({ error: "Board user context required" }); return; }
+ res.json(await svc.summary(companyId, { userId: req.actor.userId, actor: req.actor }));
+ });
+
+ router.get("/companies/:companyId/attention/tasks", async (req, res) => {
+ const companyId = req.params.companyId as string;
+ assertCompanyAccess(req, companyId);
+ assertBoard(req);
+ if (!req.actor.userId) { res.status(403).json({ error: "Board user context required" }); return; }
+ res.json(await svc.taskIndex(companyId, { userId: req.actor.userId, actor: req.actor }));
+ });
+
router.get("/companies/:companyId/attention/history", async (req, res) => {
const companyId = req.params.companyId as string;
assertCompanyAccess(req, companyId);
diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts
index b168d12beaf..cf86556ce5e 100644
--- a/server/src/routes/issues.ts
+++ b/server/src/routes/issues.ts
@@ -16449,12 +16449,23 @@ export function issueRoutes(
router.get("/issues/:id/recent-comments", async (req, res) => {
const issue = await getAccessibleResource(req, res, getIssueById(req, req.params.id as string), "Issue not found");
if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return;
+ if (issue.conversationAgentId) {
+ const recent = await svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 21 });
+ const comments = recent.slice(0, 20);
+ res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
+ comments,
+ latestCommentId: comments[0]?.id ?? null,
+ oldestCommentId: comments.at(-1)?.id ?? null,
+ hasMore: recent.length > 20,
+ }));
+ return;
+ }
const [recent, human, firstInstruction] = await Promise.all([
- svc.listComments(issue.id, { order: "desc", limit: 5 }),
- svc.listComments(issue.id, { order: "desc", limit: 1, humanOnly: true }),
+ svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 5 }),
+ svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 1, humanOnly: true }),
issue.description?.trim()
? Promise.resolve([])
- : svc.listComments(issue.id, { order: "asc", limit: 1, humanOnly: true }),
+ : svc.listComments(issue.id, { companyId: issue.companyId, order: "asc", limit: 1, humanOnly: true }),
]);
const comments = [...new Map([...firstInstruction, ...human, ...recent.slice(0, 4)].map((comment) => [comment.id, comment])).values()];
res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
@@ -16494,7 +16505,8 @@ export function issueRoutes(
limitRaw && Number.isFinite(limitRaw) && limitRaw > 0
? Math.min(Math.floor(limitRaw), MAX_ISSUE_COMMENT_LIMIT)
: null;
- const comments = await svc.listComments(id, {
+ const comments = await svc.listComments(issue.id, {
+ companyId: issue.companyId,
afterCommentId,
order,
limit,
diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts
index e9d0f0b69e2..ff110a52e18 100644
--- a/server/src/routes/openapi.ts
+++ b/server/src/routes/openapi.ts
@@ -1,6 +1,7 @@
import { MUSE_ASSET_VERSION, museStopBoundarySchema, museDetectorCleanupAttestationSchema, createGrokInvitationSchema, configureGrokRoutineSchema, verifyGrokRoutineSchema, recoverGrokRoutineSchema, pairGrokBindingSchema, stopGrokWorkSchema, retryGrokDeliverySchema, updateFastResponseSchema } from "@paperclipai/shared";
import { slackTestMessageSchema, slackTestMessageResultSchema, slackManagerAuthorizationSchema, slackSetupOptionsSchema, slackManagedProvisionSchema, slackManagedResultSchema, slackRegistrationSchema, slackSetupActionSchema, slackInstallAuthorizationSchema, slackRegistrationStateSchema, slackAppConfigurationSchema, slackAvatarStateSchema, slackAccountStateSchema } from "@paperclipai/shared";
import { experimentalApiMetadata } from "./experimental-api-metadata.js";
+import { ATTENTION_SOURCE_KINDS } from "@paperclipai/shared";
import { agentCaseHandoffSchema, caseWorkflowDocumentTextSchema, doneCaseHandoffSchema, createWorkflowSchema, updateWorkflowSchema, listWorkflowsSchema, readWorkflowQuerySchema } from "@paperclipai/shared";
import {
experimentalApiPaths,
@@ -1417,6 +1418,8 @@ const BOARD_ONLY_OPERATIONS = new Set([
"GET /api/issues/{id}/decision-receipts",
"GET /api/issues/{id}/chat-handoffs",
"GET /api/companies/{companyId}/attention/history",
+ "GET /api/companies/{companyId}/attention/summary",
+ "GET /api/companies/{companyId}/attention/tasks",
"POST /api/issues/{id}/interactions/{interactionId}/dismiss",
"POST /api/issues/{id}/retry",
"GET /api/companies/{companyId}/prompt-injection-prevention",
@@ -4596,7 +4599,7 @@ registry.registerPath({
path: "/api/issues/{id}/recent-comments",
tags: ["issues"],
summary: "Read a bounded recent task exchange",
- description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
+ description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Agent chats instead return the latest 20 contiguous comments and oldestCommentId for backward cursor paging. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
request: { params: z.object({ id: z.string() }) },
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
});
@@ -6088,8 +6091,8 @@ registry.registerPath({
path: "/api/issues/{id}/runs",
tags: ["activity"],
summary: "List runs for an issue",
- request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries.") }) },
- responses: { 200: r.ok(), 401: r.unauthorized },
+ request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries."), runIds: z.string().optional().describe("Comma-separated list of 1 to 100 run UUIDs. Returns only those runs linked to this accessible issue.") }) },
+ responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
});
registry.registerPath({
@@ -6160,6 +6163,29 @@ registry.registerPath({
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden },
});
+const attentionSummaryResponseSchema = z.object({
+ companyId: z.string().uuid(),
+ generatedAt: z.string().datetime(),
+ totalCount: z.number().int().nonnegative(),
+ deskBadgeCount: z.number().int().nonnegative(),
+ countsBySourceKind: z.record(z.enum(ATTENTION_SOURCE_KINDS), z.number().int().nonnegative()),
+}).strict();
+
+for (const [projection, responseSchema] of [
+ ["summary", attentionSummaryResponseSchema],
+ ["tasks", attentionSummaryResponseSchema.extend({ taskIds: z.array(z.string().uuid()) })],
+] as const) {
+ registry.registerPath({
+ method: "get",
+ path: `/api/companies/{companyId}/attention/${projection}`,
+ tags: ["inbox"],
+ summary: projection === "summary" ? "Get canonical attention counts" : "Get complete attention task membership",
+ description: "Requires a board user with company access. Reads the complete canonical attention set with current task privacy, addressee, dismissal, snooze, and archive rules. Returns counts without decision cards, images, plan bodies, or pagination. The tasks projection also returns distinct accessible task IDs. This read grants no resolution authority.",
+ request: { params: z.object({ companyId: z.string() }) },
+ responses: { 200: r.ok(responseSchema), 401: r.unauthorized, 403: r.forbidden },
+ });
+}
+
// ─── Decisions ──────────────────────────────────────────────────────────────
registry.registerPath({
diff --git a/server/src/routes/plugin-ui-static.ts b/server/src/routes/plugin-ui-static.ts
index ca9cae7db08..7a9efb4b249 100644
--- a/server/src/routes/plugin-ui-static.ts
+++ b/server/src/routes/plugin-ui-static.ts
@@ -32,6 +32,7 @@ import path from "node:path";
import fs from "node:fs";
import crypto from "node:crypto";
import type { Db } from "@paperclipai/db";
+import type { PluginRecord } from "@paperclipai/shared";
import { pluginRegistryService } from "../services/plugin-registry.js";
import { logger } from "../middleware/logger.js";
import { assertCompanyAccess } from "./authz.js";
@@ -192,6 +193,7 @@ export interface PluginUiStaticRouteOptions {
* Defaults to the standard `~/.paperclip/plugins/` location.
*/
localPluginDir: string;
+ resolvePackageRoot?: (plugin: PluginRecord) => string;
}
/**
@@ -304,7 +306,7 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
if (typeof devUiUrl === "string" && devUiUrl.length > 0) {
// Dev proxy is only available in development mode
- if (process.env.NODE_ENV === "production") {
+ if (options.resolvePackageRoot || process.env.NODE_ENV === "production") {
log.warn(
{ pluginId: plugin.id },
"plugin-ui-static: devUiUrl ignored in production",
@@ -405,12 +407,15 @@ export function pluginUiStaticRoutes(db: Db, options: PluginUiStaticRouteOptions
}
// Step 3: Resolve the plugin's UI directory
- const uiDir = resolvePluginUiDir(
- options.localPluginDir,
- plugin.packageName,
- manifest.entrypoints.ui,
- plugin.packagePath,
- );
+ let uiDir: string | null;
+ try {
+ uiDir = options.resolvePackageRoot
+ ? path.resolve(options.resolvePackageRoot(plugin), manifest.entrypoints.ui)
+ : resolvePluginUiDir(options.localPluginDir, plugin.packageName, manifest.entrypoints.ui, plugin.packagePath);
+ } catch {
+ res.status(503).json({ error: "Plugin package is unavailable in the current image" });
+ return;
+ }
if (!uiDir) {
log.warn(
diff --git a/server/src/secrets/local-encrypted-provider.ts b/server/src/secrets/local-encrypted-provider.ts
index ea0611bd06f..f0162fb56e8 100644
--- a/server/src/secrets/local-encrypted-provider.ts
+++ b/server/src/secrets/local-encrypted-provider.ts
@@ -94,8 +94,8 @@ function loadOrCreateMasterKey(allowCreate = true, env: LocalEncryptedKeyEnv = p
}
/** Derive a purpose-specific instance secret without exposing or reusing the vault key. */
-export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env): string {
- return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(true, env), Buffer.alloc(0),
+export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env, options: { allowCreate?: boolean } = {}): string {
+ return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(options.allowCreate ?? true, env), Buffer.alloc(0),
`paperclip:instance-secret:v1\0${purpose}`, 32)).toString("base64url");
}
diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts
index b256bf358ff..bac15a5239f 100644
--- a/server/src/services/activity.ts
+++ b/server/src/services/activity.ts
@@ -428,7 +428,8 @@ export function activityService(db: Db) {
return limit === undefined ? query : query.limit(limit);
},
- runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number) => {
+ runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number, selectedRunIds?: string[]) => {
+ if (selectedRunIds && !selectedRunIds.length) return [];
scheduleRunLivenessBackfill(companyId, issueId);
const runsQuery = db
.select({
@@ -479,6 +480,7 @@ export function activityService(db: Db) {
.where(
and(
eq(heartbeatRuns.companyId, companyId),
+ ...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []),
or(
eq(heartbeatRuns.issueId, issueId),
sql`exists (
diff --git a/server/src/services/adapter-code-policy.ts b/server/src/services/adapter-code-policy.ts
new file mode 100644
index 00000000000..49c1e822f80
--- /dev/null
+++ b/server/src/services/adapter-code-policy.ts
@@ -0,0 +1,10 @@
+import { forbidden } from "../errors.js";
+import { isCloudManagedInstance } from "./cloud-instance.js";
+
+export function assertExternalAdapterCodeAllowed(): void {
+ if (isCloudManagedInstance()) {
+ throw forbidden("Adapter installation is platform-managed on cloud-managed instances", {
+ code: "adapter_install_platform_managed",
+ });
+ }
+}
diff --git a/server/src/services/adapter-plugin-store.ts b/server/src/services/adapter-plugin-store.ts
index 13369323d4f..8ccdf3f6049 100644
--- a/server/src/services/adapter-plugin-store.ts
+++ b/server/src/services/adapter-plugin-store.ts
@@ -15,6 +15,8 @@
import fs from "node:fs";
import path from "node:path";
import { resolvePaperclipHomeDir } from "../home-paths.js";
+import { isCloudManagedInstance } from "./cloud-instance.js";
+import { assertExternalAdapterCodeAllowed } from "./adapter-code-policy.js";
// ---------------------------------------------------------------------------
// Types
@@ -43,8 +45,8 @@ interface AdapterSettings {
// Paths
// ---------------------------------------------------------------------------
-function adapterPluginPaths() {
- const paperclipDir = resolvePaperclipHomeDir();
+function adapterPluginPaths(homeOverride?: string) {
+ const paperclipDir = resolvePaperclipHomeDir(homeOverride);
return {
adapterPluginsDir: path.join(paperclipDir, "adapter-plugins"),
adapterPluginsStorePath: path.join(paperclipDir, "adapter-plugins.json"),
@@ -79,6 +81,7 @@ function ensureDirs(): string {
}
function readStore(): AdapterPluginRecord[] {
+ if (isCloudManagedInstance()) return [];
const { adapterPluginsStorePath } = adapterPluginPaths();
if (storeCache?.path === adapterPluginsStorePath) return storeCache.records;
try {
@@ -130,11 +133,16 @@ function writeSettings(settings: AdapterSettings): void {
// Public API
// ---------------------------------------------------------------------------
+export function getAdapterPluginStorePath(homeOverride?: string): string {
+ return adapterPluginPaths(homeOverride).adapterPluginsStorePath;
+}
+
export function listAdapterPlugins(): AdapterPluginRecord[] {
return readStore();
}
export function addAdapterPlugin(record: AdapterPluginRecord): void {
+ assertExternalAdapterCodeAllowed();
const store = [...readStore()];
const idx = store.findIndex((r) => r.type === record.type);
if (idx >= 0) {
@@ -146,6 +154,7 @@ export function addAdapterPlugin(record: AdapterPluginRecord): void {
}
export function removeAdapterPlugin(type: string): boolean {
+ assertExternalAdapterCodeAllowed();
const store = [...readStore()];
const idx = store.findIndex((r) => r.type === type);
if (idx < 0) return false;
@@ -159,6 +168,7 @@ export function getAdapterPluginByType(type: string): AdapterPluginRecord | unde
}
export function getAdapterPluginsDir(): string {
+ assertExternalAdapterCodeAllowed();
return ensureDirs();
}
diff --git a/server/src/services/adapter-registry-bootstrap.reconcile.test.ts b/server/src/services/adapter-registry-bootstrap.reconcile.test.ts
index 1e32e3575fe..01409c571ec 100644
--- a/server/src/services/adapter-registry-bootstrap.reconcile.test.ts
+++ b/server/src/services/adapter-registry-bootstrap.reconcile.test.ts
@@ -32,8 +32,7 @@ describe("reconcileAdapterAvailability", () => {
]);
expect(result.enabled).toEqual(["opencode_local"]);
expect(result.disabled.sort()).toEqual(["claude_local", "pi_local"]);
- expect(setAdapterDisabled).toHaveBeenCalledWith("claude_local", true);
- expect(setAdapterDisabled).toHaveBeenCalledWith("opencode_local", false);
+ expect(setAdapterDisabled).not.toHaveBeenCalled();
});
it("throws when a declared adapter has no installed implementation", () => {
diff --git a/server/src/services/adapter-registry-bootstrap.ts b/server/src/services/adapter-registry-bootstrap.ts
index c8e7f4b0f0b..e4e609eed09 100644
--- a/server/src/services/adapter-registry-bootstrap.ts
+++ b/server/src/services/adapter-registry-bootstrap.ts
@@ -1,73 +1,8 @@
-/**
- * Declarative adapter-registry bootstrap.
- *
- * One source (`PAPERCLIP_ADAPTERS` inline JSON, or `PAPERCLIP_ADAPTERS_FILE` a
- * path to a JSON file) feeds two consumers:
- * 1. Availability: reconcile the file-backed disabled-set so the picker shows
- * exactly the declared, enabled set (runs for any instance).
- * 2. k8s runtime: the same registry rides on the Kubernetes environment config
- * (see execution-policy-bootstrap) so the plugin resolves runtime
- * image/envKeys/allowFqdns/probe/defaultEnv from it.
- *
- * Parsing is pure + fails loud on malformed/invalid config, mirroring
- * execution-policy-bootstrap.
- */
-import fs from "node:fs";
-import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
+import type { AdapterRegistryEntryParsed } from "@paperclipai/shared";
import { logger } from "../middleware/logger.js";
import { listServerAdapters } from "../adapters/registry.js";
-import { setAdapterDisabled } from "./adapter-plugin-store.js";
+export { parseAdapterRegistryEnv, type AdapterRegistryEnv } from "./adapter-registry-config.js";
-export type AdapterRegistryEnv = Record;
-
-/**
- * Parse the declarative registry from env. Returns null when unconfigured
- * (built-in defaults; local/OSS unchanged). Throws on malformed/invalid input.
- */
-export function parseAdapterRegistryEnv(
- env: AdapterRegistryEnv = process.env,
-): AdapterRegistryEntryParsed[] | null {
- const inline = env.PAPERCLIP_ADAPTERS?.trim();
- const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
- if (!inline && !filePath) return null;
-
- let rawText: string;
- if (inline) {
- rawText = inline;
- } else {
- try {
- rawText = fs.readFileSync(filePath as string, "utf-8");
- } catch (err) {
- throw new Error(
- `PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
- );
- }
- }
-
- let parsed: unknown;
- try {
- parsed = JSON.parse(rawText);
- } catch (err) {
- throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
- }
-
- const result = adapterRegistrySchema.safeParse(parsed);
- if (!result.success) {
- throw new Error(
- `PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
- .map((i) => `${i.path.join(".")}: ${i.message}`)
- .join("; ")}`,
- );
- }
- return result.data;
-}
-
-/**
- * Reconcile availability: every known server adapter NOT enabled in the declared
- * registry is disabled; declared+enabled ones are enabled. Throws if a declared
- * adapterType has no installed adapter (cannot offer a harness with no
- * implementation). No-op when `registry` is null.
- */
export function reconcileAdapterAvailability(
registry: AdapterRegistryEntryParsed[] | null,
): { enabled: string[]; disabled: string[] } {
@@ -88,10 +23,9 @@ export function reconcileAdapterAvailability(
for (const type of knownTypes) {
const entry = declared.get(type);
const shouldEnable = entry !== undefined && entry.enabled !== false;
- setAdapterDisabled(type, !shouldEnable);
(shouldEnable ? enabled : disabled).push(type);
}
- logger.info({ enabled, disabled }, "reconciled adapter availability from PAPERCLIP_ADAPTERS");
+ logger.info({ enabled, disabled }, "validated adapter availability from PAPERCLIP_ADAPTERS");
return { enabled, disabled };
}
diff --git a/server/src/services/adapter-registry-config.ts b/server/src/services/adapter-registry-config.ts
new file mode 100644
index 00000000000..732f5428051
--- /dev/null
+++ b/server/src/services/adapter-registry-config.ts
@@ -0,0 +1,47 @@
+import fs from "node:fs";
+import { adapterRegistrySchema, type AdapterRegistryEntryParsed } from "@paperclipai/shared";
+
+export type AdapterRegistryEnv = Record;
+
+let cached: { inline: string | undefined; filePath: string | undefined; registry: AdapterRegistryEntryParsed[] } | undefined;
+
+export function parseAdapterRegistryEnv(
+ env: AdapterRegistryEnv = process.env,
+): AdapterRegistryEntryParsed[] | null {
+ const inline = env.PAPERCLIP_ADAPTERS?.trim();
+ const filePath = env.PAPERCLIP_ADAPTERS_FILE?.trim();
+ if (!inline && !filePath) return null;
+ if (cached && cached.inline === inline && cached.filePath === filePath) return structuredClone(cached.registry);
+
+ let rawText: string;
+ if (inline) {
+ rawText = inline;
+ } else {
+ try {
+ rawText = fs.readFileSync(filePath as string, "utf-8");
+ } catch (err) {
+ throw new Error(
+ `PAPERCLIP_ADAPTERS_FILE could not be read at "${filePath}": ${(err as Error).message}`,
+ );
+ }
+ }
+
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(rawText);
+ } catch (err) {
+ throw new Error(`PAPERCLIP_ADAPTERS must be valid JSON: ${(err as Error).message}`);
+ }
+
+ const result = adapterRegistrySchema.safeParse(parsed);
+ if (!result.success) {
+ throw new Error(
+ `PAPERCLIP_ADAPTERS failed validation: ${result.error.issues
+ .map((i) => `${i.path.join(".")}: ${i.message}`)
+ .join("; ")}`,
+ );
+ }
+ cached = { inline, filePath, registry: result.data };
+ return structuredClone(result.data);
+}
+
diff --git a/server/src/services/adapter-settings-local.test.ts b/server/src/services/adapter-settings-local.test.ts
new file mode 100644
index 00000000000..03e56fcb941
--- /dev/null
+++ b/server/src/services/adapter-settings-local.test.ts
@@ -0,0 +1,87 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+vi.mock("../adapters/registry.js", () => ({ listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }] }));
+import { adapterSettingsService, type AdapterSettingsDb } from "./adapter-settings.js";
+import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
+import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
+
+describe("local adapter settings compatibility", () => {
+ let directory: string;
+ beforeEach(() => {
+ directory = fs.mkdtempSync(path.join(os.tmpdir(), "adapter-settings-local-"));
+ vi.stubEnv("PAPERCLIP_HOME", directory);
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
+ vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
+ vi.stubEnv("PAPERCLIP_ADAPTERS", undefined);
+ vi.stubEnv("PAPERCLIP_ADAPTERS_FILE", undefined);
+ });
+ afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ fs.rmSync(directory, { recursive: true, force: true });
+ });
+
+ it("reads and writes the existing file without a database", async () => {
+ fs.writeFileSync(path.join(directory, "adapter-settings.json"), JSON.stringify({ disabledTypes: ["claude_local"] }));
+ const service = adapterSettingsService(undefined);
+ expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
+ expect(await service.setDisabled("codex_local", true)).toBe(true);
+ expect(getDisabledAdapterTypes()).toEqual(["claude_local", "codex_local"]);
+ expect(setAdapterDisabled("claude_local", false)).toBe(true);
+ expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
+ expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: ["codex_local"] });
+ });
+
+ it("loads an operator file once and leaves the saved file intact", async () => {
+ setAdapterDisabled("codex_local", true);
+ const file = path.join(directory, "operator-adapters.json");
+ fs.writeFileSync(file, JSON.stringify([{ adapterType: "codex_local", enabled: true }]));
+ const env = { PAPERCLIP_ADAPTERS_FILE: file };
+ parseAdapterRegistryEnv(env);
+ const read = vi.spyOn(fs, "readFileSync");
+ const service = adapterSettingsService(undefined, { runtimeEnv: env });
+ expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
+ expect(await service.getDisabledTypes()).toEqual(["claude_local"]);
+ expect(read).not.toHaveBeenCalled();
+ expect(getDisabledAdapterTypes()).toEqual(["codex_local"]);
+ });
+
+ it("restores the saved choice when its audit fails and audits a later retry", async () => {
+ const transaction = vi.fn(async (callback: (tx: AdapterSettingsDb) => Promise) => callback(db));
+ const db = { transaction } as unknown as AdapterSettingsDb;
+ const service = adapterSettingsService(db);
+ const audit = vi.fn().mockRejectedValueOnce(new Error("audit unavailable")).mockResolvedValue(undefined);
+
+ await expect(service.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
+ expect(await service.getDisabledTypes()).toEqual([]);
+ expect(JSON.parse(fs.readFileSync(path.join(directory, "adapter-settings.json"), "utf8"))).toEqual({ disabledTypes: [] });
+ expect(await service.setDisabled("codex_local", true, audit)).toBe(true);
+ expect(await service.getDisabledTypes()).toEqual(["codex_local"]);
+ expect(audit).toHaveBeenCalledTimes(2);
+ expect(transaction).toHaveBeenCalledTimes(2);
+ });
+
+ it("keeps a failed audit rollback from overwriting a concurrent save", async () => {
+ const db = { transaction: async (callback: (tx: AdapterSettingsDb) => Promise) => callback(db) } as unknown as AdapterSettingsDb;
+ const first = adapterSettingsService(db);
+ const second = adapterSettingsService(db);
+ let failAudit!: (reason: Error) => void;
+ const auditWait = new Promise((_resolve, reject) => { failAudit = reject; });
+ const audit = vi.fn(() => auditWait);
+ const rejected = expect(first.setDisabled("codex_local", true, audit)).rejects.toThrow("audit unavailable");
+ await vi.waitFor(() => expect(audit).toHaveBeenCalledOnce());
+ const nextAudit = vi.fn(async () => undefined);
+ const next = second.setDisabled("codex_local", true, nextAudit);
+ await Promise.resolve();
+ expect(nextAudit).not.toHaveBeenCalled();
+
+ failAudit(new Error("audit unavailable"));
+ await rejected;
+ expect(await next).toBe(true);
+ expect(await first.getDisabledTypes()).toEqual(["codex_local"]);
+ expect(nextAudit).toHaveBeenCalledOnce();
+ });
+});
diff --git a/server/src/services/adapter-settings-postgres.test.ts b/server/src/services/adapter-settings-postgres.test.ts
new file mode 100644
index 00000000000..299c4b14f35
--- /dev/null
+++ b/server/src/services/adapter-settings-postgres.test.ts
@@ -0,0 +1,228 @@
+import express from "express";
+import request from "supertest";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { activityLog, applyPendingMigrations, companies, createDb, instanceSettings, type Db } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
+
+const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() }));
+vi.mock("./adapter-plugin-store.js", () => ({
+ getDisabledAdapterTypes: legacy.get,
+ setAdapterDisabled: legacy.set,
+ listAdapterPlugins: () => [],
+ getAdapterPluginByType: () => undefined,
+}));
+vi.mock("../adapters/registry.js", () => ({
+ listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }],
+ findServerAdapter: (type: string) => ({ type }),
+ findActiveServerAdapter: (type: string) => ({ type }),
+ isOverridePaused: () => false,
+ hasActiveAdapterOverride: () => false,
+ waitForExternalAdapters: async () => {},
+}));
+vi.mock("../adapters/plugin-loader.js", () => ({}));
+import { adapterSettingsService, initializeAdapterSettings } from "./adapter-settings.js";
+import { adapterRoutes } from "../routes/adapters.js";
+import { errorHandler } from "../middleware/error-handler.js";
+import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
+import * as activities from "./activity-log.js";
+
+const externalDatabaseUrl = process.env.PAPERCLIP_ADAPTER_SETTINGS_TEST_DATABASE_URL;
+const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
+(support.supported ? describe : describe.skip)("durable managed adapter settings", () => {
+ let database: Awaited>;
+ let db: Db;
+ let peer: Db;
+ let directory: string;
+ const managed = { runtimeEnv: { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "synthetic-test-token" } };
+ const service = () => adapterSettingsService(db, managed);
+
+ beforeAll(async () => {
+ database = externalDatabaseUrl
+ ? { connectionString: externalDatabaseUrl, cleanup: async () => {} }
+ : await startEmbeddedPostgresTestDatabase("adapter-settings-");
+ if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
+ db = createDb(database.connectionString);
+ peer = createDb(database.connectionString);
+ await db.insert(companies).values({ name: "Adapter settings test", issuePrefix: "AST" });
+ }, 90_000);
+ afterAll(async () => {
+ await Promise.all([db?.$client.end({ timeout: 0 }), peer?.$client.end({ timeout: 0 })]);
+ await database?.cleanup();
+ }, 60_000);
+ beforeEach(async () => {
+ directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-"));
+ await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1"));
+ legacy.get.mockClear();
+ legacy.set.mockClear();
+ vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token");
+ });
+ afterEach(async () => {
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ await fs.rm(directory, { recursive: true, force: true });
+ });
+
+ function app() {
+ const app = express();
+ app.use(express.json());
+ app.use((req, _res, next) => {
+ req.actor = { type: "board", source: "local_implicit", userId: "local-board", isInstanceAdmin: true };
+ next();
+ });
+ app.use(adapterRoutes({ db }));
+ app.use(errorHandler);
+ return app;
+ }
+
+ it("persists across independent connections and reconstructed stores without file reads", async () => {
+ await expect(service().getDisabledTypes()).rejects.toThrow("not been initialized");
+ await service().seedIfAbsent({ disabledTypes: [] });
+ expect(await service().setDisabled("codex_local", true)).toBe(true);
+ expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
+ await db.$client.end({ timeout: 0 });
+ db = createDb(database.connectionString);
+ expect(await service().inspectStored()).toEqual({ version: 1, revision: 1, disabledTypes: ["codex_local"] });
+ expect(legacy.get).not.toHaveBeenCalled();
+ expect(legacy.set).not.toHaveBeenCalled();
+ });
+
+ it("acknowledges the committed save when its live activity event fails", async () => {
+ await service().seedIfAbsent({ disabledTypes: [] });
+ const publication = vi.spyOn(activities, "publishActivity").mockImplementation(() => { throw new Error("event unavailable"); });
+ const response = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
+ expect(response.status, JSON.stringify(response.body)).toBe(200);
+ expect(publication).toHaveBeenCalled();
+ expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual(["codex_local"]);
+ const events = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
+ expect(events.some((event) => event.details?.adapterType === "codex_local")).toBe(true);
+ });
+
+ it("preserves simultaneous first writes and concurrent removals from other connections", async () => {
+ const writers = [service(), adapterSettingsService(peer, managed)];
+ await Promise.all(Array.from({ length: 12 }, (_, index) => writers[index % 2].setDisabled(`adapter_${index}`, true)));
+ expect((await service().inspectStored())?.disabledTypes).toHaveLength(12);
+ await Promise.all([
+ writers[0].setDisabled("adapter_0", false),
+ writers[1].setDisabled("adapter_1", false),
+ writers[0].setDisabled("extra", true),
+ ]);
+ const stored = await service().inspectStored();
+ expect(stored?.revision).toBe(15);
+ expect(stored?.disabledTypes).not.toContain("adapter_0");
+ expect(stored?.disabledTypes).not.toContain("adapter_1");
+ expect(stored?.disabledTypes).toContain("extra");
+ expect(await writers[1].setDisabled("extra", true)).toBe(false);
+ expect((await service().inspectStored())?.revision).toBe(15);
+ });
+
+ it("imports legacy choices once before admission and ignores stale files after the DB record exists", async () => {
+ const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
+ const file = path.join(directory, "adapter-settings.json");
+ await fs.writeFile(file, JSON.stringify({ disabledTypes: ["codex_local", "future_adapter"] }));
+ const results = await Promise.all([
+ initializeAdapterSettings(db, { runtimeEnv }),
+ initializeAdapterSettings(peer, { runtimeEnv }),
+ ]);
+ expect(results.filter((result) => result?.seeded)).toHaveLength(1);
+ expect(await service().getDisabledTypes()).toEqual(["codex_local", "future_adapter"]);
+ await service().setDisabled("codex_local", false);
+ await fs.writeFile(file, "invalid stale JSON");
+ expect((await initializeAdapterSettings(peer, { runtimeEnv }))?.seeded).toBe(false);
+ expect(await service().getDisabledTypes()).toEqual(["future_adapter"]);
+ });
+
+ it("uses defaults only for an absent legacy file and rejects malformed or oversized input", async () => {
+ const runtimeEnv = { ...managed.runtimeEnv, PAPERCLIP_HOME: directory };
+ const file = path.join(directory, "adapter-settings.json");
+ await fs.writeFile(file, JSON.stringify({ disabledTypes: [42] }));
+ await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("Persisted adapter settings are invalid");
+ expect(await service().inspectStored()).toBeNull();
+ await fs.writeFile(file, Buffer.alloc(65_537));
+ await expect(initializeAdapterSettings(db, { runtimeEnv })).rejects.toThrow("size limit");
+ expect(await service().inspectStored()).toBeNull();
+ await fs.unlink(file);
+ expect((await initializeAdapterSettings(db, { runtimeEnv }))?.current.disabledTypes).toEqual([]);
+ expect(await service().getDisabledTypes()).toEqual([]);
+ });
+
+ it("composes atomic import and receipts with the caller transaction", async () => {
+ await expect(db.transaction(async (tx) => {
+ expect(await service().seedIfAbsent({ disabledTypes: ["codex_local"] }, tx)).toEqual({
+ seeded: true, current: { version: 1, revision: 0, disabledTypes: ["codex_local"] },
+ });
+ throw new Error("receipt failed");
+ })).rejects.toThrow("receipt failed");
+ expect(await service().inspectStored()).toBeNull();
+ await Promise.all([
+ service().seedIfAbsent({ disabledTypes: ["codex_local"] }),
+ adapterSettingsService(peer, managed).seedIfAbsent({ disabledTypes: ["claude_local"] }),
+ ]).then((results) => {
+ expect(results.filter((result) => result.seeded)).toHaveLength(1);
+ expect(results[0].current).toEqual(results[1].current);
+ });
+ });
+
+ it("applies operator restrictions without rewriting saved choices", async () => {
+ await service().setDisabled("codex_local", true);
+ const env = { ...managed.runtimeEnv, PAPERCLIP_ADAPTERS: JSON.stringify([{ adapterType: "codex_local", enabled: true }]) };
+ const overlay = adapterSettingsService(peer, { runtimeEnv: env });
+ expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
+ expect(await overlay.inspectStored()).toEqual(await service().inspectStored());
+ await expect(overlay.setDisabled("claude_local", false)).rejects.toThrow("hosting operator");
+ await expect(overlay.setDisabled("codex_local", true)).rejects.toThrow("hosting operator");
+ expect(await overlay.getDisabledTypes()).toEqual(["claude_local", "paperclip_runner"]);
+ expect(await service().getDisabledTypes()).toEqual(["codex_local"]);
+ });
+
+ it("refreshes the disabled harness at each new-agent admission", async () => {
+ await service().seedIfAbsent({ disabledTypes: [] });
+ const input = { adapterType: "codex_local", runner: "legacy" as const };
+ expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
+ await adapterSettingsService(peer, managed).setDisabled("codex_local", true);
+ await expect(resolveNewAgentRunnerForCompany(db, "company", input)).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
+ await adapterSettingsService(peer, managed).setDisabled("codex_local", false);
+ expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
+ });
+
+ it("rolls back the setting when its audit write fails", async () => {
+ await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed");
+ expect(await service().inspectStored()).toBeNull();
+ await expect(adapterSettingsService(peer, managed).getDisabledTypes()).rejects.toThrow("not been initialized");
+ });
+
+ it("publishes catalog changes across stores and records the instance mutation", async () => {
+ const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
+ expect(res.status).toBe(200);
+ expect(res.body).toEqual({ type: "codex_local", disabled: true, changed: true });
+ const catalog = await request(app()).get("/adapters");
+ expect(catalog.status).toBe(200);
+ expect(catalog.body.find((entry: { type: string }) => entry.type === "codex_local").disabled).toBe(true);
+ const [audit] = await peer.select().from(activityLog).where(eq(activityLog.action, "instance.adapter.updated"));
+ expect(audit.details).toMatchObject({ adapterType: "codex_local", disabled: true, revision: 1 });
+ });
+
+ it("returns failure when PostgreSQL rejects the update and keeps the durable choice", async () => {
+ await service().seedIfAbsent({ disabledTypes: [] });
+ await db.execute(sql`CREATE FUNCTION reject_adapter_settings() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test adapter settings failure'; END $$`);
+ await db.execute(sql`CREATE TRIGGER reject_adapter_settings BEFORE UPDATE ON instance_settings FOR EACH ROW EXECUTE FUNCTION reject_adapter_settings()`);
+ try {
+ const res = await request(app()).patch("/adapters/codex_local").send({ disabled: true });
+ expect(res.status).toBe(500);
+ expect(await adapterSettingsService(peer, managed).getDisabledTypes()).toEqual([]);
+ } finally {
+ await db.execute(sql`DROP TRIGGER reject_adapter_settings ON instance_settings`);
+ await db.execute(sql`DROP FUNCTION reject_adapter_settings()`);
+ }
+ });
+
+ it("fails closed on malformed stored settings and unavailable database", async () => {
+ await db.insert(instanceSettings).values({ singletonKey: "adapter-settings:v1", general: { version: 2, disabledTypes: [] } });
+ await expect(service().getDisabledTypes()).rejects.toThrow("Persisted adapter settings are invalid");
+ await expect(adapterSettingsService(undefined, managed).getDisabledTypes()).rejects.toThrow("database is required");
+ expect(legacy.get).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/services/adapter-settings.ts b/server/src/services/adapter-settings.ts
new file mode 100644
index 00000000000..4b9459b7165
--- /dev/null
+++ b/server/src/services/adapter-settings.ts
@@ -0,0 +1,149 @@
+import type { Db } from "@paperclipai/db";
+import { instanceSettings } from "@paperclipai/db";
+import { eq } from "drizzle-orm";
+import { open } from "node:fs/promises";
+import { constants } from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+import { listServerAdapters } from "../adapters/registry.js";
+import { forbidden } from "../errors.js";
+import { resolvePaperclipHomeDir } from "../home-paths.js";
+import { getDisabledAdapterTypes, setAdapterDisabled } from "./adapter-plugin-store.js";
+import { parseAdapterRegistryEnv } from "./adapter-registry-config.js";
+import { isCloudManagedInstance } from "./cloud-instance.js";
+
+const SINGLETON_KEY = "adapter-settings:v1";
+let localSettingsMutation: Promise = Promise.resolve();
+type Transaction = Parameters[0]>[0];
+export type AdapterSettingsDb = Pick;
+export type StoredAdapterSettings = { version: 1; revision: number; disabledTypes: string[] };
+const storedSettingsSchema = z.object({
+ version: z.literal(1),
+ revision: z.number().int().nonnegative(),
+ disabledTypes: z.array(z.string().refine((type) => type.trim().length > 0)),
+});
+
+function parseStored(value: Record): StoredAdapterSettings {
+ const parsed = storedSettingsSchema.safeParse(value);
+ if (!parsed.success) throw new Error("Persisted adapter settings are invalid");
+ return { ...parsed.data, disabledTypes: [...new Set(parsed.data.disabledTypes)].sort() };
+}
+
+export function effectiveDisabledAdapterTypes(saved: string[], knownTypes: string[] | (() => string[]), env: Record = process.env): string[] {
+ const registry = parseAdapterRegistryEnv(env);
+ if (!registry) return [...saved];
+ const enabled = new Set(registry.filter((entry) => entry.enabled !== false).map((entry) => entry.adapterType));
+ const types = typeof knownTypes === "function" ? knownTypes() : knownTypes;
+ return [...new Set(types.filter((type) => !enabled.has(type)))].sort();
+}
+
+export function adapterSettingsService(db: AdapterSettingsDb | undefined, options: { runtimeEnv?: Record } = {}) {
+ const env = options.runtimeEnv ?? process.env;
+ const managed = isCloudManagedInstance(env);
+
+ function requireDb(): AdapterSettingsDb {
+ if (!db) throw new Error("A database is required for managed adapter settings");
+ return db;
+ }
+
+ async function inspectStored(handle: AdapterSettingsDb = requireDb()): Promise {
+ const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
+ .where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).limit(1);
+ return row ? parseStored(row.general) : null;
+ }
+
+ async function seedIfAbsent(value: { disabledTypes: string[] }, handle: AdapterSettingsDb = requireDb()) {
+ const snapshot = parseStored({ version: 1, revision: 0, disabledTypes: value.disabledTypes });
+ const inserted = await handle.insert(instanceSettings).values({ singletonKey: SINGLETON_KEY, general: snapshot })
+ .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
+ const current = await inspectStored(handle);
+ if (!current) throw new Error("Adapter settings seed was not persisted");
+ return { seeded: inserted.length > 0, current };
+ }
+
+ return {
+ inspectStored,
+ seedIfAbsent,
+ async getDisabledTypes(): Promise {
+ const current = managed ? await inspectStored() : null;
+ if (managed && !current) throw new Error("Managed adapter settings have not been initialized");
+ const stored = managed ? current!.disabledTypes : getDisabledAdapterTypes();
+ return effectiveDisabledAdapterTypes(stored, () => listServerAdapters().map((adapter) => adapter.type), env);
+ },
+ async setDisabled(type: string, disabled: boolean, afterChange?: (handle: AdapterSettingsDb, current: StoredAdapterSettings) => Promise): Promise {
+ if (!type.trim()) throw new Error("Adapter type is required");
+ const registry = parseAdapterRegistryEnv(env);
+ if (registry && disabled === registry.some((entry) => entry.adapterType === type && entry.enabled !== false)) {
+ throw forbidden("Adapter availability is managed by the hosting operator", { code: "settings_operator_managed" });
+ }
+ if (!managed) {
+ const mutation = localSettingsMutation.then(async () => {
+ const previous = getDisabledAdapterTypes().includes(type);
+ const changed = setAdapterDisabled(type, disabled);
+ if (!changed || !db || !afterChange) return changed;
+ try {
+ await db.transaction(async (tx) => {
+ await afterChange(tx, { version: 1, revision: 0, disabledTypes: getDisabledAdapterTypes() });
+ });
+ } catch (error) {
+ setAdapterDisabled(type, previous);
+ throw error;
+ }
+ return true;
+ });
+ localSettingsMutation = mutation.catch(() => undefined);
+ return mutation;
+ }
+ return requireDb().transaction(async (tx) => {
+ await seedIfAbsent({ disabledTypes: [] }, tx);
+ const [row] = await tx.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, SINGLETON_KEY)).for("update");
+ if (!row) throw new Error("Adapter settings are unavailable");
+ const current = parseStored(row.general);
+ const types = new Set(current.disabledTypes);
+ if (types.has(type) === disabled) return false;
+ if (disabled) types.add(type); else types.delete(type);
+ const next: StoredAdapterSettings = { version: 1, revision: current.revision + 1, disabledTypes: [...types].sort() };
+ await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
+ await afterChange?.(tx, next);
+ return true;
+ });
+ },
+ };
+}
+
+export type AdapterSettingsService = Pick, "getDisabledTypes" | "setDisabled">;
+
+export async function initializeAdapterSettings(db: AdapterSettingsDb, options: { runtimeEnv?: Record } = {}) {
+ const env = options.runtimeEnv ?? process.env;
+ if (!isCloudManagedInstance(env)) return null;
+ const service = adapterSettingsService(db, options);
+ const current = await service.inspectStored();
+ if (current) return { seeded: false, current };
+ return service.seedIfAbsent(await readLegacyAdapterSettings(env) ?? { disabledTypes: [] });
+}
+
+export async function readLegacyAdapterSettings(env: Record = process.env): Promise<{ disabledTypes: string[] } | null> {
+ const filePath = path.join(resolvePaperclipHomeDir(env.PAPERCLIP_HOME), "adapter-settings.json");
+ try {
+ const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+ try {
+ const stat = await file.stat();
+ if (!stat.isFile() || stat.size > 65_536) throw new Error("Legacy adapter settings exceed the size limit or are not a regular file");
+ const buffer = Buffer.alloc(65_537);
+ let total = 0;
+ while (total < buffer.length) {
+ const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
+ if (bytesRead === 0) break;
+ total += bytesRead;
+ }
+ if (total > 65_536) throw new Error("Legacy adapter settings exceed the size limit");
+ const value = JSON.parse(buffer.subarray(0, total).toString("utf8"));
+ return { disabledTypes: parseStored({ version: 1, revision: 0, disabledTypes: value?.disabledTypes }).disabledTypes };
+ } finally {
+ await file.close();
+ }
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+ return null;
+ }
+}
diff --git a/server/src/services/agent-identity.ts b/server/src/services/agent-identity.ts
index 462c0393931..42615880789 100644
--- a/server/src/services/agent-identity.ts
+++ b/server/src/services/agent-identity.ts
@@ -5,6 +5,7 @@ import type { AgentPublicIdentity } from "@paperclipai/shared";
import { notFound } from "../errors.js";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import { persistActivity } from "./activity-log.js";
+import { createAgentIdentityRedactor } from "./agent-identity-redaction.js";
const publicColumns = {
algorithm: agentIdentityKeys.algorithm,
@@ -17,6 +18,16 @@ export function agentIdentityService(db: Db) {
const ownership = (companyId: string, agentId: string) => and(
eq(agentIdentityKeys.companyId, companyId), eq(agentIdentityKeys.agentId, agentId),
);
+ async function resolveIdentity(row: typeof agentIdentityKeys.$inferSelect) {
+ const privateKeyPem = await localEncryptedProvider.resolveVersion({ material: row.privateKeyMaterial, externalRef: null });
+ const privateKey = createPrivateKey(privateKeyPem);
+ const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" });
+ const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" });
+ if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" ||
+ !derived.equals(stored) || fingerprint(stored) !== row.keyId)
+ throw new Error("Agent identity public and private keys do not match");
+ return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem };
+ }
return {
// Reading an identity must never provision it or decrypt private material.
@@ -24,6 +35,10 @@ export function agentIdentityService(db: Db) {
const [row] = await db.select(publicColumns).from(agentIdentityKeys).where(ownership(companyId, agentId));
return row ? { ...row, createdAt: row.createdAt.toISOString() } : null;
},
+ async createExistingLogRedactor(companyId: string, agentId: string) {
+ const [row] = await db.select().from(agentIdentityKeys).where(ownership(companyId, agentId));
+ return createAgentIdentityRedactor(row ? (await resolveIdentity(row)).privateKeyPem : undefined);
+ },
async ensureAgentIdentity(companyId: string, agentId: string) {
return db.transaction(async (tx) => {
@@ -49,18 +64,8 @@ export function agentIdentityService(db: Db) {
details: { algorithm: "Ed25519", keyId },
});
}
- const privateKeyPem = await localEncryptedProvider.resolveVersion({
- material: row.privateKeyMaterial, externalRef: null,
- });
// Fail closed on corruption or a wrong master key; never replace a key.
- const privateKey = createPrivateKey(privateKeyPem);
- const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" });
- const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" });
- if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" ||
- !derived.equals(stored) || fingerprint(stored) !== row.keyId) {
- throw new Error("Agent identity public and private keys do not match");
- }
- return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem };
+ return resolveIdentity(row);
});
},
};
diff --git a/server/src/services/agent-runner-selection.test.ts b/server/src/services/agent-runner-selection.test.ts
index 8b6fe94df2a..22114120f57 100644
--- a/server/src/services/agent-runner-selection.test.ts
+++ b/server/src/services/agent-runner-selection.test.ts
@@ -3,6 +3,10 @@ import { agentHarnessType, agentRunner, paperclipRunnerProfileForHarness, paperc
const state = vi.hoisted(() => ({ disabled: [] as string[], overridden: new Set(), settings: {} as Record, getEnvironment: vi.fn(), bindings: vi.fn(), activeTemplate: vi.fn().mockResolvedValue(null), secretPaths: vi.fn().mockResolvedValue(new Set(["apiKey"])), managedEnvironment: vi.fn(), kubernetesEnvironment: vi.fn(), ensureKubernetesEnvironment: vi.fn(), ssh: vi.fn(), resolveEnvironment: vi.fn(), sandboxQualification: vi.fn(), pluginByKey: vi.fn() }));
vi.mock("../adapters/registry.js", () => ({ findActiveServerAdapter: (type: string) => ({ type }), hasActiveAdapterOverride: (type: string) => state.overridden.has(type), waitForExternalAdapters: async () => {} }));
vi.mock("./adapter-plugin-store.js", () => ({ getDisabledAdapterTypes: () => state.disabled }));
+vi.mock("./adapter-settings.js", async (original) => ({
+ ...await original(),
+ adapterSettingsService: () => ({ getDisabledTypes: async () => state.disabled }),
+}));
vi.mock("./instance-settings.js", () => ({ instanceSettingsService: () => ({ get: async () => state.settings }) }));
vi.mock("./environments.js", () => ({ environmentService: () => ({ getById: state.getEnvironment, listBoundCompanyIds: state.bindings, findManagedSandboxEnvironment: state.managedEnvironment, findKubernetesEnvironment: state.kubernetesEnvironment, ensureKubernetesEnvironment: state.ensureKubernetesEnvironment }) }));
vi.mock("./environment-custom-images.js", () => ({ environmentCustomImageService: () => ({ getActiveTemplate: state.activeTemplate }) }));
@@ -351,7 +355,7 @@ describe("server-owned Codex runner selection", () => {
state.settings = { general: { executionMode: "kubernetes" } };
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company");
expect(selected).toEqual({ driver: "sandbox" });
- expect(agentRunnerAvailability("codex_local", selected)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
+ expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company");
expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled();
expect(state.resolveEnvironment).not.toHaveBeenCalled();
@@ -477,14 +481,14 @@ describe("server-owned Codex runner selection", () => {
config: { provider: "daytona", snapshot: "replacement-snapshot" }, metadata: { managedByPaperclip: true } });
state.resolveEnvironment.mockRejectedValue(new Error("Saved provider secret is unavailable."));
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
- expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("paperclip");
+ expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("paperclip");
expect(await resolveNewAgentRunnerForCompany({} as never, "company", {
adapterType, defaultEnvironmentId: "sandbox", adapterConfig: { model: adapterType === "opencode_local" ? "provider/model" : undefined },
})).toMatchObject({ adapterType: "paperclip_runner" });
expect(state.sandboxQualification).not.toHaveBeenCalled();
expect(state.resolveEnvironment).not.toHaveBeenCalled();
state.disabled.push("paperclip_runner");
- expect(agentRunnerAvailability(adapterType, selected).defaultRunner).toBe("legacy");
+ expect(agentRunnerAvailability(adapterType, selected, state.disabled).defaultRunner).toBe("legacy");
});
it.each(["self-hosted", "custom-environment", "captured-template", "stale-marker"])("keeps qualification for a %s sandbox", async kind => {
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", kind === "self-hosted" ? undefined : JSON.stringify({ ...managedCloudConfig,
@@ -495,7 +499,7 @@ describe("server-owned Codex runner selection", () => {
metadata: { managedByPaperclip: kind !== "custom-environment" } });
state.activeTemplate.mockResolvedValue(kind === "captured-template" ? { id: "template", provider: "daytona", templateRef: "custom-snapshot" } : null);
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
- expect(agentRunnerAvailability("codex_local", selected).defaultRunner).toBe("legacy");
+ expect(agentRunnerAvailability("codex_local", selected, state.disabled).defaultRunner).toBe("legacy");
expect(state.sandboxQualification).toHaveBeenCalled();
});
it("ignores a stale captured image and preserves a matching custom image without resolving credentials", async () => {
@@ -507,17 +511,17 @@ describe("server-owned Codex runner selection", () => {
sourceEnvironmentConfigFingerprint: fingerprint({ ...config, snapshot: "previous-snapshot" }, { excludePaths: ["apiKey"] }) };
state.activeTemplate.mockResolvedValue(template);
const current = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
- expect(agentRunnerAvailability("codex_local", current).defaultRunner).toBe("paperclip");
+ expect(agentRunnerAvailability("codex_local", current, state.disabled).defaultRunner).toBe("paperclip");
expect(await resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "codex_local", runner: "paperclip", defaultEnvironmentId: "sandbox" }))
.toMatchObject({ adapterType: "paperclip_runner" });
expect(state.sandboxQualification).not.toHaveBeenCalled();
state.activeTemplate.mockResolvedValue({ ...template, sourceEnvironmentConfigFingerprint: fingerprint(config, { excludePaths: ["apiKey"] }) });
const custom = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
- expect(agentRunnerAvailability("codex_local", custom).defaultRunner).toBe("legacy");
+ expect(agentRunnerAvailability("codex_local", custom, state.disabled).defaultRunner).toBe("legacy");
expect(state.sandboxQualification).toHaveBeenCalled();
state.getEnvironment.mockResolvedValue({ id: "sandbox", driver: "sandbox", status: "active", config: { ...config, timeoutMs: "invalid" }, metadata: { managedByPaperclip: true } });
const invalid = await resolveAgentRunnerTargetForCompany({} as never, "company", "sandbox");
- expect(agentRunnerAvailability("codex_local", invalid).defaultRunner).toBe("legacy");
+ expect(agentRunnerAvailability("codex_local", invalid, state.disabled).defaultRunner).toBe("legacy");
expect(state.resolveEnvironment).not.toHaveBeenCalled();
});
it("checks company access before trusting a managed Cloud image", async () => {
diff --git a/server/src/services/agent-runner-selection.ts b/server/src/services/agent-runner-selection.ts
index 7515b40dc81..843d69b699a 100644
--- a/server/src/services/agent-runner-selection.ts
+++ b/server/src/services/agent-runner-selection.ts
@@ -5,6 +5,9 @@ import { resolveClaudeModel } from "@paperclipai/adapter-claude-local";
import { DEFAULT_GROK_LOCAL_MODEL } from "@paperclipai/adapter-grok-local";
import { findActiveServerAdapter, hasActiveAdapterOverride, waitForExternalAdapters } from "../adapters/registry.js";
import { getDisabledAdapterTypes } from "./adapter-plugin-store.js";
+import { adapterSettingsService, effectiveDisabledAdapterTypes } from "./adapter-settings.js";
+import { listServerAdapters } from "../adapters/registry.js";
+import { isCloudManagedInstance } from "./cloud-instance.js";
import { claudeRunnerUnsupportedSettings, PaperclipRunnerProviderProfileError, resolvePaperclipRunnerNativeProviderInput } from "./native-runtime/provider-profile.js";
import { forbidden, unprocessable } from "../errors.js";
import { runSshCommand } from "@paperclipai/adapter-utils/ssh";
@@ -24,6 +27,7 @@ export type RunnerTarget = {
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
export interface AgentRunnerSelection {
adapterType?: string | null;
+ disabledTypes?: string[];
adapterConfig?: Record;
runner?: AgentRunnerChoice;
target?: RunnerTarget;
@@ -32,8 +36,13 @@ export interface AgentRunnerSelection {
onConfigurationWarnings?: (warnings: AgentConfigurationWarning[]) => void;
}
-function runnerAdaptersAvailable(harness: string, driver: string): boolean {
- const disabled = new Set(getDisabledAdapterTypes());
+function localDisabledTypes(): string[] {
+ if (isCloudManagedInstance()) throw new Error("Managed runner selection requires a fresh adapter settings snapshot");
+ return effectiveDisabledAdapterTypes(getDisabledAdapterTypes(), () => listServerAdapters().map((adapter) => adapter.type));
+}
+
+function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes: string[]): boolean {
+ const disabled = new Set(disabledTypes);
return !!paperclipRunnerProfileForHarness(harness)
&& !disabled.has(harness) && !disabled.has("paperclip_runner")
&& !!findActiveServerAdapter(harness) && !!findActiveServerAdapter("paperclip_runner")
@@ -41,13 +50,13 @@ function runnerAdaptersAvailable(harness: string, driver: string): boolean {
&& isEnvironmentDriverSupportedForAdapter("paperclip_runner", driver);
}
-export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget()): AgentRunnerAvailability {
+export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability {
if (target.runnerTasks || target.managedCloudImage) {
- return runnerAdaptersAvailable(harness, target.driver)
+ return runnerAdaptersAvailable(harness, target.driver, disabledTypes)
? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" }
: { supportedRunners: ["legacy"], defaultRunner: "legacy" };
}
- const supported = runnerAdaptersAvailable(harness, target.driver)
+ const supported = runnerAdaptersAvailable(harness, target.driver, disabledTypes)
&& (target.driver !== "sandbox" || Boolean(target.platform && target.architecture
&& (target.platform !== "linux" || (target.libc === "glibc" && target.glibcVersion))))
&& (["claude_local", "grok_local"].includes(harness)
@@ -166,11 +175,12 @@ export function resolveNewAgentRunner(input: AgentRunnerSelection): { adapterTyp
function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; adapterConfig: Record } {
const adapterType = input.adapterType ?? "process";
let config = { ...input.adapterConfig };
+ const disabledTypes = input.disabledTypes ?? localDisabledTypes();
const harness = agentHarnessType(adapterType, config);
const native = adapterType === "paperclip_runner";
- if (paperclipRunnerProfileForHarness(adapterType) && getDisabledAdapterTypes().includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
+ if (paperclipRunnerProfileForHarness(adapterType) && disabledTypes.includes(adapterType)) throw unprocessable(`${adapterType === "grok_local" ? "Grok Build" : adapterType === "pi_local" ? "Pi" : adapterType === "opencode_local" ? "OpenCode" : adapterType === "claude_local" ? "Claude" : "Codex"} is disabled on this instance. Enable its adapter or choose another harness.`, { code: "agent_runner_unavailable" });
if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) };
- const availability = agentRunnerAvailability(harness, input.target);
+ const availability = agentRunnerAvailability(harness, input.target, disabledTypes);
const choice = input.runner ?? "auto";
const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip");
if (!useNative) {
@@ -318,13 +328,14 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string;
}
export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) {
+ input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() };
if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig))
|| input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input);
- const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId);
+ const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes);
return resolveNewAgentRunner({ ...input, target });
}
-export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null): Promise {
+export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise {
await waitForExternalAdapters();
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
@@ -356,7 +367,9 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
const qualified = await readSandboxPlatformQualification(db, environment, companyId);
if (qualified) target = { driver, ...qualified };
}
- if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver))) {
+ const disabledForSsh = driver === "ssh" && environment
+ ? disabledTypes ?? await adapterSettingsService(db).getDisabledTypes() : [];
+ if (driver === "ssh" && environment && ["codex_local", "claude_local", "opencode_local", "pi_local", "grok_local"].some(harness => runnerAdaptersAvailable(harness, driver, disabledForSsh))) {
const parsed = await resolveEnvironmentDriverConfigForRuntime(db, companyId, environment);
if (parsed.driver !== "ssh") throw unprocessable("The selected SSH environment is unavailable.");
let output: string;
diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts
index c03b40e3ced..d76f9606c9e 100644
--- a/server/src/services/ai-connections.ts
+++ b/server/src/services/ai-connections.ts
@@ -1105,7 +1105,7 @@ export function aiConnectionService(db: Db) {
async function refreshQuotaCredential(row: Parameters[0], failedValue: string, signal: AbortSignal) {
return withAccountHomeSecretMutationLock(undefined, row.connection.companyId, async () => {
const recovery = quotaCredentialRecovery(row.connection.companyId, row.grant.id);
- let savingReplacement = false;
+ let replacement: { value: string; secretId: string; version: number; epoch: number; base: string; grantUpdatedAt: number; versionSelector: "latest" | number } | undefined;
const save = () => db.transaction(async (tx) => {
// Do not wait on a transaction that might itself be waiting for the file
// lock. All database locks must be available before consuming OAuth tokens.
@@ -1117,11 +1117,20 @@ export function aiConnectionService(db: Db) {
if (!grant || grant.status !== "active") throw new Error("credentials_unavailable");
const ref = grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential");
if (!ref) throw new Error("credentials_unavailable");
- await tx.select({ id: companySecrets.id }).from(companySecrets).where(and(
+ const [secret] = await tx.select({ id: companySecrets.id, version: companySecrets.latestVersion, epoch: companySecrets.aiSessionEpoch }).from(companySecrets).where(and(
eq(companySecrets.id, ref.secretId), eq(companySecrets.companyId, companyId),
)).for("update", { noWait: true });
+ if (!secret) throw new Error("credentials_unavailable");
const current = await aiConnectionService(tx as unknown as Db).credential({ connection: row.connection, grant });
- signal.throwIfAborted();
+ if (replacement) {
+ // A lost commit response is reconciled from the authoritative secret.
+ // Reconnect, revocation and a competing saved generation always win.
+ if (current === replacement.value) return current;
+ if (ref.secretId !== replacement.secretId || secret.version !== replacement.version
+ || secret.epoch !== replacement.epoch || current !== replacement.base
+ || grant.updatedAt.getTime() !== replacement.grantUpdatedAt
+ || (ref.versionSelector ?? "latest") !== replacement.versionSelector) return current;
+ } else signal.throwIfAborted();
let pending = await recovery.read();
if (pending && (pending.secretId !== ref.secretId || pending.connectionId !== row.connection.id || pending.baseHash !== quotaCredentialHash(current))) {
// Reconnect may reuse this grant. Once the current authorized token
@@ -1141,10 +1150,9 @@ export function aiConnectionService(db: Db) {
or(sql`${heartbeatRuns.contextSnapshot}->'aiConnection'->>'grantId' = ${grant.id}`,
sql`${agents.runtimeConfig}->'aiConnection'->>'provider' = 'openai'`, eq(agents.adapterType, "codex_local")),
)).limit(1);
- if (active && !pending) throw new Error("provider_unavailable");
- let refreshed = pending?.value;
+ if (active && !pending && !replacement) throw new Error("provider_unavailable");
+ let refreshed = replacement?.value ?? pending?.value;
if (!refreshed) {
- await recovery.prepare();
const auth = JSON.parse(current);
const refreshToken = auth.tokens?.refresh_token;
if (typeof refreshToken !== "string" || !refreshToken) throw new Error("authentication_required");
@@ -1169,12 +1177,12 @@ export function aiConnectionService(db: Db) {
...(typeof tokens.refresh_token === "string" && tokens.refresh_token ? { refresh_token: tokens.refresh_token } : {}),
...(typeof tokens.id_token === "string" && tokens.id_token ? { id_token: tokens.id_token } : {}),
}, last_refresh: new Date().toISOString() });
- // This fsynced encrypted journal survives transaction rollback and host
- // restart. Never exchange the consumed token again after a failed save.
- await recovery.write({ companyId, grantId: grant.id, connectionId: row.connection.id,
- secretId: ref.secretId, baseHash: quotaCredentialHash(current), value: refreshed });
}
- savingReplacement = true;
+ // Only this logical refresh owns the replacement. Save retries must not
+ // consume another single-use token. Process loss before commit can require
+ // signing in again in v1; no secondary credential vault is created.
+ replacement ??= { value: refreshed, secretId: ref.secretId, version: secret.version, epoch: secret.epoch, base: current,
+ grantUpdatedAt: grant.updatedAt.getTime(), versionSelector: ref.versionSelector ?? "latest" };
// Persist the rotated token even if the quota deadline expires after the
// successful exchange; losing it would strand all subsequent executions.
await secretService(tx).rotate(ref.secretId, { value: refreshed, preserveAiSessionEpoch: true }, { userId: grant.subjectUserId });
@@ -1191,9 +1199,9 @@ export function aiConnectionService(db: Db) {
await recovery.clear().catch((error) => logger.warn({ err: error, grantId: row.grant.id }, "Saved quota credential recovery cleanup pending"));
return value;
} catch (error) {
- if (!savingReplacement || attempt >= 2) throw error;
- // Saving retries have no provider side effect. Persistent failures
- // retain the journal for the next quota poll, including after restart.
+ if (!replacement || attempt >= 2) throw error;
+ // Retry only the returned value. Required database reads still fail
+ // closed; exhaustion is an error, never a successful refresh.
}
}
});
diff --git a/server/src/services/attention.ts b/server/src/services/attention.ts
index dbb06445874..8d5dedd3b03 100644
--- a/server/src/services/attention.ts
+++ b/server/src/services/attention.ts
@@ -31,6 +31,8 @@ import { deriveProjectUrlKey } from "@paperclipai/shared";
import type {
AttentionDecisionVerb,
AttentionFeed,
+ AttentionSummary,
+ AttentionTaskIndex,
AttentionFeedQuery,
AttentionDetailImage,
AttentionItem,
@@ -163,6 +165,8 @@ type AttentionListOptions = AttentionFeedQuery & {
/** Internal-only escape hatch for callers that need one stable, unpaginated feed snapshot. */
allowUnscopedAll?: boolean;
excludeSources?: readonly AttentionSourceKind[];
+ /** Count the canonical complete set without card bodies or presentation reads. */
+ summaryOnly?: boolean;
};
type AttentionServiceOptions = {
@@ -554,7 +558,7 @@ function parseActivityBoundary(value: string | undefined, field: "activitySince"
return parsed;
}
-async function enrichAttentionItems(db: Db, companyId: string, items: AttentionItem[], now: number) {
+async function enrichAttentionItems(db: Db, companyId: string, items: AttentionItem[], now: number, includePresentation = true) {
if (items.length === 0) return items;
const sourceIds = [...new Set(items.map((item) => item.subject.id))];
const queueRows = await db
@@ -605,7 +609,7 @@ async function enrichAttentionItems(db: Db, companyId: string, items: AttentionI
...items.map(readMetadataAgentId),
...triageRows.map((row) => row.setByAgentId),
].filter((value): value is string => Boolean(value)))];
- const agentNameById = new Map(agentIds.length === 0 ? [] : await db
+ const agentNameById = new Map(!includePresentation || agentIds.length === 0 ? [] : await db
.select({ id: agents.id, name: agents.name })
.from(agents)
.where(and(eq(agents.companyId, companyId), inArray(agents.id, agentIds)))
@@ -1173,7 +1177,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
OPEN_DECISION_MAX_LIMIT,
);
- return {
+ const service = {
list: async (companyId: string, options: AttentionListOptions = {}): Promise => {
if (options.all && !options.queue && !options.allowUnscopedAll) {
throw badRequest("all requires a queue filter");
@@ -1187,6 +1191,9 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
]);
const includeDismissed = options.includeDismissed === true;
const now = serviceOptions.now?.() ?? Date.now();
+ const summaryOnly = options.summaryOnly === true;
+ const loadImages = (...args: Parameters) => summaryOnly
+ ? Promise.resolve(new Map()) : issueImageMap(...args);
const collected: AttentionItem[] = [];
const add = (item: AttentionItem) => {
@@ -1204,7 +1211,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
status: approvals.status,
requestedByAgentId: approvals.requestedByAgentId,
requestedByUserId: approvals.requestedByUserId,
- payload: approvals.payload,
+ payload: summaryOnly ? sql>`'{}'::jsonb` : approvals.payload,
createdAt: approvals.createdAt,
updatedAt: approvals.updatedAt,
})
@@ -1271,7 +1278,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
status: issueThreadInteractions.status,
title: issueThreadInteractions.title,
summary: issueThreadInteractions.summary,
- payload: issueThreadInteractions.payload,
+ payload: summaryOnly ? sql>`jsonb_build_object('target', ${issueThreadInteractions.payload}->'target', 'qualityReview', ${issueThreadInteractions.payload}->'qualityReview')` : issueThreadInteractions.payload,
addresseeAgentId: issueThreadInteractions.addresseeAgentId,
addresseeUserId: issueThreadInteractions.addresseeUserId,
createdByAgentId: issueThreadInteractions.createdByAgentId,
@@ -1318,22 +1325,23 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
// A pending human request remains actionable in a persistent chat even
// though its idle container is intentionally absent from work queues.
issueSummaryMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition, true),
- issueImageMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
- planDocumentMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
+ loadImages(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
+ summaryOnly ? Promise.resolve(new Map())
+ : planDocumentMap(db, companyId, visibleInteractionRows.map((row) => row.issueId), readCondition),
]);
for (const interaction of visibleInteractionRows) {
const issue = interactionIssueMap.get(interaction.issueId) ?? null;
if (!issue) continue;
const payload = readRecord(interaction.payload);
- const detail = interactionDetail({
+ const detail = summaryOnly ? null : interactionDetail({
kind: interaction.kind,
payload,
issue,
planDocument: interactionPlanDocumentMap.get(interaction.issueId) ?? null,
images: issueImages(interactionImageMap, interaction.issueId),
});
- const isPlanTarget = detail.kind === "plan_approval";
+ const isPlanTarget = interaction.kind === "request_confirmation" && isPlanDocumentTarget(payload);
const dedupKey = `interaction:${interaction.id}`;
add(createItem({
companyId,
@@ -1380,7 +1388,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
originAgentId: decisions.originAgentId,
ruleKey: decisions.ruleKey,
title: decisions.title,
- body: decisions.body,
+ body: summaryOnly ? sql`''` : decisions.body,
status: decisions.status,
expiresAt: decisions.expiresAt,
originIssueId: decisions.originIssueId,
@@ -1511,7 +1519,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
recoveryRows.flatMap((row) => [row.sourceIssueId, row.recoveryIssueId]),
readCondition,
),
- issueImageMap(db, companyId, recoveryRows.map((row) => row.sourceIssueId), readCondition),
+ loadImages(db, companyId, recoveryRows.map((row) => row.sourceIssueId), readCondition),
]);
for (const recovery of recoveryRows) {
@@ -1580,7 +1588,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const [blockedIssueSummaries, terminalBlockerSummaries, blockerImageMap, blockingIssues] = await Promise.all([
issueSummaryMap(db, companyId, blockedIssues.map((issue) => issue.id), readCondition),
issueSummaryMap(db, companyId, terminalBlockerIssueIds, readCondition),
- issueImageMap(
+ loadImages(
db,
companyId,
[...blockedIssues.map((issue) => issue.id), ...terminalBlockerIssueIds],
@@ -1721,7 +1729,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const [reviewAttentionByIssueId, reviewIssueMap, reviewImageMap] = await Promise.all([
issueService(db).listReviewAttention(companyId, reviewRows),
issueSummaryMap(db, companyId, reviewIssueIds, readCondition),
- issueImageMap(db, companyId, reviewIssueIds, readCondition),
+ loadImages(db, companyId, reviewIssueIds, readCondition),
]);
for (const review of reviewRows) {
@@ -1787,7 +1795,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
failedIssueIds,
readCondition,
),
- issueImageMap(db, companyId, failedIssueIds, readCondition),
+ loadImages(db, companyId, failedIssueIds, readCondition),
listFailedRunsWithNewerRun(db, companyId, failedRows.map((row, index) => ({
id: row.id,
agentId: row.agentId,
@@ -1962,7 +1970,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
const collectedItems = [...deduped.values()].sort(compareAttentionItems);
await decisionQueueService(db).materializeSeededQueues(companyId, collectedItems);
- const enrichedItems = await enrichAttentionItems(db, companyId, collectedItems, now);
+ const enrichedItems = await enrichAttentionItems(db, companyId, collectedItems, now, !summaryOnly);
const activitySince = parseActivityBoundary(options.activitySince, "activitySince");
const activityUntil = parseActivityBoundary(options.activityUntil, "activityUntil");
@@ -1980,6 +1988,15 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
return true;
});
+ const countsBySourceKind = emptyCounts();
+ for (const item of visibleItems) countsBySourceKind[item.sourceKind] += 1;
+ const summary: AttentionSummary = {
+ companyId, generatedAt: new Date().toISOString(), totalCount: visibleItems.length,
+ deskBadgeCount: visibleItems.filter(item => isNewToday(item, now) || isDecideNow(item, now)).length,
+ countsBySourceKind,
+ };
+ if (summaryOnly) return { ...summary, nextCursor: null, items: visibleItems };
+
const sort = options.sort ?? "activity";
if (sort !== "activity" && sort !== "decide") throw badRequest("sort must be 'activity' or 'decide'");
const selectedComparator = sort === "decide"
@@ -2051,22 +2068,23 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
}
}
}
- const countsBySourceKind = emptyCounts();
- for (const item of rankedItems) countsBySourceKind[item.sourceKind] += 1;
-
- return {
- companyId,
- generatedAt: new Date().toISOString(),
- totalCount: rankedItems.length,
- // Desk badge: distinct items that surfaced
- // today OR carry an explicit decide-by deadline due today/past. Counted
- // over the full ranked set (pre-pagination) so the sidebar badge stays
- // company-wide accurate even on a small first page.
- deskBadgeCount: rankedItems.filter((item) => isNewToday(item, now) || isDecideNow(item, now)).length,
- nextCursor,
- countsBySourceKind,
- items,
- };
+ return { ...summary, nextCursor, items };
+ },
+ };
+ return {
+ ...service,
+ summary: async (companyId: string, options: Pick = {}): Promise => {
+ const { items: _items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
+ ...options, all: true, allowUnscopedAll: true, summaryOnly: true,
+ });
+ return summary;
+ },
+ taskIndex: async (companyId: string, options: Pick = {}): Promise => {
+ const { items, nextCursor: _cursor, ...summary } = await service.list(companyId, {
+ ...options, all: true, allowUnscopedAll: true, summaryOnly: true,
+ });
+ return { ...summary, taskIds: [...new Set(items.flatMap(item => item.relatedIssue
+ ? [item.relatedIssue.id] : item.subject.kind === "issue" ? [item.subject.id] : []))] };
},
};
}
diff --git a/server/src/services/bundled-plugin-package.ts b/server/src/services/bundled-plugin-package.ts
new file mode 100644
index 00000000000..4d85c4f93f1
--- /dev/null
+++ b/server/src/services/bundled-plugin-package.ts
@@ -0,0 +1,108 @@
+import fs from "node:fs";
+import path from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import type { PaperclipPluginManifestV1, PluginRecord } from "@paperclipai/shared";
+import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
+import { assertDistributionManifestCapabilities } from "./distribution-plugin-catalog.js";
+
+export type BundledPluginRegistration = Pick;
+
+export interface BundledPluginPackageIdentity {
+ kind: "bundled_image";
+ catalogKey: string;
+ pluginKey: string;
+ packageName: string;
+ version: string;
+ digest?: string;
+}
+
+export type BundledPluginPackageInspection =
+ | { compatible: true; identity: BundledPluginPackageIdentity }
+ | { compatible: false; pluginKey: string; code: "incompatible_bundled_package" | "missing_manifest_evidence"; reason: string };
+
+function containedEntrypoint(root: string, declared: unknown, directory: boolean): string {
+ const relative = typeof declared === "string" ? declared.replace(/^\.\//, "").replace(/\/$/, "") : "";
+ if (!relative || path.isAbsolute(relative) || path.win32.isAbsolute(relative) || relative.includes("\\") || relative.split("/").length > 32 || relative.split("/").some((part) => !part || part === "." || part === "..")) {
+ throw new Error("Bundled plugin entrypoint must stay inside its package");
+ }
+ const target = fs.realpathSync(path.join(root, relative));
+ const resolvedRelative = path.relative(root, target);
+ if (!resolvedRelative || resolvedRelative.startsWith("..") || path.isAbsolute(resolvedRelative)) {
+ throw new Error("Bundled plugin entrypoint escapes its package");
+ }
+ const stat = fs.statSync(target);
+ if (directory ? !stat.isDirectory() : !stat.isFile()) throw new Error("Bundled plugin entrypoint is not prebuilt");
+ return relative;
+}
+
+export function createBundledPluginPackageResolver(installs: readonly ResolvedBundledPlugin[]) {
+ const byKey = new Map(installs.map((install) => [install.pluginKey, install]));
+
+ function read(install: ResolvedBundledPlugin) {
+ const root = fs.realpathSync(install.localPath);
+ if (root !== path.resolve(install.localPath)) throw new Error("Bundled plugin package root changed after catalog resolution");
+ containedEntrypoint(root, "package.json", false);
+ const metadataPath = path.join(root, "package.json");
+ if (fs.statSync(metadataPath).size > 128 * 1024) throw new Error("Bundled plugin package metadata exceeds verification limits");
+ const pkg = JSON.parse(fs.readFileSync(metadataPath, "utf8"));
+ if (typeof pkg.name !== "string" || typeof pkg.version !== "string" || !pkg.paperclipPlugin) {
+ throw new Error("Bundled plugin package identity or entrypoints are missing");
+ }
+ if (install.distribution && pkg.version !== install.distribution.version) throw new Error("Bundled plugin package version does not match the catalog");
+ const manifest = containedEntrypoint(root, pkg.paperclipPlugin.manifest, false);
+ const worker = containedEntrypoint(root, pkg.paperclipPlugin.worker, false);
+ const ui = pkg.paperclipPlugin.ui === undefined ? undefined : containedEntrypoint(root, pkg.paperclipPlugin.ui, true);
+ if (install.distribution && (worker !== install.distribution.entrypoints.worker || ui !== install.distribution.entrypoints.ui)) {
+ throw new Error("Bundled plugin entrypoints do not match the verified package");
+ }
+ return { root, pkg, manifest, worker, ui };
+ }
+
+ function assertManifest(install: ResolvedBundledPlugin, manifest: PaperclipPluginManifestV1, previousManifest?: PaperclipPluginManifestV1) {
+ const bundle = read(install);
+ if (manifest.id !== install.pluginKey || manifest.version !== bundle.pkg.version) throw new Error("Bundled plugin manifest identity/version does not match the image");
+ const worker = containedEntrypoint(bundle.root, manifest.entrypoints.worker, false);
+ const ui = manifest.entrypoints.ui === undefined ? undefined : containedEntrypoint(bundle.root, manifest.entrypoints.ui, true);
+ if (worker !== bundle.worker || ui !== bundle.ui) throw new Error("Bundled plugin manifest entrypoints do not match the image");
+ assertDistributionManifestCapabilities(manifest);
+ if (previousManifest) {
+ const approved = new Set(previousManifest.capabilities);
+ if (manifest.capabilities.some((capability) => !approved.has(capability))) throw new Error("Bundled plugin capabilities require approval");
+ }
+ return bundle;
+ }
+
+ function resolve(plugin: BundledPluginRegistration): string {
+ const install = byKey.get(plugin.pluginKey);
+ if (!install) throw new Error(`Bundled plugin is absent or not selected in this deployment: ${plugin.pluginKey}`);
+ const bundle = assertManifest(install, plugin.manifestJson);
+ if (plugin.packageName !== bundle.pkg.name || plugin.version !== bundle.pkg.version) throw new Error("Bundled plugin registration identity/version does not match the image");
+ return bundle.root;
+ }
+
+ return {
+ resolve,
+ inspect(plugin: BundledPluginRegistration, verifiedManifest?: PaperclipPluginManifestV1): BundledPluginPackageInspection {
+ try {
+ resolve(plugin);
+ const install = byKey.get(plugin.pluginKey)!;
+ if (!verifiedManifest) return { compatible: false, pluginKey: plugin.pluginKey, code: "missing_manifest_evidence", reason: "Image manifest evidence is required" };
+ assertManifest(install, verifiedManifest);
+ if (!isDeepStrictEqual(verifiedManifest, plugin.manifestJson)) throw new Error("Approved plugin manifest does not match the image");
+ return { compatible: true, identity: {
+ kind: "bundled_image", catalogKey: install.key, pluginKey: plugin.pluginKey,
+ packageName: plugin.packageName, version: plugin.version,
+ ...(install.distribution ? { digest: install.distribution.digest } : {}),
+ } };
+ } catch (error) {
+ return { compatible: false, pluginKey: plugin.pluginKey, code: "incompatible_bundled_package", reason: error instanceof Error ? error.message : String(error) };
+ }
+ },
+ assertActivation(input: { pluginKey?: string; packageRoot: string; manifest?: PaperclipPluginManifestV1; previousManifest?: PaperclipPluginManifestV1 }) {
+ const install = input.pluginKey ? byKey.get(input.pluginKey) : installs.find((candidate) => candidate.localPath === input.packageRoot);
+ if (!install || path.resolve(input.packageRoot) !== install.localPath) throw new Error("Bundled plugin is absent or not selected in this deployment");
+ read(install);
+ if (input.manifest) assertManifest(install, input.manifest, input.previousManifest);
+ },
+ };
+}
diff --git a/server/src/services/bundled-plugin-runtime.ts b/server/src/services/bundled-plugin-runtime.ts
new file mode 100644
index 00000000000..cae4b5e80ab
--- /dev/null
+++ b/server/src/services/bundled-plugin-runtime.ts
@@ -0,0 +1,37 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+export function resolveBundledPluginRuntimeLoader(
+ moduleDirectory = path.dirname(fileURLToPath(import.meta.url)),
+): string {
+ let directory = fs.realpathSync(moduleDirectory);
+ let imageRoot: string | undefined;
+ for (let depth = 0; depth < 5; depth++) {
+ const metadataPath = path.join(directory, "package.json");
+ const metadata = fs.lstatSync(metadataPath, { throwIfNoEntry: false });
+ if (metadata?.isFile() && metadata.size <= 128 * 1024
+ && JSON.parse(fs.readFileSync(metadataPath, "utf8")).name === "@paperclipai/server") {
+ imageRoot = path.dirname(directory);
+ break;
+ }
+ const parent = path.dirname(directory);
+ if (parent === directory) break;
+ directory = parent;
+ }
+ if (!imageRoot) throw new Error("Managed plugin server package is missing from the current image");
+ const declared = path.join(imageRoot, "cli/node_modules/tsx/dist/loader.mjs");
+ let loader: string;
+ try {
+ loader = fs.realpathSync(declared);
+ if (!fs.statSync(loader).isFile()) throw new Error("not a file");
+ } catch {
+ throw new Error("Managed plugin runtime loader is missing from the current image");
+ }
+ const roots = [path.join(imageRoot, "cli"), path.join(imageRoot, "node_modules")];
+ if (!roots.some((root) => {
+ const relative = path.relative(root, loader);
+ return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
+ })) throw new Error("Managed plugin runtime loader escapes the current image");
+ return loader;
+}
diff --git a/server/src/services/bundled-plugins.ts b/server/src/services/bundled-plugins.ts
index 9a728f452fc..cb59fa5cbae 100644
--- a/server/src/services/bundled-plugins.ts
+++ b/server/src/services/bundled-plugins.ts
@@ -1,6 +1,7 @@
import path from "node:path";
import fs from "node:fs";
import { isDeepStrictEqual } from "node:util";
+import { isCloudManagedInstance } from "./cloud-instance.js";
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
import { assertDistributionManifestCapabilities, readDistributionPluginCatalog, type DistributionPlugin } from "./distribution-plugin-catalog.js";
@@ -118,6 +119,13 @@ export const BUNDLED_PLUGIN_CATALOG: readonly BundledPluginCatalogEntry[] = [
*/
export const SELF_HOSTED_AUTO_INSTALL_KEYS: readonly string[] = ["kubernetes"];
+export function resolveManagedBundledPluginKeys(
+ keys: readonly string[] | null | undefined,
+ env: Record = process.env,
+): readonly string[] | null {
+ return keys ?? (isCloudManagedInstance(env) ? [] : null);
+}
+
export function resolveBundledCatalogRoot(
env: Record,
): string {
@@ -203,7 +211,7 @@ export function resolveBundledPluginInstalls(
`bundled plugin "${key}" resolves to "${localPath}", outside the bundled catalog root "${opts.catalogRoot}"; refusing to start`,
);
}
- resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath });
+ resolved.push({ key: entry.key, pluginKey: entry.pluginKey, localPath: opts.enforceCatalogRoot ? canonicalize(localPath) : localPath });
}
return resolved;
}
@@ -269,6 +277,7 @@ async function reconcileBundledPluginManifest(
deps: BundledPluginProvisionerDeps,
bundleManifestExists: (localPath: string) => boolean,
verifiedManifest?: PaperclipPluginManifestV1,
+ preservePackagePath = false,
): Promise<"upgrade_pending" | undefined> {
try {
// Managed reinstalls take the install path instead; this branch means the
@@ -283,7 +292,7 @@ async function reconcileBundledPluginManifest(
const approved = new Set(existing.manifestJson.capabilities ?? []);
requiresApproval = bundleManifest.capabilities.some((capability) => !approved.has(capability));
}
- const rebindPackage = install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
+ const rebindPackage = !preservePackagePath && install.distribution && existing.packagePath !== install.localPath && existing.status !== "uninstalled";
const refreshDistribution = install.distribution && !isDeepStrictEqual(bundleManifest, existing.manifestJson);
if (bundleManifest.version === existing.version && !rebindPackage && !requiresApproval && !refreshDistribution) return;
await deps.registry.update(existing.id, {
@@ -389,7 +398,7 @@ async function reenableErroredBundledPlugin(
export async function ensureBundledPlugins(
installs: readonly ResolvedBundledPlugin[],
deps: BundledPluginProvisionerDeps,
- opts: { reinstallUninstalled: boolean },
+ opts: { reinstallUninstalled: boolean; preservePackagePath?: boolean },
): Promise {
const bundleManifestExists = deps.bundleManifestExists ?? defaultBundleManifestExists;
for (const install of installs) {
@@ -411,7 +420,7 @@ export async function ensureBundledPlugins(
// existing install, because the auto-install below skips a present
// plugin. Distribution entries also replace a legacy/npm package path
// before loadAll resolves the worker. Configuration and status stay put.
- const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest);
+ const reconciledStatus = await reconcileBundledPluginManifest(existing, install, deps, bundleManifestExists, verifiedManifest, opts.preservePackagePath);
if (reconciledStatus === "upgrade_pending") continue;
if (existing.status === "error") {
await reenableErroredBundledPlugin(existing, install, deps);
diff --git a/server/src/services/cloud-bundled-package-digest.ts b/server/src/services/cloud-bundled-package-digest.ts
new file mode 100644
index 00000000000..40cacb3fa11
--- /dev/null
+++ b/server/src/services/cloud-bundled-package-digest.ts
@@ -0,0 +1,95 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
+
+export function cloudBundledPackageDigest(packageRoot: string, options: { catalogRoot?: string } = {}): string {
+ const root = fs.realpathSync(packageRoot);
+ const roots = [{ key: "package", directory: root }];
+ let imageNodeModules: string | undefined;
+ if (options.catalogRoot) {
+ const catalogRoot = fs.realpathSync(options.catalogRoot);
+ const relative = path.relative(catalogRoot, root);
+ if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) throw new Error("Image package is outside the selected catalog root");
+ imageNodeModules = path.resolve(catalogRoot, "../../node_modules");
+ roots.push(
+ { key: "plugin-sdk", directory: path.join(catalogRoot, "sdk") },
+ { key: "shared", directory: path.resolve(catalogRoot, "../shared") },
+ { key: "image-node-modules", directory: imageNodeModules },
+ );
+ }
+ roots.sort((left, right) => right.directory.length - left.directory.length);
+ const inventory = new Map();
+ const directories = new Set();
+ const files = new Set();
+ const buffer = Buffer.alloc(65_536);
+ let bytes = 0;
+ let entries = 0;
+
+ function logical(file: string): string {
+ for (const candidate of roots) {
+ const relative = path.relative(candidate.directory, file);
+ if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
+ if (relative.split(path.sep).length > 32) throw new Error("Image package exceeds the directory depth limit");
+ return `${candidate.key}/${relative.split(path.sep).join("/")}`;
+ }
+ }
+ throw new Error(options.catalogRoot ? "Image dependency escapes its authorized image roots" : "Image package symlink escapes its package");
+ }
+
+ function follow(target: string) {
+ logical(target);
+ if (imageNodeModules) {
+ const relative = path.relative(imageNodeModules, target).split(path.sep);
+ if ((relative.length === 1 && relative[0] === "")
+ || (relative[0] === ".pnpm" && (relative.length === 1 || (relative.length === 2 && relative[1] === "node_modules")))) {
+ throw new Error("Image dependency must select an individual package");
+ }
+ if (relative[0] === ".pnpm" && relative.length > 2 && relative[1] && relative[1] !== "node_modules") {
+ walk(path.join(imageNodeModules, ".pnpm", relative[1]));
+ return;
+ }
+ }
+ walk(target);
+ }
+
+ function walk(file: string) {
+ const name = logical(file);
+ const stat = fs.lstatSync(file);
+ if (stat.isSymbolicLink()) {
+ const target = fs.realpathSync(file);
+ inventory.set(name, [name, "link", logical(target), stat.mode & 0o777]);
+ follow(target);
+ } else if (stat.isDirectory()) {
+ const real = fs.realpathSync(file);
+ logical(real);
+ if (directories.has(real)) return;
+ directories.add(real);
+ for (const child of fs.readdirSync(real).sort()) {
+ if (++entries > 50_000) throw new Error("Image package exceeds the entry limit");
+ walk(path.join(real, child));
+ }
+ } else if (stat.isFile()) {
+ const real = fs.realpathSync(file);
+ if (files.has(real)) return;
+ files.add(real);
+ if (path.basename(file) === "package.json" && stat.size > 128 * 1024) throw new Error("Image package metadata exceeds the byte limit");
+ if (bytes + stat.size > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
+ const handle = fs.openSync(file, "r");
+ const hash = createHash("sha256");
+ try {
+ while (true) {
+ const count = fs.readSync(handle, buffer);
+ if (!count) break;
+ bytes += count;
+ if (bytes > 256 * 1024 * 1024) throw new Error("Image package exceeds the byte limit");
+ hash.update(buffer.subarray(0, count));
+ }
+ } finally { fs.closeSync(handle); }
+ inventory.set(name, [name, "file", `sha256:${hash.digest("hex")}`, stat.mode & 0o777]);
+ } else throw new Error("Image package contains a special file");
+ }
+ walk(root);
+ if (!inventory.size) throw new Error("Image package is empty");
+ return cloudConfigurationDigest([...inventory.values()].sort((left, right) => left[0] < right[0] ? -1 : left[0] > right[0] ? 1 : 0));
+}
diff --git a/server/src/services/cloud-configuration-evidence.ts b/server/src/services/cloud-configuration-evidence.ts
new file mode 100644
index 00000000000..63303fbf055
--- /dev/null
+++ b/server/src/services/cloud-configuration-evidence.ts
@@ -0,0 +1,74 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { open } from "node:fs/promises";
+import { z } from "zod";
+
+function canonical(value: unknown): unknown {
+ if (Array.isArray(value)) return value.map(canonical);
+ if (value && typeof value === "object") {
+ return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([key, child]) => [key, canonical(child)]));
+ }
+ return value;
+}
+
+export function cloudConfigurationDigest(value: unknown): string {
+ return `sha256:${createHash("sha256").update(JSON.stringify(canonical(value))).digest("hex")}`;
+}
+
+export const externalRegistrationEvidenceSchema = z.object({
+ sourceLocatorDigest: z.string(),
+ sourceDigest: z.string(),
+ state: z.enum(["missing", "parsed", "invalid", "unreadable"]),
+ adapterTypes: z.array(z.string()),
+ blocker: z.enum(["unsupported_external_adapters", "invalid_external_registration_metadata", "unreadable_external_registration_metadata"]).nullable(),
+}).strict();
+export type ExternalRegistrationEvidence = z.infer;
+
+const legacyRegistrationSchema = z.array(z.object({
+ type: z.string().regex(/^[a-zA-Z0-9_.-]{1,128}$/),
+ packageName: z.string().min(1).max(1024),
+ version: z.string().max(128).optional(),
+ localPath: z.string().max(4096).optional(),
+ disabled: z.boolean().optional(),
+}).passthrough()).max(100);
+
+export async function inspectLegacyExternalRegistrations(filePath: string): Promise {
+ const sourceLocatorDigest = cloudConfigurationDigest(filePath);
+ try {
+ const file = await open(filePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+ let raw: unknown;
+ try {
+ const stat = await file.stat();
+ if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid legacy registration file");
+ const buffer = Buffer.alloc(65_537);
+ let total = 0;
+ while (total < buffer.length) {
+ const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
+ if (!bytesRead) break;
+ total += bytesRead;
+ }
+ if (total > 65_536) throw new Error("oversized legacy registration file");
+ raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
+ } finally {
+ await file.close();
+ }
+ const records = legacyRegistrationSchema.safeParse(raw);
+ if (!records.success) {
+ return { sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state: "invalid" }), state: "invalid", adapterTypes: [], blocker: "invalid_external_registration_metadata" };
+ }
+ const metadata = records.data.map((record) => ({ type: record.type, packageName: record.packageName, version: record.version, disabled: record.disabled, hasLocalPath: record.localPath !== undefined }));
+ return {
+ sourceLocatorDigest, sourceDigest: cloudConfigurationDigest(metadata), state: "parsed",
+ adapterTypes: [...new Set(metadata.map((record) => record.type))].sort(),
+ blocker: metadata.length ? "unsupported_external_adapters" : null,
+ };
+ } catch (error) {
+ const absent = (error as NodeJS.ErrnoException).code === "ENOENT";
+ const invalid = error instanceof SyntaxError || (error instanceof Error && /legacy registration file/.test(error.message));
+ const state = absent ? "missing" : invalid ? "invalid" : "unreadable";
+ return {
+ sourceLocatorDigest, sourceDigest: cloudConfigurationDigest({ state }), state, adapterTypes: [],
+ blocker: absent ? null : invalid ? "invalid_external_registration_metadata" : "unreadable_external_registration_metadata",
+ };
+ }
+}
diff --git a/server/src/services/cloud-configuration-migration.ts b/server/src/services/cloud-configuration-migration.ts
new file mode 100644
index 00000000000..3ce38234408
--- /dev/null
+++ b/server/src/services/cloud-configuration-migration.ts
@@ -0,0 +1,193 @@
+import { instanceSettings, plugins, type Db } from "@paperclipai/db";
+import { eq, inArray, ne } from "drizzle-orm";
+import { z } from "zod";
+import { adapterSettingsService, readLegacyAdapterSettings } from "./adapter-settings.js";
+import { getAdapterPluginStorePath } from "./adapter-plugin-store.js";
+import { cloudRuntimeConfigService, readLegacyCloudRuntimeConfig } from "./cloud-runtime-config.js";
+import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
+import { cloudConfigurationDigest, externalRegistrationEvidenceSchema, inspectLegacyExternalRegistrations } from "./cloud-configuration-evidence.js";
+import { cloudPackageEvidenceSchema, inspectCloudPackages, qualificationBlockerSchema } from "./cloud-package-qualification.js";
+import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
+import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
+
+const RECEIPT_KEY = "cloud-configuration-migration:v1";
+const CONFIGURATION_KEYS = ["adapter-settings:v1", "cloud-runtime-config:v1"];
+type Transaction = Parameters[0]>[0];
+export type CloudConfigurationDb = Pick;
+const sourceSchema = z.object({
+ authority: z.enum(["database", "legacy_file", "default"]), sourceDigest: z.string(),
+ importedRevision: z.number().int().nonnegative(),
+}).strict();
+const qualificationSchema = z.object({
+ scope: z.literal("configuration_and_packages"), checkedAt: z.string(),
+ adapterRevision: z.number().int().nonnegative(), configRevision: z.number().int().nonnegative(),
+ configurationDigest: z.string(), packageEvidenceDigest: z.string(),
+ packageInspectionDurationMs: z.number().nonnegative(),
+ expectedPackageEvidenceDigest: z.string().nullable(),
+ configurationPackagesEligible: z.boolean(), samePackageOverlapEligible: z.boolean(),
+ packages: z.array(cloudPackageEvidenceSchema), blockers: z.array(qualificationBlockerSchema),
+ overlapBlockers: z.array(qualificationBlockerSchema),
+}).strict();
+const receiptSchema = z.object({
+ version: z.literal(1), revision: z.number().int().nonnegative(),
+ migration: z.object({ completedAt: z.string(), adapter: sourceSchema, config: sourceSchema, externalRegistrations: externalRegistrationEvidenceSchema }).strict(),
+ qualification: qualificationSchema.nullable(), lastPackageMismatch: qualificationSchema.nullable(),
+}).strict();
+export type CloudConfigurationReceipt = z.infer;
+
+function parseReceipt(value: unknown): CloudConfigurationReceipt {
+ const result = receiptSchema.safeParse(value);
+ if (!result.success) throw new Error("Persisted Cloud configuration migration receipt is invalid");
+ return result.data;
+}
+
+export function cloudConfigurationMigrationService(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv; now?: () => Date } = {}) {
+ const env = options.runtimeEnv ?? process.env;
+ const now = () => (options.now?.() ?? new Date()).toISOString();
+ const adapters = adapterSettingsService(db, { runtimeEnv: env });
+ const config = cloudRuntimeConfigService(db);
+
+ function assertManaged() {
+ if (!isCloudManagedInstance(env)) throw new Error("Cloud configuration qualification requires a managed instance");
+ }
+ async function readRow(handle: CloudConfigurationDb, lock = false) {
+ const query = handle.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, RECEIPT_KEY));
+ const [row] = lock ? await query.for("update") : await query;
+ return row ?? null;
+ }
+ async function currentConfiguration(handle: CloudConfigurationDb) {
+ await handle.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
+ .orderBy(instanceSettings.singletonKey).for("update");
+ const adapter = await adapters.inspectStored(handle);
+ const runtime = await config.inspectStored(handle);
+ if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
+ return { adapter, runtime };
+ }
+ async function registrationSnapshot(handle: CloudConfigurationDb, lock = false) {
+ const query = handle.select({
+ pluginKey: plugins.pluginKey, packageName: plugins.packageName, version: plugins.version,
+ manifestJson: plugins.manifestJson, status: plugins.status,
+ }).from(plugins).where(ne(plugins.status, "uninstalled")).orderBy(plugins.pluginKey);
+ return lock ? query.for("share") : query;
+ }
+ async function save(handle: CloudConfigurationDb, id: string, receipt: CloudConfigurationReceipt) {
+ const validated = parseReceipt(receipt);
+ const updated = await handle.update(instanceSettings).set({ general: validated, updatedAt: new Date() }).where(eq(instanceSettings.id, id)).returning({ id: instanceSettings.id });
+ if (updated.length !== 1) throw new Error("Cloud configuration migration receipt was not persisted");
+ return validated;
+ }
+
+ return {
+ async inspectStored(): Promise {
+ const row = await readRow(db);
+ return row ? parseReceipt(row.general) : null;
+ },
+ async migrate(): Promise {
+ assertManaged();
+ return db.transaction(async (tx) => {
+ const inserted = await tx.insert(instanceSettings).values({ singletonKey: RECEIPT_KEY, general: { version: 1 } })
+ .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
+ const row = await readRow(tx, true);
+ if (!row) throw new Error("Cloud configuration migration receipt is unavailable");
+ if (!inserted.length) {
+ const receipt = parseReceipt(row.general);
+ await currentConfiguration(tx);
+ return receipt;
+ }
+ await tx.select({ id: instanceSettings.id }).from(instanceSettings).where(inArray(instanceSettings.singletonKey, CONFIGURATION_KEYS))
+ .orderBy(instanceSettings.singletonKey).for("update");
+ const storedAdapter = await adapters.inspectStored(tx);
+ const storedConfig = await config.inspectStored(tx);
+ const adapterCandidate = storedAdapter ? null : await readLegacyAdapterSettings(env);
+ const configCandidate = storedConfig ? null : await readLegacyCloudRuntimeConfig(env);
+ const adapterSeed = storedAdapter ? null : await adapters.seedIfAbsent({ disabledTypes: adapterCandidate?.disabledTypes ?? [] }, tx);
+ const configSeed = storedConfig ? null : await config.seedIfAbsent(configCandidate, tx);
+ const adapter = storedAdapter ?? adapterSeed!.current;
+ const runtime = storedConfig ?? configSeed!.current;
+ const adapterFromDatabase = Boolean(storedAdapter || !adapterSeed?.seeded);
+ const configFromDatabase = Boolean(storedConfig || !configSeed?.seeded);
+ const receipt: CloudConfigurationReceipt = {
+ version: 1, revision: 0,
+ migration: {
+ completedAt: now(),
+ adapter: { authority: adapterFromDatabase ? "database" : adapterCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(adapterFromDatabase ? { disabledTypes: adapter.disabledTypes } : adapterCandidate), importedRevision: adapter.revision },
+ config: { authority: configFromDatabase ? "database" : configCandidate ? "legacy_file" : "default", sourceDigest: cloudConfigurationDigest(configFromDatabase ? runtime.config : configCandidate), importedRevision: runtime.revision },
+ externalRegistrations: await inspectLegacyExternalRegistrations(getAdapterPluginStorePath(env.PAPERCLIP_HOME)),
+ },
+ qualification: null, lastPackageMismatch: null,
+ };
+ return save(tx, row.id, receipt);
+ });
+ },
+ async recheckLegacyExternalRegistrations(expectedRevision: number): Promise {
+ assertManaged();
+ return db.transaction(async (tx) => {
+ const row = await readRow(tx, true);
+ if (!row) throw new Error("Cloud configuration migration has not completed");
+ const receipt = parseReceipt(row.general);
+ if (receipt.revision !== expectedRevision) throw new Error("Cloud configuration migration receipt revision conflict");
+ const filePath = getAdapterPluginStorePath(env.PAPERCLIP_HOME);
+ if (cloudConfigurationDigest(filePath) !== receipt.migration.externalRegistrations.sourceLocatorDigest) {
+ throw new Error("External registration recheck requires the original legacy source location");
+ }
+ const evidence = await inspectLegacyExternalRegistrations(filePath);
+ if (evidence.state !== "parsed") throw new Error("External registration recheck requires a present, valid registration file");
+ receipt.migration.externalRegistrations = evidence;
+ receipt.revision += 1;
+ receipt.qualification = null;
+ return save(tx, row.id, receipt);
+ });
+ },
+ async qualify(input: {
+ installs: readonly ResolvedBundledPlugin[];
+ catalogRoot?: string;
+ loadManifest: (packageRoot: string) => Promise;
+ expectedPackageEvidenceDigest?: string;
+ imageCatalogUnavailable?: boolean;
+ }) {
+ assertManaged();
+ const adapter = await adapters.inspectStored();
+ const runtime = await config.inspectStored();
+ if (!adapter || !runtime) throw new Error("Cloud configuration authority is incomplete");
+ const configurationBefore = cloudConfigurationDigest({ adapter, runtime });
+ const registrationsBefore = await registrationSnapshot(db);
+ const evidence = await inspectCloudPackages({ ...input, registrations: registrationsBefore });
+ return db.transaction(async (tx) => {
+ const row = await readRow(tx, true);
+ if (!row) throw new Error("Cloud configuration migration has not completed");
+ const receipt = parseReceipt(row.general);
+ const current = await currentConfiguration(tx);
+ const registrations = await registrationSnapshot(tx, true);
+ const blockers = [...evidence.blockers];
+ if (configurationBefore !== cloudConfigurationDigest(current) || cloudConfigurationDigest(registrationsBefore) !== cloudConfigurationDigest(registrations)) {
+ blockers.push({ code: "configuration_changed_during_inspection", reason: "Configuration or plugin registration changed during image inspection; run qualification again" });
+ }
+ if (input.imageCatalogUnavailable) blockers.push({ code: "current_image_catalog_unavailable", reason: "The current image package catalog could not be verified" });
+ const external = receipt.migration.externalRegistrations;
+ if (external.blocker) blockers.push({ code: external.blocker, reason: "Legacy external adapter registration metadata requires an explicit operator resolution" });
+ const overlapBlockers = input.expectedPackageEvidenceDigest && input.expectedPackageEvidenceDigest !== evidence.packageEvidenceDigest
+ ? [{ code: "package_evidence_changed", reason: "Current packages do not match the expected package evidence digest" }] : [];
+ const qualification: z.infer = {
+ scope: "configuration_and_packages", checkedAt: now(),
+ adapterRevision: current.adapter.revision, configRevision: current.runtime.revision,
+ configurationDigest: cloudConfigurationDigest(current), packageEvidenceDigest: evidence.packageEvidenceDigest,
+ packageInspectionDurationMs: evidence.packageInspectionDurationMs,
+ expectedPackageEvidenceDigest: input.expectedPackageEvidenceDigest ?? null,
+ configurationPackagesEligible: blockers.length === 0,
+ samePackageOverlapEligible: Boolean(input.expectedPackageEvidenceDigest && blockers.length === 0 && overlapBlockers.length === 0),
+ packages: evidence.packages, blockers, overlapBlockers,
+ };
+ receipt.revision += 1;
+ receipt.qualification = qualification;
+ if (evidence.blockers.length || input.imageCatalogUnavailable || overlapBlockers.length) receipt.lastPackageMismatch = qualification;
+ await save(tx, row.id, receipt);
+ return { receiptRevision: receipt.revision, ...qualification };
+ });
+ },
+ };
+}
+
+export async function migrateCloudConfiguration(db: CloudConfigurationDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
+ if (!isCloudManagedInstance(options.runtimeEnv ?? process.env)) return null;
+ return cloudConfigurationMigrationService(db, options).migrate();
+}
diff --git a/server/src/services/cloud-configuration-qualification.ts b/server/src/services/cloud-configuration-qualification.ts
new file mode 100644
index 00000000000..c96c97ae8e5
--- /dev/null
+++ b/server/src/services/cloud-configuration-qualification.ts
@@ -0,0 +1,33 @@
+import type { Db } from "@paperclipai/db";
+import { BUNDLED_PLUGIN_CATALOG, resolveBundledCatalogRoot, resolveBundledPluginInstalls, resolveManagedBundledPluginKeys, type ResolvedBundledPlugin } from "./bundled-plugins.js";
+import { readDistributionPluginCatalog } from "./distribution-plugin-catalog.js";
+import { createBundledPluginPackageResolver } from "./bundled-plugin-package.js";
+import { cloudConfigurationMigrationService } from "./cloud-configuration-migration.js";
+import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
+import { getManagedInstanceConfig } from "./managed-config.js";
+import { pluginLoader } from "./plugin-loader.js";
+
+export async function qualifyCloudConfigurationForCurrentImage(db: Db, options: {
+ runtimeEnv?: CloudInstanceEnv;
+ expectedPackageEvidenceDigest?: string;
+} = {}) {
+ const env = options.runtimeEnv ?? process.env;
+ if (!isCloudManagedInstance(env)) return null;
+ const service = cloudConfigurationMigrationService(db, { runtimeEnv: env });
+ const catalogRoot = resolveBundledCatalogRoot(env);
+ let installs: ResolvedBundledPlugin[];
+ try {
+ const distribution = readDistributionPluginCatalog(catalogRoot, BUNDLED_PLUGIN_CATALOG);
+ installs = resolveBundledPluginInstalls(resolveManagedBundledPluginKeys(getManagedInstanceConfig(env)?.plugins.autoInstall, env) ?? [], {
+ catalogRoot, env, enforceCatalogRoot: true, distributionPlugins: distribution,
+ });
+ } catch {
+ return service.qualify({ installs: [], loadManifest: async () => null, imageCatalogUnavailable: true, expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
+ }
+ const resolver = createBundledPluginPackageResolver(installs);
+ const loader = pluginLoader(db, {
+ prebuiltOnly: true, enableLocalFilesystem: false, enableNpmDiscovery: false,
+ assertPackageActivation: resolver.assertActivation, resolvePackageRoot: resolver.resolve,
+ });
+ return service.qualify({ installs, catalogRoot, loadManifest: (root) => loader.loadManifest(root), expectedPackageEvidenceDigest: options.expectedPackageEvidenceDigest });
+}
diff --git a/server/src/services/cloud-package-qualification.ts b/server/src/services/cloud-package-qualification.ts
new file mode 100644
index 00000000000..d0d75f30cf7
--- /dev/null
+++ b/server/src/services/cloud-package-qualification.ts
@@ -0,0 +1,72 @@
+import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
+import { z } from "zod";
+import { createBundledPluginPackageResolver, type BundledPluginRegistration } from "./bundled-plugin-package.js";
+import type { ResolvedBundledPlugin } from "./bundled-plugins.js";
+import { distributionBundleDigest } from "./distribution-plugin-catalog.js";
+import { cloudConfigurationDigest } from "./cloud-configuration-evidence.js";
+import { cloudBundledPackageDigest } from "./cloud-bundled-package-digest.js";
+
+export const qualificationBlockerSchema = z.object({
+ code: z.string(), pluginKey: z.string().optional(), reason: z.string(),
+ registeredVersion: z.string().optional(), imageVersion: z.string().optional(),
+}).strict();
+export type QualificationBlocker = z.infer;
+export const cloudPackageEvidenceSchema = z.object({
+ catalogKey: z.string(), pluginKey: z.string(), packageDigest: z.string().nullable(),
+ imageManifestDigest: z.string().nullable(), approvedManifestDigest: z.string().nullable(),
+}).strict();
+export type CloudPackageEvidence = z.infer;
+
+export async function inspectCloudPackages(input: {
+ installs: readonly ResolvedBundledPlugin[];
+ registrations: readonly BundledPluginRegistration[];
+ catalogRoot?: string;
+ loadManifest: (packageRoot: string) => Promise;
+}) {
+ const started = performance.now();
+ const resolver = createBundledPluginPackageResolver(input.installs);
+ const registrations = new Map(input.registrations.map((record) => [record.pluginKey, record]));
+ const selected = new Set(input.installs.map((install) => install.pluginKey));
+ const blockers: QualificationBlocker[] = input.registrations.filter((record) => !selected.has(record.pluginKey)).map((record) => ({
+ code: "unsupported_plugin_registration", pluginKey: record.pluginKey, reason: "An installed plugin is not selected from the current image", registeredVersion: record.version,
+ }));
+ const packages: CloudPackageEvidence[] = [];
+ for (const install of [...input.installs].sort((left, right) => left.pluginKey < right.pluginKey ? -1 : left.pluginKey > right.pluginKey ? 1 : 0)) {
+ const registration = registrations.get(install.pluginKey);
+ const evidence: CloudPackageEvidence = {
+ catalogKey: install.key, pluginKey: install.pluginKey, packageDigest: null,
+ imageManifestDigest: null, approvedManifestDigest: registration ? cloudConfigurationDigest(registration.manifestJson) : null,
+ };
+ packages.push(evidence);
+ let stage = "missing_bundled_package";
+ try {
+ resolver.assertActivation({ packageRoot: install.localPath });
+ stage = "unverified_package_integrity";
+ evidence.packageDigest = install.distribution ? distributionBundleDigest(install.localPath) : cloudBundledPackageDigest(install.localPath, { catalogRoot: input.catalogRoot });
+ if (install.distribution && evidence.packageDigest !== install.distribution.digest) {
+ blockers.push({ code: "package_integrity_mismatch", pluginKey: install.pluginKey, reason: "Current package bytes do not match the verified distribution digest" });
+ continue;
+ }
+ stage = "missing_manifest_evidence";
+ const manifest = await input.loadManifest(install.localPath);
+ if (!manifest) {
+ blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image has no verified plugin manifest" });
+ continue;
+ }
+ evidence.imageManifestDigest = cloudConfigurationDigest(manifest);
+ if (!registration) {
+ blockers.push({ code: "missing_plugin_registration", pluginKey: install.pluginKey, reason: "The current image package is not registered in the database", imageVersion: manifest.version });
+ continue;
+ }
+ const inspected = resolver.inspect(registration, manifest);
+ if (!inspected.compatible) blockers.push({
+ code: inspected.code, pluginKey: install.pluginKey, reason: inspected.reason,
+ registeredVersion: registration.version, imageVersion: manifest.version,
+ });
+ } catch {
+ blockers.push({ code: stage, pluginKey: install.pluginKey, reason: "The current image package could not be verified" });
+ }
+ }
+ const packageEvidenceDigest = cloudConfigurationDigest(packages.map(({ approvedManifestDigest: _approved, ...evidence }) => evidence));
+ return { packages, blockers, packageEvidenceDigest, packageInspectionDurationMs: Math.round(performance.now() - started) };
+}
diff --git a/server/src/services/cloud-runtime-config-snapshot.ts b/server/src/services/cloud-runtime-config-snapshot.ts
new file mode 100644
index 00000000000..e97cd534b24
--- /dev/null
+++ b/server/src/services/cloud-runtime-config-snapshot.ts
@@ -0,0 +1,129 @@
+import {
+ authConfigSchema,
+ databaseBackupConfigSchema,
+ llmConfigSchema,
+ loggingConfigSchema,
+ paperclipConfigSchema,
+ secretsConfigSchema,
+ secretsLocalEncryptedConfigSchema,
+ serverConfigSchema,
+ storageConfigSchema,
+ storageLocalDiskConfigSchema,
+ storageS3ConfigSchema,
+ type PaperclipConfig,
+} from "@paperclipai/shared";
+import { z } from "zod";
+
+const publicUrl = z.string().url().refine((value) => {
+ const url = new URL(value);
+ return !url.username && !url.password && !url.search && !url.hash;
+});
+
+export const cloudRuntimeConfigSchema = z.object({
+ server: serverConfigSchema.strict(),
+ auth: authConfigSchema.extend({ publicBaseUrl: publicUrl.optional() }).strict(),
+ backup: databaseBackupConfigSchema.strict(),
+ storage: storageConfigSchema.extend({
+ localDisk: storageLocalDiskConfigSchema.strict(),
+ s3: storageS3ConfigSchema.extend({ endpoint: publicUrl.optional() }).strict(),
+ }).strict(),
+ secrets: secretsConfigSchema.extend({
+ localEncrypted: secretsLocalEncryptedConfigSchema.strict(),
+ }).strict(),
+ telemetry: z.object({ enabled: z.boolean() }).strict(),
+ logging: loggingConfigSchema.strict(),
+ updates: z.object({ checkEnabled: z.boolean() }).strict().optional(),
+ llm: llmConfigSchema.omit({ apiKey: true }).strict().optional(),
+}).strict();
+
+export type CloudRuntimeConfig = z.infer;
+export type StoredCloudRuntimeConfig = { version: 1; revision: number; config: CloudRuntimeConfig | null };
+
+const storedSchema = z.object({
+ version: z.literal(1),
+ revision: z.number().int().nonnegative(),
+ config: cloudRuntimeConfigSchema.nullable(),
+}).strict();
+
+export function parseStoredCloudRuntimeConfig(value: unknown): StoredCloudRuntimeConfig {
+ const parsed = storedSchema.safeParse(value);
+ if (!parsed.success || Buffer.byteLength(JSON.stringify(parsed.data), "utf8") > 65_536) {
+ throw new Error("Persisted Cloud runtime configuration is invalid");
+ }
+ const full = parsed.data.config === null ? { success: true } : paperclipConfigSchema.safeParse(toPaperclipConfig(parsed.data.config));
+ if (!full.success) throw new Error("Persisted Cloud runtime configuration is invalid");
+ return parsed.data;
+}
+
+export function toPaperclipConfig(config: CloudRuntimeConfig | null): PaperclipConfig | null {
+ if (config === null) return null;
+ const { backup, ...settings } = structuredClone(config);
+ return {
+ $meta: { version: 1, updatedAt: "1970-01-01T00:00:00.000Z", source: "configure" },
+ database: {
+ mode: "postgres",
+ embeddedPostgresDataDir: "",
+ embeddedPostgresPort: 54329,
+ backup,
+ },
+ ...settings,
+ };
+}
+
+export function projectCloudRuntimeConfig(source: PaperclipConfig): CloudRuntimeConfig {
+ const raw = {
+ server: {
+ deploymentMode: source.server.deploymentMode,
+ exposure: source.server.exposure,
+ bind: source.server.bind,
+ customBindHost: source.server.customBindHost,
+ host: source.server.host,
+ port: source.server.port,
+ allowedHostnames: source.server.allowedHostnames,
+ serveUi: source.server.serveUi,
+ },
+ auth: {
+ baseUrlMode: source.auth.baseUrlMode,
+ publicBaseUrl: source.auth.publicBaseUrl,
+ disableSignUp: source.auth.disableSignUp,
+ },
+ backup: {
+ enabled: source.database.backup.enabled,
+ intervalMinutes: source.database.backup.intervalMinutes,
+ retentionDays: source.database.backup.retentionDays,
+ dir: source.database.backup.dir,
+ },
+ storage: {
+ provider: source.storage.provider,
+ localDisk: { baseDir: source.storage.localDisk.baseDir },
+ s3: {
+ bucket: source.storage.s3.bucket,
+ region: source.storage.s3.region,
+ endpoint: source.storage.s3.endpoint,
+ prefix: source.storage.s3.prefix,
+ forcePathStyle: source.storage.s3.forcePathStyle,
+ },
+ },
+ secrets: {
+ provider: source.secrets.provider,
+ strictMode: source.secrets.strictMode,
+ localEncrypted: { keyFilePath: source.secrets.localEncrypted.keyFilePath },
+ },
+ telemetry: { enabled: source.telemetry.enabled },
+ logging: { mode: source.logging.mode, logDir: source.logging.logDir },
+ updates: source.updates ? { checkEnabled: source.updates.checkEnabled } : undefined,
+ llm: source.llm ? { provider: source.llm.provider } : undefined,
+ };
+ return parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config: raw }).config!;
+}
+
+let snapshot: StoredCloudRuntimeConfig | null = null;
+
+export function setCloudRuntimeConfigSnapshot(value: StoredCloudRuntimeConfig | null): void {
+ snapshot = value ? parseStoredCloudRuntimeConfig(value) : null;
+}
+
+export function getCloudRuntimeConfigSnapshot(): StoredCloudRuntimeConfig {
+ if (!snapshot) throw new Error("Cloud runtime configuration has not been initialized");
+ return structuredClone(snapshot);
+}
diff --git a/server/src/services/cloud-runtime-config.ts b/server/src/services/cloud-runtime-config.ts
new file mode 100644
index 00000000000..61082ae07c7
--- /dev/null
+++ b/server/src/services/cloud-runtime-config.ts
@@ -0,0 +1,105 @@
+import { constants } from "node:fs";
+import { open } from "node:fs/promises";
+import path from "node:path";
+import { eq } from "drizzle-orm";
+import { instanceSettings, type Db } from "@paperclipai/db";
+import { paperclipConfigSchema } from "@paperclipai/shared";
+import { resolvePaperclipConfigPathForInstance } from "@paperclipai/shared/home-paths";
+import { isCloudManagedInstance, type CloudInstanceEnv } from "./cloud-instance.js";
+import {
+ parseStoredCloudRuntimeConfig,
+ projectCloudRuntimeConfig,
+ setCloudRuntimeConfigSnapshot,
+ type CloudRuntimeConfig,
+ type StoredCloudRuntimeConfig,
+} from "./cloud-runtime-config-snapshot.js";
+
+export const CLOUD_RUNTIME_CONFIG_KEY = "cloud-runtime-config:v1";
+type Transaction = Parameters[0]>[0];
+export type CloudRuntimeConfigDb = Pick;
+
+export function cloudRuntimeConfigService(db: CloudRuntimeConfigDb) {
+ async function inspectStored(handle: CloudRuntimeConfigDb = db): Promise {
+ const [row] = await handle.select({ general: instanceSettings.general }).from(instanceSettings)
+ .where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).limit(1);
+ return row ? parseStoredCloudRuntimeConfig(row.general) : null;
+ }
+
+ async function seedIfAbsent(config: CloudRuntimeConfig | null, handle: CloudRuntimeConfigDb = db) {
+ const next = parseStoredCloudRuntimeConfig({ version: 1, revision: 0, config });
+ const inserted = await handle.insert(instanceSettings)
+ .values({ singletonKey: CLOUD_RUNTIME_CONFIG_KEY, general: next })
+ .onConflictDoNothing({ target: instanceSettings.singletonKey }).returning({ id: instanceSettings.id });
+ const current = await inspectStored(handle);
+ if (!current) throw new Error("Cloud runtime configuration seed was not persisted");
+ return { seeded: inserted.length > 0, current };
+ }
+
+ return {
+ inspectStored,
+ seedIfAbsent,
+ async replace(config: CloudRuntimeConfig | null, expectedRevision: number, handle: CloudRuntimeConfigDb = db) {
+ const validated = parseStoredCloudRuntimeConfig({ version: 1, revision: expectedRevision, config });
+ return handle.transaction(async (tx) => {
+ const [row] = await tx.select().from(instanceSettings)
+ .where(eq(instanceSettings.singletonKey, CLOUD_RUNTIME_CONFIG_KEY)).for("update");
+ if (!row) throw new Error("Cloud runtime configuration has not been initialized");
+ const current = parseStoredCloudRuntimeConfig(row.general);
+ if (current.revision !== expectedRevision) throw new Error("Cloud runtime configuration revision conflict");
+ const next = { ...validated, revision: current.revision + 1 };
+ await tx.update(instanceSettings).set({ general: next, updatedAt: new Date() }).where(eq(instanceSettings.id, row.id));
+ return next;
+ });
+ },
+ };
+}
+
+export function legacyCloudRuntimeConfigPath(env: CloudInstanceEnv = process.env): string {
+ return env.PAPERCLIP_CONFIG
+ ? path.resolve(env.PAPERCLIP_CONFIG)
+ : resolvePaperclipConfigPathForInstance({ homeDir: env.PAPERCLIP_HOME, instanceId: env.PAPERCLIP_INSTANCE_ID });
+}
+
+export async function readLegacyCloudRuntimeConfig(env: CloudInstanceEnv = process.env): Promise {
+ let raw: unknown;
+ try {
+ const file = await open(legacyCloudRuntimeConfigPath(env), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+ try {
+ const stat = await file.stat();
+ if (!stat.isFile() || stat.size > 65_536) throw new Error("invalid file");
+ const buffer = Buffer.alloc(65_537);
+ let total = 0;
+ while (total < buffer.length) {
+ const { bytesRead } = await file.read(buffer, total, buffer.length - total, total);
+ if (bytesRead === 0) break;
+ total += bytesRead;
+ }
+ if (total > 65_536) throw new Error("oversized file");
+ raw = JSON.parse(buffer.subarray(0, total).toString("utf8"));
+ } finally {
+ await file.close();
+ }
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw new Error("Legacy Cloud runtime configuration could not be read safely");
+ return null;
+ }
+ const parsed = paperclipConfigSchema.safeParse(raw);
+ if (!parsed.success) throw new Error("Legacy Cloud runtime configuration is invalid");
+ if (parsed.data.llm?.provider === "openai" && parsed.data.llm.apiKey?.trim() && !env.OPENAI_API_KEY?.trim()) {
+ throw new Error("Move the legacy OpenAI API key to OPENAI_API_KEY before initializing Cloud runtime configuration");
+ }
+ return projectCloudRuntimeConfig(parsed.data);
+}
+
+export async function initializeCloudRuntimeConfig(db: CloudRuntimeConfigDb, options: { runtimeEnv?: CloudInstanceEnv } = {}) {
+ const env = options.runtimeEnv ?? process.env;
+ if (!isCloudManagedInstance(env)) return null;
+ setCloudRuntimeConfigSnapshot(null);
+ const service = cloudRuntimeConfigService(db);
+ const current = await service.inspectStored();
+ const result = current
+ ? { seeded: false, current }
+ : await service.seedIfAbsent(await readLegacyCloudRuntimeConfig(env));
+ setCloudRuntimeConfigSnapshot(result.current);
+ return result;
+}
diff --git a/server/src/services/customer-success-inspection.ts b/server/src/services/customer-success-inspection.ts
index c4a5504ed97..a73d1673dcd 100644
--- a/server/src/services/customer-success-inspection.ts
+++ b/server/src/services/customer-success-inspection.ts
@@ -20,7 +20,7 @@ import {
import { deriveBundleState } from "./agent-instructions.js";
import { readInstructionBytes } from "./agent-instruction-files.js";
import { skillPackageStore } from "./skill-package-store.js";
-import { getRunLogStore } from "./run-log-store.js";
+import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
// This is a reviewed catalog of readers, not a proxy for existing GET routes.
// Select credentials out of queries; never load encrypted identity records.
@@ -311,9 +311,9 @@ export async function readCustomerSuccessResource(
}
if (q.operation === "runs.log") {
const run = await ownRow(tx, schema.heartbeatRuns, q.companyId!, q.resourceId!);
- if (!run.logRef || run.logStore !== "local_file") return unavailable("run_log_unavailable");
+ if (!run.logRef || !isRunLogStoreType(run.logStore)) return unavailable("run_log_unavailable");
const result = await getRunLogStore().read(
- { store: "local_file", logRef: String(run.logRef) },
+ { store: run.logStore, logRef: String(run.logRef) },
{
offset: q.range?.start ?? q.offset ?? 0,
limitBytes: q.range ? q.range.end - q.range.start + 1 : 256000,
diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts
index 67cfa6173df..9195f1fcb12 100644
--- a/server/src/services/delivery-work-notifications.ts
+++ b/server/src/services/delivery-work-notifications.ts
@@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
+ runEvidenceMigration: "run-evidence-migration",
+ providerTraceRetention: "provider-trace-retention",
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
skillVersionRetention: "skill-version-retention",
databaseBackups: "database-backups",
diff --git a/server/src/services/execution-preparation-timing.test.ts b/server/src/services/execution-preparation-timing.test.ts
index c267d1ee68d..7b1703ae7a9 100644
--- a/server/src/services/execution-preparation-timing.test.ts
+++ b/server/src/services/execution-preparation-timing.test.ts
@@ -1,6 +1,6 @@
import { getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
-import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming } from "./execution-preparation-timing.js";
+import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming, withRunResponsePresentationTiming } from "./execution-preparation-timing.js";
import { ROOT_CONTEXT, trace, SpanStatusCode, type Context } from "@opentelemetry/api";
import { createHash } from "node:crypto";
@@ -45,6 +45,7 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async
try {
await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);
await expect(withRunWorkspaceFinalizationTiming("run", () => 42)).resolves.toBe(42);
+ await expect(withRunResponsePresentationTiming("run", () => 42)).resolves.toBe(42);
for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation",
"skills.inventory", "skills.inventory.wait", "skills.cache.acquire", "skills.version.materialize", "skills.consume",
"workspace_sync.seed_read_copy", "workspace_sync.seed_read_hash", "workspace_sync.seed_read_touch"] as const) {
@@ -69,7 +70,10 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async
}
});
-it("parents concurrent deferred copyback phases to their own finalization roots and preserves failure", async () => {
+it.each([
+ [withRunWorkspaceFinalizationTiming, "task.workspace_finalization", "workspace_sync.restore"],
+ [withRunResponsePresentationTiming, "task.response_presentation", "response.transaction"],
+] as const)("parents concurrent deferred phases to their own %s root and preserves failure", async (withTiming, rootName, phase) => {
const spans: Array<{ name: string; attributes: Record; context: Context;
span: { end: ReturnType; setStatus: ReturnType } }> = [];
handles.startActiveSpan.mockImplementation((name, options, context, callback) => {
@@ -77,16 +81,16 @@ it("parents concurrent deferred copyback phases to their own finalization roots
spans.push({ name, attributes: options.attributes, context, span });
return callback(span);
});
- await Promise.all(["first", "second"].map(runId => withRunWorkspaceFinalizationTiming(runId, async () => {
+ await Promise.all(["first", "second"].map(runId => withTiming(runId, async () => {
await Promise.resolve();
- await measureRunPhase("workspace_sync.restore", async () => {});
+ await measureRunPhase(phase, async () => {});
})));
for (const runId of ["first", "second"]) {
const hash = createHash("sha256").update(runId).digest("hex").slice(0, 12);
const owned = spans.filter(span => span.attributes["paperclip.task.run.run_id"] === hash);
expect(owned).toHaveLength(2);
- const root = owned.find(span => span.name === "task.workspace_finalization")!;
- const child = owned.find(span => span.name === "workspace_sync.restore")!;
+ const root = owned.find(span => span.name === rootName)!;
+ const child = owned.find(span => span.name === phase)!;
expect(root.context).toBe(ROOT_CONTEXT);
expect(trace.getSpan(child.context)).toBe(root.span);
expect(root.span.end).toHaveBeenCalledOnce();
@@ -94,7 +98,7 @@ it("parents concurrent deferred copyback phases to their own finalization roots
}
const original = new Error("private finalization failure");
const work = vi.fn(() => { throw original; });
- await expect(withRunWorkspaceFinalizationTiming("failed", work)).rejects.toBe(original);
+ await expect(withTiming("failed", work)).rejects.toBe(original);
expect(work).toHaveBeenCalledOnce();
expect(spans.at(-1)!.span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
expect(spans.at(-1)!.span.end).toHaveBeenCalledOnce();
diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts
index 9041b76f8f3..9ee7f8a189e 100644
--- a/server/src/services/execution-preparation-timing.ts
+++ b/server/src/services/execution-preparation-timing.ts
@@ -199,6 +199,14 @@ export function withRunWorkspaceFinalizationTiming(runId: string, work: () =>
return measure(runId, "task.workspace_finalization", work, true);
}
+export function withRunResponsePresentationTiming(runId: string, work: () => T | Promise): Promise {
+ const owned = ownership.getStore();
+ if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) {
+ return (async () => work())();
+ }
+ return measure(runId, "task.response_presentation", work, true);
+}
+
export async function measureRunPhase(phase: ExecutionPhase, work: () => T | Promise, counts?: { fileCount?: number; bytes?: number }): Promise {
const owned = ownership.getStore();
if (!endpointEnabled() || !owned || owned.ended) return work();
diff --git a/server/src/services/feedback.ts b/server/src/services/feedback.ts
index 1f9ceb78721..e3e4d70b766 100644
--- a/server/src/services/feedback.ts
+++ b/server/src/services/feedback.ts
@@ -47,7 +47,8 @@ import {
sanitizeFeedbackValue,
sha256Digest,
} from "./feedback-redaction.js";
-import { getRunLogStore } from "./run-log-store.js";
+import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
+import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js";
import { getOperatorSettingDefaults } from "./setting-defaults.js";
const FEEDBACK_SCHEMA_VERSION = "paperclip-feedback-envelope-v2";
@@ -318,17 +319,18 @@ async function findMatchingFile(
return search(rootDir, 0);
}
-async function readFullRunLog(run: {
+async function readFullRunLog(db: Db, run: {
logStore: string | null;
logRef: string | null;
}) {
- if (run.logStore !== "local_file" || !run.logRef) return null;
+ if (!isRunLogStoreType(run.logStore) || !run.logRef) return null;
const store = getRunLogStore();
+ if (run.logStore === "s3_segments") await recoverSettledNativeRunLog(db, store, run.logRef).catch(() => undefined);
let offset = 0;
let combined = "";
while (true) {
- const result = await store.read({ store: "local_file", logRef: run.logRef }, {
+ const result = await store.read({ store: run.logStore, logRef: run.logRef }, {
offset,
limitBytes: 512_000,
}).catch(() => null);
@@ -1526,7 +1528,7 @@ async function buildFeedbackTraceBundleFromRow(
.from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, run.id))
.orderBy(asc(heartbeatRunEvents.seq));
- const logText = await readFullRunLog(run);
+ const logText = await readFullRunLog(db, run);
const logEntries = parseRunLogEntries(logText);
const stdoutText = logEntries
.filter((entry) => entry.stream === "stdout")
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 984186fe408..4f69300ef43 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -26,7 +26,10 @@ import { createHeartbeatPreparationTiming } from "./heartbeat/preparation-timing
import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkingCopy } from "./heartbeat/instruction-working-copy.js";
import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js";
import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js";
+import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js";
import { createHeartbeatRunOutput } from "./heartbeat/run-output.js";
+import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.js";
+import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability";
import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js";
import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
@@ -371,7 +374,7 @@ import {
type NativeSessionBackend,
} from "../vendor/paperclip-runner/index.js";
-import { getRunLogStore } from "./run-log-store.js";
+import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
import {
providerTraceStore,
} from "./provider-trace-store.js";
@@ -2090,6 +2093,7 @@ export function heartbeatService(
}
try {
providerTraceCapture = await traceStore.prepare({
+ assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner),
runId: run.id,
companyId: run.companyId,
provider:
@@ -3131,6 +3135,8 @@ export function heartbeatService(
const runOutput = createHeartbeatRunOutput(db, {
run, runId, issueId, runLogStore, identityRedactor, secretKeys,
appendRunEvent, getCurrentUserRedactionOptions,
+ assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner),
+ runtimeLogSecretValues: observabilitySecretValues(observability),
});
const goalCheckpointSession: {
current: {
@@ -3216,7 +3222,7 @@ export function heartbeatService(
message: "run started",
});
- const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent } =
+ const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onPersistedNativeLog, onAdapterMeta, onAdapterEvent } =
await timePreparation("run_output", () => runOutput.begin(currentRun));
if (runScopedMentionedSkillKeys.length > 0) {
await onLog(
@@ -3450,6 +3456,7 @@ export function heartbeatService(
identityRedactor,
appendIdentityRedactedLog,
onLog,
+ onPersistedNativeLog,
onAdapterEvent,
onAdapterMeta,
},
@@ -4072,6 +4079,9 @@ export function heartbeatService(
companyId: string;
logStore: string | null;
logRef: string | null;
+ runtimeMode?: string | null;
+ status?: string;
+ logBytes?: number | null;
},
opts?: { offset?: number; limitBytes?: number; alignStartToRecord?: boolean; signal?: AbortSignal },
) => {
@@ -4084,12 +4094,14 @@ export function heartbeatService(
if (!run) throw notFound("Heartbeat run not found");
if (!run.logStore || !run.logRef) throw notFound("Run log not found");
- const result = await runLogStore.read(
+ if (!isRunLogStoreType(run.logStore)) throw notFound("Run log backend not supported");
+ const result = await readRecoverableRunLog(db, runLogStore,
{
- store: run.logStore as "local_file",
+ store: run.logStore,
logRef: run.logRef,
},
opts,
+ run,
);
return {
diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts
index 1dc15b06e9c..62ce16df460 100644
--- a/server/src/services/heartbeat/run-completion.test.ts
+++ b/server/src/services/heartbeat/run-completion.test.ts
@@ -1,3 +1,5 @@
+import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
+import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { randomUUID } from "node:crypto";
import type { Db } from "@paperclipai/db";
import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
@@ -158,6 +160,57 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object));
});
+ it("settles the accepted native exact rate-card price without replacing it with zero", async () => {
+ const f = await fixture({ runtimeMode: "native" });
+ const runnerInstanceId = randomUUID();
+ const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId,
+ provider: { kind: "codex", model: "gpt-6.1-sol" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
+ const event = (sourceSeq: number, eventType: string, payload: Record): PrpEvent => ({
+ sourceKind: "runner", sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `rate-card-${sourceSeq}`,
+ turnId: "price-turn", eventType, emittedAt: new Date().toISOString(), payload,
+ } as PrpEvent);
+ await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true,
+ runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } }));
+ await delivery.capture(event(2, "turn.completed", {}));
+ const accepted = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id });
+ expect(accepted?.costStatus).toBe("estimated");
+ const exact = accepted!.costUsdExact;
+ f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...accepted, usageComplete: accepted!.complete };
+ await f.completion.completeRun(f.input);
+ const saved = await read(f.run.id);
+ expect(saved).toMatchObject({ status: "succeeded", costAccountingPending: false, costAccountedAt: expect.any(Date),
+ usageJson: { costUsdExact: exact, costStatus: "estimated" } });
+ const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
+ expect(ledger).toHaveLength(1);
+ expect(ledger[0].costCents).toBe(100.2);
+ });
+
+ it("retains newer accepted native usage when completion consumed an older receipt", async () => {
+ const f = await fixture({ runtimeMode: "native", usageJson: { sessionRotated: true, configFreshness: { version: 0 } } });
+ const runnerInstanceId = randomUUID();
+ const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId,
+ provider: { kind: "codex", model: "test-model" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
+ const event = (sourceSeq: number, eventType: string, inputTokens = 100): PrpEvent => ({
+ sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `racing-${sourceSeq}`, turnId: "turn-1",
+ emittedAt: new Date().toISOString(), eventType, payload: eventType === "usage.reported"
+ ? { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens, outputTokens: 10, providerCostUsd: inputTokens / 1000 } } : {},
+ } as PrpEvent);
+ await delivery.capture(event(1, "usage.reported"));
+ await delivery.capture(event(2, "turn.completed"));
+ const acceptedA = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id });
+ await delivery.capture(event(3, "usage.reported", 120));
+ f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...acceptedA, usageComplete: acceptedA!.complete };
+ await f.completion.completeRun(f.input);
+ expect(await read(f.run.id)).toMatchObject({ status: "succeeded", costAccountingPending: true, costAccountedAt: null,
+ usageJson: { inputTokens: 120, costUsd: 0.12, accountingReceiptSequence: 3, accountingReceiptReady: false, sessionRotated: false, configFreshness: { version: 1 } } });
+ expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
+ await delivery.capture(event(4, "turn.completed", 120));
+ expect(await (await import("../run-cost-accounting.js")).accountRunCost(db, f.run.id)).toBe(true);
+ const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
+ expect(ledger).toHaveLength(1);
+ expect(ledger[0].costCents).toBe(12);
+ });
+
it.each([
{ timedOut: true, expected: "timed_out", errorCode: "timeout" },
{ signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" },
diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts
index 39bb4a509ab..c82816f384b 100644
--- a/server/src/services/heartbeat/run-completion.ts
+++ b/server/src/services/heartbeat/run-completion.ts
@@ -1,3 +1,4 @@
+import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
import { measureRunPhase } from "../execution-preparation-timing.js";
import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js";
@@ -686,7 +687,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,
),
- usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },
+ usageJson: preserveAcceptedRunUsageProjectionSql({ ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) }),
finishedAt:
persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,
updatedAt: new Date(),
diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts
index 1922364615a..00a27c1faf2 100644
--- a/server/src/services/heartbeat/run-lifecycle.ts
+++ b/server/src/services/heartbeat/run-lifecycle.ts
@@ -1,3 +1,4 @@
+import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyExternalConversationSettlement } from "../external-conversation-settlement-notifications.js";
@@ -1202,6 +1203,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
.set({
status,
...patch,
+ ...(patch?.usageJson ? { usageJson: preserveAcceptedRunUsageProjectionSql(patch.usageJson) } : {}),
executionStatusDeliveryId: randomUUID(),
updatedAt: new Date(),
})
diff --git a/server/src/services/heartbeat/run-log-authority.ts b/server/src/services/heartbeat/run-log-authority.ts
new file mode 100644
index 00000000000..705035cf617
--- /dev/null
+++ b/server/src/services/heartbeat/run-log-authority.ts
@@ -0,0 +1,65 @@
+import { and, eq } from "drizzle-orm";
+import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
+import { legacyControllerBootId } from "../legacy-controller-lease.js";
+import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
+
+export function createRunLogWriterAuthority(
+ db: Db,
+ admitted: typeof heartbeatRuns.$inferSelect,
+ nativeLeaseOwner?: string,
+) {
+ return async () => {
+ const [run] = await db
+ .select({
+ id: heartbeatRuns.id,
+ runtimeMode: heartbeatRuns.runtimeMode,
+ controllerBootId: heartbeatRuns.controllerBootId,
+ leaseExpiresAt: heartbeatRuns.controllerLeaseExpiresAt,
+ logStore: heartbeatRuns.logStore,
+ logRef: heartbeatRuns.logRef,
+ })
+ .from(heartbeatRuns)
+ .where(
+ and(
+ eq(heartbeatRuns.id, admitted.id),
+ eq(heartbeatRuns.companyId, admitted.companyId),
+ eq(heartbeatRuns.agentId, admitted.agentId),
+ eq(heartbeatRuns.status, "running"),
+ ),
+ );
+ if (!run) throw new Error("run_log_writer_authority_lost");
+ if (run.runtimeMode !== "native") {
+ if (
+ run.controllerBootId !== legacyControllerBootId ||
+ !run.leaseExpiresAt ||
+ run.leaseExpiresAt <= new Date()
+ )
+ throw new Error("run_log_writer_authority_lost");
+ return;
+ }
+ const [owner] = await db
+ .select({
+ leaseOwner: nativeRunFinalizations.leaseOwner,
+ bootId: nativeRunFinalizations.controllerBootId,
+ expiresAt: nativeRunFinalizations.leaseExpiresAt,
+ })
+ .from(nativeRunFinalizations)
+ .where(
+ and(
+ eq(nativeRunFinalizations.runId, admitted.id),
+ eq(nativeRunFinalizations.companyId, admitted.companyId),
+ ),
+ );
+
+ if (!owner && !run.logStore && !run.logRef && !admitted.logStore && !admitted.logRef) return;
+ const identity = await currentNativeControllerIdentity();
+ if (
+ !owner ||
+ owner.bootId !== identity.bootId ||
+ !owner.expiresAt ||
+ owner.expiresAt <= new Date() ||
+ (nativeLeaseOwner !== undefined && owner.leaseOwner !== nativeLeaseOwner)
+ )
+ throw new Error("run_log_writer_authority_lost");
+ };
+}
diff --git a/server/src/services/heartbeat/run-output.test.ts b/server/src/services/heartbeat/run-output.test.ts
index bcfb7c3c2bc..f457f11f55c 100644
--- a/server/src/services/heartbeat/run-output.test.ts
+++ b/server/src/services/heartbeat/run-output.test.ts
@@ -7,6 +7,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createDurableRunLogStore } from "../run-log-store.js";
import type { StorageProvider } from "../../storage/types.js";
import { createHeartbeatRunOutput, isNewRunLog } from "./run-output.js";
+import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
+import { formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js";
+import { generateKeyPairSync } from "node:crypto";
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
@@ -45,15 +48,24 @@ function createRunRowDb(row: Partial) {
return {
update: () => ({
set: (values: Partial) => ({
- where: async () => { Object.assign(row, values); },
+ where: async () => {
+ Object.assign(row, values);
+ },
}),
}),
} as unknown as Db;
}
const newRunRow = {
- id: "run-1", companyId: "company-1", agentId: "agent-1", status: "running",
- logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0, lastOutputAt: null,
+ id: "run-1",
+ companyId: "company-1",
+ agentId: "agent-1",
+ status: "running",
+ logStore: null,
+ logRef: null,
+ logBytes: null,
+ lastOutputSeq: 0,
+ lastOutputAt: null,
} as Partial;
async function beginAttempt(store: ReturnType, row: Partial) {
@@ -81,11 +93,26 @@ afterEach(async () => {
describe("heartbeat run output log recovery", () => {
it("treats only a run row with no recorded log or output as new", () => {
- expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(true);
- expect(isNewRunLog({ logStore: "local_file", logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);
- expect(isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);
- expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(false);
- expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(false);
+ expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(
+ true,
+ );
+ expect(
+ isNewRunLog({
+ logStore: "local_file",
+ logRef: "c/a/r.ndjson",
+ logBytes: null,
+ lastOutputSeq: 0,
+ }),
+ ).toBe(false);
+ expect(
+ isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 }),
+ ).toBe(false);
+ expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(
+ false,
+ );
+ expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(
+ false,
+ );
});
it("makes no remote log recovery request for a new run", async () => {
@@ -116,8 +143,121 @@ describe("heartbeat run output log recovery", () => {
await second.onLog("stdout", "attempt two\n");
await restartedStore.finalize(second.output.handle!);
- const lines = objects.get(row.logRef!)!.toString().trim().split("\n").map((line) => JSON.parse(line));
+ const lines = objects
+ .get(row.logRef!)!
+ .toString()
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(line));
expect(lines.map((line) => line.chunk)).toEqual(["attempt one\n", "attempt two\n"]);
expect(lines.map((line) => line.seq)).toEqual([1, 2]);
});
});
+
+describe("object-backed diagnostic recording", () => {
+ it("applies identity redaction to a native key-bearing event before S3 admission", async () => {
+ const privateKeyPem = generateKeyPairSync("ed25519")
+ .privateKey.export({ type: "pkcs8", format: "pem" })
+ .toString();
+ const row = { ...newRunRow, runtimeMode: "native" } as Run;
+ const db = createRunRowDb(row) as Db & { select: () => unknown };
+ db.select = (() => ({
+ from: () => ({ where: () => ({ orderBy: () => ({ limit: async () => [] }) }) }),
+ })) as typeof db.select;
+ const chunks: string[] = [];
+ const output = createHeartbeatRunOutput(db, {
+ run: row,
+ runId: row.id,
+ issueId: null,
+ secretKeys: new Set(),
+ identityRedactor: createAgentIdentityRedactor(privateKeyPem),
+ runLogStore: {
+ begin: async () => ({
+ store: "s3_segments",
+ logRef: "company/agent/run",
+ initialSourceCursor: 0,
+ }),
+ append: async () => 0,
+ enqueue: async (_handle, event) => {
+ chunks.push(event.chunk);
+ return { committed: Promise.resolve(100) };
+ },
+ flush: async () => undefined,
+ },
+ appendRunEvent: vi.fn(),
+ getCurrentUserRedactionOptions: async () => ({ enabled: false }),
+ });
+ const handlers = await output.begin(row);
+ await handlers.onPersistedNativeLog!(
+ 1,
+ formatNativeRunLogEvent({
+ schema: "paperclip.prp.event.v1",
+ schemaVersion: 1,
+ runId: row.id,
+ sourceEventId: "source:1",
+ sourceInstanceId: "source",
+ sourceKind: "runner",
+ sourceSeq: 1,
+ normalizedSessionId: "session",
+ eventType: "harness.diagnostic",
+ priority: 1,
+ emittedAt: new Date().toISOString(),
+ payload: { text: privateKeyPem },
+ }),
+ );
+ expect(chunks).toHaveLength(1);
+ expect(chunks[0]).not.toContain(privateKeyPem.split("\n")[1]!);
+ expect(chunks[0]).toContain("REDACTED");
+ });
+ it("admits a burst without serial network waits, publishes after receipts, and coalesces progress", async () => {
+ const { publishLiveEvent } = await import("../live-events.js");
+ vi.mocked(publishLiveEvent).mockClear();
+ const row = { ...newRunRow };
+ const writes: Array> = [];
+ const db = {
+ update: () => ({
+ set: (values: Partial) => ({
+ where: async () => {
+ writes.push(values);
+ Object.assign(row, values);
+ },
+ }),
+ }),
+ } as unknown as Db;
+ const commit: Array<() => void> = [];
+ const recorder = createHeartbeatRunOutput(db, {
+ run: row as Run,
+ runId: row.id!,
+ issueId: null,
+ runLogStore: {
+ begin: async () => ({
+ store: "s3_segments",
+ logRef: "company/agent/run",
+ initialBytes: 0,
+ initialSequence: 10,
+ }),
+ append: () => new Promise((resolve) => commit.push(() => resolve(100))),
+ },
+ identityRedactor: { chunk: (_stream, chunk) => chunk, redact: (value) => value },
+ secretKeys: new Set(),
+ appendRunEvent: vi.fn(),
+ getCurrentUserRedactionOptions: async () => ({ enabled: false }),
+ });
+ const { onLog } = await recorder.begin(row as Run);
+ for (let i = 0; i < 100; i++) await onLog("stdout", `line ${i}\n`);
+ expect(commit).toHaveLength(100);
+ expect(publishLiveEvent).not.toHaveBeenCalled();
+ for (const resolve of commit) resolve();
+ await vi.waitFor(() =>
+ expect(
+ vi
+ .mocked(publishLiveEvent)
+ .mock.calls.filter(([event]) => event.type === "heartbeat.run.log"),
+ ).toHaveLength(100),
+ );
+ await recorder.flushOutputProgress({ force: true });
+ expect(row.lastOutputSeq).toBe(110);
+ expect(row.lastOutputBytes).toBe(10000);
+ expect(writes.filter((write) => write.lastOutputSeq !== undefined)).toHaveLength(2);
+ });
+});
diff --git a/server/src/services/heartbeat/run-output.ts b/server/src/services/heartbeat/run-output.ts
index 62d929938ff..71a606a53a4 100644
--- a/server/src/services/heartbeat/run-output.ts
+++ b/server/src/services/heartbeat/run-output.ts
@@ -5,9 +5,15 @@ import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../l
import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { touchHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js";
import { publishLiveEvent } from "../live-events.js";
-import type { RunLogHandle, RunLogStore } from "../run-log-store.js";
+import { isRunLogStoreType, type RunLogHandle, type RunLogStore } from "../run-log-store.js";
import { appendExcerpt, compactRunLogChunk } from "./run-log.js";
-import { isHeartbeatRunRuntimeStatusActive, publishHeartbeatRunRuntimeProgress, type createHeartbeatLifecycle } from "./run-lifecycle.js";
+import { createNativeRunLogExporter } from "../native-runtime/native-run-log-export.js";
+import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
+import {
+ isHeartbeatRunRuntimeStatusActive,
+ publishHeartbeatRunRuntimeProgress,
+ type createHeartbeatLifecycle,
+} from "./run-lifecycle.js";
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
export const ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS = 60 * 1000;
@@ -19,11 +25,13 @@ export interface HeartbeatRunOutputInput {
run: Run;
runId: string;
issueId: string | null;
- runLogStore: Pick;
+ runLogStore: Pick;
identityRedactor: Pick, "chunk" | "redact">;
secretKeys: Set;
appendRunEvent: ReturnType["appendRunEvent"];
getCurrentUserRedactionOptions: () => Promise;
+ assertWriterAuthority?: () => Promise;
+ runtimeLogSecretValues?: readonly string[];
}
/**
@@ -35,15 +43,26 @@ export interface HeartbeatRunOutputInput {
export function isNewRunLog(
run: Pick,
): boolean {
- return run.logStore == null
- && run.logRef == null
- && run.logBytes == null
- && Number(run.lastOutputSeq ?? 0) === 0;
+ return (
+ run.logStore == null &&
+ run.logRef == null &&
+ run.logBytes == null &&
+ Number(run.lastOutputSeq ?? 0) === 0
+ );
}
/** One recorder per execution attempt; the log store itself stays service-scoped. */
export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) {
- const { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions } = input;
+ const {
+ run,
+ runId,
+ issueId,
+ runLogStore,
+ identityRedactor,
+ secretKeys,
+ appendRunEvent,
+ getCurrentUserRedactionOptions,
+ } = input;
let handle: RunLogHandle | null = null;
let stdoutExcerpt = "";
let stderrExcerpt = "";
@@ -59,7 +78,14 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
} | null;
} = { pending: null };
let persistedLogBytes = Number(run.logBytes ?? 0);
+ let progressWrite: Promise | null = null;
+ let recordingFailureReported = false;
const flushOutputProgress = async (opts?: { force?: boolean }) => {
+ if (progressWrite) {
+ await progressWrite;
+ if (opts?.force) return flushOutputProgress(opts);
+ return;
+ }
const pendingOutputProgress = outputProgressState.pending;
if (!pendingOutputProgress) return;
const shouldFlush =
@@ -68,7 +94,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >=
ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS;
if (!shouldFlush) return;
- await db
+ progressWrite = db
.update(heartbeatRuns)
.set({
lastOutputAt: pendingOutputProgress.at,
@@ -77,111 +103,181 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
lastOutputBytes: pendingOutputProgress.bytes,
updatedAt: new Date(),
})
- .where(eq(heartbeatRuns.id, run.id));
- lastOutputFlushAt = pendingOutputProgress.at;
- outputProgressState.pending = null;
+ .where(eq(heartbeatRuns.id, run.id))
+ .then(() => {
+ lastOutputFlushAt = pendingOutputProgress.at;
+ if (outputProgressState.pending === pendingOutputProgress)
+ outputProgressState.pending = null;
+ })
+ .finally(() => {
+ progressWrite = null;
+ });
+ await progressWrite;
};
// Bind handlers to the run row persisted at execution start. Progress writes
// above use only the immutable run ID from the initial row.
async function begin(currentRun: Run) {
const run = currentRun;
+ if (run.logStore != null && !isRunLogStoreType(run.logStore))
+ throw new Error("Run log backend is not supported");
handle = await runLogStore.begin({
companyId: run.companyId,
agentId: run.agentId,
runId,
newRun: isNewRunLog(run),
+ assertWriterAuthority: input.assertWriterAuthority,
+ previousStore: run.logStore ?? undefined,
});
+ outputSeq = handle.initialSequence ?? outputSeq;
+ persistedLogBytes = handle.initialBytes ?? persistedLogBytes;
await db
.update(heartbeatRuns)
.set({
logStore: handle.store,
logRef: handle.logRef,
+ ...(handle.store === "s3_segments" ? { logBytes: null, logSha256: null } : {}),
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, runId));
- const currentUserRedactionOptions =
- await getCurrentUserRedactionOptions();
- const appendIdentityRedactedLog = async (stream: "stdout" | "stderr", chunk: string) => {
+ const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
+ if (
+ handle.store === "s3_segments" &&
+ run.runtimeMode === "native" &&
+ input.runtimeLogSecretValues?.length
+ ) {
+ const registry = createRunSecretRedactionRegistry(db);
+ for (const value of new Set(input.runtimeLogSecretValues))
+ await registry.register(run.companyId, runId, value);
+ }
+ const reportRecordingFailure = async () => {
+ if (recordingFailureReported) return;
+ recordingFailureReported = true;
+ await appendRunEvent(currentRun, {
+ eventType: "lifecycle",
+ stream: "system",
+ level: "warn",
+ message: "Run log persistence is incomplete",
+ payload: { reason: "object_log_persistence_failed" },
+ }).catch(() => undefined);
+ };
+ const appendIdentityRedactedLog = async (
+ stream: "stdout" | "stderr",
+ chunk: string,
+ waitForCommit = true,
+ sourceCursor?: number,
+ ) => {
const sanitizedChunk = compactRunLogChunk(
redactCurrentUserText(chunk, currentUserRedactionOptions),
);
- if (stream === "stdout")
- stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
- if (stream === "stderr")
- stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
+ if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
+ if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
const ts = new Date().toISOString();
outputSeq += 1;
const chunkSeq = outputSeq;
- let appendedBytes = 0;
+ let committed: Promise = Promise.resolve(0);
if (handle) {
- appendedBytes = await runLogStore.append(handle, {
+ const event = {
stream,
chunk: sanitizedChunk,
ts,
seq: chunkSeq,
- });
+ sourceCursor,
+ };
+ committed = runLogStore.enqueue
+ ? (await runLogStore.enqueue(handle, event)).committed
+ : runLogStore.append(handle, event);
+ }
+ const recorded = committed.then(async (appendedBytes) => {
persistedLogBytes += appendedBytes;
- }
- outputProgressState.pending = {
- at: new Date(ts),
- seq: chunkSeq,
- stream,
- bytes: persistedLogBytes,
- };
- await flushOutputProgress();
-
- // Streamed CLI output is real run activity: keep the in-memory
- // runtime status ("Working... / X ago") fresh between structured
- // events so sandbox runs with mid-run log streaming never show a
- // minutes-stale timestamp. Throttled to avoid churning the live
- // event stream on every 250ms tail chunk.
- const logActivityAt = new Date(ts);
- if (
- isHeartbeatRunRuntimeStatusActive(run.status) &&
- logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
- ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
- ) {
- lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
- const touchedStatus = touchHeartbeatRunRuntimeStatus({
- companyId: run.companyId,
- issueId,
- agentId: run.agentId,
- runId: run.id,
- at: logActivityAt,
- });
- if (touchedStatus)
- publishHeartbeatRunRuntimeProgress(touchedStatus);
- }
-
- const payloadChunk =
- sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
- ? sanitizedChunk.slice(
- sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES,
- )
- : sanitizedChunk;
-
- publishLiveEvent({
- companyId: run.companyId,
- type: "heartbeat.run.log",
- payload: {
- runId: run.id,
- agentId: run.agentId,
- issueId,
- ts,
+ outputProgressState.pending = {
+ at: new Date(ts),
seq: chunkSeq,
stream,
- chunk: payloadChunk,
- logRef: handle?.logRef,
- logStore: handle?.store,
- truncated: payloadChunk.length !== sanitizedChunk.length,
- },
+ bytes: persistedLogBytes,
+ };
+ await flushOutputProgress();
+
+ // Streamed CLI output is real run activity: keep the in-memory
+ // runtime status ("Working... / X ago") fresh between structured
+ // events so sandbox runs with mid-run log streaming never show a
+ // minutes-stale timestamp. Throttled to avoid churning the live
+ // event stream on every 250ms tail chunk.
+ const logActivityAt = new Date(ts);
+ if (
+ isHeartbeatRunRuntimeStatusActive(run.status) &&
+ logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
+ ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
+ ) {
+ lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
+ const touchedStatus = touchHeartbeatRunRuntimeStatus({
+ companyId: run.companyId,
+ issueId,
+ agentId: run.agentId,
+ runId: run.id,
+ at: logActivityAt,
+ });
+ if (touchedStatus) publishHeartbeatRunRuntimeProgress(touchedStatus);
+ }
+
+ const payloadChunk =
+ sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
+ ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
+ : sanitizedChunk;
+
+ publishLiveEvent({
+ companyId: run.companyId,
+ type: "heartbeat.run.log",
+ payload: {
+ runId: run.id,
+ agentId: run.agentId,
+ issueId,
+ ts,
+ seq: chunkSeq,
+ stream,
+ chunk: payloadChunk,
+ logRef: handle?.logRef,
+ logStore: handle?.store,
+ truncated: payloadChunk.length !== sanitizedChunk.length,
+ },
+ });
});
+ if (waitForCommit) await recorded;
+ else void recorded.catch(reportRecordingFailure);
};
- const onLog = (stream: "stdout" | "stderr", chunk: string) =>
- appendIdentityRedactedLog(stream, identityRedactor.chunk(stream, chunk));
+ const onLog = (stream: "stdout" | "stderr", chunk: string) => {
+ const external = handle?.store === "s3_segments";
+ const recorded = appendIdentityRedactedLog(
+ stream,
+ identityRedactor.chunk(stream, chunk),
+ !external,
+ );
+ return external ? recorded.catch(reportRecordingFailure) : recorded;
+ };
+ const nativeExporter =
+ handle.store === "s3_segments" && run.runtimeMode === "native"
+ ? createNativeRunLogExporter({
+ db,
+ companyId: run.companyId,
+ runId,
+ afterCursor: handle.initialSourceCursor ?? 0,
+ enqueue: (cursor, chunk) =>
+ appendIdentityRedactedLog(
+ "stdout",
+ identityRedactor.chunk("stdout", chunk),
+ false,
+ cursor,
+ ),
+ flush: () => runLogStore.flush!(handle!),
+ })
+ : undefined;
+ if (nativeExporter) await nativeExporter.recover();
+ const onPersistedNativeLog = nativeExporter
+ ? (cursor: number, chunk: string) =>
+ nativeExporter.record(cursor, chunk).catch(reportRecordingFailure)
+ : undefined;
const onAdapterMeta = async (meta: AdapterInvocationMeta) => {
meta = identityRedactor.redact(meta);
if (meta.env && secretKeys.size > 0) {
@@ -214,16 +310,29 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
});
};
- return { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent };
+ return {
+ currentUserRedactionOptions,
+ appendIdentityRedactedLog,
+ onLog,
+ onPersistedNativeLog,
+ onAdapterMeta,
+ onAdapterEvent,
+ };
}
return {
begin,
- get handle() { return handle; },
+ get handle() {
+ return handle;
+ },
outputProgressState,
flushOutputProgress,
// Finalization awaits I/O and can receive more output before persisting excerpts.
- get stdoutExcerpt() { return stdoutExcerpt; },
- get stderrExcerpt() { return stderrExcerpt; },
+ get stdoutExcerpt() {
+ return stdoutExcerpt;
+ },
+ get stderrExcerpt() {
+ return stderrExcerpt;
+ },
};
}
diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts
index af4d682291e..c2030afa7d1 100644
--- a/server/src/services/heartbeat/run-state.ts
+++ b/server/src/services/heartbeat/run-state.ts
@@ -355,6 +355,9 @@ const heartbeatRunLogAccessColumns = {
...heartbeatRunReadAccessColumns,
logStore: heartbeatRuns.logStore,
logRef: heartbeatRuns.logRef,
+ runtimeMode: heartbeatRuns.runtimeMode,
+ status: heartbeatRuns.status,
+ logBytes: heartbeatRuns.logBytes,
} as const;
const heartbeatRunIssueControlColumns = {
diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts
index d8b65929c6a..f39b180fbde 100644
--- a/server/src/services/heartbeat/runtime-execution.test.ts
+++ b/server/src/services/heartbeat/runtime-execution.test.ts
@@ -1,9 +1,12 @@
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
+
+import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
+import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";
+import { agents, budgetPolicies, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";
import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { trace } from "@opentelemetry/api";
@@ -213,6 +216,38 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
expectCleanup(input);
});
+ it("returns the accepted exact native rate-card price and preserves its live budget hold", async () => {
+ const { input, run } = await fixture(true);
+ input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home };
+ input.runtime.nativeExecution!.provider = { kind: "codex", model: "gpt-6.1-sol" };
+ await db.insert(budgetPolicies).values({ companyId: run.companyId, scopeType: "agent", scopeId: input.agent.id,
+ metric: "billed_cents", windowKind: "lifetime", amount: 100, notifyEnabled: false });
+ let exact: string | undefined;
+ vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => {
+ expect(native.onUsage).toBeUndefined();
+ const delivery = await createNativeUsageDelivery(db, { companyId: run.companyId, runId: run.id,
+ runnerInstanceId: native.runnerInstanceId, provider: native.execution.provider,
+ billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
+ const event = (sourceSeq: number, eventType: string, payload: Record): PrpEvent => ({
+ sourceKind: "runner", sourceInstanceId: native.runnerInstanceId, runId: run.id, sourceSeq, sourceEventId: `priced-${sourceSeq}`,
+ turnId: "priced-turn", eventType, emittedAt: new Date().toISOString(), payload,
+ } as PrpEvent);
+ await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true,
+ runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } }));
+ await delivery.capture(event(2, "turn.completed", {}));
+ const saved = (await read(run.id)).usageJson!;
+ exact = saved.costUsdExact as string;
+ expect(Number(exact)).toBe(1.002);
+ expect(saved.budgetExposure).toMatchObject({ costCents: "100.2000000" });
+ expect((await db.select().from(agents).where(eq(agents.id, input.agent.id)))[0].lifecycleHolds).toContain("budget");
+ return { exitCode: 0, signal: null, timedOut: false, usageComplete: true, costUsd: null, model: "gpt-6.1-sol" };
+ });
+ const result = await executeHeartbeatRuntime(db, input);
+ if (!result.dispatched) throw new Error("expected native dispatch");
+ expect(result.adapterResult).toMatchObject({ costUsdExact: exact, costStatus: "estimated", usageComplete: true,
+ pricingProvenance: { source: "rate_card" }, usage: { inputTokens: 1000, outputTokens: 100000 } });
+ });
+
it.each(["expected", "wrong"])("finalizes a plugin worktree through its owner (branch: %s)", async branch => {
const { input } = await fixture(true);
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId,
diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts
index 69edcfdbee6..61a728872b9 100644
--- a/server/src/services/heartbeat/runtime-execution.ts
+++ b/server/src/services/heartbeat/runtime-execution.ts
@@ -1,4 +1,6 @@
import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js";
+
+import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { writeExecutionControl } from "../execution-control-notifications.js";
import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js";
@@ -34,7 +36,7 @@ import {
NativeWorkspaceFinalizationOwnershipLostError,
type NativeWorkspaceFinalizationOwnership,
} from "../native-runtime/native-workspace-finalization-ownership.js";
-import { createRunUsageRecorder } from "../usage-receipts.js";
+import { createRunUsageRecorder, readAcceptedRunUsageCheckpoint } from "../usage-receipts.js";
import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { isConversation } from "../agent-conversations.js";
@@ -173,6 +175,7 @@ export interface HeartbeatRuntimeExecutionInput {
>;
};
output: Pick & {
+ onPersistedNativeLog?: (cursor: number, chunk: string) => Promise;
onAdapterEvent: NonNullable;
onAdapterMeta: AdapterExecutionContext["onMeta"];
identityRedactor: ReturnType;
@@ -496,9 +499,13 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
adapterFinalizeOutcome = status;
};
- const usageRecorder = await measureRunPhase("execution.usage_startup", () =>
+ const producerOwnsUsage = usesNativeUsageDelivery({
+ runnerd: nativeRuntimeResolution.kind === "native" && agent.adapterType === "paperclip_runner",
+ remote: executionTarget?.kind === "remote", provider: nativeExecution?.provider,
+ });
+ const usageRecorder = producerOwnsUsage ? undefined : await measureRunPhase("execution.usage_startup", () =>
createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }));
- input.effects.onUsageCaptureReady(usageRecorder.persistFailure);
+ input.effects.onUsageCaptureReady(usageRecorder?.persistFailure ?? (async () => {}));
let adapterResult: AdapterExecutionResult;
const runGoalControlRequestId = readNonEmptyString(
context.goalControlRequestId,
@@ -630,10 +637,11 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
goalCheckpointSession.current = { params, displayId };
},
onLog,
+ onPersistedNativeLog: input.output.onPersistedNativeLog,
onEvent: onAdapterEvent,
instructionWorkingCopy: nativeInstructionWorkingCopy(),
- onUsage: async receipt => { await usageRecorder.capture(receipt); },
+ onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
preparationSpans: nativeRunnerPreparationSpans,
// Bootstrap with executable/home discovery while keeping
// configured provider values and the server-selected
@@ -835,7 +843,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
onLog,
onMeta: onAdapterMeta,
onEvent: onAdapterEvent,
- onUsage: async receipt => { await usageRecorder.capture(receipt); },
+ onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
onExecutionPhase: executionControl.phases.enter,
startupTraceContext: getStartupTraceContext(),
onRuntimeProgress: async (progress) => {
@@ -914,8 +922,14 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };
if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {
- const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
- adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
+ if (usageRecorder) {
+ const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
+ adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
+ } else {
+ const accepted = await measureRunPhase("execution.usage", () => readAcceptedRunUsageCheckpoint(db, { companyId: run.companyId, runId: run.id }));
+ adapterResult = { ...adapterResult, ...accepted, usageComplete: accepted?.complete ?? false,
+ ...(!accepted ? { costStatus: "unpriced" as const } : {}) };
+ }
} else {
// Stop may already own the terminal result. Preserve its metadata
// while durably recording the proof needed to release admission.
diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts
index e066949c630..42c65bb5d02 100644
--- a/server/src/services/issues.ts
+++ b/server/src/services/issues.ts
@@ -189,7 +189,8 @@ import {
resolveIssueGoalId,
resolveNextIssueGoalId,
} from "./issue-goal-fallback.js";
-import { getRunLogStore } from "./run-log-store.js";
+import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
+import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js";
import { getDefaultCompanyGoal } from "./goals.js";
import { assertAssignableAgent } from "./agent-assignability.js";
import {
@@ -6712,13 +6713,14 @@ export async function readIssueCommentRunLogText(run: {
logStore: string | null;
logRef: string | null;
logBytes: number | null;
-}) {
- if (run.logStore !== "local_file" || !run.logRef) return "";
+}, db?: Db) {
+ if (!isRunLogStoreType(run.logStore) || !run.logRef) return "";
// A timed-out finalization leaves size unknown even when earlier entries
// exist. Read those logs within the same byte budget as a known-size log.
if (run.logBytes !== null && (!Number.isFinite(run.logBytes) || run.logBytes <= 0)) return "";
const logRef = run.logRef;
+ const storeType = run.logStore;
const store = getRunLogStore();
let offset = 0;
let content = "";
@@ -6727,6 +6729,8 @@ export async function readIssueCommentRunLogText(run: {
let readTimer: NodeJS.Timeout | undefined;
const readChunks = async () => {
+ if (db && storeType === "s3_segments") await recoverSettledNativeRunLog(db, store, logRef, undefined, controller.signal)
+ .catch(() => { controller.signal.throwIfAborted(); });
while (nextOffset !== undefined) {
controller.signal.throwIfAborted();
const remainingBytes =
@@ -6734,7 +6738,7 @@ export async function readIssueCommentRunLogText(run: {
Buffer.byteLength(content, "utf8");
if (remainingBytes <= 0) break;
const chunk = await store.read(
- { store: "local_file", logRef },
+ { store: storeType, logRef },
{
offset,
limitBytes: Math.min(ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES, remainingBytes),
@@ -7259,7 +7263,7 @@ export function issueService(db: Db) {
);
await Promise.all(
batch.map(async (run) => {
- logByRunId.set(run.runId, await readIssueCommentRunLogText(run));
+ logByRunId.set(run.runId, await readIssueCommentRunLogText(run, db));
}),
);
}
@@ -12587,6 +12591,7 @@ export function issueService(db: Db) {
order?: "asc" | "desc";
limit?: number | null;
humanOnly?: boolean;
+ companyId?: string;
},
) => {
const order = opts?.order === "asc" ? "asc" : "desc";
@@ -12597,6 +12602,7 @@ export function issueService(db: Db) {
: null;
const conditions = [eq(issueComments.issueId, issueId)];
+ if (opts?.companyId) conditions.push(eq(issueComments.companyId, opts.companyId));
if (opts?.humanOnly) conditions.push(or(
eq(issueComments.authorType, "user"),
and(isNull(issueComments.authorType), isNotNull(issueComments.authorUserId)),
diff --git a/server/src/services/native-runtime/__fixtures__/usage-receiver.ts b/server/src/services/native-runtime/__fixtures__/usage-receiver.ts
new file mode 100644
index 00000000000..8b5a2805397
--- /dev/null
+++ b/server/src/services/native-runtime/__fixtures__/usage-receiver.ts
@@ -0,0 +1,45 @@
+import { resolve } from "node:path";
+import { createDb } from "@paperclipai/db";
+import { DurablePrpControlPlane, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint } from "../../../vendor/paperclip-runner/index.js";
+import { createNativeUsageDelivery } from "../native-usage-delivery.js";
+
+type Start = {
+ kind: "start"; requestId: number; connectionString: string; directory: string; port?: number;
+ identity: ConstructorParameters[0]["identity"];
+ input: Parameters[1];
+ checkpoint?: [string, string][];
+};
+let core: DurablePrpControlPlane | undefined;
+let checkpoint: DurablePrpCheckpoint | undefined;
+process.on("message", async (message: Start | { kind: "command"; requestId: number; type: string; commandId: string } | { kind: "stop"; requestId: number }) => {
+ try {
+ if (message.kind === "start") {
+ const delivery = await createNativeUsageDelivery(createDb(message.connectionString), message.input);
+ if (message.checkpoint) decodeDurablePrpCheckpoint(new Map(message.checkpoint)).restoreSessionDirectory(message.directory);
+ core = new DurablePrpControlPlane({ stateDirectory: resolve(message.directory, "control-plane"), identity: message.identity,
+ expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"c".repeat(64)}`,
+ onCommittedEvent: async event => {
+ try { await delivery.capture(event); }
+ catch (error) { process.send?.({ kind: "capture-failed", cause: error instanceof Error ? error.message : String(error) }); throw error; }
+ },
+ });
+ core.setCheckpointPersistence(async saved => { checkpoint = saved; });
+ await core.start(message.port);
+ const ticket = message.checkpoint ? undefined : core.issueBootstrapTicket();
+ await core.flushCheckpoint();
+ process.send?.({ requestId: message.requestId, connectUrl: core.connectUrl, ticket });
+ } else if (message.kind === "command") {
+ core!.queueCommand(message.type, {}, message.commandId, true);
+ process.send?.({ requestId: message.requestId });
+ } else {
+ await core?.stop();
+ await core?.drainPendingConnectionProcessing();
+ await core?.flushCheckpoint();
+ process.send?.({ requestId: message.requestId, checkpoint: [...checkpoint!.records] }, () => process.exit(0));
+ }
+ } catch (error) {
+ process.send?.({ requestId: message.requestId, error: error instanceof Error ? error.message : String(error) });
+ }
+});
+
+process.send?.({ kind: "ready" });
diff --git a/server/src/services/native-runtime/native-github-access.test.ts b/server/src/services/native-runtime/native-github-access.test.ts
index 444715a1b14..1f8b07172b1 100644
--- a/server/src/services/native-runtime/native-github-access.test.ts
+++ b/server/src/services/native-runtime/native-github-access.test.ts
@@ -3,6 +3,7 @@ import { spawn, execFile } from "node:child_process";
import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises";
import { promisify } from "node:util";
import type { CommandManagedDuplexChannel, CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
+import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability";
import { startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -39,7 +40,7 @@ function localDuplexChannel(command: readonly string[], cwd: string): CommandMan
async function broker(resolveCredentials = vi.fn(async (binding: ReturnType) => ({
status: "available", env: { GH_TOKEN: `fixture-${binding.runId}` },
})), remote = false, bridgeUnavailable = false, onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise,
-duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) {
+duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean; recorder?: DuplexObservabilityRecorder }) {
const root = await mkdtemp(path.join(tmpdir(), "native-github-reuse-"));
cleanups.push(() => rm(root, { recursive: true, force: true }));
const bin = path.join(root, "real-bin");
@@ -61,7 +62,7 @@ duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) {
});
const target = remote ? { kind: "remote" as const, transport: "sandbox" as const, providerKey: "test",
remoteCwd: root, runner: { execute, ...(duplex ? { openDuplexChannel } : {}) }, streamRunLogs: false,
- ...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, effectiveCapabilities: {
+ ...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, duplexObservabilityRecorder: duplex.recorder, effectiveCapabilities: {
reusableLeases: true, nativeSyncIn: false, nativeSyncOut: false, persistentProcessSessions: false,
independentControlCommands: true, incrementalSessionOutput: false, concurrentSyncOperations: false,
duplexCommandStream: duplex.capability, runnerWebSocketIngress: false,
@@ -122,9 +123,13 @@ it.each([false, true])("resolves managed GitHub credentials from OpenCode shell
expect(childEnvironment.githubTokenPresent).toBe(false);
}, 30000);
it("uses the selected duplex transport across credential rotation without file polling", async () => {
- const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true });
+ const recorder = { recordSpan: vi.fn(), incrementCounter: vi.fn(), emitEvent: vi.fn() };
+ const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true, recorder });
expect(b.bridgeModes).toEqual(["http2_v1"]);
expect(b.openDuplexChannel).toHaveBeenCalledOnce();
+ expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.channel_open", dimensions: {
+ provider: "other", transport: "http2", outcome: "ok",
+ } }));
const commandsAfterStartup = b.execute.mock.calls.length;
expect((await request(b)).status).toBe(403);
const releaseA = b.activate(run("a"));
@@ -135,6 +140,8 @@ it("uses the selected duplex transport across credential rotation without file p
releaseA();
expect(await (await request(b)).json()).toMatchObject({ env: { GH_TOKEN: "fixture-b" } });
expect(b.resolveCredentials.mock.calls.map(([binding]) => binding.runId)).toEqual(["a", "b"]);
+ expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.request" }));
+ expect(recorder.emitEvent).not.toHaveBeenCalledWith(expect.objectContaining({ dimensions: expect.objectContaining({ transport: "file" }) }));
expect(b.execute.mock.calls).toHaveLength(commandsAfterStartup);
releaseB();
await b.stop();
diff --git a/server/src/services/native-runtime/native-github-access.ts b/server/src/services/native-runtime/native-github-access.ts
index 87233394538..870dbbbbac9 100644
--- a/server/src/services/native-runtime/native-github-access.ts
+++ b/server/src/services/native-runtime/native-github-access.ts
@@ -3,6 +3,7 @@ import { createServer } from "node:http";
import {
cleanupGitHubOperationLaunchers,
adapterExecutionTargetEnablesSandboxDuplexBridge,
+ adapterExecutionTargetDuplexObservabilityRecorder,
githubOperationLauncherDirectory,
prepareGitHubOperationLaunchers,
startAdapterExecutionTargetPaperclipBridge,
@@ -107,6 +108,7 @@ export async function createNativeGitHubAccess(input: {
hostApiToken: token,
hostApiUrl: url,
enableSandboxDuplexBridge: duplexRequested,
+ duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(input.target),
onLog: input.onLog,
});
if (input.target?.kind === "remote" && !bridge) {
diff --git a/server/src/services/native-runtime/native-response-presentation-work.ts b/server/src/services/native-runtime/native-response-presentation-work.ts
index aac11b54e3f..cad63e0f53c 100644
--- a/server/src/services/native-runtime/native-response-presentation-work.ts
+++ b/server/src/services/native-runtime/native-response-presentation-work.ts
@@ -7,6 +7,7 @@ import type { NativeResponsePresentationScope } from "../native-response-present
import { onCommittedChatResponseAuthorityRegistered } from "../durable-chat-wakeup.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
import { recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
+import { measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js";
export function registerNativeResponsePresentationWork(db: Db,
coordinator: ReturnType, canRun: () => boolean) {
@@ -20,27 +21,35 @@ export function nativeResponsePresentationWork(db: Db) {
const query = nativeResponsePresentationCandidates(scope, after, runId).as("pending_native_response");
return db.select().from(query).orderBy(asc(query.id));
};
- async function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise {
+ function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise {
+ const work = () => loadWithinTrace(scope, signal, canRun);
+ return scope?.runId ? withRunResponsePresentationTiming(scope.runId, work) : work();
+ }
+ async function loadWithinTrace(scope: NativeResponsePresentationScope | undefined, signal: AbortSignal | undefined,
+ canRun: () => boolean): Promise {
const entries: Entry[] = [];
let after: string | undefined;
while (canRun() && !signal?.aborted) {
- const rows = await candidates(scope, after);
+ const rows = await measureRunPhase("response.resolve", () => candidates(scope, after));
for (const row of rows) entries.push({ id: row.id, companyId: row.companyId, issueId: row.issueId, agentId: row.agentId, dueAt: row.nextAttemptAt?.getTime() ?? 0 });
if (rows.length < 100) return entries;
after = rows[rows.length - 1].id;
}
throw new Error("Native response presentation discovery stopped");
}
- async function check(entry: Entry, signal?: AbortSignal): Promise {
+ function check(entry: Entry, signal?: AbortSignal): Promise {
+ return withRunResponsePresentationTiming(entry.id, () => checkWithinTrace(entry, signal));
+ }
+ async function checkWithinTrace(entry: Entry, signal?: AbortSignal): Promise {
if (signal?.aborted) throw new Error("Native response presentation recovery stopped");
const scope = { companyId: entry.companyId, issueId: entry.issueId };
- const [current] = await candidates(scope, undefined, entry.id);
+ const [current] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id));
if (!current) return null;
if (current.nextAttemptAt && current.nextAttemptAt.getTime() > Date.now()) return { ...entry, dueAt: current.nextAttemptAt.getTime() };
if (signal?.aborted) throw new Error("Native response presentation recovery stopped");
const recoveryStartedAt = Date.now();
await recoverCommittedNativeChatResponse(db, { ...scope, runId: entry.id, onlyWhenDue: true });
- const [remaining] = await candidates(scope, undefined, entry.id);
+ const [remaining] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id));
return remaining?.nextAttemptAt && remaining.nextAttemptAt.getTime() > recoveryStartedAt
? { ...entry, dueAt: remaining.nextAttemptAt.getTime() } : null;
}
diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts
index ba2017eb9a8..202d2b4eac3 100644
--- a/server/src/services/native-runtime/native-run-finalizer.ts
+++ b/server/src/services/native-runtime/native-run-finalizer.ts
@@ -67,6 +67,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueService } from "../issues.js";
import { publishChatPublicationCommitSignal } from "../chat-publication-reconciliation.js";
import { nativeSha256 } from "./canonical.js";
+import { beginRunPhase, measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js";
import {
readNativeBoardResponseWaitOrigin,
readNativeBoardResponseWaitSource,
@@ -736,11 +737,20 @@ export async function repairCommittedNativeChatResponse(
export async function recoverCommittedNativeChatResponse(
db: Db,
input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean },
+): Promise {
+ return withRunResponsePresentationTiming(input.runId, () => recoverCommittedNativeChatResponseInScope(db, input));
+}
+
+async function recoverCommittedNativeChatResponseInScope(
+ db: Db,
+ input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean },
): Promise {
let agentId: string | null = null;
let materialized = false;
+ const entered = beginRunPhase("response.transaction.entry");
try {
- materialized = await db.transaction(async (transaction) => {
+ materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
+ entered();
const tx = transaction as unknown as Db;
@@ -1022,10 +1032,12 @@ export async function recoverCommittedNativeChatResponse(
await projectCommittedRun({ db: tx, run, coordinator });
agentId = run.agentId;
return true;
- });
+ }));
} catch (error) {
if (error instanceof CommittedChatResponseAuthorizationError) return false;
throw error;
+ } finally {
+ entered();
}
if (materialized && agentId)
publishChatPublicationCommitSignal({
diff --git a/server/src/services/native-runtime/native-run-log-export.integration.test.ts b/server/src/services/native-runtime/native-run-log-export.integration.test.ts
new file mode 100644
index 00000000000..582bab13407
--- /dev/null
+++ b/server/src/services/native-runtime/native-run-log-export.integration.test.ts
@@ -0,0 +1,474 @@
+import { randomUUID, createHash, generateKeyPairSync } from "node:crypto";
+import { eq, sql } from "drizzle-orm";
+import {
+ createDb,
+ heartbeatRunEvents,
+ heartbeatRuns,
+ agentIdentityKeys,
+ type Db,
+} from "@paperclipai/db";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import {
+ S3Client,
+ CreateBucketCommand,
+ DeleteBucketCommand,
+ DeleteObjectsCommand,
+ ListObjectsV2Command,
+} from "@aws-sdk/client-s3";
+import { createS3StorageProvider } from "../../storage/s3-provider.js";
+import { createObjectRunLogStore } from "../object-run-log-store.js";
+import { createNativeRunLogExporter, formatNativeRunLogEvent } from "./native-run-log-export.js";
+import { recoverSettledNativeRunLog, readRecoverableRunLog } from "./native-run-log-recovery.js";
+import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
+import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
+import { createNativeRunLogTestTables } from "../../__tests__/helpers/native-run-log-database.js";
+import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
+import type { StorageProvider } from "../../storage/types.js";
+import { appendNativeCredentialUpdate } from "./native-credential-updates.js";
+import { appendRetainedNativeCleanupEvent } from "./native-session-executor.js";
+import { idleWorkSnapshot } from "../task-admission.js";
+import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
+import { localEncryptedProvider } from "../../secrets/local-encrypted-provider.js";
+import { createHeartbeatRunOutput } from "../heartbeat/run-output.js";
+import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
+import { ObjectJournalConflictError } from "../object-byte-journal.js";
+
+const url = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL");
+const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
+describe.skipIf(!url || !endpoint)("native run log source recovery", () => {
+ const schema = `native_log_${randomUUID().replaceAll("-", "")}`,
+ bucket = `pc-native-log-${randomUUID()}`;
+ let db: Db,
+ client: S3Client,
+ bucketCreated = false;
+ beforeAll(async () => {
+ db = createDb(url!, { maxConnections: 1, idleTimeoutSeconds: 0 });
+ await db.execute(sql.raw(`create schema ${schema}`));
+ await db.execute(sql.raw(`set search_path=${schema}`));
+ await createNativeRunLogTestTables(db);
+ client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
+ await client.send(new CreateBucketCommand({ Bucket: bucket }));
+ bucketCreated = true;
+ });
+ afterEach(() => vi.unstubAllEnvs());
+ afterAll(async () => {
+ if (client && bucketCreated) {
+ for (;;) {
+ const page = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
+ if (!page.Contents?.length) break;
+ await client.send(
+ new DeleteObjectsCommand({
+ Bucket: bucket,
+ Delete: { Objects: page.Contents.map((item) => ({ Key: item.Key! })) },
+ }),
+ );
+ }
+ await client.send(new DeleteBucketCommand({ Bucket: bucket }));
+ client.destroy();
+ }
+ if (db) {
+ await db.execute(sql.raw(`drop schema ${schema} cascade`));
+ await db.$client.end();
+ }
+ });
+ async function fixture() {
+ const binding = {
+ companyId: randomUUID(),
+ runId: randomUUID(),
+ agentId: randomUUID(),
+ sessionId: randomUUID(),
+ issueId: randomUUID(),
+ sourceInstanceId: randomUUID(),
+ controlPlaneSourceInstanceId: randomUUID(),
+ completionContractId: randomUUID(),
+ completionContractSha256: "test",
+ };
+ await db.execute(
+ sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${binding.runId},${binding.companyId},${binding.agentId},'running')`,
+ );
+ const provider = createS3StorageProvider({
+ endpoint,
+ bucket,
+ region: "us-east-1",
+ forcePathStyle: true,
+ });
+ const store = (storage: StorageProvider = provider) =>
+ createObjectRunLogStore({
+ provider: storage,
+ keyPrefix: "logs",
+ flushMs: 30_000,
+ assertWriterAuthority: async (input) => {
+ await input.assertWriterAuthority?.();
+ },
+ });
+ const begin = { companyId: binding.companyId, agentId: binding.agentId, runId: binding.runId };
+ const port = new PaperclipControlPlanePort(db, binding);
+ const event = (seq: number): PrpEvent => ({
+ schema: "paperclip.prp.event.v1",
+ schemaVersion: 1,
+ sourceInstanceId: binding.sourceInstanceId,
+ sourceEventId: `${binding.sourceInstanceId}:${seq}`,
+ sourceSeq: seq,
+ sourceKind: "runner",
+ runId: binding.runId,
+ normalizedSessionId: binding.sessionId,
+ eventType: "item.delta",
+ priority: 2,
+ emittedAt: new Date().toISOString(),
+ payload: { text: `step ${seq} 再開` },
+ });
+ return { binding, store, begin, port, event, provider };
+ }
+ it("replays acknowledged native records into a new writer, repairs gaps, and ignores late duplicates", async () => {
+ const f = await fixture(),
+ a = f.store(),
+ h = await a.begin(f.begin);
+ await db.execute(
+ sql`update heartbeat_runs set log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
+ );
+ const events = [f.event(1), f.event(2), f.event(3)];
+ const cursors = await Promise.all(events.slice(0, 2).map((event) => f.port.appendEvent(event)));
+ const receipt = await a.enqueue!(h, {
+ stream: "stdout",
+ chunk: formatNativeRunLogEvent(events[0]!),
+ ts: "then",
+ seq: 1,
+ sourceCursor: cursors[0]!.cursor,
+ });
+ await a.flush!(h);
+ await receipt.committed;
+ const b = f.store(),
+ successor = await b.begin(f.begin);
+ let seq = successor.initialSequence!;
+ const exporter = createNativeRunLogExporter({
+ db,
+ companyId: f.binding.companyId,
+ runId: f.binding.runId,
+ afterCursor: successor.initialSourceCursor!,
+ enqueue: async (sourceCursor, chunk) => {
+ const queued = await b.enqueue!(successor, {
+ stream: "stdout",
+ chunk,
+ ts: "now",
+ seq: ++seq,
+ sourceCursor,
+ });
+ void queued.committed.catch(() => undefined);
+ },
+ flush: () => b.flush!(successor),
+ });
+ const third = await f.port.appendEvent(events[2]!);
+ await exporter.record(third.cursor, formatNativeRunLogEvent(events[2]!));
+ await exporter.record(cursors[0]!.cursor, formatNativeRunLogEvent(events[0]!));
+ await b.finalize(successor);
+ const entries = (await b.read(successor)).content
+ .trim()
+ .split("\n")
+ .map((line) => JSON.parse(JSON.parse(line).chunk).event);
+ expect(entries.map((event) => event.sourceSeq)).toEqual([1, 2, 3]);
+ expect(
+ await db
+ .select()
+ .from(heartbeatRunEvents)
+ .where(eq(heartbeatRunEvents.runId, f.binding.runId)),
+ ).toHaveLength(3);
+ });
+ it("repairs a settled run on demand after its controller lease expires, without extending source retention", async () => {
+ const f = await fixture(),
+ a = f.store(),
+ h = await a.begin(f.begin);
+ await f.port.appendEvent(f.event(1));
+ await db.execute(
+ sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
+ );
+ await db.execute(
+ sql`insert into native_run_finalizations values(${f.binding.runId},${f.binding.companyId},now()+interval '1 minute')`,
+ );
+ const b = f.store();
+ await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
+ expect((await b.read(h)).content).toBe("");
+ await db.execute(
+ sql`update native_run_finalizations set lease_expires_at=now()-interval '1 second' where run_id=${f.binding.runId}`,
+ );
+ await Promise.all(
+ [1, 2].map(() =>
+ recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })),
+ ),
+ );
+ expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1);
+ const [run] = await db
+ .select({ bytes: heartbeatRuns.logBytes })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, f.binding.runId));
+ expect(run!.bytes).toBeGreaterThan(0);
+ });
+ it.each(["hash", "payload", "identity"])(
+ "rejects a corrupt canonical source %s without marking the partial archive complete",
+ async (kind) => {
+ const f = await fixture(),
+ a = f.store(),
+ h = await a.begin(f.begin);
+ await f.port.appendEvent(f.event(1));
+ if (kind === "hash")
+ await db.execute(
+ sql`update heartbeat_run_events set source_payload_sha256='wrong' where run_id=${f.binding.runId}`,
+ );
+ else if (kind === "payload")
+ await db.execute(
+ sql`update heartbeat_run_events set payload='{}'::jsonb where run_id=${f.binding.runId}`,
+ );
+ else
+ await db.execute(
+ sql`update heartbeat_run_events set source_instance_id='wrong' where run_id=${f.binding.runId}`,
+ );
+ await db.execute(
+ sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
+ );
+ await expect(
+ recoverSettledNativeRunLog(db, f.store(), h.logRef, async () => ({ enabled: false })),
+ ).rejects.toThrow("integrity");
+ const [run] = await db
+ .select({ bytes: heartbeatRuns.logBytes })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, f.binding.runId));
+ expect(run!.bytes).toBeNull();
+ },
+ );
+ it("ignores credential and retained cleanup projections when replaying canonical log sources", async () => {
+ const f = await fixture(),
+ a = f.store(),
+ h = await a.begin(f.begin);
+ await f.port.appendEvent(f.event(1));
+ const credential = (sourceSeq: number): PrpEvent => ({
+ ...f.event(sourceSeq),
+ schema: "paperclip.prp.event.v3",
+ schemaVersion: 3,
+ priority: 0,
+ eventType: "auth.credentials.updated",
+ payload: {
+ schema: "paperclip.auth.credentials.updated.v1",
+ provider: "codex",
+ bindingId: "test-auth",
+ revision: 1,
+ detection: "credential_store",
+ },
+ });
+ await f.port.appendEvent(credential(2));
+ await db.execute(sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef},
+ native_session_id=${f.binding.sessionId}, runner_instance_id=${f.binding.sourceInstanceId} where id=${f.binding.runId}`);
+ await appendNativeCredentialUpdate(db, {
+ companyId: f.binding.companyId,
+ agentId: f.binding.agentId,
+ normalizedSessionId: f.binding.sessionId,
+ runnerInstanceId: f.binding.sourceInstanceId,
+ event: credential(20),
+ });
+ await appendRetainedNativeCleanupEvent(db, {
+ companyId: f.binding.companyId,
+ agentId: f.binding.agentId,
+ runId: f.binding.runId,
+ nativeSessionId: f.binding.sessionId,
+ runnerInstanceId: f.binding.sourceInstanceId,
+ requestId: "test-cleanup",
+ event: f.event(21),
+ });
+ await appendHeartbeatRunEvent(db, {
+ companyId: f.binding.companyId,
+ agentId: f.binding.agentId,
+ runId: f.binding.runId,
+ eventType: "workspace.diff.recorded",
+ payload: { complete: true },
+ nativeSource: {
+ sourceInstanceId: "provider-trace-reproject:test",
+ sourceEventId: "provider-trace-reproject:test:change",
+ sourceSeq: 1,
+ protocolSchemaVersion: 1,
+ canonicalPayload: { complete: true },
+ },
+ });
+ const b = f.store();
+ await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
+ const entries = (await b.read(h)).content.trim().split("\n");
+ expect(entries).toHaveLength(2);
+ expect(JSON.parse(JSON.parse(entries[0]!).chunk).event.sourceSeq).toBe(1);
+ expect(JSON.parse(JSON.parse(entries[1]!).chunk).event.eventType).toBe(
+ "auth.credentials.updated",
+ );
+ const [run] = await db
+ .select({ bytes: heartbeatRuns.logBytes })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, f.binding.runId));
+ expect(run!.bytes).toBeGreaterThan(0);
+ });
+ it("aborts in-flight archive writes when its last reader cancels, then resumes without loss", async () => {
+ const f = await fixture(),
+ a = f.store(),
+ h = await a.begin(f.begin),
+ baseline = idleWorkSnapshot().active;
+ await f.port.appendEvent(f.event(1));
+ await db.execute(
+ sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
+ );
+ let entered = false,
+ aborted = false;
+ const blocked: StorageProvider = {
+ ...f.provider,
+ async putObjectConditional(input) {
+ if (input.objectKey.includes("/objects/")) {
+ entered = true;
+ await new Promise((_resolve, reject) =>
+ input.signal!.addEventListener(
+ "abort",
+ () => {
+ aborted = true;
+ reject(input.signal!.reason);
+ },
+ { once: true },
+ ),
+ );
+ }
+ return f.provider.putObjectConditional!(input);
+ },
+ };
+ const controller = new AbortController();
+ const repairing = recoverSettledNativeRunLog(
+ db,
+ f.store(blocked),
+ h.logRef,
+ async () => ({ enabled: false }),
+ controller.signal,
+ );
+ const rejected = expect(repairing).rejects.toThrow("reader cancelled");
+ const lastReader = new AbortController();
+ const joined = recoverSettledNativeRunLog(
+ db,
+ f.store(blocked),
+ h.logRef,
+ async () => ({ enabled: false }),
+ lastReader.signal,
+ );
+ const joinedRejected = expect(joined).rejects.toThrow("last reader cancelled");
+ await vi.waitFor(() => expect(entered).toBe(true));
+ controller.abort(new Error("reader cancelled"));
+ await rejected;
+ expect(aborted).toBe(false);
+ lastReader.abort(new Error("last reader cancelled"));
+ await joinedRejected;
+ await vi.waitFor(() => {
+ expect(aborted).toBe(true);
+ expect(idleWorkSnapshot().active).toBe(baseline);
+ });
+ const b = f.store();
+ await vi.waitFor(async () => {
+ await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
+ });
+ expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1);
+ });
+ it("redacts the existing agent identity and registered secrets before publishing recovered native output", async () => {
+ vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "e".repeat(64));
+ const f = await fixture();
+ const keys = generateKeyPairSync("ed25519"),
+ privateKeyPem = keys.privateKey.export({ type: "pkcs8", format: "pem" }).toString();
+ const publicKeyPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString();
+ const keyId = `sha256:${createHash("sha256")
+ .update(keys.publicKey.export({ type: "spki", format: "der" }))
+ .digest("base64url")}`;
+ const encrypted = await localEncryptedProvider.createSecret({ value: privateKeyPem });
+ await db.insert(agentIdentityKeys).values({
+ agentId: f.binding.agentId,
+ companyId: f.binding.companyId,
+ algorithm: "Ed25519",
+ keyId,
+ publicKeyPem,
+ privateKeyMaterial: encrypted.material,
+ });
+ const secret = "registered-native-fixture-secret";
+ const redactor = createAgentIdentityRedactor(privateKeyPem);
+ redactor.values.push(secret);
+ const run = {
+ id: f.binding.runId,
+ companyId: f.binding.companyId,
+ agentId: f.binding.agentId,
+ status: "running",
+ runtimeMode: "native",
+ logStore: null,
+ logRef: null,
+ logBytes: null,
+ lastOutputSeq: 0,
+ lastOutputAt: null,
+ } as typeof heartbeatRuns.$inferSelect;
+ const output = createHeartbeatRunOutput(db, {
+ run,
+ runId: run.id,
+ issueId: null,
+ runLogStore: f.store(),
+ identityRedactor: redactor,
+ secretKeys: new Set(),
+ runtimeLogSecretValues: [secret],
+ appendRunEvent: async () => undefined,
+ getCurrentUserRedactionOptions: async () => ({ enabled: false }),
+ });
+ await output.begin(run);
+ const h = output.handle!;
+ await f.port.appendEvent({ ...f.event(1), payload: { text: `${privateKeyPem}\n${secret}` } });
+ await db.execute(
+ sql`update heartbeat_runs set status='succeeded', log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`,
+ );
+ const b = f.store();
+ await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
+ const content = (await b.read(h)).content;
+ expect(content).not.toContain(privateKeyPem.split("\n")[1]!);
+ expect(content).not.toContain(secret);
+ expect(content).toContain("REDACTED");
+ const [snapshot] = await db
+ .select({ context: heartbeatRuns.contextSnapshot })
+ .from(heartbeatRuns)
+ .where(eq(heartbeatRuns.id, f.binding.runId));
+ expect(JSON.stringify(snapshot!.context)).not.toContain(privateKeyPem.split("\n")[1]!);
+ });
+ it("serves the published snapshot when another writer wins a repair claim", async () => {
+ const f = await fixture(),
+ a = f.store(),
+ h = await a.begin(f.begin);
+ const event = f.event(1),
+ receipt = await f.port.appendEvent(event);
+ const queued = await a.enqueue!(h, {
+ stream: "stdout",
+ chunk: formatNativeRunLogEvent(event),
+ ts: "then",
+ seq: 1,
+ sourceCursor: receipt.cursor,
+ });
+ await a.flush!(h);
+ await queued.committed;
+ await f.port.appendEvent(f.event(2));
+ await db.execute(
+ sql`update heartbeat_runs set status='succeeded',log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`,
+ );
+ let enter!: () => void, release!: () => void;
+ const entered = new Promise((resolve) => {
+ enter = resolve;
+ }),
+ blocked = new Promise((resolve) => {
+ release = resolve;
+ });
+ const store = {
+ ...f.store(),
+ async begin() {
+ enter();
+ await blocked;
+ throw new ObjectJournalConflictError();
+ },
+ };
+ const repairing = recoverSettledNativeRunLog(db, store, h.logRef, async () => ({
+ enabled: false,
+ }));
+ const rejected = expect(repairing).rejects.toBeInstanceOf(ObjectJournalConflictError);
+ await entered;
+ const read = readRecoverableRunLog(db, store, h);
+ release();
+ await rejected;
+ expect((await read).content.trim().split("\n")).toHaveLength(1);
+ await a.finalize(h);
+ });
+});
diff --git a/server/src/services/native-runtime/native-run-log-export.ts b/server/src/services/native-runtime/native-run-log-export.ts
new file mode 100644
index 00000000000..d8693c56d64
--- /dev/null
+++ b/server/src/services/native-runtime/native-run-log-export.ts
@@ -0,0 +1,210 @@
+import { and, asc, desc, eq, gt, isNotNull, lte, not, or, sql } from "drizzle-orm";
+import { StringDecoder } from "node:string_decoder";
+import { heartbeatRunEvents, type Db } from "@paperclipai/db";
+import { validatePrpEvent, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
+import { nativeSha256 } from "./canonical.js";
+
+export function formatNativeRunLogEvent(event: PrpEvent) {
+ return `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`;
+}
+export function nativeRunLogSourceWhere(companyId: string, runId: string) {
+ return and(
+ eq(heartbeatRunEvents.companyId, companyId),
+ eq(heartbeatRunEvents.runId, runId),
+ isNotNull(heartbeatRunEvents.sourceEventId),
+ not(
+ sql`coalesce((${or(
+ and(
+ eq(heartbeatRunEvents.eventType, "auth.credentials.updated"),
+ sql`${heartbeatRunEvents.sourceEventId} like 'credentials:%'`,
+ sql`${heartbeatRunEvents.sourceInstanceId} like '%:credentials'`,
+ ),
+ and(
+ eq(heartbeatRunEvents.eventType, "native.cleanup.event"),
+ sql`${heartbeatRunEvents.sourceEventId} like 'cleanup:%'`,
+ sql`${heartbeatRunEvents.sourceInstanceId} like '%:cleanup:%'`,
+ ),
+ and(
+ eq(heartbeatRunEvents.eventType, "workspace.diff.recorded"),
+ sql`${heartbeatRunEvents.sourceEventId} like 'provider-trace-reproject:%'`,
+ sql`${heartbeatRunEvents.sourceInstanceId} like 'provider-trace-reproject:%'`,
+ ),
+ )}), false)`,
+ ),
+ );
+}
+async function sourcePage(
+ db: Db,
+ companyId: string,
+ runId: string,
+ after: number,
+ through?: number,
+ signal?: AbortSignal,
+) {
+ signal?.throwIfAborted();
+ const rows = await db
+ .select({
+ seq: heartbeatRunEvents.seq,
+ payload: heartbeatRunEvents.payload,
+ sourceInstanceId: heartbeatRunEvents.sourceInstanceId,
+ sourceEventId: heartbeatRunEvents.sourceEventId,
+ sourceSeq: heartbeatRunEvents.sourceSeq,
+ sha256: heartbeatRunEvents.sourcePayloadSha256,
+ })
+ .from(heartbeatRunEvents)
+ .where(
+ and(
+ nativeRunLogSourceWhere(companyId, runId),
+ gt(heartbeatRunEvents.seq, after),
+ through === undefined ? undefined : lte(heartbeatRunEvents.seq, through),
+ ),
+ )
+ .orderBy(asc(heartbeatRunEvents.seq))
+ .limit(16);
+ signal?.throwIfAborted();
+ return rows.map((row) => {
+ const validated = validatePrpEvent(row.payload?.prpEvent);
+ if (
+ !validated.ok ||
+ validated.event.runId !== runId ||
+ validated.event.sourceInstanceId !== row.sourceInstanceId ||
+ validated.event.sourceEventId !== row.sourceEventId ||
+ validated.event.sourceSeq !== row.sourceSeq ||
+ nativeSha256(validated.event) !== row.sha256
+ )
+ throw new Error("native_run_log_source_integrity_error");
+ return { cursor: row.seq, event: validated.event };
+ });
+}
+
+export function createNativeRunLogCoverageVerifier(db: Db, companyId: string, runId: string) {
+ const decoder = new StringDecoder("utf8");
+ let text = "",
+ cursor = 0;
+ let page: Awaited> = [];
+ const unproven = () => new Error("native_log_source_coverage_unproven");
+ async function next() {
+ if (!page.length) page = await sourcePage(db, companyId, runId, cursor);
+ return page[0];
+ }
+ async function line(value: string) {
+ if (!value.trim()) return;
+ if (Buffer.byteLength(value) > 1024 * 1024) throw unproven();
+ let chunk: unknown;
+ try {
+ chunk = JSON.parse(value).chunk;
+ } catch {
+ throw unproven();
+ }
+ if (typeof chunk !== "string" || !chunk.startsWith('{"type":"paperclip.prp.event"')) return;
+ let event: PrpEvent;
+ try {
+ const parsed = validatePrpEvent(JSON.parse(chunk).event);
+ if (!parsed.ok) throw unproven();
+ event = parsed.event;
+ } catch {
+ throw unproven();
+ }
+ const expected = await next();
+ if (
+ !expected ||
+ event.runId !== runId ||
+ event.sourceInstanceId !== expected.event.sourceInstanceId ||
+ event.sourceEventId !== expected.event.sourceEventId ||
+ event.sourceSeq !== expected.event.sourceSeq ||
+ event.normalizedSessionId !== expected.event.normalizedSessionId ||
+ event.schemaVersion !== expected.event.schemaVersion ||
+ event.eventType !== expected.event.eventType
+ )
+ throw unproven();
+ cursor = expected.cursor;
+ page.shift();
+ }
+ return {
+ async write(bytes: Buffer) {
+ text += decoder.write(bytes);
+ let boundary: number;
+ while ((boundary = text.indexOf("\n")) !== -1) {
+ await line(text.slice(0, boundary));
+ text = text.slice(boundary + 1);
+ }
+ if (Buffer.byteLength(text) > 1024 * 1024) throw unproven();
+ },
+ async finish() {
+ text += decoder.end();
+ if (text) await line(text);
+ if (await next()) throw unproven();
+ return cursor;
+ },
+ };
+}
+
+export function createNativeRunLogExporter(input: {
+ db: Db;
+ companyId: string;
+ runId: string;
+ afterCursor: number;
+ enqueue: (cursor: number, chunk: string) => Promise;
+ flush: () => Promise;
+ signal?: AbortSignal;
+}) {
+ let cursor = input.afterCursor;
+ let queue = Promise.resolve();
+ let queuedBytes = 0,
+ queuedRecords = 0;
+ const sourceWhere = nativeRunLogSourceWhere(input.companyId, input.runId);
+ async function replay(through?: number) {
+ input.signal?.throwIfAborted();
+ if (through === undefined) {
+ const [last] = await input.db
+ .select({ seq: heartbeatRunEvents.seq })
+ .from(heartbeatRunEvents)
+ .where(sourceWhere)
+ .orderBy(desc(heartbeatRunEvents.seq))
+ .limit(1);
+ through = last?.seq ?? cursor;
+ }
+ while (cursor < through) {
+ const rows = await sourcePage(
+ input.db,
+ input.companyId,
+ input.runId,
+ cursor,
+ through,
+ input.signal,
+ );
+ if (!rows.length) break;
+ for (const row of rows) {
+ input.signal?.throwIfAborted();
+ await input.enqueue(row.cursor, formatNativeRunLogEvent(row.event));
+ cursor = row.cursor;
+ }
+ await input.flush();
+ }
+ }
+ return {
+ recover: () => replay(),
+ async record(sourceCursor: number, chunk: string) {
+ if (!Number.isSafeInteger(sourceCursor) || sourceCursor < 1)
+ throw new Error("invalid_native_run_log_cursor");
+ const bytes = Buffer.byteLength(chunk);
+ if (queuedRecords >= 32 || queuedBytes + bytes > 8 * 1024 * 1024)
+ throw new Error("native_run_log_admission_capacity_exceeded");
+ queuedRecords++;
+ queuedBytes += bytes;
+ const work = queue
+ .then(async () => {
+ if (sourceCursor <= cursor) return;
+ if (sourceCursor > cursor + 1) await replay(sourceCursor - 1);
+ await input.enqueue(sourceCursor, chunk);
+ cursor = sourceCursor;
+ })
+ .finally(() => {
+ queuedRecords--;
+ queuedBytes -= bytes;
+ });
+ queue = work.catch(() => undefined);
+ await work;
+ },
+ };
+}
diff --git a/server/src/services/native-runtime/native-run-log-recovery.ts b/server/src/services/native-runtime/native-run-log-recovery.ts
new file mode 100644
index 00000000000..b3fd0c36aaf
--- /dev/null
+++ b/server/src/services/native-runtime/native-run-log-recovery.ts
@@ -0,0 +1,223 @@
+import { and, eq, gt, inArray, isNull, sql } from "drizzle-orm";
+import {
+ heartbeatRuns,
+ heartbeatRunEvents,
+ nativeRunFinalizations,
+ type Db,
+} from "@paperclipai/db";
+import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../log-redaction.js";
+import type { RunLogHandle, RunLogReadOptions, RunLogStore } from "../run-log-store.js";
+import { instanceSettingsService } from "../instance-settings.js";
+import { compactRunLogChunk } from "../heartbeat/run-log.js";
+import { beginIdleTrackedWork } from "../task-admission.js";
+import { agentIdentityService } from "../agent-identity.js";
+import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
+import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
+import { logger } from "../../middleware/logger.js";
+import { createNativeRunLogExporter, nativeRunLogSourceWhere } from "./native-run-log-export.js";
+
+interface Recovery {
+ work: Promise;
+ controller: AbortController;
+ readers: number;
+}
+const recoveries = new WeakMap>();
+function awaitRecovery(recovery: Recovery, signal?: AbortSignal) {
+ signal?.throwIfAborted();
+ recovery.readers++;
+ return new Promise((resolve, reject) => {
+ let finished = false;
+ const finish = () => {
+ if (finished) return false;
+ finished = true;
+ signal?.removeEventListener("abort", abort);
+ recovery.readers--;
+ return true;
+ };
+ const abort = () => {
+ if (!finish()) return;
+ if (!recovery.readers) recovery.controller.abort(signal!.reason);
+ reject(signal!.reason);
+ };
+ signal?.addEventListener("abort", abort, { once: true });
+ recovery.work.then(
+ () => {
+ if (finish()) resolve();
+ },
+ (error) => {
+ if (finish()) reject(error);
+ },
+ );
+ });
+}
+
+export async function readRecoverableRunLog(
+ db: Db,
+ store: RunLogStore,
+ handle: RunLogHandle,
+ options?: RunLogReadOptions,
+ known?: { runtimeMode?: string | null; status?: string; logBytes?: number | null },
+) {
+ if (
+ handle.store === "s3_segments" &&
+ known?.runtimeMode !== "legacy" &&
+ known?.status !== "running" &&
+ (known?.logBytes === undefined || known.logBytes === null)
+ )
+ await recoverSettledNativeRunLog(db, store, handle.logRef, undefined, options?.signal).catch(
+ () => {
+ options?.signal?.throwIfAborted();
+ logger.warn(
+ { runId: handle.logRef.split("/").at(-1) },
+ "Native run log repair is incomplete; serving the published prefix",
+ );
+ },
+ );
+ return store.read(handle, options);
+}
+
+export async function recoverSettledNativeRunLog(
+ db: Db,
+ store: RunLogStore,
+ logRef: string,
+ getRedactionOptions: () => Promise = async () => ({
+ enabled: (await instanceSettingsService(db).getGeneral()).censorUsernameInLogs,
+ }),
+ requestSignal?: AbortSignal,
+) {
+ requestSignal?.throwIfAborted();
+ const binding = /^([a-f0-9-]+)\/([a-f0-9-]+)\/([a-f0-9-]+)$/.exec(logRef);
+ if (!binding) throw new Error("invalid_native_run_log_reference");
+ let pending = recoveries.get(db);
+ if (!pending) {
+ pending = new Map();
+ recoveries.set(db, pending);
+ }
+ const existing = pending.get(logRef);
+ if (existing) return awaitRecovery(existing, requestSignal);
+ const controller = new AbortController();
+ const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(30_000)]);
+ const finishWork = beginIdleTrackedWork();
+ const runId = binding[3]!,
+ companyId = binding[1]!;
+ const eligible = () =>
+ and(
+ eq(heartbeatRuns.id, runId),
+ eq(heartbeatRuns.companyId, companyId),
+ eq(heartbeatRuns.agentId, binding[2]!),
+ eq(heartbeatRuns.logStore, "s3_segments"),
+ eq(heartbeatRuns.logRef, logRef),
+ eq(heartbeatRuns.runtimeMode, "native"),
+ inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled", "timed_out"]),
+ isNull(heartbeatRuns.logBytes),
+ sql`not exists (select 1 from ${nativeRunFinalizations} where ${and(
+ eq(nativeRunFinalizations.runId, runId),
+ eq(nativeRunFinalizations.companyId, companyId),
+ gt(nativeRunFinalizations.leaseExpiresAt, new Date()),
+ )})`,
+ );
+ const recover = (async () => {
+ signal.throwIfAborted();
+ const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eligible());
+ signal.throwIfAborted();
+ if (!run) return;
+ const [redaction, identityRedactor, secrets] = await Promise.all([
+ getRedactionOptions(),
+ agentIdentityService(db).createExistingLogRedactor(companyId, binding[2]!),
+ createRunSecretRedactionRegistry(db).redactionValuesForRun(companyId, runId),
+ ]);
+ for (const value of secrets)
+ identityRedactor.values.push(...createAgentIdentityRedactor(value).values);
+ identityRedactor.values.sort((left, right) => right.length - left.length);
+ const handle = await store.begin({
+ companyId,
+ agentId: binding[2]!,
+ runId,
+ previousStore: "s3_segments",
+ signal,
+ assertWriterAuthority: async () => {
+ signal.throwIfAborted();
+ const [current] = await db
+ .select({ id: heartbeatRuns.id })
+ .from(heartbeatRuns)
+ .where(eligible());
+ if (!current) throw new Error("native_run_log_recovery_authority_lost");
+ },
+ });
+ let sequence = handle.initialSequence ?? 0;
+ let lastSourceCursor = handle.initialSourceCursor ?? 0;
+ const receipts: Promise[] = [];
+ let failure: unknown;
+ const exporter = createNativeRunLogExporter({
+ db,
+ companyId,
+ runId,
+ afterCursor: handle.initialSourceCursor ?? 0,
+ signal,
+ enqueue: async (sourceCursor, chunk) => {
+ const receipt = await store.enqueue!(handle, {
+ stream: "stdout",
+ ts: new Date().toISOString(),
+ seq: ++sequence,
+ sourceCursor,
+ chunk: compactRunLogChunk(
+ redactCurrentUserText(identityRedactor.chunk("stdout", chunk), redaction),
+ ),
+ });
+ receipts.push(
+ receipt.committed.catch((error) => {
+ failure ??= error;
+ return 0;
+ }),
+ );
+ lastSourceCursor = sourceCursor;
+ },
+ flush: async () => {
+ await store.flush!(handle);
+ await Promise.all(receipts.splice(0));
+ if (failure) throw failure;
+ },
+ });
+ try {
+ await exporter.recover();
+ } catch (error) {
+ failure ??= error;
+ } finally {
+ const summary = await store.finalize(handle, { computeDigest: false });
+ if (summary.bytes === null) failure ??= new Error("native_run_log_recovery_incomplete");
+ if (!failure && summary.bytes !== null) {
+ signal.throwIfAborted();
+ const updated = await db
+ .update(heartbeatRuns)
+ .set({
+ logBytes: summary.bytes,
+ logSha256: summary.sha256,
+ lastOutputSeq: sequence,
+ lastOutputBytes: summary.bytes,
+ })
+ .where(
+ and(
+ eligible(),
+ sql`not exists (select 1 from ${heartbeatRunEvents} where ${and(
+ nativeRunLogSourceWhere(companyId, runId),
+ gt(heartbeatRunEvents.seq, lastSourceCursor),
+ )})`,
+ ),
+ )
+ .returning({ id: heartbeatRuns.id });
+ if (!updated.length) throw new Error("native_run_log_recovery_source_changed");
+ }
+ }
+ if (failure) throw failure;
+ })();
+ const recovery: Recovery = {
+ controller,
+ readers: 0,
+ work: recover.finally(() => {
+ pending.delete(logRef);
+ finishWork();
+ }),
+ };
+ pending.set(logRef, recovery);
+ await awaitRecovery(recovery, requestSignal);
+}
diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts
index d04c1290a2b..3cedb542b5b 100644
--- a/server/src/services/native-runtime/native-session-executor.test.ts
+++ b/server/src/services/native-runtime/native-session-executor.test.ts
@@ -111,6 +111,10 @@ vi.mock("../usage-receipts.js", async (importOriginal) => ({
createRunUsageRecorder: usageRecorder.create,
}));
+vi.mock("./native-usage-delivery.js", () => ({
+ createNativeUsageDelivery: vi.fn(async () => ({ capture: async () => undefined })),
+}));
+
const githubAccess = vi.hoisted(() => ({
activate: vi.fn((_binding: { runId: string }) => vi.fn()),
stop: vi.fn(async () => undefined),
@@ -141,6 +145,7 @@ type BackendFactoryOptions = {
};
type RunnerTransportOptions = {
+ onAccountingEvent?: (event: PrpEvent) => Promise;
environment?: NodeJS.ProcessEnv;
acpxRuntimeDirectory?: string;
adoptExistingRunner?: { pid: number; isAlive: () => Promise | boolean };
@@ -250,8 +255,8 @@ vi.mock("./paperclip-control-plane-port.js", async importOriginal => {
return { ...original, PaperclipControlPlanePort: class extends original.PaperclipControlPlanePort {
constructor(...args: ConstructorParameters) {
super(...args);
- accountingEvents.committed = args[2]?.onCommittedEvent;
- accountingEvents.duplicate = args[2]?.onDuplicateEvent;
+ accountingEvents.committed = args[2]?.onCommittedEvent ? event => args[2]!.onCommittedEvent!(event, 1) : undefined;
+ accountingEvents.duplicate = args[2]?.onDuplicateEvent ? event => args[2]!.onDuplicateEvent!(event, 1) : undefined;
}
} };
});
@@ -7159,6 +7164,36 @@ describe("native terminal-turn accounting", () => {
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true }));
expect(usageRecorder.create).not.toHaveBeenCalled();
});
+ it("binds retained warm transport usage to the successor run authority", async () => {
+ const directory = await mkdtemp(join(tmpdir(), "native-warm-usage-"));
+ const previous = process.env.PAPERCLIP_RUNNER_STATE_DIR;
+ process.env.PAPERCLIP_RUNNER_STATE_DIR = directory;
+ const first = { ...execution, binding: { ...execution.binding, runId: "first-warm-usage", executionWorkspaceId: "warm-usage-workspace" },
+ session: { ...execution.session, normalizedSessionId: "warm-usage-session", lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60000 } } } as NativeExecutionInputV1;
+ const second = { ...first, binding: { ...first.binding, runId: "second-warm-usage" } };
+ const firstCapture = vi.fn(async (_event: PrpEvent) => {}), secondCapture = vi.fn(async (_event: PrpEvent) => {});
+ try {
+ await createRunnerdBackend({ db: leaseDb(first), execution: first, runnerInstanceId: "warm-producer", onAccountingEvent: firstCapture });
+ state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
+ const retainedHook = state.createTransport.mock.calls.at(-1)![0].onAccountingEvent!;
+ const firstReport = { eventType: "usage.reported", runId: first.binding.runId, sourceInstanceId: "warm-producer" } as PrpEvent;
+ await retainedHook(firstReport);
+ expect(firstCapture).toHaveBeenCalledExactlyOnceWith(firstReport);
+ await rm(state.createTransport.mock.calls.at(-1)![0].stateDirectory!, { recursive: true, force: true });
+ await createRunnerdBackend({ db: leaseDb(second), execution: second, runnerInstanceId: "warm-producer", onAccountingEvent: secondCapture });
+ await expect(retainedHook({ ...firstReport, eventType: "run.attached" })).resolves.toBeUndefined();
+ expect(secondCapture).not.toHaveBeenCalled();
+ const secondReport = { ...firstReport, runId: second.binding.runId };
+ await retainedHook(secondReport);
+ expect(secondCapture).toHaveBeenCalledExactlyOnceWith(secondReport);
+ expect(firstCapture).toHaveBeenCalledTimes(1);
+ await expect(retainedHook(firstReport)).rejects.toThrow("native_usage_run_binding_mismatch");
+ } finally {
+ if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR; else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous;
+ await rm(directory, { recursive: true, force: true });
+ }
+ });
+
it("uses the canonical recorder for partial and completed usage when no onUsage callback is supplied", async () => {
const updates: Array<{ table: unknown; values: Record }> = [];
state.execute.mockReset().mockImplementationOnce(async () => {
@@ -12367,7 +12402,14 @@ describe("runnerd provider runtime wiring", () => {
}
});
- it.each(["github", "assigned"] as const)("cleans run-owned access after overlapping executor failure: %s", async failure => {
+ it.each((["codex", "opencode"] as const).flatMap(provider =>
+ (["github", "assigned"] as const).map(failure => ({ provider, failure })),
+ ))("cleans run-owned access after overlapping $provider executor failure: $failure", async ({ provider, failure }) => {
+ const current = {
+ ...execution,
+ provider: provider === "codex" ? execution.provider : { kind: "opencode", model: null },
+ session: { ...execution.session, driverKind: provider === "codex" ? "codex_app_server" : "opencode_server" },
+ } as NativeExecutionInputV1;
const issueService = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({
update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })),
} as unknown as ReturnType);
@@ -12393,7 +12435,7 @@ describe("runnerd provider runtime wiring", () => {
state.createBackend.mockClear();
let settled = false;
const running = executePaperclipNativeSession({
- db: leaseDb(execution), execution, runnerInstanceId: "runner-overlap-cleanup",
+ db: leaseDb(current), execution: current, runnerInstanceId: "runner-overlap-cleanup",
useRunnerd: true, managedGitHub: true,
runnerEnvironment: {
PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned",
@@ -12419,7 +12461,7 @@ describe("runnerd provider runtime wiring", () => {
expect(state.execute).not.toHaveBeenCalled();
expect(state.createBackend).not.toHaveBeenCalled();
if (failure === "assigned") {
- expect(activate).toHaveBeenCalledWith(execution.binding);
+ expect(activate).toHaveBeenCalledWith(current.binding);
expect(deactivate).toHaveBeenCalledOnce();
expect(stop).toHaveBeenCalledOnce();
}
diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts
index 9cfd4f07a88..a8558bc09ed 100644
--- a/server/src/services/native-runtime/native-session-executor.ts
+++ b/server/src/services/native-runtime/native-session-executor.ts
@@ -1,4 +1,8 @@
+import { createNativeUsageDelivery } from "./native-usage-delivery.js";
+import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js";
+export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js";
import { measureRunnerOperation, recordRunnerTimingReceipt } from "./runner-operation-timing.js";
+import { formatNativeRunLogEvent } from "./native-run-log-export.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
import { releaseRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js";
@@ -107,7 +111,7 @@ import type {
AdapterExecutionResult,
AdapterRuntimeEvent,
} from "../../adapters/index.js";
-import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared";
+import type { NativeFinalizationResult } from "@paperclipai/shared";
import type {
DurablePrpControlPlane,
HarnessRuntimeRequestResolution,
@@ -1754,12 +1758,13 @@ export async function synchronizeCompletedProviderPlan(input: {
};
}
-class SessionToolAuthorityEpoch {
+class SessionAuthorityEpoch {
readonly runId: string;
#authority: PaperclipRunnerToolAuthority;
#revoked = false;
- constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace) {
+ constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace,
+ private readonly onAccountingEvent?: (event: PrpEvent) => Promise) {
this.runId = runId;
this.#authority = authority;
}
@@ -1775,6 +1780,14 @@ class SessionToolAuthorityEpoch {
}
}
+ async acceptAccountingEvent(event: PrpEvent) {
+ if (!isNativeUsageAccountingEvent(event)) return;
+ this.#assertCurrent();
+ if (event.runId !== this.runId) throw new Error("native_usage_run_binding_mismatch");
+ await this.onAccountingEvent?.(event);
+ this.#assertCurrent();
+ }
+
definitions() {
this.#assertCurrent();
return this.#authority.definitions();
@@ -1795,7 +1808,7 @@ class SessionToolAuthorityEpoch {
}
}
-const sessionToolAuthorityEpochs = new Map();
+const sessionToolAuthorityEpochs = new Map();
const initializingSessionToolAuthorities = new Set();
const executingRunnerdSessionScopes = new Map();
const executingNativeOwnerScopes = new Map();
@@ -7873,6 +7886,7 @@ export async function executePaperclipNativeSession(input: {
/** Internal, run-owned file inspection lifetime; never supplied by tool arguments. */
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise;
+ onPersistedNativeLog?: (cursor: number, chunk: string) => Promise;
onEvent?: (event: AdapterRuntimeEvent) => Promise;
/** Only this run's registered private instruction entry.
* Probe at terminal; persist only after owned shutdown. */
@@ -8618,7 +8632,15 @@ async function executePaperclipNativeSessionWithinScope(
for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent);
}
let fallbackUsageRecorder: ReturnType | undefined;
+ const producerOwnsUsage = usesNativeUsageDelivery({ runnerd: input.useRunnerd === true,
+ remote: input.runnerExecutionTarget?.kind === "remote", provider: input.execution.provider });
+ const producerUsage = producerOwnsUsage ? await createNativeUsageDelivery(input.db, {
+ companyId: input.execution.binding.companyId, runId: input.execution.binding.runId,
+ runnerInstanceId: effectiveRunnerInstanceId, provider: input.execution.provider,
+ billing: resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity),
+ }) : undefined;
const persistAccountingUsage = async (usage: Record | null, complete: boolean) => {
+ if (producerOwnsUsage) return;
const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" });
const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined;
complete = complete && accountingUsage !== undefined;
@@ -8721,7 +8743,7 @@ async function executePaperclipNativeSessionWithinScope(
attributes:{eventKind:event.eventType}});
}
},
- onCommittedEvent: async (event) => {
+ onCommittedEvent: async (event, cursor) => {
if (event.eventType === "run.result.proposed") {
const at = Date.now();
await trace.record({name:"completion.tool_accepted",parentName:"native.session.execute",startedAtMs:at,endedAtMs:at});
@@ -8874,7 +8896,8 @@ async function executePaperclipNativeSessionWithinScope(
});
trace.activate(taskSettleScope);
}
- if (input.onLog)
+ if (input.onPersistedNativeLog) await input.onPersistedNativeLog(cursor, formatNativeRunLogEvent(event));
+ else if (input.onLog)
await input.onLog(
"stdout",
`${JSON.stringify({ type: "paperclip.prp.event", event })}\n`,
@@ -8953,12 +8976,10 @@ async function executePaperclipNativeSessionWithinScope(
},
);
},
- onDuplicateEvent: async (event) => {
+ onDuplicateEvent: async (event, cursor) => {
+ await input.onPersistedNativeLog?.(cursor, formatNativeRunLogEvent(event));
if (!firstTokenRecorded) observeDeliverableText(event);
await observeAccountingEvent(event);
- // A crash can happen after the event commit but before its callback
- // finishes. Recover only idempotent durable projections here; activity,
- // publication, logging, trace, and metric effects remain committed-only.
await liveQuestions.observe(event);
await projectSessionGoalEvent(event);
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
@@ -9230,7 +9251,8 @@ async function executePaperclipNativeSessionWithinScope(
return input.runnerEnvironment;
};
const overlapGitHubPreparation = needsCoreGitHubAccess(input) && input.useRunnerd && input.backend === undefined
- && input.runnerExecutionTarget?.kind === "remote" && input.execution.provider.kind === "codex"
+ && input.runnerExecutionTarget?.kind === "remote"
+ && (input.execution.provider.kind === "codex" || input.execution.provider.kind === "opencode")
&& input.execution.schema !== "paperclip.native-execution-input.v6"
&& input.execution.schema !== "paperclip.native-execution-input.v8";
if (!overlapGitHubPreparation) await prepareRunnerEnvironment();
@@ -9243,6 +9265,7 @@ async function executePaperclipNativeSessionWithinScope(
input.useRunnerd && input.backend === undefined
? await timeController("backend_setup", () => createRunnerdBackend({
...input,
+ onAccountingEvent: producerUsage?.capture,
controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? {
bootId: controller.bootId,
runnerArtifact: restoredController ? { version: restoredController.runnerVersion, digest: restoredController.runnerDigest } : undefined,
@@ -9289,7 +9312,7 @@ async function executePaperclipNativeSessionWithinScope(
persistWarmNativeCheckpoint(remoteWarmRecovery.previousExecution,
nativeSessionConfigDigest(remoteWarmRecovery.previousExecution, "remote"), await recovered.session.snapshot());
persistedWarmSession = loadWarmNativeCheckpoint(input.execution, warmConfigDigest!, "remote");
- runnerdBackend = await createRunnerdBackend({ ...input, runnerInstanceId: effectiveRunnerInstanceId,
+ runnerdBackend = await createRunnerdBackend({ ...input, onAccountingEvent: producerUsage?.capture, runnerInstanceId: effectiveRunnerInstanceId,
controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? { bootId: controller.bootId } : undefined,
durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, trace, toolTrace });
}
@@ -10334,151 +10357,6 @@ async function executePaperclipNativeSessionWithinScope(
return adapterResult;
}
-function numericUsageField(
- usage: Record | null,
- keys: string[],
-): number | undefined {
- if (!usage) return undefined;
- for (const key of keys) {
- const value = usage[key];
- if (typeof value === "number" && Number.isFinite(value) && value >= 0)
- return value;
- }
- return undefined;
-}
-
-function nativeUsageMeasurement(usage: Record) {
- const nestedUsage = record(usage.usage);
- // An explicit partial delta must not fall through to a full session total.
- if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
- if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
- const candidates = [
- record(usage.runDelta),
- record(nestedUsage.runDelta),
- record(usage.total),
- record(nestedUsage.total),
- record(usage.cumulative),
- record(nestedUsage.cumulative),
- nestedUsage,
- usage,
- ];
- return (
- candidates.find(
- (candidate) =>
- numericUsageField(candidate, [
- "inputTokens",
- "input",
- "promptTokens",
- "outputTokens",
- "output",
- "completionTokens",
- ]) !== undefined,
- ) ?? usage
- );
-}
-
-export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string {
- if (provider.kind === "acpx") {
- if (provider.agent === "cursor") return "cursor";
- if (provider.agent === "copilot") return "github";
- if (provider.agent === "pi") return "openrouter";
- }
- return "openai";
-}
-
-function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
- const turns = new Map | null; terminal: boolean }>();
- const sequences = new Map();
- let currentTurn: string | undefined;
- const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
- const snapshot = () => {
- if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
- let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
- let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
- for (const [turnId, turn] of turns) {
- const normalized = normalizeNativeUsage(turn.usage);
- complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
- inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
- outputTokens += normalized?.outputTokens ?? 0;
- cacheReadTokens += normalized?.cachedInputTokens ?? 0;
- cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
- const price = nativeUsageCostUsd(turn.usage, provider);
- if (price === undefined) costIncomplete = true;
- else { cost += usdToUnits(price); costKnown = true; }
- }
- // A free empty attempt contributes no price evidence for later paid work.
- // Keep an incomplete zero subtotal absent so complete aggregate tokens can
- // still receive an estimate. Preserve positive known spend and true zero totals.
- const retainCost = costKnown && (!costIncomplete || cost > 0n);
- const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
- ...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
- const current = currentTurn ? turns.get(currentTurn) : undefined;
- return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
- ...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
- turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
- };
- return {
- observe(event: PrpEvent) {
- if (!event) return false;
- const payload = record(event.payload);
- const terminal = terminalTypes.includes(event.eventType);
- if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
- // Historical replay can revisit an earlier report after restoration.
- // Re-persist its aggregate, but never rewind or count that report twice.
- if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
- if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
- sequences.set(event.sourceInstanceId, event.sourceSeq);
- }
- const key = event.turnId ?? "unattributed";
- const turn = turns.get(key) ?? { usage: null, terminal: false };
- if (event.eventType === "turn.started") {
- currentTurn = event.turnId;
- turn.usage = null;
- turn.terminal = false;
- } else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
- if (payload.kind === "usage") {
- const usage = record(payload.usage);
- turn.usage = Object.hasOwn(usage, "runDelta")
- ? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
- }
- if (terminal) turn.terminal = true;
- turns.set(key, turn);
- return true;
- },
- snapshot,
- finish(turnId: string | null) {
- const turn = turnId ? turns.get(turnId) : undefined;
- if (turn) turn.terminal = true;
- return snapshot();
- },
- };
-}
-
-export function nativeUsageCostUsd(
- usage: Record | null,
- provider?: NativeExecutionInput["provider"],
-) {
- // The ACP normalization contract fills absent per-turn cost with zero and
- // reports actual cost cumulatively. Until it carries an authoritative run
- // delta with provenance, neither value is a candidate's billed USD receipt.
- if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
- if (!usage) return undefined;
- const measurement = nativeUsageMeasurement(usage);
- const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
- const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
- ?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
- // PRP currently defaults an absent providerCostUsd to zero. That zero is
- // not proof of a free request when the runtime observed token consumption.
- if (direct !== undefined && direct > 0) return direct;
- const normalized = normalizeNativeUsage(usage);
- if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
- const cost = record(hasRunDelta ? measurement.cost : usage.cost);
- const currency =
- typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
- if (currency !== "USD") return undefined;
- return numericUsageField(cost, ["amount", "total"]);
-}
-
/** Billing identity follows the selected runtime and resolved credentials.
* Unknown authentication remains unknown; a provider name does not prove billing mode. */
export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick): Pick {
@@ -10511,33 +10389,6 @@ export function resolveNativeBilling(provider: NativeExecutionInput["provider"],
return { provider: "openai", biller: billingType === "metered_api" ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType };
}
-export function normalizeNativeUsage(usage: Record | null, options: { inputIncludesCacheReads?: boolean } = {}) {
- if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
- const measurement = nativeUsageMeasurement(usage);
- const cache = record(measurement.cache);
- const count = (source: Record, keys: string[]) => {
- for (const key of keys) {
- if (source[key] === undefined) continue;
- const value = source[key];
- return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
- }
- return undefined;
- };
- const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
- const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
- const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
- const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
- if (input === undefined || outputTokens === undefined
- || ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
- const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
- + (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
- if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
- return {
- inputTokens, outputTokens,
- ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
- ...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
- };
-}
function processEnvironment(
environment: NodeJS.ProcessEnv,
@@ -11923,6 +11774,7 @@ export async function createRunnerdBackend(input: {
casesEnabled?: boolean;
db: Db;
controllerPersistence?: { bootId: string; runnerArtifact?: { version: string; digest: string } };
+ onAccountingEvent?: (event: PrpEvent) => Promise;
execution: NativeExecutionInput;
runnerInstanceId: string;
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
@@ -12238,13 +12090,14 @@ async function createRunnerdBackendWithinSessionClaim(
syncIssueExternalObjects: input.syncIssueExternalObjects,
enqueueWakeup: input.enqueueWakeup,
});
- const authorityEpoch = new SessionToolAuthorityEpoch(
+ const authorityEpoch = new SessionAuthorityEpoch(
input.execution.binding.runId,
authority,
input.toolTrace,
+ input.onAccountingEvent,
);
let dynamicTools: Awaited<
- ReturnType
+ ReturnType
>;
try {
dynamicTools = await timeController("tool_authority", () => authorityEpoch.definitions());
@@ -12346,7 +12199,7 @@ async function createRunnerdBackendWithinSessionClaim(
authority.binding.readRemoteTaskFile = file => owner.readWorkspaceFile(file.contentRef, file.byteSize);
dynamicTools = await authorityEpoch.definitions();
}
- const executeTool = (call: Parameters[0]) => {
+ const executeTool = (call: Parameters[0]) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
return current.execute(call);
@@ -12441,7 +12294,7 @@ async function createRunnerdBackendWithinSessionClaim(
};
const backendOptions = {
onSpawn: input.onSpawn, dynamicTools,
- dynamicToolHandler: async (call: Parameters[0]) => {
+ dynamicToolHandler: async (call: Parameters[0]) => {
await assertExternalNativeRunAuthority(input.db, externalExecution); return authorityEpoch.execute(call);
},
completionFeedback: async (result: PrpStructuredRunResult) => {
@@ -13782,7 +13635,7 @@ async function createRunnerdBackendWithinSessionClaim(
markSpawned?.();
}
const executeCurrentToolAuthority = (
- call: Parameters[0],
+ call: Parameters[0],
) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
@@ -13830,6 +13683,11 @@ async function createRunnerdBackendWithinSessionClaim(
onSpawn: input.onSpawn,
...(input.execution.provider.kind === "codex" ? { disableCodexWebSearch: "webSearch" in input.execution.provider && input.execution.provider.webSearch === "disabled" } : {}),
homeMode: preconfiguredHome ? "preconfigured" : "isolated",
+ onAccountingEvent: input.onAccountingEvent ? async event => {
+ const current = sessionToolAuthorityEpochs.get(sessionScopeId);
+ if (!current) throw new Error("native_session_authority_unavailable");
+ await current.acceptAccountingEvent(event);
+ } : undefined,
onCredentialsUpdated: (event) => appendNativeCredentialUpdate(input.db, {
companyId: input.execution.binding.companyId,
agentId: input.execution.binding.agentId,
diff --git a/server/src/services/native-runtime/native-usage-accounting.test.ts b/server/src/services/native-runtime/native-usage-accounting.test.ts
new file mode 100644
index 00000000000..7e40c08fd3f
--- /dev/null
+++ b/server/src/services/native-runtime/native-usage-accounting.test.ts
@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vitest";
+import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
+import { createNativeTurnAccounting, nativeUsageCostUsd, normalizeNativeUsage, usesNativeUsageDelivery } from "./native-usage-accounting.js";
+
+const provider = { kind: "opencode" as const, model: "test-model" };
+const event = (sourceSeq: number, eventType: string, turnId = "turn-1", payload: Record