diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 8a2c2831316..49b12390240 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -453,6 +453,9 @@ jobs: packages/paperclip-runner/runner/target/release/deps/paperclip_runnerd-* packages/paperclip-runner/runner/target/release/paperclip-runnerd packages/paperclip-runner/runner/target/release/paperclip-runnerd.d + packages/paperclip-runner/runner/target/release/.fingerprint/paperclip-process-table-* + packages/paperclip-runner/runner/target/release/deps/libpaperclip_process_table-* + packages/paperclip-runner/runner/target/release/deps/paperclip_process_table-*.d key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/acpx-profiles.json', 'packages/paperclip-runner/antigravity-distributions.json', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} - name: Mark the restored Runner release crate current @@ -1007,6 +1010,9 @@ jobs: packages/paperclip-runner/runner/target/release/deps/paperclip_runnerd-* packages/paperclip-runner/runner/target/release/paperclip-runnerd packages/paperclip-runner/runner/target/release/paperclip-runnerd.d + packages/paperclip-runner/runner/target/release/.fingerprint/paperclip-process-table-* + packages/paperclip-runner/runner/target/release/deps/libpaperclip_process_table-* + packages/paperclip-runner/runner/target/release/deps/paperclip_process_table-*.d key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/acpx-profiles.json', 'packages/paperclip-runner/antigravity-distributions.json', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} - name: Mark the restored Runner release crate current @@ -1251,6 +1257,9 @@ jobs: packages/paperclip-runner/runner/target/release/deps/paperclip_runnerd-* packages/paperclip-runner/runner/target/release/paperclip-runnerd packages/paperclip-runner/runner/target/release/paperclip-runnerd.d + packages/paperclip-runner/runner/target/release/.fingerprint/paperclip-process-table-* + packages/paperclip-runner/runner/target/release/deps/libpaperclip_process_table-* + packages/paperclip-runner/runner/target/release/deps/paperclip_process_table-*.d key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/acpx-profiles.json', 'packages/paperclip-runner/antigravity-distributions.json', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} - name: Mark the restored Runner release crate current @@ -1592,6 +1601,9 @@ jobs: packages/paperclip-runner/runner/target/release/deps/paperclip_runnerd-* packages/paperclip-runner/runner/target/release/paperclip-runnerd packages/paperclip-runner/runner/target/release/paperclip-runnerd.d + packages/paperclip-runner/runner/target/release/.fingerprint/paperclip-process-table-* + packages/paperclip-runner/runner/target/release/deps/libpaperclip_process_table-* + packages/paperclip-runner/runner/target/release/deps/paperclip_process_table-*.d key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/acpx-profiles.json', 'packages/paperclip-runner/antigravity-distributions.json', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} - name: Mark the restored Runner release crate current diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index b0319840ec1..99c551d0122 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -450,7 +450,7 @@ jobs: - name: Clean debug test variants before publishing workspace binaries if: ${{ matrix.lane == 'rust' && github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha && steps.runner_debug_crate.outputs.cache-hit != 'true' }} working-directory: packages/paperclip-runner - run: cargo clean --profile dev --manifest-path runner/Cargo.toml -p paperclip-runner-core + run: cargo clean --profile dev --manifest-path runner/Cargo.toml -p paperclip-runner-core -p paperclip-process-table - name: Warm debug dependencies for the shared Runner cache # Protocol tests need debug binaries. Populate their dependencies in @@ -497,6 +497,9 @@ jobs: packages/paperclip-runner/runner/target/release/deps/paperclip_runnerd-* packages/paperclip-runner/runner/target/release/paperclip-runnerd packages/paperclip-runner/runner/target/release/paperclip-runnerd.d + packages/paperclip-runner/runner/target/release/.fingerprint/paperclip-process-table-* + packages/paperclip-runner/runner/target/release/deps/libpaperclip_process_table-* + packages/paperclip-runner/runner/target/release/deps/paperclip_process_table-*.d key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/acpx-profiles.json', 'packages/paperclip-runner/antigravity-distributions.json', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} - name: Build the Runner release crate for its cache @@ -518,6 +521,9 @@ jobs: packages/paperclip-runner/runner/target/release/deps/paperclip_runnerd-* packages/paperclip-runner/runner/target/release/paperclip-runnerd packages/paperclip-runner/runner/target/release/paperclip-runnerd.d + packages/paperclip-runner/runner/target/release/.fingerprint/paperclip-process-table-* + packages/paperclip-runner/runner/target/release/deps/libpaperclip_process_table-* + packages/paperclip-runner/runner/target/release/deps/paperclip_process_table-*.d key: runner-release-crate-v1-${{ steps.runner_release_crate_key.outputs.scope }}-${{ hashFiles('packages/paperclip-runner/rust-toolchain.toml', 'packages/paperclip-runner/acpx-profiles.json', 'packages/paperclip-runner/antigravity-distributions.json', 'packages/paperclip-runner/runner/Cargo.toml', 'packages/paperclip-runner/runner/Cargo.lock', 'packages/paperclip-runner/runner/crates/**', 'packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/test-fixtures/**', 'packages/paperclip-runner/test/fixtures/**', 'tests/runner-e2e/fixtures/**') }} build: diff --git a/packages/paperclip-runner/runner/Cargo.lock b/packages/paperclip-runner/runner/Cargo.lock index 809b81962db..9d4d7de39e1 100644 --- a/packages/paperclip-runner/runner/Cargo.lock +++ b/packages/paperclip-runner/runner/Cargo.lock @@ -1820,6 +1820,13 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "paperclip-process-table" +version = "0.0.0" +dependencies = [ + "libc", +] + [[package]] name = "paperclip-runner-core" version = "0.0.0" @@ -1838,6 +1845,7 @@ dependencies = [ "notify", "num-bigint", "num-traits", + "paperclip-process-table", "reqwest", "rustix", "rustls", diff --git a/packages/paperclip-runner/runner/Cargo.toml b/packages/paperclip-runner/runner/Cargo.toml index 40b0ac1b1c4..05e26ff4236 100644 --- a/packages/paperclip-runner/runner/Cargo.toml +++ b/packages/paperclip-runner/runner/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/runner-core"] +members = ["crates/process-table", "crates/runner-core"] resolver = "2" [profile.dev] diff --git a/packages/paperclip-runner/runner/crates/process-table/Cargo.toml b/packages/paperclip-runner/runner/crates/process-table/Cargo.toml new file mode 100644 index 00000000000..74a21e27759 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/process-table/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "paperclip-process-table" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2" diff --git a/packages/paperclip-runner/runner/crates/process-table/src/lib.rs b/packages/paperclip-runner/runner/crates/process-table/src/lib.rs new file mode 100644 index 00000000000..99d3af9d337 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/process-table/src/lib.rs @@ -0,0 +1,243 @@ +//! Reads the host process table: PID, parent, process group and start time. +//! +//! `paperclip-runner-core` forbids unsafe code. macOS exposes this table only +//! through libproc, so the two libproc calls live here, each checked against +//! the size the kernel reports. Linux reads `/proc` without unsafe code. + +#![deny(unsafe_op_in_unsafe_fn)] + +/// One process as the kernel reported it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ProcessEntry { + pub pid: i32, + pub ppid: i32, + pub pgid: i32, + /// Start time in a platform unit (macOS: microseconds since the epoch; + /// Linux: clock ticks since boot). Compare only with the same host. + pub started: u64, + pub zombie: bool, +} + +/// One read of the whole table. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Snapshot { + pub entries: Vec, + /// Listed processes that still exist but whose details could not be + /// read. A caller must not treat them as exited. + pub unreadable: Vec, +} + +/// The result of reading one process. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Read { + Present(ProcessEntry), + /// The process does not exist. + Gone, + /// The process may exist, but its details could not be read. + Unreadable, +} + +impl Read { + pub fn entry(self) -> Option { + match self { + Self::Present(entry) => Some(entry), + Self::Gone | Self::Unreadable => None, + } + } +} + +/// Lists every process. `None` when the table cannot be read at all, or on +/// an unsupported platform. +pub fn list() -> Option { + let mut snapshot = Snapshot::default(); + for pid in platform::pids()? { + match read(pid) { + Read::Present(entry) => snapshot.entries.push(entry), + Read::Gone => {} + Read::Unreadable => snapshot.unreadable.push(pid), + } + } + Some(snapshot) +} + +/// Reads one process. +pub fn read(pid: i32) -> Read { + if pid <= 0 { + return Read::Gone; + } + platform::read(pid) +} + +#[cfg(target_os = "macos")] +mod platform { + use super::{ProcessEntry, Read}; + use std::mem::{size_of, MaybeUninit}; + + /// `SZOMB` from . + const SZOMB: u32 = 5; + /// Room for processes started between counting and listing. + const LIST_HEADROOM: usize = 256; + + pub(super) fn pids() -> Option> { + // SAFETY: a null buffer of size zero asks only for the PID count. + let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + let capacity = usize::try_from(count).ok()?.checked_add(LIST_HEADROOM)?; + let mut pids: Vec = vec![0; capacity]; + let bytes = i32::try_from(capacity.checked_mul(size_of::())?).ok()?; + // SAFETY: `pids` owns `bytes` writable bytes for the call's duration. + let listed = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), bytes) }; + let listed = usize::try_from(listed).ok()?; + if listed == 0 { + return None; + } + pids.truncate(listed.min(capacity)); + Some(pids.into_iter().filter(|pid| *pid > 0).collect()) + } + + pub(super) fn read(pid: i32) -> Read { + let mut info = MaybeUninit::::zeroed(); + let Ok(size) = i32::try_from(size_of::()) else { + return Read::Unreadable; + }; + // SAFETY: `info` is a zeroed `proc_bsdinfo` of exactly `size` bytes. + let read = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if read != size { + // A short read with ESRCH means the process is gone. Anything + // else leaves it unresolved. + let error = std::io::Error::last_os_error(); + return if read <= 0 && error.raw_os_error() == Some(libc::ESRCH) { + Read::Gone + } else { + Read::Unreadable + }; + } + // SAFETY: the kernel filled all `size` bytes, and every field is a + // plain integer or byte array for which any bit pattern is valid. + let info = unsafe { info.assume_init() }; + let (Ok(ppid), Ok(pgid)) = (i32::try_from(info.pbi_ppid), i32::try_from(info.pbi_pgid)) + else { + return Read::Unreadable; + }; + Read::Present(ProcessEntry { + pid, + ppid, + pgid, + started: info + .pbi_start_tvsec + .saturating_mul(1_000_000) + .saturating_add(info.pbi_start_tvusec), + zombie: info.pbi_status == SZOMB, + }) + } +} + +#[cfg(target_os = "linux")] +mod platform { + use super::{ProcessEntry, Read}; + + pub(super) fn pids() -> Option> { + let entries = std::fs::read_dir("/proc").ok()?; + Some( + entries + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().to_str()?.parse::().ok()) + .collect(), + ) + } + + pub(super) fn read(pid: i32) -> Read { + match std::fs::read_to_string(format!("/proc/{pid}/stat")) { + Ok(stat) => parse_stat(pid, &stat).map_or(Read::Unreadable, Read::Present), + // The process exited between listing and reading. + Err(error) + if error.kind() == std::io::ErrorKind::NotFound + || error.raw_os_error() == Some(3) => + { + Read::Gone + } + Err(_) => Read::Unreadable, + } + } + + /// `/proc//stat` fields after the parenthesized command name, which + /// may itself contain spaces and parentheses. + pub(super) fn parse_stat(pid: i32, stat: &str) -> Option { + let rest = &stat[stat.rfind(')')? + 1..]; + let fields: Vec<&str> = rest.split_whitespace().collect(); + let state = *fields.first()?; + Some(ProcessEntry { + pid, + ppid: fields.get(1)?.parse().ok()?, + pgid: fields.get(2)?.parse().ok()?, + started: fields.get(19)?.parse().ok()?, + zombie: state == "Z" || state == "X", + }) + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +mod platform { + use super::Read; + + pub(super) fn pids() -> Option> { + None + } + + pub(super) fn read(_pid: i32) -> Read { + Read::Unreadable + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn lists_this_process_with_its_parent_and_group() { + let me = i32::try_from(std::process::id()).unwrap(); + let listed = list().expect("process table"); + let entry = listed + .entries + .iter() + .find(|entry| entry.pid == me) + .expect("self"); + assert_eq!(Read::Present(*entry), read(me)); + assert!(entry.ppid > 0); + assert!(entry.pgid > 0); + assert!(entry.started > 0); + assert!(!entry.zombie); + } + + #[cfg(any(target_os = "linux", target_os = "macos"))] + #[test] + fn reports_an_exited_process_as_gone() { + let mut child = std::process::Command::new("/bin/sh") + .arg("-c") + .arg("exit 0") + .spawn() + .unwrap(); + let pid = i32::try_from(child.id()).unwrap(); + child.wait().unwrap(); + assert_eq!(read(pid), Read::Gone); + } + + #[cfg(target_os = "linux")] + #[test] + fn parses_linux_stat_with_spaces_and_parentheses_in_the_name() { + let stat = "4242 (a (b) c) S 1 4200 4200 0 -1 4194560 0 0 0 0 0 0 0 0 20 0 1 0 987654 0 0"; + let parsed = platform::parse_stat(4242, stat).unwrap(); + assert_eq!( + (parsed.ppid, parsed.pgid, parsed.started, parsed.zombie), + (1, 4200, 987654, false) + ); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/Cargo.toml b/packages/paperclip-runner/runner/crates/runner-core/Cargo.toml index 08ded319dba..6a8771d229a 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/Cargo.toml +++ b/packages/paperclip-runner/runner/crates/runner-core/Cargo.toml @@ -19,6 +19,7 @@ hmac.workspace = true jsonschema.workspace = true num-bigint.workspace = true num-traits.workspace = true +paperclip-process-table = { path = "../process-table" } notify.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs index 554713259a1..3e7d16b9f32 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs @@ -31,6 +31,8 @@ pub mod managed_provider; pub mod managed_provider_backend; pub mod native_provider_backend; pub mod oom_score; +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod process_lineage; pub mod process_supervisor; pub mod provider_backend; pub mod provider_bridge; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_lineage.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_lineage.rs new file mode 100644 index 00000000000..04893cfb35a --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_lineage.rs @@ -0,0 +1,511 @@ +//! Tracks every process a supervised process starts, including processes that +//! leave its process group. +//! +//! A supervised process runs in its own process group, and signalling that +//! group stops it and the children that stay in the group. Provider tools do +//! not stay there. Claude Code, for example, starts each Bash tool command in a +//! new session (`setsid`). A tool command such as `nohup ./job.sh &` then +//! leaves a job in that session whose wrapper has already exited, so the job's +//! parent is init. Neither signalling the owned group nor walking the parent +//! chain at teardown finds it. +//! +//! The tracker records identity while the lineage is still visible. It samples +//! the process table and records each descendant by PID and start time. When a +//! descendant leads a new process group or session, it records that group or +//! session with the leader's start time. Later, a process belongs to the +//! lineage when its parent belongs, or when it is in a recorded group or +//! session and started no earlier than that group's leader. Membership never +//! depends on a command name or path. A process that inherits the supervised +//! process's own session or an ancestor's group is never claimed through that +//! session or group, because only groups and sessions that a recorded +//! descendant leads are recorded. + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +/// How often the background sampler reads the process table. A tool wrapper +/// must live at least this long for the tracker to record its session. Claude +/// Code's Bash wrapper sources a shell snapshot first and lives far longer. +const SAMPLE_INTERVAL: Duration = Duration::from_millis(50); + +pub(crate) use paperclip_process_table::ProcessEntry; + +/// One read of the process table. Session IDs are read only on demand. +pub(crate) trait ProcessTable { + fn entries(&self) -> &[ProcessEntry]; + /// Processes that exist but whose details could not be read. On macOS + /// these are other users' processes; a provider's tools run as the + /// runner's user, so only a recorded member here is a concern. + fn unreadable(&self) -> &[i32] { + &[] + } + fn session_of(&self, pid: i32) -> Option; +} + +struct LiveProcessTable { + snapshot: paperclip_process_table::Snapshot, +} + +impl ProcessTable for LiveProcessTable { + fn entries(&self) -> &[ProcessEntry] { + &self.snapshot.entries + } + + fn unreadable(&self) -> &[i32] { + &self.snapshot.unreadable + } + + fn session_of(&self, pid: i32) -> Option { + let pid = rustix::process::Pid::from_raw(pid)?; + rustix::process::getsid(Some(pid)) + .ok() + .map(|sid| sid.as_raw_nonzero().get()) + } +} + +/// The lineage as one table read saw it. +#[derive(Debug, Default)] +pub(crate) struct Observation { + /// Live members, with the identity that read reported. + pub live: Vec, + /// Members that still exist but could not be read. Their exit is not + /// proven, so teardown cannot succeed while any remain. + pub unresolved: Vec, +} + +impl Observation { + pub(crate) fn is_empty(&self) -> bool { + self.live.is_empty() && self.unresolved.is_empty() + } +} + +#[derive(Default)] +pub(crate) struct LineageState { + root: Option<(i32, u64)>, + /// Live processes known to belong to the lineage, by PID and start time. + members: HashMap, + /// Groups led by a recorded member, keyed by ID, with the leader's start. + groups: HashMap, + /// Sessions led by a recorded member, keyed by ID, with the leader's start. + sessions: HashMap, +} + +impl LineageState { + pub(crate) fn new(root_pid: i32) -> Self { + Self { + root: Some((root_pid, 0)), + ..Self::default() + } + } + + /// Updates membership from one table read and returns the live members. + /// Callers must apply reads in the order they were taken; an older read + /// would forget members that a newer one recorded. + pub(crate) fn observe(&mut self, table: &dyn ProcessTable) -> Observation { + let entries = table.entries(); + let by_pid: HashMap = entries.iter().map(|e| (e.pid, e)).collect(); + let unreadable: HashSet = table.unreadable().iter().copied().collect(); + + // Resolve the root's start time on first sight. A root that is gone + // contributes nothing further; its recorded lineage still does. + if let Some((root_pid, root_started)) = self.root { + match by_pid.get(&root_pid) { + Some(entry) if root_started == 0 => { + self.root = Some((root_pid, entry.started)); + self.members.insert(root_pid, entry.started); + } + Some(entry) if entry.started == root_started && !entry.zombie => { + self.members.insert(root_pid, root_started); + } + _ => {} + } + } + + // Forget members that exited, and groups or sessions whose leader PID + // now names a different process. A group or session ID cannot be + // reused while any process is still in it, so a new process with the + // leader's PID means the recorded one is gone. A member that could + // not be read is kept: its exit is not established. + self.members.retain(|pid, started| match by_pid.get(pid) { + Some(e) => e.started == *started && !e.zombie, + None => unreadable.contains(pid), + }); + self.groups + .retain(|id, started| by_pid.get(id).is_none_or(|e| e.started == *started)); + self.sessions + .retain(|id, started| by_pid.get(id).is_none_or(|e| e.started == *started)); + + // Children are listed under their parents, and a recorded group or + // session can admit more members, so iterate until nothing changes. + let mut sessions_cache: HashMap> = HashMap::new(); + let mut session_of = |pid: i32| -> Option { + *sessions_cache + .entry(pid) + .or_insert_with(|| table.session_of(pid)) + }; + loop { + let mut changed = false; + for entry in entries { + if entry.zombie || self.members.contains_key(&entry.pid) { + continue; + } + let parent_is_member = self + .members + .get(&entry.ppid) + .is_some_and(|parent_started| *parent_started <= entry.started); + let in_group = self + .groups + .get(&entry.pgid) + .is_some_and(|leader_started| *leader_started <= entry.started); + let in_session = !parent_is_member + && !in_group + && !self.sessions.is_empty() + && session_of(entry.pid).is_some_and(|sid| { + self.sessions + .get(&sid) + .is_some_and(|leader_started| *leader_started <= entry.started) + }); + if parent_is_member || in_group || in_session { + self.members.insert(entry.pid, entry.started); + changed = true; + } + } + // Record groups and sessions that a member leads. A member that + // was seen before its setsid/setpgid is re-checked on every read. + for entry in entries { + let Some(started) = self.members.get(&entry.pid).copied() else { + continue; + }; + if entry.pgid == entry.pid && !self.groups.contains_key(&entry.pid) { + self.groups.insert(entry.pid, started); + changed = true; + } + if !self.sessions.contains_key(&entry.pid) + && session_of(entry.pid) == Some(entry.pid) + { + self.sessions.insert(entry.pid, started); + changed = true; + } + } + if !changed { + break; + } + } + + let mut unresolved: Vec = self + .members + .keys() + .copied() + .filter(|pid| !by_pid.contains_key(pid)) + .collect(); + unresolved.sort_unstable(); + Observation { + live: entries + .iter() + .filter(|e| !e.zombie && self.members.get(&e.pid) == Some(&e.started)) + .copied() + .collect(), + unresolved, + } + } +} + +fn read_process_table() -> Option { + paperclip_process_table::list().map(|snapshot| LiveProcessTable { snapshot }) +} + +/// Reads the table while holding the state, so reads are applied in the +/// order they were taken. `None` when the table could not be read. +fn observe_now(state: &Mutex) -> Option { + let mut state = state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let table = read_process_table()?; + Some(state.observe(&table)) +} + +/// Records a supervised process's lineage from spawn until it is reaped. +pub(crate) struct LineageTracker { + state: Arc>, + stop: Arc, + sampler: Option>, +} + +impl LineageTracker { + /// Starts recording. Fails when the sampler cannot run, so a caller never + /// supervises a process whose tools it could not stop. + pub(crate) fn start(root_pid: u32) -> Result { + let root_pid = i32::try_from(root_pid) + .map_err(|_| format!("process ID {root_pid} is out of range"))?; + let state = Arc::new(Mutex::new(LineageState::new(root_pid))); + let stop = Arc::new(AtomicBool::new(false)); + let sampler_state = Arc::clone(&state); + let sampler_stop = Arc::clone(&stop); + let sampler = thread::Builder::new() + .name("process-lineage".to_owned()) + .spawn(move || { + while !sampler_stop.load(Ordering::Acquire) { + let _ = observe_now(&sampler_state); + thread::park_timeout(SAMPLE_INTERVAL); + } + }) + .map_err(|error| format!("the process lineage sampler could not start: {error}"))?; + Ok(Self { + state, + stop, + sampler: Some(sampler), + }) + } + + /// Reads the table once now, so lineage visible at this moment is kept. + pub(crate) fn sample(&self) { + let _ = observe_now(&self.state); + } + + /// Stops every live process in the lineage and proves it is gone. + /// + /// Each pass re-reads the table and signals only processes whose PID and + /// start time still match a recorded member. The first passes send TERM; + /// after `grace`, they send KILL. Fails with the members still alive at + /// the deadline, or when the table could not be read to prove that none + /// remain. + pub(crate) fn terminate(&self, grace: Duration, deadline: Duration) -> Result<(), String> { + let started = Instant::now(); + let mut terminated: HashSet<(i32, u64)> = HashSet::new(); + loop { + let elapsed = started.elapsed(); + let Some(observation) = observe_now(&self.state) else { + if elapsed >= deadline { + return Err("the process table could not be read".to_owned()); + } + thread::sleep(Duration::from_millis(10)); + continue; + }; + if observation.is_empty() { + return Ok(()); + } + if elapsed >= deadline { + let alive = observation + .live + .iter() + .map(|entry| entry.pid.to_string()) + .collect::>() + .join(","); + let unresolved = observation + .unresolved + .iter() + .map(i32::to_string) + .collect::>() + .join(","); + return Err(format!( + "still alive: [{alive}]; unreadable: [{unresolved}]" + )); + } + for entry in &observation.live { + let first_signal = terminated.insert((entry.pid, entry.started)); + if elapsed >= grace { + signal_member(entry, rustix::process::Signal::KILL); + } else if first_signal { + signal_member(entry, rustix::process::Signal::TERM); + } + } + thread::sleep(Duration::from_millis(10)); + } + } + + fn stop_sampler(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + sampler.thread().unpark(); + let _ = sampler.join(); + } + } +} + +impl Drop for LineageTracker { + fn drop(&mut self) { + self.stop_sampler(); + } +} + +/// Signals one member after re-checking that its PID still names the same +/// process. The re-check and the signal are not atomic, but a PID is not +/// reused within the few microseconds between them in practice. +fn signal_member(entry: &ProcessEntry, signal: rustix::process::Signal) { + let Some(current) = paperclip_process_table::read(entry.pid).entry() else { + return; + }; + if current.started != entry.started || current.zombie { + return; + } + if let Some(pid) = rustix::process::Pid::from_raw(entry.pid) { + let _ = rustix::process::kill_process(pid, signal); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct FixedTable { + entries: Vec, + unreadable: Vec, + sessions: HashMap, + } + + impl ProcessTable for FixedTable { + fn entries(&self) -> &[ProcessEntry] { + &self.entries + } + fn unreadable(&self) -> &[i32] { + &self.unreadable + } + fn session_of(&self, pid: i32) -> Option { + self.sessions.get(&pid).copied() + } + } + + fn entry(pid: i32, ppid: i32, pgid: i32, started: u64) -> ProcessEntry { + ProcessEntry { + pid, + ppid, + pgid, + started, + zombie: false, + } + } + + fn table(entries: Vec<(ProcessEntry, i32)>) -> FixedTable { + FixedTable { + unreadable: Vec::new(), + sessions: entries.iter().map(|(e, sid)| (e.pid, *sid)).collect(), + entries: entries.into_iter().map(|(e, _)| e).collect(), + } + } + + impl LineageState { + fn observe_live(&mut self, table: &FixedTable) -> Vec { + self.observe(table).live + } + } + + fn pids(entries: &[ProcessEntry]) -> Vec { + let mut pids: Vec = entries.iter().map(|e| e.pid).collect(); + pids.sort_unstable(); + pids + } + + #[test] + fn keeps_a_reparented_job_from_a_recorded_tool_session() { + // runnerd 10 (session 1) supervises sidecar 100 (group 100). The + // sidecar starts a tool wrapper 200 in a new session, which starts + // job 201 and exits. The job is then a child of init. + let mut state = LineageState::new(100); + let before = table(vec![ + (entry(1, 0, 1, 1), 1), + (entry(10, 1, 10, 5), 1), + (entry(100, 10, 100, 10), 1), + (entry(200, 100, 200, 20), 200), + (entry(201, 200, 200, 21), 200), + ]); + assert_eq!(pids(&state.observe_live(&before)), vec![100, 200, 201]); + + let after = table(vec![ + (entry(1, 0, 1, 1), 1), + (entry(10, 1, 10, 5), 1), + (entry(100, 10, 100, 10), 1), + (entry(201, 1, 200, 21), 200), + (entry(202, 201, 200, 30), 200), + ]); + assert_eq!(pids(&state.observe_live(&after)), vec![100, 201, 202]); + } + + #[test] + fn finds_a_job_that_left_the_group_through_its_recorded_session() { + let mut state = LineageState::new(100); + state.observe(&table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(200, 100, 200, 20), 200), + ])); + // The wrapper has exited and its job made its own group, but the job + // is still in the wrapper's session. + let after = table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(300, 1, 300, 25), 200), + ]); + assert_eq!(pids(&state.observe_live(&after)), vec![100, 300]); + } + + #[test] + fn never_claims_processes_through_the_supervisors_own_session_or_group() { + let mut state = LineageState::new(100); + // 50 shares the supervisor's session and group but is not a descendant. + let observed = state.observe_live(&table(vec![ + (entry(10, 1, 10, 5), 1), + (entry(50, 1, 10, 6), 1), + (entry(100, 10, 100, 10), 1), + (entry(60, 1, 100, 11), 1), + ])); + // 60 joined the sidecar's own group, which the sidecar leads. + assert_eq!(pids(&observed), vec![60, 100]); + } + + #[test] + fn rejects_a_reused_group_or_session_id_and_older_processes() { + let mut state = LineageState::new(100); + state.observe(&table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(200, 100, 200, 20), 200), + ])); + // PID 200 now names a different process: the recorded group and + // session are gone, so its group members are not ours. Process 210 + // reports group 200 but started before the recorded leader. + let reused = table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(200, 1, 200, 90), 200), + (entry(201, 200, 200, 91), 200), + (entry(210, 1, 200, 15), 200), + ]); + assert_eq!(pids(&state.observe_live(&reused)), vec![100]); + } + + #[test] + fn records_a_session_created_after_the_member_was_first_seen() { + let mut state = LineageState::new(100); + // First read: the fork has not called setsid yet. + state.observe(&table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(200, 100, 100, 20), 1), + ])); + state.observe(&table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(200, 100, 200, 20), 200), + ])); + let after = table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(201, 1, 200, 21), 200), + ]); + assert_eq!(pids(&state.observe_live(&after)), vec![100, 201]); + } + + #[test] + fn keeps_an_unreadable_member_unresolved_until_it_is_gone() { + let mut state = LineageState::new(100); + state.observe(&table(vec![ + (entry(100, 10, 100, 10), 1), + (entry(200, 100, 200, 20), 200), + ])); + let mut unreadable = table(vec![(entry(100, 10, 100, 10), 1)]); + unreadable.unreadable = vec![200]; + let observation = state.observe(&unreadable); + assert_eq!(pids(&observation.live), vec![100]); + assert_eq!(observation.unresolved, vec![200]); + assert!(!observation.is_empty()); + + let gone = state.observe(&table(vec![(entry(100, 10, 100, 10), 1)])); + assert!(gone.unresolved.is_empty()); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs index 81488ca64d4..eff002ac44f 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs @@ -27,6 +27,10 @@ use sha2::{Digest, Sha256}; use crate::local_runner::LocalRunnerError; const PROCESS_OUTPUT_QUEUE_CAPACITY: usize = 256; +/// How long, after the shutdown grace, teardown keeps sending KILL to the +/// processes a supervised process started before it reports them as alive. +#[cfg(any(target_os = "linux", target_os = "macos"))] +const LINEAGE_KILL_PROOF_WINDOW: Duration = Duration::from_secs(5); const VERIFIED_RUNTIME_EXECUTABLE_ENV: &str = "PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE"; const VERIFIED_COMMONJS_ARTIFACT_LOADER: &str = r#"const fs=require("node:fs");const Module=require("node:module");const filename=process.argv[1];const source=fs.readFileSync(filename,"utf8").replace(/^#![^\r\n]*(?:\r?\n|$)/,"");const artifact=new Module(filename);artifact.filename=filename;artifact.paths=[];artifact._compile(source,filename);"#; @@ -913,6 +917,9 @@ pub struct SupervisedProcess { process_group_id: u32, shutdown_grace: Duration, finished: bool, + /// Every process this one starts, including tools that leave its group. + #[cfg(any(target_os = "linux", target_os = "macos"))] + lineage: Option, #[cfg(target_os = "macos")] _temporary_executables: Vec, #[cfg(target_os = "macos")] @@ -1079,6 +1086,33 @@ impl SupervisedProcess { )) })?; let process_group_id = child.id(); + // Record the lineage from the start. Without it, teardown could not + // stop tools that leave the group, so a process is never supervised + // without it: stop the one just started and report the failure. + #[cfg(any(target_os = "linux", target_os = "macos"))] + let lineage = match crate::process_lineage::LineageTracker::start(process_group_id) { + Ok(lineage) => lineage, + Err(reason) => { + // The host may be out of threads or processes, so signal with + // a direct system call rather than a `kill` child process. The + // new process has had no time to start tools of its own. + let group = i32::try_from(process_group_id) + .ok() + .and_then(rustix::process::Pid::from_raw); + let signalled = group.is_some_and(|group| { + rustix::process::kill_process_group(group, rustix::process::Signal::KILL) + .is_ok() + }); + if !signalled { + let _ = child.kill(); + } + let _ = child.wait(); + return Err(LocalRunnerError::invalid(format!( + "failed to start supervised process {}: {reason}", + program.display() + ))); + } + }; crate::oom_score::release_supervised_process(process_group_id); let stdin = child .stdin @@ -1105,6 +1139,8 @@ impl SupervisedProcess { process_group_id, shutdown_grace, finished: false, + #[cfg(any(target_os = "linux", target_os = "macos"))] + lineage: Some(lineage), #[cfg(target_os = "macos")] _temporary_executables: Vec::new(), #[cfg(target_os = "macos")] @@ -1221,28 +1257,60 @@ impl SupervisedProcess { pub fn wait(&mut self) -> Result { if self.finished { - return self.child.wait().map(exit_fact).map_err(|error| { + let fact = self.child.wait().map(exit_fact).map_err(|error| { LocalRunnerError::invalid(format!("failed to inspect retired child: {error}")) - }); + })?; + self.terminate_lineage(self.shutdown_grace)?; + return Ok(fact); } let status = self.child.wait().map_err(|error| { LocalRunnerError::invalid(format!("failed to wait for process: {error}")) })?; // The group leader can exit while descendants remain alive. Reap the - // leader first, then clear any remaining members of its private group. - // A caller that must exclude escaped or re-parented descendants still - // needs a separately inherited lifetime fence. + // leader first, then clear any remaining members of its private group, + // then every recorded process that left the group. #[cfg(unix)] signal_process_group(self.process_group_id, "KILL"); self.finished = true; + self.terminate_lineage(self.shutdown_grace)?; Ok(exit_fact(status)) } + /// Stops the processes this one started outside its own group, such as a + /// tool that a provider ran in a new session and that outlived its + /// wrapper. Fails while any recorded process is still alive, so a caller + /// never treats the provider as stopped while its tools still run. + fn terminate_lineage(&mut self, grace: Duration) -> Result<(), LocalRunnerError> { + #[cfg(any(target_os = "linux", target_os = "macos"))] + if let Some(lineage) = self.lineage.as_ref() { + lineage + .terminate(grace, grace + LINEAGE_KILL_PROOF_WINDOW) + .map_err(|reason| { + LocalRunnerError::invalid(format!( + "processes started by the supervised process were not proven stopped: {reason}" + )) + })?; + self.lineage = None; + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + let _ = grace; + Ok(()) + } + + fn sample_lineage(&self) { + #[cfg(any(target_os = "linux", target_os = "macos"))] + if let Some(lineage) = self.lineage.as_ref() { + lineage.sample(); + } + } + pub fn terminate_group(&mut self) -> Result { if self.finished { return self.wait(); } self.stdin.take(); + // Record the lineage visible now, before signals start re-parenting it. + self.sample_lineage(); #[cfg(unix)] signal_process_group(self.process_group_id, "TERM"); #[cfg(not(unix))] @@ -1254,6 +1322,7 @@ impl SupervisedProcess { #[cfg(unix)] signal_process_group(self.process_group_id, "KILL"); self.finished = true; + self.terminate_lineage(self.shutdown_grace)?; return Ok(fact); } if Instant::now() >= deadline { @@ -1271,10 +1340,17 @@ impl SupervisedProcess { /// Fence this owned process group when an existing deadline leaves no grace. pub fn terminate_group_immediately(&mut self) -> Result { + let fact = self.fence_group_immediately()?; + self.terminate_lineage(Duration::ZERO)?; + Ok(fact) + } + + fn fence_group_immediately(&mut self) -> Result { if self.finished { return self.wait(); } self.stdin.take(); + self.sample_lineage(); #[cfg(unix)] let signal = { use rustix::process::{kill_process_group, Pid, Signal}; @@ -1295,6 +1371,19 @@ impl SupervisedProcess { "owned process group disappeared before leader exit was observed", )) } + // macOS refuses to signal a group whose only member is the + // exited, unreaped leader. Reaping it settles the group; the + // lineage pass that follows proves no other member remains. + Err(rustix::io::Errno::PERM) => { + if let Some(exit) = self.try_wait()? { + self.finished = true; + return Ok(exit); + } + Err(LocalRunnerError::invalid(format!( + "failed to signal owned process group: {}", + rustix::io::Errno::PERM + ))) + } Err(error) => Err(LocalRunnerError::invalid(format!( "failed to signal owned process group: {error}" ))), @@ -1345,6 +1434,8 @@ impl Drop for SupervisedProcess { fn drop(&mut self) { if !self.finished { let _ = self.terminate_group(); + } else { + let _ = self.terminate_lineage(self.shutdown_grace); } } } @@ -1388,6 +1479,103 @@ mod tests { use super::*; use std::time::{SystemTime, UNIX_EPOCH}; + /// Plays a provider whose tool runs `nohup job &` in a new session, the + /// way Claude Code runs a Bash tool command. The wrapper reports the job's + /// PID and exits, so the job is re-parented and outside the owned group. + fn spawn_provider_with_escaped_tool_job( + name: &str, + ) -> ( + SupervisedProcess, + PathBuf, + paperclip_process_table::ProcessEntry, + ) { + let directory = std::env::temp_dir().join(format!( + "paperclip-escaped-tool-{name}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&directory).unwrap(); + let pid_file = directory.join("job.pid"); + let tool = r#"sleep 0.3; nohup /bin/sh -c 'while :; do sleep 1; done' >/dev/null 2>&1 & echo $! > "$0.tmp"; mv "$0.tmp" "$0"; exit 0"#; + let wrapper = format!( + "use POSIX (); POSIX::setsid() or die; exec '/bin/sh', '-c', q{{{tool}}}, $ARGV[0];" + ); + let provider = r#"perl -e "$1" "$2" & printf 'ready\n'; while :; do sleep 1; done"#; + let process = SupervisedProcess::spawn( + Path::new("/bin/sh"), + &[ + "-c".to_owned(), + provider.to_owned(), + "provider".to_owned(), + wrapper, + pid_file.display().to_string(), + ], + Duration::from_millis(500), + 1024, + ) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let job = loop { + if let Some(pid) = fs::read_to_string(&pid_file) + .ok() + .and_then(|text| text.trim().parse::().ok()) + { + let job = paperclip_process_table::read(pid) + .entry() + .expect("tool job is running"); + // Wait until the wrapper has exited and the job is orphaned. + if paperclip_process_table::read(job.ppid) + .entry() + .is_none_or(|parent| parent.pgid != job.pgid) + { + break job; + } + } + assert!(Instant::now() < deadline, "tool job did not start"); + thread::sleep(Duration::from_millis(20)); + }; + assert_ne!(job.pgid, i32::try_from(process.process_group_id()).unwrap()); + (process, directory, job) + } + + fn assert_stopped_and_clean_up(job: paperclip_process_table::ProcessEntry, directory: &Path) { + let alive = paperclip_process_table::read(job.pid) + .entry() + .is_some_and(|current| current.started == job.started && !current.zombie); + if alive { + // Never leave the escaped job behind when the assertion fails. + if let Some(group) = rustix::process::Pid::from_raw(job.pgid) { + let _ = rustix::process::kill_process_group(group, rustix::process::Signal::KILL); + } + } + let _ = fs::remove_dir_all(directory); + assert!(!alive, "tool job {} outlived its provider", job.pid); + } + + #[test] + fn terminating_the_group_stops_a_tool_job_that_left_its_session() { + let (mut process, directory, job) = spawn_provider_with_escaped_tool_job("terminate"); + process.terminate_group().unwrap(); + assert_stopped_and_clean_up(job, &directory); + } + + #[test] + fn immediate_fence_stops_a_tool_job_that_left_its_session() { + let (mut process, directory, job) = spawn_provider_with_escaped_tool_job("immediate"); + process.terminate_group_immediately().unwrap(); + assert_stopped_and_clean_up(job, &directory); + } + + #[test] + fn dropping_the_provider_stops_a_tool_job_that_left_its_session() { + let (process, directory, job) = spawn_provider_with_escaped_tool_job("drop"); + drop(process); + assert_stopped_and_clean_up(job, &directory); + } + #[test] #[cfg(unix)] fn immediate_group_fence_does_not_wait_for_shutdown_grace() { diff --git a/scripts/__tests__/runner-debug-crate-cache.test.mjs b/scripts/__tests__/runner-debug-crate-cache.test.mjs index 08df149227d..cd890c319fa 100644 --- a/scripts/__tests__/runner-debug-crate-cache.test.mjs +++ b/scripts/__tests__/runner-debug-crate-cache.test.mjs @@ -37,6 +37,28 @@ test("every debug binary must be complete before the restored units are marked f assert.throws(() => debugContract(root, { ...metadata, workspace_members: [] }), /Review/); }); +test("the process-table library is cached with the crate, and any other workspace member needs review", t => { + const root = mkdtempSync(path.join(os.tmpdir(), "runner-debug-cache-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const table = { id: "table", name: "paperclip-process-table", targets: [{ name: "paperclip_process_table", kind: ["lib"] }] }; + const withTable = { workspace_members: ["core", "table"], packages: [...metadata.packages, table] }; + const contract = debugContract(root, withTable); + assert.ok(contract.cachedPaths.some(entry => entry.endsWith("/deps/libpaperclip_process_table-*"))); + assert.ok(contract.required.some(item => item.label === "the process-table lib rlib")); + for (const item of contract.required) { + const filename = path.join(root, item.glob.replaceAll("*", "0123456789abcdef")); + mkdirSync(path.dirname(filename), { recursive: true }); + writeFileSync(filename, "cached artifact"); + } + assert.equal(markCurrent({ root, ...contract }).action, "marked"); + rmSync(path.join(root, contract.required.find(item => item.label === "the process-table lib rlib").glob.replaceAll("*", "0123456789abcdef"))); + assert.match(markCurrent({ root, ...contract }).reason, /process-table lib rlib/); + const tableWithBin = { ...table, targets: [...table.targets, { name: "probe", kind: ["bin"] }] }; + assert.throws(() => debugContract(root, { ...withTable, packages: [...metadata.packages, tableWithBin] }), /Review/); + const other = { id: "other", name: "paperclip-other", targets: [{ name: "other", kind: ["lib"] }] }; + assert.throws(() => debugContract(root, { workspace_members: ["core", "other"], packages: [...metadata.packages, other] }), /Review/); +}); + test("debug cache keys invalidate source, included inputs, deletions and compiler flags", t => { const root = mkdtempSync(path.join(os.tmpdir(), "runner-debug-key-")); t.after(() => rmSync(root, { recursive: true, force: true })); diff --git a/scripts/__tests__/runner-release-crate-cache.test.mjs b/scripts/__tests__/runner-release-crate-cache.test.mjs index aa242ae6bca..1be0097a6c5 100644 --- a/scripts/__tests__/runner-release-crate-cache.test.mjs +++ b/scripts/__tests__/runner-release-crate-cache.test.mjs @@ -76,6 +76,9 @@ test("the script's path list and key inputs cover what the release build of the `${RELEASE_DIR}/deps/paperclip_runnerd-*`, `${RELEASE_DIR}/paperclip-runnerd`, `${RELEASE_DIR}/paperclip-runnerd.d`, + `${RELEASE_DIR}/.fingerprint/paperclip-process-table-*`, + `${RELEASE_DIR}/deps/libpaperclip_process_table-*`, + `${RELEASE_DIR}/deps/paperclip_process_table-*.d`, ]); assert.deepEqual(HASHED_INPUTS, [ `${RUNNER_PACKAGE}/rust-toolchain.toml`, @@ -451,9 +454,11 @@ function layRestoredCrate(root) { const release = path.join(root, RELEASE_DIR); const lib = path.join(release, ".fingerprint/paperclip-runner-core-0b6c4654c2710743"); const bin = path.join(release, ".fingerprint/paperclip-runner-core-46cce78dbc4840c2"); + const table = path.join(release, ".fingerprint/paperclip-process-table-5a1b2c3d4e5f6071"); mkdirSync(path.join(release, "deps"), { recursive: true }); mkdirSync(lib, { recursive: true }); mkdirSync(bin, { recursive: true }); + mkdirSync(table, { recursive: true }); const files = { [path.join(lib, "lib-paperclip_runner_core")]: "0123456789abcdef", [path.join(lib, "lib-paperclip_runner_core.json")]: "{}", @@ -469,6 +474,12 @@ function layRestoredCrate(root) { [path.join(release, "deps/paperclip_runnerd-46cce78dbc4840c2.d")]: "d", [path.join(release, "paperclip-runnerd")]: "ELF", [path.join(release, "paperclip-runnerd.d")]: "d", + // The process-table workspace library's lib unit. + [path.join(table, "lib-paperclip_process_table")]: "1111222233334444", + [path.join(table, "dep-lib-paperclip_process_table")]: "dep", + [path.join(release, "deps/libpaperclip_process_table-5a1b2c3d4e5f6071.rlib")]: "rlib", + [path.join(release, "deps/libpaperclip_process_table-5a1b2c3d4e5f6071.rmeta")]: "rmeta", + [path.join(release, "deps/paperclip_process_table-5a1b2c3d4e5f6071.d")]: "d", // A dependency's artifacts sit beside the crate's and must not be touched. [path.join(release, "deps/libserde-1234567890abcdef.rlib")]: "dep rlib", }; @@ -477,7 +488,7 @@ function layRestoredCrate(root) { writeFileSync(file, content); utimesSync(file, old, old); } - for (const dir of [lib, bin, path.join(release, "deps"), path.join(release, ".fingerprint"), release]) utimesSync(dir, old, old); + for (const dir of [lib, bin, table, path.join(release, "deps"), path.join(release, ".fingerprint"), release]) utimesSync(dir, old, old); return { release, files: Object.keys(files), old }; } @@ -492,7 +503,7 @@ test("mark-current makes every restored artifact newer than the checkout and lea const result = markCurrent({ root, now: new Date(checkedOutAt + 5000) }); assert.equal(result.action, "marked"); - assert.equal(result.files, 14); + assert.equal(result.files, 19); assert.equal(result.bytes, files.filter((file) => !file.includes("libserde")).reduce((sum, file) => sum + statSync(file).size, 0)); for (const file of files) { @@ -509,7 +520,7 @@ test("mark-current makes every restored artifact newer than the checkout and lea const printed = JSON.parse(execFileSync(process.execPath, [script, "mark-current"], { encoding: "utf8", env: { ...process.env, GITHUB_WORKSPACE: root } })); assert.equal(printed.schema, "paperclip.runner-release-crate.v1"); assert.equal(printed.action, "marked"); - assert.equal(printed.files, 14); + assert.equal(printed.files, 19); } finally { rmSync(root, { recursive: true, force: true }); } @@ -528,6 +539,10 @@ const REQUIRED_FILES = { "the bin fingerprint hash": ".fingerprint/paperclip-runner-core-46cce78dbc4840c2/bin-paperclip-runnerd", "the bin dep-info": ".fingerprint/paperclip-runner-core-46cce78dbc4840c2/dep-bin-paperclip-runnerd", "the bin under deps": "deps/paperclip_runnerd-46cce78dbc4840c2", + "the process-table lib fingerprint hash": ".fingerprint/paperclip-process-table-5a1b2c3d4e5f6071/lib-paperclip_process_table", + "the process-table lib dep-info": ".fingerprint/paperclip-process-table-5a1b2c3d4e5f6071/dep-lib-paperclip_process_table", + "the process-table lib rlib": "deps/libpaperclip_process_table-5a1b2c3d4e5f6071.rlib", + "the process-table lib rmeta": "deps/libpaperclip_process_table-5a1b2c3d4e5f6071.rmeta", }; const NOT_NEEDED = [ ".fingerprint/paperclip-runner-core-0b6c4654c2710743/lib-paperclip_runner_core.json", @@ -537,6 +552,7 @@ const NOT_NEEDED = [ "deps/paperclip_runnerd-46cce78dbc4840c2.d", "paperclip-runnerd", "paperclip-runnerd.d", + "deps/paperclip_process_table-5a1b2c3d4e5f6071.d", ]; test("mark-current discards a restore that lacks any file cargo needs for either unit, naming the file, so cargo sees a plain miss", () => { @@ -579,7 +595,7 @@ test("mark-current discards a restore that lacks any file cargo needs for either rmSync(path.join(release, file)); const result = markCurrent({ root: intact }); assert.equal(result.action, "marked", `${file} missing must not discard the restore`); - assert.equal(result.files, 13, file); + assert.equal(result.files, 18, file); } finally { rmSync(intact, { recursive: true, force: true }); } diff --git a/scripts/runner-debug-crate-cache.mjs b/scripts/runner-debug-crate-cache.mjs index dff1e16be0b..930bab3f5c4 100644 --- a/scripts/runner-debug-crate-cache.mjs +++ b/scripts/runner-debug-crate-cache.mjs @@ -4,7 +4,7 @@ import { appendFileSync, existsSync, globSync, readFileSync, statSync } from "no import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { computeScope, HASHED_INPUTS, markCurrent, readRustcVersion, RUNNER_PACKAGE } from "./runner-release-crate-cache.mjs"; +import { computeScope, HASHED_INPUTS, markCurrent, processTableRequired, readRustcVersion, RUNNER_PACKAGE } from "./runner-release-crate-cache.mjs"; const directory = `${RUNNER_PACKAGE}/runner/target/debug`; export const DEBUG_INPUTS = [...HASHED_INPUTS, `${RUNNER_PACKAGE}/package.json`, ".cargo/**", `${RUNNER_PACKAGE}/.cargo/**`, @@ -20,18 +20,28 @@ function readMetadata(root) { })); } +// The workspace is the Runner crate plus its process-table library, which +// has only a lib target. Any other shape needs this contract reviewed. +const PROCESS_TABLE = "paperclip-process-table"; + export function debugContract(root, metadata = readMetadata(root)) { const packages = metadata.packages.filter(pkg => metadata.workspace_members.includes(pkg.id)); - if (packages.length !== 1 || packages[0].name !== "paperclip-runner-core") { + const core = packages.filter(pkg => pkg.name === "paperclip-runner-core"); + const others = packages.filter(pkg => pkg.name !== "paperclip-runner-core"); + const libraryOnly = pkg => pkg.name === PROCESS_TABLE + && pkg.targets.every(target => target.kind.every(kind => kind === "lib")); + if (core.length !== 1 || others.length > 1 || !others.every(libraryOnly)) { throw new Error("Review the debug artifact contract for the changed Runner workspace"); } - const bins = packages[0].targets.filter(target => target.kind.includes("bin")).map(target => target.name).sort(); + const withProcessTable = others.length === 1; + const bins = core[0].targets.filter(target => target.kind.includes("bin")).map(target => target.name).sort(); if (!bins.includes("paperclip-runnerd")) throw new Error("Missing Runner binary target"); const fingerprint = `${directory}/.fingerprint/paperclip-runner-core-*`; const cachedPaths = [fingerprint, `${directory}/deps/libpaperclip_runner_core-*`, `${directory}/deps/paperclip_runner_core-*.d`, ...bins.flatMap(bin => [ `${directory}/deps/${bin.replaceAll("-", "_")}-*`, `${directory}/${bin}`, `${directory}/${bin}.d`, - ])]; + ]), ...(withProcessTable ? [`${directory}/.fingerprint/${PROCESS_TABLE}-*`, + `${directory}/deps/libpaperclip_process_table-*`, `${directory}/deps/paperclip_process_table-*.d`] : [])]; const required = [ { label: "lib fingerprint", glob: `${fingerprint}/lib-paperclip_runner_core` }, { label: "lib dep-info", glob: `${fingerprint}/dep-lib-paperclip_runner_core` }, @@ -42,6 +52,7 @@ export function debugContract(root, metadata = readMetadata(root)) { { label: `${bin} dep-info`, glob: `${fingerprint}/dep-bin-${bin}` }, { label: `${bin} executable`, glob: `${directory}/deps/${bin.replaceAll("-", "_")}-*`, exclude: /\.(d|dSYM)$/ }, ]), + ...(withProcessTable ? processTableRequired(directory) : []), ]; return { bins, cachedPaths, required }; } diff --git a/scripts/runner-release-crate-cache.mjs b/scripts/runner-release-crate-cache.mjs index 3bc01a2b6ae..5be815b3a65 100644 --- a/scripts/runner-release-crate-cache.mjs +++ b/scripts/runner-release-crate-cache.mjs @@ -43,6 +43,12 @@ export const CACHED_PATHS = [ `${RELEASE_DIR}/deps/paperclip_runnerd-*`, `${RELEASE_DIR}/paperclip-runnerd`, `${RELEASE_DIR}/paperclip-runnerd.d`, + // The process-table library is the crate's one workspace dependency. Cargo + // rebuilds a dependent whose workspace dependency is newer than it, so the + // library's lib unit is cached and marked current with the crate. + `${RELEASE_DIR}/.fingerprint/paperclip-process-table-*`, + `${RELEASE_DIR}/deps/libpaperclip_process_table-*`, + `${RELEASE_DIR}/deps/paperclip_process_table-*.d`, ]; // Every file the release build of the crate reads, as hashFiles() patterns: @@ -81,8 +87,20 @@ export const REQUIRED = [ { label: "the bin fingerprint hash", glob: `${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*/bin-paperclip-runnerd` }, { label: "the bin dep-info", glob: `${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*/dep-bin-paperclip-runnerd` }, { label: "the bin under deps", glob: `${RELEASE_DIR}/deps/paperclip_runnerd-*`, exclude: /\.d$/ }, + ...processTableRequired(RELEASE_DIR), ]; +/** The lib unit of the process-table workspace dependency, in `directory`. */ +export function processTableRequired(directory) { + const fingerprint = `${directory}/.fingerprint/paperclip-process-table-*`; + return [ + { label: "the process-table lib fingerprint hash", glob: `${fingerprint}/lib-paperclip_process_table` }, + { label: "the process-table lib dep-info", glob: `${fingerprint}/dep-lib-paperclip_process_table` }, + { label: "the process-table lib rlib", glob: `${directory}/deps/libpaperclip_process_table-*.rlib` }, + { label: "the process-table lib rmeta", glob: `${directory}/deps/libpaperclip_process_table-*.rmeta` }, + ]; +} + const SCHEMA = "paperclip.runner-release-crate.v1"; function keySafe(value) {