diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index 8ada4807851..cc04223f4b7 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -612,6 +612,14 @@ jobs: with: ref: ${{ needs.authorize_manual.outputs.target_sha }} persist-credentials: false + - name: Checkout trusted read-only executable observer + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.workflow_sha }} + path: .trusted-source-observer + sparse-checkout: scripts/ci/source-observer/executable_holders.py + sparse-checkout-cone-mode: false + persist-credentials: false - name: Record immutable source and planned checks env: SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }} @@ -621,6 +629,9 @@ jobs: run: | set -euo pipefail mkdir -p remote-source-verification + git -C .trusted-source-observer rev-parse HEAD > remote-source-verification/observer-source.txt + test "$(cat remote-source-verification/observer-source.txt)" = "${{ github.workflow_sha }}" + sha256sum .trusted-source-observer/scripts/ci/source-observer/executable_holders.py > remote-source-verification/observer-helper.sha256 git rev-parse HEAD > remote-source-verification/source.txt test "$(cat remote-source-verification/source.txt)" = "$SOURCE_SHA" printf '%s\n' "$EXPECTED_LOCK_SHA256" > remote-source-verification/expected-resolved-lock.sha256 @@ -719,9 +730,21 @@ jobs: progress_pid= checks_started_at=$(date +%s) trap 'if [ -n "$progress_pid" ]; then kill "$progress_pid" 2>/dev/null || true; fi' EXIT + observe_holders() { + # Exact executable identity only: no argv/environment reads or signals. + timeout --signal=TERM --kill-after=2s 10s python3 -B \ + .trusted-source-observer/scripts/ci/source-observer/executable_holders.py \ + --target "$GITHUB_WORKSPACE/packages/paperclip-runner/dist/bin/paperclip-runnerd" \ + > "remote-source-verification/check-$index-holders-$1.json" \ + 2> "remote-source-verification/check-$index-holders-$1.stderr" + observer_status=$? + printf '%s\n' "$observer_status" > "remote-source-verification/check-$index-holders-$1.status" + if [ "$observer_status" -ne 0 ]; then status=1; fi + } while IFS= read -r check; do index=$((index + 1)) echo "Starting $check" + observe_holders before printf 'running\n' > "remote-source-verification/check-$index.status" # test:run executes all groups serially. Recorded server durations # alone exceed 85 minutes; ordinary CI distributes them across shards. @@ -758,6 +781,7 @@ jobs: printf '%s\t%s\n' "$check_status" "$check" >> remote-source-verification/check-status.tsv if [ "$check_status" -ne 0 ]; then status=1; fi echo "Finished $check (exit $check_status)" + observe_holders after done < remote-source-verification/commands.txt exit "$status" - name: Audit source and lock closure even on check failure diff --git a/scripts/ci/source-observer/executable_holders.py b/scripts/ci/source-observer/executable_holders.py new file mode 100644 index 00000000000..8afabe70aac --- /dev/null +++ b/scripts/ci/source-observer/executable_holders.py @@ -0,0 +1,87 @@ +"""Read-only Linux evidence for the exact staged Runner executable; never signal.""" +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import stat +import time + + +def identity(path): + value = path.stat() + return {"dev": str(value.st_dev), "ino": str(value.st_ino), + "size": value.st_size, "mode": stat.S_IMODE(value.st_mode)} + + +def process_identity(path): + # comm may contain spaces or parentheses. Do not retain it, argv, or env. + value = (path / "stat").read_text() + fields = value[value.rindex(")") + 2:].split() + return {"pid": int(path.name), "state": fields[0], "ppid": int(fields[1]), + "processGroup": int(fields[2]), "startTimeTicks": fields[19]} + + +def inspect(target, proc=Path("/proc"), max_pids=32768, seconds=8): + target = Path(os.path.abspath(target)) + begin = time.monotonic() + result = {"schema": "paperclip.exact-executable-holders/v1", + "observedAt": datetime.now(timezone.utc).isoformat(), + "target": str(target), "targetBefore": None, "holders": [], + "permissionDenied": 0, "racedMatches": 0, "scanned": 0, + "complete": True, "signalsSent": 0} + try: + if target.is_symlink() or not stat.S_ISREG(target.stat().st_mode): + raise ValueError("Staged executable is not a regular non-symlink file") + result["targetBefore"] = identity(target) + except FileNotFoundError: + pass # The first typecheck has not staged the daemon yet. + for entry in proc.iterdir(): + if not entry.name.isdecimal(): + continue + if result["scanned"] >= max_pids or time.monotonic() - begin >= seconds: + result.update(complete=False, stopReason="bounded_scan_limit") + break + result["scanned"] += 1 + matched = False + try: + exe = entry / "exe" + link = os.readlink(exe) + executable = identity(exe) + pinned = result["targetBefore"] + by_inode = bool(pinned and all(executable[k] == pinned[k] for k in ("dev", "ino"))) + by_path = link in (str(target), str(target) + " (deleted)") + if not (by_inode or by_path): + continue + matched = True + before = process_identity(entry) + after = process_identity(entry) + if before != after or identity(exe) != executable or os.readlink(exe) != link: + result["racedMatches"] += 1 + continue + result["holders"].append({**before, "executable": executable, + "matchesTargetInode": by_inode, + "matchesTargetPath": by_path, + "deletedExecutable": link.endswith(" (deleted)")}) + except FileNotFoundError: + if matched: + result["racedMatches"] += 1 + except PermissionError: + result["permissionDenied"] += 1 + try: + result["targetAfter"] = identity(target) + except FileNotFoundError: + result["targetAfter"] = None + result["targetChangedDuringScan"] = result["targetBefore"] != result["targetAfter"] + result["complete"] = (result["complete"] and not result["permissionDenied"] + and not result["racedMatches"] and not result["targetChangedDuringScan"]) + result["holders"].sort(key=lambda row: row["pid"]) + result["elapsedSeconds"] = round(time.monotonic() - begin, 6) + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--target", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(inspect(args.target), sort_keys=True)) diff --git a/scripts/ci/source-observer/test_executable_holders.py b/scripts/ci/source-observer/test_executable_holders.py new file mode 100644 index 00000000000..c9dfc7d423f --- /dev/null +++ b/scripts/ci/source-observer/test_executable_holders.py @@ -0,0 +1,113 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import executable_holders as observer + + +class HolderTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.proc = self.root / "proc" + self.proc.mkdir() + self.target = self.root / "paperclip-runnerd" + self.target.write_bytes(b"exact daemon") + + def process(self, pid, executable): + root = self.proc / str(pid) + root.mkdir() + (root / "exe").symlink_to(executable) + # Include malicious-looking comm; it must never appear in output. + (root / "stat").write_text(f"{pid} (secret ) argv) S 7 8 " + "0 " * 16 + "12345 0") + (root / "environ").write_text("DO_NOT_READ_SECRET") + (root / "cmdline").write_text("DO_NOT_READ_SECRET") + return root + + def inspect(self, **kw): + return observer.inspect(self.target, self.proc, **kw) + + def test_exact_inode_and_only_numeric_process_fields(self): + self.process(12, self.target) + other = self.root / "other" + other.write_bytes(b"foreign") + self.process(13, other) + result = self.inspect() + self.assertTrue(result["complete"]) + self.assertEqual([r["pid"] for r in result["holders"]], [12]) + self.assertEqual(result["holders"][0]["startTimeTicks"], "12345") + self.assertNotIn("secret", json.dumps(result).lower()) + self.assertEqual(result["signalsSent"], 0) + + def test_inode_alias_is_still_holder(self): + alias = self.root / "alias" + alias.hardlink_to(self.target) + self.process(12, alias) + row = self.inspect()["holders"][0] + self.assertTrue(row["matchesTargetInode"]) + self.assertFalse(row["matchesTargetPath"]) + + def test_deleted_path_preserved_even_after_replacement(self): + deleted = Path(str(self.target) + " (deleted)") + deleted.write_bytes(b"old") + self.process(12, deleted) + row = self.inspect()["holders"][0] + self.assertTrue(row["matchesTargetPath"]) + self.assertTrue(row["deletedExecutable"]) + self.assertFalse(row["matchesTargetInode"]) + + def test_target_not_built_yet(self): + self.target.unlink() + self.assertIsNone(self.inspect()["targetBefore"]) + + def test_symlink_target_rejected(self): + self.target.unlink() + self.target.symlink_to(self.root / "foreign") + with self.assertRaisesRegex(ValueError, "non-symlink"): + self.inspect() + + def test_bounded_scan_is_explicitly_incomplete(self): + self.process(12, self.target) + result = self.inspect(max_pids=0) + self.assertFalse(result["complete"]) + self.assertEqual(result["stopReason"], "bounded_scan_limit") + + def test_process_identity_race_is_not_attested(self): + self.process(12, self.target) + with patch.object(observer, "process_identity", side_effect=[{"pid": 12}, {"pid": 99}]): + result = self.inspect() + self.assertFalse(result["complete"]) + self.assertEqual(result["holders"], []) + self.assertEqual(result["racedMatches"], 1) + + def test_permission_denial_is_explicitly_incomplete(self): + self.process(12, self.target) + with patch.object(observer.os, "readlink", side_effect=PermissionError): + result = self.inspect() + self.assertFalse(result["complete"]) + self.assertEqual(result["permissionDenied"], 1) + + def test_target_replacement_during_scan_is_incomplete(self): + real = observer.identity + calls = 0 + + def changed(path): + nonlocal calls + row = real(path) + if path == self.target: + calls += 1 + if calls == 2: + row["ino"] = "replacement" + return row + + with patch.object(observer, "identity", side_effect=changed): + result = self.inspect() + self.assertTrue(result["targetChangedDuringScan"]) + self.assertFalse(result["complete"]) + + +if __name__ == "__main__": + unittest.main()