From 2f90cb31ae77367fafcb4a0d6a453303be6d3028 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 1 Oct 2026 16:21:11 -0500 Subject: [PATCH] ci(runner): retain Intel pack and stop on uncertain cleanup Archive and verify the complete tested pack before startup checks. Make cleanup uncertainty sticky across test cases, retain scratch after interrupted launches, and enforce the reviewed 16 GiB seven-day artifact storage bound without truncation. Co-Authored-By: Paperclip --- .github/workflows/docker-runner-check.yml | 11 ++- scripts/ci/pi-intel/admit_evidence.py | 50 +++++++++++++ scripts/ci/pi-intel/lifecycle.py | 40 ++++++++++ scripts/ci/pi-intel/retain_pack.py | 35 +++++++++ scripts/ci/pi-intel/test_ci_helpers.py | 89 +++++++++++++++++++++++ scripts/ci/pi-intel/verify.py | 35 ++++++--- 6 files changed, 247 insertions(+), 13 deletions(-) create mode 100644 scripts/ci/pi-intel/admit_evidence.py create mode 100644 scripts/ci/pi-intel/lifecycle.py create mode 100644 scripts/ci/pi-intel/retain_pack.py create mode 100644 scripts/ci/pi-intel/test_ci_helpers.py diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index f4392aa09b7..3fc209b8073 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -202,11 +202,20 @@ jobs: python3 -B trusted-ci/scripts/ci/pi-intel/verify.py \ --source "$GITHUB_WORKSPACE/candidate" \ --output "$GITHUB_WORKSPACE/pi-intel-evidence" - - name: Retain exact input, independent output, test, and cleanup evidence + - name: Admit complete evidence within the reserved storage bound if: always() + id: pi_intel_evidence + timeout-minutes: 2 + run: | + python3 -B trusted-ci/scripts/ci/pi-intel/admit_evidence.py \ + --evidence "$GITHUB_WORKSPACE/pi-intel-evidence" \ + --github-output "$GITHUB_OUTPUT" + - name: Retain exact input, independent output, test, and cleanup evidence + if: always() && steps.pi_intel_evidence.outputs.admitted == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: pi-native-intel-${{ github.run_id }} + include-hidden-files: true path: pi-intel-evidence/ retention-days: 7 if-no-files-found: error diff --git a/scripts/ci/pi-intel/admit_evidence.py b/scripts/ci/pi-intel/admit_evidence.py new file mode 100644 index 00000000000..f5bc2809374 --- /dev/null +++ b/scripts/ci/pi-intel/admit_evidence.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Bound the complete evidence upload; never truncate the tested pack or diagnostics.""" +import argparse +import json +from pathlib import Path +import stat + +MAX_BYTES = 16 * 1024 ** 3 +RETENTION_DAYS = 7 + +def regular_bytes(root): + total = 0 + for path in root.rglob('*'): + mode = path.lstat().st_mode + if stat.S_ISREG(mode): total += path.stat().st_size + elif not stat.S_ISDIR(mode): raise RuntimeError('Evidence contains a symlink or special entry: ' + str(path)) + return total + +def admit_evidence(root, maximum_bytes=MAX_BYTES): + root = Path(root) + if root.is_symlink() or not root.is_dir(): raise RuntimeError('Evidence root missing or linked') + admission = root/'artifact-admission.json' + receipt = root/'receipt.json' + data = {'status':'checking', 'regularFileBytes':0, 'maximumBytes':maximum_bytes, + 'retentionDays':RETENTION_DAYS, 'truncated':False, + 'costScope':'Artifact storage only; standard public-runner minutes are separate.', + 'priceUsdPerGiBMonth':0.25, 'conservativeMonthDays':28, + 'maximumEstimatedStorageUsd':round(maximum_bytes / 1024**3 * RETENTION_DAYS / 28 * 0.25, 6), + 'billingPolicyUrl':'https://docs.github.com/en/billing/concepts/product-billing/github-actions'} + # Include these receipt bytes themselves; settle the decimal-size field before admission. + for _ in range(10): + admission.write_text(json.dumps(data, indent=2)+'\n') + if receipt.exists(): + full = json.loads(receipt.read_text()); full['artifactStorage'] = data + receipt.write_text(json.dumps(full, indent=2)+'\n') + actual = regular_bytes(root) + status = 'admitted_complete_evidence' if actual <= maximum_bytes else 'rejected_storage_cap' + if data['regularFileBytes'] == actual and data['status'] == status: return data + data.update(regularFileBytes=actual, status=status) + raise RuntimeError('Evidence size accounting did not stabilize') + +if __name__ == '__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--evidence',type=Path,required=True) + parser.add_argument('--github-output',type=Path,required=True) + args=parser.parse_args();result=admit_evidence(args.evidence) + admitted=result['status']=='admitted_complete_evidence' + with args.github_output.open('a') as output:output.write('admitted='+str(admitted).lower()+'\n') + print(json.dumps(result)) + raise SystemExit(0 if admitted else 1) diff --git a/scripts/ci/pi-intel/lifecycle.py b/scripts/ci/pi-intel/lifecycle.py new file mode 100644 index 00000000000..b250bcb4eeb --- /dev/null +++ b/scripts/ci/pi-intel/lifecycle.py @@ -0,0 +1,40 @@ +"""Sticky launch/cleanup gate shared by execution and the narrow regressions.""" +class CleanupUncertain(RuntimeError): + pass + +class Lifecycle: + def __init__(self): + self.pending = False + self.uncertain = False + + @property + def safe(self): + return not self.pending and not self.uncertain + + def begin(self): + if not self.safe: + raise CleanupUncertain('Previous launch/cleanup is unresolved; no next command') + # Set before Popen: an interrupted launch may have no trustworthy handle. + self.pending = True + + def failed(self): + self.uncertain = True + + def confirmed(self): + if self.uncertain: + raise CleanupUncertain('An earlier cleanup failure cannot be cleared') + self.pending = False + + +def run_test_cases(cases, run, lifecycle): + failures = [] + for label, command in cases: + if not lifecycle.safe: + raise CleanupUncertain('Unresolved owner prohibits subsequent test launch') + try: + run(command, label) + except RuntimeError as error: + failures.append(str(error)) + if not lifecycle.safe: + raise CleanupUncertain('Test cleanup uncertain; stopping without another test') from error + return failures diff --git a/scripts/ci/pi-intel/retain_pack.py b/scripts/ci/pi-intel/retain_pack.py new file mode 100644 index 00000000000..b06ac2ab176 --- /dev/null +++ b/scripts/ci/pi-intel/retain_pack.py @@ -0,0 +1,35 @@ +"""Retain exact tested bytes, internal symlinks, and modes outside owned scratch.""" +import hashlib +from pathlib import Path +import tarfile +from source_guard import require, sha + +def retain_pack(pack, output, inventory): + pack, output = Path(pack), Path(output) + require(not output.exists() and not output.resolve().is_relative_to(pack.resolve()), 'Archive must be new and outside pack') + with tarfile.open(output, 'w:gz', compresslevel=1, dereference=False) as archive: + archive.add(pack, arcname='provider-pack') + expected = {row['path']: row for row in inventory} + observed = set() + with tarfile.open(output, 'r:gz') as archive: + for member in archive: + if member.name == 'provider-pack': + require(member.isdir(), 'Archive root must be directory') + continue + relative = member.name.removeprefix('provider-pack/') + require(member.name.startswith('provider-pack/') and relative in expected and relative not in observed, 'Unexpected archive member') + row = expected[relative]; observed.add(relative) + require(member.mode == row['mode'], 'Archive mode mismatch') + if row['kind'] == 'directory': + require(member.isdir(), 'Archive directory mismatch') + elif row['kind'] == 'symlink': + require(member.issym() and member.linkname == row['target'], 'Archive symlink mismatch') + else: + require(member.isfile() and member.size == row['size'], 'Archive file mismatch') + digest = hashlib.sha256() + with archive.extractfile(member) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): digest.update(chunk) + require(digest.hexdigest() == row['sha256'], 'Archive bytes mismatch') + require(observed == set(expected), 'Archive omitted pack members') + return {'path':str(output), 'sha256':sha(output), 'bytes':output.stat().st_size, + 'inventoryVerified':True, 'internalSymlinksAndModesPreserved':True} diff --git a/scripts/ci/pi-intel/test_ci_helpers.py b/scripts/ci/pi-intel/test_ci_helpers.py new file mode 100644 index 00000000000..300e67f721c --- /dev/null +++ b/scripts/ci/pi-intel/test_ci_helpers.py @@ -0,0 +1,89 @@ +import os +from pathlib import Path +import shutil +import tarfile +import tempfile +import unittest +from unittest.mock import Mock +from lifecycle import CleanupUncertain, Lifecycle, run_test_cases +from retain_pack import retain_pack +from source_guard import tree, sha +from admit_evidence import admit_evidence, regular_bytes + +class CleanupRegression(unittest.TestCase): + def test_first_cleanup_failure_prevents_second_test_and_scratch_removal(self): + lifecycle = Lifecycle(); launched = []; remove_scratch = Mock() + def run(command, label): + lifecycle.begin(); launched.append(label) + try: + raise RuntimeError('owner.stop inspection failed') + finally: + lifecycle.failed() + with self.assertRaises(CleanupUncertain): + run_test_cases([('startup', []), ('sdk', [])], run, lifecycle) + if lifecycle.safe: remove_scratch() + self.assertEqual(launched, ['startup']); remove_scratch.assert_not_called() + with self.assertRaises(CleanupUncertain): lifecycle.confirmed() + with self.assertRaises(CleanupUncertain): lifecycle.begin() + self.assertFalse(lifecycle.safe) + + def test_interrupted_launch_without_handle_retains_scratch(self): + lifecycle = Lifecycle(); lifecycle.begin() + # Popen may be interrupted before Python receives a handle: pending is unsafe. + launched = Mock() + with self.assertRaises(CleanupUncertain): + run_test_cases([('sdk', [])], launched, lifecycle) + launched.assert_not_called(); self.assertFalse(lifecycle.safe) + + def test_assertion_failure_with_confirmed_cleanup_allows_independent_sdk_test(self): + lifecycle = Lifecycle(); launched = [] + def run(command, label): + lifecycle.begin(); launched.append(label); lifecycle.confirmed() + if label == 'startup': raise RuntimeError('test assertion failed') + failures = run_test_cases([('startup', []), ('sdk', [])], run, lifecycle) + self.assertEqual(launched, ['startup', 'sdk']) + self.assertEqual(failures, ['test assertion failed']); self.assertTrue(lifecycle.safe) + +class PackRetentionRegression(unittest.TestCase): + def test_full_pack_survives_scratch_removal_with_bytes_links_and_modes(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root); pack = root/'scratch'/'provider-pack'; pack.mkdir(parents=True) + (pack/'bin').mkdir(); (pack/'bin'/'node').write_bytes(b'exact-tested-bytes') + (pack/'bin'/'node').chmod(0o755); (pack/'node').symlink_to('bin/node') + (pack/'.hidden').write_bytes(b'inventory-must-include-hidden'); (pack/'.hidden').chmod(0o444) + inventory = tree(pack, allow_links=True) + archive = root/'retained-pack.tar.gz' + receipt = retain_pack(pack, archive, inventory) + self.assertEqual(receipt['sha256'], sha(archive)); self.assertTrue(receipt['inventoryVerified']) + shutil.rmtree(root/'scratch') + restored = root/'restored'; restored.mkdir() + # This archive was created above and every member was verified against our owned inventory. + # The data filter intentionally rewrites read-only modes, so it cannot test exact restoration. + with tarfile.open(archive) as tar: tar.extractall(restored, filter='fully_trusted') + self.assertEqual(tree(restored/'provider-pack', allow_links=True), inventory) + self.assertEqual(os.readlink(restored/'provider-pack/node'), 'bin/node') + +class ArtifactStorageRegression(unittest.TestCase): + def test_complete_evidence_size_includes_receipts_and_hidden_files(self): + with tempfile.TemporaryDirectory() as root: + root=Path(root); (root/'receipt.json').write_text('{}'); (root/'.hidden').write_bytes(b'exact') + result=admit_evidence(root, maximum_bytes=1024*1024) + self.assertEqual(result['status'], 'admitted_complete_evidence') + self.assertEqual(result['regularFileBytes'], regular_bytes(root)) + self.assertEqual(result['retentionDays'], 7) + self.assertFalse(result['truncated']) + + def test_over_cap_rejects_upload_without_truncating_pack(self): + with tempfile.TemporaryDirectory() as root: + root=Path(root); pack=root/'provider-pack.tar.gz'; pack.write_bytes(b'x'*4096) + before=sha(pack); result=admit_evidence(root, maximum_bytes=1024) + self.assertEqual(result['status'], 'rejected_storage_cap') + self.assertEqual(sha(pack), before) + self.assertEqual(result['regularFileBytes'], regular_bytes(root)) + + def test_symlink_cannot_escape_storage_accounting(self): + with tempfile.TemporaryDirectory() as root: + root=Path(root); (root/'escape').symlink_to('/etc/hosts') + with self.assertRaisesRegex(RuntimeError, 'symlink'): admit_evidence(root) + +if __name__ == '__main__': unittest.main() diff --git a/scripts/ci/pi-intel/verify.py b/scripts/ci/pi-intel/verify.py index b353e59ba93..adfff513a9d 100644 --- a/scripts/ci/pi-intel/verify.py +++ b/scripts/ci/pi-intel/verify.py @@ -5,6 +5,8 @@ from pathlib import Path from closed_inventory import closed_tree from owned_processes import OwnedProcesses,atomic_json import source_guard +from lifecycle import Lifecycle, run_test_cases +from retain_pack import retain_pack HERE=Path(__file__).resolve().parent INPUTS=json.loads((HERE/'profile-inputs.json').read_text()) @@ -36,28 +38,30 @@ def execute(args): 'RUSTUP_TOOLCHAIN':'1.97.1','CARGO_BUILD_JOBS':'2','npm_config_audit':'false','npm_config_fund':'false', 'npm_config_update_notifier':'false','npm_config_package_import_method':'copy'} Path(env['HOME']).mkdir(mode=0o700) - active=None;owner=None;cleanup_ok=True + active=None;owner=None;lifecycle=Lifecycle() def stop_signal(signum,_):raise InterruptedError('CI stopped with signal '+str(signum)) signal.signal(signal.SIGTERM,stop_signal) def run(command,label,timeout,cwd=stage,command_env=None,owned=False): - nonlocal active,owner,cleanup_ok + nonlocal active,owner + lifecycle.begin() row={'label':label,'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'command':list(map(str,command)),'deadlineSeconds':timeout,'status':'running'} receipt['commands'].append(row);save();start=time.monotonic() with (out/(label+'.log')).open('x') as log: try: + if owned:owner=OwnedProcesses(out/(label+'-processes.json'),scratch,stage,pack/'dist/cli/acpx-runtime-sidecar.cjs') active=subprocess.Popen(row['command'],cwd=cwd,env=command_env or env,stdout=log,stderr=log,start_new_session=True) - if owned:cleanup_ok=False;owner=OwnedProcesses(out/(label+'-processes.json'),scratch,stage,pack/'dist/cli/acpx-runtime-sidecar.cjs') while active.poll() is None: if owner:owner.observe(active.pid) require(time.monotonic()-start