diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index f4392aa09b7..3fc209b8073 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -202,11 +202,20 @@ jobs: python3 -B trusted-ci/scripts/ci/pi-intel/verify.py \ --source "$GITHUB_WORKSPACE/candidate" \ --output "$GITHUB_WORKSPACE/pi-intel-evidence" - - name: Retain exact input, independent output, test, and cleanup evidence + - name: Admit complete evidence within the reserved storage bound if: always() + id: pi_intel_evidence + timeout-minutes: 2 + run: | + python3 -B trusted-ci/scripts/ci/pi-intel/admit_evidence.py \ + --evidence "$GITHUB_WORKSPACE/pi-intel-evidence" \ + --github-output "$GITHUB_OUTPUT" + - name: Retain exact input, independent output, test, and cleanup evidence + if: always() && steps.pi_intel_evidence.outputs.admitted == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: pi-native-intel-${{ github.run_id }} + include-hidden-files: true path: pi-intel-evidence/ retention-days: 7 if-no-files-found: error diff --git a/scripts/ci/pi-intel/admit_evidence.py b/scripts/ci/pi-intel/admit_evidence.py new file mode 100644 index 00000000000..f5bc2809374 --- /dev/null +++ b/scripts/ci/pi-intel/admit_evidence.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Bound the complete evidence upload; never truncate the tested pack or diagnostics.""" +import argparse +import json +from pathlib import Path +import stat + +MAX_BYTES = 16 * 1024 ** 3 +RETENTION_DAYS = 7 + +def regular_bytes(root): + total = 0 + for path in root.rglob('*'): + mode = path.lstat().st_mode + if stat.S_ISREG(mode): total += path.stat().st_size + elif not stat.S_ISDIR(mode): raise RuntimeError('Evidence contains a symlink or special entry: ' + str(path)) + return total + +def admit_evidence(root, maximum_bytes=MAX_BYTES): + root = Path(root) + if root.is_symlink() or not root.is_dir(): raise RuntimeError('Evidence root missing or linked') + admission = root/'artifact-admission.json' + receipt = root/'receipt.json' + data = {'status':'checking', 'regularFileBytes':0, 'maximumBytes':maximum_bytes, + 'retentionDays':RETENTION_DAYS, 'truncated':False, + 'costScope':'Artifact storage only; standard public-runner minutes are separate.', + 'priceUsdPerGiBMonth':0.25, 'conservativeMonthDays':28, + 'maximumEstimatedStorageUsd':round(maximum_bytes / 1024**3 * RETENTION_DAYS / 28 * 0.25, 6), + 'billingPolicyUrl':'https://docs.github.com/en/billing/concepts/product-billing/github-actions'} + # Include these receipt bytes themselves; settle the decimal-size field before admission. + for _ in range(10): + admission.write_text(json.dumps(data, indent=2)+'\n') + if receipt.exists(): + full = json.loads(receipt.read_text()); full['artifactStorage'] = data + receipt.write_text(json.dumps(full, indent=2)+'\n') + actual = regular_bytes(root) + status = 'admitted_complete_evidence' if actual <= maximum_bytes else 'rejected_storage_cap' + if data['regularFileBytes'] == actual and data['status'] == status: return data + data.update(regularFileBytes=actual, status=status) + raise RuntimeError('Evidence size accounting did not stabilize') + +if __name__ == '__main__': + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--evidence',type=Path,required=True) + parser.add_argument('--github-output',type=Path,required=True) + args=parser.parse_args();result=admit_evidence(args.evidence) + admitted=result['status']=='admitted_complete_evidence' + with args.github_output.open('a') as output:output.write('admitted='+str(admitted).lower()+'\n') + print(json.dumps(result)) + raise SystemExit(0 if admitted else 1) diff --git a/scripts/ci/pi-intel/lifecycle.py b/scripts/ci/pi-intel/lifecycle.py new file mode 100644 index 00000000000..b250bcb4eeb --- /dev/null +++ b/scripts/ci/pi-intel/lifecycle.py @@ -0,0 +1,40 @@ +"""Sticky launch/cleanup gate shared by execution and the narrow regressions.""" +class CleanupUncertain(RuntimeError): + pass + +class Lifecycle: + def __init__(self): + self.pending = False + self.uncertain = False + + @property + def safe(self): + return not self.pending and not self.uncertain + + def begin(self): + if not self.safe: + raise CleanupUncertain('Previous launch/cleanup is unresolved; no next command') + # Set before Popen: an interrupted launch may have no trustworthy handle. + self.pending = True + + def failed(self): + self.uncertain = True + + def confirmed(self): + if self.uncertain: + raise CleanupUncertain('An earlier cleanup failure cannot be cleared') + self.pending = False + + +def run_test_cases(cases, run, lifecycle): + failures = [] + for label, command in cases: + if not lifecycle.safe: + raise CleanupUncertain('Unresolved owner prohibits subsequent test launch') + try: + run(command, label) + except RuntimeError as error: + failures.append(str(error)) + if not lifecycle.safe: + raise CleanupUncertain('Test cleanup uncertain; stopping without another test') from error + return failures diff --git a/scripts/ci/pi-intel/retain_pack.py b/scripts/ci/pi-intel/retain_pack.py new file mode 100644 index 00000000000..b06ac2ab176 --- /dev/null +++ b/scripts/ci/pi-intel/retain_pack.py @@ -0,0 +1,35 @@ +"""Retain exact tested bytes, internal symlinks, and modes outside owned scratch.""" +import hashlib +from pathlib import Path +import tarfile +from source_guard import require, sha + +def retain_pack(pack, output, inventory): + pack, output = Path(pack), Path(output) + require(not output.exists() and not output.resolve().is_relative_to(pack.resolve()), 'Archive must be new and outside pack') + with tarfile.open(output, 'w:gz', compresslevel=1, dereference=False) as archive: + archive.add(pack, arcname='provider-pack') + expected = {row['path']: row for row in inventory} + observed = set() + with tarfile.open(output, 'r:gz') as archive: + for member in archive: + if member.name == 'provider-pack': + require(member.isdir(), 'Archive root must be directory') + continue + relative = member.name.removeprefix('provider-pack/') + require(member.name.startswith('provider-pack/') and relative in expected and relative not in observed, 'Unexpected archive member') + row = expected[relative]; observed.add(relative) + require(member.mode == row['mode'], 'Archive mode mismatch') + if row['kind'] == 'directory': + require(member.isdir(), 'Archive directory mismatch') + elif row['kind'] == 'symlink': + require(member.issym() and member.linkname == row['target'], 'Archive symlink mismatch') + else: + require(member.isfile() and member.size == row['size'], 'Archive file mismatch') + digest = hashlib.sha256() + with archive.extractfile(member) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): digest.update(chunk) + require(digest.hexdigest() == row['sha256'], 'Archive bytes mismatch') + require(observed == set(expected), 'Archive omitted pack members') + return {'path':str(output), 'sha256':sha(output), 'bytes':output.stat().st_size, + 'inventoryVerified':True, 'internalSymlinksAndModesPreserved':True} diff --git a/scripts/ci/pi-intel/test_ci_helpers.py b/scripts/ci/pi-intel/test_ci_helpers.py new file mode 100644 index 00000000000..300e67f721c --- /dev/null +++ b/scripts/ci/pi-intel/test_ci_helpers.py @@ -0,0 +1,89 @@ +import os +from pathlib import Path +import shutil +import tarfile +import tempfile +import unittest +from unittest.mock import Mock +from lifecycle import CleanupUncertain, Lifecycle, run_test_cases +from retain_pack import retain_pack +from source_guard import tree, sha +from admit_evidence import admit_evidence, regular_bytes + +class CleanupRegression(unittest.TestCase): + def test_first_cleanup_failure_prevents_second_test_and_scratch_removal(self): + lifecycle = Lifecycle(); launched = []; remove_scratch = Mock() + def run(command, label): + lifecycle.begin(); launched.append(label) + try: + raise RuntimeError('owner.stop inspection failed') + finally: + lifecycle.failed() + with self.assertRaises(CleanupUncertain): + run_test_cases([('startup', []), ('sdk', [])], run, lifecycle) + if lifecycle.safe: remove_scratch() + self.assertEqual(launched, ['startup']); remove_scratch.assert_not_called() + with self.assertRaises(CleanupUncertain): lifecycle.confirmed() + with self.assertRaises(CleanupUncertain): lifecycle.begin() + self.assertFalse(lifecycle.safe) + + def test_interrupted_launch_without_handle_retains_scratch(self): + lifecycle = Lifecycle(); lifecycle.begin() + # Popen may be interrupted before Python receives a handle: pending is unsafe. + launched = Mock() + with self.assertRaises(CleanupUncertain): + run_test_cases([('sdk', [])], launched, lifecycle) + launched.assert_not_called(); self.assertFalse(lifecycle.safe) + + def test_assertion_failure_with_confirmed_cleanup_allows_independent_sdk_test(self): + lifecycle = Lifecycle(); launched = [] + def run(command, label): + lifecycle.begin(); launched.append(label); lifecycle.confirmed() + if label == 'startup': raise RuntimeError('test assertion failed') + failures = run_test_cases([('startup', []), ('sdk', [])], run, lifecycle) + self.assertEqual(launched, ['startup', 'sdk']) + self.assertEqual(failures, ['test assertion failed']); self.assertTrue(lifecycle.safe) + +class PackRetentionRegression(unittest.TestCase): + def test_full_pack_survives_scratch_removal_with_bytes_links_and_modes(self): + with tempfile.TemporaryDirectory() as root: + root = Path(root); pack = root/'scratch'/'provider-pack'; pack.mkdir(parents=True) + (pack/'bin').mkdir(); (pack/'bin'/'node').write_bytes(b'exact-tested-bytes') + (pack/'bin'/'node').chmod(0o755); (pack/'node').symlink_to('bin/node') + (pack/'.hidden').write_bytes(b'inventory-must-include-hidden'); (pack/'.hidden').chmod(0o444) + inventory = tree(pack, allow_links=True) + archive = root/'retained-pack.tar.gz' + receipt = retain_pack(pack, archive, inventory) + self.assertEqual(receipt['sha256'], sha(archive)); self.assertTrue(receipt['inventoryVerified']) + shutil.rmtree(root/'scratch') + restored = root/'restored'; restored.mkdir() + # This archive was created above and every member was verified against our owned inventory. + # The data filter intentionally rewrites read-only modes, so it cannot test exact restoration. + with tarfile.open(archive) as tar: tar.extractall(restored, filter='fully_trusted') + self.assertEqual(tree(restored/'provider-pack', allow_links=True), inventory) + self.assertEqual(os.readlink(restored/'provider-pack/node'), 'bin/node') + +class ArtifactStorageRegression(unittest.TestCase): + def test_complete_evidence_size_includes_receipts_and_hidden_files(self): + with tempfile.TemporaryDirectory() as root: + root=Path(root); (root/'receipt.json').write_text('{}'); (root/'.hidden').write_bytes(b'exact') + result=admit_evidence(root, maximum_bytes=1024*1024) + self.assertEqual(result['status'], 'admitted_complete_evidence') + self.assertEqual(result['regularFileBytes'], regular_bytes(root)) + self.assertEqual(result['retentionDays'], 7) + self.assertFalse(result['truncated']) + + def test_over_cap_rejects_upload_without_truncating_pack(self): + with tempfile.TemporaryDirectory() as root: + root=Path(root); pack=root/'provider-pack.tar.gz'; pack.write_bytes(b'x'*4096) + before=sha(pack); result=admit_evidence(root, maximum_bytes=1024) + self.assertEqual(result['status'], 'rejected_storage_cap') + self.assertEqual(sha(pack), before) + self.assertEqual(result['regularFileBytes'], regular_bytes(root)) + + def test_symlink_cannot_escape_storage_accounting(self): + with tempfile.TemporaryDirectory() as root: + root=Path(root); (root/'escape').symlink_to('/etc/hosts') + with self.assertRaisesRegex(RuntimeError, 'symlink'): admit_evidence(root) + +if __name__ == '__main__': unittest.main() diff --git a/scripts/ci/pi-intel/verify.py b/scripts/ci/pi-intel/verify.py index b353e59ba93..adfff513a9d 100644 --- a/scripts/ci/pi-intel/verify.py +++ b/scripts/ci/pi-intel/verify.py @@ -5,6 +5,8 @@ from pathlib import Path from closed_inventory import closed_tree from owned_processes import OwnedProcesses,atomic_json import source_guard +from lifecycle import Lifecycle, run_test_cases +from retain_pack import retain_pack HERE=Path(__file__).resolve().parent INPUTS=json.loads((HERE/'profile-inputs.json').read_text()) @@ -36,28 +38,30 @@ def execute(args): 'RUSTUP_TOOLCHAIN':'1.97.1','CARGO_BUILD_JOBS':'2','npm_config_audit':'false','npm_config_fund':'false', 'npm_config_update_notifier':'false','npm_config_package_import_method':'copy'} Path(env['HOME']).mkdir(mode=0o700) - active=None;owner=None;cleanup_ok=True + active=None;owner=None;lifecycle=Lifecycle() def stop_signal(signum,_):raise InterruptedError('CI stopped with signal '+str(signum)) signal.signal(signal.SIGTERM,stop_signal) def run(command,label,timeout,cwd=stage,command_env=None,owned=False): - nonlocal active,owner,cleanup_ok + nonlocal active,owner + lifecycle.begin() row={'label':label,'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'command':list(map(str,command)),'deadlineSeconds':timeout,'status':'running'} receipt['commands'].append(row);save();start=time.monotonic() with (out/(label+'.log')).open('x') as log: try: + if owned:owner=OwnedProcesses(out/(label+'-processes.json'),scratch,stage,pack/'dist/cli/acpx-runtime-sidecar.cjs') active=subprocess.Popen(row['command'],cwd=cwd,env=command_env or env,stdout=log,stderr=log,start_new_session=True) - if owned:cleanup_ok=False;owner=OwnedProcesses(out/(label+'-processes.json'),scratch,stage,pack/'dist/cli/acpx-runtime-sidecar.cjs') while active.poll() is None: if owner:owner.observe(active.pid) require(time.monotonic()-start