ci(runner): retain Intel pack and stop on uncertain cleanup

Archive and verify the complete tested pack before startup checks. Make cleanup uncertainty sticky across test cases, retain scratch after interrupted launches, and enforce the reviewed 16 GiB seven-day artifact storage bound without truncation.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/docker-runner-check.yml

@@ -202,11 +202,20 @@ jobs:
           python3 -B trusted-ci/scripts/ci/pi-intel/verify.py \
             --source "$GITHUB_WORKSPACE/candidate" \
             --output "$GITHUB_WORKSPACE/pi-intel-evidence"
-      - name: Retain exact input, independent output, test, and cleanup evidence
+      - name: Admit complete evidence within the reserved storage bound
         if: always()
+        id: pi_intel_evidence
+        timeout-minutes: 2
+        run: |
+          python3 -B trusted-ci/scripts/ci/pi-intel/admit_evidence.py \
+            --evidence "$GITHUB_WORKSPACE/pi-intel-evidence" \
+            --github-output "$GITHUB_OUTPUT"
+      - name: Retain exact input, independent output, test, and cleanup evidence
+        if: always() && steps.pi_intel_evidence.outputs.admitted == 'true'
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: pi-native-intel-${{ github.run_id }}
+          include-hidden-files: true
           path: pi-intel-evidence/
           retention-days: 7
           if-no-files-found: error

scripts/ci/pi-intel/admit_evidence.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Bound the complete evidence upload; never truncate the tested pack or diagnostics."""
+import argparse
+import json
+from pathlib import Path
+import stat
+
+MAX_BYTES = 16 * 1024 ** 3
+RETENTION_DAYS = 7
+
+def regular_bytes(root):
+    total = 0
+    for path in root.rglob('*'):
+        mode = path.lstat().st_mode
+        if stat.S_ISREG(mode): total += path.stat().st_size
+        elif not stat.S_ISDIR(mode): raise RuntimeError('Evidence contains a symlink or special entry: ' + str(path))
+    return total
+
+def admit_evidence(root, maximum_bytes=MAX_BYTES):
+    root = Path(root)
+    if root.is_symlink() or not root.is_dir(): raise RuntimeError('Evidence root missing or linked')
+    admission = root/'artifact-admission.json'
+    receipt = root/'receipt.json'
+    data = {'status':'checking', 'regularFileBytes':0, 'maximumBytes':maximum_bytes,
+            'retentionDays':RETENTION_DAYS, 'truncated':False,
+            'costScope':'Artifact storage only; standard public-runner minutes are separate.',
+            'priceUsdPerGiBMonth':0.25, 'conservativeMonthDays':28,
+            'maximumEstimatedStorageUsd':round(maximum_bytes / 1024**3 * RETENTION_DAYS / 28 * 0.25, 6),
+            'billingPolicyUrl':'https://docs.github.com/en/billing/concepts/product-billing/github-actions'}
+    # Include these receipt bytes themselves; settle the decimal-size field before admission.
+    for _ in range(10):
+        admission.write_text(json.dumps(data, indent=2)+'\n')
+        if receipt.exists():
+            full = json.loads(receipt.read_text()); full['artifactStorage'] = data
+            receipt.write_text(json.dumps(full, indent=2)+'\n')
+        actual = regular_bytes(root)
+        status = 'admitted_complete_evidence' if actual <= maximum_bytes else 'rejected_storage_cap'
+        if data['regularFileBytes'] == actual and data['status'] == status: return data
+        data.update(regularFileBytes=actual, status=status)
+    raise RuntimeError('Evidence size accounting did not stabilize')
+
+if __name__ == '__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--evidence',type=Path,required=True)
+    parser.add_argument('--github-output',type=Path,required=True)
+    args=parser.parse_args();result=admit_evidence(args.evidence)
+    admitted=result['status']=='admitted_complete_evidence'
+    with args.github_output.open('a') as output:output.write('admitted='+str(admitted).lower()+'\n')
+    print(json.dumps(result))
+    raise SystemExit(0 if admitted else 1)

scripts/ci/pi-intel/lifecycle.py

@@ -0,0 +1,40 @@
+"""Sticky launch/cleanup gate shared by execution and the narrow regressions."""
+class CleanupUncertain(RuntimeError):
+    pass
+
+class Lifecycle:
+    def __init__(self):
+        self.pending = False
+        self.uncertain = False
+
+    @property
+    def safe(self):
+        return not self.pending and not self.uncertain
+
+    def begin(self):
+        if not self.safe:
+            raise CleanupUncertain('Previous launch/cleanup is unresolved; no next command')
+        # Set before Popen: an interrupted launch may have no trustworthy handle.
+        self.pending = True
+
+    def failed(self):
+        self.uncertain = True
+
+    def confirmed(self):
+        if self.uncertain:
+            raise CleanupUncertain('An earlier cleanup failure cannot be cleared')
+        self.pending = False
+
+
+def run_test_cases(cases, run, lifecycle):
+    failures = []
+    for label, command in cases:
+        if not lifecycle.safe:
+            raise CleanupUncertain('Unresolved owner prohibits subsequent test launch')
+        try:
+            run(command, label)
+        except RuntimeError as error:
+            failures.append(str(error))
+            if not lifecycle.safe:
+                raise CleanupUncertain('Test cleanup uncertain; stopping without another test') from error
+    return failures

scripts/ci/pi-intel/retain_pack.py

@@ -0,0 +1,35 @@
+"""Retain exact tested bytes, internal symlinks, and modes outside owned scratch."""
+import hashlib
+from pathlib import Path
+import tarfile
+from source_guard import require, sha
+
+def retain_pack(pack, output, inventory):
+    pack, output = Path(pack), Path(output)
+    require(not output.exists() and not output.resolve().is_relative_to(pack.resolve()), 'Archive must be new and outside pack')
+    with tarfile.open(output, 'w:gz', compresslevel=1, dereference=False) as archive:
+        archive.add(pack, arcname='provider-pack')
+    expected = {row['path']: row for row in inventory}
+    observed = set()
+    with tarfile.open(output, 'r:gz') as archive:
+        for member in archive:
+            if member.name == 'provider-pack':
+                require(member.isdir(), 'Archive root must be directory')
+                continue
+            relative = member.name.removeprefix('provider-pack/')
+            require(member.name.startswith('provider-pack/') and relative in expected and relative not in observed, 'Unexpected archive member')
+            row = expected[relative]; observed.add(relative)
+            require(member.mode == row['mode'], 'Archive mode mismatch')
+            if row['kind'] == 'directory':
+                require(member.isdir(), 'Archive directory mismatch')
+            elif row['kind'] == 'symlink':
+                require(member.issym() and member.linkname == row['target'], 'Archive symlink mismatch')
+            else:
+                require(member.isfile() and member.size == row['size'], 'Archive file mismatch')
+                digest = hashlib.sha256()
+                with archive.extractfile(member) as stream:
+                    for chunk in iter(lambda: stream.read(1024 * 1024), b''): digest.update(chunk)
+                require(digest.hexdigest() == row['sha256'], 'Archive bytes mismatch')
+    require(observed == set(expected), 'Archive omitted pack members')
+    return {'path':str(output), 'sha256':sha(output), 'bytes':output.stat().st_size,
+            'inventoryVerified':True, 'internalSymlinksAndModesPreserved':True}

scripts/ci/pi-intel/test_ci_helpers.py

@@ -0,0 +1,89 @@
+import os
+from pathlib import Path
+import shutil
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import Mock
+from lifecycle import CleanupUncertain, Lifecycle, run_test_cases
+from retain_pack import retain_pack
+from source_guard import tree, sha
+from admit_evidence import admit_evidence, regular_bytes
+
+class CleanupRegression(unittest.TestCase):
+    def test_first_cleanup_failure_prevents_second_test_and_scratch_removal(self):
+        lifecycle = Lifecycle(); launched = []; remove_scratch = Mock()
+        def run(command, label):
+            lifecycle.begin(); launched.append(label)
+            try:
+                raise RuntimeError('owner.stop inspection failed')
+            finally:
+                lifecycle.failed()
+        with self.assertRaises(CleanupUncertain):
+            run_test_cases([('startup', []), ('sdk', [])], run, lifecycle)
+        if lifecycle.safe: remove_scratch()
+        self.assertEqual(launched, ['startup']); remove_scratch.assert_not_called()
+        with self.assertRaises(CleanupUncertain): lifecycle.confirmed()
+        with self.assertRaises(CleanupUncertain): lifecycle.begin()
+        self.assertFalse(lifecycle.safe)
+
+    def test_interrupted_launch_without_handle_retains_scratch(self):
+        lifecycle = Lifecycle(); lifecycle.begin()
+        # Popen may be interrupted before Python receives a handle: pending is unsafe.
+        launched = Mock()
+        with self.assertRaises(CleanupUncertain):
+            run_test_cases([('sdk', [])], launched, lifecycle)
+        launched.assert_not_called(); self.assertFalse(lifecycle.safe)
+
+    def test_assertion_failure_with_confirmed_cleanup_allows_independent_sdk_test(self):
+        lifecycle = Lifecycle(); launched = []
+        def run(command, label):
+            lifecycle.begin(); launched.append(label); lifecycle.confirmed()
+            if label == 'startup': raise RuntimeError('test assertion failed')
+        failures = run_test_cases([('startup', []), ('sdk', [])], run, lifecycle)
+        self.assertEqual(launched, ['startup', 'sdk'])
+        self.assertEqual(failures, ['test assertion failed']); self.assertTrue(lifecycle.safe)
+
+class PackRetentionRegression(unittest.TestCase):
+    def test_full_pack_survives_scratch_removal_with_bytes_links_and_modes(self):
+        with tempfile.TemporaryDirectory() as root:
+            root = Path(root); pack = root/'scratch'/'provider-pack'; pack.mkdir(parents=True)
+            (pack/'bin').mkdir(); (pack/'bin'/'node').write_bytes(b'exact-tested-bytes')
+            (pack/'bin'/'node').chmod(0o755); (pack/'node').symlink_to('bin/node')
+            (pack/'.hidden').write_bytes(b'inventory-must-include-hidden'); (pack/'.hidden').chmod(0o444)
+            inventory = tree(pack, allow_links=True)
+            archive = root/'retained-pack.tar.gz'
+            receipt = retain_pack(pack, archive, inventory)
+            self.assertEqual(receipt['sha256'], sha(archive)); self.assertTrue(receipt['inventoryVerified'])
+            shutil.rmtree(root/'scratch')
+            restored = root/'restored'; restored.mkdir()
+            # This archive was created above and every member was verified against our owned inventory.
+            # The data filter intentionally rewrites read-only modes, so it cannot test exact restoration.
+            with tarfile.open(archive) as tar: tar.extractall(restored, filter='fully_trusted')
+            self.assertEqual(tree(restored/'provider-pack', allow_links=True), inventory)
+            self.assertEqual(os.readlink(restored/'provider-pack/node'), 'bin/node')
+
+class ArtifactStorageRegression(unittest.TestCase):
+    def test_complete_evidence_size_includes_receipts_and_hidden_files(self):
+        with tempfile.TemporaryDirectory() as root:
+            root=Path(root); (root/'receipt.json').write_text('{}'); (root/'.hidden').write_bytes(b'exact')
+            result=admit_evidence(root, maximum_bytes=1024*1024)
+            self.assertEqual(result['status'], 'admitted_complete_evidence')
+            self.assertEqual(result['regularFileBytes'], regular_bytes(root))
+            self.assertEqual(result['retentionDays'], 7)
+            self.assertFalse(result['truncated'])
+
+    def test_over_cap_rejects_upload_without_truncating_pack(self):
+        with tempfile.TemporaryDirectory() as root:
+            root=Path(root); pack=root/'provider-pack.tar.gz'; pack.write_bytes(b'x'*4096)
+            before=sha(pack); result=admit_evidence(root, maximum_bytes=1024)
+            self.assertEqual(result['status'], 'rejected_storage_cap')
+            self.assertEqual(sha(pack), before)
+            self.assertEqual(result['regularFileBytes'], regular_bytes(root))
+
+    def test_symlink_cannot_escape_storage_accounting(self):
+        with tempfile.TemporaryDirectory() as root:
+            root=Path(root); (root/'escape').symlink_to('/etc/hosts')
+            with self.assertRaisesRegex(RuntimeError, 'symlink'): admit_evidence(root)
+
+if __name__ == '__main__': unittest.main()

scripts/ci/pi-intel/verify.py

@@ -5,6 +5,8 @@ from pathlib import Path
 from closed_inventory import closed_tree
 from owned_processes import OwnedProcesses,atomic_json
 import source_guard
+from lifecycle import Lifecycle, run_test_cases
+from retain_pack import retain_pack
 
 HERE=Path(__file__).resolve().parent
 INPUTS=json.loads((HERE/'profile-inputs.json').read_text())
@@ -36,28 +38,30 @@ def execute(args):
          'RUSTUP_TOOLCHAIN':'1.97.1','CARGO_BUILD_JOBS':'2','npm_config_audit':'false','npm_config_fund':'false',
          'npm_config_update_notifier':'false','npm_config_package_import_method':'copy'}
     Path(env['HOME']).mkdir(mode=0o700)
-    active=None;owner=None;cleanup_ok=True
+    active=None;owner=None;lifecycle=Lifecycle()
     def stop_signal(signum,_):raise InterruptedError('CI stopped with signal '+str(signum))
     signal.signal(signal.SIGTERM,stop_signal)
     def run(command,label,timeout,cwd=stage,command_env=None,owned=False):
-        nonlocal active,owner,cleanup_ok
+        nonlocal active,owner
+        lifecycle.begin()
         row={'label':label,'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'command':list(map(str,command)),'deadlineSeconds':timeout,'status':'running'}
         receipt['commands'].append(row);save();start=time.monotonic()
         with (out/(label+'.log')).open('x') as log:
             try:
+                if owned:owner=OwnedProcesses(out/(label+'-processes.json'),scratch,stage,pack/'dist/cli/acpx-runtime-sidecar.cjs')
                 active=subprocess.Popen(row['command'],cwd=cwd,env=command_env or env,stdout=log,stderr=log,start_new_session=True)
-                if owned:cleanup_ok=False;owner=OwnedProcesses(out/(label+'-processes.json'),scratch,stage,pack/'dist/cli/acpx-runtime-sidecar.cjs')
                 while active.poll() is None:
                     if owner:owner.observe(active.pid)
                     require(time.monotonic()-start<timeout,'Command deadline exceeded: '+label)
                     time.sleep(.1 if owner else .5)
                 if owner:owner.observe(active.pid)
             finally:
-                if active:
+                try:
+                    # No reliable returned handle means launch ownership is uncertain.
+                    require(active is not None,'Launch interrupted without a reliable process handle')
                     if owner:
                         shutdown=owner.stop(active.pid);row['shutdown']=shutdown
-                        cleanup_ok=shutdown['status']=='stopped' and not owner.live()
-                        require(cleanup_ok,'Owned descendants remain')
+                        require(shutdown['status']=='stopped' and not owner.live(),'Owned descendants remain')
                     elif active.poll() is None:
                         require(os.getpgid(active.pid)==active.pid,'Process group identity changed')
                         os.killpg(active.pid,signal.SIGTERM)
@@ -65,7 +69,13 @@ def execute(args):
                         except subprocess.TimeoutExpired:os.killpg(active.pid,signal.SIGKILL)
                     active.wait(timeout=10)
                     row.update(exitCode=active.returncode,durationSeconds=round(time.monotonic()-start,3),logSha256=sha(out/(label+'.log')),status='passed' if active.returncode==0 else 'failed')
-                    active=None;owner=None;save()
+                    lifecycle.confirmed()
+                    active=None;owner=None
+                except BaseException as error:
+                    lifecycle.failed()
+                    row.update(status='cleanup_uncertain',cleanupErrorType=type(error).__name__)
+                    raise
+                finally:save()
         require(row['exitCode']==0,'Command failed without retry: '+label)
         return (out/(label+'.log')).read_text()
     def source_inventory():
@@ -124,6 +134,8 @@ def execute(args):
         source_guard.verify_source(stage,SOURCE,PIN['resolvedLockSha256'],pack,authority)
         atomic_json(out/'pack-inventory.json',closed_tree(pack));shutil.copy2(pack/'provider-pack.json',out/'provider-pack.json');shutil.copy2(daemon,out/'paperclip-runnerd')
         receipt.update(packVerification=verified,packManifestSha256=sha(out/'provider-pack.json'),packInventorySha256=sha(out/'pack-inventory.json'),daemonSha256=sha(daemon),nodeSha256=sha(node),buildInputsMatchLocal=True,outputDigestsAssumedEqual=False)
+        receipt['providerPackArchive']=retain_pack(pack,out/'provider-pack.tar.gz',authority['pack'])
+        save()
         test=stage/'packages/paperclip-runner/test/pi-closed-startup.test.mjs'
         require(sha(test)=='8967cf9c8cd130b68bf9d64abef8cb8d352af00646e2288b341d8c6ae758b47a','Closed startup test changed')
         receipt['testSourceSha256']={n:sha(stage/'packages/paperclip-runner/test'/n) for n in ['pi-closed-startup.test.mjs','pi-native-package-contract.test.mjs','pi-acp-package-contract.test.mjs']};save()
@@ -134,10 +146,8 @@ def execute(args):
           'PAPERCLIP_TEST_PI_RUNTIME_ROOT':str(pack/'provider-assets/pi/darwin-x64/runtime/node_modules/@earendil-works/pi-coding-agent'),
           'PAPERCLIP_TEST_PI_ACP_PACKAGE':str(pack/'provider-assets/pi/darwin-x64/runtime/node_modules/pi-acp/package.json')}
         # The observer is passed only to the test process; runtime children receive no NODE_OPTIONS.
-        failures=[]
-        for label,command in [('closed-startup',[node,'--import',HERE/'retain-test-state.mjs','--test',test]),('native-extension-contracts',[node,'--test',stage/'packages/paperclip-runner/test/pi-native-package-contract.test.mjs',stage/'packages/paperclip-runner/test/pi-acp-package-contract.test.mjs'])]:
-            try:run(command,label,180,command_env=testenv,owned=True)
-            except RuntimeError as e:failures.append(str(e))
+        cases=[('closed-startup',[node,'--import',HERE/'retain-test-state.mjs','--test',test]),('native-extension-contracts',[node,'--test',stage/'packages/paperclip-runner/test/pi-native-package-contract.test.mjs',stage/'packages/paperclip-runner/test/pi-acp-package-contract.test.mjs'])]
+        failures=run_test_cases(cases,lambda command,label:run(command,label,180,command_env=testenv,owned=True),lifecycle)
         source_guard.verify_source(stage,SOURCE,PIN['resolvedLockSha256'],pack,authority)
         require(source_inventory()==json.loads((out/'source-input-inventory.json').read_text()),'Tracked source changed')
         require(closed_tree(pack)==json.loads((out/'pack-inventory.json').read_text()),'Pack changed during tests')
@@ -149,8 +159,9 @@ def execute(args):
     finally:
         if scratch.exists():
             st=scratch.stat();require((st.st_dev,st.st_ino,st.st_uid)==identity and st.st_uid==os.getuid(),'Scratch ownership changed')
-            if cleanup_ok:shutil.rmtree(scratch);receipt['scratchCleanup']='removed_verified_owned_root'
+            if lifecycle.safe:shutil.rmtree(scratch);receipt['scratchCleanup']='removed_verified_owned_root'
             else:receipt['scratchCleanup']='retained_unresolved_processes'
+        receipt['cleanupUncertain']=not lifecycle.safe
         receipt['finishedAt']=datetime.datetime.now(datetime.timezone.utc).isoformat();save()
 
 if __name__=='__main__':