ci(runner): verify closed pack hardlinks and asset presence

Co-Authored-By: Paperclip <noreply@paperclip.ing>

scripts/ci/pi-intel/closed_inventory.py

@@ -15,7 +15,7 @@ def closed_tree(root):
         row = {'path': path.relative_to(root).as_posix(), 'mode': stat.S_IMODE(info.st_mode)}
         if stat.S_ISLNK(info.st_mode):
             if not path.resolve(strict=True).is_relative_to(root.resolve()):
-                raise RuntimeError('Symlink escapes pack')
+                raise RuntimeError(f'Symlink escapes pack: path={row["path"]!r}')
             row.update(kind='symlink', target=os.readlink(path))
         elif stat.S_ISDIR(info.st_mode):
             row.update(kind='directory')
@@ -29,7 +29,7 @@ def closed_tree(root):
             row.update(kind='file', sha256=digest.hexdigest(), size=info.st_size,
                        hardlinkGroup=groups[identity][0][0].relative_to(root).as_posix(), links=info.st_nlink)
         else:
-            raise RuntimeError('Nonregular pack file')
+            raise RuntimeError(f'Nonregular pack file: path={row["path"]!r} type={stat.S_IFMT(info.st_mode):#o} links={info.st_nlink}')
         rows.append(row)
     for identity, members in groups.items():
         for path, before in members:
@@ -37,5 +37,5 @@ def closed_tree(root):
             if ((after.st_dev, after.st_ino) != identity or after.st_nlink != len(members)
                 or after.st_size != before.st_size or after.st_mtime_ns != before.st_mtime_ns
                 or after.st_mode != before.st_mode):
-                raise RuntimeError('Pack hard link escapes inventory or file changed')
+                raise RuntimeError(f'Pack hard link escapes inventory or file changed: path={path.relative_to(root).as_posix()!r} links={after.st_nlink} inventoriedLinks={len(members)}')
     return rows

scripts/ci/pi-intel/retain_pack.py

@@ -1,23 +1,28 @@
 """Retain exact tested bytes, internal symlinks, and modes outside owned scratch."""
 import hashlib
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 import tarfile
 from source_guard import require, sha
 
-def retain_pack(pack, output, inventory):
-    pack, output = Path(pack), Path(output)
-    require(not output.exists() and not output.resolve().is_relative_to(pack.resolve()), 'Archive must be new and outside pack')
-    with tarfile.open(output, 'w:gz', compresslevel=1, dereference=False) as archive:
-        archive.add(pack, arcname='provider-pack')
+def archive_relative(name):
+    path = PurePosixPath(name)
+    require(name.startswith('provider-pack/') and not path.is_absolute()
+            and '..' not in path.parts and path.as_posix() == name, 'Unsafe archive member path')
+    return name.removeprefix('provider-pack/')
+
+def verify_archive(output, inventory):
     expected = {row['path']: row for row in inventory}
+    require(len(expected) == len(inventory), 'Duplicate inventory member')
     observed = set()
+    regular_members = set()
+    archived_groups = {}
     with tarfile.open(output, 'r:gz') as archive:
         for member in archive:
             if member.name == 'provider-pack':
                 require(member.isdir(), 'Archive root must be directory')
                 continue
-            relative = member.name.removeprefix('provider-pack/')
-            require(member.name.startswith('provider-pack/') and relative in expected and relative not in observed, 'Unexpected archive member')
+            relative = archive_relative(member.name)
+            require(relative in expected and relative not in observed, 'Unexpected archive member')
             row = expected[relative]; observed.add(relative)
             require(member.mode == row['mode'], 'Archive mode mismatch')
             if row['kind'] == 'directory':
@@ -25,11 +30,35 @@ def retain_pack(pack, output, inventory):
             elif row['kind'] == 'symlink':
                 require(member.issym() and member.linkname == row['target'], 'Archive symlink mismatch')
             else:
-                require(member.isfile() and member.size == row['size'], 'Archive file mismatch')
+                require(row['kind'] == 'file', 'Unknown inventory member kind')
+                if member.islnk():
+                    target = archive_relative(member.linkname)
+                    require(target in regular_members, 'Archive hard link target must be an earlier regular file')
+                    target_row = expected[target]
+                    require(row.get('links', 1) > 1 and member.size == 0
+                            and row.get('hardlinkGroup') == target_row.get('hardlinkGroup')
+                            and all(row[key] == target_row[key] for key in ('sha256', 'size', 'mode', 'links')),
+                            'Archive hard link identity mismatch')
+                else:
+                    require(member.isfile() and member.size == row['size'], 'Archive file mismatch')
+                    if row.get('links', 1) > 1:
+                        group = row['hardlinkGroup']
+                        require(group not in archived_groups, 'Archive lost hard link identity')
+                        archived_groups[group] = relative
+                    regular_members.add(relative)
                 digest = hashlib.sha256()
+                size = 0
                 with archive.extractfile(member) as stream:
-                    for chunk in iter(lambda: stream.read(1024 * 1024), b''): digest.update(chunk)
-                require(digest.hexdigest() == row['sha256'], 'Archive bytes mismatch')
+                    for chunk in iter(lambda: stream.read(1024 * 1024), b''):
+                        digest.update(chunk); size += len(chunk)
+                require(size == row['size'] and digest.hexdigest() == row['sha256'], 'Archive bytes mismatch')
     require(observed == set(expected), 'Archive omitted pack members')
+
+def retain_pack(pack, output, inventory):
+    pack, output = Path(pack), Path(output)
+    require(not output.exists() and not output.resolve().is_relative_to(pack.resolve()), 'Archive must be new and outside pack')
+    with tarfile.open(output, 'w:gz', compresslevel=1, dereference=False) as archive:
+        archive.add(pack, arcname='provider-pack')
+    verify_archive(output, inventory)
     return {'path':str(output), 'sha256':sha(output), 'bytes':output.stat().st_size,
             'inventoryVerified':True, 'internalSymlinksAndModesPreserved':True}

scripts/ci/pi-intel/source_guard.py

@@ -5,6 +5,7 @@ import os
 from pathlib import Path
 import stat
 import subprocess
+from closed_inventory import closed_tree
 
 ASSETS = 'packages/paperclip-runner/provider-assets'
 
@@ -32,7 +33,8 @@ def tree(root, allow_links=False):
         elif stat.S_ISDIR(info.st_mode):
             item.update(kind='directory')
         else:
-            require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1, 'Inventory requires private regular files')
+            require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1,
+                    f'Inventory requires private regular files: path={item["path"]!r} type={stat.S_IFMT(info.st_mode):#o} links={info.st_nlink}')
             item.update(kind='file', sha256=sha(path), size=info.st_size)
         result.append(item)
     return result
@@ -62,14 +64,26 @@ def verify_pack(pack, source, pins):
     check(payload)
     return manifest
 
+def capture_authority(stage, pack):
+    stage, pack = Path(stage), Path(pack)
+    return {'schema': 'pi-intel/build-authority/v1', 'pack': closed_tree(pack),
+            'dist': tree(stage / 'packages/paperclip-runner/dist'),
+            'packAssets': tree(pack / 'provider-assets'),
+            'sourceAssets': tree(stage / ASSETS) if os.path.lexists(stage / ASSETS) else None}
+
 def verify_authority(stage, pack, expected):
     """Recompute every byte before the generated-directory source exception."""
     stage, pack = Path(stage), Path(pack)
     require(expected is not None, 'Generated assets require a captured immutable inventory')
-    require(tree(pack, allow_links=True) == expected['pack'], 'Immutable pack inventory drift')
+    require(expected.get('schema') == 'pi-intel/build-authority/v1', 'Unknown build authority schema')
+    require(closed_tree(pack) == expected['pack'], 'Immutable pack inventory drift')
     require(tree(stage / 'packages/paperclip-runner/dist') == expected['dist'], 'Controller dist inventory drift')
-    assets = tree(stage / ASSETS)
-    require(assets == expected['assets'] == tree(pack / 'provider-assets'), 'Generated asset inventory drift')
+    pack_assets = tree(pack / 'provider-assets')
+    require(pack_assets == expected['packAssets'], 'Packed asset inventory drift')
+    source_assets = tree(stage / ASSETS) if os.path.lexists(stage / ASSETS) else None
+    require(source_assets == expected['sourceAssets'], 'Source asset presence or inventory drift')
+    if source_assets is not None:
+        require(source_assets == pack_assets, 'Source and packed asset inventories differ')
     # Every packed dist file must match the actual local controller copy.
     for item in tree(pack / 'dist'):
         if item['kind'] == 'file':
@@ -82,11 +96,11 @@ def verify_source(stage, source, lock_sha, pack=None, authority=None):
     require(head == source, 'Stage HEAD changed')
     require(sha(stage / 'pnpm-lock.yaml') == lock_sha, 'Private overlay changed')
     generated = os.path.lexists(stage / ASSETS)
-    if generated:
-        require(pack is not None, 'Generated assets cannot be admitted before pack inventory')
+    if authority is not None:
+        require(pack is not None, 'Build authority requires the captured pack')
         verify_authority(stage, pack, authority)
     else:
-        require(authority is None, 'Verified generated authority disappeared')
+        require(not generated, 'Generated assets cannot be admitted before pack inventory')
     status = subprocess.check_output(['git', 'status', '--porcelain=v1', '-z', '--untracked-files=all'], cwd=stage)
     entries = [entry.decode('utf8') for entry in status.split(b'\0') if entry]
     require(' M pnpm-lock.yaml' in entries, 'Expected unstaged private lock overlay missing')
@@ -95,5 +109,5 @@ def verify_source(stage, source, lock_sha, pack=None, authority=None):
             continue
         require(generated and entry.startswith('?? ' + ASSETS + '/'), 'Unexpected source/untracked change')
         path = entry[3:]
-        require(path in {ASSETS + '/' + row['path'] for row in authority['assets'] if row['kind'] == 'file'}, 'Untracked asset is outside verified inventory')
+        require(path in {ASSETS + '/' + row['path'] for row in authority['sourceAssets'] if row['kind'] == 'file'}, 'Untracked asset is outside verified inventory')
     return {'head': head, 'lockSha256': lock_sha, 'generatedAssetsAdmitted': generated, 'statusEntries': entries}

scripts/ci/pi-intel/test_ci_helpers.py

@@ -1,4 +1,5 @@
 import os
+import io
 from pathlib import Path
 import shutil
 import subprocess
@@ -7,8 +8,9 @@ import tempfile
 import unittest
 from unittest.mock import Mock, patch
 from lifecycle import CleanupUncertain, Lifecycle, run_test_cases
-from retain_pack import retain_pack
-from source_guard import tree, sha
+from retain_pack import retain_pack, verify_archive
+from closed_inventory import closed_tree
+from source_guard import tree, sha, capture_authority, verify_authority, verify_source, ASSETS
 from admit_evidence import admit_evidence, regular_bytes
 from verify import tool_invocation, verify_tool_invocation
 
@@ -47,6 +49,129 @@ class CleanupRegression(unittest.TestCase):
         self.assertEqual(failures, ['test assertion failed']); self.assertTrue(lifecycle.safe)
 
 class PackRetentionRegression(unittest.TestCase):
+    def hardlinked_pack(self, root):
+        pack = root/'pack'; pack.mkdir()
+        (pack/'a').write_bytes(b'exact-tested-bytes'); (pack/'a').chmod(0o755)
+        os.link(pack/'a', pack/'b')
+        (pack/'c').write_bytes(b'different-bytes'); (pack/'c').chmod(0o755)
+        return pack
+
+    def test_internal_hardlinks_survive_full_archive_roundtrip(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root)
+            # This is the exact pre-fix CI failure; strict inventories stay strict.
+            with self.assertRaisesRegex(RuntimeError, "private regular files: path='a'.*links=2"):
+                tree(pack, allow_links=True)
+            inventory = closed_tree(pack)
+            archive = root/'retained.tar.gz'; retain_pack(pack, archive, inventory)
+            shutil.rmtree(pack)
+            with tarfile.open(archive) as tar: tar.extractall(root/'restored', filter='fully_trusted')
+            restored = root/'restored'/'provider-pack'
+            self.assertEqual(closed_tree(restored), inventory)
+            self.assertEqual((restored/'a').stat().st_ino, (restored/'b').stat().st_ino)
+
+    def test_external_hardlink_and_special_file_fail_closed(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root)
+            os.link(pack/'a', root/'outside')
+            with self.assertRaisesRegex(RuntimeError, "escapes inventory.*path='a'.*links=3.*inventoriedLinks=2"):
+                closed_tree(pack)
+            (root/'outside').unlink(); os.mkfifo(pack/'fifo')
+            with self.assertRaisesRegex(RuntimeError, "Nonregular pack file: path='fifo'"):
+                closed_tree(pack)
+
+    def test_archive_rejects_escaping_cyclic_and_wrong_hardlink_targets(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root); inventory = closed_tree(pack)
+            for index, target in enumerate(('../outside', '/outside', 'provider-pack/../outside',
+                                             'provider-pack/b', 'provider-pack/missing', 'provider-pack/c')):
+                with self.subTest(target=target):
+                    archive = root/f'invalid-{index}.tar.gz'
+                    with tarfile.open(archive, 'w:gz') as tar:
+                        for name in ('a', 'c'): tar.add(pack/name, arcname='provider-pack/'+name)
+                        link = tarfile.TarInfo('provider-pack/b'); link.type = tarfile.LNKTYPE
+                        link.linkname = target; link.mode = 0o755; tar.addfile(link)
+                    with self.assertRaises(RuntimeError): verify_archive(archive, inventory)
+
+    def test_archive_rejects_changed_bytes_mode_and_missing_members(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root); inventory = closed_tree(pack)
+            for index, change in enumerate(('bytes', 'mode', 'missing')):
+                with self.subTest(change=change):
+                    archive = root/f'invalid-{index}.tar.gz'
+                    with tarfile.open(archive, 'w:gz') as tar:
+                        for name in ('a', 'b', 'c'):
+                            if change == 'missing' and name == 'c': continue
+                            entry = tar.gettarinfo(pack/name, arcname='provider-pack/'+name)
+                            if change == 'mode' and name == 'b': entry.mode = 0o600
+                            data = (pack/name).read_bytes()
+                            if change == 'bytes' and name == 'a': data = b'x' * len(data)
+                            tar.addfile(entry, io.BytesIO(data) if entry.isfile() else None)
+                    with self.assertRaises(RuntimeError): verify_archive(archive, inventory)
+
+    def test_archive_rejects_special_members_and_lost_link_identity(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root); inventory = closed_tree(pack)
+            for index, change in enumerate(('special', 'escaping', 'lost-link')):
+                with self.subTest(change=change):
+                    archive = root/f'invalid-{index}.tar.gz'
+                    with tarfile.open(archive, 'w:gz') as tar:
+                        for name in ('a', 'b', 'c'):
+                            data = (pack/name).read_bytes()
+                            entry = tarfile.TarInfo('provider-pack/'+name); entry.mode = 0o755
+                            entry.size = len(data)
+                            if name == 'b' and change == 'special': entry.type = tarfile.FIFOTYPE; entry.size = 0
+                            if name == 'b' and change == 'escaping': entry.name = 'provider-pack/../outside'
+                            tar.addfile(entry, io.BytesIO(data) if entry.isfile() else None)
+                    with self.assertRaises(RuntimeError): verify_archive(archive, inventory)
+
+    def test_full_pack_authority_recheck_rejects_mutation(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root); stage = root/'stage'
+            dist = stage/'packages/paperclip-runner/dist'; dist.mkdir(parents=True)
+            assets = stage/'packages/paperclip-runner/provider-assets'; assets.mkdir()
+            (pack/'dist').mkdir(); (pack/'provider-assets').mkdir()
+            authority = capture_authority(stage, pack)
+            verify_authority(stage, pack, authority)
+            (pack/'a').write_bytes(b'changed')
+            with self.assertRaisesRegex(RuntimeError, 'Immutable pack inventory drift'):
+                verify_authority(stage, pack, authority)
+
+    def test_source_assets_absence_and_presence_are_both_bound(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root); stage = root/'stage'
+            dist = stage/'packages/paperclip-runner/dist'; dist.mkdir(parents=True)
+            assets = stage/ASSETS
+            (pack/'dist').mkdir(); (pack/'provider-assets').mkdir()
+            (stage/'pnpm-lock.yaml').write_bytes(b'resolved-lock')
+            git_status = b' M pnpm-lock.yaml\0'
+            def verify(expected):
+                with patch('source_guard.subprocess.check_output', side_effect=['frozen-source\n', git_status]):
+                    return verify_source(stage, 'frozen-source', sha(stage/'pnpm-lock.yaml'), pack, expected)
+            absent = capture_authority(stage, pack)
+            self.assertIsNone(absent['sourceAssets'])
+            self.assertFalse(verify(absent)['generatedAssetsAdmitted'])
+            assets.mkdir()
+            with self.assertRaisesRegex(RuntimeError, 'Source asset presence or inventory drift'): verify(absent)
+            present = capture_authority(stage, pack)
+            self.assertEqual(present['sourceAssets'], [])
+            self.assertTrue(verify(present)['generatedAssetsAdmitted'])
+            assets.rmdir()
+            with self.assertRaisesRegex(RuntimeError, 'Source asset presence or inventory drift'): verify(present)
+
+    def test_pack_and_source_assets_keep_strict_single_link_contract(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); pack = self.hardlinked_pack(root); stage = root/'stage'
+            (stage/'packages/paperclip-runner/dist').mkdir(parents=True)
+            (pack/'dist').mkdir(); (pack/'provider-assets').mkdir()
+            assets = stage/ASSETS; assets.mkdir()
+            for directory in (pack/'provider-assets', assets):
+                with self.subTest(directory=directory.name):
+                    (directory/'a').write_bytes(b'asset'); os.link(directory/'a', directory/'b')
+                    with self.assertRaisesRegex(RuntimeError, 'Inventory requires private regular files'):
+                        capture_authority(stage, pack)
+                    (directory/'a').unlink(); (directory/'b').unlink()
+
     def test_full_pack_survives_scratch_removal_with_bytes_links_and_modes(self):
         with tempfile.TemporaryDirectory() as root:
             root = Path(root); pack = root/'scratch'/'provider-pack'; pack.mkdir(parents=True)

scripts/ci/pi-intel/verify.py

@@ -145,7 +145,7 @@ def execute(args):
         run([node,'packages/paperclip-runner/scripts/build-provider-pack.mjs',pack,'--candidate-providers=pi,copilot,cursor'],'pack-build',600)
         verified=json.loads(run([node,HERE/'verify-pack.mjs',pack,SOURCE,'darwin','x64'],'pack-verify',180))
         require(sha(pack/'node_modules/node/bin/node')==PIN['nodeSha256'],'Pack Node mismatch')
-        authority={'pack':source_guard.tree(pack,allow_links=True),'dist':source_guard.tree(stage/'packages/paperclip-runner/dist'),'assets':source_guard.tree(stage/source_guard.ASSETS)}
+        authority=source_guard.capture_authority(stage,pack)
         source_guard.verify_source(stage,SOURCE,PIN['resolvedLockSha256'],pack,authority)
         atomic_json(out/'pack-inventory.json',closed_tree(pack));shutil.copy2(pack/'provider-pack.json',out/'provider-pack.json');shutil.copy2(daemon,out/'paperclip-runnerd')
         receipt.update(packVerification=verified,packManifestSha256=sha(out/'provider-pack.json'),packInventorySha256=sha(out/'pack-inventory.json'),daemonSha256=sha(daemon),nodeSha256=sha(node),buildInputsMatchLocal=True,outputDigestsAssumedEqual=False)