ci(runner): add bounded hosted E2E support recheck

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/docker-runner-check.yml

@@ -19,6 +19,10 @@ on:
         description: "With verify_source, run only the complete pnpm test:run suite (180-minute bound)"
         type: boolean
         default: false
+      source_e2e_support_only:
+        description: "Run only E2E support typecheck, complete unit suite and catalog on the exact reviewed coverage source"
+        type: boolean
+        default: false
       verify_runner:
         description: "Run the complete offline Runner verify command on GitHub-hosted Ubuntu without building an image"
         type: boolean
@@ -32,11 +36,11 @@ on:
         type: boolean
         default: false
       expected_source_sha:
-        description: "Require this immutable target commit (required for verify_runner or verify_source)"
+        description: "Require this immutable target commit (required for verify_runner, verify_source or source_e2e_support_only)"
         type: string
         required: false
       expected_resolved_lock_sha256:
-        description: "Require this reviewed resolved dependency lock (required for verify_runner or verify_source)"
+        description: "Require this reviewed resolved dependency lock (required for verify_runner, verify_source or source_e2e_support_only)"
         type: string
         required: false
       verify_public_install:
@@ -61,7 +65,7 @@ permissions: {}
 
 concurrency:
   # Publishing a newer image must not discard an earlier verification's evidence.
-  group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }}
+  group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }}
   cancel-in-progress: true
 
 jobs:
@@ -113,10 +117,11 @@ jobs:
           VERIFY_RUNNER: ${{ inputs.verify_runner }}
           VERIFY_SOURCE: ${{ inputs.verify_source }}
           SOURCE_ROOT_TESTS_ONLY: ${{ inputs.source_root_tests_only }}
+          SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }}
           IMAGE_MODE: ${{ inputs.publish_eval_image || inputs.verify_public_install || inputs.build_eval_viewer }}
           VERIFY_PI_INTEL: ${{ inputs.verify_pi_intel }}
           DIAGNOSE_AJV: ${{ inputs.diagnose_ajv_pack }}
-          OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel }}
+          OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel || inputs.source_e2e_support_only }}
         run: |
           set -euo pipefail
           test "$REPOSITORY" = paperclipai/paperclip
@@ -131,10 +136,21 @@ jobs:
           if [ "$SOURCE_ROOT_TESTS_ONLY" = true ]; then
             test "$VERIFY_SOURCE" = true
           fi
-          if [ "$VERIFY_RUNNER" = true ] || [ "$VERIFY_SOURCE" = true ]; then
+          if [ "$VERIFY_RUNNER" = true ] || [ "$VERIFY_SOURCE" = true ] || [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then
             [[ "$EXPECTED_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
             [[ "$EXPECTED_LOCK_SHA256" =~ ^[0-9a-f]{64}$ ]]
           fi
+          if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then
+            test "$VERIFY_SOURCE" != true
+            test "$SOURCE_ROOT_TESTS_ONLY" != true
+            test "$VERIFY_RUNNER" != true
+            test "$VERIFY_PI_INTEL" != true
+            test "$DIAGNOSE_AJV" != true
+            test "$IMAGE_MODE" != true
+            test "$EXPECTED_SOURCE_SHA" = 5cd6d3d5237e3e15394fcc1c7e754c30a63c5169
+            test "$EXPECTED_LOCK_SHA256" = 38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba
+            test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public
+          fi
           if [ "$VERIFY_SOURCE" = true ]; then
             test "$VERIFY_RUNNER" != true
             test "$IMAGE_MODE" != true
@@ -296,11 +312,11 @@ jobs:
           retention-days: 14
 
   manual_source_verify:
-    name: Full source verification on GitHub-hosted Ubuntu
-    if: github.event_name == 'workflow_dispatch' && inputs.verify_source
+    name: ${{ inputs.source_e2e_support_only && 'Focused E2E support verification on GitHub-hosted Ubuntu' || 'Full source verification on GitHub-hosted Ubuntu' }}
+    if: github.event_name == 'workflow_dispatch' && (inputs.verify_source || inputs.source_e2e_support_only)
     needs: authorize_manual
     runs-on: ubuntu-latest
-    timeout-minutes: ${{ inputs.source_root_tests_only && 200 || 295 }}
+    timeout-minutes: ${{ inputs.source_e2e_support_only && 45 || inputs.source_root_tests_only && 200 || 295 }}
     permissions:
       contents: read
     env:
@@ -319,6 +335,7 @@ jobs:
           SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
           EXPECTED_LOCK_SHA256: ${{ inputs.expected_resolved_lock_sha256 }}
           SOURCE_ROOT_TESTS_ONLY: ${{ inputs.source_root_tests_only }}
+          SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }}
         run: |
           set -euo pipefail
           mkdir -p remote-source-verification
@@ -339,6 +356,16 @@ jobs:
           if [ "$SOURCE_ROOT_TESTS_ONLY" = true ]; then
             printf '%s\n' 'pnpm test:run' > remote-source-verification/commands.txt
           fi
+          if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then
+            printf '%s  pnpm-lock.yaml\n' e11d69fa8702a906c293c1cb307c71df90d3b1f1617b3c23ebf17fa9a87fec79 | sha256sum --check --strict
+            git ls-tree -r -z HEAD > remote-source-verification/source-tree.zlist
+            git archive --format=tar HEAD | sha256sum > remote-source-verification/source-archive.sha256
+            printf '%s\n' \
+              'pnpm test:e2e:runner:typecheck' \
+              'pnpm test:e2e:runner:unit' \
+              'node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/launch.ts --list' \
+              > remote-source-verification/commands.txt
+          fi
           index=0
           while IFS= read -r _check; do
             index=$((index + 1))
@@ -389,12 +416,15 @@ jobs:
             pnpm --filter @paperclipai/plugin-sdk ensure-build-deps \
             > remote-source-verification/test-build-deps.log 2>&1
       - name: Run every source check and retain each result
-        timeout-minutes: ${{ inputs.source_root_tests_only && 182 || 273 }}
+        timeout-minutes: ${{ inputs.source_e2e_support_only && 28 || inputs.source_root_tests_only && 182 || 273 }}
+        env:
+          SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }}
         run: |
           set -uo pipefail
           status=0
           index=0
           progress_pid=
+          checks_started_at=$(date +%s)
           trap 'if [ -n "$progress_pid" ]; then kill "$progress_pid" 2>/dev/null || true; fi' EXIT
           while IFS= read -r check; do
             index=$((index + 1))
@@ -404,6 +434,12 @@ jobs:
             # alone exceed 85 minutes; ordinary CI distributes them across shards.
             check_timeout=15m
             if [ "$check" = 'pnpm test:run' ]; then check_timeout=180m; fi
+            remaining=900
+            if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then
+              # Leave job time for final auditing and complete evidence upload.
+              remaining=$((1380 - $(date +%s) + checks_started_at))
+              if [ "$remaining" -lt 900 ] && [ "$remaining" -gt 0 ]; then check_timeout="${remaining}s"; fi
+            fi
             date -u +%FT%TZ > "remote-source-verification/check-$index.started-at"
             started_at=$(date +%s)
             (
@@ -414,7 +450,12 @@ jobs:
             ) &
             progress_pid=$!
             check_status=0
-            timeout --verbose --signal=TERM --kill-after=15s "$check_timeout" bash -c "$check" > "remote-source-verification/check-$index.log" 2>&1 || check_status=$?
+            if [ "$remaining" -gt 0 ]; then
+              timeout --verbose --signal=TERM --kill-after=15s "$check_timeout" bash -c "$check" > "remote-source-verification/check-$index.log" 2>&1 || check_status=$?
+            else
+              check_status=124
+              printf 'Not launched: focused command budget exhausted.\n' > "remote-source-verification/check-$index.log"
+            fi
             kill "$progress_pid" 2>/dev/null || true
             wait "$progress_pid" 2>/dev/null || true
             progress_pid=
@@ -426,6 +467,26 @@ jobs:
             echo "Finished $check (exit $check_status)"
           done < remote-source-verification/commands.txt
           exit "$status"
+      - name: Audit focused source and lock closure even on check failure
+        if: always() && inputs.source_e2e_support_only
+        timeout-minutes: 2
+        env:
+          SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
+          EXPECTED_LOCK_SHA256: ${{ inputs.expected_resolved_lock_sha256 }}
+        run: |
+          set -uo pipefail
+          mkdir -p remote-source-verification
+          status=0
+          date -u +%FT%TZ > remote-source-verification/final-audit.started-at
+          git rev-parse HEAD > remote-source-verification/final-source.txt
+          test "$(cat remote-source-verification/final-source.txt)" = "$SOURCE_SHA" || status=1
+          git status --porcelain=v1 --untracked-files=no > remote-source-verification/final-source-status.txt
+          git diff -- . ':(exclude)pnpm-lock.yaml' > remote-source-verification/final-source.diff
+          git diff --quiet -- . ':(exclude)pnpm-lock.yaml' || status=1
+          printf '%s  pnpm-lock.yaml\n' "$EXPECTED_LOCK_SHA256" | sha256sum --check --strict > remote-source-verification/final-overlay-check.log 2>&1 || status=1
+          date -u +%FT%TZ > remote-source-verification/final-audit.finished-at
+          printf '%s\n' "$status" > remote-source-verification/final-audit.status
+          exit "$status"
       - name: Capture the final lock state even on resolution failure
         if: always()
         run: |
@@ -435,13 +496,37 @@ jobs:
             sha256sum pnpm-lock.yaml > remote-source-verification/final-lock.sha256
             git diff -- pnpm-lock.yaml > remote-source-verification/final-lock.diff
           fi
+      - name: Admit complete focused evidence within the storage bound
+        if: always() && inputs.source_e2e_support_only
+        id: source_e2e_evidence
+        timeout-minutes: 2
+        run: |
+          python3 - <<'PYTHON'
+          import hashlib, json, os, pathlib
+          root = pathlib.Path('remote-source-verification')
+          files, size, limit = [], 0, 256 * 1024 * 1024
+          for path in sorted(root.rglob('*')):
+              if path.is_symlink(): raise RuntimeError('Evidence symlink is not allowed')
+              if path.is_dir(): continue
+              if not path.is_file(): raise RuntimeError('Nonregular evidence is not allowed')
+              size += path.stat().st_size
+              if size > limit: raise RuntimeError('Complete evidence exceeds 256 MiB bound')
+              digest = hashlib.sha256()
+              with path.open('rb') as source:
+                  for chunk in iter(lambda: source.read(1024 * 1024), b''): digest.update(chunk)
+              files.append({'path': str(path.relative_to(root)), 'bytes': path.stat().st_size, 'sha256': digest.hexdigest()})
+          inventory = json.dumps({'bytes': size, 'files': files}, indent=2) + '\n'
+          if not files or size + len(inventory.encode()) > limit: raise RuntimeError('Complete evidence exceeds bound or is absent')
+          (root / 'evidence-inventory.json').write_text(inventory)
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as output: output.write('admitted=true\n')
+          PYTHON
       - name: Retain source verification evidence even on failure
-        if: always()
+        if: always() && (!inputs.source_e2e_support_only || steps.source_e2e_evidence.outputs.admitted == 'true')
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: ${{ inputs.source_root_tests_only && 'source-root-tests' || 'source-full-verification' }}-${{ github.run_id }}
+          name: ${{ inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || 'source-full-verification' }}-${{ github.run_id }}
           path: remote-source-verification/
-          retention-days: 14
+          retention-days: ${{ inputs.source_e2e_support_only && 7 || 14 }}
 
   manual_ajv_pack_diagnostic:
     name: Pinned AJV directory-pack diagnostic on Linux
@@ -506,7 +591,7 @@ jobs:
 
   manual_image:
     name: Build and verify on EC2
-    if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel
+    if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel && !inputs.source_e2e_support_only
     needs: authorize_manual
     runs-on: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
     timeout-minutes: 90

scripts/ci/test_source_e2e_mode.py

@@ -0,0 +1,136 @@
+"""Offline regressions for the reviewed, exclusive hosted support dispatch."""
+import os
+from pathlib import Path
+import re
+import shutil
+import sys
+import subprocess
+import tempfile
+import textwrap
+import unittest
+
+ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW = '.github/workflows/docker-runner-check.yml'
+SOURCE = '5cd6d3d5237e3e15394fcc1c7e754c30a63c5169'
+LOCK = '38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba'
+INTEL_SOURCE = '5eba61ece929dcfb2b0c1761825a2d9e557c2554'
+BASELINE = 'b58907578'
+
+
+def job(text, name):
+    return re.search(r'^  ' + name + r':\n.*?(?=^  [a-z_]+:\n|\Z)', text, re.M | re.S).group()
+
+
+def script(block, name):
+    step = block.split('      - name: ' + name + '\n', 1)[1].split('\n      - ', 1)[0]
+    return textwrap.dedent(step.split('        run: |\n', 1)[1]).rstrip() + '\n'
+
+
+class SupportMode(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.workflow = (ROOT / WORKFLOW).read_text()
+        cls.authorize = script(job(cls.workflow, 'authorize_manual'), 'Authorize an explicit maintainer image build')
+
+    def admission(self, **changes):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            fake = root / 'gh'
+            fake.write_text('''#!/usr/bin/env python3
+import os, sys
+path = sys.argv[2]
+if path == 'users/maintainer': print('1')
+elif path == 'repos/paperclipai/paperclip/git/ref/heads/coverage': print(os.environ['FAKE_TARGET'])
+elif path == 'repos/paperclipai/paperclip': print(os.environ['FAKE_VISIBILITY'])
+else: raise RuntimeError('Unexpected offline GitHub request')
+''')
+            fake.chmod(0o755)
+            env = dict(PATH=str(root) + os.pathsep + str(Path(shutil.which('jq')).parent) + os.pathsep + os.defpath,
+                       REPOSITORY='paperclipai/paperclip', REPOSITORY_ID='1170821064', ACTOR_ID='1',
+                       TRIGGERING_ACTOR='maintainer', ALLOWED_IDS='[1]', TARGET_BRANCH='coverage',
+                       EXPECTED_SOURCE_SHA=SOURCE, EXPECTED_LOCK_SHA256=LOCK, FAKE_TARGET=SOURCE,
+                       FAKE_VISIBILITY='public', VERIFY_RUNNER='false', VERIFY_SOURCE='false',
+                       SOURCE_ROOT_TESTS_ONLY='false', SOURCE_E2E_SUPPORT_ONLY='true', IMAGE_MODE='false',
+                       VERIFY_PI_INTEL='false', DIAGNOSE_AJV='false', OTHER_MODE='true',
+                       GITHUB_OUTPUT=str(root / 'output'))
+            env.update(changes)
+            result = subprocess.run(['bash', '-c', self.authorize], env=env, capture_output=True, text=True, timeout=5)
+            return result.returncode, (root / 'output').read_text() if (root / 'output').exists() else ''
+
+    def test_exact_reviewed_source_and_overlay_are_admitted(self):
+        self.assertEqual(self.admission(), (0, 'target_sha=' + SOURCE + '\n'))
+
+    def test_every_other_execution_mode_conflicts(self):
+        for mode in ('VERIFY_RUNNER', 'VERIFY_SOURCE', 'SOURCE_ROOT_TESTS_ONLY', 'IMAGE_MODE', 'VERIFY_PI_INTEL', 'DIAGNOSE_AJV'):
+            with self.subTest(mode=mode):
+                status, output = self.admission(**{mode: 'true'})
+                self.assertNotEqual(status, 0); self.assertEqual(output, '')
+
+    def test_wrong_missing_or_moving_source_and_lock_fail(self):
+        for changes in ({'EXPECTED_SOURCE_SHA': ''}, {'EXPECTED_SOURCE_SHA': INTEL_SOURCE},
+                        {'FAKE_TARGET': INTEL_SOURCE}, {'EXPECTED_LOCK_SHA256': ''},
+                        {'EXPECTED_LOCK_SHA256': 'a' * 64}, {'FAKE_VISIBILITY': 'private'}, {'ACTOR_ID': '2'}):
+            with self.subTest(changes=changes):
+                status, output = self.admission(**changes)
+                self.assertNotEqual(status, 0); self.assertEqual(output, '')
+
+    def test_original_intel_admission_and_entire_job_are_unchanged(self):
+        self.assertEqual(self.admission(SOURCE_E2E_SUPPORT_ONLY='false', VERIFY_PI_INTEL='true',
+                                       EXPECTED_SOURCE_SHA=INTEL_SOURCE, FAKE_TARGET=INTEL_SOURCE),
+                         (0, 'target_sha=' + INTEL_SOURCE + '\n'))
+        baseline = subprocess.check_output(['git', 'show', BASELINE + ':' + WORKFLOW], cwd=ROOT, text=True)
+        self.assertEqual(job(self.workflow, 'manual_pi_intel'), job(baseline, 'manual_pi_intel'))
+        for name in ('manual_runner_verify', 'manual_ajv_pack_diagnostic'):
+            self.assertEqual(job(self.workflow, name), job(baseline, name))
+
+    def test_focused_commands_are_complete_support_only_and_image_is_excluded(self):
+        source = job(self.workflow, 'manual_source_verify')
+        plan = script(source, 'Record immutable source and planned checks')
+        focused = plan.split('if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then', 1)[1].split('\nfi', 1)[0]
+        self.assertEqual(re.findall(r"^    '([^']+)'", focused, re.M), [
+            'pnpm test:e2e:runner:typecheck', 'pnpm test:e2e:runner:unit',
+            'node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/launch.ts --list'])
+        self.assertIn('!inputs.source_e2e_support_only', job(self.workflow, 'manual_image'))
+        self.assertIn("inputs.source_e2e_support_only && 'source-e2e-support'", self.workflow)
+        self.assertNotIn('packages: write', source)
+        self.assertNotIn('secrets.', source)
+        self.assertIn('persist-credentials: false', source)
+        self.assertIn('1380 - $(date +%s) + checks_started_at', source)
+        self.assertIn('check_timeout=15m', source)
+        self.assertIn('inputs.source_e2e_support_only && 45', source)
+        self.assertIn('256 * 1024 * 1024', source)
+        self.assertIn('inputs.source_e2e_support_only && 7 || 14', source)
+
+    def test_complete_failure_evidence_is_admitted_and_unsafe_or_oversize_is_not(self):
+        source = script(job(self.workflow, 'manual_source_verify'), 'Admit complete focused evidence within the storage bound')
+        admission = source.split("<<'PYTHON'\n", 1)[1].rsplit('PYTHON', 1)[0]
+        for scenario in ('failed-check', 'oversize', 'symlink', 'absent'):
+            with self.subTest(scenario=scenario), tempfile.TemporaryDirectory() as temp:
+                root = Path(temp); evidence = root / 'remote-source-verification'; evidence.mkdir()
+                if scenario == 'failed-check':
+                    (evidence / 'check-1.log').write_text('real failure output\n')
+                    (evidence / 'check-1.status').write_text('1\n')
+                elif scenario == 'oversize':
+                    with (evidence / 'large.log').open('wb') as output: output.truncate(256 * 1024 * 1024 + 1)
+                elif scenario == 'symlink': (evidence / 'link').symlink_to(root / 'outside')
+                output = root / 'output'
+                result = subprocess.run([sys.executable, '-c', admission], cwd=root,
+                                        env={'GITHUB_OUTPUT': str(output)}, capture_output=True, timeout=5)
+                if scenario == 'failed-check':
+                    self.assertEqual(result.returncode, 0, result.stderr)
+                    self.assertEqual(output.read_text(), 'admitted=true\n')
+                    self.assertIn('real failure output', (evidence / 'check-1.log').read_text())
+                    self.assertTrue((evidence / 'evidence-inventory.json').is_file())
+                else:
+                    self.assertNotEqual(result.returncode, 0)
+                    self.assertFalse(output.exists())
+
+    def test_all_embedded_shell_scripts_parse(self):
+        for number, match in enumerate(re.finditer(r'^        run: \|\n((?:          .*\n|\n)+)', self.workflow, re.M)):
+            # The heredoc retains its Python as data for bash syntax validation.
+            with self.subTest(block=number):
+                subprocess.run(['bash', '-n'], input=textwrap.dedent(match.group(1)), text=True, check=True)
+
+
+if __name__ == '__main__':
+    unittest.main()