diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index 3fc209b807..8c217defdd 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -19,6 +19,10 @@ on: description: "With verify_source, run only the complete pnpm test:run suite (180-minute bound)" type: boolean default: false + source_e2e_support_only: + description: "Run only E2E support typecheck, complete unit suite and catalog on the exact reviewed coverage source" + type: boolean + default: false verify_runner: description: "Run the complete offline Runner verify command on GitHub-hosted Ubuntu without building an image" type: boolean @@ -32,11 +36,11 @@ on: type: boolean default: false expected_source_sha: - description: "Require this immutable target commit (required for verify_runner or verify_source)" + description: "Require this immutable target commit (required for verify_runner, verify_source or source_e2e_support_only)" type: string required: false expected_resolved_lock_sha256: - description: "Require this reviewed resolved dependency lock (required for verify_runner or verify_source)" + description: "Require this reviewed resolved dependency lock (required for verify_runner, verify_source or source_e2e_support_only)" type: string required: false verify_public_install: @@ -61,7 +65,7 @@ permissions: {} concurrency: # Publishing a newer image must not discard an earlier verification's evidence. - group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} + group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} cancel-in-progress: true jobs: @@ -113,10 +117,11 @@ jobs: VERIFY_RUNNER: ${{ inputs.verify_runner }} VERIFY_SOURCE: ${{ inputs.verify_source }} SOURCE_ROOT_TESTS_ONLY: ${{ inputs.source_root_tests_only }} + SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }} IMAGE_MODE: ${{ inputs.publish_eval_image || inputs.verify_public_install || inputs.build_eval_viewer }} VERIFY_PI_INTEL: ${{ inputs.verify_pi_intel }} DIAGNOSE_AJV: ${{ inputs.diagnose_ajv_pack }} - OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel }} + OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel || inputs.source_e2e_support_only }} run: | set -euo pipefail test "$REPOSITORY" = paperclipai/paperclip @@ -131,10 +136,21 @@ jobs: if [ "$SOURCE_ROOT_TESTS_ONLY" = true ]; then test "$VERIFY_SOURCE" = true fi - if [ "$VERIFY_RUNNER" = true ] || [ "$VERIFY_SOURCE" = true ]; then + if [ "$VERIFY_RUNNER" = true ] || [ "$VERIFY_SOURCE" = true ] || [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then [[ "$EXPECTED_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$EXPECTED_LOCK_SHA256" =~ ^[0-9a-f]{64}$ ]] fi + if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then + test "$VERIFY_SOURCE" != true + test "$SOURCE_ROOT_TESTS_ONLY" != true + test "$VERIFY_RUNNER" != true + test "$VERIFY_PI_INTEL" != true + test "$DIAGNOSE_AJV" != true + test "$IMAGE_MODE" != true + test "$EXPECTED_SOURCE_SHA" = 5cd6d3d5237e3e15394fcc1c7e754c30a63c5169 + test "$EXPECTED_LOCK_SHA256" = 38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba + test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public + fi if [ "$VERIFY_SOURCE" = true ]; then test "$VERIFY_RUNNER" != true test "$IMAGE_MODE" != true @@ -296,11 +312,11 @@ jobs: retention-days: 14 manual_source_verify: - name: Full source verification on GitHub-hosted Ubuntu - if: github.event_name == 'workflow_dispatch' && inputs.verify_source + name: ${{ inputs.source_e2e_support_only && 'Focused E2E support verification on GitHub-hosted Ubuntu' || 'Full source verification on GitHub-hosted Ubuntu' }} + if: github.event_name == 'workflow_dispatch' && (inputs.verify_source || inputs.source_e2e_support_only) needs: authorize_manual runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.source_root_tests_only && 200 || 295 }} + timeout-minutes: ${{ inputs.source_e2e_support_only && 45 || inputs.source_root_tests_only && 200 || 295 }} permissions: contents: read env: @@ -319,6 +335,7 @@ jobs: SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }} EXPECTED_LOCK_SHA256: ${{ inputs.expected_resolved_lock_sha256 }} SOURCE_ROOT_TESTS_ONLY: ${{ inputs.source_root_tests_only }} + SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }} run: | set -euo pipefail mkdir -p remote-source-verification @@ -339,6 +356,16 @@ jobs: if [ "$SOURCE_ROOT_TESTS_ONLY" = true ]; then printf '%s\n' 'pnpm test:run' > remote-source-verification/commands.txt fi + if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then + printf '%s pnpm-lock.yaml\n' e11d69fa8702a906c293c1cb307c71df90d3b1f1617b3c23ebf17fa9a87fec79 | sha256sum --check --strict + git ls-tree -r -z HEAD > remote-source-verification/source-tree.zlist + git archive --format=tar HEAD | sha256sum > remote-source-verification/source-archive.sha256 + printf '%s\n' \ + 'pnpm test:e2e:runner:typecheck' \ + 'pnpm test:e2e:runner:unit' \ + 'node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/launch.ts --list' \ + > remote-source-verification/commands.txt + fi index=0 while IFS= read -r _check; do index=$((index + 1)) @@ -389,12 +416,15 @@ jobs: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps \ > remote-source-verification/test-build-deps.log 2>&1 - name: Run every source check and retain each result - timeout-minutes: ${{ inputs.source_root_tests_only && 182 || 273 }} + timeout-minutes: ${{ inputs.source_e2e_support_only && 28 || inputs.source_root_tests_only && 182 || 273 }} + env: + SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }} run: | set -uo pipefail status=0 index=0 progress_pid= + checks_started_at=$(date +%s) trap 'if [ -n "$progress_pid" ]; then kill "$progress_pid" 2>/dev/null || true; fi' EXIT while IFS= read -r check; do index=$((index + 1)) @@ -404,6 +434,12 @@ jobs: # alone exceed 85 minutes; ordinary CI distributes them across shards. check_timeout=15m if [ "$check" = 'pnpm test:run' ]; then check_timeout=180m; fi + remaining=900 + if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then + # Leave job time for final auditing and complete evidence upload. + remaining=$((1380 - $(date +%s) + checks_started_at)) + if [ "$remaining" -lt 900 ] && [ "$remaining" -gt 0 ]; then check_timeout="${remaining}s"; fi + fi date -u +%FT%TZ > "remote-source-verification/check-$index.started-at" started_at=$(date +%s) ( @@ -414,7 +450,12 @@ jobs: ) & progress_pid=$! check_status=0 - timeout --verbose --signal=TERM --kill-after=15s "$check_timeout" bash -c "$check" > "remote-source-verification/check-$index.log" 2>&1 || check_status=$? + if [ "$remaining" -gt 0 ]; then + timeout --verbose --signal=TERM --kill-after=15s "$check_timeout" bash -c "$check" > "remote-source-verification/check-$index.log" 2>&1 || check_status=$? + else + check_status=124 + printf 'Not launched: focused command budget exhausted.\n' > "remote-source-verification/check-$index.log" + fi kill "$progress_pid" 2>/dev/null || true wait "$progress_pid" 2>/dev/null || true progress_pid= @@ -426,6 +467,26 @@ jobs: echo "Finished $check (exit $check_status)" done < remote-source-verification/commands.txt exit "$status" + - name: Audit focused source and lock closure even on check failure + if: always() && inputs.source_e2e_support_only + timeout-minutes: 2 + env: + SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }} + EXPECTED_LOCK_SHA256: ${{ inputs.expected_resolved_lock_sha256 }} + run: | + set -uo pipefail + mkdir -p remote-source-verification + status=0 + date -u +%FT%TZ > remote-source-verification/final-audit.started-at + git rev-parse HEAD > remote-source-verification/final-source.txt + test "$(cat remote-source-verification/final-source.txt)" = "$SOURCE_SHA" || status=1 + git status --porcelain=v1 --untracked-files=no > remote-source-verification/final-source-status.txt + git diff -- . ':(exclude)pnpm-lock.yaml' > remote-source-verification/final-source.diff + git diff --quiet -- . ':(exclude)pnpm-lock.yaml' || status=1 + printf '%s pnpm-lock.yaml\n' "$EXPECTED_LOCK_SHA256" | sha256sum --check --strict > remote-source-verification/final-overlay-check.log 2>&1 || status=1 + date -u +%FT%TZ > remote-source-verification/final-audit.finished-at + printf '%s\n' "$status" > remote-source-verification/final-audit.status + exit "$status" - name: Capture the final lock state even on resolution failure if: always() run: | @@ -435,13 +496,37 @@ jobs: sha256sum pnpm-lock.yaml > remote-source-verification/final-lock.sha256 git diff -- pnpm-lock.yaml > remote-source-verification/final-lock.diff fi + - name: Admit complete focused evidence within the storage bound + if: always() && inputs.source_e2e_support_only + id: source_e2e_evidence + timeout-minutes: 2 + run: | + python3 - <<'PYTHON' + import hashlib, json, os, pathlib + root = pathlib.Path('remote-source-verification') + files, size, limit = [], 0, 256 * 1024 * 1024 + for path in sorted(root.rglob('*')): + if path.is_symlink(): raise RuntimeError('Evidence symlink is not allowed') + if path.is_dir(): continue + if not path.is_file(): raise RuntimeError('Nonregular evidence is not allowed') + size += path.stat().st_size + if size > limit: raise RuntimeError('Complete evidence exceeds 256 MiB bound') + digest = hashlib.sha256() + with path.open('rb') as source: + for chunk in iter(lambda: source.read(1024 * 1024), b''): digest.update(chunk) + files.append({'path': str(path.relative_to(root)), 'bytes': path.stat().st_size, 'sha256': digest.hexdigest()}) + inventory = json.dumps({'bytes': size, 'files': files}, indent=2) + '\n' + if not files or size + len(inventory.encode()) > limit: raise RuntimeError('Complete evidence exceeds bound or is absent') + (root / 'evidence-inventory.json').write_text(inventory) + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: output.write('admitted=true\n') + PYTHON - name: Retain source verification evidence even on failure - if: always() + if: always() && (!inputs.source_e2e_support_only || steps.source_e2e_evidence.outputs.admitted == 'true') uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ${{ inputs.source_root_tests_only && 'source-root-tests' || 'source-full-verification' }}-${{ github.run_id }} + name: ${{ inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || 'source-full-verification' }}-${{ github.run_id }} path: remote-source-verification/ - retention-days: 14 + retention-days: ${{ inputs.source_e2e_support_only && 7 || 14 }} manual_ajv_pack_diagnostic: name: Pinned AJV directory-pack diagnostic on Linux @@ -506,7 +591,7 @@ jobs: manual_image: name: Build and verify on EC2 - if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel + if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel && !inputs.source_e2e_support_only needs: authorize_manual runs-on: runs-on/fleet=paperclip-public-pr-x64/env=public-ci timeout-minutes: 90 diff --git a/scripts/ci/test_source_e2e_mode.py b/scripts/ci/test_source_e2e_mode.py new file mode 100644 index 0000000000..07e750c758 --- /dev/null +++ b/scripts/ci/test_source_e2e_mode.py @@ -0,0 +1,136 @@ +"""Offline regressions for the reviewed, exclusive hosted support dispatch.""" +import os +from pathlib import Path +import re +import shutil +import sys +import subprocess +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = '.github/workflows/docker-runner-check.yml' +SOURCE = '5cd6d3d5237e3e15394fcc1c7e754c30a63c5169' +LOCK = '38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba' +INTEL_SOURCE = '5eba61ece929dcfb2b0c1761825a2d9e557c2554' +BASELINE = 'b58907578' + + +def job(text, name): + return re.search(r'^ ' + name + r':\n.*?(?=^ [a-z_]+:\n|\Z)', text, re.M | re.S).group() + + +def script(block, name): + step = block.split(' - name: ' + name + '\n', 1)[1].split('\n - ', 1)[0] + return textwrap.dedent(step.split(' run: |\n', 1)[1]).rstrip() + '\n' + + +class SupportMode(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.workflow = (ROOT / WORKFLOW).read_text() + cls.authorize = script(job(cls.workflow, 'authorize_manual'), 'Authorize an explicit maintainer image build') + + def admission(self, **changes): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + fake = root / 'gh' + fake.write_text('''#!/usr/bin/env python3 +import os, sys +path = sys.argv[2] +if path == 'users/maintainer': print('1') +elif path == 'repos/paperclipai/paperclip/git/ref/heads/coverage': print(os.environ['FAKE_TARGET']) +elif path == 'repos/paperclipai/paperclip': print(os.environ['FAKE_VISIBILITY']) +else: raise RuntimeError('Unexpected offline GitHub request') +''') + fake.chmod(0o755) + env = dict(PATH=str(root) + os.pathsep + str(Path(shutil.which('jq')).parent) + os.pathsep + os.defpath, + REPOSITORY='paperclipai/paperclip', REPOSITORY_ID='1170821064', ACTOR_ID='1', + TRIGGERING_ACTOR='maintainer', ALLOWED_IDS='[1]', TARGET_BRANCH='coverage', + EXPECTED_SOURCE_SHA=SOURCE, EXPECTED_LOCK_SHA256=LOCK, FAKE_TARGET=SOURCE, + FAKE_VISIBILITY='public', VERIFY_RUNNER='false', VERIFY_SOURCE='false', + SOURCE_ROOT_TESTS_ONLY='false', SOURCE_E2E_SUPPORT_ONLY='true', IMAGE_MODE='false', + VERIFY_PI_INTEL='false', DIAGNOSE_AJV='false', OTHER_MODE='true', + GITHUB_OUTPUT=str(root / 'output')) + env.update(changes) + result = subprocess.run(['bash', '-c', self.authorize], env=env, capture_output=True, text=True, timeout=5) + return result.returncode, (root / 'output').read_text() if (root / 'output').exists() else '' + + def test_exact_reviewed_source_and_overlay_are_admitted(self): + self.assertEqual(self.admission(), (0, 'target_sha=' + SOURCE + '\n')) + + def test_every_other_execution_mode_conflicts(self): + for mode in ('VERIFY_RUNNER', 'VERIFY_SOURCE', 'SOURCE_ROOT_TESTS_ONLY', 'IMAGE_MODE', 'VERIFY_PI_INTEL', 'DIAGNOSE_AJV'): + with self.subTest(mode=mode): + status, output = self.admission(**{mode: 'true'}) + self.assertNotEqual(status, 0); self.assertEqual(output, '') + + def test_wrong_missing_or_moving_source_and_lock_fail(self): + for changes in ({'EXPECTED_SOURCE_SHA': ''}, {'EXPECTED_SOURCE_SHA': INTEL_SOURCE}, + {'FAKE_TARGET': INTEL_SOURCE}, {'EXPECTED_LOCK_SHA256': ''}, + {'EXPECTED_LOCK_SHA256': 'a' * 64}, {'FAKE_VISIBILITY': 'private'}, {'ACTOR_ID': '2'}): + with self.subTest(changes=changes): + status, output = self.admission(**changes) + self.assertNotEqual(status, 0); self.assertEqual(output, '') + + def test_original_intel_admission_and_entire_job_are_unchanged(self): + self.assertEqual(self.admission(SOURCE_E2E_SUPPORT_ONLY='false', VERIFY_PI_INTEL='true', + EXPECTED_SOURCE_SHA=INTEL_SOURCE, FAKE_TARGET=INTEL_SOURCE), + (0, 'target_sha=' + INTEL_SOURCE + '\n')) + baseline = subprocess.check_output(['git', 'show', BASELINE + ':' + WORKFLOW], cwd=ROOT, text=True) + self.assertEqual(job(self.workflow, 'manual_pi_intel'), job(baseline, 'manual_pi_intel')) + for name in ('manual_runner_verify', 'manual_ajv_pack_diagnostic'): + self.assertEqual(job(self.workflow, name), job(baseline, name)) + + def test_focused_commands_are_complete_support_only_and_image_is_excluded(self): + source = job(self.workflow, 'manual_source_verify') + plan = script(source, 'Record immutable source and planned checks') + focused = plan.split('if [ "$SOURCE_E2E_SUPPORT_ONLY" = true ]; then', 1)[1].split('\nfi', 1)[0] + self.assertEqual(re.findall(r"^ '([^']+)'", focused, re.M), [ + 'pnpm test:e2e:runner:typecheck', 'pnpm test:e2e:runner:unit', + 'node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/launch.ts --list']) + self.assertIn('!inputs.source_e2e_support_only', job(self.workflow, 'manual_image')) + self.assertIn("inputs.source_e2e_support_only && 'source-e2e-support'", self.workflow) + self.assertNotIn('packages: write', source) + self.assertNotIn('secrets.', source) + self.assertIn('persist-credentials: false', source) + self.assertIn('1380 - $(date +%s) + checks_started_at', source) + self.assertIn('check_timeout=15m', source) + self.assertIn('inputs.source_e2e_support_only && 45', source) + self.assertIn('256 * 1024 * 1024', source) + self.assertIn('inputs.source_e2e_support_only && 7 || 14', source) + + def test_complete_failure_evidence_is_admitted_and_unsafe_or_oversize_is_not(self): + source = script(job(self.workflow, 'manual_source_verify'), 'Admit complete focused evidence within the storage bound') + admission = source.split("<<'PYTHON'\n", 1)[1].rsplit('PYTHON', 1)[0] + for scenario in ('failed-check', 'oversize', 'symlink', 'absent'): + with self.subTest(scenario=scenario), tempfile.TemporaryDirectory() as temp: + root = Path(temp); evidence = root / 'remote-source-verification'; evidence.mkdir() + if scenario == 'failed-check': + (evidence / 'check-1.log').write_text('real failure output\n') + (evidence / 'check-1.status').write_text('1\n') + elif scenario == 'oversize': + with (evidence / 'large.log').open('wb') as output: output.truncate(256 * 1024 * 1024 + 1) + elif scenario == 'symlink': (evidence / 'link').symlink_to(root / 'outside') + output = root / 'output' + result = subprocess.run([sys.executable, '-c', admission], cwd=root, + env={'GITHUB_OUTPUT': str(output)}, capture_output=True, timeout=5) + if scenario == 'failed-check': + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), 'admitted=true\n') + self.assertIn('real failure output', (evidence / 'check-1.log').read_text()) + self.assertTrue((evidence / 'evidence-inventory.json').is_file()) + else: + self.assertNotEqual(result.returncode, 0) + self.assertFalse(output.exists()) + + def test_all_embedded_shell_scripts_parse(self): + for number, match in enumerate(re.finditer(r'^ run: \|\n((?: .*\n|\n)+)', self.workflow, re.M)): + # The heredoc retains its Python as data for bash syntax validation. + with self.subTest(block=number): + subprocess.run(['bash', '-n'], input=textwrap.dedent(match.group(1)), text=True, check=True) + + +if __name__ == '__main__': + unittest.main()