diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 3250469823f..0c4fa59387a 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -1132,7 +1132,7 @@ jobs: fi publish_history: - name: Publish trusted-summary S3 history and structured Pages bundle + name: Publish S3 history and Pages bundle with declared screenshots needs: [authorize, catalog, report] if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true' runs-on: ubuntu-latest @@ -1183,7 +1183,7 @@ jobs: role-to-assume: ${{ vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_HISTORY_AWS_REGION }} - - name: Publish trusted summary image to S3 and prune the Pages bundle + - name: Publish trusted summary and declared screenshots to public bundles env: PAPERCLIP_RUNNER_E2E_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report/normalized RUNNER_E2E_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_HISTORY_S3_BUCKET }} @@ -1196,7 +1196,7 @@ jobs: if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' run: echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}" >> "$GITHUB_OUTPUT" - - name: Package structured-only dashboard for GitHub Pages + - name: Package pruned dashboard with declared screenshots for GitHub Pages if: vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 with: @@ -1204,7 +1204,7 @@ jobs: path: runner-e2e-merged-report/pages pages: - name: Publish latest structured dashboard + name: Publish latest dashboard with declared screenshots needs: [report, publish_history] if: always() && needs.report.outputs.history_source_ready == 'true' && needs.publish_history.result == 'success' && vars.RUNNER_FULL_STACK_E2E_PUBLISH_PAGES == 'true' runs-on: ubuntu-latest diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index cbf336f4597..ed9fed5caa7 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -189,10 +189,11 @@ Packaged, access-controlled evidence is written beneath outcomes, sanitized fixture/API metadata, a result record, JUnit, HTML, and a blob report. Failures additionally retain the Playwright trace/video, browser diagnostics, failure screenshot, and sanitized Paperclip/run logs when -produced. Provider/UI PNG and WebM files remain limited to the local results -directory and access-controlled GitHub Actions artifact because secrets can be -rendered into pixels. SVG is active content and is rejected from the packaged -evidence entirely. +produced. WebM files remain limited to the local results directory and +access-controlled GitHub Actions artifact. Declared PNG screenshots are also +published with permanent campaign dashboards; fixture authors must therefore +keep credentials and other private data out of every captured UI state. SVG is +active content and is rejected from the packaged evidence entirely. Every completed local campaign also writes `tests/runner-e2e/results//dashboard.html`. The self-contained page @@ -205,22 +206,30 @@ usage is labeled `unavailable` or `unpriced`; it is never presented as zero cost. The CI report job stages the same portable site at `normalized/index.html` inside the access-controlled merged report artifact. -Permanent publication uses two explicit bundles. The CloudFront-backed S3 -history contains one publisher-generated `public-images/campaign-summary.png`. +Permanent publication uses two explicit bundles. Both retain only normalized +result PNG files with the explicit `public-runner-fixture` publication marker, +including marked `failure.png` captures, so every campaign dashboard has its +screenshot thumbnails and gallery. The capture helper adds this marker only +for the reviewed runner fixture and blocks public capture outside the exact +issue route for the fixture that the harness created. A blocked failure capture +remains private. The CloudFront-backed S3 +history also contains one publisher-generated +`public-images/campaign-summary.png`. Trusted publisher code renders it offline from fixed catalog labels and sanitized status/count/duration fields; provider output, error text, comments, and target-produced pixels are never inputs. The PNG must pass a 12 MiB bound and signature validation before entering the immutable manifest. S3 also retains allowlisted inert per-attempt evidence (`.json`, `.log`, `.md`, and `.txt`); `.log` copies have already passed exact-value/key-shape scanning and -redaction. The GitHub Pages bundle is regenerated separately and remains -structured-only. +redaction. The GitHub Pages bundle is regenerated separately with the same +declared-screenshot boundary. Both public bundles exclude video, archives, raw/unallowlisted logs, SVG or other active content, generated Playwright/blob/HTML report trees, and -per-attempt XML. The root `junit.xml` remains public because the report -aggregator builds it from fixed markup and XML-escaped fields. Full evidence -remains available only in the access-controlled workflow artifact. +undeclared PNG files, and per-attempt XML. The root `junit.xml` remains public +because the report aggregator builds it from fixed markup and XML-escaped +fields. Full evidence remains available only in the access-controlled workflow +artifact. ### Billing interpretation @@ -388,9 +397,12 @@ bundle digest fails closed. GitHub Pages remains the stable latest dashboard. Enable Pages with GitHub Actions as its source and set `RUNNER_FULL_STACK_E2E_PUBLISH_PAGES=true`. The publisher creates an S3 stage with the trusted synthetic summary PNG and a -separate structured-only Pages stage. Neither surface publishes provider/UI -screenshots, video, archives, generated reports, SVG/active content, databases, -Paperclip homes, workspaces, raw/unallowlisted logs, or credentials. +separate Pages stage. Both surfaces publish only per-result PNG screenshots +with the explicit `public-runner-fixture` marker alongside sanitized structured +evidence. The runner capture helper refuses to mark a screenshot outside the +exact live fixture issue route. Neither surface publishes video, archives, +SVG/active content, databases, Paperclip homes, workspaces, raw/unallowlisted +logs, or credentials. See [FIXTURES.md](./FIXTURES.md) before adding or changing a profile, environment, task, matcher, or future Paperclip object fixture. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 06f01adfd45..25a7a87333b 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -179,26 +179,36 @@ latest pointers are mutable, and S3 versioning makes those updates recoverable. ## Public evidence boundary CloudFront and GitHub Pages are public. Fixture identifiers, timing, token -usage, costs, normalized results, and allowlisted inert structured per-attempt -evidence are expected public data. The CloudFront-backed S3 history also -publishes one synthetic campaign-summary PNG generated by trusted publisher -code solely from fixed catalog labels and sanitized numeric/status fields. -Provider output, comments, error text, and provider/UI screenshots are never -rendered into it. Video, archives, generated -Playwright/blob/HTML report trees, SVG or other active content, credentials, -Paperclip homes, databases, workspaces, master keys, raw/unredacted logs, and -unallowlisted files are not public. Allowlisted `.log` copies must pass the -existing exact-value/key-shape scan and redaction boundary. +usage, costs, normalized results, allowlisted inert structured per-attempt +evidence, and trusted runner PNG screenshots are expected public data. Each +public screenshot must carry the explicit `public-runner-fixture` marker in +the normalized result. This includes a `failure.png` capture. Screenshot paths +must be safe PNG basenames and must be tied to the exact normalized execution +ID and attempt. The runner capture helper accepts only the exact issue route +for the live fixture that the harness created. Other issue routes, credential +pages, setup pages, and administration pages fail closed. The +CloudFront-backed S3 history also publishes one +synthetic campaign-summary PNG generated by trusted publisher code solely from +fixed catalog labels and sanitized numeric/status fields. Video, archives, +generated Playwright/blob/HTML report trees, SVG or other active content, +credentials, Paperclip homes, databases, workspaces, master keys, +raw/unredacted logs, unmarked images, and unallowlisted files are not public. +Allowlisted `.log` copies must pass the existing exact-value/key-shape scan and +redaction boundary. The packaged evidence uploaded as a 30-day GitHub Actions artifact has a -different, access-controlled boundary. Text is exact-value and key-shape -scanned and redacted. PNG and WebM are raw-byte scanned; SVG is rejected during -packaging because it is active content. A raster image cannot be exhaustively -secret-scanned by bytes, so every target/provider-produced image remains -access-controlled and is removed at the public boundary. +different, broader boundary. Text is exact-value and key-shape scanned and +redacted. PNG and WebM are raw-byte scanned; SVG is rejected during packaging +because it is active content. Raster pixels cannot be exhaustively +secret-scanned by bytes, so fixture authors must treat every marked capture as +public and must never extend the allowed task route to credentials, secrets, +private user data, or other non-public content. Adding or changing a marked +capture requires review of the visible page state. Videos remain +access-controlled. Before permanent publication, the campaign publisher creates a separate S3 -stage and retains only allowlisted `.json`, `.log`, `.md`, and `.txt` evidence. +stage and retains only allowlisted `.json`, `.log`, `.md`, and `.txt` evidence, +result PNGs with the explicit `public-runner-fixture` marker. It then launches publisher-only Chromium with networking blocked to render one `public-images/campaign-summary.png`. That fixed-path PNG is capped at 12 MiB and its signature is validated. Per-attempt XML is excluded because browsers @@ -208,10 +218,9 @@ constructs from fixed markup and XML-escaped fields. Videos, archives, raw/unallowlisted logs, SVG, undeclared images, generated reports, and symlinks fail closed or are removed before the immutable manifest is calculated. -GitHub Pages is built from a second structured-only stage without the summary -PNG. This keeps Pages small while the CloudFront-backed S3 dashboard can show a -useful visual generated without target content. A leak fails the cell and -withholds the unsafe file. +GitHub Pages is built from a second stage without the synthetic summary PNG but +with the same trusted-fixture screenshot allowlist. A leak detected by the +existing packager scan fails the cell and withholds the unsafe file. Rotate the affected credential immediately if a secret-scanning failure or unexpected public object is observed. Preserve the access-controlled Actions diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 9a9910fb56d..820615ff923 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -981,7 +981,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {

Full-stack acceptance campaign

${html(input.title)}

-

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Visual evidence is retained in the access-controlled workflow artifact; public history contains inert structured evidence and a trusted synthetic campaign summary.

+

A browser-verified matrix of runner profiles, execution environments, and deterministic task contracts. Declared PNG screenshots and sanitized structured evidence are retained with every published campaign; additional diagnostic evidence remains in the access-controlled workflow artifact.

@@ -1020,7 +1020,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ${suiteSections} ${historySection} -
Generated ${html(input.generatedAt)}${input.catalog.length} catalog executions · Public history excludes provider-produced visual evidence
+
Generated ${html(input.generatedAt)}${input.catalog.length} catalog executions · Declared screenshots and sanitized structured evidence published