Retain continuation screenshots and required state evidence

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/continuation-cases.ts

@@ -69,3 +69,11 @@ export const continuationTasks: readonly RunnerTaskFixture[] =
     buildVisibleMarker: (nonce) => continuationScenario(id, nonce).marker,
     buildMatchers: () => [],
   }));
+
+export function continuationScreenshotFile(
+  phase: "initial" | "answered" | "revised" | "final",
+) {
+  return phase === "final"
+    ? "final-state.png"
+    : `question-continuation-${phase}.png`;
+}

tests/runner-e2e/continuation-flow.ts

@@ -7,7 +7,10 @@ import {
   collectChatRunEvidence,
   type ChatRun,
 } from "./chat-flow.js";
-import { continuationScenario } from "./continuation-cases.js";
+import {
+  continuationScenario,
+  continuationScreenshotFile,
+} from "./continuation-cases.js";
 import {
   gradeContinuation,
   type ContinuationCheckpoint,
@@ -128,11 +131,12 @@ export async function runContinuationFlow(input: {
       checkpoints,
       checks,
     });
+    await input.evidence("api-state.json", checkpoints.at(-1));
     await open();
     await input.capture(
       phase,
       `Continuation: ${phase}`,
-      `continuation-${phase}.png`,
+      continuationScreenshotFile(phase),
     );
   }
   async function answer() {

tests/runner-e2e/continuation.test.ts

@@ -1,3 +1,8 @@
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { packageEvidence } from "./evidence.js";
+import { continuationScreenshotFile } from "./continuation-cases.js";
 import { describe, expect, it } from "vitest";
 import {
   CONTINUATION_CASES,
@@ -118,3 +123,31 @@ describe("continuation behavioral evaluation", () => {
     expect(failures(r)).toContain("updated-output");
   });
 });
+
+it("packages all continuation checkpoints using the shared evidence rules", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "continuation-evidence-"));
+  try {
+    const privateDir = path.join(root, "private");
+    await mkdir(path.join(privateDir, "snapshots"), { recursive: true });
+    const files = ["initial", "answered", "revised", "final"].map((phase) =>
+      continuationScreenshotFile(phase as "initial"),
+    );
+    for (const file of files)
+      await writeFile(
+        path.join(privateDir, file),
+        Buffer.from("89504e470d0a1a0a", "hex"),
+      );
+    await writeFile(path.join(privateDir, "snapshots/api-state.json"), "{}");
+    const result = await packageEvidence({
+      privateDir,
+      uploadDir: path.join(root, "upload"),
+      secrets: [],
+      expectPassScreenshot: true,
+    });
+    expect(result.files).toEqual(expect.arrayContaining(files));
+    expect(result.missing).not.toContain("final-state.png");
+    expect(result.missing).not.toContain("snapshots/api-state.json");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});