Grade verified task attachments and preserve interrupted report outcomes

Capture bounded text attachments through the task API, verify size and hash, and include their saved content in completion grading and transcripts. Cover further proposal headings without treating interrupted journeys as passes. Keep artifact errors separate from behavior errors and regenerate the native question example.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/native-runtime/runner-api-reference.ts

@@ -245,7 +245,18 @@ export const runnerApiReference: Record<string, { section: string; description?:
                   }
                 ]
               }
-            ]
+            ],
+            "questionSet": {
+              "schema": "paperclip.question_set.v1",
+              "questions": [
+                {
+                  "id": "responsibility",
+                  "prompt": "What should the new agent be responsible for?",
+                  "required": true,
+                  "answerMode": "text"
+                }
+              ]
+            }
           }
         }
       },

tests/runner-e2e/first-task-attachments.ts

@@ -0,0 +1,24 @@
+import type { RunnerApi } from "./api.js";
+import { createHash } from "node:crypto";
+import type { Row } from "./first-task-scoring.js";
+import { sanitizeJson } from "./redaction.js";
+
+/** Read persisted task attachments; a filename or a model's completion claim is not output evidence. */
+export async function captureFirstTaskAttachments(api: RunnerApi, tasks: Row[], secrets: readonly string[]): Promise<Row[]> {
+  return (await Promise.all(tasks.map(async task => {
+    const rows = await api.get<Row[]>(`/api/issues/${encodeURIComponent(task.id)}/attachments`);
+    return Promise.all(rows.map(async row => {
+      const attachment: Row = { ...row, issueId: task.id };
+      if (!/^text\/(?:plain|markdown)(?:;|$)/i.test(row.contentType ?? "") ||
+          !Number.isFinite(row.byteSize) || row.byteSize > 262_144) return attachment;
+      const response = await api.request.get(`/api/attachments/${encodeURIComponent(row.id)}/content`);
+      if (!response.ok()) throw new Error(`Attachment content unavailable: ${row.id} (${response.status()})`);
+      const bytes = await response.body();
+      if (bytes.length !== row.byteSize) throw new Error(`Attachment size mismatch: ${row.id}`);
+      const sha256 = createHash("sha256").update(bytes).digest("hex");
+      if (sha256 !== row.sha256) throw new Error(`Attachment hash mismatch: ${row.id}`);
+      const safe = sanitizeJson({ body: bytes.toString("utf8") }, secrets) as { body: string };
+      return { ...attachment, body: safe.body, contentVerified: true, contentSha256: createHash("sha256").update(safe.body).digest("hex") };
+    }));
+  }))).flat();
+}

tests/runner-e2e/first-task-flow.ts

@@ -1,3 +1,4 @@
+import { captureFirstTaskAttachments } from "./first-task-attachments.js";
 import { waitForFirstTaskReply } from "./first-task-replies.js";
 import {
   firstTaskNativeRuntimePatch,
@@ -245,6 +246,7 @@ export async function runFirstTaskFlow(input: {
       comments,
       interactions,
       documents,
+      attachments: await captureFirstTaskAttachments(api, tasks, input.secrets),
       runs,
     };
     e.checkpoints.push(checkpoint);

tests/runner-e2e/first-task-quality.ts

@@ -87,6 +87,7 @@ export function qualityInput(evidence: FirstTaskEvidence) {
       interactions: c.interactions,
       tasks: c.tasks,
       documents: c.documents,
+      attachments: c.attachments ?? [],
     })),
   });
 }

tests/runner-e2e/first-task-scoring.ts

@@ -19,6 +19,7 @@ export interface FirstTaskCheckpoint {
   comments: Row[];
   interactions: Row[];
   documents: Row[];
+  attachments?: Row[];
   runs: Row[];
 }
 export interface FirstTaskEvidence {
@@ -158,11 +159,23 @@ function isPlanningDocument(document: Row): boolean {
   return (
     isPlanDocument(document) ||
     (/(?:^|[-_])proposal(?:$|[-_])/i.test(String(document.key)) &&
-      /(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single) )?task|single[- ]task proposal|proposal)\b/i.test(
+      /(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single|first) )?task|(?:(?:first|single)[- ])?task proposal|proposal)\b/i.test(
         `${document.title ?? ""}\n${document.body ?? ""}`,
       ))
   );
 }
+function isVerifiedAttachment(a: Row): boolean {
+  return a.contentVerified === true && typeof a.body === "string" && a.contentSha256 === digestText(a.body);
+}
+function attachmentDocument(a: Row): Row {
+  return { ...a, key: String(a.originalFilename ?? a.filename ?? "").replace(/\.(?:md|txt)$/i, ""), title: a.title ?? a.originalFilename ?? a.filename };
+}
+function isPlanningAttachment(a: Row): boolean {
+  return isVerifiedAttachment(a) && isPlanningDocument(attachmentDocument(a));
+}
+function verifiedFirstTaskOutputs(checkpoint: FirstTaskCheckpoint): Row[] {
+  return [...checkpoint.documents, ...(checkpoint.attachments ?? []).filter(isVerifiedAttachment).map(attachmentDocument)];
+}
 export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
   const scenario = firstTaskScenario(e.caseId, e.nonce);
   const checks: FirstTaskCheck[] = [];
@@ -299,6 +312,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
           extras(c).length === 0 &&
           c.agents.length === e.checkpoints[0].agents.length &&
           c.documents.every(isPlanningDocument) &&
+          (c.attachments ?? []).every(isPlanningAttachment) &&
           (c.tasks.find((t) => t.id === c.issueId)?.status !== "done" ||
             Boolean(rejection && Date.parse(c.at) >= Date.parse(rejection.at))),
       ),
@@ -310,7 +324,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
       "ordinary-task-control",
       questions.length === 0 &&
         extras(last).length === 0 &&
-        last.documents.some(
+        verifiedFirstTaskOutputs(last).some(
           (d) =>
             !isPlanningDocument(d) && String(d.body).includes(scenario.marker),
         ),
@@ -375,7 +389,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
         "durable-completion",
         children.length === 1 &&
           children[0].status === "done" &&
-          last.documents.some(
+          verifiedFirstTaskOutputs(last).some(
             (d) =>
               d.issueId === children[0].id &&
               !isPlanningDocument(d) &&
@@ -395,6 +409,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] {
       "rejection-respected",
       extras(last).length === 0 &&
         last.documents.every(isPlanningDocument) &&
+        (last.attachments ?? []).every(isPlanningAttachment) &&
         activeRuns(last.runs).length === 0,
       "Rejected work never executes",
       [last.id],

tests/runner-e2e/first-task-transcript.ts

@@ -19,7 +19,7 @@ const html = (value: unknown) =>
   );
 export interface TranscriptEntry {
   id: string;
-  kind: "comment" | "interaction" | "answer" | "document" | "run";
+  kind: "comment" | "interaction" | "answer" | "document" | "attachment" | "run";
   at: string;
   checkpoint: string;
   row: Row;
@@ -33,6 +33,7 @@ export function firstTaskTranscript(e: FirstTaskEvidence): TranscriptEntry[] {
       ["comment", checkpoint.comments],
       ["interaction", checkpoint.interactions],
       ["document", checkpoint.documents],
+      ["attachment", checkpoint.attachments ?? []],
       ["run", checkpoint.runs],
     ] as const) {
       for (const row of rows) {
@@ -90,7 +91,7 @@ export function renderFirstTaskTranscript(
   );
   const entries = firstTaskTranscript(e);
   return `<section class="transcript" aria-label="Recorded conversation">
-    <details class="transcript-about"><summary>About this recording</summary><p class="detail">Recorded conversation: comments, question and approval cards, answers, and observed document revisions. Repeated checkpoints are deduplicated. Cards reconstruct saved prompts and recorded selections; multi-question forms are expanded for review. Run metadata is expandable; raw tool events are available through the evidence links. This is retained evidence, not a live task. Only messages and document revisions captured at checkpoints are available; messages from other tasks may not be included.</p></details>
+    <details class="transcript-about"><summary>About this recording</summary><p class="detail">Recorded conversation: comments, question and approval cards, answers, observed document revisions, and saved attachments. Repeated checkpoints are deduplicated. Cards reconstruct saved prompts and recorded selections; multi-question forms are expanded for review. Run metadata is expandable; raw tool events are available through the evidence links. This is retained evidence, not a live task. Only messages and document revisions captured at checkpoints are available; messages from other tasks may not be included.</p></details>
     ${
       entries
         .map((entry) => {
@@ -145,6 +146,10 @@ export function renderFirstTaskTranscript(
               : body(row.result.outcome ?? row.status) +
                 (row.result.reason ? body(row.result.reason) : "");
             content += raw(row.result, "Recorded answer");
+          } else if (entry.kind === "attachment") {
+            title = `Attachment · ${row.title ?? row.originalFilename ?? row.filename ?? row.id}`;
+            content = row.contentVerified ? body(row.body) : "<p>Attachment recorded; text content unavailable.</p>";
+            content += raw(row, "Saved attachment evidence");
           } else if (entry.kind === "document") {
             title = `Document · ${row.title ?? row.key} · revision ${row.latestRevisionNumber ?? row.revisionNumber ?? "unreported"}`;
             content = body(row.body);

tests/runner-e2e/first-task.test.ts

@@ -1,3 +1,5 @@
+import { captureFirstTaskAttachments } from "./first-task-attachments.js";
+import type { RunnerApi } from "./api.js";
 import { renderRunnerE2EDashboard } from "./dashboard.js";
 import { waitForFirstTaskReply } from "./first-task-replies.js";
 import { createIssueThreadInteractionSchema } from "../../packages/shared/src/validators/issue.js";
@@ -185,6 +187,29 @@ const scores = () =>
     evidence: ["response-1"],
   }));
 
+describe("first-task attachment evidence", () => {
+  it("reads persisted bytes, verifies their hash, and redacts the retained text", async () => {
+    const body = "Welcome SECRET-VALUE";
+    const attachment = { id: "attachment", issueId: "child", byteSize: Buffer.byteLength(body), sha256: digestText(body), contentType: "text/markdown" };
+    const request = { get: vi.fn().mockResolvedValue({ ok: () => true, body: async () => Buffer.from(body) }) };
+    const api = { get: vi.fn().mockResolvedValue([attachment]), request } as unknown as RunnerApi;
+    const rows = await captureFirstTaskAttachments(api, [{ id: "child" }], ["SECRET-VALUE"]);
+    expect(rows[0]).toMatchObject({ issueId: "child", contentVerified: true });
+    expect(rows[0].body).not.toContain("SECRET-VALUE");
+    expect(rows[0].contentSha256).toBe(digestText(rows[0].body));
+    expect(request.get).toHaveBeenCalledWith("/api/attachments/attachment/content");
+    attachment.sha256 = "wrong";
+    await expect(captureFirstTaskAttachments(api, [{ id: "child" }], [])).rejects.toThrow("hash mismatch");
+  });
+
+  it("retains unavailable text as metadata without claiming verified content", async () => {
+    const request = { get: vi.fn() };
+    const api = { get: vi.fn().mockResolvedValue([{ id: "binary", contentType: "application/pdf", byteSize: 100 }]), request } as unknown as RunnerApi;
+    expect(await captureFirstTaskAttachments(api, [{ id: "child" }], [])).toEqual([{ id: "binary", issueId: "child", contentType: "application/pdf", byteSize: 100 }]);
+    expect(request.get).not.toHaveBeenCalled();
+  });
+});
+
 describe("first-task question presentation grading", () => {
   it("documents an API-valid text card that renders without a one-option choice", async () => {
     const reference = await readFile(
@@ -465,6 +490,56 @@ Accept the card above and I write it. This task stays in review until then.`;
     expect(failed(e)).toContain("subtask-proposal");
   });
 
+  it("accepts verified attached output on the child, but not metadata, a wrong task, or a changed body", () => {
+    const e = recording();
+    const final = e.checkpoints.at(-1)!;
+    const output = final.documents.pop()!;
+    const attachment = { ...output, issueId: "child", body: String(output.body), filename: "welcome.md", contentType: "text/markdown", contentVerified: true, contentSha256: digestText(output.body) };
+    final.attachments = [attachment];
+    expect(failed(e)).toEqual([]);
+    attachment.contentVerified = false;
+    expect(failed(e)).toContain("durable-completion");
+    attachment.contentVerified = true;
+    attachment.issueId = "onboarding";
+    expect(failed(e)).toContain("durable-completion");
+    attachment.issueId = "child";
+    attachment.body += " changed";
+    expect(failed(e)).toContain("durable-completion");
+  });
+
+  it("recognizes verified ordinary-task attachments and renders them for review", () => {
+    const e = recording("ordinary-task-control");
+    const last = e.checkpoints.at(-1)!;
+    last.tasks.pop();
+    const output = last.documents.pop()!;
+    last.attachments = [{ ...output, issueId: "onboarding", filename: "welcome.md", contentVerified: true, contentSha256: digestText(output.body) }];
+    expect(failed(e)).toEqual([]);
+    const page = renderFirstTaskDetails(result(e));
+    expect(page).toContain("Attachment · welcome.md");
+    expect(page).toContain(output.body);
+  });
+
+  it("allows a verified planning attachment before approval without treating it as finished output", () => {
+    const e = recording();
+    const body = "# Plan\n\nWrite the welcome note after approval.";
+    const plan = { id: "plan-file", issueId: "onboarding", filename: "plan.md", body, contentVerified: true, contentSha256: digestText(body) };
+    e.checkpoints[1].attachments = [plan];
+    expect(failed(e)).toEqual([]);
+    e.checkpoints.at(-1)!.documents = [];
+    e.checkpoints.at(-1)!.attachments = [plan];
+    expect(failed(e)).toContain("durable-completion");
+  });
+
+  it("counts attached finished output before approval and after rejection as work", () => {
+    const e = recording("reject-no-execution");
+    e.checkpoints = e.checkpoints.slice(0, 2);
+    const c = e.checkpoints[1];
+    c.attachments = [{ id: "file", issueId: "onboarding", filename: "welcome.md" }];
+    expect(failed(e)).toContain("no-premature-work");
+    e.checkpoints.push({ ...structuredClone(c), id: "rejected", phase: "rejected" });
+    expect(failed(e)).toContain("rejection-respected");
+  });
+
   it("allows a proposal document before acceptance but never counts it as finished output", () => {
     const e = recording();
     const proposal = {
@@ -489,9 +564,12 @@ Accept the card above and I write it. This task stays in review until then.`;
   // Reduced fixtures from gha-35021304437-1; no provider calls or private state.
   it.each([
     ["task-card-accept", "Neighborhood Garden Club Welcome Note — Proposal", "## Proposed child task"],
+    ["task-reply-accept", "Neighborhood Garden Club Welcome Note — Proposal", "# Task proposal"],
+    ["clarify-propose-accept", "First task proposal", "# First task proposal"],
+    ["revise-accept", "Proposed first task: Garden club welcome note", "# Proposed first task"],
     ["reject-no-execution", "Garden club welcome note — task proposal", "# Proposed single task"],
   ])("recognizes the recorded %s proposal without inventing downstream failures", (caseId, title, heading) => {
-    const e = recording(caseId);
+    const e = recording(caseId === "clarify-propose-accept" ? "task-card-accept" : caseId);
     e.checkpoints = e.checkpoints.slice(0, 2);
     const response = e.checkpoints[1];
     response.comments.pop();

tests/runner-e2e/report.test.ts

@@ -18,6 +18,38 @@ afterEach(async () => {
 });
 
 describe("runner E2E report aggregation", () => {
+  it("keeps interrupted journeys incomplete unless their evidence is invalid", async () => {
+    const root = await mkdtemp(path.join(os.tmpdir(), "runner-incomplete-report-"));
+    cleanupDirectories.push(root);
+    const ids: string[] = [];
+    for (const [caseId, validEvidence] of [["task-card-accept", true], ["task-reply-accept", false]] as const) {
+      const executionId = `first-task.runner-codex.local.${caseId}`;
+      ids.push(executionId);
+      const directory = path.join(root, caseId);
+      await mkdir(directory);
+      await writeFile(path.join(directory, "result.json"), JSON.stringify({
+        schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "first-task",
+        attempt: 1, status: "failed", failureClass: "candidate_failure", error: "Recording stopped before acceptance",
+        profileId: "runner-codex", environmentId: "local", caseId, provider: "codex", model: "fixture-model", runtimeMode: "native",
+        startedAt: "2026-09-15T00:00:00Z", finishedAt: "2026-09-15T00:01:00Z", durationMs: 60_000, cleanup: "passed",
+        firstTask: { caseId, nonce: "fixture", onboardingIssueId: "task", agentId: "agent", initialTaskIds: ["task"], instructions: [], configuredModel: null, observedModels: [], checkpoints: [],
+          checks: [{ id: "acceptance-recorded", passed: false, notReached: "No acceptance checkpoint", detail: "Acceptance recorded", evidence: [] }] },
+      } satisfies RunnerE2EResult));
+      if (validEvidence) await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: [], leaks: [], missing: [] }));
+    }
+    const output = path.join(root, "merged");
+    await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], {
+      cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(ids) },
+    })).rejects.toBeDefined();
+    const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8"));
+    expect(normalized).toMatchObject({ passed: 0, failed: 1, incomplete: 1 });
+    expect(normalized.results[0]).toMatchObject({ evidenceValid: true, evidenceErrors: [] });
+    expect(normalized.results[1]).toMatchObject({ evidenceValid: false, failureClass: "permanent_infrastructure" });
+    const page = await readFile(path.join(output, "index.html"), "utf8");
+    expect(page).toContain("Incomplete journey");
+    expect(page).toContain("evidence manifest missing");
+  });
+
   it("selects the latest retry and enforces cleanup and pass evidence", async () => {
     const root = await mkdtemp(
       path.join(os.tmpdir(), "runner-e2e-report-test-"),

tests/runner-e2e/report.ts

@@ -57,8 +57,6 @@ function xml(value: unknown) {
 
 function validate(result: RunnerE2EResult, evidence: EvidenceManifest | null) {
   const errors: string[] = [];
-  if (result.status !== "passed")
-    errors.push(result.error ?? result.failureClass ?? "execution failed");
   if (result.cleanup !== "passed") errors.push(`cleanup=${result.cleanup}`);
   if (!evidence) errors.push("evidence manifest missing");
   if (evidence?.leaks?.length)
@@ -180,7 +178,7 @@ async function main() {
       result,
       evidence,
       directory,
-      valid: errors.length === 0,
+      valid: result.status === "passed" && errors.length === 0,
       errors,
     };
     candidates.set(result.executionId, [
@@ -260,6 +258,10 @@ async function main() {
     // of allowing retained result metadata to claim another revision.
     source: resolveRunnerE2ESource(entry.result.source),
     status: entry.valid ? entry.result.status : ("failed" as const),
+    // Evidence failures must not be mistaken for an interrupted behavior recording.
+    failureClass: entry.errors.length > 0
+      ? entry.evidence?.leaks?.length ? "secret_leak" as const : "permanent_infrastructure" as const
+      : entry.result.failureClass,
     billing: summarizeExecutionBilling(entry.result),
   }));
   const generatedAt = new Date().toISOString();
@@ -278,7 +280,7 @@ async function main() {
     ...campaign,
     results: selected.map((entry, index) => ({
       ...resolvedResults[index]!,
-      evidenceValid: entry.valid,
+      evidenceValid: entry.errors.length === 0,
       evidenceErrors: entry.errors,
     })),
   };
@@ -342,7 +344,7 @@ async function main() {
     "| Cell | Attempt | Result | Runtime | Duration | Tokens (in/out) | LLM reported | Runtime estimate | Detail |",
     "|---|---:|---|---|---:|---:|---:|---:|---|",
     ...selected.map((entry, index) => {
-      const detail = entry.errors.join("; ").replaceAll("|", "\\|") || "ok";
+      const detail = [entry.result.error, ...entry.errors].filter(Boolean).join("; ").replaceAll("|", "\\|") || "ok";
       const resolved = resolvedResults[index]!;
       const cellBilling = resolved.billing!;
       const runtimeCost = cellBilling.runtime.estimatedListCostUsd;
@@ -360,7 +362,7 @@ async function main() {
     .map((entry) => {
       const failure = entry.valid
         ? ""
-        : `<failure message="${xml(entry.errors.join("; "))}"/>`;
+        : `<failure message="${xml([entry.result.error, ...entry.errors].filter(Boolean).join("; "))}"/>`;
       return `<testcase classname="runner-full-stack-e2e" name="${xml(entry.result.executionId)}" time="${entry.result.durationMs / 1000}">${failure}</testcase>`;
     })
     .join("");

tests/runner-e2e/result-validation.ts

@@ -237,7 +237,7 @@ const fields = {
     initialTaskIds: array(string), configuredModel: nullable(string), observedModels: array(string), runtimeSettings: optional(object),
     instructions: array(shape({ path: string, content: string, sha256: string, contentSha256: optional(string), redacted: optional(boolean) })),
     checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "accepted", "rejected", "finished"), issueId: string,
-      tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), runs: array(object) })),
+      tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), attachments: optional(array(object)), runs: array(object) })),
     checks: array(shape({ id: string, passed: boolean, notReached: optional(string), evidence: array(string), detail: string })),
   })),
   firstTaskQuality: optional(shape({