From 1c5b9c8b1422c254a90d6cdcb909043578e2ed0b Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 15 Sep 2026 22:58:50 -0500 Subject: [PATCH] Grade verified task attachments and preserve interrupted report outcomes Capture bounded text attachments through the task API, verify size and hash, and include their saved content in completion grading and transcripts. Cover further proposal headings without treating interrupted journeys as passes. Keep artifact errors separate from behavior errors and regenerate the native question example. Co-Authored-By: Paperclip --- .../native-runtime/runner-api-reference.ts | 13 ++- tests/runner-e2e/first-task-attachments.ts | 24 ++++++ tests/runner-e2e/first-task-flow.ts | 2 + tests/runner-e2e/first-task-quality.ts | 1 + tests/runner-e2e/first-task-scoring.ts | 21 ++++- tests/runner-e2e/first-task-transcript.ts | 9 ++- tests/runner-e2e/first-task.test.ts | 80 ++++++++++++++++++- tests/runner-e2e/report.test.ts | 32 ++++++++ tests/runner-e2e/report.ts | 14 ++-- tests/runner-e2e/result-validation.ts | 2 +- 10 files changed, 184 insertions(+), 14 deletions(-) create mode 100644 tests/runner-e2e/first-task-attachments.ts diff --git a/server/src/services/native-runtime/runner-api-reference.ts b/server/src/services/native-runtime/runner-api-reference.ts index f201b6a861c..0ec14a7d61a 100644 --- a/server/src/services/native-runtime/runner-api-reference.ts +++ b/server/src/services/native-runtime/runner-api-reference.ts @@ -245,7 +245,18 @@ export const runnerApiReference: Record { + return (await Promise.all(tasks.map(async task => { + const rows = await api.get(`/api/issues/${encodeURIComponent(task.id)}/attachments`); + return Promise.all(rows.map(async row => { + const attachment: Row = { ...row, issueId: task.id }; + if (!/^text\/(?:plain|markdown)(?:;|$)/i.test(row.contentType ?? "") || + !Number.isFinite(row.byteSize) || row.byteSize > 262_144) return attachment; + const response = await api.request.get(`/api/attachments/${encodeURIComponent(row.id)}/content`); + if (!response.ok()) throw new Error(`Attachment content unavailable: ${row.id} (${response.status()})`); + const bytes = await response.body(); + if (bytes.length !== row.byteSize) throw new Error(`Attachment size mismatch: ${row.id}`); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + if (sha256 !== row.sha256) throw new Error(`Attachment hash mismatch: ${row.id}`); + const safe = sanitizeJson({ body: bytes.toString("utf8") }, secrets) as { body: string }; + return { ...attachment, body: safe.body, contentVerified: true, contentSha256: createHash("sha256").update(safe.body).digest("hex") }; + })); + }))).flat(); +} diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index 5aca5df8f46..dfe923162a6 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -1,3 +1,4 @@ +import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; import { firstTaskNativeRuntimePatch, @@ -245,6 +246,7 @@ export async function runFirstTaskFlow(input: { comments, interactions, documents, + attachments: await captureFirstTaskAttachments(api, tasks, input.secrets), runs, }; e.checkpoints.push(checkpoint); diff --git a/tests/runner-e2e/first-task-quality.ts b/tests/runner-e2e/first-task-quality.ts index f77a6fbe790..b2dfd526042 100644 --- a/tests/runner-e2e/first-task-quality.ts +++ b/tests/runner-e2e/first-task-quality.ts @@ -87,6 +87,7 @@ export function qualityInput(evidence: FirstTaskEvidence) { interactions: c.interactions, tasks: c.tasks, documents: c.documents, + attachments: c.attachments ?? [], })), }); } diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index 7c8b8814219..e9f8b1b58b2 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -19,6 +19,7 @@ export interface FirstTaskCheckpoint { comments: Row[]; interactions: Row[]; documents: Row[]; + attachments?: Row[]; runs: Row[]; } export interface FirstTaskEvidence { @@ -158,11 +159,23 @@ function isPlanningDocument(document: Row): boolean { return ( isPlanDocument(document) || (/(?:^|[-_])proposal(?:$|[-_])/i.test(String(document.key)) && - /(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single) )?task|single[- ]task proposal|proposal)\b/i.test( + /(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single|first) )?task|(?:(?:first|single)[- ])?task proposal|proposal)\b/i.test( `${document.title ?? ""}\n${document.body ?? ""}`, )) ); } +function isVerifiedAttachment(a: Row): boolean { + return a.contentVerified === true && typeof a.body === "string" && a.contentSha256 === digestText(a.body); +} +function attachmentDocument(a: Row): Row { + return { ...a, key: String(a.originalFilename ?? a.filename ?? "").replace(/\.(?:md|txt)$/i, ""), title: a.title ?? a.originalFilename ?? a.filename }; +} +function isPlanningAttachment(a: Row): boolean { + return isVerifiedAttachment(a) && isPlanningDocument(attachmentDocument(a)); +} +function verifiedFirstTaskOutputs(checkpoint: FirstTaskCheckpoint): Row[] { + return [...checkpoint.documents, ...(checkpoint.attachments ?? []).filter(isVerifiedAttachment).map(attachmentDocument)]; +} export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { const scenario = firstTaskScenario(e.caseId, e.nonce); const checks: FirstTaskCheck[] = []; @@ -299,6 +312,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { extras(c).length === 0 && c.agents.length === e.checkpoints[0].agents.length && c.documents.every(isPlanningDocument) && + (c.attachments ?? []).every(isPlanningAttachment) && (c.tasks.find((t) => t.id === c.issueId)?.status !== "done" || Boolean(rejection && Date.parse(c.at) >= Date.parse(rejection.at))), ), @@ -310,7 +324,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "ordinary-task-control", questions.length === 0 && extras(last).length === 0 && - last.documents.some( + verifiedFirstTaskOutputs(last).some( (d) => !isPlanningDocument(d) && String(d.body).includes(scenario.marker), ), @@ -375,7 +389,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "durable-completion", children.length === 1 && children[0].status === "done" && - last.documents.some( + verifiedFirstTaskOutputs(last).some( (d) => d.issueId === children[0].id && !isPlanningDocument(d) && @@ -395,6 +409,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "rejection-respected", extras(last).length === 0 && last.documents.every(isPlanningDocument) && + (last.attachments ?? []).every(isPlanningAttachment) && activeRuns(last.runs).length === 0, "Rejected work never executes", [last.id], diff --git a/tests/runner-e2e/first-task-transcript.ts b/tests/runner-e2e/first-task-transcript.ts index 16cac40cbcb..26ef60b243a 100644 --- a/tests/runner-e2e/first-task-transcript.ts +++ b/tests/runner-e2e/first-task-transcript.ts @@ -19,7 +19,7 @@ const html = (value: unknown) => ); export interface TranscriptEntry { id: string; - kind: "comment" | "interaction" | "answer" | "document" | "run"; + kind: "comment" | "interaction" | "answer" | "document" | "attachment" | "run"; at: string; checkpoint: string; row: Row; @@ -33,6 +33,7 @@ export function firstTaskTranscript(e: FirstTaskEvidence): TranscriptEntry[] { ["comment", checkpoint.comments], ["interaction", checkpoint.interactions], ["document", checkpoint.documents], + ["attachment", checkpoint.attachments ?? []], ["run", checkpoint.runs], ] as const) { for (const row of rows) { @@ -90,7 +91,7 @@ export function renderFirstTaskTranscript( ); const entries = firstTaskTranscript(e); return `
-
About this recording

Recorded conversation: comments, question and approval cards, answers, and observed document revisions. Repeated checkpoints are deduplicated. Cards reconstruct saved prompts and recorded selections; multi-question forms are expanded for review. Run metadata is expandable; raw tool events are available through the evidence links. This is retained evidence, not a live task. Only messages and document revisions captured at checkpoints are available; messages from other tasks may not be included.

+
About this recording

Recorded conversation: comments, question and approval cards, answers, observed document revisions, and saved attachments. Repeated checkpoints are deduplicated. Cards reconstruct saved prompts and recorded selections; multi-question forms are expanded for review. Run metadata is expandable; raw tool events are available through the evidence links. This is retained evidence, not a live task. Only messages and document revisions captured at checkpoints are available; messages from other tasks may not be included.

${ entries .map((entry) => { @@ -145,6 +146,10 @@ export function renderFirstTaskTranscript( : body(row.result.outcome ?? row.status) + (row.result.reason ? body(row.result.reason) : ""); content += raw(row.result, "Recorded answer"); + } else if (entry.kind === "attachment") { + title = `Attachment · ${row.title ?? row.originalFilename ?? row.filename ?? row.id}`; + content = row.contentVerified ? body(row.body) : "

Attachment recorded; text content unavailable.

"; + content += raw(row, "Saved attachment evidence"); } else if (entry.kind === "document") { title = `Document · ${row.title ?? row.key} · revision ${row.latestRevisionNumber ?? row.revisionNumber ?? "unreported"}`; content = body(row.body); diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index 72dc346c6b6..10990a86cec 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -1,3 +1,5 @@ +import { captureFirstTaskAttachments } from "./first-task-attachments.js"; +import type { RunnerApi } from "./api.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; import { createIssueThreadInteractionSchema } from "../../packages/shared/src/validators/issue.js"; @@ -185,6 +187,29 @@ const scores = () => evidence: ["response-1"], })); +describe("first-task attachment evidence", () => { + it("reads persisted bytes, verifies their hash, and redacts the retained text", async () => { + const body = "Welcome SECRET-VALUE"; + const attachment = { id: "attachment", issueId: "child", byteSize: Buffer.byteLength(body), sha256: digestText(body), contentType: "text/markdown" }; + const request = { get: vi.fn().mockResolvedValue({ ok: () => true, body: async () => Buffer.from(body) }) }; + const api = { get: vi.fn().mockResolvedValue([attachment]), request } as unknown as RunnerApi; + const rows = await captureFirstTaskAttachments(api, [{ id: "child" }], ["SECRET-VALUE"]); + expect(rows[0]).toMatchObject({ issueId: "child", contentVerified: true }); + expect(rows[0].body).not.toContain("SECRET-VALUE"); + expect(rows[0].contentSha256).toBe(digestText(rows[0].body)); + expect(request.get).toHaveBeenCalledWith("/api/attachments/attachment/content"); + attachment.sha256 = "wrong"; + await expect(captureFirstTaskAttachments(api, [{ id: "child" }], [])).rejects.toThrow("hash mismatch"); + }); + + it("retains unavailable text as metadata without claiming verified content", async () => { + const request = { get: vi.fn() }; + const api = { get: vi.fn().mockResolvedValue([{ id: "binary", contentType: "application/pdf", byteSize: 100 }]), request } as unknown as RunnerApi; + expect(await captureFirstTaskAttachments(api, [{ id: "child" }], [])).toEqual([{ id: "binary", issueId: "child", contentType: "application/pdf", byteSize: 100 }]); + expect(request.get).not.toHaveBeenCalled(); + }); +}); + describe("first-task question presentation grading", () => { it("documents an API-valid text card that renders without a one-option choice", async () => { const reference = await readFile( @@ -465,6 +490,56 @@ Accept the card above and I write it. This task stays in review until then.`; expect(failed(e)).toContain("subtask-proposal"); }); + it("accepts verified attached output on the child, but not metadata, a wrong task, or a changed body", () => { + const e = recording(); + const final = e.checkpoints.at(-1)!; + const output = final.documents.pop()!; + const attachment = { ...output, issueId: "child", body: String(output.body), filename: "welcome.md", contentType: "text/markdown", contentVerified: true, contentSha256: digestText(output.body) }; + final.attachments = [attachment]; + expect(failed(e)).toEqual([]); + attachment.contentVerified = false; + expect(failed(e)).toContain("durable-completion"); + attachment.contentVerified = true; + attachment.issueId = "onboarding"; + expect(failed(e)).toContain("durable-completion"); + attachment.issueId = "child"; + attachment.body += " changed"; + expect(failed(e)).toContain("durable-completion"); + }); + + it("recognizes verified ordinary-task attachments and renders them for review", () => { + const e = recording("ordinary-task-control"); + const last = e.checkpoints.at(-1)!; + last.tasks.pop(); + const output = last.documents.pop()!; + last.attachments = [{ ...output, issueId: "onboarding", filename: "welcome.md", contentVerified: true, contentSha256: digestText(output.body) }]; + expect(failed(e)).toEqual([]); + const page = renderFirstTaskDetails(result(e)); + expect(page).toContain("Attachment · welcome.md"); + expect(page).toContain(output.body); + }); + + it("allows a verified planning attachment before approval without treating it as finished output", () => { + const e = recording(); + const body = "# Plan\n\nWrite the welcome note after approval."; + const plan = { id: "plan-file", issueId: "onboarding", filename: "plan.md", body, contentVerified: true, contentSha256: digestText(body) }; + e.checkpoints[1].attachments = [plan]; + expect(failed(e)).toEqual([]); + e.checkpoints.at(-1)!.documents = []; + e.checkpoints.at(-1)!.attachments = [plan]; + expect(failed(e)).toContain("durable-completion"); + }); + + it("counts attached finished output before approval and after rejection as work", () => { + const e = recording("reject-no-execution"); + e.checkpoints = e.checkpoints.slice(0, 2); + const c = e.checkpoints[1]; + c.attachments = [{ id: "file", issueId: "onboarding", filename: "welcome.md" }]; + expect(failed(e)).toContain("no-premature-work"); + e.checkpoints.push({ ...structuredClone(c), id: "rejected", phase: "rejected" }); + expect(failed(e)).toContain("rejection-respected"); + }); + it("allows a proposal document before acceptance but never counts it as finished output", () => { const e = recording(); const proposal = { @@ -489,9 +564,12 @@ Accept the card above and I write it. This task stays in review until then.`; // Reduced fixtures from gha-35021304437-1; no provider calls or private state. it.each([ ["task-card-accept", "Neighborhood Garden Club Welcome Note — Proposal", "## Proposed child task"], + ["task-reply-accept", "Neighborhood Garden Club Welcome Note — Proposal", "# Task proposal"], + ["clarify-propose-accept", "First task proposal", "# First task proposal"], + ["revise-accept", "Proposed first task: Garden club welcome note", "# Proposed first task"], ["reject-no-execution", "Garden club welcome note — task proposal", "# Proposed single task"], ])("recognizes the recorded %s proposal without inventing downstream failures", (caseId, title, heading) => { - const e = recording(caseId); + const e = recording(caseId === "clarify-propose-accept" ? "task-card-accept" : caseId); e.checkpoints = e.checkpoints.slice(0, 2); const response = e.checkpoints[1]; response.comments.pop(); diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index f1e7b0e21f0..93e48a6d5a7 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -18,6 +18,38 @@ afterEach(async () => { }); describe("runner E2E report aggregation", () => { + it("keeps interrupted journeys incomplete unless their evidence is invalid", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-incomplete-report-")); + cleanupDirectories.push(root); + const ids: string[] = []; + for (const [caseId, validEvidence] of [["task-card-accept", true], ["task-reply-accept", false]] as const) { + const executionId = `first-task.runner-codex.local.${caseId}`; + ids.push(executionId); + const directory = path.join(root, caseId); + await mkdir(directory); + await writeFile(path.join(directory, "result.json"), JSON.stringify({ + schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "first-task", + attempt: 1, status: "failed", failureClass: "candidate_failure", error: "Recording stopped before acceptance", + profileId: "runner-codex", environmentId: "local", caseId, provider: "codex", model: "fixture-model", runtimeMode: "native", + startedAt: "2026-09-15T00:00:00Z", finishedAt: "2026-09-15T00:01:00Z", durationMs: 60_000, cleanup: "passed", + firstTask: { caseId, nonce: "fixture", onboardingIssueId: "task", agentId: "agent", initialTaskIds: ["task"], instructions: [], configuredModel: null, observedModels: [], checkpoints: [], + checks: [{ id: "acceptance-recorded", passed: false, notReached: "No acceptance checkpoint", detail: "Acceptance recorded", evidence: [] }] }, + } satisfies RunnerE2EResult)); + if (validEvidence) await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: [], leaks: [], missing: [] })); + } + const output = path.join(root, "merged"); + await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(ids) }, + })).rejects.toBeDefined(); + const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")); + expect(normalized).toMatchObject({ passed: 0, failed: 1, incomplete: 1 }); + expect(normalized.results[0]).toMatchObject({ evidenceValid: true, evidenceErrors: [] }); + expect(normalized.results[1]).toMatchObject({ evidenceValid: false, failureClass: "permanent_infrastructure" }); + const page = await readFile(path.join(output, "index.html"), "utf8"); + expect(page).toContain("Incomplete journey"); + expect(page).toContain("evidence manifest missing"); + }); + it("selects the latest retry and enforces cleanup and pass evidence", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-report-test-"), diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 95d7394df91..fa8f0309ad7 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -57,8 +57,6 @@ function xml(value: unknown) { function validate(result: RunnerE2EResult, evidence: EvidenceManifest | null) { const errors: string[] = []; - if (result.status !== "passed") - errors.push(result.error ?? result.failureClass ?? "execution failed"); if (result.cleanup !== "passed") errors.push(`cleanup=${result.cleanup}`); if (!evidence) errors.push("evidence manifest missing"); if (evidence?.leaks?.length) @@ -180,7 +178,7 @@ async function main() { result, evidence, directory, - valid: errors.length === 0, + valid: result.status === "passed" && errors.length === 0, errors, }; candidates.set(result.executionId, [ @@ -260,6 +258,10 @@ async function main() { // of allowing retained result metadata to claim another revision. source: resolveRunnerE2ESource(entry.result.source), status: entry.valid ? entry.result.status : ("failed" as const), + // Evidence failures must not be mistaken for an interrupted behavior recording. + failureClass: entry.errors.length > 0 + ? entry.evidence?.leaks?.length ? "secret_leak" as const : "permanent_infrastructure" as const + : entry.result.failureClass, billing: summarizeExecutionBilling(entry.result), })); const generatedAt = new Date().toISOString(); @@ -278,7 +280,7 @@ async function main() { ...campaign, results: selected.map((entry, index) => ({ ...resolvedResults[index]!, - evidenceValid: entry.valid, + evidenceValid: entry.errors.length === 0, evidenceErrors: entry.errors, })), }; @@ -342,7 +344,7 @@ async function main() { "| Cell | Attempt | Result | Runtime | Duration | Tokens (in/out) | LLM reported | Runtime estimate | Detail |", "|---|---:|---|---|---:|---:|---:|---:|---|", ...selected.map((entry, index) => { - const detail = entry.errors.join("; ").replaceAll("|", "\\|") || "ok"; + const detail = [entry.result.error, ...entry.errors].filter(Boolean).join("; ").replaceAll("|", "\\|") || "ok"; const resolved = resolvedResults[index]!; const cellBilling = resolved.billing!; const runtimeCost = cellBilling.runtime.estimatedListCostUsd; @@ -360,7 +362,7 @@ async function main() { .map((entry) => { const failure = entry.valid ? "" - : ``; + : ``; return `${failure}`; }) .join(""); diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 04acce693fc..be7a0368aa8 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -237,7 +237,7 @@ const fields = { initialTaskIds: array(string), configuredModel: nullable(string), observedModels: array(string), runtimeSettings: optional(object), instructions: array(shape({ path: string, content: string, sha256: string, contentSha256: optional(string), redacted: optional(boolean) })), checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "accepted", "rejected", "finished"), issueId: string, - tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), runs: array(object) })), + tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), attachments: optional(array(object)), runs: array(object) })), checks: array(shape({ id: string, passed: boolean, notReached: optional(string), evidence: array(string), detail: string })), })), firstTaskQuality: optional(shape({