diff --git a/server/src/services/native-runtime/runner-api-reference.ts b/server/src/services/native-runtime/runner-api-reference.ts index f201b6a861c..0ec14a7d61a 100644 --- a/server/src/services/native-runtime/runner-api-reference.ts +++ b/server/src/services/native-runtime/runner-api-reference.ts @@ -245,7 +245,18 @@ export const runnerApiReference: Record { + return (await Promise.all(tasks.map(async task => { + const rows = await api.get(`/api/issues/${encodeURIComponent(task.id)}/attachments`); + return Promise.all(rows.map(async row => { + const attachment: Row = { ...row, issueId: task.id }; + if (!/^text\/(?:plain|markdown)(?:;|$)/i.test(row.contentType ?? "") || + !Number.isFinite(row.byteSize) || row.byteSize > 262_144) return attachment; + const response = await api.request.get(`/api/attachments/${encodeURIComponent(row.id)}/content`); + if (!response.ok()) throw new Error(`Attachment content unavailable: ${row.id} (${response.status()})`); + const bytes = await response.body(); + if (bytes.length !== row.byteSize) throw new Error(`Attachment size mismatch: ${row.id}`); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + if (sha256 !== row.sha256) throw new Error(`Attachment hash mismatch: ${row.id}`); + const safe = sanitizeJson({ body: bytes.toString("utf8") }, secrets) as { body: string }; + return { ...attachment, body: safe.body, contentVerified: true, contentSha256: createHash("sha256").update(safe.body).digest("hex") }; + })); + }))).flat(); +} diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index 5aca5df8f46..dfe923162a6 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -1,3 +1,4 @@ +import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; import { firstTaskNativeRuntimePatch, @@ -245,6 +246,7 @@ export async function runFirstTaskFlow(input: { comments, interactions, documents, + attachments: await captureFirstTaskAttachments(api, tasks, input.secrets), runs, }; e.checkpoints.push(checkpoint); diff --git a/tests/runner-e2e/first-task-quality.ts b/tests/runner-e2e/first-task-quality.ts index f77a6fbe790..b2dfd526042 100644 --- a/tests/runner-e2e/first-task-quality.ts +++ b/tests/runner-e2e/first-task-quality.ts @@ -87,6 +87,7 @@ export function qualityInput(evidence: FirstTaskEvidence) { interactions: c.interactions, tasks: c.tasks, documents: c.documents, + attachments: c.attachments ?? [], })), }); } diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index 7c8b8814219..e9f8b1b58b2 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -19,6 +19,7 @@ export interface FirstTaskCheckpoint { comments: Row[]; interactions: Row[]; documents: Row[]; + attachments?: Row[]; runs: Row[]; } export interface FirstTaskEvidence { @@ -158,11 +159,23 @@ function isPlanningDocument(document: Row): boolean { return ( isPlanDocument(document) || (/(?:^|[-_])proposal(?:$|[-_])/i.test(String(document.key)) && - /(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single) )?task|single[- ]task proposal|proposal)\b/i.test( + /(?:^|\n)(?:#+\s*)?(?:proposed (?:(?:child|single|first) )?task|(?:(?:first|single)[- ])?task proposal|proposal)\b/i.test( `${document.title ?? ""}\n${document.body ?? ""}`, )) ); } +function isVerifiedAttachment(a: Row): boolean { + return a.contentVerified === true && typeof a.body === "string" && a.contentSha256 === digestText(a.body); +} +function attachmentDocument(a: Row): Row { + return { ...a, key: String(a.originalFilename ?? a.filename ?? "").replace(/\.(?:md|txt)$/i, ""), title: a.title ?? a.originalFilename ?? a.filename }; +} +function isPlanningAttachment(a: Row): boolean { + return isVerifiedAttachment(a) && isPlanningDocument(attachmentDocument(a)); +} +function verifiedFirstTaskOutputs(checkpoint: FirstTaskCheckpoint): Row[] { + return [...checkpoint.documents, ...(checkpoint.attachments ?? []).filter(isVerifiedAttachment).map(attachmentDocument)]; +} export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { const scenario = firstTaskScenario(e.caseId, e.nonce); const checks: FirstTaskCheck[] = []; @@ -299,6 +312,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { extras(c).length === 0 && c.agents.length === e.checkpoints[0].agents.length && c.documents.every(isPlanningDocument) && + (c.attachments ?? []).every(isPlanningAttachment) && (c.tasks.find((t) => t.id === c.issueId)?.status !== "done" || Boolean(rejection && Date.parse(c.at) >= Date.parse(rejection.at))), ), @@ -310,7 +324,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "ordinary-task-control", questions.length === 0 && extras(last).length === 0 && - last.documents.some( + verifiedFirstTaskOutputs(last).some( (d) => !isPlanningDocument(d) && String(d.body).includes(scenario.marker), ), @@ -375,7 +389,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "durable-completion", children.length === 1 && children[0].status === "done" && - last.documents.some( + verifiedFirstTaskOutputs(last).some( (d) => d.issueId === children[0].id && !isPlanningDocument(d) && @@ -395,6 +409,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "rejection-respected", extras(last).length === 0 && last.documents.every(isPlanningDocument) && + (last.attachments ?? []).every(isPlanningAttachment) && activeRuns(last.runs).length === 0, "Rejected work never executes", [last.id], diff --git a/tests/runner-e2e/first-task-transcript.ts b/tests/runner-e2e/first-task-transcript.ts index 16cac40cbcb..26ef60b243a 100644 --- a/tests/runner-e2e/first-task-transcript.ts +++ b/tests/runner-e2e/first-task-transcript.ts @@ -19,7 +19,7 @@ const html = (value: unknown) => ); export interface TranscriptEntry { id: string; - kind: "comment" | "interaction" | "answer" | "document" | "run"; + kind: "comment" | "interaction" | "answer" | "document" | "attachment" | "run"; at: string; checkpoint: string; row: Row; @@ -33,6 +33,7 @@ export function firstTaskTranscript(e: FirstTaskEvidence): TranscriptEntry[] { ["comment", checkpoint.comments], ["interaction", checkpoint.interactions], ["document", checkpoint.documents], + ["attachment", checkpoint.attachments ?? []], ["run", checkpoint.runs], ] as const) { for (const row of rows) { @@ -90,7 +91,7 @@ export function renderFirstTaskTranscript( ); const entries = firstTaskTranscript(e); return `
-
About this recording

Recorded conversation: comments, question and approval cards, answers, and observed document revisions. Repeated checkpoints are deduplicated. Cards reconstruct saved prompts and recorded selections; multi-question forms are expanded for review. Run metadata is expandable; raw tool events are available through the evidence links. This is retained evidence, not a live task. Only messages and document revisions captured at checkpoints are available; messages from other tasks may not be included.

+
About this recording

Recorded conversation: comments, question and approval cards, answers, observed document revisions, and saved attachments. Repeated checkpoints are deduplicated. Cards reconstruct saved prompts and recorded selections; multi-question forms are expanded for review. Run metadata is expandable; raw tool events are available through the evidence links. This is retained evidence, not a live task. Only messages and document revisions captured at checkpoints are available; messages from other tasks may not be included.

${ entries .map((entry) => { @@ -145,6 +146,10 @@ export function renderFirstTaskTranscript( : body(row.result.outcome ?? row.status) + (row.result.reason ? body(row.result.reason) : ""); content += raw(row.result, "Recorded answer"); + } else if (entry.kind === "attachment") { + title = `Attachment · ${row.title ?? row.originalFilename ?? row.filename ?? row.id}`; + content = row.contentVerified ? body(row.body) : "

Attachment recorded; text content unavailable.

"; + content += raw(row, "Saved attachment evidence"); } else if (entry.kind === "document") { title = `Document · ${row.title ?? row.key} · revision ${row.latestRevisionNumber ?? row.revisionNumber ?? "unreported"}`; content = body(row.body); diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index 72dc346c6b6..10990a86cec 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -1,3 +1,5 @@ +import { captureFirstTaskAttachments } from "./first-task-attachments.js"; +import type { RunnerApi } from "./api.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; import { createIssueThreadInteractionSchema } from "../../packages/shared/src/validators/issue.js"; @@ -185,6 +187,29 @@ const scores = () => evidence: ["response-1"], })); +describe("first-task attachment evidence", () => { + it("reads persisted bytes, verifies their hash, and redacts the retained text", async () => { + const body = "Welcome SECRET-VALUE"; + const attachment = { id: "attachment", issueId: "child", byteSize: Buffer.byteLength(body), sha256: digestText(body), contentType: "text/markdown" }; + const request = { get: vi.fn().mockResolvedValue({ ok: () => true, body: async () => Buffer.from(body) }) }; + const api = { get: vi.fn().mockResolvedValue([attachment]), request } as unknown as RunnerApi; + const rows = await captureFirstTaskAttachments(api, [{ id: "child" }], ["SECRET-VALUE"]); + expect(rows[0]).toMatchObject({ issueId: "child", contentVerified: true }); + expect(rows[0].body).not.toContain("SECRET-VALUE"); + expect(rows[0].contentSha256).toBe(digestText(rows[0].body)); + expect(request.get).toHaveBeenCalledWith("/api/attachments/attachment/content"); + attachment.sha256 = "wrong"; + await expect(captureFirstTaskAttachments(api, [{ id: "child" }], [])).rejects.toThrow("hash mismatch"); + }); + + it("retains unavailable text as metadata without claiming verified content", async () => { + const request = { get: vi.fn() }; + const api = { get: vi.fn().mockResolvedValue([{ id: "binary", contentType: "application/pdf", byteSize: 100 }]), request } as unknown as RunnerApi; + expect(await captureFirstTaskAttachments(api, [{ id: "child" }], [])).toEqual([{ id: "binary", issueId: "child", contentType: "application/pdf", byteSize: 100 }]); + expect(request.get).not.toHaveBeenCalled(); + }); +}); + describe("first-task question presentation grading", () => { it("documents an API-valid text card that renders without a one-option choice", async () => { const reference = await readFile( @@ -465,6 +490,56 @@ Accept the card above and I write it. This task stays in review until then.`; expect(failed(e)).toContain("subtask-proposal"); }); + it("accepts verified attached output on the child, but not metadata, a wrong task, or a changed body", () => { + const e = recording(); + const final = e.checkpoints.at(-1)!; + const output = final.documents.pop()!; + const attachment = { ...output, issueId: "child", body: String(output.body), filename: "welcome.md", contentType: "text/markdown", contentVerified: true, contentSha256: digestText(output.body) }; + final.attachments = [attachment]; + expect(failed(e)).toEqual([]); + attachment.contentVerified = false; + expect(failed(e)).toContain("durable-completion"); + attachment.contentVerified = true; + attachment.issueId = "onboarding"; + expect(failed(e)).toContain("durable-completion"); + attachment.issueId = "child"; + attachment.body += " changed"; + expect(failed(e)).toContain("durable-completion"); + }); + + it("recognizes verified ordinary-task attachments and renders them for review", () => { + const e = recording("ordinary-task-control"); + const last = e.checkpoints.at(-1)!; + last.tasks.pop(); + const output = last.documents.pop()!; + last.attachments = [{ ...output, issueId: "onboarding", filename: "welcome.md", contentVerified: true, contentSha256: digestText(output.body) }]; + expect(failed(e)).toEqual([]); + const page = renderFirstTaskDetails(result(e)); + expect(page).toContain("Attachment · welcome.md"); + expect(page).toContain(output.body); + }); + + it("allows a verified planning attachment before approval without treating it as finished output", () => { + const e = recording(); + const body = "# Plan\n\nWrite the welcome note after approval."; + const plan = { id: "plan-file", issueId: "onboarding", filename: "plan.md", body, contentVerified: true, contentSha256: digestText(body) }; + e.checkpoints[1].attachments = [plan]; + expect(failed(e)).toEqual([]); + e.checkpoints.at(-1)!.documents = []; + e.checkpoints.at(-1)!.attachments = [plan]; + expect(failed(e)).toContain("durable-completion"); + }); + + it("counts attached finished output before approval and after rejection as work", () => { + const e = recording("reject-no-execution"); + e.checkpoints = e.checkpoints.slice(0, 2); + const c = e.checkpoints[1]; + c.attachments = [{ id: "file", issueId: "onboarding", filename: "welcome.md" }]; + expect(failed(e)).toContain("no-premature-work"); + e.checkpoints.push({ ...structuredClone(c), id: "rejected", phase: "rejected" }); + expect(failed(e)).toContain("rejection-respected"); + }); + it("allows a proposal document before acceptance but never counts it as finished output", () => { const e = recording(); const proposal = { @@ -489,9 +564,12 @@ Accept the card above and I write it. This task stays in review until then.`; // Reduced fixtures from gha-35021304437-1; no provider calls or private state. it.each([ ["task-card-accept", "Neighborhood Garden Club Welcome Note — Proposal", "## Proposed child task"], + ["task-reply-accept", "Neighborhood Garden Club Welcome Note — Proposal", "# Task proposal"], + ["clarify-propose-accept", "First task proposal", "# First task proposal"], + ["revise-accept", "Proposed first task: Garden club welcome note", "# Proposed first task"], ["reject-no-execution", "Garden club welcome note — task proposal", "# Proposed single task"], ])("recognizes the recorded %s proposal without inventing downstream failures", (caseId, title, heading) => { - const e = recording(caseId); + const e = recording(caseId === "clarify-propose-accept" ? "task-card-accept" : caseId); e.checkpoints = e.checkpoints.slice(0, 2); const response = e.checkpoints[1]; response.comments.pop(); diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index f1e7b0e21f0..93e48a6d5a7 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -18,6 +18,38 @@ afterEach(async () => { }); describe("runner E2E report aggregation", () => { + it("keeps interrupted journeys incomplete unless their evidence is invalid", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-incomplete-report-")); + cleanupDirectories.push(root); + const ids: string[] = []; + for (const [caseId, validEvidence] of [["task-card-accept", true], ["task-reply-accept", false]] as const) { + const executionId = `first-task.runner-codex.local.${caseId}`; + ids.push(executionId); + const directory = path.join(root, caseId); + await mkdir(directory); + await writeFile(path.join(directory, "result.json"), JSON.stringify({ + schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "first-task", + attempt: 1, status: "failed", failureClass: "candidate_failure", error: "Recording stopped before acceptance", + profileId: "runner-codex", environmentId: "local", caseId, provider: "codex", model: "fixture-model", runtimeMode: "native", + startedAt: "2026-09-15T00:00:00Z", finishedAt: "2026-09-15T00:01:00Z", durationMs: 60_000, cleanup: "passed", + firstTask: { caseId, nonce: "fixture", onboardingIssueId: "task", agentId: "agent", initialTaskIds: ["task"], instructions: [], configuredModel: null, observedModels: [], checkpoints: [], + checks: [{ id: "acceptance-recorded", passed: false, notReached: "No acceptance checkpoint", detail: "Acceptance recorded", evidence: [] }] }, + } satisfies RunnerE2EResult)); + if (validEvidence) await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: [], leaks: [], missing: [] })); + } + const output = path.join(root, "merged"); + await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(ids) }, + })).rejects.toBeDefined(); + const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")); + expect(normalized).toMatchObject({ passed: 0, failed: 1, incomplete: 1 }); + expect(normalized.results[0]).toMatchObject({ evidenceValid: true, evidenceErrors: [] }); + expect(normalized.results[1]).toMatchObject({ evidenceValid: false, failureClass: "permanent_infrastructure" }); + const page = await readFile(path.join(output, "index.html"), "utf8"); + expect(page).toContain("Incomplete journey"); + expect(page).toContain("evidence manifest missing"); + }); + it("selects the latest retry and enforces cleanup and pass evidence", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-report-test-"), diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 95d7394df91..fa8f0309ad7 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -57,8 +57,6 @@ function xml(value: unknown) { function validate(result: RunnerE2EResult, evidence: EvidenceManifest | null) { const errors: string[] = []; - if (result.status !== "passed") - errors.push(result.error ?? result.failureClass ?? "execution failed"); if (result.cleanup !== "passed") errors.push(`cleanup=${result.cleanup}`); if (!evidence) errors.push("evidence manifest missing"); if (evidence?.leaks?.length) @@ -180,7 +178,7 @@ async function main() { result, evidence, directory, - valid: errors.length === 0, + valid: result.status === "passed" && errors.length === 0, errors, }; candidates.set(result.executionId, [ @@ -260,6 +258,10 @@ async function main() { // of allowing retained result metadata to claim another revision. source: resolveRunnerE2ESource(entry.result.source), status: entry.valid ? entry.result.status : ("failed" as const), + // Evidence failures must not be mistaken for an interrupted behavior recording. + failureClass: entry.errors.length > 0 + ? entry.evidence?.leaks?.length ? "secret_leak" as const : "permanent_infrastructure" as const + : entry.result.failureClass, billing: summarizeExecutionBilling(entry.result), })); const generatedAt = new Date().toISOString(); @@ -278,7 +280,7 @@ async function main() { ...campaign, results: selected.map((entry, index) => ({ ...resolvedResults[index]!, - evidenceValid: entry.valid, + evidenceValid: entry.errors.length === 0, evidenceErrors: entry.errors, })), }; @@ -342,7 +344,7 @@ async function main() { "| Cell | Attempt | Result | Runtime | Duration | Tokens (in/out) | LLM reported | Runtime estimate | Detail |", "|---|---:|---|---|---:|---:|---:|---:|---|", ...selected.map((entry, index) => { - const detail = entry.errors.join("; ").replaceAll("|", "\\|") || "ok"; + const detail = [entry.result.error, ...entry.errors].filter(Boolean).join("; ").replaceAll("|", "\\|") || "ok"; const resolved = resolvedResults[index]!; const cellBilling = resolved.billing!; const runtimeCost = cellBilling.runtime.estimatedListCostUsd; @@ -360,7 +362,7 @@ async function main() { .map((entry) => { const failure = entry.valid ? "" - : ``; + : ``; return `${failure}`; }) .join(""); diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 04acce693fc..be7a0368aa8 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -237,7 +237,7 @@ const fields = { initialTaskIds: array(string), configuredModel: nullable(string), observedModels: array(string), runtimeSettings: optional(object), instructions: array(shape({ path: string, content: string, sha256: string, contentSha256: optional(string), redacted: optional(boolean) })), checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "accepted", "rejected", "finished"), issueId: string, - tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), runs: array(object) })), + tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), attachments: optional(array(object)), runs: array(object) })), checks: array(shape({ id: string, passed: boolean, notReached: optional(string), evidence: array(string), detail: string })), })), firstTaskQuality: optional(shape({