diff --git a/doc/evals.md b/doc/evals.md index f3bd414277a..2e156379b48 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -377,6 +377,12 @@ and startup evidence under Diagnostics. Optional `presentation` metadata in a retained campaign supplies its title, description, notes, and evidence links inside the shared report shell when the dashboard is regenerated. +Standard runner Product E2E attempts also retain a sanitized `taskChat` recording +for Test details: task briefs, complete saved messages, requests and answers, +and native agent progress. Related task threads are separate; incomplete capture +is labeled. This automatic capture runs before teardown on success and failure. +See the [Product E2E evidence contract](../tests/runner-e2e/README.md#evidence-and-cleanup). + Existing public histories are available at [Runner protocol history](https://d1p6rlowie26tp.cloudfront.net/runner-protocol-evals/index.html) and [Runner Product E2E history](https://d1p6rlowie26tp.cloudfront.net/runner-e2e/). diff --git a/tests/e2e/runner-e2e-dashboard.spec.ts b/tests/e2e/runner-e2e-dashboard.spec.ts index e9cc5c5237b..f52a66ea59b 100644 --- a/tests/e2e/runner-e2e-dashboard.spec.ts +++ b/tests/e2e/runner-e2e-dashboard.spec.ts @@ -7,6 +7,7 @@ import { } from "../runner-e2e/dashboard.js"; import { buildRunnerCampaign } from "../runner-e2e/history.js"; import type { MatrixExecution, RunnerE2EResult } from "../runner-e2e/types.js"; +import { projectTaskChatThread } from "../runner-e2e/task-chat.js"; const reportUrl = "http://runner-e2e.test/campaign/"; @@ -73,6 +74,27 @@ function resultFor( }; } +test("expanding test details exposes the complete saved task conversation", async ({ page }) => { + const execution = runnerExecutionById("core-compatibility.runner-codex.local.message-marker"); + const entry = resultFor(execution, "passed"); + const longReply = "Here is the result.\n\n" + "Retained paragraph. ".repeat(200) + "END OF COMPLETE REPLY"; + entry.result.taskChat = { schema: "paperclip.runner-e2e.task-chat/v1", capturedAt: "2026-10-28T12:01:00Z", + status: "complete", missing: [], threads: [projectTaskChatThread( + { id: "task", title: "Find the saved artifact", identifier: "E2E-1", description: "Find our saved report.", createdAt: "2026-10-28T12:00:00Z" }, + [{ id: "request", body: "Please include the download link.", createdAt: "2026-10-28T12:00:01Z" }, + { id: "reply", authorAgentId: "agent", body: longReply, createdAt: "2026-10-28T12:00:02Z" }], [], new Map([["agent", "Morgan"]]))] }; + await openReport(page, renderRunnerE2EDashboard({ title: "Task chat recording", catalog: [execution], entries: [entry], expected: [execution.id], generatedAt: "2026-10-28T12:01:00Z" })); + const chat = page.getByRole("region", { name: "Task chat", exact: true }); + await expect(chat).not.toBeVisible(); + await page.getByText("Test details", { exact: true }).click(); + await expect(chat).toBeVisible(); + await expect(chat.getByText("Find our saved report.", { exact: true })).toBeVisible(); + await expect(chat.getByText("Please include the download link.", { exact: true })).toBeVisible(); + await expect(chat.getByText("Morgan", { exact: true })).toBeVisible(); + await expect(chat.locator(".transcript-text").last()).toHaveText(longReply); + await expect(chat.getByText(/END OF COMPLETE REPLY/)).toBeVisible(); +}); + test("filters execution previews and pages through the filtered screenshot gallery", async ({ page, }) => { diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 4ffcb521fe8..3820b8f9541 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1055,6 +1055,24 @@ missing, repeated, or out-of-order sequence numbers and fails capture after 100 full pages instead of grading a truncated stream. Original incomplete captures remain failed evidence; qualifying a fix requires a new live attempt. +Every standard runner attempt automatically captures `taskChat` before fixture +teardown, including failed attempts. Test details show each task's saved brief, +full comments, question/approval prompts and recorded answers in a chat layout, +plus native agent progress when available. Delegated tasks have their own +threads; standing chats remain included after a journey restores the Agent Chat +setting. Comments are paginated and message bodies are retained without display +truncation. Missing reads, repeated pages, and capture limits are labeled partial +or unavailable without changing the workflow's grade. Browser-only attempts with +no tasks show no chat section. Existing reports without `taskChat` continue to +render normally; screenshots cannot reconstruct a historical conversation. + +`taskChat` is a closed, credential-redacted public projection in the normalized +result, so local, CI, regenerated, and published dashboards share the same +recording. It contains visible conversation text, not raw run logs, reasoning, +tool arguments or provider/session metadata. Collection uses the public API, +has a 30-second request-admission budget, and caps task/comment discovery at +100 pages; incomplete collection is never labeled complete. + Every completed local campaign also writes `tests/runner-e2e/results//dashboard.html`. The self-contained page shows every selected execution with screenshot thumbnails, grouped by suite and case. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index c3dde9aef93..d000473077d 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -322,6 +322,17 @@ in the retained Actions artifact. Credential redaction does not remove hidden reasoning or provider session identities, so a text extension never admits a file to the public bundle. Graded results remain in `normalized-results.json`. +The normalized result may also contain the reviewed `taskChat` projection: +task titles/briefs, complete saved comments, visible question/approval copy and +recorded answers, agent display names, timestamps, and explicitly classified +native assistant progress. It is captured only from the disposable fixture +company and credential-redacted before retention. Only these presentation +fields are admitted by the closed result validator. Arbitrary interaction +payloads, tool arguments/results, provider/session identities, reasoning and +raw event envelopes are excluded. No per-attempt snapshot or log becomes public +as a consequence. Fixture authors must keep task conversation text public just +as they do the visible content of marked screenshots. + The packaged evidence uploaded as a 30-day GitHub Actions artifact has a different, broader boundary. Text is exact-value and key-shape scanned and redacted. PNG and WebM are raw-byte scanned; SVG is rejected during packaging diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 424bb37ae9b..65ba170db72 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -10,6 +10,7 @@ import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js"; import { STARTUP_REUSE_LAYERS } from "../../packages/adapter-utils/src/startup-reuse.js"; import { renderFirstTaskDetails } from "./first-task-report.js"; +import { renderTaskChat } from "./task-chat-report.js"; import { summarizeCampaignFeedback } from "./feedback.js"; import { renderCampaignFeedbackReview } from "./feedback-dispositions.js"; import { renderEvalOperationsReport } from "./operations-report.js"; @@ -160,6 +161,8 @@ function renderCase( const outcome = entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""; const passed = state === "passed" && entry?.result.status === "passed"; const firstTaskChecks = entry?.result.firstTask?.checks; + const taskChatCoversOnboarding = entry?.result.taskChat?.status === "complete" && + entry.result.taskChat.threads.some(thread => thread.issueId === entry.result.firstTask?.onboardingIssueId); const productChecks = entry?.result.productJourney?.checks; const notReached = firstTaskChecks?.filter((c) => c.notReached).length ?? 0; const matcherResults = firstTaskChecks?.length @@ -323,7 +326,8 @@ function renderCase( ` : ""} ${matcherResults.length || notReached ? `` : ""} ${billingStrip} - ${renderFirstTaskDetails(entry?.result)} + ${renderFirstTaskDetails(entry?.result, !taskChatCoversOnboarding)} + ${renderTaskChat(entry?.result.taskChat)} ${links}
Diagnostics diff --git a/tests/runner-e2e/first-task-report.ts b/tests/runner-e2e/first-task-report.ts index ac06264640c..3dc9fcdeed7 100644 --- a/tests/runner-e2e/first-task-report.ts +++ b/tests/runner-e2e/first-task-report.ts @@ -8,14 +8,14 @@ const html = (value: unknown) => c ]!, ); -export function renderFirstTaskDetails(result?: RunnerE2EResult) { +export function renderFirstTaskDetails(result?: RunnerE2EResult, includeConversation = true) { const e = result?.firstTask; if (!e) return ""; const prefix = `first-task-${encodeURIComponent(result!.executionId)}`; const q = result?.firstTaskQuality; const ref = (id: string) => `${prefix}-${encodeURIComponent(id)}`; const last = e.checkpoints.at(-1); - return `
Read full conversation${renderFirstTaskTranscript(e, (id) => `#${ref(id)}`)}
Onboarding instructions, approval timeline, and quality + return `${includeConversation ? `
Read full conversation${renderFirstTaskTranscript(e, (id) => `#${ref(id)}`)}
` : ""}
Onboarding instructions, approval timeline, and quality

Source: ${html(e.source?.sha ?? result?.source?.sha ?? "Unknown")} · ${html(e.source?.ref ?? result?.source?.ref ?? "Unknown")} ${e.source?.dirty ? "(working tree modified; compare instruction hashes)" : ""}

${e.runtimeSettings?.onboardingRuntime ? `

Setup: ${html((e.runtimeSettings.onboardingRuntime as { mode: string }).mode)}.

` : ""}

Configured model: ${html(e.configuredModel ?? "Provider default")}. Observed: ${html(e.observedModels.join(", ") || "Not reported")}.

diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 950750f0984..97dc40f9d1f 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -8,6 +8,8 @@ import { afterEach, describe, expect, it } from "vitest"; import { STARTUP_REUSE_LAYERS, STARTUP_REUSE_SCHEMA, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; import type { RunnerE2EResult } from "./types.js"; import { classifyLatency, observationTiming } from "./latency-timing.js"; +import { projectTaskChatThread } from "./task-chat.js"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; const execFileAsync = promisify(execFile); const repositoryRoot = path.resolve(import.meta.dirname, "../.."); @@ -21,6 +23,44 @@ afterEach(async () => { }); describe("runner E2E report aggregation", () => { + it("retains task conversations through aggregation and public dashboard regeneration", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-task-chat-report-")); + cleanupDirectories.push(root); + const directory = path.join(root, "attempt-1"); + await mkdir(directory); + const executionId = "core-compatibility.runner-codex.local.message-marker"; + const at = "2026-10-28T12:00:00Z"; + const result: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility", attempt: 1, + status: "passed", profileId: "runner-codex", environmentId: "local", caseId: "message-marker", + provider: "codex", model: "fixture", runtimeMode: "native", startedAt: at, finishedAt: at, + durationMs: 0, cleanup: "passed", taskChat: { + schema: "paperclip.runner-e2e.task-chat/v1", capturedAt: at, status: "complete", missing: [], + threads: [projectTaskChatThread({ id: "task", title: "Saved task conversation", description: "Original user request", createdAt: at }, + [{ id: "reply", authorAgentId: "agent", createdAt: at, body: "Entire persisted response " }], [], new Map([["agent", "Morgan"]]))], + }, + }; + await writeFile(path.join(directory, "result.json"), JSON.stringify(result)); + await writeFile(path.join(directory, "final-state.png"), "fixture-png"); + await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] })); + const output = path.join(root, "merged"); + await execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]) }, + }); + const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")); + expect(normalized.results[0].taskChat).toEqual(result.taskChat); + const regenerated = path.join(root, "regenerated"); + await regenerateRunnerDashboard({ bundle: output, outputDirectory: regenerated, historyFile: null }); + for (const folder of [output, regenerated]) { + const page = await readFile(path.join(folder, "index.html"), "utf8"); + expect(page).toContain('aria-label="Task chat"'); + expect(page).toContain("Original user request"); + expect(page).toContain("Entire persisted response <script>unsafe()</script>"); + expect(page).not.toContain(""); + } + }); + it("publishes retained passing results but exits unsuccessfully when the request journal is damaged", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-damaged-ledger-report-")); cleanupDirectories.push(root); diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 67506fb947e..59c7685f349 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -335,6 +335,17 @@ const fields = { }), ), ), + taskChat: optional(closedShape({ + schema: oneOf("paperclip.runner-e2e.task-chat/v1"), capturedAt: date, + status: oneOf("complete", "partial", "unavailable"), + missing: array(oneOf("agent-names", "task-discovery", "chat-discovery", "task-read", "comments", "interactions", "progress", "thread-projection", "capture")), + threads: array(closedShape({ issueId: string, identifier: nullable(string), title: string, + status: oneOf("complete", "partial"), messages: array(closedShape({ id: string, at: nullable(date), + role: oneOf("user", "agent", "system"), author: string, + kind: oneOf("description", "comment", "interaction", "answer", "progress"), body: string, + })), + })), + })), firstTask: optional(shape({ caseId: string, nonce: string, onboardingIssueId: string, agentId: string, source: optional(shape({ sha: string, ref: string, dirty: boolean })), diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index e5c672538c1..97a429fecb1 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,4 +1,5 @@ import { createWorkerTiming } from "./worker-timing.js"; +import { captureTaskChat, type TaskChatEvidence } from "./task-chat.js"; import { FEEDBACK_PERSISTENCE_SUITE, gradeFeedbackPersistence } from "./feedback-persistence.js"; import { EvalAttemptTrace, evalBraintrustConfig, EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js"; import { captureFollowupStopFailureApiState } from "./followup-stop-evidence.js"; @@ -659,6 +660,7 @@ for (const execution of executions) { let fixtures: LiveFixtureValues | undefined; let reviewProvider: Awaited> | undefined; let issue: IssueRecord | undefined; + let taskChat: TaskChatEvidence | undefined; let workflowCase: { id: string; identifier?: string } | undefined; let selectedRuns: RunRecord[] = []; let feedback = emptyFeedbackSummary(); @@ -3198,6 +3200,15 @@ for (const execution of executions) { }, assertions: beforeEnvironmentTeardownAssertions, teardown: async () => { + try { + taskChat = await captureTaskChat({ get: route => api.get(route, { timeout: 5_000 }), companyId: fixtures!.company.id, + issueIds: [...new Set([issue?.id, ...selectedRuns.flatMap(run => [run.nativeIssueId, + run.contextSnapshot?.issueId as string | undefined])].filter((id): id is string => Boolean(id)))], secrets }); + } catch { + // Supplemental evidence must never prevent owned fixture cleanup. + taskChat = { schema: "paperclip.runner-e2e.task-chat/v1", capturedAt: new Date().toISOString(), + status: "unavailable", threads: [], missing: ["capture"] }; + } await workerTiming.enter("feedback"); trace?.phase("feedback"); try { @@ -3339,6 +3350,7 @@ for (const execution of executions) { provider: execution.profile.provider, model: firstTaskEvidence ? firstTaskEvidence.observedModels[0] ?? firstTaskEvidence.configuredModel ?? "provider-default (unreported)" : execution.profile.model, ...(firstTaskEvidence ? { firstTask: firstTaskEvidence } : {}), + ...(taskChat ? { taskChat } : {}), ...(completionQuality.length ? { completionQuality } : {}), ...(publicMcpUsage ? { publicMcp: publicMcpUsage } : {}), runtimeMode: execution.profile.expectedRuntimeMode, diff --git a/tests/runner-e2e/task-chat-progress.ts b/tests/runner-e2e/task-chat-progress.ts new file mode 100644 index 00000000000..bff128de4ea --- /dev/null +++ b/tests/runner-e2e/task-chat-progress.ts @@ -0,0 +1,47 @@ +import type { TaskChatMessage } from "./task-chat.js"; + +type Row = Record; +const text = (value: unknown) => typeof value === "string" ? value : ""; + +/** Read only provider-neutral, explicitly public assistant progress. Completion + * comments own final replies. Reasoning, tool payloads and runtime identities + * never enter the conversation projection. */ +export function taskChatProgress(runId: string, events: Row[], author: string): TaskChatMessage[] { + const items = new Map(); + const seen = new Set(); + const ordered = events.flatMap(event => { + const envelope = event.payload?.prpEvent; + const tails = envelope?.payload?.outputTails; + if (!Array.isArray(tails) || !/^(item|turn|session)\.(completed|failed|cancelled|interrupted|closed)$/.test(event.eventType)) return [event]; + // The runtime may hold a possible credential prefix until completion. + // Preserve its redacted display tail, just as the task page does. + return [...tails.flatMap((tail, index) => typeof tail?.payload?.text !== "string" ? [] : [{ + ...event, eventType: "item.delta", payload: { prpEvent: { ...envelope, + eventType: "item.delta", itemId: tail.itemId, payload: tail.payload, + sourceEventId: `${envelope.sourceEventId ?? event.seq}:tail:${index}`, + } }, + }]), event]; + }); + for (const event of ordered) { + const envelope = event.payload?.prpEvent; + if (envelope?.schema !== "paperclip.prp.event.v1" || envelope.schemaVersion !== 1 || + envelope.runId !== runId || envelope.eventType !== event.eventType || + !["item.started", "item.delta", "item.completed"].includes(event.eventType)) continue; + if (envelope.sourceEventId && seen.has(envelope.sourceEventId)) continue; + if (envelope.sourceEventId) seen.add(envelope.sourceEventId); + const p = envelope.payload ?? {}, item = p.item ?? p; + const id = text(envelope.itemId) || text(p.itemId) || text(item.id); + if (!id) continue; + const previous = items.get(id); + const body = text(p.text) || text(item.text); + items.set(id, { + kind: (text(p.kind) || text(item.kind) || text(item.type) || previous?.kind || "").replaceAll("_", "").toLowerCase(), + channel: text(p.channel) || text(item.channel) || previous?.channel || "", + body: event.eventType === "item.completed" && body ? body : (previous?.body ?? "") + body, + at: previous?.at ?? (typeof event.createdAt === "string" && Number.isFinite(Date.parse(event.createdAt)) ? event.createdAt : null), + }); + } + return [...items].filter(([, item]) => ["agentmessage", "assistantmessage"].includes(item.kind) && item.channel === "progress" && item.body) + .map(([, item], index) => ({ id: `progress:${runId}:${index + 1}`, at: item.at, role: "agent", author, + kind: "progress", body: item.body })); +} diff --git a/tests/runner-e2e/task-chat-report.ts b/tests/runner-e2e/task-chat-report.ts new file mode 100644 index 00000000000..a1601b61436 --- /dev/null +++ b/tests/runner-e2e/task-chat-report.ts @@ -0,0 +1,23 @@ +import type { TaskChatEvidence } from "./task-chat.js"; + +const html = (value: unknown) => String(value ?? "").replace(/[&<>"']/g, + c => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]!); + +export function renderTaskChat(evidence?: TaskChatEvidence) { + if (!evidence || (!evidence.threads.length && evidence.status === "complete")) return ""; + return `
+

Task chat

+ ${evidence.status !== "complete" ? `

Conversation capture ${html(evidence.status)}. Some task history could not be collected (${evidence.missing.map(html).join(", ")}).

` : ""} + ${evidence.threads.map(thread => `
+ ${html(thread.identifier ? `${thread.identifier}: ` : "")}${html(thread.title)} · ${thread.messages.length} entries${thread.status === "partial" ? " · partial capture" : ""} +
+

Recorded task brief, messages, requests and answers, with native agent progress when available. This is a read-only recording of the saved task conversation.

+ ${thread.messages.length ? thread.messages.map(message => `
+
${html(message.author)}${message.kind === "answer" ? "Response" : ""}
+
${html(message.body)}
+ ${message.at ? `
` : ""} +
`).join("") : '

No messages were saved on this task.

'} +
+
`).join("")} +
`; +} diff --git a/tests/runner-e2e/task-chat.test.ts b/tests/runner-e2e/task-chat.test.ts new file mode 100644 index 00000000000..ae35c286397 --- /dev/null +++ b/tests/runner-e2e/task-chat.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { captureTaskChat, projectTaskChatThread, type TaskChatEvidence } from "./task-chat.js"; +import { renderTaskChat } from "./task-chat-report.js"; +import { taskChatProgress } from "./task-chat-progress.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import type { RunnerE2EResult } from "./types.js"; + +const time = "2026-10-28T12:00:00Z"; +const issue = { id: "task", companyId: "company", identifier: "E2E-1", title: "Write the release note", description: "Please write a release note.", createdAt: time }; +const question = { id: "question", createdAt: "2026-10-28T12:00:02Z", resolvedAt: "2026-10-28T12:00:03Z", kind: "ask_user_questions", status: "answered", + payload: { questionSet: { questions: [{ id: "tone", prompt: "Which tone?", options: [{ id: "warm", label: "Warm" }, { id: "formal", label: "Formal" }] }] }, + toolAction: { argumentsJson: "PRIVATE_TOOL_ARGUMENTS" } }, + result: { answers: { tone: { selectedOptionIds: ["warm"], text: "Include the launch date" } }, providerSessionId: "PRIVATE_SESSION" } }; +const comment = (i: number) => ({ id: `comment-${i}`, body: `Message ${i}`, authorAgentId: i % 2 ? "agent" : null, + authorUserId: i % 2 ? null : "PRIVATE_USER", createdAt: new Date(Date.parse(time) + i * 1000).toISOString() }); + +function fixture(overrides: { comments?: ReturnType[]; fail?: string; empty?: boolean; chats?: boolean } = {}) { + const comments = overrides.comments ?? [comment(1), comment(4)]; + const calls: string[] = []; + const get = async (route: string): Promise => { + calls.push(route); + if (overrides.fail && route.includes(overrides.fail)) throw new Error("PRIVATE_API_RESPONSE"); + const url = new URL(route, "http://fixture.test"); + let response: unknown; + if (url.pathname.endsWith("/agents")) response = [{ id: "agent", name: "Morgan", apiKey: "PRIVATE_AGENT_KEY" }]; + else if (url.pathname.endsWith("/experimental")) response = { enableAgentChat: overrides.chats ?? false }; + else if (url.pathname.endsWith("/chats")) response = [{ ...issue, id: "chat", title: "Chat with Morgan", description: null }]; + else if (url.pathname.endsWith("/issues")) response = overrides.empty ? [] : [issue]; + else if (url.pathname.endsWith("/comments")) { + const cursor = comments.findIndex(c => c.id === url.searchParams.get("after")); + response = comments.slice(cursor + 1, cursor + 101); + } else if (url.pathname.endsWith("/interactions")) response = [question]; + else if (url.pathname.endsWith("/runs")) response = []; + else if (url.pathname === "/api/issues/chat") response = { ...issue, id: "chat", title: "Chat with Morgan", description: null }; + else throw new Error(`Unexpected route ${route}`); + return structuredClone(response) as T; + }; + return { get, calls }; +} + +describe("automatic task conversation evidence", () => { + it("reads a long saved conversation beyond the first page and retains every full message", async () => { + const comments = Array.from({ length: 1105 }, (_, i) => comment(i)); + comments[1104]!.body = "Final reply " + "Long text. ".repeat(2000); + const api = fixture({ comments }); + const e = await captureTaskChat({ get: api.get, companyId: "company", secrets: [] }); + expect(e.status).toBe("complete"); + const messages = e.threads[0]!.messages.filter(m => m.kind === "comment"); + expect(messages).toHaveLength(1105); + expect(messages.at(-1)?.body).toBe(comments[1104]!.body); + expect(new Set(messages.map(m => m.id)).size).toBe(1105); + expect(api.calls.filter(route => route.includes("/comments"))).toHaveLength(12); + expect(renderTaskChat(e)).toContain(comments[1104]!.body); + }); + + it("includes the standing chat after its setting is restored, plus the delegated task", async () => { + const api = fixture(); + const e = await captureTaskChat({ get: api.get, companyId: "company", issueIds: ["chat", "chat"], secrets: [] }); + expect(e.threads.map(t => t.issueId)).toEqual(["task", "chat"]); + expect(e.status).toBe("complete"); + expect(api.calls).not.toContain("/api/companies/company/chats"); + expect(e.threads[0]!.messages.map(m => m.kind)).toEqual(["description", "comment", "interaction", "answer", "comment"]); + const body = e.threads[0]!.messages.find(m => m.kind === "answer")!.body; + expect(body).toContain("Warm"); + expect(body).toContain("Include the launch date"); + for (const privateValue of ["PRIVATE_USER", "PRIVATE_AGENT_KEY", "PRIVATE_TOOL_ARGUMENTS", "PRIVATE_SESSION"]) + expect(JSON.stringify(e)).not.toContain(privateValue); + }); + + it("redacts credentials and escapes untrusted chat HTML without cutting off messages", async () => { + const c = comment(1); c.body = 'A secret: fixture-secret-123 '; + const e = await captureTaskChat({ get: fixture({ comments: [c] }).get, companyId: "company", secrets: ["fixture-secret-123"] }); + const page = renderTaskChat(e); + expect(page).not.toContain("fixture-secret-123"); + expect(page).not.toContain(" { + const api = fixture(); + const requests: string[] = []; + const e = await captureTaskChat({ companyId: "company", secrets: [], get: async (route: string): Promise => { + requests.push(route); + if (route.endsWith("/runs")) return [{ runId: "run", runtimeMode: "native", agentId: "agent" }] as T; + if (route.startsWith("/api/heartbeat-runs/run/events?")) { + const after = Number(new URL(route, "http://fixture.test").searchParams.get("afterSeq")); + if (!after) return Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "runtime.status" })) as T; + return [{ seq: 1001, eventType: "item.completed", createdAt: time, payload: { prpEvent: { + schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", itemId: "provider-item", eventType: "item.completed", + payload: { kind: "agent_message", channel: "progress", text: "I found the file." }, + } } }] as T; + } + return api.get(route); + } }); + expect(e.status).toBe("complete"); + expect(e.threads[0]!.messages).toContainEqual(expect.objectContaining({ kind: "progress", body: "I found the file.", author: "Morgan" })); + expect(requests.some(route => route.includes("afterSeq=1000"))).toBe(true); + expect(JSON.stringify(e)).not.toContain("provider-item"); + expect(requests.some(route => route.includes("undefined"))).toBe(false); + }); + + it("retains every item-review verdict and reason with its visible label", () => { + const review = { id: "review", kind: "request_item_verdicts", status: "resolved", createdAt: time, resolvedAt: time, + payload: { prompt: "Review both changes", items: [{ id: "a", label: "Release note" }, { id: "b", label: "Announcement" }] }, + result: { items: [{ id: "a", verdict: "approve", reason: "Ready to publish" }, { id: "b", verdict: "reject", reason: "Fix the date", metadata: "PRIVATE_VERDICT_METADATA" }] } }; + const thread = projectTaskChatThread(issue, [], [review], new Map()); + const answer = thread.messages.find(message => message.kind === "answer")!; + for (const content of ["Release note", "approve", "Ready to publish", "Announcement", "reject", "Fix the date"]) expect(answer.body).toContain(content); + expect(JSON.stringify(thread)).not.toContain("PRIVATE_VERDICT_METADATA"); + }); + + it("keeps company boundaries and does not read a foreign task's messages", async () => { + const api = fixture({ empty: true }); + const calls: string[] = []; + const e = await captureTaskChat({ companyId: "company", issueIds: ["foreign"], secrets: [], get: async (route: string): Promise => { + calls.push(route); + return route === "/api/issues/foreign" ? { ...issue, id: "foreign", companyId: "other", description: "PRIVATE_FOREIGN_TASK" } as T : api.get(route); + } }); + expect(e.status).toBe("unavailable"); + expect(e.missing).toContain("task-read"); + expect(calls.some(route => route.includes("/foreign/comments"))).toBe(false); + expect(JSON.stringify(e)).not.toContain("PRIVATE_FOREIGN_TASK"); + }); + + it("keeps available history and labels failed capture without retaining the API error body", async () => { + const e = await captureTaskChat({ get: fixture({ fail: "/interactions" }).get, companyId: "company", secrets: [] }); + expect(e).toMatchObject({ status: "partial", missing: ["interactions"], threads: [{ status: "partial" }] }); + expect(renderTaskChat(e)).toContain("Message 4"); + expect(renderTaskChat(e)).toContain("partial capture"); + expect(JSON.stringify(e)).not.toContain("PRIVATE_API_RESPONSE"); + }); + + it("rejects repeated pages and retains the already collected prefix as partial", async () => { + const api = fixture({ comments: Array.from({ length: 100 }, (_, i) => comment(i)) }); + const e = await captureTaskChat({ get: route => api.get(route.replace(/&after=[^&]+/, "")), companyId: "company", secrets: [] }); + expect(e.status).toBe("partial"); + expect(e.missing).toContain("comments"); + expect(e.threads[0]!.messages.filter(m => m.kind === "comment")).toHaveLength(100); + }); + + it("has no chat section for a browser-only test with no tasks", async () => { + const e = await captureTaskChat({ get: fixture({ empty: true }).get, companyId: "company", secrets: [] }); + expect(e).toMatchObject({ status: "complete", threads: [] }); + expect(renderTaskChat(e)).toBe(""); + expect(renderTaskChat()).toBe(""); + }); + + it("survives normalized result validation and rejects raw fields inside the public projection", () => { + const taskChat: TaskChatEvidence = { schema: "paperclip.runner-e2e.task-chat/v1", capturedAt: time, status: "complete", missing: [], + threads: [projectTaskChatThread(issue, [comment(1)], [], new Map())] }; + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: "core-compatibility.runner-codex.local.message-marker", + suiteId: "core-compatibility", attempt: 1, status: "failed", profileId: "runner-codex", environmentId: "local", caseId: "message-marker", + runtimeMode: "native", provider: "codex", model: "fixture", startedAt: time, finishedAt: time, durationMs: 0, cleanup: "passed", taskChat }; + expect(() => validateRetainedRunnerResult(JSON.parse(JSON.stringify(result)))).not.toThrow(); + Object.assign(taskChat.threads[0]!.messages[0]!, { rawProviderEnvelope: {} }); + expect(() => validateRetainedRunnerResult(result)).toThrow(); + }); + + it("reassembles native progress, prefers completed text, and withholds reasoning and final duplicates", () => { + const event = (eventType: string, itemId: string, payload: object, seq: number) => ({ eventType, createdAt: time, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", sourceEventId: `event-${seq}`, itemId, eventType, payload } } }); + const events = [event("item.started", "provider-item", { kind: "agent_message", channel: "progress" }, 1), + event("item.delta", "provider-item", { text: "Checking " }, 2), event("item.delta", "provider-item", { text: "files" }, 3), + event("item.completed", "provider-item", { text: "Checking files now." }, 4), + event("item.completed", "reasoning", { kind: "reasoning", channel: "progress", text: "PRIVATE_REASONING" }, 5), + event("item.completed", "final", { kind: "agent_message", channel: "final", text: "Final duplicate" }, 6)]; + const progress = taskChatProgress("run", events, "Morgan"); + expect(progress).toEqual([{ id: "progress:run:1", at: time, role: "agent", author: "Morgan", kind: "progress", body: "Checking files now." }]); + expect(taskChatProgress("other-run", events, "Morgan")).toEqual([]); + expect(taskChatProgress("run", [...events.slice(0, 3), events[2]!], "Morgan")[0]!.body).toBe("Checking files"); + const terminal = event("turn.completed", "", { outputTails: [{ itemId: "provider-item", payload: { text: " now." } }] }, 7); + expect(taskChatProgress("run", [...events.slice(0, 3), terminal], "Morgan")[0]!.body).toBe("Checking files now."); + }); +}); diff --git a/tests/runner-e2e/task-chat.ts b/tests/runner-e2e/task-chat.ts new file mode 100644 index 00000000000..bfe2937d61e --- /dev/null +++ b/tests/runner-e2e/task-chat.ts @@ -0,0 +1,189 @@ +import { sanitizeJson } from "./redaction.js"; +import { questionReportAnswers } from "./interaction-report.js"; +import { collectRunEvents } from "./run-observations.js"; +import { taskChatProgress } from "./task-chat-progress.js"; + +type Row = Record; +interface TaskChatRun { + runId: string; + agentId: string; + runtimeMode?: "legacy" | "native"; +} +export interface TaskChatMessage { + id: string; + at: string | null; + role: "user" | "agent" | "system"; + author: string; + kind: "description" | "comment" | "interaction" | "answer" | "progress"; + body: string; +} +export interface TaskChatThread { + issueId: string; + identifier: string | null; + title: string; + status: "complete" | "partial"; + messages: TaskChatMessage[]; +} +export interface TaskChatEvidence { + schema: "paperclip.runner-e2e.task-chat/v1"; + capturedAt: string; + status: "complete" | "partial" | "unavailable"; + threads: TaskChatThread[]; + /** Fixed diagnostic labels only; API errors may contain private response bodies. */ + missing: string[]; +} + +const rows = (value: unknown): Row[] => { + if (!Array.isArray(value) || value.some(row => !row || typeof row !== "object" || Array.isArray(row))) { + throw new Error("Invalid task chat page"); + } + return value; +}; +const text = (value: unknown): string => typeof value === "string" ? value : ""; +const at = (value: unknown): string | null => typeof value === "string" && Number.isFinite(Date.parse(value)) ? value : null; +const lines = (values: unknown[]) => values.filter(value => typeof value === "string" && value.length).join("\n\n"); + +/** Only visible card copy and recorded choices cross the public boundary. + * Tool arguments, credentials, provider envelopes and arbitrary metadata do not. */ +function cardBody(row: Row): string { + const p = row.payload ?? {}; + const questions = p.questionSet?.questions ?? p.questions ?? []; + return lines([row.title, p.title, p.prompt, p.detailsMarkdown, p.questionSet?.title, p.questionSet?.description, + ...rows(questions).map(q => lines([q.prompt, q.helpText, + ...rows(q.options ?? []).map(option => lines([option.label, option.description])), + ])), + ...rows(p.items ?? p.options ?? []).map(item => lines([item.label, item.title, item.description])), + p.acceptLabel, p.rejectLabel, `Status: ${text(row.status) || "unknown"}`]); +} +function answerBody(row: Row): string { + const questions = rows(row.payload?.questionSet?.questions ?? row.payload?.questions ?? []); + return lines([row.result?.outcome, + ...questionReportAnswers({ ...row, id: text(row.id) }).map(answer => { + const q = questions.find(q => q.id === answer.questionId); + const selected = answer.optionIds ?? answer.selectedOptionIds ?? []; + return lines([q?.prompt, ...(Array.isArray(selected) ? selected.map(id => + rows(q?.options ?? []).find(option => option.id === id)?.label ?? text(id)) : []), + answer.otherText ?? answer.customText ?? answer.text]); + }), row.result?.reason, + ...(Array.isArray(row.result?.selectedOptionIds) ? row.result.selectedOptionIds.map((id: string) => + rows(row.payload?.items ?? row.payload?.options ?? []).find(item => item.id === id)?.label ?? id) : []), + ...rows(row.result?.items ?? []).map(result => lines([ + rows(row.payload?.items ?? []).find(item => item.id === result.id)?.label ?? text(result.id), + result.verdict, result.reason, + ])), + ]); +} + +export function projectTaskChatThread(issue: Row, comments: Row[], interactions: Row[], names: ReadonlyMap): TaskChatThread { + const messages: TaskChatMessage[] = []; + if (text(issue.description)) messages.push({ id: `description:${issue.id}`, at: at(issue.createdAt), + role: "user", author: "Task brief", kind: "description", body: issue.description }); + for (const c of comments) { + const role = c.authorAgentId || c.authorType === "agent" ? "agent" : c.authorType === "system" ? "system" : "user"; + messages.push({ id: `comment:${c.id}`, at: at(c.createdAt), role, + author: role === "agent" ? names.get(c.authorAgentId) ?? "Agent" : role === "user" ? "You" : "System", + kind: "comment", body: text(c.body) }); + } + for (const card of interactions) { + messages.push({ id: `interaction:${card.id}`, at: at(card.createdAt), role: "agent", author: "Request", + kind: "interaction", body: cardBody(card) }); + if (card.result && card.resolvedAt) messages.push({ id: `answer:${card.id}`, at: at(card.resolvedAt), + role: card.resolvedByAgentId ? "agent" : "user", + author: card.resolvedByAgentId ? names.get(card.resolvedByAgentId) ?? "Agent" : "You", + kind: "answer", body: answerBody(card) || text(card.status) }); + } + messages.sort((a, b) => (a.at ?? "").localeCompare(b.at ?? "") || a.id.localeCompare(b.id)); + return { issueId: text(issue.id), identifier: text(issue.identifier) || null, title: text(issue.title), status: "complete", messages }; +} + +/** Called once after run retirement and before the disposable instance is removed. + * Discovery includes delegated tasks; explicit IDs retain standing chats even + * when the journey has restored the disabled Agent Chat setting. */ +export async function captureTaskChat(input: { + get: (route: string) => Promise; + companyId: string; + issueIds?: readonly string[]; + secrets: readonly string[]; +}): Promise { + const deadlineAt = Date.now() + 30_000; + const get = (route: string): Promise => { + if (Date.now() >= deadlineAt) return Promise.reject(new Error("Task chat collection deadline")); + return input.get(route); + }; + const evidence: TaskChatEvidence = { schema: "paperclip.runner-e2e.task-chat/v1", capturedAt: new Date().toISOString(), + status: "complete", threads: [], missing: [] }; + const issues = new Map(); + let names = new Map(); + try { + names = new Map(rows(await get(`/api/companies/${input.companyId}/agents`)).map(agent => [agent.id, text(agent.name) || "Agent"])); + } catch { evidence.missing.push("agent-names"); } + try { + let cursor = ""; + for (let page = 0; ; page++) { + if (page === 100) throw new Error("Task discovery exceeded page bound"); + const batch = rows(await get(`/api/companies/${input.companyId}/issues?includeAgentCreatedTasks=true&includeRoutineExecutions=true&includePluginOperations=true&sortField=id&sortDir=asc&limit=100${cursor ? `&afterId=${encodeURIComponent(cursor)}` : ""}`)); + if (batch.length > 100) throw new Error("Oversized task discovery page"); + for (const issue of batch) { + if (!text(issue.id) || issue.id <= cursor || issues.has(issue.id) || issue.companyId !== input.companyId) throw new Error("Invalid task discovery page"); + issues.set(issue.id, issue); + cursor = issue.id; + } + if (batch.length < 100) break; + } + } catch { evidence.missing.push("task-discovery"); } + // Avoid a disabled-chat 404: the setting is often restored by the journey. + try { + const settings = await get("/api/instance/settings/experimental"); + if (settings.enableAgentChat) for (const issue of rows(await get(`/api/companies/${input.companyId}/chats`))) { + if (issue.companyId !== input.companyId || !text(issue.id)) throw new Error("Foreign chat"); + issues.set(issue.id, issue); + } + } catch { evidence.missing.push("chat-discovery"); } + for (const id of new Set(input.issueIds)) { + if (issues.has(id)) continue; + try { + const issue = await get(`/api/issues/${encodeURIComponent(id)}`); + if (issue.companyId !== input.companyId || issue.id !== id) throw new Error("Foreign task"); + issues.set(id, issue); + } catch { evidence.missing.push("task-read"); } + } + for (const issue of issues.values()) { + const comments: Row[] = []; + let interactions: Row[] = []; + let complete = true; + try { + let cursor = ""; + const seen = new Set(); + for (let page = 0; ; page++) { + if (page === 100) throw new Error("Comment capture exceeded page bound"); + const batch = rows(await get(`/api/issues/${issue.id}/comments?order=asc&limit=100${cursor ? `&after=${encodeURIComponent(cursor)}` : ""}`)); + if (batch.length > 100) throw new Error("Oversized comment page"); + for (const comment of batch) { + if (!text(comment.id) || seen.has(comment.id)) throw new Error("Repeated comment page"); + seen.add(comment.id); comments.push(comment); cursor = comment.id; + } + if (batch.length < 100) break; + } + } catch { complete = false; evidence.missing.push("comments"); } + try { interactions = rows(await get(`/api/issues/${issue.id}/interactions`)); } + catch { complete = false; evidence.missing.push("interactions"); } + try { + const thread = projectTaskChatThread(issue, comments, interactions, names); + try { + const runs = await get(`/api/issues/${issue.id}/runs`); + rows(runs); + for (const run of runs.filter(run => run.runtimeMode === "native")) { + if (!text(run.runId)) throw new Error("Native task run has no runId"); + const events = await collectRunEvents((afterSeq, limit) => get(`/api/heartbeat-runs/${encodeURIComponent(run.runId)}/events?afterSeq=${afterSeq}&limit=${limit}`)); + thread.messages.push(...taskChatProgress(run.runId, events, names.get(run.agentId) ?? "Agent")); + } + thread.messages.sort((a, b) => (a.at ?? "").localeCompare(b.at ?? "") || a.id.localeCompare(b.id)); + } catch { complete = false; evidence.missing.push("progress"); } + thread.status = complete ? "complete" : "partial"; + evidence.threads.push(thread); + } catch { evidence.missing.push("thread-projection"); } + } + evidence.missing = [...new Set(evidence.missing)]; + if (evidence.missing.length) evidence.status = evidence.threads.length ? "partial" : "unavailable"; + return sanitizeJson(evidence, input.secrets) as TaskChatEvidence; +} diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 0feaf41a0c3..39b14e6b53e 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -365,6 +365,7 @@ export interface RunnerE2EResult { }>; productJourney?: import("./product-journeys/types.js").JourneyEvidence; firstTask?: import("./first-task-scoring.js").FirstTaskEvidence; + taskChat?: import("./task-chat.js").TaskChatEvidence; providerConnection?: import("./connection-evidence.js").ConnectionEvidence; githubReviewJourney?: import("./github-review-evidence.js").GitHubReviewJourneyEvidence; completionQuality?: import("./completion-quality.js").CompletionQualityRecord[];