fix(evals): retire failed chat work before bounded diagnostics

tests/runner-e2e/chat-catalog-failure-retirement.test.ts

@@ -0,0 +1,95 @@
+import { readFile } from "node:fs/promises";
+import { stripTypeScriptTypes } from "node:module";
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+const control = vi.hoisted(() => ({
+  collectorEntered: Promise.resolve(), collectorRelease: Promise.resolve(), enter: () => {}, release: () => {}, completeQuery: false, stall: true, savedState: { label: "before cleanup" },
+}));
+beforeEach(() => {
+  control.completeQuery = false; control.stall = true; control.savedState = { label: "before cleanup" };
+  control.collectorEntered = new Promise<void>(resolve => { control.enter = resolve; });
+  control.collectorRelease = new Promise<void>(resolve => { control.release = resolve; });
+});
+afterEach(() => vi.useRealTimers());
+vi.mock("./sdk-canary-prefixture.js", () => ({ assertSdkCanaryFingerprintMatches: () => {} }));
+vi.mock("./chat-catalog-registry.js", () => ({
+  findChatCatalogCase: () => ({}),
+  loadChatCatalogImplementation: async () => ({ scenarioId: "questions-and-approvals.multi-part-question", attemptTimeoutMs: 300000,
+    expectedRunCount: 2, turnTimeoutMs: 300000, setupPolicy: { readiness: "preserve-scenario", funding: "preserve-scenario" }, setup: async () => control.savedState, cleanup: async () => { control.savedState.label = "after cleanup"; }, observe: async () => ({ state: control.savedState }), grade: () => ({ passed: false, checks: [{ id: "not-completed", passed: false, detail: "Pending native continuation" }] }) }),
+}));
+vi.mock("./chat-catalog-fingerprint.js", () => ({ chatCatalogCaseFingerprint: () => ({}), normalizedChatCatalogExecution: () => ({}) }));
+vi.mock("./chat-query-bindings.js", () => ({ resolveChatCatalogQuery: async () => { if (control.completeQuery) return { renderedQuery: "Original request" }; throw new Error("Original attempt failed"); } }));
+vi.mock("./chat-rendered-links.js", () => ({ collectConversationLinks: async () => [] }));
+vi.mock("./chat-catalog-run-policy.js", () => ({ gradeChatCatalogRuns: () => ({ checks: [] }) }));
+vi.mock("./chat-flow.js", () => ({ sendChatMessage: async () => {}, collectChatRunEvidence: async () => { control.enter(); if (control.stall) await control.collectorRelease; return {}; } }));
+vi.mock("./api.js", () => ({ pollUntil: async ({ load }: { load: () => Promise<unknown> }) => { await load(); throw new Error("Original attempt failed"); } }));
+import { runChatCatalogFlow } from "./chat-catalog-flow.js";
+it("does not let a stalled failure collector prevent outer retirement of an owned pending native", async () => {
+  let retired = false;
+  const run = { id: "pending-native", companyId: "company", agentId: "ceo", status: "running", runtimeMode: "native" };
+  const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original question-form request" },
+    input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } },
+      api: { baseURL: "http://127.0.0.1", get: async () => [] }, page: { locator: () => ({ count: async () => 0 }) }, evidence: async () => {} },
+    refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => [run] };
+  Object.assign(context.input, { retireFailedAttempt: async () => { retired = true; run.status = "cancelled"; } });
+  const attempt = runChatCatalogFlow(context as never).finally(async () => { retired = true; run.status = "cancelled"; });
+  const rejected = expect(attempt).rejects.toThrow("Original attempt failed");
+  await control.collectorEntered;
+  const retiredBeforeCollectorReleased = retired;
+  control.release();
+  await rejected;
+  expect(retired).toBe(true);
+  expect(retiredBeforeCollectorReleased).toBe(true);
+});
+
+it("bounds failed collection and cannot write evidence after the collector later resolves", async () => {
+  vi.useFakeTimers();
+  const writes: string[] = [], order: string[] = [];
+  const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" },
+    input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company" }, agent: { id: "ceo" } },
+      api: { baseURL: "http://127.0.0.1", get: async () => [] }, page: { locator: () => ({ count: async () => 0 }) }, evidence: async (name: string) => { writes.push(name); },
+      retireFailedAttempt: async () => { order.push("retired"); } },
+    refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => [{ id: "pending", companyId: "company", status: "cancelled" }] };
+  const attempt = runChatCatalogFlow(context as never);
+  const rejected = expect(attempt).rejects.toThrow("Original attempt failed");
+  await control.collectorEntered;
+  expect(order).toEqual(["retired"]);
+  await vi.advanceTimersByTimeAsync(5000);
+  await rejected;
+  expect(writes).toContain("chat-catalog-evidence-error.json");
+  const finishedWrites = [...writes];
+  control.release();
+  await vi.advanceTimersByTimeAsync(0);
+  expect(writes).toEqual(finishedWrites);
+});
+it("preserves the primary failure when retirement and failure evidence persistence fail", async () => {
+  const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" },
+    input: { execution: { task: { turnTimeoutMs: 0 } }, evidence: async (name: string) => {
+      if (name === "chat-catalog-evidence-error.json") throw new Error("Evidence storage failed");
+    }, retireFailedAttempt: async () => { throw new Error("Retirement failed"); } } };
+  await expect(runChatCatalogFlow(context as never)).rejects.toThrow("Original attempt failed");
+});
+
+it("retains completed observation and conversation before scenario cleanup mutates state", async () => {
+  control.completeQuery = true; control.stall = false;
+  const snapshots: Record<string, any> = {};
+  const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" },
+    input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } },
+      api: { baseURL: "http://127.0.0.1", get: async (route: string) => route.endsWith("/budgets/overview") ? { policies: [] } : route.startsWith("/api/agents/") ? { adapterConfig: { timeoutSec: 0 } } : route.endsWith("/interactions") ? [{ id: "question" }] : [] },
+      page: { locator: () => ({ count: async () => 0 }) }, evidence: async (name: string, value: unknown) => { snapshots[name] = structuredClone(value); }, retireFailedAttempt: async () => {} },
+    refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => [] };
+  await expect(runChatCatalogFlow(context as never)).rejects.toThrow("Original attempt failed");
+  expect(snapshots["api-state.json"]).toMatchObject({ issue: { id: "source" }, comments: [], interactions: [{ id: "question" }], observation: { state: { label: "before cleanup" } } });
+  expect(control.savedState.label).toBe("after cleanup");
+});
+it("executes the real outer failure-capture guard without entering a held duplicate API read", async () => {
+  const source = await readFile(new URL("./runner.spec.ts", import.meta.url), "utf8");
+  const start = source.indexOf("    const captureFailureApiState = async () => {"), end = source.indexOf("\n\n    page.on", start);
+  const body = stripTypeScriptTypes(source.slice(start, end), { mode: "strip" });
+  let reads = 0;
+  const capture = new Function("catalogFailureCaptureAttempted", "fixtures", "api", body + "; return captureFailureApiState;")(true, { company: { id: "company" } }, { get: async () => { reads++; return new Promise(() => {}); } });
+  const primary = new Error("Original attempt failed");
+  const caller = (async () => { try { throw primary; } catch (error) { await capture(); throw error; } })();
+  await expect(caller).rejects.toBe(primary);
+  expect(reads).toBe(0);
+  expect(source).toContain("if (!(error instanceof SetupHealthAbort)) await captureFailureApiState();");
+});

tests/runner-e2e/chat-catalog-flow.ts

@@ -35,11 +35,12 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
   let observation: unknown;
   let grade: ChatCatalogGrade | undefined;
   let failed = false;
+  let failure: unknown;
   let renderedQuery = scenario.query;
   let persistedQuery = scenario.query;
   let queryAttachments: ChatQueryAttachmentReceipt[] = [];
   let setupBoundary: ChatCatalogSetupBoundary = { humanCommentIds: [], runIds: [] };
-  async function conversation() {
+  async function conversation(persist = true) {
     await context.refreshIssue();
     const issue = context.issue();
     const runs = await context.allRuns();
@@ -56,7 +57,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
       sourceId: issue.id, leadId: input.fixtures.agent.id, comments, runs, projects, activity,
     }) : [];
     const evidence = { issue, runs, comments, renderedLinks };
-    await input.evidence("chat-catalog-conversation.json", evidence);
+    if (persist) await input.evidence("chat-catalog-conversation.json", evidence);
     return evidence;
   }
   async function refresh() {
@@ -220,26 +221,65 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) {
       leadId: input.fixtures.agent.id, observation });
   } catch (error) {
     failed = true;
+    failure = error;
     throw error;
   } finally {
-    try {
-      const finalConversation = await conversation();
-      if (state) await refresh();
-      const runs = await context.allRuns();
-      const runEvidence = await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run)));
-      await input.evidence("chat-catalog-runs.json", runEvidence);
-      await input.evidence("api-state.json", { capturePhase: failed ? "failure" : "final",
-        ...finalConversation, runs, runEvidence });
-    } catch {
-      await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id });
-      if (!failed) throw new Error("Scenario evidence collection failed");
-    } finally {
+    if (failed) {
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const failureSnapshot: Record<string, unknown> = { capturePhase: "failure" };
       try {
-        if (state) await implementation.cleanup?.(context, state);
-        await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" });
-      } catch (error) {
-        await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" });
-        if (!failed) throw error;
+        failureSnapshot.observation = observation === undefined ? null : structuredClone(observation);
+        await input.retireFailedAttempt?.(failure);
+        const collection = (async () => {
+          const finalConversation = await conversation(false);
+          Object.assign(failureSnapshot, finalConversation);
+          const interactions = finalConversation.issue ? await input.api.get(`/api/issues/${finalConversation.issue.id}/interactions`) : [];
+          failureSnapshot.interactions = interactions;
+          const runEvidence = await Promise.all(finalConversation.runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run)));
+          return { ...failureSnapshot, runEvidence };
+        })();
+        const evidence = await Promise.race([collection, new Promise<undefined>(resolve => {
+          timer = setTimeout(() => resolve(undefined), 5_000);
+        })]);
+        if (evidence) {
+          await input.evidence("chat-catalog-runs.json", evidence.runEvidence);
+          await input.evidence("api-state.json", evidence);
+        } else {
+          await input.evidence("api-state.json", structuredClone({ ...failureSnapshot, incomplete: true }));
+          await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id, reason: "failure_collection_deadline" });
+        }
+      } catch {
+        await input.evidence("api-state.json", structuredClone({ ...failureSnapshot, incomplete: true })).catch(() => {});
+        await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id }).catch(() => {});
+      } finally {
+        if (timer) clearTimeout(timer);
+        try {
+          if (state) await implementation.cleanup?.(context, state);
+          await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" });
+        } catch {
+          await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" }).catch(() => {});
+        }
+      }
+    } else {
+      try {
+        const finalConversation = await conversation();
+        if (state) await refresh();
+        const runs = await context.allRuns();
+        const runEvidence = await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run)));
+        await input.evidence("chat-catalog-runs.json", runEvidence);
+        await input.evidence("api-state.json", { capturePhase: failed ? "failure" : "final",
+          ...finalConversation, runs, runEvidence });
+      } catch {
+        await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id });
+        if (!failed) throw new Error("Scenario evidence collection failed");
+      } finally {
+        try {
+          if (state) await implementation.cleanup?.(context, state);
+          await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" });
+        } catch (error) {
+          await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" });
+          if (!failed) throw error;
+        }
       }
     }
   }

tests/runner-e2e/chat-flow.ts

@@ -388,6 +388,7 @@ export interface ChatFlowInput {
   privateInputs?: ChatPrivateInputs;
   deadlineAt: number;
   restart: () => Promise<void>;
+  retireFailedAttempt?: (error: unknown) => Promise<void>;
   observe: (issue: ChatIssue, runs: ChatRun[]) => void;
   capture: (id: string, label: string, file: string) => Promise<void>;
   evidence: (name: string, data: unknown) => Promise<void>;

tests/runner-e2e/cleanup-verification.test.ts

@@ -1,5 +1,5 @@
 import { expect, it } from "vitest";
-import { runCleanupWithObservers, verifyCleanupAssertions } from "./cleanup-verification.js";
+import { ownedActiveCleanupRunIds, runCleanupWithObservers, verifyCleanupAssertions } from "./cleanup-verification.js";
 
 it("retains a failed cleanup proof and still closes every later observer", async () => {
   let closed = 0;
@@ -37,3 +37,13 @@ it("collects remote proof before environment destruction, preserving every clean
   expect(result.errors.map(error => (error as Error).message)).toEqual(["cancel failed", "receipt missing", "delete failed"]);
   expect(result.checks).toContainEqual({ id: "sealed", passed: true, detail: "sealed before deletion" });
 });
+
+it("cancels only independently listed active owned-company runs", () => {
+  expect(ownedActiveCleanupRunIds([
+    { id: "owned", companyId: "attempt-company", status: "running" },
+    { id: "foreign", companyId: "other-company", status: "running" },
+    { id: "unscoped", status: "running" },
+    { id: "finished", companyId: "attempt-company", status: "succeeded" },
+    { id: "owned", companyId: "attempt-company", status: "running" },
+  ], "attempt-company", new Set(["succeeded"]))).toEqual(["owned"]);
+});

tests/runner-e2e/cleanup-verification.ts

@@ -60,3 +60,7 @@ export async function runCleanupWithObservers(input: {
   try { await input.teardown(); } catch (error) { errors.push(error); }
   return { checks: verification.checks, errors };
 }
+
+export function ownedActiveCleanupRunIds(runs: readonly { id: string; companyId?: string; status: string }[], companyId: string, terminal: ReadonlySet<string>) {
+  return [...new Set(runs.filter(run => run.companyId === companyId && !terminal.has(run.status)).map(run => run.id).filter(Boolean))];
+}

tests/runner-e2e/private-framework-report.test.ts

@@ -202,3 +202,35 @@ it("rejects unrecognized command credentials in a registered plain-text file", a
   const owner = await privateCellEvidenceOwner({ root: f.root, private: true, runId: "123", attempt: "1", executionIds: [id], cellReportProducer: await inspectCellReportProducer() }, id);
   await expect(packageEvidence({ privateDir: f.cell, uploadDir: path.join(f.root, "upload"), secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } })).rejects.toThrow("requires redaction");
 });
+
+it.skipIf(!process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE)("seals an actual failed browser screenshot, trace and blob report without losing declared attachments", async () => {
+  const f = await fixture(), tests = path.join(f.root, "browser-tests"); await mkdir(tests);
+  const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test");
+  await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('real failed browser result',async({page},info)=>{await page.setContent('<h1>Dummy retained page</h1>');await info.attach('dummy-content',{body:Buffer.from('nonprivate dummy attachment'),contentType:'text/plain'});await test.step('actual failed browser assertion',async()=>expect(await page.locator('h1').innerText()).toBe('different heading'));});`);
+  const config = path.join(tests, "config.cjs");
+  await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, outputDir: path.join(f.source, "playwright-output"), use: { launchOptions: { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE }, screenshot: "only-on-failure", trace: "retain-on-failure" }, reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`);
+  await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" }, timeout: 30000 }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); });
+  await rm(path.join(f.cell, "playwright-output"), { recursive: true });
+  for (const folder of ["playwright-output", "html-report", "blob-report"]) await cp(path.join(f.source, folder), path.join(f.cell, folder), { recursive: true });
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  const retained = await validatePrivateFrameworkReport(f.cell);
+  expect((retained.report.suites as any)[0].specs[0].tests[0].results[0].status).toBe("failed");
+  expect(retained.attachments.some(a => a.path.endsWith(".png"))).toBe(true);
+  expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true);
+  const { packageEvidence } = await import("./evidence.js");
+  const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js");
+  const { sealStandardOriginalEvidence, sealManualResultEvidence } = await import("./manual-private-handoff.js");
+  const { validateSealedOriginals, validateSealedInventory } = await import("./manual-private-transfer.js");
+  const { MANUAL_PRIVATE_PROTOCOL, manualCellKind } = await import("./manual-private-retention.js");
+  const id = "chat-features.runner-opencode.local.questions-and-approvals.multi-part-question";
+  const context: ManualPrivateHandoff = { root: await realpath(f.root), private: true, protocol: MANUAL_PRIVATE_PROTOCOL, runId: "123", attempt: "1", workflowSha: "a".repeat(40), targetSha: "b".repeat(40), executionIds: [id], cellReportProducer: await inspectCellReportProducer() };
+  const owner = await privateCellEvidenceOwner(context, id), upload = path.join(context.root, "results", `gha-123-1-${id}/chat-features/runner-opencode/local/questions-and-approvals.multi-part-question/attempt-1`);
+  await writeFile(path.join(f.cell, "result.json"), '{"status":"failed","error":"actual failed browser assertion"}');
+  const evidence = await packageEvidence({ privateDir: f.cell, uploadDir: upload, secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } });
+  expect(evidence.leaks).toEqual([]);
+  await sealStandardOriginalEvidence({ context, executionId: id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false });
+  await referenceSealedFrameworkAttachments(upload, path.join(context.root, "originals", manualCellKind(id)), id);
+  await sealManualResultEvidence(context, []);
+  expect((await validateSealedOriginals(context.root, id, context)).complete).toBe(true);
+  expect((await validateSealedInventory(path.join(context.root, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", context)).complete).toBe(true);
+});

tests/runner-e2e/runner.spec.ts

@@ -48,7 +48,7 @@ import { summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-
 import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js";
 import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js";
 import { runNativeProviderLossFlow } from "./native-provider-loss-flow.js";
-import { mayAllocateRemoteResources, runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js";
+import { ownedActiveCleanupRunIds, mayAllocateRemoteResources, runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js";
 import { completionDelivery, type CompletionObservation } from "./completion-updates.js";
 import { runInstructionPersistenceFlow } from "./instruction-persistence.js";
 import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js";
@@ -773,13 +773,17 @@ for (const execution of executions) {
         setupProbeLeaseId: fixtures.setupProbe?.leaseId, setupLeaseIds: fixtures.setupLeases?.map(lease => lease.leaseId) });
     };
 
-    const cancelActiveRunsForCleanup = async () => {
+    const cancelActiveRunsForCleanup = async (budgetMs?: number) => {
       if (!issue && !(companyRunFlow && fixtures)) return;
       const cleanupIssueId = issue?.id;
+      const cleanupDeadline = budgetMs === undefined ? undefined : Date.now() + budgetMs;
+      const cleanupRequestOptions = () => cleanupDeadline === undefined ? undefined : { timeout: Math.max(1, cleanupDeadline - Date.now()) };
       const runs = await api.get<RunRecord[]>(
-        companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`,
+        companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`, cleanupRequestOptions(),
       );
-      const activeRunIds = [
+      const activeRunIds = companyRunFlow && fixtures
+        ? ownedActiveCleanupRunIds(runs, fixtures.company.id, TERMINAL_RUN_STATUSES)
+        : [
         ...new Set(
           runs
             .filter((run) => !TERMINAL_RUN_STATUSES.has(run.status))
@@ -791,14 +795,14 @@ for (const execution of executions) {
         ),
       ];
       for (const runId of activeRunIds) {
-        await api.post(`/api/heartbeat-runs/${runId}/cancel`);
+        await api.post(`/api/heartbeat-runs/${runId}/cancel`, undefined, cleanupRequestOptions());
       }
       if (activeRunIds.length === 0) return;
       const activeIds = new Set(activeRunIds);
       await pollUntil({
         label: `cleanup cancellation for issue ${cleanupIssueId}`,
-        deadlineAt: Date.now() + 45_000,
-        load: () => api.get<RunRecord[]>(companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`),
+        deadlineAt: cleanupDeadline ?? Date.now() + 45_000,
+        load: () => api.get<RunRecord[]>(companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`, cleanupRequestOptions()),
         accept: (currentRuns) =>
           currentRuns
             .filter((run) => activeIds.has(run.id))
@@ -807,8 +811,9 @@ for (const execution of executions) {
       });
     };
 
+    let catalogFailureCaptureAttempted = false;
     const captureFailureApiState = async () => {
-      if (!fixtures) return;
+      if (catalogFailureCaptureAttempted || !fixtures) return;
       if (execution.task.flow === "followup_stop_resume") {
         const captured = await captureFollowupStopFailureApiState({ api, companyId: fixtures.company.id,
           agentId: fixtures.agent.id, issueId: issue?.id,
@@ -1389,6 +1394,14 @@ for (const execution of executions) {
           page, api, fixtures, execution, nonce, workspacePath,
           deadlineAt: startedAtMs + deadlineMs,
           restart: () => restartChatServer(page, () => restartIsolatedPaperclipServer({ api, requestId: `chat-${nonce}`, deadlineAt: startedAtMs + deadlineMs })),
+          retireFailedAttempt: async error => {
+            catalogFailureCaptureAttempted = true;
+            primaryError ??= error;
+            try { await writeSanitizedJson(snapshotsDir, "failed-attempt-checkpoint.json", {
+              schema: "paperclip.runner-e2e.failed-attempt-checkpoint/v1", executionId: execution.id,
+              qualification: "unqualified", featureOutcome: "unknown", phase: "before_owned_retirement",
+            }, secrets); } finally { await cancelActiveRunsForCleanup(20_000); }
+          },
           observe: (chatIssue, chatRuns) => { issue = chatIssue; selectedRuns = chatRuns; },
           capture: ordinaryChatCapture(execution.suite.id, capturePrivateScreenshot, captureScreenshot),
           evidence: completionEvidence,
@@ -3247,9 +3260,12 @@ for (const execution of executions) {
         if (checks.some(check => !check.passed)) primaryError ??= new Error("Feedback persistence matcher failed");
       }
 
-      const traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id),
-        cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions });
-      await saveTrace();
+      let traceReceipt: Awaited<ReturnType<NonNullable<typeof trace>["export"]>> | undefined;
+      try {
+        traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id),
+          cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions });
+        await saveTrace();
+      } catch (error) { primaryError ??= error; }
       const finishedAtMs = Date.now();
       const runtimeUsage = fixtureRuntimeCostCoverage(buildRuntimeUsage({
         environmentId: execution.environment.id,