From 8c01c75733bfb41840c40023210281ad85bfb990 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 22:56:15 -0600 Subject: [PATCH] fix(evals): retire failed chat work before bounded diagnostics --- .../chat-catalog-failure-retirement.test.ts | 95 +++++++++++++++++++ tests/runner-e2e/chat-catalog-flow.ts | 78 +++++++++++---- tests/runner-e2e/chat-flow.ts | 1 + tests/runner-e2e/cleanup-verification.test.ts | 12 ++- tests/runner-e2e/cleanup-verification.ts | 4 + .../private-framework-report.test.ts | 32 +++++++ tests/runner-e2e/runner.spec.ts | 38 +++++--- 7 files changed, 229 insertions(+), 31 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-failure-retirement.test.ts diff --git a/tests/runner-e2e/chat-catalog-failure-retirement.test.ts b/tests/runner-e2e/chat-catalog-failure-retirement.test.ts new file mode 100644 index 00000000000..968187078f6 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-failure-retirement.test.ts @@ -0,0 +1,95 @@ +import { readFile } from "node:fs/promises"; +import { stripTypeScriptTypes } from "node:module"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +const control = vi.hoisted(() => ({ + collectorEntered: Promise.resolve(), collectorRelease: Promise.resolve(), enter: () => {}, release: () => {}, completeQuery: false, stall: true, savedState: { label: "before cleanup" }, +})); +beforeEach(() => { + control.completeQuery = false; control.stall = true; control.savedState = { label: "before cleanup" }; + control.collectorEntered = new Promise(resolve => { control.enter = resolve; }); + control.collectorRelease = new Promise(resolve => { control.release = resolve; }); +}); +afterEach(() => vi.useRealTimers()); +vi.mock("./sdk-canary-prefixture.js", () => ({ assertSdkCanaryFingerprintMatches: () => {} })); +vi.mock("./chat-catalog-registry.js", () => ({ + findChatCatalogCase: () => ({}), + loadChatCatalogImplementation: async () => ({ scenarioId: "questions-and-approvals.multi-part-question", attemptTimeoutMs: 300000, + expectedRunCount: 2, turnTimeoutMs: 300000, setupPolicy: { readiness: "preserve-scenario", funding: "preserve-scenario" }, setup: async () => control.savedState, cleanup: async () => { control.savedState.label = "after cleanup"; }, observe: async () => ({ state: control.savedState }), grade: () => ({ passed: false, checks: [{ id: "not-completed", passed: false, detail: "Pending native continuation" }] }) }), +})); +vi.mock("./chat-catalog-fingerprint.js", () => ({ chatCatalogCaseFingerprint: () => ({}), normalizedChatCatalogExecution: () => ({}) })); +vi.mock("./chat-query-bindings.js", () => ({ resolveChatCatalogQuery: async () => { if (control.completeQuery) return { renderedQuery: "Original request" }; throw new Error("Original attempt failed"); } })); +vi.mock("./chat-rendered-links.js", () => ({ collectConversationLinks: async () => [] })); +vi.mock("./chat-catalog-run-policy.js", () => ({ gradeChatCatalogRuns: () => ({ checks: [] }) })); +vi.mock("./chat-flow.js", () => ({ sendChatMessage: async () => {}, collectChatRunEvidence: async () => { control.enter(); if (control.stall) await control.collectorRelease; return {}; } })); +vi.mock("./api.js", () => ({ pollUntil: async ({ load }: { load: () => Promise }) => { await load(); throw new Error("Original attempt failed"); } })); +import { runChatCatalogFlow } from "./chat-catalog-flow.js"; +it("does not let a stalled failure collector prevent outer retirement of an owned pending native", async () => { + let retired = false; + const run = { id: "pending-native", companyId: "company", agentId: "ceo", status: "running", runtimeMode: "native" }; + const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original question-form request" }, + input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } }, + api: { baseURL: "http://127.0.0.1", get: async () => [] }, page: { locator: () => ({ count: async () => 0 }) }, evidence: async () => {} }, + refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => [run] }; + Object.assign(context.input, { retireFailedAttempt: async () => { retired = true; run.status = "cancelled"; } }); + const attempt = runChatCatalogFlow(context as never).finally(async () => { retired = true; run.status = "cancelled"; }); + const rejected = expect(attempt).rejects.toThrow("Original attempt failed"); + await control.collectorEntered; + const retiredBeforeCollectorReleased = retired; + control.release(); + await rejected; + expect(retired).toBe(true); + expect(retiredBeforeCollectorReleased).toBe(true); +}); + +it("bounds failed collection and cannot write evidence after the collector later resolves", async () => { + vi.useFakeTimers(); + const writes: string[] = [], order: string[] = []; + const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" }, + input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company" }, agent: { id: "ceo" } }, + api: { baseURL: "http://127.0.0.1", get: async () => [] }, page: { locator: () => ({ count: async () => 0 }) }, evidence: async (name: string) => { writes.push(name); }, + retireFailedAttempt: async () => { order.push("retired"); } }, + refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => [{ id: "pending", companyId: "company", status: "cancelled" }] }; + const attempt = runChatCatalogFlow(context as never); + const rejected = expect(attempt).rejects.toThrow("Original attempt failed"); + await control.collectorEntered; + expect(order).toEqual(["retired"]); + await vi.advanceTimersByTimeAsync(5000); + await rejected; + expect(writes).toContain("chat-catalog-evidence-error.json"); + const finishedWrites = [...writes]; + control.release(); + await vi.advanceTimersByTimeAsync(0); + expect(writes).toEqual(finishedWrites); +}); +it("preserves the primary failure when retirement and failure evidence persistence fail", async () => { + const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" }, + input: { execution: { task: { turnTimeoutMs: 0 } }, evidence: async (name: string) => { + if (name === "chat-catalog-evidence-error.json") throw new Error("Evidence storage failed"); + }, retireFailedAttempt: async () => { throw new Error("Retirement failed"); } } }; + await expect(runChatCatalogFlow(context as never)).rejects.toThrow("Original attempt failed"); +}); + +it("retains completed observation and conversation before scenario cleanup mutates state", async () => { + control.completeQuery = true; control.stall = false; + const snapshots: Record = {}; + const context = { scenario: { id: "questions-and-approvals.multi-part-question", query: "Original request" }, + input: { execution: { task: { turnTimeoutMs: 0 } }, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } }, + api: { baseURL: "http://127.0.0.1", get: async (route: string) => route.endsWith("/budgets/overview") ? { policies: [] } : route.startsWith("/api/agents/") ? { adapterConfig: { timeoutSec: 0 } } : route.endsWith("/interactions") ? [{ id: "question" }] : [] }, + page: { locator: () => ({ count: async () => 0 }) }, evidence: async (name: string, value: unknown) => { snapshots[name] = structuredClone(value); }, retireFailedAttempt: async () => {} }, + refreshIssue: async () => {}, issue: () => ({ id: "source", companyId: "company" }), allRuns: async () => [] }; + await expect(runChatCatalogFlow(context as never)).rejects.toThrow("Original attempt failed"); + expect(snapshots["api-state.json"]).toMatchObject({ issue: { id: "source" }, comments: [], interactions: [{ id: "question" }], observation: { state: { label: "before cleanup" } } }); + expect(control.savedState.label).toBe("after cleanup"); +}); +it("executes the real outer failure-capture guard without entering a held duplicate API read", async () => { + const source = await readFile(new URL("./runner.spec.ts", import.meta.url), "utf8"); + const start = source.indexOf(" const captureFailureApiState = async () => {"), end = source.indexOf("\n\n page.on", start); + const body = stripTypeScriptTypes(source.slice(start, end), { mode: "strip" }); + let reads = 0; + const capture = new Function("catalogFailureCaptureAttempted", "fixtures", "api", body + "; return captureFailureApiState;")(true, { company: { id: "company" } }, { get: async () => { reads++; return new Promise(() => {}); } }); + const primary = new Error("Original attempt failed"); + const caller = (async () => { try { throw primary; } catch (error) { await capture(); throw error; } })(); + await expect(caller).rejects.toBe(primary); + expect(reads).toBe(0); + expect(source).toContain("if (!(error instanceof SetupHealthAbort)) await captureFailureApiState();"); +}); diff --git a/tests/runner-e2e/chat-catalog-flow.ts b/tests/runner-e2e/chat-catalog-flow.ts index 954f67718cc..e7af4a05a7b 100644 --- a/tests/runner-e2e/chat-catalog-flow.ts +++ b/tests/runner-e2e/chat-catalog-flow.ts @@ -35,11 +35,12 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { let observation: unknown; let grade: ChatCatalogGrade | undefined; let failed = false; + let failure: unknown; let renderedQuery = scenario.query; let persistedQuery = scenario.query; let queryAttachments: ChatQueryAttachmentReceipt[] = []; let setupBoundary: ChatCatalogSetupBoundary = { humanCommentIds: [], runIds: [] }; - async function conversation() { + async function conversation(persist = true) { await context.refreshIssue(); const issue = context.issue(); const runs = await context.allRuns(); @@ -56,7 +57,7 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { sourceId: issue.id, leadId: input.fixtures.agent.id, comments, runs, projects, activity, }) : []; const evidence = { issue, runs, comments, renderedLinks }; - await input.evidence("chat-catalog-conversation.json", evidence); + if (persist) await input.evidence("chat-catalog-conversation.json", evidence); return evidence; } async function refresh() { @@ -220,26 +221,65 @@ export async function runChatCatalogFlow(context: ChatCatalogContext) { leadId: input.fixtures.agent.id, observation }); } catch (error) { failed = true; + failure = error; throw error; } finally { - try { - const finalConversation = await conversation(); - if (state) await refresh(); - const runs = await context.allRuns(); - const runEvidence = await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run))); - await input.evidence("chat-catalog-runs.json", runEvidence); - await input.evidence("api-state.json", { capturePhase: failed ? "failure" : "final", - ...finalConversation, runs, runEvidence }); - } catch { - await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id }); - if (!failed) throw new Error("Scenario evidence collection failed"); - } finally { + if (failed) { + let timer: ReturnType | undefined; + const failureSnapshot: Record = { capturePhase: "failure" }; try { - if (state) await implementation.cleanup?.(context, state); - await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" }); - } catch (error) { - await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" }); - if (!failed) throw error; + failureSnapshot.observation = observation === undefined ? null : structuredClone(observation); + await input.retireFailedAttempt?.(failure); + const collection = (async () => { + const finalConversation = await conversation(false); + Object.assign(failureSnapshot, finalConversation); + const interactions = finalConversation.issue ? await input.api.get(`/api/issues/${finalConversation.issue.id}/interactions`) : []; + failureSnapshot.interactions = interactions; + const runEvidence = await Promise.all(finalConversation.runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run))); + return { ...failureSnapshot, runEvidence }; + })(); + const evidence = await Promise.race([collection, new Promise(resolve => { + timer = setTimeout(() => resolve(undefined), 5_000); + })]); + if (evidence) { + await input.evidence("chat-catalog-runs.json", evidence.runEvidence); + await input.evidence("api-state.json", evidence); + } else { + await input.evidence("api-state.json", structuredClone({ ...failureSnapshot, incomplete: true })); + await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id, reason: "failure_collection_deadline" }); + } + } catch { + await input.evidence("api-state.json", structuredClone({ ...failureSnapshot, incomplete: true })).catch(() => {}); + await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id }).catch(() => {}); + } finally { + if (timer) clearTimeout(timer); + try { + if (state) await implementation.cleanup?.(context, state); + await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" }); + } catch { + await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" }).catch(() => {}); + } + } + } else { + try { + const finalConversation = await conversation(); + if (state) await refresh(); + const runs = await context.allRuns(); + const runEvidence = await Promise.all(runs.map(run => collectChatRunEvidence(chatRunEvidenceApi(input), run))); + await input.evidence("chat-catalog-runs.json", runEvidence); + await input.evidence("api-state.json", { capturePhase: failed ? "failure" : "final", + ...finalConversation, runs, runEvidence }); + } catch { + await input.evidence("chat-catalog-evidence-error.json", { incomplete: true, scenarioId: scenario.id }); + if (!failed) throw new Error("Scenario evidence collection failed"); + } finally { + try { + if (state) await implementation.cleanup?.(context, state); + await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "complete", instanceCleanup: "owned by existing launcher" }); + } catch (error) { + await input.evidence("chat-catalog-cleanup.json", { scenarioId: scenario.id, caseCleanup: "failed" }); + if (!failed) throw error; + } } } } diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index cfbb38ef246..e9e8422cb9b 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -388,6 +388,7 @@ export interface ChatFlowInput { privateInputs?: ChatPrivateInputs; deadlineAt: number; restart: () => Promise; + retireFailedAttempt?: (error: unknown) => Promise; observe: (issue: ChatIssue, runs: ChatRun[]) => void; capture: (id: string, label: string, file: string) => Promise; evidence: (name: string, data: unknown) => Promise; diff --git a/tests/runner-e2e/cleanup-verification.test.ts b/tests/runner-e2e/cleanup-verification.test.ts index fdca9d33df0..5cc43fdcce2 100644 --- a/tests/runner-e2e/cleanup-verification.test.ts +++ b/tests/runner-e2e/cleanup-verification.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { runCleanupWithObservers, verifyCleanupAssertions } from "./cleanup-verification.js"; +import { ownedActiveCleanupRunIds, runCleanupWithObservers, verifyCleanupAssertions } from "./cleanup-verification.js"; it("retains a failed cleanup proof and still closes every later observer", async () => { let closed = 0; @@ -37,3 +37,13 @@ it("collects remote proof before environment destruction, preserving every clean expect(result.errors.map(error => (error as Error).message)).toEqual(["cancel failed", "receipt missing", "delete failed"]); expect(result.checks).toContainEqual({ id: "sealed", passed: true, detail: "sealed before deletion" }); }); + +it("cancels only independently listed active owned-company runs", () => { + expect(ownedActiveCleanupRunIds([ + { id: "owned", companyId: "attempt-company", status: "running" }, + { id: "foreign", companyId: "other-company", status: "running" }, + { id: "unscoped", status: "running" }, + { id: "finished", companyId: "attempt-company", status: "succeeded" }, + { id: "owned", companyId: "attempt-company", status: "running" }, + ], "attempt-company", new Set(["succeeded"]))).toEqual(["owned"]); +}); diff --git a/tests/runner-e2e/cleanup-verification.ts b/tests/runner-e2e/cleanup-verification.ts index 2e935b226b0..5f1c1422d09 100644 --- a/tests/runner-e2e/cleanup-verification.ts +++ b/tests/runner-e2e/cleanup-verification.ts @@ -60,3 +60,7 @@ export async function runCleanupWithObservers(input: { try { await input.teardown(); } catch (error) { errors.push(error); } return { checks: verification.checks, errors }; } + +export function ownedActiveCleanupRunIds(runs: readonly { id: string; companyId?: string; status: string }[], companyId: string, terminal: ReadonlySet) { + return [...new Set(runs.filter(run => run.companyId === companyId && !terminal.has(run.status)).map(run => run.id).filter(Boolean))]; +} diff --git a/tests/runner-e2e/private-framework-report.test.ts b/tests/runner-e2e/private-framework-report.test.ts index 95b46f40d04..d220ab1ac6a 100644 --- a/tests/runner-e2e/private-framework-report.test.ts +++ b/tests/runner-e2e/private-framework-report.test.ts @@ -202,3 +202,35 @@ it("rejects unrecognized command credentials in a registered plain-text file", a const owner = await privateCellEvidenceOwner({ root: f.root, private: true, runId: "123", attempt: "1", executionIds: [id], cellReportProducer: await inspectCellReportProducer() }, id); await expect(packageEvidence({ privateDir: f.cell, uploadDir: path.join(f.root, "upload"), secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } })).rejects.toThrow("requires redaction"); }); + +it.skipIf(!process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE)("seals an actual failed browser screenshot, trace and blob report without losing declared attachments", async () => { + const f = await fixture(), tests = path.join(f.root, "browser-tests"); await mkdir(tests); + const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); + await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('real failed browser result',async({page},info)=>{await page.setContent('

Dummy retained page

');await info.attach('dummy-content',{body:Buffer.from('nonprivate dummy attachment'),contentType:'text/plain'});await test.step('actual failed browser assertion',async()=>expect(await page.locator('h1').innerText()).toBe('different heading'));});`); + const config = path.join(tests, "config.cjs"); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, outputDir: path.join(f.source, "playwright-output"), use: { launchOptions: { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE }, screenshot: "only-on-failure", trace: "retain-on-failure" }, reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`); + await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" }, timeout: 30000 }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); }); + await rm(path.join(f.cell, "playwright-output"), { recursive: true }); + for (const folder of ["playwright-output", "html-report", "blob-report"]) await cp(path.join(f.source, folder), path.join(f.cell, folder), { recursive: true }); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const retained = await validatePrivateFrameworkReport(f.cell); + expect((retained.report.suites as any)[0].specs[0].tests[0].results[0].status).toBe("failed"); + expect(retained.attachments.some(a => a.path.endsWith(".png"))).toBe(true); + expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true); + const { packageEvidence } = await import("./evidence.js"); + const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js"); + const { sealStandardOriginalEvidence, sealManualResultEvidence } = await import("./manual-private-handoff.js"); + const { validateSealedOriginals, validateSealedInventory } = await import("./manual-private-transfer.js"); + const { MANUAL_PRIVATE_PROTOCOL, manualCellKind } = await import("./manual-private-retention.js"); + const id = "chat-features.runner-opencode.local.questions-and-approvals.multi-part-question"; + const context: ManualPrivateHandoff = { root: await realpath(f.root), private: true, protocol: MANUAL_PRIVATE_PROTOCOL, runId: "123", attempt: "1", workflowSha: "a".repeat(40), targetSha: "b".repeat(40), executionIds: [id], cellReportProducer: await inspectCellReportProducer() }; + const owner = await privateCellEvidenceOwner(context, id), upload = path.join(context.root, "results", `gha-123-1-${id}/chat-features/runner-opencode/local/questions-and-approvals.multi-part-question/attempt-1`); + await writeFile(path.join(f.cell, "result.json"), '{"status":"failed","error":"actual failed browser assertion"}'); + const evidence = await packageEvidence({ privateDir: f.cell, uploadDir: upload, secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } }); + expect(evidence.leaks).toEqual([]); + await sealStandardOriginalEvidence({ context, executionId: id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false }); + await referenceSealedFrameworkAttachments(upload, path.join(context.root, "originals", manualCellKind(id)), id); + await sealManualResultEvidence(context, []); + expect((await validateSealedOriginals(context.root, id, context)).complete).toBe(true); + expect((await validateSealedInventory(path.join(context.root, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", context)).complete).toBe(true); +}); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 0137923de8d..0e5c4f8f34b 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -48,7 +48,7 @@ import { summarizeStartupReuse } from "../../packages/adapter-utils/src/startup- import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js"; import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js"; import { runNativeProviderLossFlow } from "./native-provider-loss-flow.js"; -import { mayAllocateRemoteResources, runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js"; +import { ownedActiveCleanupRunIds, mayAllocateRemoteResources, runCleanupWithObservers, verifyCleanupAssertions, type CleanupAssertion } from "./cleanup-verification.js"; import { completionDelivery, type CompletionObservation } from "./completion-updates.js"; import { runInstructionPersistenceFlow } from "./instruction-persistence.js"; import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js"; @@ -773,13 +773,17 @@ for (const execution of executions) { setupProbeLeaseId: fixtures.setupProbe?.leaseId, setupLeaseIds: fixtures.setupLeases?.map(lease => lease.leaseId) }); }; - const cancelActiveRunsForCleanup = async () => { + const cancelActiveRunsForCleanup = async (budgetMs?: number) => { if (!issue && !(companyRunFlow && fixtures)) return; const cleanupIssueId = issue?.id; + const cleanupDeadline = budgetMs === undefined ? undefined : Date.now() + budgetMs; + const cleanupRequestOptions = () => cleanupDeadline === undefined ? undefined : { timeout: Math.max(1, cleanupDeadline - Date.now()) }; const runs = await api.get( - companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`, + companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`, cleanupRequestOptions(), ); - const activeRunIds = [ + const activeRunIds = companyRunFlow && fixtures + ? ownedActiveCleanupRunIds(runs, fixtures.company.id, TERMINAL_RUN_STATUSES) + : [ ...new Set( runs .filter((run) => !TERMINAL_RUN_STATUSES.has(run.status)) @@ -791,14 +795,14 @@ for (const execution of executions) { ), ]; for (const runId of activeRunIds) { - await api.post(`/api/heartbeat-runs/${runId}/cancel`); + await api.post(`/api/heartbeat-runs/${runId}/cancel`, undefined, cleanupRequestOptions()); } if (activeRunIds.length === 0) return; const activeIds = new Set(activeRunIds); await pollUntil({ label: `cleanup cancellation for issue ${cleanupIssueId}`, - deadlineAt: Date.now() + 45_000, - load: () => api.get(companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`), + deadlineAt: cleanupDeadline ?? Date.now() + 45_000, + load: () => api.get(companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`, cleanupRequestOptions()), accept: (currentRuns) => currentRuns .filter((run) => activeIds.has(run.id)) @@ -807,8 +811,9 @@ for (const execution of executions) { }); }; + let catalogFailureCaptureAttempted = false; const captureFailureApiState = async () => { - if (!fixtures) return; + if (catalogFailureCaptureAttempted || !fixtures) return; if (execution.task.flow === "followup_stop_resume") { const captured = await captureFollowupStopFailureApiState({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id, issueId: issue?.id, @@ -1389,6 +1394,14 @@ for (const execution of executions) { page, api, fixtures, execution, nonce, workspacePath, deadlineAt: startedAtMs + deadlineMs, restart: () => restartChatServer(page, () => restartIsolatedPaperclipServer({ api, requestId: `chat-${nonce}`, deadlineAt: startedAtMs + deadlineMs })), + retireFailedAttempt: async error => { + catalogFailureCaptureAttempted = true; + primaryError ??= error; + try { await writeSanitizedJson(snapshotsDir, "failed-attempt-checkpoint.json", { + schema: "paperclip.runner-e2e.failed-attempt-checkpoint/v1", executionId: execution.id, + qualification: "unqualified", featureOutcome: "unknown", phase: "before_owned_retirement", + }, secrets); } finally { await cancelActiveRunsForCleanup(20_000); } + }, observe: (chatIssue, chatRuns) => { issue = chatIssue; selectedRuns = chatRuns; }, capture: ordinaryChatCapture(execution.suite.id, capturePrivateScreenshot, captureScreenshot), evidence: completionEvidence, @@ -3247,9 +3260,12 @@ for (const execution of executions) { if (checks.some(check => !check.passed)) primaryError ??= new Error("Feedback persistence matcher failed"); } - const traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id), - cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions }); - await saveTrace(); + let traceReceipt: Awaited["export"]>> | undefined; + try { + traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id), + cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions }); + await saveTrace(); + } catch (error) { primaryError ??= error; } const finishedAtMs = Date.now(); const runtimeUsage = fixtureRuntimeCostCoverage(buildRuntimeUsage({ environmentId: execution.environment.id,