From 7a7c26f6688ec6b0e0454562a43fd48ab1a81c97 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 11:15:18 -0500 Subject: [PATCH] test: retain automatic repository success evidence --- tests/runner-e2e/automatic-repository-flow.ts | 7 +++++-- tests/runner-e2e/automatic-repository.test.ts | 17 ++++++++++++++++- tests/runner-e2e/automatic-repository.ts | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/automatic-repository-flow.ts b/tests/runner-e2e/automatic-repository-flow.ts index c676b7345a8..2a1b4693568 100644 --- a/tests/runner-e2e/automatic-repository-flow.ts +++ b/tests/runner-e2e/automatic-repository-flow.ts @@ -6,7 +6,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { createTaskThroughUi } from "./user-actions.js"; import { readRegisteredArtifacts } from "./registered-artifact.js"; -import { automaticRepositoryFiles, automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; +import { automaticRepositoryApiState, automaticRepositoryFiles, automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; @@ -61,7 +61,10 @@ export async function runAutomaticRepositoryFlow(input: { } async function save(phase: string) { const checks = gradeAutomaticRepository(observation); - if (issue) input.observe(issue, runs, checks); + if (issue) { + input.observe(issue, runs, checks); + await input.evidence("api-state.json", automaticRepositoryApiState(phase, issue, runs, checks)); + } await input.evidence("automatic-repository.json", { schema: "paperclip.e2e.automatic-repository.v1", phase, observation, checks, runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source, finalizationReason: run.resultJson?.finalizationReasonCode, startedAt: run.startedAt, finishedAt: run.finishedAt })) }); diff --git a/tests/runner-e2e/automatic-repository.test.ts b/tests/runner-e2e/automatic-repository.test.ts index 4657a5d3598..825ed08a0f7 100644 --- a/tests/runner-e2e/automatic-repository.test.ts +++ b/tests/runner-e2e/automatic-repository.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; +import { automaticRepositoryApiState, automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; const sha = "a".repeat(40); const reference = "c".repeat(64); @@ -48,3 +48,18 @@ it.each>([ ])("rejects a plausible incomplete or wrong result %j", fault => { expect(gradeAutomaticRepository({ ...valid, ...fault }).some(check => !check.passed)).toBe(true); }); + +it("retains required lifecycle evidence while excluding provider content and unrelated task data", () => { + const issue = { id: "task", companyId: "company", status: "done", description: "private description", privateToken: "private token" }; + const run = { id: "successor", companyId: "company", agentId: "agent", nativeIssueId: "task", status: "succeeded", + wakeupRequestId: "dispatch", contextSnapshot: { source: "native_status_decision", nativeStatusWakeIntentId: "intent", paperclipWake: { prompt: "private prompt" } }, + resultJson: { finalizationPhase: "committed", nativeResult: { summary: "private provider text" } }, privateConfig: { secret: "private secret" } }; + for (const phase of ["completed", "failed"]) { + const snapshot = automaticRepositoryApiState(phase, issue, [run], gradeAutomaticRepository(valid)); + expect(snapshot).toMatchObject({ capturePhase: phase, issue: { id: "task", status: "done" }, + runs: [{ id: "successor", wakeupRequestId: "dispatch", source: "native_status_decision", nativeStatusWakeIntentId: "intent", finalizationPhase: "committed" }] }); + expect(JSON.stringify(snapshot)).not.toMatch(/private|description|paperclipWake|nativeResult/); + } + const malformed = automaticRepositoryApiState("failed", issue, [{ ...run, contextSnapshot: { source: { private: true }, nativeStatusWakeIntentId: { private: true } }, resultJson: { finalizationPhase: { private: true } } }], []); + expect(malformed.runs[0]).toMatchObject({ source: null, nativeStatusWakeIntentId: null, finalizationPhase: null }); +}); diff --git a/tests/runner-e2e/automatic-repository.ts b/tests/runner-e2e/automatic-repository.ts index 27140059279..321ed86b907 100644 --- a/tests/runner-e2e/automatic-repository.ts +++ b/tests/runner-e2e/automatic-repository.ts @@ -54,6 +54,25 @@ export interface AutomaticRepositoryObservation { monitorScheduled: boolean; } +export function automaticRepositoryApiState(phase: string, issue: Record, runs: readonly { + id?: string; companyId?: string; agentId?: string; issueId?: string | null; nativeIssueId?: string | null; + status?: string; runtimeMode?: string; nativePhase?: string | null; startedAt?: string | null; finishedAt?: string | null; + errorCode?: string | null; wakeupRequestId?: string | null; + contextSnapshot?: { source?: unknown; nativeStatusWakeIntentId?: unknown } | null; + resultJson?: { finalizationPhase?: unknown } | null; +}[], checks: ReturnType) { + return { capturePhase: phase, + issue: { id: issue.id, companyId: issue.companyId, identifier: issue.identifier, status: issue.status, + assigneeAgentId: issue.assigneeAgentId, executionWorkspaceId: issue.executionWorkspaceId, statusVersion: issue.statusVersion }, + runs: runs.map(run => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, issueId: run.issueId, + nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, nativePhase: run.nativePhase, + startedAt: run.startedAt, finishedAt: run.finishedAt, errorCode: run.errorCode, wakeupRequestId: run.wakeupRequestId, + source: typeof run.contextSnapshot?.source === "string" ? run.contextSnapshot.source : null, + nativeStatusWakeIntentId: typeof run.contextSnapshot?.nativeStatusWakeIntentId === "string" ? run.contextSnapshot.nativeStatusWakeIntentId : null, + finalizationPhase: typeof run.resultJson?.finalizationPhase === "string" ? run.resultJson.finalizationPhase : null })), + checks }; +} + export function automaticRepositoryStopReason(issue: { status: string }, runs: readonly { status: string }[]) { if (runs.length > 2) return "Unexpected extra provider admission"; if (runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) return "A provider admission failed";