From 57cabfc6b38355afc3346ec581096b294587ebb2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 11:21:31 -0500 Subject: [PATCH] test: observe the real active provider checkpoint before Stop --- tests/runner-e2e/followup-stop-resume-flow.ts | 17 +++++++++++------ tests/runner-e2e/followup-stop-resume.test.ts | 9 ++++++++- tests/runner-e2e/followup-stop-resume.ts | 3 +++ tests/runner-e2e/registered-artifact.test.ts | 18 +++++++++++++++++- tests/runner-e2e/registered-artifact.ts | 8 ++++++++ 5 files changed, 47 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/followup-stop-resume-flow.ts b/tests/runner-e2e/followup-stop-resume-flow.ts index ea85f33d3ed..130f8511f83 100644 --- a/tests/runner-e2e/followup-stop-resume-flow.ts +++ b/tests/runner-e2e/followup-stop-resume-flow.ts @@ -5,11 +5,11 @@ import path from "node:path"; import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; -import { readRegisteredArtifacts } from "./registered-artifact.js"; +import { readRegisteredArtifacts, RegisteredArtifactNotFoundError } from "./registered-artifact.js"; import { captureStockHarness, gradeStockHire } from "./stock-harness.js"; import { createTaskThroughUi, submitTaskReply } from "./user-actions.js"; import { waitForRunDelivery } from "./run-delivery.js"; -import { exactSyntheticJson, followupStopFiles, gradeFollowupStopResume, STOP_QUIET_WINDOW_MS, type FollowupStopObservation } from "./followup-stop-resume.js"; +import { isActiveStopCheckpointRun, exactSyntheticJson, followupStopFiles, gradeFollowupStopResume, STOP_QUIET_WINDOW_MS, type FollowupStopObservation } from "./followup-stop-resume.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; @@ -86,11 +86,16 @@ export async function runFollowupStopResumeFlow(input: { load: async () => { await refresh(); const run = runs[0]; - if (runs.length !== 1 || run?.status !== "running" || run.nativePhase !== "running") return false; + if (runs.length !== 1 || !isActiveStopCheckpointRun(run)) return false; const events = prpEvents(await runEvents(run.id), run.id); - if (!events.some(event => event.eventType === "turn.started") || events.some(event => event.eventType === "run.result.accepted")) return false; - const [checkpoint] = await readRegisteredArtifacts(api, issue!.id, run.id, [files.checkpoint]); - if (!checkpoint || !exactSyntheticJson(checkpoint.content, { sourceSha256, label: files.initialLabel })) throw new Error("Checkpoint does not match the independent source"); + if (!events.some(event => event.eventType === "turn.started" && event.sourceKind === "runner") || events.some(event => event.eventType === "run.result.accepted")) return false; + const checkpoint = await readRegisteredArtifacts(api, issue!.id, run.id, [files.checkpoint]) + .then(([artifact]) => artifact).catch(error => { + if (error instanceof RegisteredArtifactNotFoundError) return null; + throw error; + }); + if (!checkpoint) return false; + if (!exactSyntheticJson(checkpoint.content, { sourceSha256, label: files.initialLabel })) throw new Error("Checkpoint does not match the independent source"); const view = await api.get(`/api/issues/${issue!.id}/workspace`); if (await realpath(view.workspace.cwd) !== workspace || !issue!.executionWorkspaceId) throw new Error("Stop/resume did not use its configured workspace"); observation.initial = { runId: run.id, workspaceId: issue!.executionWorkspaceId, ...retainedRuntime(run), checkpointVerified: true }; diff --git a/tests/runner-e2e/followup-stop-resume.test.ts b/tests/runner-e2e/followup-stop-resume.test.ts index bdba28b7ee1..7fa8063fe8a 100644 --- a/tests/runner-e2e/followup-stop-resume.test.ts +++ b/tests/runner-e2e/followup-stop-resume.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { runnerMatrix } from "./catalog.js"; -import { exactSyntheticJson, gradeFollowupStopResume, type FollowupStopObservation } from "./followup-stop-resume.js"; +import { isActiveStopCheckpointRun, exactSyntheticJson, gradeFollowupStopResume, type FollowupStopObservation } from "./followup-stop-resume.js"; function fixture(): FollowupStopObservation { return { binding: { companyId: "company", issueId: "task", agentId: "agent" }, @@ -65,3 +65,10 @@ describe("one task across follow-up, Stop and explicit resumption", () => { cell.suite.definitionMetadata?.stagingQualification === false && cell.task.automaticRetryPolicy === "single_attempt")).toBe(true); }); }); + +it("recognizes the production active-provider phase before checkpoint publication", () => { + expect(isActiveStopCheckpointRun({ status: "running", nativePhase: "provider_running" })).toBe(true); + for (const run of [undefined, { status: "queued", nativePhase: "provider_running" }, + { status: "succeeded", nativePhase: "committed" }, { status: "running", nativePhase: "result_accepted" }, + { status: "running", nativePhase: "workspace_finalizing" }]) expect(isActiveStopCheckpointRun(run)).toBe(false); +}); diff --git a/tests/runner-e2e/followup-stop-resume.ts b/tests/runner-e2e/followup-stop-resume.ts index 655c4a4061a..4e59340ed95 100644 --- a/tests/runner-e2e/followup-stop-resume.ts +++ b/tests/runner-e2e/followup-stop-resume.ts @@ -25,6 +25,9 @@ export const followupStopTask: RunnerTaskFixture = { ].join("\n"); }, buildMatchers: () => [], }; +export function isActiveStopCheckpointRun(run: { status?: string; nativePhase?: string | null } | undefined) { + return run?.status === "running" && run.nativePhase === "provider_running"; +} export function exactSyntheticJson(content: string, expected: Record): boolean { let parsed: unknown; try { parsed = JSON.parse(content); } catch { return false; } diff --git a/tests/runner-e2e/registered-artifact.test.ts b/tests/runner-e2e/registered-artifact.test.ts index b0754d56fb1..e01f93682e3 100644 --- a/tests/runner-e2e/registered-artifact.test.ts +++ b/tests/runner-e2e/registered-artifact.test.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { expect, it, vi } from "vitest"; import type { RunnerApi } from "./api.js"; -import { readRegisteredArtifacts } from "./registered-artifact.js"; +import { readRegisteredArtifacts, RegisteredArtifactNotFoundError } from "./registered-artifact.js"; import { evaluateMatcher } from "./matchers.js"; function fixture() { @@ -54,3 +54,19 @@ it("does not accept a filename or unchecked content as downloaded evidence", asy expect((await evaluateMatcher(matcher, { artifacts: [{ name: "proof.txt", content: "verified\n" }] })).passed).toBe(false); expect((await evaluateMatcher(matcher, { artifacts: [{ name: "proof.txt", contentVerified: true, content: "wrong" }] })).passed).toBe(false); }); + +it("distinguishes a checkpoint not yet registered from corrupt or unauthorized evidence", async () => { + const { api, product, attachment, get, download } = fixture(); + get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [] : [attachment]); + await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toBeInstanceOf(RegisteredArtifactNotFoundError); + expect(download).not.toHaveBeenCalled(); + get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [product] : [attachment]); + expect((await readRegisteredArtifacts(api, "issue", "run", ["proof.txt"]))[0]?.contentVerified).toBe(true); + get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [product, { ...product, id: "duplicate" }] : [attachment]); + await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.not.toBeInstanceOf(RegisteredArtifactNotFoundError); + get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [product] : [attachment]); + download.mockResolvedValue({ ok: () => false, status: () => 403 }); + await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.not.toBeInstanceOf(RegisteredArtifactNotFoundError); + get.mockRejectedValue(new Error("read access revoked")); + await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toThrow("read access revoked"); +}); diff --git a/tests/runner-e2e/registered-artifact.ts b/tests/runner-e2e/registered-artifact.ts index 33b4df8b8b4..9531119fe36 100644 --- a/tests/runner-e2e/registered-artifact.ts +++ b/tests/runner-e2e/registered-artifact.ts @@ -3,6 +3,13 @@ import type { IssueAttachment } from "../../packages/shared/src/types/issue.js"; import type { IssueWorkProduct } from "../../packages/shared/src/types/work-product.js"; import type { RunnerApi } from "./api.js"; +export class RegisteredArtifactNotFoundError extends Error { + constructor(artifactName: string) { + super(`Expected one registered artifact ${artifactName} from the tested run; observed 0`); + this.name = "RegisteredArtifactNotFoundError"; + } +} + const REGISTERED_ARTIFACT_STATUSES = new Set(["active", "ready_for_review", "approved"]); /** Registration, attribution and downloaded bytes must agree independently. */ @@ -16,6 +23,7 @@ export async function readRegisteredArtifacts(api: RunnerApi, issueId: string, r const registered = products.filter(product => product.issueId === issueId && product.createdByRunId === runId && product.type === "artifact" && REGISTERED_ARTIFACT_STATUSES.has(product.status) && product.title === name); + if (registered.length === 0) throw new RegisteredArtifactNotFoundError(name); if (registered.length !== 1) throw new Error(`Expected one registered artifact ${name} from the tested run; observed ${registered.length}`); const product = registered[0]!; const attachmentId = product.metadata?.attachmentId;