test: observe the real active provider checkpoint before Stop

tests/runner-e2e/followup-stop-resume-flow.ts

@@ -5,11 +5,11 @@ import path from "node:path";
 import { expect, type Page } from "@playwright/test";
 import { pollUntil, type RunnerApi } from "./api.js";
 import { collectRunEvents } from "./run-observations.js";
-import { readRegisteredArtifacts } from "./registered-artifact.js";
+import { readRegisteredArtifacts, RegisteredArtifactNotFoundError } from "./registered-artifact.js";
 import { captureStockHarness, gradeStockHire } from "./stock-harness.js";
 import { createTaskThroughUi, submitTaskReply } from "./user-actions.js";
 import { waitForRunDelivery } from "./run-delivery.js";
-import { exactSyntheticJson, followupStopFiles, gradeFollowupStopResume, STOP_QUIET_WINDOW_MS, type FollowupStopObservation } from "./followup-stop-resume.js";
+import { isActiveStopCheckpointRun, exactSyntheticJson, followupStopFiles, gradeFollowupStopResume, STOP_QUIET_WINDOW_MS, type FollowupStopObservation } from "./followup-stop-resume.js";
 import type { LiveFixtureValues } from "./live-fixtures.js";
 import type { MatrixExecution } from "./types.js";
 
@@ -86,11 +86,16 @@ export async function runFollowupStopResumeFlow(input: {
       load: async () => {
         await refresh();
         const run = runs[0];
-        if (runs.length !== 1 || run?.status !== "running" || run.nativePhase !== "running") return false;
+        if (runs.length !== 1 || !isActiveStopCheckpointRun(run)) return false;
         const events = prpEvents(await runEvents(run.id), run.id);
-        if (!events.some(event => event.eventType === "turn.started") || events.some(event => event.eventType === "run.result.accepted")) return false;
-        const [checkpoint] = await readRegisteredArtifacts(api, issue!.id, run.id, [files.checkpoint]);
-        if (!checkpoint || !exactSyntheticJson(checkpoint.content, { sourceSha256, label: files.initialLabel })) throw new Error("Checkpoint does not match the independent source");
+        if (!events.some(event => event.eventType === "turn.started" && event.sourceKind === "runner") || events.some(event => event.eventType === "run.result.accepted")) return false;
+        const checkpoint = await readRegisteredArtifacts(api, issue!.id, run.id, [files.checkpoint])
+          .then(([artifact]) => artifact).catch(error => {
+            if (error instanceof RegisteredArtifactNotFoundError) return null;
+            throw error;
+          });
+        if (!checkpoint) return false;
+        if (!exactSyntheticJson(checkpoint.content, { sourceSha256, label: files.initialLabel })) throw new Error("Checkpoint does not match the independent source");
         const view = await api.get<Row>(`/api/issues/${issue!.id}/workspace`);
         if (await realpath(view.workspace.cwd) !== workspace || !issue!.executionWorkspaceId) throw new Error("Stop/resume did not use its configured workspace");
         observation.initial = { runId: run.id, workspaceId: issue!.executionWorkspaceId, ...retainedRuntime(run), checkpointVerified: true };

tests/runner-e2e/followup-stop-resume.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { runnerMatrix } from "./catalog.js";
-import { exactSyntheticJson, gradeFollowupStopResume, type FollowupStopObservation } from "./followup-stop-resume.js";
+import { isActiveStopCheckpointRun, exactSyntheticJson, gradeFollowupStopResume, type FollowupStopObservation } from "./followup-stop-resume.js";
 
 function fixture(): FollowupStopObservation {
   return { binding: { companyId: "company", issueId: "task", agentId: "agent" },
@@ -65,3 +65,10 @@ describe("one task across follow-up, Stop and explicit resumption", () => {
       cell.suite.definitionMetadata?.stagingQualification === false && cell.task.automaticRetryPolicy === "single_attempt")).toBe(true);
   });
 });
+
+it("recognizes the production active-provider phase before checkpoint publication", () => {
+  expect(isActiveStopCheckpointRun({ status: "running", nativePhase: "provider_running" })).toBe(true);
+  for (const run of [undefined, { status: "queued", nativePhase: "provider_running" },
+    { status: "succeeded", nativePhase: "committed" }, { status: "running", nativePhase: "result_accepted" },
+    { status: "running", nativePhase: "workspace_finalizing" }]) expect(isActiveStopCheckpointRun(run)).toBe(false);
+});

tests/runner-e2e/followup-stop-resume.ts

@@ -25,6 +25,9 @@ export const followupStopTask: RunnerTaskFixture = {
     ].join("\n");
   }, buildMatchers: () => [],
 };
+export function isActiveStopCheckpointRun(run: { status?: string; nativePhase?: string | null } | undefined) {
+  return run?.status === "running" && run.nativePhase === "provider_running";
+}
 export function exactSyntheticJson(content: string, expected: Record<string, string>): boolean {
   let parsed: unknown;
   try { parsed = JSON.parse(content); } catch { return false; }

tests/runner-e2e/registered-artifact.test.ts

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 import { expect, it, vi } from "vitest";
 import type { RunnerApi } from "./api.js";
-import { readRegisteredArtifacts } from "./registered-artifact.js";
+import { readRegisteredArtifacts, RegisteredArtifactNotFoundError } from "./registered-artifact.js";
 import { evaluateMatcher } from "./matchers.js";
 
 function fixture() {
@@ -54,3 +54,19 @@ it("does not accept a filename or unchecked content as downloaded evidence", asy
   expect((await evaluateMatcher(matcher, { artifacts: [{ name: "proof.txt", content: "verified\n" }] })).passed).toBe(false);
   expect((await evaluateMatcher(matcher, { artifacts: [{ name: "proof.txt", contentVerified: true, content: "wrong" }] })).passed).toBe(false);
 });
+
+it("distinguishes a checkpoint not yet registered from corrupt or unauthorized evidence", async () => {
+  const { api, product, attachment, get, download } = fixture();
+  get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [] : [attachment]);
+  await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toBeInstanceOf(RegisteredArtifactNotFoundError);
+  expect(download).not.toHaveBeenCalled();
+  get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [product] : [attachment]);
+  expect((await readRegisteredArtifacts(api, "issue", "run", ["proof.txt"]))[0]?.contentVerified).toBe(true);
+  get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [product, { ...product, id: "duplicate" }] : [attachment]);
+  await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.not.toBeInstanceOf(RegisteredArtifactNotFoundError);
+  get.mockImplementation(async (path: string) => path.endsWith("work-products") ? [product] : [attachment]);
+  download.mockResolvedValue({ ok: () => false, status: () => 403 });
+  await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.not.toBeInstanceOf(RegisteredArtifactNotFoundError);
+  get.mockRejectedValue(new Error("read access revoked"));
+  await expect(readRegisteredArtifacts(api, "issue", "run", ["proof.txt"])).rejects.toThrow("read access revoked");
+});

tests/runner-e2e/registered-artifact.ts

@@ -3,6 +3,13 @@ import type { IssueAttachment } from "../../packages/shared/src/types/issue.js";
 import type { IssueWorkProduct } from "../../packages/shared/src/types/work-product.js";
 import type { RunnerApi } from "./api.js";
 
+export class RegisteredArtifactNotFoundError extends Error {
+  constructor(artifactName: string) {
+    super(`Expected one registered artifact ${artifactName} from the tested run; observed 0`);
+    this.name = "RegisteredArtifactNotFoundError";
+  }
+}
+
 const REGISTERED_ARTIFACT_STATUSES = new Set(["active", "ready_for_review", "approved"]);
 
 /** Registration, attribution and downloaded bytes must agree independently. */
@@ -16,6 +23,7 @@ export async function readRegisteredArtifacts(api: RunnerApi, issueId: string, r
     const registered = products.filter(product => product.issueId === issueId &&
       product.createdByRunId === runId && product.type === "artifact" &&
       REGISTERED_ARTIFACT_STATUSES.has(product.status) && product.title === name);
+    if (registered.length === 0) throw new RegisteredArtifactNotFoundError(name);
     if (registered.length !== 1) throw new Error(`Expected one registered artifact ${name} from the tested run; observed ${registered.length}`);
     const product = registered[0]!;
     const attachmentId = product.metadata?.attachmentId;