diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index a86949df8e7..767e2f247a9 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -71,6 +71,12 @@ Cloud adapter admits at most 30 minutes and $10. Select `runner-codex-mini` explicitly for its `gpt-5.4-mini` pilot; existing profile defaults remain unchanged. There are no automatic story retries. One campaign may create at most one disposable workspace. +Workspace contention can defer a child before provider execution. The journey +keeps that run and follows only its unique, scoped durable replacement when the +server proves that no provider work started. A scheduled replacement remains +pending and consumes the run limit. It cannot satisfy completion; the child must +still finish through native execution and pass the independent artifact checks. + ## Evidence and lifecycle Results use the existing campaign/result schemas and dashboard. Each result @@ -130,6 +136,9 @@ request admission, setup/agent phase transitions, run observation, cost receipts trace IDs, and feedback collection. A normalized operations receipt belongs only to the onboarding attempt; descendants do not repeat shared setup charges. Unknown cost remains unknown. Raw feedback, traces, and ledgers stay private. +Invocation evidence distinguishes native event inspection from a proven +`provider_not_started` workspace deferral. Both require complete event reads. +Pending replacements and contradictory execution evidence remain partial. The sibling **Operationalize eval runs** implementation currently admits local cells and binds its gateway to loopback. That is not Cloud reachability or budget diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 2d1545a339f..c45e00c2981 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -2840,6 +2840,12 @@ have complete native invocation coverage. An empty saved-feedback list does not prove that a tool was available, and a completed tool receipt does not by itself prove a saved submission. +Invocation receipts distinguish inspected native runs (`native_events`) from +terminal workspace deferrals that provably started no provider work +(`provider_not_started`). The latter requires the server's explicit receipt and +an exhaustive scan without provider events. Pending retries, actual legacy +execution, contradictory receipts, and incomplete event reads remain partial. + Results and retained-attempt receipts contain only counts, closed statuses, and body fingerprints. Campaign reports group identical fingerprints across attempts into distinct items awaiting review. Feedback presence does not change product diff --git a/tests/runner-e2e/feedback.test.ts b/tests/runner-e2e/feedback.test.ts index 6cb8ead6bb4..dce4d204e8e 100644 --- a/tests/runner-e2e/feedback.test.ts +++ b/tests/runner-e2e/feedback.test.ts @@ -103,6 +103,124 @@ it.each([{ runtimeMode: "legacy", status: "succeeded" }, { runtimeMode: "native" expect(summary.invocationCollection).toBe("partial"); }); +function workspaceDeferral(): Record { + return { + id: "run", companyId: "company", agentId: "agent", issueId: "child", status: "cancelled", runtimeMode: "legacy", + errorCode: "workspace_busy", startedAt: "2026-10-28T03:35:01.660Z", finishedAt: "2026-10-28T03:35:02.268Z", + runtimeModeResolvedAt: null, processStartedAt: null, processPid: null, processStartTicks: null, + runnerInstanceId: null, nativeSessionId: null, sessionIdAfter: null, lastOutputSeq: 0, costUsd: null, + usageJson: { ledgerScope: { issueId: "child" } }, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { + executionRecovery: { kind: "workspace_wait", providerWorkStarted: false }, + cancellation: { source: "control_plane", expected: true, initiator: { type: "system" } }, + workspaceBusy: { holderRunId: "parent-run", holderIssueId: "parent", deferralAttempt: 0 }, + }, + }; +} + +async function collectWorkspaceDeferral(run: Record, events: Record[] = []) { + let retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => { + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) return events; + if (path === "/api/heartbeat-runs/run") return run; + return [{ id: "run", status: run.status }]; + }) }); + return { summary, retained: retained! }; +} + +it("records complete no-provider coverage for a proven workspace deferral without claiming native execution", async () => { + const { summary, retained } = await collectWorkspaceDeferral(workspaceDeferral(), [ + { seq: 1, eventType: "lifecycle", stream: "system", payload: { retryScheduled: true } }, + ]); + expect(summary).toMatchObject({ collection: "complete", invocationCollection: "complete", runsInspected: 1, + invocations: { completed: 0, failed: 0, unknown: 0 }, toolAvailability: "unknown" }); + expect(retained.runCoverage).toEqual([{ runId: "run", classification: "provider_not_started" }]); +}); + +it.each([ + ["pending successor", { status: "scheduled_retry", startedAt: null, finishedAt: null }], + ["actual legacy work", { status: "succeeded", errorCode: null, resultJson: {} }], + ["provider work", { resultJson: { executionRecovery: { kind: "workspace_wait", providerWorkStarted: true } } }], + ["native process", { runnerInstanceId: "runner" }], + ["native selection", { runtimeMode: "native", runtimeModeResolvedAt: "2026-10-28T03:35:01.700Z" }], + ["local process", { processPid: 123 }], + ["usage", { usageJson: { inputTokens: 1 } }], + ["charge", { costUsd: "0.01" }], +] as const)("keeps invocation coverage partial for %s despite a workspace marker", async (_label, overrides) => { + const { summary, retained } = await collectWorkspaceDeferral({ ...workspaceDeferral(), ...overrides }); + expect(summary.invocationCollection).toBe("partial"); + expect(retained.runCoverage).toEqual([{ runId: "run", classification: "incomplete" }]); +}); + +it.each([ + event(1, "tool.execution.started", { name: "submit_complaint", executionId: "call" }), + event(1, "turn.started", {}), + { seq: 1, eventType: "adapter.invoke", stream: "system", payload: {} }, + { seq: 1, eventType: "stdout", stream: "stdout", payload: {} }, + { seq: 1, eventType: "lifecycle", stream: "system", sourceEventId: "native-event", payload: {} }, + { seq: 1, eventType: "lifecycle", stream: "system", runId: "foreign-run", payload: {} }, + { seq: 1, eventType: "unknown", stream: "system", payload: {} }, +])("rejects contradictory or unknown events when classifying a workspace deferral: %o", async contradictory => { + const { summary, retained } = await collectWorkspaceDeferral(workspaceDeferral(), [contradictory]); + expect(summary.invocationCollection).toBe("partial"); + expect(retained.runCoverage).toEqual([{ runId: "run", classification: "incomplete" }]); +}); + +it("requires an exhaustive event scan before recording no-provider coverage", async () => { + let retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => { + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) throw new Error("unavailable"); + if (path === "/api/heartbeat-runs/run") return workspaceDeferral(); + return [{ id: "run", status: "cancelled" }]; + }) }); + expect(summary).toMatchObject({ invocationCollection: "partial", runsInspected: 0 }); + expect(retained!.runCoverage).toEqual([{ runId: "run", classification: "incomplete" }]); + expect(retained!.diagnostics).toContain("events_read_failed"); +}); + +it("examines all event pages before accepting no-provider coverage", async () => { + let pages = 0, retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => { + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) { + pages++; + return pages === 1 ? Array.from({ length: 100 }, (_, index) => ({ seq: index + 1, eventType: "lifecycle", stream: "system" })) + : [event(101, "tool.execution.started", { name: "submit_suggestion", executionId: "call" })]; + } + if (path === "/api/heartbeat-runs/run") return workspaceDeferral(); + return [{ id: "run", status: "cancelled" }]; + }) }); + expect(pages).toBe(2); + expect(summary).toMatchObject({ invocationCollection: "partial", invocations: { unknown: 1 } }); + expect(retained!.runCoverage).toEqual([{ runId: "run", classification: "incomplete" }]); +}); + +it("does not complete feedback while a workspace deferral's replacement remains scheduled", async () => { + let retained: FeedbackEvidence | undefined; + const runs = [ + { id: "parent-opening", status: "succeeded", runtimeMode: "native" }, + { id: "parent-resumed", status: "succeeded", runtimeMode: "native" }, + workspaceDeferral(), + { id: "replacement", status: "scheduled_retry", runtimeMode: "legacy", retryOfRunId: "run", startedAt: null }, + ]; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => { + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) return []; + if (path.startsWith("/api/heartbeat-runs/")) return runs.find(run => run.id === path.split("/").at(-1)); + return runs; + }) }); + expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", runsInspected: 4 }); + expect(retained!.runCoverage).toEqual([ + { runId: "parent-opening", classification: "native_events" }, + { runId: "parent-resumed", classification: "native_events" }, + { runId: "run", classification: "provider_not_started" }, + { runId: "replacement", classification: "incomplete" }, + ]); +}); + it("captures feedback after cancellation failure but before destructive teardown, and still tears down if persistence fails", async () => { const calls: string[] = []; const result = await runCleanupWithObservers({ diff --git a/tests/runner-e2e/feedback.ts b/tests/runner-e2e/feedback.ts index 7a24b21cb6d..f75d57988eb 100644 --- a/tests/runner-e2e/feedback.ts +++ b/tests/runner-e2e/feedback.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import type { AgentCommentaryDetail, AgentCommentaryPage } from "../../packages/shared/src/validators/agent-commentary.js"; import type { RunnerApi } from "./api.js"; +import { isProviderUnstartedWorkspaceDeferral } from "./product-journeys/workspace-deferral.js"; type Row = Record; const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; @@ -23,11 +24,17 @@ interface FeedbackInvocation { kind: "complaint" | "suggestion"; outcome: "completed" | "failed" | "unknown"; } +export interface FeedbackRunCoverage { + runId: string; + classification: "native_events" | "provider_not_started" | "incomplete"; +} export interface FeedbackEvidence { version: 1; companyId: string; entries: AgentCommentaryDetail[]; invocations: FeedbackInvocation[]; + /** Absent in historical evidence; no-provider coverage is not native execution. */ + runCoverage?: FeedbackRunCoverage[]; diagnostics: Array<"feedback_read_failed" | "feedback_detail_failed" | "feedback_pagination_incomplete" | "run_read_failed" | "run_detail_failed" | "run_limit_reached" | "events_read_failed" | "deadline_reached" | "evidence_limit_reached">; summary: FeedbackSummary; } @@ -72,6 +79,15 @@ export function observeFeedbackInvocations(runId: string, events: readonly Row[] return [...calls.values()]; } +function onlyControlPlaneEvents(runId: string, events: readonly Row[]): boolean { + return events.every(event => event.stream === "system" + && (event.runId === undefined || event.runId === runId) + && ["lifecycle", "phase"].includes(String(event.eventType)) + && row(event.payload).prpEvent === undefined + && [event.sourceInstanceId, event.sourceEventId, event.sourceSeq, event.protocolSchemaVersion] + .every(value => value === undefined || value === null)); +} + export function feedbackReview(entries: readonly AgentCommentaryDetail[]): FeedbackSummary["review"] { const groups = new Map(); for (const entry of entries) { @@ -117,7 +133,8 @@ export async function collectAttemptFeedback(input: { ...(summary.collection !== "complete" ? { collection: summary.collection } : {}), ...(summary.invocationCollection !== "complete" ? { invocationCollection: summary.invocationCollection } : {}), }); - const evidence: FeedbackEvidence = { version: 1, companyId: input.companyId, entries: [], invocations: [], diagnostics: [], summary }; + const runCoverage: FeedbackRunCoverage[] = []; + const evidence: FeedbackEvidence = { version: 1, companyId: input.companyId, entries: [], invocations: [], runCoverage, diagnostics: [], summary }; let retainedBytes = 0, checkpointBytes = 0, eventBytes = 0, eventCount = 0; const diagnose = (code: FeedbackEvidence["diagnostics"][number]) => { if (!evidence.diagnostics.includes(code)) evidence.diagnostics.push(code); }; const get = (url: string) => { @@ -213,23 +230,26 @@ export async function collectAttemptFeedback(input: { summary.invocationCollection = runs.length < 1000 ? "complete" : "partial"; if (runs.length >= 1000) diagnose("run_limit_reached"); for (const listed of runs) { - let run: { id: string; status?: string; runtimeMode?: string } = listed; + let run: Row & { id: string; status?: string; runtimeMode?: string } = listed; + let detailRead = false; try { const detail = await get(`/api/heartbeat-runs/${encodeURIComponent(listed.id)}`); if (detail.id !== listed.id) throw new Error("Invalid run detail"); run = detail; + detailRead = true; } catch { summary.invocationCollection = "partial"; summary.collection = summary.collection === "complete" ? "partial" : summary.collection; diagnose("run_detail_failed"); } - if (!["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(run.status ?? "")) { + const terminal = ["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(run.status ?? ""); + if (!terminal) { summary.collection = summary.collection === "complete" ? "partial" : summary.collection; summary.invocationCollection = "partial"; } - if (run.runtimeMode !== "native") summary.invocationCollection = "partial"; const events: Row[] = []; let afterSeq = 0; + let eventsComplete = false; try { for (let pageNumber = 0; pageNumber < 100; pageNumber++) { const page = await get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=100`); @@ -248,10 +268,19 @@ export async function collectAttemptFeedback(input: { if (pageNumber === 99) { diagnose("evidence_limit_reached"); throw new Error("Event limit reached"); } } summary.runsInspected++; + eventsComplete = true; } catch { summary.invocationCollection = "partial"; diagnose("events_read_failed"); } + const workspaceWait = run.errorCode === "workspace_busy" || row(row(run.resultJson).executionRecovery).kind === "workspace_wait"; + const classification: FeedbackRunCoverage["classification"] = detailRead && terminal && eventsComplete + ? isProviderUnstartedWorkspaceDeferral(run) && onlyControlPlaneEvents(run.id, events) + ? "provider_not_started" + : run.runtimeMode === "native" && !workspaceWait ? "native_events" : "incomplete" + : "incomplete"; + runCoverage.push({ runId: run.id, classification }); + if (classification === "incomplete") summary.invocationCollection = "partial"; for (const invocation of observeFeedbackInvocations(run.id, events)) { const bytes = Buffer.byteLength(JSON.stringify(invocation)) + 1; if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break; } diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 980822013fd..003d998ab4c 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -99,7 +99,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); const suiteDefinitionHash = definitionHash.digest("hex"); const expected = config.journeys.map(id => `product-journeys.${config.id}.${id === "cloud-onboarding" ? config.modelProfileId : "browser"}.${config.executionEnvironment}.${id}`); let reportPersisted = false; diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts index 2e559aa8584..3908de12cb3 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts @@ -274,4 +274,166 @@ describe("Cloud canonical onboarding first task", () => { } }); expect((await onboardingFirstTask(f.input)).runIds).toContain("obsolete-wake"); }); + it("follows a proven pre-provider workspace wait to its native child execution without retrying the story", async () => { + let waiting = true; + vi.mocked(connectionDelay).mockImplementation(async () => { waiting = false; }); + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push(workspaceDeferral()); + const child = state.runs.find(run => run.id === "child-run")!; + Object.assign(child, { retryOfRunId: "workspace-wait", scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: 1 }); + child.contextSnapshot.retryOfRunId = "workspace-wait"; + child.contextSnapshot.retryReason = "workspace_busy"; + child.contextSnapshot.wakeReason = "workspace_busy_retry"; + if (!waiting) return; + Object.assign(child, { status: "scheduled_retry", runtimeMode: "legacy", runtimeModeResolvedAt: null, + nativeIssueId: null, startedAt: null, finishedAt: null, scheduledRetryAt: "2026-01-01T00:01:30Z", usageJson: null, lastOutputSeq: 0 }); + state.tasks.forEach(task => { task.status = "in_progress"; }); + state.runs[1]!.status = "running"; + state.runs = state.runs.filter(run => run.id !== "completion-run"); + state.documents = []; state.attachments = []; + } }); + const result = await onboardingFirstTask(f.input); + expect(result.runIds).toEqual(["proposal-run", "acceptance-run", "child-run", "workspace-wait", "completion-run"]); + expect(f.events.filter(event => event === "click:Continue")).toHaveLength(1); + expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1); + expect(result.artifact.originatingRunId).toBe("child-run"); + expect(connectionDelay).toHaveBeenCalledOnce(); + }); + it.each(["transient_failure", "ai_connection_busy", "ai_connection_pool_wait", "max_turns_continuation", "issue_disposition_repair"])("starts the workspace wait counter independently of %s", async reason => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push({ ...workspaceDeferral(), scheduledRetryReason: reason, scheduledRetryAttempt: 3 }); + linkWorkspaceRetry(state.runs.find(run => run.id === "child-run")!); + } }); + await expect(onboardingFirstTask(f.input)).resolves.toMatchObject({ artifact: { originatingRunId: "child-run" } }); + expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1); + }); + it("does not reset an existing workspace wait counter", async () => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push({ ...workspaceDeferral(), scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: 3 }); + linkWorkspaceRetry(state.runs.find(run => run.id === "child-run")!); + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_workspace_retry_lineage_mismatch" }); + }); + it.each([ + ["missing proof", (run: Row) => { run.resultJson = {}; }], + ["provider started", (run: Row) => { run.resultJson.executionRecovery.providerWorkStarted = true; }], + ["unexpected cancellation", (run: Row) => { run.resultJson.cancellation.expected = false; }], + ["operator cancellation", (run: Row) => { run.resultJson.cancellation.initiator.type = "user"; }], + ["runtime selected", (run: Row) => { run.runtimeModeResolvedAt = "2026-01-01T00:00:00Z"; }], + ["native session", (run: Row) => { run.nativeSessionId = "native-session"; }], + ["provider process", (run: Row) => { run.processStartedAt = "2026-01-01T00:00:00Z"; }], + ["provider output", (run: Row) => { run.lastOutputSeq = 1; }], + ["provider usage", (run: Row) => { run.usageJson.outputTokens = 1; }], + ["provider cost", (run: Row) => { run.costUsd = 0.001; }], + ["native execution input", (run: Row) => { run.runnerProfileJson.nativeExecutionInput = {}; }], + ] as const)("rejects workspace cancellation with %s even if a later child succeeded", async (_label, change) => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + const deferred = workspaceDeferral(); change(deferred); state.runs.push(deferred); + linkWorkspaceRetry(state.runs.find(run => run.id === "child-run")!); + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" }); + expect(f.events).not.toContain("visible:download"); + }); + it.each([ + ["ambiguous successor", (child: Row, state: { runs: Row[] }) => { state.runs.push({ ...child, id: "duplicate-retry" }); }], + ["wrong company", (child: Row) => { child.companyId = "other-company"; }], + ["wrong agent", (child: Row) => { child.agentId = "other-agent"; }], + ["conflicting issue", (child: Row) => { child.contextSnapshot.issueId = "parent"; }], + ["missing context lineage", (child: Row) => { delete child.contextSnapshot.retryOfRunId; }], + ["conflicting context lineage", (child: Row) => { child.contextSnapshot.retryOfRunId = "other-run"; }], + ["different retry reason", (child: Row) => { child.scheduledRetryReason = "transient_failure"; }], + ["different wake reason", (child: Row) => { child.contextSnapshot.wakeReason = "manual_retry"; }], + ["skipped attempt", (child: Row) => { child.scheduledRetryAttempt = 3; }], + ] as const)("rejects workspace retry with %s", async (_label, change) => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push(workspaceDeferral()); + const child = state.runs.find(run => run.id === "child-run")!; linkWorkspaceRetry(child); change(child, state); + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_workspace_retry_lineage_mismatch" }); + expect(f.events).not.toContain("visible:download"); + }); + it.each(["failed", "cancelled", "interrupted", "timed_out", "unknown"])("does not hide a %s successor", async status => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push(workspaceDeferral()); + const child = state.runs.find(run => run.id === "child-run")!; linkWorkspaceRetry(child); child.status = status; + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_failed" }); + }); + it.each(["missing", "scheduled"])("never passes a %s workspace successor even when tasks claim completion", async mode => { + let now = Date.now(); vi.spyOn(Date, "now").mockImplementation(() => now); + vi.mocked(connectionDelay).mockImplementation(async ms => { now += ms; }); + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push(workspaceDeferral()); + if (mode === "missing") return; + const child = state.runs.find(run => run.id === "child-run")!; + Object.assign(child, { status: "scheduled_retry", runtimeMode: "legacy", runtimeModeResolvedAt: null, + nativeIssueId: null, startedAt: null, finishedAt: null, scheduledRetryAt: "2026-01-01T00:01:30Z", usageJson: null, lastOutputSeq: 0 }); + linkWorkspaceRetry(child); + } }); + f.input.deadlineAt = now + 2000; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_deadline" }); + expect(f.events).not.toContain("visible:download"); + }); + it("counts the pre-provider cancellation against the campaign run cap", async () => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push(workspaceDeferral()); linkWorkspaceRetry(state.runs.find(run => run.id === "child-run")!); + } }); + f.input.maxRuns = 4; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "campaign_run_limit_exceeded" }); + expect(f.observeRun).toHaveBeenCalledTimes(5); + }); + it("waits for the durable successor when cancellation becomes visible first", async () => { + let waiting = true; + vi.mocked(connectionDelay).mockImplementation(async () => { waiting = false; }); + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + state.runs.push(workspaceDeferral()); + if (!waiting) linkWorkspaceRetry(state.runs.find(run => run.id === "child-run")!); + } }); + await expect(onboardingFirstTask(f.input)).resolves.toMatchObject({ artifact: { originatingRunId: "child-run" } }); + expect(connectionDelay).toHaveBeenCalledOnce(); + expect(f.events.filter(event => event === "click:Approve")).toHaveLength(1); + }); + it("follows repeated proven workspace admission waits within the same run budget", async () => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + const first = workspaceDeferral(); + const second = { ...workspaceDeferral(), id: "second-wait", contextSnapshot: { issueId: "child" } }; + linkWorkspaceRetry(second); + state.runs.push(first, second); + linkWorkspaceRetry(state.runs.find(run => run.id === "child-run")!, "second-wait", 2); + } }); + const result = await onboardingFirstTask(f.input); + expect(result.runIds).toHaveLength(6); + expect(result.runIds).toContain("workspace-wait"); + expect(result.runIds).toContain("second-wait"); + expect(result.artifact.originatingRunId).toBe("child-run"); + }); }); + +function linkWorkspaceRetry(run: Row, sourceId = "workspace-wait", attempt = 1) { + Object.assign(run, { retryOfRunId: sourceId, scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: attempt }); + Object.assign(run.contextSnapshot, { retryOfRunId: sourceId, retryReason: "workspace_busy", wakeReason: "workspace_busy_retry" }); +} + +function workspaceDeferral(): Row { + return { id: "workspace-wait", companyId: "company", agentId: "agent", issueId: "child", + status: "cancelled", errorCode: "workspace_busy", runtimeMode: "legacy", runtimeModeResolvedAt: null, + nativeIssueId: null, nativeSessionId: null, runnerInstanceId: null, processPid: null, processGroupId: null, + processStartedAt: null, sessionIdAfter: null, externalRunId: null, exitCode: null, signal: null, + startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", lastOutputSeq: 0, + usageJson: { ledgerScope: { companyId: "company", agentId: "agent", issueId: "child" } }, costUsd: null, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + contextSnapshot: { issueId: "child" }, resultJson: { + executionRecovery: { kind: "workspace_wait", providerWorkStarted: false }, + cancellation: { source: "control_plane", expected: true, initiator: { type: "system" } }, + } }; +} diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.ts index 474bd615a84..1ea4c693dd7 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.ts @@ -7,6 +7,7 @@ import { isBlockedUnstartedWake, isTerminalUnstartedWake } from "../non-executio import { JourneyError, type JourneyContext, type OnboardingArtifactEvidence, type TargetSession } from "./types.js"; import { retainOnboardingArtifact } from "./artifact-evidence.js"; import { preAcceptanceTaskStatus, preAcceptanceViolations, type OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; +import { isProviderUnstartedWorkspaceDeferral } from "./workspace-deferral.js"; export interface OnboardingFirstTaskInput { context: JourneyContext; @@ -40,6 +41,26 @@ function fail(code: string): never { throw new JourneyError("failed", code); } const runIssueId = (run: Row) => run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId; const isProposal = (row: Row) => ["request_confirmation", "request_checkbox_confirmation"].includes(row.kind); const isPlanningDocument = (row: Row) => /^(?:plan|proposal)$/.test(row.key); +const isPendingRun = (run: Row) => activeRuns([run]).length > 0 || run.status === "scheduled_retry"; + +function sameWorkspaceRetry(source: Row, successor: Row): boolean { + const issueIds = (run: Row) => [run.nativeIssueId, run.issueId, run.contextSnapshot?.issueId].filter(value => value != null); + const sourceIssues = issueIds(source), successorIssues = issueIds(successor); + return source.id !== successor.id && source.companyId === successor.companyId && source.agentId === successor.agentId && + sourceIssues.length > 0 && successorIssues.length > 0 && [...sourceIssues, ...successorIssues].every(id => id === sourceIssues[0]) && + successor.retryOfRunId === source.id && successor.contextSnapshot?.retryOfRunId === source.id && + successor.scheduledRetryReason === "workspace_busy" && successor.contextSnapshot?.retryReason === "workspace_busy" && + successor.contextSnapshot?.wakeReason === "workspace_busy_retry" && + Number.isSafeInteger(successor.scheduledRetryAttempt) && + successor.scheduledRetryAttempt === (source.scheduledRetryReason === "workspace_busy" ? source.scheduledRetryAttempt ?? 0 : 0) + 1; +} + +function unstartedScheduledWorkspaceRetry(run: Row): boolean { + return run.status === "scheduled_retry" && run.startedAt === null && run.finishedAt === null && + Number.isFinite(Date.parse(run.scheduledRetryAt)) && run.runtimeMode === "legacy" && run.runtimeModeResolvedAt === null && + ["nativeIssueId", "nativeSessionId", "runnerInstanceId", "processPid", "processStartedAt", "sessionIdAfter"].every(key => run[key] == null) && + run.lastOutputSeq === 0 && run.usageJson == null; +} /** Exercise the ordinary onboarding conversation; APIs observe, never submit its actions. */ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Promise { @@ -87,11 +108,38 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom return { tasks, runs, interactions, comments, documents, attachments }; }; const assertRuns = (snapshot: Snapshot, allowedIssues: string[]) => { + const runs = [...seen.values()]; + let workspacePending = false; + const successors = new Map(); + for (const source of runs.filter(isProviderUnstartedWorkspaceDeferral)) { + const linked = runs.filter(run => run.retryOfRunId === source.id || run.contextSnapshot?.retryOfRunId === source.id); + if (linked.length > 1 || (linked[0] && !sameWorkspaceRetry(source, linked[0]))) fail("onboarding_workspace_retry_lineage_mismatch"); + if (linked[0]) successors.set(source.id, linked[0]); + else workspacePending = true; + } for (const run of seen.values()) { if (run.companyId !== company.id || run.agentId !== agent.id || !allowedIssues.includes(runIssueId(run))) fail("onboarding_run_scope_mismatch"); + if (isProviderUnstartedWorkspaceDeferral(run)) { + const chain = new Set(); + let tail: Row | undefined = run; + while (tail && isProviderUnstartedWorkspaceDeferral(tail)) { + if (chain.has(tail.id)) fail("onboarding_workspace_retry_lineage_mismatch"); + chain.add(tail.id); tail = successors.get(tail.id); + } + if (!tail || isPendingRun(tail)) workspacePending = true; + else if (tail.status !== "succeeded" || tail.runtimeMode !== "native") fail("onboarding_first_task_failed"); + continue; + } + if (run.status === "scheduled_retry") { + const source = seen.get(run.retryOfRunId); + if (!source || !isProviderUnstartedWorkspaceDeferral(source) || !sameWorkspaceRetry(source, run) || !unstartedScheduledWorkspaceRetry(run)) fail("onboarding_workspace_retry_lineage_mismatch"); + workspacePending = true; + continue; + } if (activeRuns([run]).length || run.status === "succeeded" || isBlockedUnstartedWake(run) || isTerminalUnstartedWake(run, snapshot.tasks)) continue; fail("onboarding_first_task_failed"); } + return workspacePending; }; const assertBeforeAcceptance = (snapshot: Snapshot, parentId: string, checkpoint: OnboardingPreAcceptanceEvidence["checkpoint"]) => { const observed = { @@ -106,7 +154,7 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom input.onPreAcceptanceFailure?.({ version: 1, checkpoint, violations, ...observed }); fail("onboarding_work_before_acceptance"); } - assertRuns(snapshot, [parentId]); + return assertRuns(snapshot, [parentId]); }; let snapshot = await load(); @@ -141,12 +189,12 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom let proposal: Row | undefined; while (!proposal) { snapshot = await load(parent.id); - assertBeforeAcceptance(snapshot, parent.id, "awaiting-proposal"); + const workspacePending = assertBeforeAcceptance(snapshot, parent.id, "awaiting-proposal"); const pending = snapshot.interactions.filter(row => row.status === "pending"); const proposals = pending.filter(isProposal); if (proposals.length > 1 || pending.some(row => row.kind === "ask_user_questions" && row.id !== opening[0]!.id)) fail("onboarding_proposal_scope_unclear"); // Let the producing turn finish before acceptance; this journey does not test overlap. - if (proposals.length === 1 && seen.size && !activeRuns([...seen.values()]).length) proposal = proposals[0]; + if (proposals.length === 1 && seen.size && !workspacePending && ![...seen.values()].some(isPendingRun)) proposal = proposals[0]; else await wait(); } const target = proposal.payload?.target; @@ -178,9 +226,10 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom if (!child && currentParent.status === "done") fail("onboarding_completed_without_child"); if (child && !childId) { childId = child.id; await input.own({ kind: "issue", name: title, id: child.id }); } if (childId && child?.id !== childId) fail("onboarding_approved_child_mismatch"); - assertRuns(snapshot, [parent.id, ...(childId ? [childId] : [])]); + const workspacePending = assertRuns(snapshot, [parent.id, ...(childId ? [childId] : [])]); if (snapshot.attachments.some(row => row.issueId !== childId)) fail("onboarding_artifact_created_by_parent"); - const settled = child && currentParent.status === "done" && firstTaskCompletionSettled(snapshot.tasks, [parent.id], parent.id) && !activeRuns([...seen.values()]).length; + const settled = child && currentParent.status === "done" && firstTaskCompletionSettled(snapshot.tasks, [parent.id], parent.id) && + !workspacePending && ![...seen.values()].some(isPendingRun); if (settled) { const artifacts = snapshot.attachments.filter(row => row.issueId === child.id && row.originalFilename === "connection-proof.json"); if (artifacts.length !== 1) fail("onboarding_child_artifact_missing"); diff --git a/tests/runner-e2e/product-journeys/workspace-deferral.ts b/tests/runner-e2e/product-journeys/workspace-deferral.ts new file mode 100644 index 00000000000..2a831c25545 --- /dev/null +++ b/tests/runner-e2e/product-journeys/workspace-deferral.ts @@ -0,0 +1,37 @@ +type RecordValue = Record; +const object = (value: unknown): RecordValue => value !== null && typeof value === "object" && !Array.isArray(value) + ? value as RecordValue : {}; + +function hasProviderUsage(value: unknown): boolean { + if (value == null) return false; + if (typeof value !== "object" || Array.isArray(value)) return true; + return Object.entries(value).some(([key, entry]) => { + if (entry == null) return false; + if (["inputTokens", "cachedInputTokens", "outputTokens", "totalTokens", "costUsd", "costUsdExact"].includes(key)) return entry !== 0; + if (key === "accountingReceiptReady") return entry !== true; + // Assigned before provider dispatch; identifies the accounting scope, not usage. + if (key === "ledgerScope") return typeof entry !== "object" || Array.isArray(entry); + return true; + }); +} + +/** Admission may set startedAt before selecting a runtime. Only this explicit + * control-plane receipt, without contradictory provider evidence, proves no turn + * ran. A legacy default alone is never proof of either execution or nonexecution. */ +export function isProviderUnstartedWorkspaceDeferral(run: RecordValue): boolean { + const result = object(run.resultJson); + const recovery = object(result.executionRecovery); + const cancellation = object(result.cancellation); + const profile = object(run.runnerProfileJson); + return run.status === "cancelled" && run.errorCode === "workspace_busy" && + recovery.kind === "workspace_wait" && recovery.providerWorkStarted === false && + cancellation.source === "control_plane" && cancellation.expected === true && object(cancellation.initiator).type === "system" && + run.runtimeMode === "legacy" && run.runtimeModeResolvedAt === null && + ["nativeIssueId", "nativeSessionId", "nativePhase", "runnerInstanceId", "processPid", "processGroupId", "processStartedAt", + "sessionIdAfter", "externalRunId", "exitCode", "signal", "logStore", "logRef", "lastOutputAt", "lastOutputStream"].every(key => run[key] == null) && + run.lastOutputSeq === 0 && + (run.lastOutputBytes == null || run.lastOutputBytes === 0) && + profile.nativeExecutionInput == null && profile.sessionCheckpoint == null && + !hasProviderUsage(run.usageJson) && (run.costUsd == null || run.costUsd === 0) && + ["costUsd", "costUsdExact", "cost_usd", "total_cost_usd"].every(key => result[key] == null || result[key] === 0); +}