diff --git a/tests/runner-e2e/product-journeys/agent-evidence.test.ts b/tests/runner-e2e/product-journeys/agent-evidence.test.ts index 507018dd673..2a02beda846 100644 --- a/tests/runner-e2e/product-journeys/agent-evidence.test.ts +++ b/tests/runner-e2e/product-journeys/agent-evidence.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { fixtureReplyStoredOnce, validateAgentTaskState } from "./agent-flows.js"; +import { agentTaskSettled, fixtureReplyStoredOnce, startedTaskRuns, validateAgentTaskState } from "./agent-flows.js"; import { SLUGIFY_REVISION } from "../everyday-cases.js"; import { activeNativeTurnStarted, nativeCancellationHasNoLaterWork, nativeStopAcknowledged, nativeWorkEvents } from "../stop-redirect-flow.js"; import { parseTargetConfig } from "./catalog.js"; @@ -75,6 +75,48 @@ describe("deployed agent journey evidence", () => { s.taskRuns[0]!.companyId = "other"; expect(() => validateAgentTaskState(context, "owned-task", s, 3, ["run"])).toThrow(); }); + it("ignores only a bound wake the control plane retired before provider work started", () => { + // Observed on QA: an automation wake cancelled pre-start while completed runs awaited accounting. + const retired = (s: ReturnType) => { + (s.taskRuns as Array>).push({ id: "retired", companyId: "qa-company", agentId: s.taskRuns[0]!.agentId, issueId: "owned-task", status: "cancelled", + startedAt: null, contextSnapshot: {}, resultJson: { + executionRecovery: { kind: "accounting_wait", providerWorkStarted: false }, + cancellation: { source: "control_plane", expected: true, initiator: { type: "system" } } } }); + return s.taskRuns.at(-1)! as Record; + }; + const s = state(); retired(s); + expect(() => validateAgentTaskState(context, "owned-task", s, 3)).not.toThrow(); + expect(s.taskRuns).toHaveLength(2); + expect(() => validateAgentTaskState(context, "owned-task", s, 1)).toThrow("agent_task_binding_mismatch"); + const variants: Array<(run: Record) => void> = [ + run => { run.startedAt = "2026-11-05T19:17:00Z"; }, + run => { delete run.startedAt; }, + run => { run.status = "failed"; }, + run => { run.resultJson.cancellation.expected = false; }, + run => { run.resultJson.cancellation.source = "operator"; }, + run => { run.resultJson.executionRecovery.providerWorkStarted = true; }, + run => { delete run.resultJson.executionRecovery; }, + run => { run.agentId = "other"; }, + run => { run.issueId = "other"; }, + ]; + for (const change of variants) { const v = state(); change(retired(v)); expect(() => validateAgentTaskState(context, "owned-task", v, 3)).toThrow(); } + }); + it("settles a completed task without counting a wake retired before provider start", () => { + const run = (id: string, extra: Record = {}): Record => ({ id, status: "succeeded", startedAt: "2026-11-05T19:15:17Z", ...extra }); + const retired = run("retired", { status: "cancelled", startedAt: null, resultJson: { + executionRecovery: { kind: "accounting_wait", providerWorkStarted: false }, + cancellation: { source: "control_plane", expected: true, initiator: { type: "system" } } } }); + const task = (taskRuns: Array>, status = "done") => ({ currentIssue: { id: "owned-task", status }, + taskRuns, comments: [], interactions: [], attachments: [] }) as unknown as Parameters[0]; + expect(agentTaskSettled(task([run("first"), run("second"), retired]), 2)).toBe(true); + expect(startedTaskRuns(task([run("first"), run("second"), retired]))).toHaveLength(2); + // Negative controls: a started or unexpected extra run, missing work, open work or an unfinished task. + expect(agentTaskSettled(task([run("first"), run("second"), { ...retired, startedAt: "2026-11-05T19:17:19Z" }]), 2)).toBe(false); + expect(agentTaskSettled(task([run("first"), run("second"), { ...retired, resultJson: { ...retired.resultJson, cancellation: { ...retired.resultJson.cancellation, expected: false } } }]), 2)).toBe(false); + expect(agentTaskSettled(task([run("first"), retired]), 2)).toBe(false); + expect(agentTaskSettled(task([run("first"), run("second", { status: "running" })]), 2)).toBe(false); + expect(agentTaskSettled(task([run("first"), run("second"), retired], "in_progress"), 2)).toBe(false); + }); it("requires explicit paid run limits for a mixed deployment campaign", () => { const config = { version: 1, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", executionEnvironment: "daytona", journeys: ["task-create", "ask-question", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect"], diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index ea0f8f42214..c1883704e67 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -42,6 +42,29 @@ export function fixtureReplyStoredOnce(before: Row[], after: Row[], userId: stri return added.length === 1 && typeof added[0].body === "string" && added[0].body.replaceAll("\\_", "_") === message; } +/** + * A wake the control plane retired before admission, for example while completed + * runs await accounting. It never started provider work, so it has no runtime + * binding to grade. Started, unexpected or user cancellations do not qualify. + */ +export function retiredBeforeProviderStart(run: Row) { + const result = run.resultJson; + return run.status === "cancelled" && run.startedAt === null + && result?.cancellation?.source === "control_plane" && result.cancellation.expected === true + && result.executionRecovery?.providerWorkStarted === false; +} + +/** Runs that count toward a journey's expected work; retired wakes stay in the raw state and run limit. */ +export function startedTaskRuns(state: Pick) { + return state.taskRuns.filter(run => !retiredBeforeProviderStart(run)); +} + +/** The task is done after exactly `count` started runs, with nothing active or pending. */ +export function agentTaskSettled(state: TaskState, count: number) { + return state.currentIssue.status === "done" && startedTaskRuns(state).length === count && state.taskRuns.every(run => !active(run)) + && state.interactions.every(row => row.status !== "pending"); +} + /** Owned-task observations only: other work in a persistent QA company cannot affect this grade. */ export function validateAgentTaskState(context: JourneyContext, issueId: string, state: TaskState, maxRuns: number, allowedCancelled: string[] = []) { const agent = context.agent!; @@ -51,6 +74,8 @@ export function validateAgentTaskState(context: JourneyContext, issueId: string, const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; const boundIssueId = run.issueId ?? run.nativeIssueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId; if (run.companyId !== context.company.id || run.agentId !== agent.id || boundIssueId !== issueId) throw new JourneyError("failed", "agent_run_binding_mismatch"); + // The retired wake stays in the raw record and in the run limit above. + if (retiredBeforeProviderStart(run)) continue; if (!active(run) && (run.runtimeMode !== "native" || run.contextSnapshot?.paperclipEnvironment?.id !== context.environmentId || provider?.kind !== agent.provider || provider?.model !== agent.model)) throw new JourneyError("failed", "agent_run_binding_mismatch"); if (!["queued", "running", "succeeded"].includes(run.status) && !(run.status === "cancelled" && allowedCancelled.includes(run.id))) throw new JourneyError("failed", "agent_run_failed"); @@ -103,8 +128,7 @@ export async function runAgentJourney(id: AgentJourney, context: JourneyContext, return state; }; const settled = (issue: Row, count: number) => waitFor(options.deadlineAt, () => read(issue), state => - state.currentIssue.status === "done" && state.taskRuns.length === count && state.taskRuns.every(run => !active(run)) - && state.interactions.every(row => row.status !== "pending"), "agent_task_not_completed"); + agentTaskSettled(state, count), "agent_task_not_completed"); const finalReply = async (issue: Row, state: TaskState, marker?: string) => { const runs = state.taskRuns.filter(run => run.status === "succeeded"); if (!runs.length || runs.some(run => !Number.isFinite(Date.parse(run.finishedAt)))) throw new JourneyError("failed", "final_run_time_missing"); @@ -195,7 +219,7 @@ export async function runAgentJourney(id: AgentJourney, context: JourneyContext, const fixture = everydayTasks.find(task => task.id === `service-${phase}`)!; const issue = await create(`-${phase}`, `${fixture.buildPrompt(nonce)}\nUse only the connected service named "${service.name}". Do not use any other service.`); const pending = await waitFor(options.deadlineAt, () => read(issue), state => - state.taskRuns.length === 1 && !state.taskRuns.some(active) && state.interactions.some(row => row.status === "pending"), "protected_action_not_requested"); + startedTaskRuns(state).length === 1 && !state.taskRuns.some(active) && state.interactions.some(row => row.status === "pending"), "protected_action_not_requested"); check(`${phase}-waited-for-decision`, pending.currentIssue.status !== "done" && service.invocationCount() === callsBefore); await open(issue); await context.page.reload(); const decision = await decideProtectedActionThroughUi({ page: context.page, rows: pending.interactions as any, diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index 8edb160c5e0..baaa071a452 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -8,9 +8,11 @@ import { answerResumeQuestionThroughUi } from "../question-resume-ui.js"; import { createTaskThroughUi } from "../user-actions.js"; import { validateAgentTaskState } from "./agent-flows.js"; import { clarificationQuestionFailure, gradeClarificationEvidence, type ClarificationReadback } from "./clarification-evidence.js"; +import { openDocumentAuthoring } from "./task-document-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; +export const DOCUMENT_READBACK_MS = 120_000; interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } export async function waitForClarificationDocument(visible: Locator, answer: string, deadlineAt: number) { @@ -108,8 +110,13 @@ export async function runClarificationJourney(context: JourneyContext, session: const href = `/${encodeURIComponent(context.company.issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}#document-${encodeURIComponent(document.key)}`; await context.page.goto(new URL(href, context.api.origin).href, { waitUntil: "domcontentloaded" }); await context.page.reload({ waitUntil: "domcontentloaded" }); + // With the task chat shell and Artifacts V1, saved documents render in the + // properties pane's Artifacts tab, not the center column. Open it as a user would. + await openDocumentAuthoring(context); const visible = context.page.locator(`[id=${JSON.stringify(`document-${document.key}`)}]:visible, [id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`); - await waitForClarificationDocument(visible, scenario.answer, options.deadlineAt); + // The API already returned the saved document, so this is a bounded UI readback, + // not agent work time: a missing view must not hold the worker until the case deadline. + await waitForClarificationDocument(visible, scenario.answer, Math.min(options.deadlineAt, Date.now() + DOCUMENT_READBACK_MS)); const persisted = await context.api.get(`/api/issues/${issue.id}/documents/${encodeURIComponent(document.key)}`); readback = { issueId: persisted.issueId, key: persisted.key, revisionId: persisted.latestRevisionId, body: persisted.body, visibleBody: await visible.innerText() }; await options.capture("reloaded", "Saved answers after reload", "clarification-reloaded.png");