diff --git a/doc/evals.md b/doc/evals.md index 8f2ec714b82..37b160e28f1 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -137,6 +137,13 @@ It requires a fresh owned agent with no prior work runs and uses three provider Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. +The explicit [onboarding approval-history journey](../tests/runner-e2e/README.md#first-task-onboarding) +continues an interview through partial plan acceptance, approved hiring, and a +separate execution confirmation. It requires the assigned child's real execution, +saved output, unchanged approved scope, and reload persistence. Deterministic +PostgreSQL producer/consumer tests run in ordinary CI; native provider cells +require an explicit paid selection and a recorded live result. + The deployment-targeted [saved skill journey](../tests/runner-e2e/product-journeys/saved-skill-use.md) checks one company skill assigned by key, its exact admitted immutable version, a bound native source read/load, and a run-authored saved result from skill-only inputs. It uses the reusable Product journey target and remains unqualified until a real target run passes. The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) diff --git a/server/src/__tests__/onboarding-approval-history.test.ts b/server/src/__tests__/onboarding-approval-history.test.ts new file mode 100644 index 00000000000..2b942fc4b14 --- /dev/null +++ b/server/src/__tests__/onboarding-approval-history.test.ts @@ -0,0 +1,142 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { and, eq } from "drizzle-orm"; +import { activityLog, agents, companyMemberships, heartbeatRuns, issues, issueThreadInteractions } from "@paperclipai/db"; +import { createLocalAgentJwt } from "../agent-auth-jwt.js"; +import { documentService } from "../services/documents.js"; +import { initializeRunIdentity } from "../services/run-identity.js"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { recoveryPostgresSupport as support } from "./helpers/recovery-postgres.js"; +import { startRunnerApiTestServer } from "./helpers/runner-api-server.js"; + +if (!support.supported) console.warn(`Skipping onboarding approval history tests: ${support.reason}`); +describe.skipIf(!support.supported)("onboarding approval producer/consumer contract", () => { + let server: Awaited>; + let decisionUserId: string; + let cookie: string; + const originalSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET; + beforeAll(async () => { + process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID(); + server = await startRunnerApiTestServer({ humanSessions: true }); + const board = await server.fixture({ connectionScenario: "fresh", disableWakeOnDemand: true }); + decisionUserId = board.responsibleUserId!; + cookie = await server.humanSession(decisionUserId); + }, 60_000); + afterAll(async () => { + await server?.close(); + if (originalSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; + else process.env.PAPERCLIP_AGENT_JWT_SECRET = originalSecret; + }); + + type Fixture = Awaited>; + const selected = { id: "discovery", label: "Read-only discovery", description: "Write the architecture document." }; + const request = async (f: Fixture, kind: "confirmation" | "checkbox", key: string, payload: Record = {}) => { + const receipt = await f.authority.execute({ tool: "request_human_input", callId: randomUUID(), arguments: { + interactionKind: kind, idempotencyKey: key, title: key, prompt: key, continuationPolicy: "wake_assignee", payload: { version: 1, ...payload }, + } }) as { interaction: { id: string } }; + return receipt.interaction.id; + }; + const accept = async (f: Fixture, id: string, selectedOptionIds?: string[]) => { + const response = await fetch(`${server.apiUrl}/api/issues/${f.issueId}/interactions/${id}/accept`, { + method: "POST", headers: { cookie, "Content-Type": "application/json", origin: server.apiUrl }, + body: JSON.stringify(selectedOptionIds ? { selectedOptionIds } : {}), + }); + const data = await response.json(); + expect(response.status, JSON.stringify(data)).toBe(200); + expect(data).toMatchObject({ status: "accepted", resolvedByUserId: decisionUserId }); + }; + const approvedHistory = async (kind: "confirmation" | "checkbox") => { + const f = await server.fixture({ disableWakeOnDemand: true, connectionScenario: "fresh" }); + await server.db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", + principalId: decisionUserId, status: "active", membershipRole: "member" }); + await server.db.update(issues).set({ originKind: "onboarding_first_task" }).where(eq(issues.id, f.issueId)); + const plan = (await documentService(server.db).upsertIssueDocument({ issueId: f.issueId, key: "plan", format: "markdown", + body: "Step 1: read-only discovery and architecture document. Step 2: implementation needs separate approval.", + createdByAgentId: f.agentId, createdByRunId: f.runId })).document; + const planId = await request(f, kind, "Approve plan", { + target: { type: "issue_document", key: "plan", issueId: f.issueId, documentId: plan.id, revisionId: plan.latestRevisionId }, + ...(kind === "checkbox" ? { options: [selected, { id: "implementation", label: "Implement the application" }], + defaultSelectedOptionIds: [selected.id, "implementation"], minSelected: 1 } : {}), + }); + await accept(f, planId, kind === "checkbox" ? [selected.id] : undefined); + const hiringId = await request(f, "checkbox", "Hire a platform engineer", { + options: [{ id: "engineer", label: "Hire one Platform Engineer" }], minSelected: 1, defaultSelectedOptionIds: ["engineer"], + }); + await accept(f, hiringId, ["engineer"]); + // Agent creation is fixture setup here; the live journey exercises hiring. + const workerId = randomUUID(); + await server.db.insert(agents).values({ id: workerId, companyId: f.companyId, name: "Platform Engineer", + status: "active", adapterType: "paperclip_runner", runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: false } } }); + const executionId = await request(f, "confirmation", "Start Step 1: Discovery"); + return { f, plan, planId, hiringId, executionId, workerId }; + }; + + const resume = async (f: Fixture) => { + const runId = randomUUID(); + await server.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.runId)); + await server.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + responsibleUserId: f.responsibleUserId, status: "running", runtimeMode: "native", nativeIssueId: f.issueId, + invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId: f.issueId } }); + await server.db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, f.issueId)); + await initializeRunIdentity(server.db, { companyId: f.companyId, runId, issueId: f.issueId, + responsibleUserId: f.responsibleUserId!, cause: "instruction" }); + return { ...f, runId, authority: new PaperclipRunnerToolAuthority(server.db, { ...f, runId }) }; + }; + + for (const kind of ["confirmation", "checkbox"] as const) { + it.each(["native", "REST issue", "REST child", "decomposition"] as const)( + `${kind} plan → hiring → separate execution approval → %s, including later-run context`, async path => { + const history = await approvedHistory(kind); + const { plan, workerId } = history; + let f = history.f; + const nativeInput = { tool: "create_task", callId: randomUUID(), arguments: { + title: "Step 1: Discovery and architecture", assigneeActorId: workerId, status: "todo", idempotencyKey: "step-1", + } }; + const before = await f.snapshot(); + // Plan and hiring acceptance alone cannot answer a separate pending execution request. + await expect(f.authority.execute(nativeInput)).rejects.toMatchObject({ status: 403 }); + expect((await f.snapshot()).issues).toEqual(before.issues); + await accept(f, history.executionId); + f = await resume(f); + const context = await f.authority.execute({ tool: "get_task_context", callId: randomUUID(), arguments: {} }) as { + acceptedPlan: { revisionId: string; approvedOptions?: unknown[] }; + }; + expect(context.acceptedPlan).toMatchObject({ revisionId: plan.latestRevisionId, markdown: plan.body }); + if (kind === "checkbox") expect(context.acceptedPlan.approvedOptions).toEqual([selected]); + else expect(context.acceptedPlan).not.toHaveProperty("approvedOptions"); + + const token = createLocalAgentJwt(f.agentId, f.companyId, "paperclip_runner", f.runId, f.responsibleUserId)!; + const post = async () => { + const route = path === "decomposition" ? `/api/issues/${f.issueId}/accepted-plan-decompositions` + : path === "REST child" ? `/api/issues/${f.issueId}/children` : `/api/companies/${f.companyId}/issues`; + const child = { title: "Step 1: Discovery and architecture", status: "backlog", workMode: "standard", priority: "medium", assigneeAgentId: workerId }; + const body = path === "decomposition" ? { acceptedPlanRevisionId: plan.latestRevisionId, children: [child] } + : path === "REST child" ? child : { ...child, parentId: f.issueId, idempotencyKey: "step-1" }; + const response = await fetch(`${server.apiUrl}${route}`, { method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, body: JSON.stringify(body) }); + const data = await response.json(); + expect(response.ok, JSON.stringify(data)).toBe(true); + return path === "decomposition" ? data.childIssueIds[0] as string : (data.issue?.id ?? data.id) as string; + }; + let childId: string; + if (path === "native") { + const created = await f.authority.execute(nativeInput) as { task: { id: string }; scheduledWakeIds: string[] }; + childId = created.task.id; + expect(created.scheduledWakeIds).toEqual([`created-child:${childId}`]); + const reconnected = new PaperclipRunnerToolAuthority(server.db, f); + expect(await reconnected.execute({ ...nativeInput, callId: randomUUID() })).toEqual(created); + } else { + childId = await post(); + // The child REST schema has no idempotency key; replay belongs to the other entry points. + if (path !== "REST child") expect(await post()).toBe(childId); + } + const children = await server.db.select().from(issues).where(and(eq(issues.companyId, f.companyId), eq(issues.parentId, f.issueId))); + expect(children).toMatchObject([{ id: childId, assigneeAgentId: workerId, title: "Step 1: Discovery and architecture" }]); + const creationEvents = await server.db.select().from(activityLog).where(and(eq(activityLog.companyId, f.companyId), eq(activityLog.entityId, childId))); + expect(creationEvents.filter(event => ["issue.created", "issue.child_created"].includes(event.action))).toHaveLength(1); + const [saved] = await server.db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, history.planId)); + expect(saved.status).toBe("accepted"); + if (kind === "checkbox") expect(saved.result).toMatchObject({ selectedOptionIds: [selected.id] }); + }); + } +}); diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index ce803748eb6..ece2625998f 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -69,6 +69,16 @@ retain their original definition and do not qualify the added Plan cell. ## Suites and matrices +`onboarding-approval-history` uses the existing first-task fixture registry path +with the production wizard and native Codex/ACPX Claude profiles. Its explicit +`interview-hire-execute` case has seven expected work turns, one attempt, a $5 +company task budget, twelve total run records, and a fifteen-minute deadline. +The model creates its plan, approval cards, writer, child task, and saved output +through production tools; the harness answers the interview and clicks normal +approval controls. Fixture code does not insert approval rows or hire the writer. +The ordinary first-task cleanup owns the complete disposable company, including +the new writer. See [the journey contract](README.md#first-task-onboarding). + A `RunnerSuiteFixture` declares one durable testing purpose: stable ID, label, description, profiles, environments, cases, expected size, and definition or ranking metadata. Its execution IDs are globally prefixed as diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 34f3183846b..61f3e8e9494 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1763,6 +1763,39 @@ Revision delivery checks exclude preserved originals by their content hash, even ## First-task onboarding +The explicit-only `onboarding-approval-history` suite extends the interview path +through three separate human decisions: a saved plan checkbox selection, a +writer hiring checkbox, and an untargeted Start drafting confirmation. The user +selects drafting and clears publishing. The model must hire the approved writer, +assign one child, and produce the saved welcome document through that child's +own execution. Persisted decisions, the unchanged plan revision and selection, +creation timestamps, one child work run, and readback after browser reload own +the grade. The exact saved output revision must name the writer and its actual +child run as author. A successful parent run or lead-authored output fails. + +It has one local cell each for native Codex and ACPX Claude. It reuses the +production wizard, first-task fixtures, $5 company task budget, twelve-run +maximum, fifteen-minute attempt deadline, five-minute response waits, screenshots, +private diagnostics, sanitized public evidence, and disposable-company cleanup. +Expect seven provider work turns, including the writer's child. The outer +runner accepts one to seven parent heartbeat records; it does not count the +child as a parent run. The oracle independently requires all three human +decisions and exactly one writer execution. Setup probes are accounted separately. +The case permits one attempt and is excluded from `--all`. Run it explicitly when +qualifying changes to onboarding or approval semantics: + +```sh +pnpm test:e2e:runner -- --list --suite onboarding-approval-history +pnpm test:e2e:runner -- --id onboarding-approval-history.runner-codex.local.interview-hire-execute +``` + +The deterministic PostgreSQL contract in +`server/src/__tests__/onboarding-approval-history.test.ts` runs in the ordinary +Vitest suite. It uses native-produced cards, authenticated human acceptance, +and a later native run to check standard and checkbox plan approvals across +native creation, both REST creation routes, decomposition, context, and replay. +Journey discovery and grader calibration do not establish a live provider pass. + `first-task` is a suite in the main Runner E2E catalog. A full `pnpm test:e2e:runner -- --all` run (or an unfiltered full GitHub Actions campaign) includes its 52 executions alongside the other suites in one shared dashboard, diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 1c6969ebcf7..0922095aad6 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,17 @@ # Runner E2E security for a public repository +The explicit `onboarding-approval-history` suite uses only synthetic garden-club +facts in an owned disposable onboarding company. The human selects drafting and +clears publishing; the scenario forbids external publication. The hired writer +must use the lead's existing provider/AI connection through normal hiring. +Credentials remain in the existing encrypted fixture path. It makes no direct +database writes and changes no shared company. Public first-task projection +retains synthetic checkbox option IDs/labels, selected IDs, plan document/revision +IDs, task/agent/run bindings, and the exact saved output revision's agent/run +authorship so the approval history can be regraded. Provider +session IDs, account configuration, raw traces, and full instructions retain +their existing private-only treatment. Registration is separate from live proof. + The deployment-targeted `saved-skill-use` journey creates one synthetic skill on the campaign-owned agent. Its random receipt reference and numeric inputs occur only in the saved skill source. Exact source, immutable version, ordinary assignment, native read diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6a37e2ab92f..d8706ffd9f1 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -329,11 +329,11 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 2, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(822); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(822); + expect(validateRunnerCatalog()).toHaveLength(824); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(824); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index e9a8f8ad6af..14889bb6006 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -44,7 +44,8 @@ import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; import { REMOTE_WORKSPACE_STORAGE_SUITE, remoteWorkspaceStoragePrompt } from "./remote-workspace-storage.js"; import { CONNECTION_GUIDANCE_SUITE, CONNECTION_GUIDANCE_BUDGET_CENTS, connectionGuidanceTasks, connectionGuidanceDefinitionDigest } from "./connection-guidance-cases.js"; -import { firstTaskTasks } from "./first-task-cases.js"; +import { firstTaskTasks, onboardingApprovalHistoryTask } from "./first-task-cases.js"; +import { APPROVAL_HISTORY_SUITE, approvalHistoryDefinitionDigest } from "./first-task-approval-history.js"; import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; @@ -1785,6 +1786,18 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ paidCalls: "one skill turn, two ordered-comment turns, three chat turns per profile", }, }, + { + id: APPROVAL_HISTORY_SUITE, label: "Onboarding approval history", manualOnly: true, + description: "Interview, partial plan approval, approved hiring, a separate execution confirmation, and the writer’s saved result.", + groups: ["onboarding", "native"], + profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)), + environments: [localEnvironment], tasks: [onboardingApprovalHistoryTask], expectedMatrixSize: 2, + definitionMetadata: { version: 1, sourceDigest: approvalHistoryDefinitionDigest(), scheduling: "explicit-only", + automaticRetryPolicy: "single_attempt", maximumAttemptsPerCell: 1, expectedWorkRuns: 7, + companyTaskBudgetCents: 500, maximumCompanyRuns: 12, attemptTimeoutMs: 900_000, + instructions: "production-wizard-and-first-task", approvalSequence: "plan-checkbox-subset/hiring-checkbox/execution-confirmation", + oracle: "persisted-human-decisions/current-plan-revision/assigned-child/one-worker-execution/saved-document/reload" }, + }, { id: "first-task", label: "First-task onboarding", description: "Production onboarding, first replies, approval, and durable task execution.", diff --git a/tests/runner-e2e/first-task-approval-history-flow.ts b/tests/runner-e2e/first-task-approval-history-flow.ts new file mode 100644 index 00000000000..a5d7d9be184 --- /dev/null +++ b/tests/runner-e2e/first-task-approval-history-flow.ts @@ -0,0 +1,51 @@ +import { expect, type Page } from "@playwright/test"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { DRAFT_OPTION, PUBLISH_OPTION, approvalHistoryWriter } from "./first-task-approval-history.js"; +import type { FirstTaskCheckpoint, Row } from "./first-task-scoring.js"; + +/** The model produces every proposal and hire; the harness only operates ordinary approval controls. */ +export async function runApprovalHistoryDecisions(input: { + page: Page; api: Pick; issueId: string; marker: string; deadlineAt: number; + allRuns: () => Promise; + snapshot: (phase: FirstTaskCheckpoint["phase"], at?: string) => Promise; + settle: (prior: Set, completion?: boolean) => Promise; + capture: (id: string, label: string, file: string) => Promise; +}) { + const cards = () => input.api.get(`/api/issues/${input.issueId}/interactions`); + const approve = async (card: Row, phase: FirstTaskCheckpoint["phase"]) => { + const before = new Set((await input.allRuns()).map(run => run.id)); + const at = new Date().toISOString(); + await input.page.getByRole("button", { name: card.payload.acceptLabel + ?? (card.kind === "request_confirmation" ? "Approve" : "Confirm selection"), exact: true }).last().click(); + await pollUntil({ label: `saved ${phase} decision`, deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000), intervalMs: 250, + load: cards, accept: rows => rows.some(row => row.id === card.id && row.status === "accepted" && row.resolvedByUserId), + }); + await input.snapshot(phase, at); + await input.settle(before, phase === "accepted"); + }; + const pending = async (predicate: (row: Row) => boolean) => { + const found = (await cards()).filter(row => row.status === "pending" && predicate(row)); + expect(found, "One saved approval for this stage must exist").toHaveLength(1); + return found[0]!; + }; + const plan = await pending(row => row.kind === "request_checkbox_confirmation" && row.payload?.target?.key === "plan"); + expect(plan.payload.options.map((option: Row) => option.label).sort()).toEqual([DRAFT_OPTION, PUBLISH_OPTION].sort()); + await input.page.getByRole("checkbox", { name: DRAFT_OPTION, exact: true }).last().check(); + await input.page.getByRole("checkbox", { name: PUBLISH_OPTION, exact: true }).last().uncheck(); + await approve(plan, "plan-accepted"); + await input.snapshot("hiring-proposed"); + await input.capture("hiring-proposed", "Hiring waits for approval", "hiring-proposed.png"); + const writer = approvalHistoryWriter(input.marker); + const hiring = await pending(row => row.kind === "request_checkbox_confirmation" && !row.payload?.target + && row.payload?.options?.length === 1 && row.payload.options[0].label.includes(writer)); + await input.page.getByRole("checkbox", { name: hiring.payload.options[0].label, exact: true }).last().check(); + await approve(hiring, "hiring-accepted"); + await input.snapshot("execution-proposed"); + await input.capture("execution-proposed", "Execution waits for Start drafting", "execution-proposed.png"); + const execution = await pending(row => row.kind === "request_confirmation" && !row.payload?.target + && /start drafting/i.test(row.payload?.prompt ?? "")); + await approve(execution, "accepted"); + await input.snapshot("finished"); + await input.page.reload({ waitUntil: "domcontentloaded" }); + await input.snapshot("finished"); +} diff --git a/tests/runner-e2e/first-task-approval-history.test.ts b/tests/runner-e2e/first-task-approval-history.test.ts new file mode 100644 index 00000000000..3fbfeef4797 --- /dev/null +++ b/tests/runner-e2e/first-task-approval-history.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Page } from "@playwright/test"; +import { runnerMatrix } from "./catalog.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { firstTaskScenario } from "./first-task-cases.js"; +import { APPROVAL_HISTORY_CASE, APPROVAL_HISTORY_SUITE, DRAFT_OPTION, PUBLISH_OPTION, approvalHistoryWriter, readApprovalHistoryRevision } from "./first-task-approval-history.js"; +import { gradeFirstTask, snapshotInstruction, type FirstTaskCheckpoint, type FirstTaskEvidence, type Row } from "./first-task-scoring.js"; +import { projectFirstTaskResult } from "./first-task-flow.js"; +import { runApprovalHistoryDecisions } from "./first-task-approval-history-flow.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import type { RunnerE2EResult } from "./types.js"; +import { matchesRunCount } from "./run-count.js"; + +function recording(): FirstTaskEvidence { + const { marker } = firstTaskScenario(APPROVAL_HISTORY_CASE, "history"); + const at = (minute: number) => `2026-11-01T00:${String(minute).padStart(2, "0")}:00Z`; + const parent = { id: "parent", title: "First task", status: "in_review", description: "/first-task" }; + const plan = { id: "plan", issueId: "parent", key: "plan", title: "Plan", body: `Draft then publish ${marker}`, latestRevisionId: "revision" }; + const planCard = { id: "plan-card", kind: "request_checkbox_confirmation", status: "pending", createdAt: at(2), payload: { + options: [{ id: "draft", label: DRAFT_OPTION }, { id: "publish", label: PUBLISH_OPTION }], defaultSelectedOptionIds: ["draft", "publish"], + target: { type: "issue_document", key: "plan", documentId: "plan", revisionId: "revision" }, + } }; + const hireCard = { id: "hire-card", kind: "request_checkbox_confirmation", status: "pending", createdAt: at(4), + payload: { options: [{ id: "writer", label: `Hire ${approvalHistoryWriter(marker)}` }] } }; + const executionCard = { id: "execution-card", kind: "request_confirmation", status: "pending", createdAt: at(6), payload: { prompt: "Start drafting" } }; + const resolve = (card: Row, minute: number, ids?: string[]) => ({ ...card, status: "accepted", resolvedByUserId: "human", resolvedAt: at(minute), + result: { outcome: "accepted", ...(ids ? { selectedOptionIds: ids } : {}) } }); + const checkpoints: FirstTaskCheckpoint[] = []; + const checkpoint = (phase: FirstTaskCheckpoint["phase"], minute: number, updates: Partial = {}) => { + const value = structuredClone({ id: `${phase}-${minute}`, phase, at: at(minute), issueId: parent.id, + tasks: [parent], agents: [{ id: "lead" }], comments: [], interactions: [], documents: [], runs: [], + ...checkpoints.at(-1), ...updates }); + Object.assign(value, { id: `${phase}-${minute}`, phase, at: at(minute) }); + checkpoints.push(value); + return value; + }; + checkpoint("opening", 0); + checkpoint("response", 1, { comments: [{ id: "interview-comment", authorAgentId: "lead", body: "Tell me about the club." }], + interactions: [{ id: "interview", kind: "ask_user_questions", status: "answered", payload: { + questions: ["goal", "audience", "done"].map(id => ({ id, answerMode: "text", prompt: id })), + } }], runs: [{ id: "interview-run", status: "succeeded" }] }); + checkpoint("clarified", 2, { interactions: [planCard], documents: [plan] }); + checkpoint("plan-accepted", 3, { interactions: [resolve(planCard, 3, ["draft"])] }); + checkpoint("hiring-proposed", 4, { interactions: [resolve(planCard, 3, ["draft"]), hireCard] }); + checkpoint("hiring-accepted", 5, { interactions: [resolve(planCard, 3, ["draft"]), resolve(hireCard, 5, ["writer"])] }); + checkpoint("execution-proposed", 6, { agents: [{ id: "lead" }, { id: "writer", name: approvalHistoryWriter(marker), createdAt: at(5) }], + interactions: [resolve(planCard, 3, ["draft"]), resolve(hireCard, 5, ["writer"]), executionCard] }); + checkpoint("accepted", 7, { interactions: [resolve(planCard, 3, ["draft"]), resolve(hireCard, 5, ["writer"]), resolve(executionCard, 7)] }); + checkpoint("finished", 8, { tasks: [{ ...parent, status: "done" }, { id: "child", parentId: "parent", assigneeAgentId: "writer", status: "done", createdAt: at(7) }], + documents: [plan, { id: "welcome", issueId: "child", key: "welcome", latestRevisionId: "welcome-revision", + body: `Welcome, beginners. Join our free Saturday meetup ${marker}.`, savedRevision: { id: "welcome-revision", documentId: "welcome", + issueId: "child", key: "welcome", body: `Welcome, beginners. Join our free Saturday meetup ${marker}.`, + createdByAgentId: "writer", createdByRunId: "child-run", createdByUserId: null } }], + runs: [{ id: "parent-run", nativeIssueId: "parent", agentId: "lead", status: "succeeded", startedAt: at(7) }, + { id: "child-run", nativeIssueId: "child", agentId: "writer", status: "succeeded", runtimeMode: "native", startedAt: at(7) }] }); + checkpoint("finished", 9); + return { caseId: APPROVAL_HISTORY_CASE, nonce: "history", onboardingIssueId: "parent", agentId: "lead", initialTaskIds: ["parent"], + configuredModel: "fixture-model", observedModels: ["fixture-model"], instructions: [snapshotInstruction("AGENTS.md", "Persona"), + snapshotInstruction("first-task/SKILL.md", "First-task instructions")], checkpoints, checks: [] }; +} +const failures = (e: FirstTaskEvidence) => gradeFirstTask(e).filter(check => !check.passed).map(check => check.id); + +describe("onboarding approval history journey", () => { + it("registers two explicit native cells without changing the default paid campaign", () => { + const cells = selectRunnerExecutions(parseRunnerSelectors(["--suite", APPROVAL_HISTORY_SUITE])); + expect(cells.map(cell => cell.id)).toEqual([`onboarding-approval-history.runner-codex.local.${APPROVAL_HISTORY_CASE}`, + `onboarding-approval-history.runner-acpx-claude.local.${APPROVAL_HISTORY_CASE}`]); + expect(cells.every(cell => cell.task.automaticRetryPolicy === "single_attempt" && cell.task.expectedRunCount === 7)).toBe(true); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => cell.suite.id === APPROVAL_HISTORY_SUITE)).toBe(false); + expect(runnerMatrix.filter(cell => cell.suite.id === "first-task")).toHaveLength(52); + }); + + it("accepts the complete interview, partial plan approval, hiring, execution, and reload recording", () => { + expect(failures(recording())).toEqual([]); + }); + + it.each(runnerMatrix.filter(cell => cell.suite.id === APPROVAL_HISTORY_SUITE))( + "completed evidence satisfies the outer runner's status and parent-run contract for $id", execution => { + const evidence = recording(); + // Seven work turns include the writer: only six belong to the parent. + const leadRuns = ["interview-run", "plan-run", "hiring-request-run", "hire-run", "create-child-run", "parent-run"].map((id, index) => ({ + id, nativeIssueId: evidence.onboardingIssueId, agentId: evidence.agentId, status: "succeeded", runtimeMode: "native", + contextSnapshot: { issueId: evidence.onboardingIssueId }, startedAt: `2026-11-01T00:0${[1, 2, 3, 5, 7, 8][index]}:00Z`, + })); + for (const checkpoint of evidence.checkpoints.filter(c => c.phase === "finished")) { + checkpoint.runs = [...leadRuns, ...checkpoint.runs.filter(run => run.agentId !== evidence.agentId)]; + } + expect(failures(evidence)).toEqual([]); + const finished = evidence.checkpoints.at(-1)!; + const parent = finished.tasks.find(task => task.id === evidence.onboardingIssueId)!; + const parentRuns = finished.runs.filter(run => run.nativeIssueId === parent.id); + expect(parent.status).toBe(execution.task.expectedTerminalState.issue); + expect(parentRuns).not.toHaveLength(0); + expect(parentRuns.every(run => run.status === execution.task.expectedTerminalState.run)).toBe(true); + expect(matchesRunCount(execution.task, parentRuns.length)).toBe(true); + expect(matchesRunCount(execution.task, 0)).toBe(false); + expect(matchesRunCount(execution.task, execution.task.expectedRunCount + 1)).toBe(false); + }); + + it("rejects a completed writer task when the onboarding parent remains in review", () => { + const evidence = recording(); + for (const checkpoint of evidence.checkpoints.filter(c => c.phase === "finished")) { + checkpoint.tasks.find(task => task.id === evidence.onboardingIssueId)!.status = "in_review"; + } + expect(failures(evidence)).toContain("reload-retains-completion"); + }); + + it("reads the exact saved document revision through its production API", async () => { + const document = recording().checkpoints.at(-1)!.documents[1]; + const get = vi.fn(async (_path: string) => document.savedRevision); + const read = await readApprovalHistoryRevision({ get: async (path: string) => await get(path) as T }, document); + expect(get).toHaveBeenCalledExactlyOnceWith("/api/issues/child/documents/welcome/revisions/welcome-revision"); + expect(read.savedRevision).toMatchObject({ createdByAgentId: "writer", createdByRunId: "child-run" }); + }); + + it.each(["id", "documentId", "issueId", "key", "body"])("rejects a mismatched saved revision %s", async field => { + const document = recording().checkpoints.at(-1)!.documents[1]; + const get = async () => ({ ...document.savedRevision, [field]: "foreign" }) as T; + await expect(readApprovalHistoryRevision({ get }, document)).rejects.toThrow("onboarding_saved_revision_binding_mismatch"); + }); + + it("clears the unchecked plan step and operates all three saved approval controls before reload", async () => { + const evidence = recording(); + const events: string[] = []; + const clickedAt: number[] = []; + const state = (phase: FirstTaskCheckpoint["phase"]) => structuredClone(evidence.checkpoints.find(c => c.phase === phase)!); + let current = state("clarified"); + const selections = new Set(["draft", "publish"]); + const page = { getByRole: (_role: string, { name }: { name: string }) => ({ last: () => ({ + check: async () => { events.push(`check:${name}`); if (name === DRAFT_OPTION) selections.add("draft"); }, + uncheck: async () => { events.push(`uncheck:${name}`); selections.delete("publish"); }, + click: async () => { + clickedAt.push(Date.now()); + // Work can begin while the acceptance response is still in flight. + vi.setSystemTime(Date.now() + 5_000); + const pending = current.interactions.find(card => card.status === "pending")!; + events.push(`accept:${pending.id}`); + current = state(pending.id === "plan-card" ? "plan-accepted" : pending.id === "hire-card" ? "hiring-accepted" : "accepted"); + if (pending.id === "plan-card") current.interactions[0].result.selectedOptionIds = [...selections]; + }, + }) }), reload: async () => { events.push("reload"); } } as unknown as Page; + const snapshot = vi.fn(async (phase: FirstTaskCheckpoint["phase"], _at?: string) => { events.push(phase); return structuredClone(current); }); + vi.useFakeTimers({ toFake: ["Date"] }); + try { + await runApprovalHistoryDecisions({ page, api: { get: async () => structuredClone(current.interactions) as T }, issueId: "parent", + marker: firstTaskScenario(APPROVAL_HISTORY_CASE, "history").marker, deadlineAt: Date.now() + 30_000, + allRuns: async () => current.runs, snapshot, capture: vi.fn(async () => undefined), + settle: async (_runs, completion) => { + current = state(completion ? "finished" : current.phase === "plan-accepted" ? "hiring-proposed" : "execution-proposed"); + }, + }); + expect(snapshot.mock.calls.filter(([phase]) => ["plan-accepted", "hiring-accepted", "accepted"].includes(phase)).map(([, at]) => at)) + .toEqual(clickedAt.map(at => new Date(at).toISOString())); + } finally { vi.useRealTimers(); } + expect([...selections]).toEqual(["draft"]); + expect(events).toEqual([`check:${DRAFT_OPTION}`, `uncheck:${PUBLISH_OPTION}`, "accept:plan-card", "plan-accepted", "hiring-proposed", + `check:Hire ${approvalHistoryWriter(firstTaskScenario(APPROVAL_HISTORY_CASE, "history").marker)}`, "accept:hire-card", "hiring-accepted", + "execution-proposed", "accept:execution-card", "accepted", "finished", "reload", "finished"]); + }); + + it.each void, string]>([ + ["approved run without a child", e => { e.checkpoints.at(-1)!.tasks.pop(); }, "approved-writer-executes-once"], + ["missing execution decision", e => { e.checkpoints.at(-1)!.interactions.pop(); }, "three-separate-human-approvals"], + ["agent-only execution acceptance", e => { e.checkpoints.at(-1)!.interactions.at(-1)!.resolvedByAgentId = "lead"; }, "three-separate-human-approvals"], + ["hiring accepted with no writer selected", e => { e.checkpoints.at(-1)!.interactions[1].result.selectedOptionIds = []; }, "three-separate-human-approvals"], + ["execution confirmation targeted to the plan", e => { e.checkpoints.at(-1)!.interactions.at(-1)!.payload.target = { key: "plan" }; }, "three-separate-human-approvals"], + ["new task before execution approval", e => { e.checkpoints.find(c => c.phase === "execution-proposed")!.tasks.push({ id: "early" }); }, "no-premature-work"], + ["hire before hiring approval", e => { e.checkpoints.find(c => c.phase === "hiring-proposed")!.agents.push({ id: "early" }); }, "no-premature-work"], + ["defaults substituted for actual choices", e => { e.checkpoints.at(-1)!.interactions[0].result.selectedOptionIds = ["draft", "publish"]; }, "selected-plan-scope-retained"], + ["changed plan revision", e => { e.checkpoints.at(-1)!.documents[0].latestRevisionId = "changed"; }, "selected-plan-scope-retained"], + ["task assigned to the lead", e => { e.checkpoints.at(-1)!.tasks[1].assigneeAgentId = "lead"; }, "approved-writer-executes-once"], + ["duplicate child execution", e => { e.checkpoints.at(-1)!.runs.push({ ...e.checkpoints.at(-1)!.runs[1], id: "duplicate" }); }, "approved-writer-executes-once"], + ["no saved output after successful runs", e => { e.checkpoints.at(-1)!.documents.pop(); }, "durable-completion"], + ["no persisted output after reload", e => { e.checkpoints.at(-1)!.documents.pop(); }, "reload-retains-completion"], + ["lead-authored writer output", e => { e.checkpoints.at(-1)!.documents[1].savedRevision.createdByAgentId = "lead"; }, "writer-authored-output"], + ["output from the wrong child run", e => { e.checkpoints.at(-1)!.documents[1].savedRevision.createdByRunId = "parent-run"; }, "writer-authored-output"], + ["missing saved revision proof", e => { delete e.checkpoints.at(-1)!.documents[1].savedRevision; }, "writer-authored-output"], + ["saved revision body mismatch", e => { e.checkpoints.at(-1)!.documents[1].savedRevision.body = "Different output"; }, "writer-authored-output"], + ["output without a current revision reference", e => { delete e.checkpoints.at(-1)!.documents[1].latestRevisionId; }, "writer-authored-output"], + ["writer switched to legacy execution", e => { e.checkpoints.at(-1)!.runs[1].runtimeMode = "legacy"; }, "writer-authored-output"], + ["changed saved output after reload", e => { e.checkpoints.at(-1)!.documents[1].body += " Changed."; }, "reload-retains-completion"], + ["duplicate approval left pending", e => { e.checkpoints.at(-1)!.interactions.push({ id: "duplicate", status: "pending", kind: "request_confirmation" }); }, "no-pending-approvals"], + ["missing intermediate approval observations", e => { e.checkpoints = e.checkpoints.filter(c => c.phase !== "hiring-accepted"); }, "approval-history-stages"], + ])("rejects %s", (_name, mutate, expected) => { + const evidence = recording(); + mutate(evidence); + expect(failures(evidence)).toContain(expected); + }); + + it("retains enough public evidence to regrade actual checkbox choices and the later child run", () => { + const evidence = recording(); + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", suiteId: APPROVAL_HISTORY_SUITE, + executionId: `${APPROVAL_HISTORY_SUITE}.runner-codex.local.${APPROVAL_HISTORY_CASE}`, attempt: 1, status: "passed", + profileId: "runner-codex", environmentId: "local", caseId: APPROVAL_HISTORY_CASE, provider: "openai", model: "fixture-model", + runtimeMode: "native", runIds: ["parent-run", "child-run"], startedAt: "2026-11-01T00:00:00Z", finishedAt: "2026-11-01T00:09:00Z", + durationMs: 540_000, cleanup: "passed", firstTask: evidence }; + const projected = projectFirstTaskResult(result); + validateRetainedRunnerResult(projected); + expect(failures(projected.firstTask!)).toEqual([]); + expect(projected.firstTask!.checkpoints.at(-1)!.interactions[0].result.selectedOptionIds).toEqual(["draft"]); + }); +}); diff --git a/tests/runner-e2e/first-task-approval-history.ts b/tests/runner-e2e/first-task-approval-history.ts new file mode 100644 index 00000000000..6393768c7dc --- /dev/null +++ b/tests/runner-e2e/first-task-approval-history.ts @@ -0,0 +1,121 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerApi } from "./api.js"; +import type { FirstTaskCheck, FirstTaskCheckpoint, FirstTaskEvidence, Row } from "./first-task-scoring.js"; + +export const APPROVAL_HISTORY_SUITE = "onboarding-approval-history"; +export const APPROVAL_HISTORY_CASE = "interview-hire-execute"; +export const DRAFT_OPTION = "Draft welcome note"; +export const PUBLISH_OPTION = "Publish welcome note"; +export const approvalHistoryWriter = (marker: string) => `Garden Writer ${marker}`; +export const approvalHistoryFacts = (marker: string) => [ + "We run a neighborhood garden club for beginners with a free Saturday meetup.", + `Our first deliverable is a two-sentence welcome note saved as a task document, including ${marker}.`, + `Please save a two-step plan and ask me to review it with exactly two checkboxes labeled "${DRAFT_OPTION}" and "${PUBLISH_OPTION}". Initially select both; I will choose the scope.`, + "Plan acceptance authorizes only my selected steps. Do not hire or create a task yet.", + `After plan acceptance, propose hiring exactly one writer named "${approvalHistoryWriter(marker)}" with a separate saved checkbox confirmation. Wait for that approval before hiring.`, + "Use the same provider, model, and AI connection as yourself for the writer. No outside services or additional agents are needed.", + 'After hiring, ask me a separate saved ordinary confirmation labeled "Start drafting" before creating the writer’s task. This is an execution confirmation; keep the saved plan revision unchanged.', + "After I accept Start drafting, assign exactly one child task to that writer, wait for its execution, verify the saved welcome document, and finish this onboarding task. Never publish the note externally.", +].join(" "); + +export function approvalHistoryDefinitionDigest() { + const hash = createHash("sha256"); + for (const file of ["first-task-approval-history.ts", "first-task-approval-history-flow.ts", "first-task-cases.ts", "first-task-flow.ts", "first-task-scoring.ts", "first-task-fixtures.ts"]) { + hash.update(file).update("\0").update(readFileSync(new URL(file, import.meta.url))).update("\0"); + } + return hash.digest("hex"); +} + +/** Read the exact immutable revision observed on the document, including its actual writer/run. */ +export async function readApprovalHistoryRevision(api: Pick, document: Row): Promise { + if (typeof document.latestRevisionId !== "string" || !document.latestRevisionId) throw new Error("onboarding_saved_revision_missing"); + const savedRevision = await api.get(`/api/issues/${document.issueId}/documents/${encodeURIComponent(document.key)}/revisions/${document.latestRevisionId}`); + if (!savedRevision || savedRevision.id !== document.latestRevisionId || savedRevision.documentId !== document.id + || savedRevision.issueId !== document.issueId || savedRevision.key !== document.key || savedRevision.body !== document.body) { + throw new Error("onboarding_saved_revision_binding_mismatch"); + } + return { ...document, savedRevision }; +} + +const acceptedByHuman = (card: Row | undefined) => Boolean(card?.status === "accepted" && card.result?.outcome === "accepted" + && card.resolvedByUserId && !card.resolvedByAgentId && !card.resolvedByRunId && Number.isFinite(Date.parse(card.resolvedAt))); +const planCard = (c: FirstTaskCheckpoint) => c.interactions.find(card => card.kind === "request_checkbox_confirmation" + && card.payload?.target?.type === "issue_document" && card.payload.target.key === "plan"); +const hiringCard = (c: FirstTaskCheckpoint, writer: string) => c.interactions.find(card => card.kind === "request_checkbox_confirmation" + && !card.payload?.target && card.payload?.options?.length === 1 && card.payload.options[0].label.includes(writer)); +const executionCard = (c: FirstTaskCheckpoint) => c.interactions.find(card => card.kind === "request_confirmation" + && !card.payload?.target && /start drafting/i.test(card.payload?.prompt ?? "")); + +/** Independent persisted-state oracle; successful provider runs cannot replace any approval or output. */ +export function gradeApprovalHistory(e: FirstTaskEvidence, marker: string): FirstTaskCheck[] { + const checks: FirstTaskCheck[] = []; + const last = e.checkpoints.at(-1); + if (!last) return [{ id: "approval-history-recorded", passed: false, detail: "Missing onboarding approval checkpoints", evidence: [] }]; + const add = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail, evidence: e.checkpoints.map(c => c.id) }); + const writer = approvalHistoryWriter(marker); + const plan = planCard(last), hiring = hiringCard(last, writer), execution = executionCard(last); + const draft = plan?.payload.options?.find((option: Row) => option.label === DRAFT_OPTION); + const selectedOnlyDraft = (card: Row | undefined) => Boolean(draft && acceptedByHuman(card) + && card?.result.selectedOptionIds?.length === 1 && card.result.selectedOptionIds[0] === draft.id); + const approvedHire = (c: FirstTaskCheckpoint) => { + const card = hiringCard(c, writer); + return acceptedByHuman(card) && card?.result.selectedOptionIds?.length === 1 + && card.result.selectedOptionIds[0] === card.payload.options[0].id; + }; + const newAgents = (c: FirstTaskCheckpoint) => c.agents.filter(agent => !e.checkpoints[0].agents.some(initial => initial.id === agent.id)); + const children = (c: FirstTaskCheckpoint) => c.tasks.filter(task => !e.initialTaskIds.includes(task.id)); + add("no-premature-work", e.checkpoints.every(c => { + const hired = approvedHire(c), executing = acceptedByHuman(executionCard(c)); + return (hired || newAgents(c).length === 0) && (executing || (children(c).length === 0 + && c.documents.every(doc => doc.key === "plan" || /(?:^|[-_])proposal(?:$|[-_])/.test(doc.key)) + && !(c.attachments?.length) && c.tasks.find(task => task.id === e.onboardingIssueId)?.status !== "done")); + }), "Hiring and execution each wait for their own human decision at every checkpoint"); + add("three-separate-human-approvals", acceptedByHuman(plan) && approvedHire(last) && acceptedByHuman(execution) + && new Set([plan?.id, hiring?.id, execution?.id]).size === 3 + && Date.parse(plan!.resolvedAt) <= Date.parse(hiring!.createdAt) + && Date.parse(hiring!.resolvedAt) <= Date.parse(execution!.createdAt), + "A revision-bound checkbox plan, a hiring checkbox, and a later untargeted execution confirmation were independently accepted"); + const afterPlan = e.checkpoints.filter(c => ["plan-accepted", "hiring-proposed", "hiring-accepted", "execution-proposed", "accepted", "finished"].includes(c.phase)); + add("selected-plan-scope-retained", afterPlan.length > 0 && afterPlan.every(c => { + const card = planCard(c); + if (!card) return false; + return selectedOnlyDraft(card) && card.id === plan?.id && card.payload.target.revisionId === plan?.payload.target.revisionId + && c.documents.some(doc => doc.issueId === e.onboardingIssueId && doc.key === "plan" + && (!card.payload.target.documentId || doc.id === card.payload.target.documentId) + && doc.latestRevisionId === card.payload.target.revisionId); + }), "Only the selected draft option and exact accepted plan revision survive hiring, execution, and reload"); + const workers = newAgents(last), tasks = children(last); + add("approved-writer-executes-once", workers.length === 1 && workers[0].name === writer + && approvedHire(last) && Date.parse(workers[0].createdAt) >= Date.parse(hiring!.resolvedAt) + && tasks.length === 1 && tasks[0].parentId === e.onboardingIssueId && tasks[0].assigneeAgentId === workers[0].id + && acceptedByHuman(execution) && Date.parse(tasks[0].createdAt) >= Date.parse(execution!.resolvedAt) + && last.runs.filter(run => (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId) === tasks[0].id + && run.agentId === workers[0].id && run.startedAt).length === 1, + "Exactly the approved writer owns one task and starts one child execution after Start drafting is accepted"); + const stages = ["plan-accepted", "hiring-proposed", "hiring-accepted", "execution-proposed", "accepted", "finished"]; + add("approval-history-stages", stages.every(phase => e.checkpoints.some(c => c.phase === phase)), "Every approval and waiting boundary was observed"); + add("no-pending-approvals", last.interactions.every(card => card.status !== "pending"), "Completion leaves no pending human request or duplicate approval"); + const finished = e.checkpoints.filter(c => c.phase === "finished"); + add("writer-authored-output", workers.length === 1 && tasks.length === 1 && finished.length >= 2 + && finished.slice(-2).every(c => c.documents.some(doc => { + const revision = doc.savedRevision; + if (!revision || typeof doc.latestRevisionId !== "string") return false; + const run = c.runs.find(row => row.id === revision.createdByRunId); + return doc.issueId === tasks[0].id && doc.key !== "plan" && String(doc.body).includes(marker) + && revision.id === doc.latestRevisionId && revision.documentId === doc.id && revision.issueId === doc.issueId + && revision.key === doc.key && revision.body === doc.body && revision.createdByAgentId === workers[0].id + && !revision.createdByUserId && run?.agentId === workers[0].id && run.status === "succeeded" && run.runtimeMode === "native" + && (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId) === tasks[0].id && Boolean(run.startedAt); + })), "The saved output's exact revision was authored by the approved writer in its actual child execution"); + const output = (c: FirstTaskCheckpoint) => c.documents.filter(doc => doc.issueId === tasks[0]?.id && doc.key !== "plan") + .map(doc => ({ id: doc.id, key: doc.key, revisionId: doc.latestRevisionId, body: doc.body })) + .sort((a, b) => a.key.localeCompare(b.key) || a.id.localeCompare(b.id)); + add("reload-retains-completion", finished.length >= 2 && tasks.length === 1 && tasks[0].status === "done" + && last.tasks.find(task => task.id === e.onboardingIssueId)?.status === "done" + && finished.slice(-2).every(c => c.tasks.some(task => task.id === tasks[0].id && task.status === "done") + && c.documents.some(doc => doc.issueId === tasks[0].id && doc.key !== "plan" && String(doc.body).includes(marker))) + && JSON.stringify(output(finished.at(-2)!)) === JSON.stringify(output(last)), + "Reload retains the completed assigned child, saved welcome document, and completed onboarding parent"); + return checks; +} diff --git a/tests/runner-e2e/first-task-cases.ts b/tests/runner-e2e/first-task-cases.ts index 8244f0ead12..18d0e5a4676 100644 --- a/tests/runner-e2e/first-task-cases.ts +++ b/tests/runner-e2e/first-task-cases.ts @@ -1,4 +1,5 @@ import type { RunnerTaskFixture } from "./types.js"; +import { APPROVAL_HISTORY_CASE, approvalHistoryFacts } from "./first-task-approval-history.js"; export const FIRST_TASK_CASES = [ ["interview-first-response", "Interview: first response", "interview", 1], @@ -35,12 +36,13 @@ export const FIRST_TASK_CASES = [ ["revise-accept", "Revise scope before accepting", "task", 4], ["reject-no-execution", "Decline proposed work", "task", 2], ] as const; -export type FirstTaskCaseId = (typeof FIRST_TASK_CASES)[number][0]; +const approvalHistoryCase = [APPROVAL_HISTORY_CASE, "Interview, plan scope, hiring, and execution approval", "interview", 7] as const; +export type FirstTaskCaseId = (typeof FIRST_TASK_CASES)[number][0] | typeof APPROVAL_HISTORY_CASE; export type FirstTaskOpening = (typeof FIRST_TASK_CASES)[number][2]; export const firstTaskMarker = (nonce: string) => `GARDEN${nonce.replace(/[^a-z0-9]/gi, "")}`; export function firstTaskScenario(id: string, nonce: string) { - const entry = FIRST_TASK_CASES.find((row) => row[0] === id); + const entry = FIRST_TASK_CASES.find((row) => row[0] === id) ?? (id === APPROVAL_HISTORY_CASE ? approvalHistoryCase : undefined); if (!entry) throw new Error(`Unknown first-task case ${id}`); const marker = firstTaskMarker(nonce); const originalMarker = id === "revise-accept" ? `DRAFT${marker}` : marker; @@ -51,7 +53,7 @@ export function firstTaskScenario(id: string, nonce: string) { opening: entry[2], marker, originalMarker, - facts, + facts: id === APPROVAL_HISTORY_CASE ? approvalHistoryFacts(marker) : facts, firstResponseOnly: id.endsWith("first-response") || id === "ordinary-task-control", prompt: @@ -68,8 +70,7 @@ export function firstTaskScenario(id: string, nonce: string) { "No, do not proceed. I do not want this work done. Please acknowledge that and stop.", }; } -export const firstTaskTasks: readonly RunnerTaskFixture[] = - FIRST_TASK_CASES.map(([id, label, , expectedRunCount]) => ({ +const buildFirstTaskFixture = ([id, label, , expectedRunCount]: readonly [string, string, string, number]): RunnerTaskFixture => ({ id, label, groups: ["onboarding"], @@ -82,4 +83,11 @@ export const firstTaskTasks: readonly RunnerTaskFixture[] = buildPrompt: (nonce) => firstTaskScenario(id, nonce).prompt, buildVisibleMarker: firstTaskMarker, buildMatchers: () => [], // Checkpoints and state effects are graded by first-task-scoring. - })); + }); +export const firstTaskTasks: readonly RunnerTaskFixture[] = FIRST_TASK_CASES.map(buildFirstTaskFixture); +export const onboardingApprovalHistoryTask = { + ...buildFirstTaskFixture(approvalHistoryCase), + expectedTerminalState: { issue: "done", run: "succeeded" } as const, + minimumExpectedRunCount: 1, + automaticRetryPolicy: "single_attempt" as const, +}; diff --git a/tests/runner-e2e/first-task-fixtures.ts b/tests/runner-e2e/first-task-fixtures.ts index 45f1d39df68..c20552ae2de 100644 --- a/tests/runner-e2e/first-task-fixtures.ts +++ b/tests/runner-e2e/first-task-fixtures.ts @@ -4,6 +4,7 @@ import type { RunnerApi } from "./api.js"; import { deleteDaytonaEnvironment, installFixtureDaytonaProvider, type LiveFixtureValues } from "./live-fixtures.js"; import { isImmutableDaytonaImage } from "./catalog.js"; import type { CredentialName, MatrixExecution } from "./types.js"; +import { APPROVAL_HISTORY_SUITE } from "./first-task-approval-history.js"; export const MANAGED_FIRST_TASK_CELL = "confirmation-replies.runner-codex.daytona.task-card-accept"; export function isManagedFirstTask(execution: Pick): boolean { @@ -63,7 +64,7 @@ export async function provisionFirstTaskFixtures(input: { const { api, execution, nonce, company, credentials } = input; const managed = isManagedFirstTask(execution); if ( - !["first-task", "completion-updates", "confirmation-replies"].includes(execution.suite.id) || execution.task.flow !== "first_task" || + !["first-task", "completion-updates", "confirmation-replies", APPROVAL_HISTORY_SUITE].includes(execution.suite.id) || execution.task.flow !== "first_task" || (execution.environment.id !== "local" && !managed) || ![ "legacy-codex", diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index f131ee9131c..d1201dae5f7 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -6,6 +6,8 @@ import { firstTaskRejectionReplyRecorded, isFirstTaskRejectionCancellation } fro import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; +import { APPROVAL_HISTORY_CASE, readApprovalHistoryRevision } from "./first-task-approval-history.js"; +import { runApprovalHistoryDecisions } from "./first-task-approval-history-flow.js"; import { observeCompletionUpdate } from "./completion-update-flow.js"; import { provisionFirstTaskFixtures, @@ -75,16 +77,19 @@ export function projectFirstTaskResult(result: RunnerE2EResult): RunnerE2EResult const answerRows: Row[] = Array.isArray(answers) ? rows(answers) : answers && typeof answers === "object" ? Object.entries(answers).map(([questionId, answer]) => ({ ...answer as Row, questionId })) : []; return { - ...pick(row, ["id", "issueId", "kind", "status", "createdAt", "resolvedAt", "resolvedByAgentId"]), + ...pick(row, ["id", "issueId", "kind", "status", "createdAt", "resolvedAt", "resolvedByAgentId", "resolvedByRunId", "sourceRunId"]), ...(row.resolvedByUserId ? { resolvedByUserId: "fixture-board" } : {}), payload: { ...pick(payload, ["version", "title", "prompt", "detailsMarkdown", "acceptLabel", "rejectLabel", "rejectRequiresReason", "rejectReasonLabel", "submitLabel"]), ...(payload?.questions ? questionSet(payload) : {}), ...(payload?.questionSet ? { questionSet: questionSet(payload.questionSet) } : {}), - ...(payload?.target ? { target: pick(payload.target, ["type", "key", "label", "revisionNumber"]) } : {}), + ...(payload?.target ? { target: pick(payload.target, ["type", "key", "label", "revisionNumber", "issueId", "documentId", "revisionId"]) } : {}), + ...(payload?.options ? { options: rows(payload.options).map(option => pick(option, ["id", "label", "description"])) } : {}), + ...(payload?.defaultSelectedOptionIds ? { defaultSelectedOptionIds: strings(payload.defaultSelectedOptionIds) } : {}), }, ...(row.result ? { result: { ...pick(row.result, ["outcome", "reason", "commentId"]), + ...(row.result.selectedOptionIds ? { selectedOptionIds: strings(row.result.selectedOptionIds) } : {}), answers: answerRows.map(answer => ({ ...pick(answer, ["id", "questionId", "text", "otherText", "customText"]), optionIds: strings(answer.optionIds ?? answer.selectedOptionIds), @@ -112,14 +117,17 @@ export function projectFirstTaskResult(result: RunnerE2EResult): RunnerE2EResult checkpoints: e.checkpoints.map(c => ({ id: c.id, at: c.at, phase: c.phase, issueId: c.issueId, tasks: c.tasks.map(t => pick(t, ["id", "identifier", "title", "description", "status", "parentId", "assigneeAgentId", "createdAt", "completedAt"])), - agents: c.agents.map(a => ({ ...pick(a, ["id", "name", "adapterType"]), + agents: c.agents.map(a => ({ ...pick(a, ["id", "name", "adapterType", "createdAt"]), adapterConfig: pick(a.adapterConfig, ["provider", "model", "modelReasoningEffort", "lifecycleMode", "codexPermissionMode"]) })), comments: c.comments.map(comment => ({ ...pick(comment, ["id", "issueId", "body", "authorAgentId", "createdAt"]), ...(comment.authorUserId ? { authorUserId: "fixture-board" } : {}) })), interactions: c.interactions.map(interaction), - documents: c.documents.map(d => pick(d, ["id", "issueId", "key", "title", "body", "format", "latestRevisionId", "latestRevisionNumber", "createdAt", "updatedAt"])), + documents: c.documents.map(d => ({ ...pick(d, ["id", "issueId", "key", "title", "body", "format", "latestRevisionId", "latestRevisionNumber", "createdAt", "updatedAt"]), + ...(d.savedRevision ? { savedRevision: { ...pick(d.savedRevision, ["id", "documentId", "issueId", "key", "body", "createdByAgentId", "createdByRunId"]), + ...(d.savedRevision.createdByUserId ? { createdByUserId: "fixture-board" } : {}) } } : {}), + })), attachments: (c.attachments ?? []).map(a => pick(a, ["id", "issueId", "title", "originalFilename", "filename", "body", "contentVerified", "contentSha256", "byteSize", "createdAt"])), - runs: c.runs.map(run => ({ ...pick(run, ["id", "issueId", "status", "startedAt", "finishedAt", "runtimeMode", "driverKind"]), + runs: c.runs.map(run => ({ ...pick(run, ["id", "issueId", "nativeIssueId", "agentId", "status", "startedAt", "finishedAt", "runtimeMode", "driverKind"]), ...pick({ model: run.resultJson?.model ?? run.usageJson?.model }, ["model"]) })), })), checks: e.checks.map(check => ({ @@ -469,13 +477,15 @@ export async function runFirstTaskFlow(input: { `/api/issues/${t.id}/documents`, ); return Promise.all( - summaries.map(async (d) => ({ - ...(await api.get( + summaries.map(async (d) => { + const document = { ...(await api.get( `/api/issues/${t.id}/documents/${encodeURIComponent(d.key)}`, )), key: d.key, issueId: t.id, - })), + }; + return scenario.id === APPROVAL_HISTORY_CASE ? readApprovalHistoryRevision(api, document) : document; + }), ); }), ) @@ -728,7 +738,7 @@ export async function runFirstTaskFlow(input: { if (!scenario.firstResponseOnly) { if (["interview", "ambiguous"].includes(scenario.opening)) { const facts = - scenario.id === "interview-plan-accept" + ["interview-plan-accept", APPROVAL_HISTORY_CASE].includes(scenario.id) ? `${scenario.facts} Please write a plan for me to review, before doing the work.` : scenario.facts; const pending = ( @@ -785,7 +795,10 @@ export async function runFirstTaskFlow(input: { if (scenario.id === "revise-accept") await turn(scenario.revision, "revised"); assertBeforeAcceptance(); - if (scenario.id === "reject-no-execution") + if (scenario.id === APPROVAL_HISTORY_CASE) { + await runApprovalHistoryDecisions({ page, api, issueId: issue.id, marker: scenario.marker, deadlineAt, + allRuns, snapshot, settle, capture: input.capture }); + } else if (scenario.id === "reject-no-execution") await turn(scenario.rejection, "rejected"); else if (["task-card-accept", "accept-while-running"].includes(scenario.id)) { const pending = ( diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index fa7b49e0807..9ccd20151d0 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -5,6 +5,7 @@ import { sanitizeJson } from "./redaction.js"; import { createHash } from "node:crypto"; import { firstTaskScenario } from "./first-task-cases.js"; import { completionDelivery } from "./completion-updates.js"; +import { APPROVAL_HISTORY_CASE, approvalHistoryWriter, gradeApprovalHistory } from "./first-task-approval-history.js"; export type Row = { id: string; [key: string]: any }; export interface FirstTaskCheckpoint { id: string; @@ -14,6 +15,10 @@ export interface FirstTaskCheckpoint { | "response" | "clarified" | "revised" + | "plan-accepted" + | "hiring-proposed" + | "hiring-accepted" + | "execution-proposed" | "accepted" | "rejected" | "finished"; @@ -311,7 +316,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "Propose a task for the concrete request", [first.id], ); - if (scenario.opening === "plan" || e.caseId === "interview-plan-accept") { + if (scenario.opening === "plan" || ["interview-plan-accept", APPROVAL_HISTORY_CASE].includes(e.caseId)) { const proposal = e.checkpoints.find((c) => c.phase === "clarified") ?? first; add( @@ -328,7 +333,9 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { c.phase !== "opening" && (!approval || Date.parse(c.at) < Date.parse(approval.at)), ); - if (scenario.opening !== "ordinary") { + if (e.caseId === APPROVAL_HISTORY_CASE) { + checks.push(...gradeApprovalHistory(e, scenario.marker)); + } else if (scenario.opening !== "ordinary") { add( "no-premature-work", before.every( @@ -402,12 +409,14 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { ); } else { const children = extras(last); + const expectedAssignee = e.caseId === APPROVAL_HISTORY_CASE + ? last.agents.find(agent => agent.name === approvalHistoryWriter(scenario.marker))?.id : e.agentId; add( "one-scoped-subtask", children.length === 1 && children[0].parentId === e.onboardingIssueId && - children[0].assigneeAgentId === e.agentId, - "Exactly one approved subtask belongs to this onboarding issue and agent", + Boolean(expectedAssignee) && children[0].assigneeAgentId === expectedAssignee, + "Exactly one approved subtask belongs to this onboarding issue and its intended agent", [last.id], notReached, ); diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index a5b30654497..5e9231cdd78 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -1591,7 +1591,7 @@ describe("managed onboarding through the production wizard", () => { expect(h.api.get).toHaveBeenCalledExactlyOnceWith("/api/instance/settings"); expect(h.api.patch).toHaveBeenLastCalledWith("/api/instance/settings", { defaultEnvironmentId: previousDefault }); expect(h.state.experimental.enableAgentChat).toBe(true); - expect(h.api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true }); + expect(h.api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true, timeout: 30_000 }); }, ); @@ -1722,7 +1722,7 @@ describe("managed onboarding through the production wizard", () => { expect(fixtures.setupLeases).toEqual([]); expect(JSON.stringify(fixtures)).not.toContain("synthetic"); await fixtures.teardown(); - expect(api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true }); + expect(api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true, timeout: 30_000 }); }); it.each(["missing-key", "mutable-image", "other-cell"])("rejects %s before admitting a managed browser fixture", async mode => { diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 157f2ffce49..5c0386a76c9 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -71,6 +71,7 @@ associate(["first-task", "confirmation-replies", "completion-updates"], Object.f "revise-accept", "reject-no-execution", ].map((id) => [id, ["onboarding/first-task"]]))); associate(["first-task"], { "ordinary-task-control": ["tasks/completion"] }); +associate(["onboarding-approval-history"], { "interview-hire-execute": ["onboarding/first-task", "questions-and-approvals/continuation", "tasks/completion"] }); associate(["confirmation-replies"], { "confirmation-ambiguous": ["questions-and-approvals/confirmations", "questions-and-approvals/audience"], "unanswered-question-return": ["questions-and-approvals/dismiss-and-reopen", "questions-and-approvals/continuation"], diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 9906e0d4869..1469a322867 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -353,7 +353,7 @@ const fields = { source: optional(shape({ sha: string, ref: string, dirty: boolean })), initialTaskIds: array(string), configuredModel: nullable(string), observedModels: array(string), runtimeSettings: optional(object), instructions: array(shape({ path: string, content: string, sha256: string, contentSha256: optional(string), redacted: optional(boolean) })), - checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "accepted", "rejected", "finished"), issueId: string, + checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "plan-accepted", "hiring-proposed", "hiring-accepted", "execution-proposed", "accepted", "rejected", "finished"), issueId: string, tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), attachments: optional(array(object)), runs: array(object) })), checks: array(shape({ id: string, passed: boolean, notReached: optional(string), evidence: array(string), detail: string })), })),