From 7c66b6b1f60bdc5742a9a2acc41b87e3aa4f1b8f Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 4 Nov 2026 21:46:55 -0600 Subject: [PATCH] test(onboarding): cover plan, hiring, and execution approval history (#19069) ## Thinking Path > - Paperclip manages agent work and human decisions. > - Onboarding can approve a plan, a hire, and an execution step in separate turns. > - Tests covered single approvals more closely than that full history. > - The repaired gate must retain valid plan approval through later decisions. > - Every consumer must recognize the saved confirmation kind and selected scope. > - This change tests that contract and registers a complete interview-to-execution journey. ## Linked Issues or Issue Description Refs #19043, #17914, #18283. The earlier tests admitted one accepted proposal at a time. Decomposition fixtures used standard confirmations. The direct-task E2E path forbids hiring. Those cases did not cover an accepted checkbox plan followed by hiring and a separate untargeted execution approval. ## What Changed - Add eight PostgreSQL contract cases for standard and checkbox plan approvals. Native tools create the cards. An authenticated human accepts them through HTTP. A later native run reads the approved revision and actual selected scope. - Exercise native task creation, generic REST creation, child REST creation, and accepted-plan decomposition. Check the assigned child and one creation audit event. Check durable replay on the three entry points that support it. A pending execution confirmation still prevents creation. - Register the explicit-only `onboarding-approval-history` suite for local native Codex and ACPX Claude. The browser chooses the interview, selects only drafting in the saved plan, approves one writer hire, and approves a separate Start drafting confirmation. The model must hire the writer and use that writer's own child execution to save the welcome document. - Grade each human decision, the unchanged plan revision and selection, approval timing, one assigned child and worker execution, the exact writer-authored saved revision, no pending requests, and persistence after reload. Calibrate the oracle against missing, forged, premature, duplicate, and changed outcomes. - Preserve synthetic option choices, document/run bindings, and saved revision authorship in the existing public first-task evidence projection. Keep private provider/account data excluded. Document the workflow, evidence, billing, and cleanup bounds. - Align the approval-history fixture with its completion oracle: the onboarding parent must be Done. Test both native profiles against graded completion evidence and reject a completed child whose parent remains In review. Count parent heartbeats separately from the writer: accept one to seven parent records while retaining the twelve-company-run cap and the independent three-approval/one-writer oracle. - Retain the existing 52-cell first-task suite. The new suite has one attempt, seven expected work turns, a twelve-run maximum, a fifteen-minute deadline, and the existing $5 company task budget. It is excluded from `--all`. - Update two stale cleanup assertions to include the existing thirty-second deletion timeout. ## Verification - Red control: temporarily restore the pre-fix admission module and run the new checkbox/native history case. It fails with the expected `First-task proposal requires human acceptance` error after plan, hiring, and execution acceptance. Restore the repaired module after the probe. - The final post-rebase history matrix and existing onboarding safety suite passed together: 74 tests, no skips. - Final focused Runner E2E support passed: 413 Vitest tests and all 370 Node support checks. This covers catalog discovery, journey controls, wrong-outcome calibration, public evidence, and existing first-task behavior. - Review controls: four authorship cases fail before the revision check. A delayed browser-response control fails before the approval-time fix. Both pass after repair. The final oracle requires the exact current document revision, writer ID, native child run ID, matching body, and readback after reload. - Credential-free discovery lists both new native cells. No provider call was made. `OPENAI_API_KEY` and `ANTHROPIC_API_KEY` are absent, so the real provider journey remains unqualified. - The earlier broad credential-free E2E support run reported 6,728 passing cases, three skipped cases, and nine failures in three unrelated files. All nine reproduce with the changed E2E source restored to the unchanged baseline: three expired overhead-fixture cases, one stale Pi profile-version assertion, and five workflow security assertions. This change does not alter those files, paid workflow policy, or provider pins. - Previous full workspace typecheck/build and [Linux CI](https://github.com/paperclipai/paperclip/actions/runs/41921754638) passed on `4f431f5aae99d072cc75027e90d7b847b4c342ca`, including the new eight-case PostgreSQL file. Ponytail subsequently found a fixture/oracle terminal-status mismatch. Both native-cell controls fail on that fixture (`done` observed versus `in_review` expected), while the parent-in-review negative control passes. A second comparison uses the actual `matchesRunCount` predicate: graded evidence with six parent runs and one writer run fails the old exact-seven-parent requirement. Both fixes pass these controls, all 413 focused support tests, all 370 Node checks, and the owning E2E project typecheck. [Fresh CI](https://github.com/paperclipai/paperclip/actions/runs/41936699686) passed on `91c980b110252da451b92eda597c168517de1276`: 48 successful jobs and one expected skip. Greptile reviewed this head at 5/5 with no unresolved findings. Ponytail's terminal-status finding is fixed and its thread is resolved; its current-head re-review remains queued. Live provider cells remain unqualified. ## Risks The production API and approval gate are unchanged. The new paid journey can expose model or product failures at an intermediate approval boundary. It retains those failures and uses the existing diagnostics and cleanup. Registration and deterministic tests do not establish a live provider pass. The unrelated baseline support failures remain explicit. ## Model Used OpenAI GPT-6 through Codex, with reasoning, repository tools, and code execution. The exact serving model ID and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass (focused tests; unrelated baseline failures are recorded above) - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- doc/evals.md | 7 + .../onboarding-approval-history.test.ts | 142 ++++++++++++ tests/runner-e2e/FIXTURES.md | 10 + tests/runner-e2e/README.md | 33 +++ tests/runner-e2e/SECURITY.md | 12 ++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 15 +- .../first-task-approval-history-flow.ts | 51 +++++ .../first-task-approval-history.test.ts | 202 ++++++++++++++++++ .../runner-e2e/first-task-approval-history.ts | 121 +++++++++++ tests/runner-e2e/first-task-cases.ts | 20 +- tests/runner-e2e/first-task-fixtures.ts | 3 +- tests/runner-e2e/first-task-flow.ts | 33 ++- tests/runner-e2e/first-task-scoring.ts | 17 +- tests/runner-e2e/first-task.test.ts | 4 +- tests/runner-e2e/journey-features.ts | 1 + tests/runner-e2e/result-validation.ts | 2 +- 17 files changed, 651 insertions(+), 28 deletions(-) create mode 100644 server/src/__tests__/onboarding-approval-history.test.ts create mode 100644 tests/runner-e2e/first-task-approval-history-flow.ts create mode 100644 tests/runner-e2e/first-task-approval-history.test.ts create mode 100644 tests/runner-e2e/first-task-approval-history.ts diff --git a/doc/evals.md b/doc/evals.md index 8f2ec714b82..37b160e28f1 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -137,6 +137,13 @@ It requires a fresh owned agent with no prior work runs and uses three provider Cloud fleet operations and trusted CI remain private. Local eval gateway qualification does not qualify Cloud routes, and these selections stay outside `--all`. +The explicit [onboarding approval-history journey](../tests/runner-e2e/README.md#first-task-onboarding) +continues an interview through partial plan acceptance, approved hiring, and a +separate execution confirmation. It requires the assigned child's real execution, +saved output, unchanged approved scope, and reload persistence. Deterministic +PostgreSQL producer/consumer tests run in ordinary CI; native provider cells +require an explicit paid selection and a recorded live result. + The deployment-targeted [saved skill journey](../tests/runner-e2e/product-journeys/saved-skill-use.md) checks one company skill assigned by key, its exact admitted immutable version, a bound native source read/load, and a run-authored saved result from skill-only inputs. It uses the reusable Product journey target and remains unqualified until a real target run passes. The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) diff --git a/server/src/__tests__/onboarding-approval-history.test.ts b/server/src/__tests__/onboarding-approval-history.test.ts new file mode 100644 index 00000000000..2b942fc4b14 --- /dev/null +++ b/server/src/__tests__/onboarding-approval-history.test.ts @@ -0,0 +1,142 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { and, eq } from "drizzle-orm"; +import { activityLog, agents, companyMemberships, heartbeatRuns, issues, issueThreadInteractions } from "@paperclipai/db"; +import { createLocalAgentJwt } from "../agent-auth-jwt.js"; +import { documentService } from "../services/documents.js"; +import { initializeRunIdentity } from "../services/run-identity.js"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { recoveryPostgresSupport as support } from "./helpers/recovery-postgres.js"; +import { startRunnerApiTestServer } from "./helpers/runner-api-server.js"; + +if (!support.supported) console.warn(`Skipping onboarding approval history tests: ${support.reason}`); +describe.skipIf(!support.supported)("onboarding approval producer/consumer contract", () => { + let server: Awaited>; + let decisionUserId: string; + let cookie: string; + const originalSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET; + beforeAll(async () => { + process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID(); + server = await startRunnerApiTestServer({ humanSessions: true }); + const board = await server.fixture({ connectionScenario: "fresh", disableWakeOnDemand: true }); + decisionUserId = board.responsibleUserId!; + cookie = await server.humanSession(decisionUserId); + }, 60_000); + afterAll(async () => { + await server?.close(); + if (originalSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; + else process.env.PAPERCLIP_AGENT_JWT_SECRET = originalSecret; + }); + + type Fixture = Awaited>; + const selected = { id: "discovery", label: "Read-only discovery", description: "Write the architecture document." }; + const request = async (f: Fixture, kind: "confirmation" | "checkbox", key: string, payload: Record = {}) => { + const receipt = await f.authority.execute({ tool: "request_human_input", callId: randomUUID(), arguments: { + interactionKind: kind, idempotencyKey: key, title: key, prompt: key, continuationPolicy: "wake_assignee", payload: { version: 1, ...payload }, + } }) as { interaction: { id: string } }; + return receipt.interaction.id; + }; + const accept = async (f: Fixture, id: string, selectedOptionIds?: string[]) => { + const response = await fetch(`${server.apiUrl}/api/issues/${f.issueId}/interactions/${id}/accept`, { + method: "POST", headers: { cookie, "Content-Type": "application/json", origin: server.apiUrl }, + body: JSON.stringify(selectedOptionIds ? { selectedOptionIds } : {}), + }); + const data = await response.json(); + expect(response.status, JSON.stringify(data)).toBe(200); + expect(data).toMatchObject({ status: "accepted", resolvedByUserId: decisionUserId }); + }; + const approvedHistory = async (kind: "confirmation" | "checkbox") => { + const f = await server.fixture({ disableWakeOnDemand: true, connectionScenario: "fresh" }); + await server.db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", + principalId: decisionUserId, status: "active", membershipRole: "member" }); + await server.db.update(issues).set({ originKind: "onboarding_first_task" }).where(eq(issues.id, f.issueId)); + const plan = (await documentService(server.db).upsertIssueDocument({ issueId: f.issueId, key: "plan", format: "markdown", + body: "Step 1: read-only discovery and architecture document. Step 2: implementation needs separate approval.", + createdByAgentId: f.agentId, createdByRunId: f.runId })).document; + const planId = await request(f, kind, "Approve plan", { + target: { type: "issue_document", key: "plan", issueId: f.issueId, documentId: plan.id, revisionId: plan.latestRevisionId }, + ...(kind === "checkbox" ? { options: [selected, { id: "implementation", label: "Implement the application" }], + defaultSelectedOptionIds: [selected.id, "implementation"], minSelected: 1 } : {}), + }); + await accept(f, planId, kind === "checkbox" ? [selected.id] : undefined); + const hiringId = await request(f, "checkbox", "Hire a platform engineer", { + options: [{ id: "engineer", label: "Hire one Platform Engineer" }], minSelected: 1, defaultSelectedOptionIds: ["engineer"], + }); + await accept(f, hiringId, ["engineer"]); + // Agent creation is fixture setup here; the live journey exercises hiring. + const workerId = randomUUID(); + await server.db.insert(agents).values({ id: workerId, companyId: f.companyId, name: "Platform Engineer", + status: "active", adapterType: "paperclip_runner", runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: false } } }); + const executionId = await request(f, "confirmation", "Start Step 1: Discovery"); + return { f, plan, planId, hiringId, executionId, workerId }; + }; + + const resume = async (f: Fixture) => { + const runId = randomUUID(); + await server.db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.runId)); + await server.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + responsibleUserId: f.responsibleUserId, status: "running", runtimeMode: "native", nativeIssueId: f.issueId, + invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId: f.issueId } }); + await server.db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, f.issueId)); + await initializeRunIdentity(server.db, { companyId: f.companyId, runId, issueId: f.issueId, + responsibleUserId: f.responsibleUserId!, cause: "instruction" }); + return { ...f, runId, authority: new PaperclipRunnerToolAuthority(server.db, { ...f, runId }) }; + }; + + for (const kind of ["confirmation", "checkbox"] as const) { + it.each(["native", "REST issue", "REST child", "decomposition"] as const)( + `${kind} plan → hiring → separate execution approval → %s, including later-run context`, async path => { + const history = await approvedHistory(kind); + const { plan, workerId } = history; + let f = history.f; + const nativeInput = { tool: "create_task", callId: randomUUID(), arguments: { + title: "Step 1: Discovery and architecture", assigneeActorId: workerId, status: "todo", idempotencyKey: "step-1", + } }; + const before = await f.snapshot(); + // Plan and hiring acceptance alone cannot answer a separate pending execution request. + await expect(f.authority.execute(nativeInput)).rejects.toMatchObject({ status: 403 }); + expect((await f.snapshot()).issues).toEqual(before.issues); + await accept(f, history.executionId); + f = await resume(f); + const context = await f.authority.execute({ tool: "get_task_context", callId: randomUUID(), arguments: {} }) as { + acceptedPlan: { revisionId: string; approvedOptions?: unknown[] }; + }; + expect(context.acceptedPlan).toMatchObject({ revisionId: plan.latestRevisionId, markdown: plan.body }); + if (kind === "checkbox") expect(context.acceptedPlan.approvedOptions).toEqual([selected]); + else expect(context.acceptedPlan).not.toHaveProperty("approvedOptions"); + + const token = createLocalAgentJwt(f.agentId, f.companyId, "paperclip_runner", f.runId, f.responsibleUserId)!; + const post = async () => { + const route = path === "decomposition" ? `/api/issues/${f.issueId}/accepted-plan-decompositions` + : path === "REST child" ? `/api/issues/${f.issueId}/children` : `/api/companies/${f.companyId}/issues`; + const child = { title: "Step 1: Discovery and architecture", status: "backlog", workMode: "standard", priority: "medium", assigneeAgentId: workerId }; + const body = path === "decomposition" ? { acceptedPlanRevisionId: plan.latestRevisionId, children: [child] } + : path === "REST child" ? child : { ...child, parentId: f.issueId, idempotencyKey: "step-1" }; + const response = await fetch(`${server.apiUrl}${route}`, { method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, body: JSON.stringify(body) }); + const data = await response.json(); + expect(response.ok, JSON.stringify(data)).toBe(true); + return path === "decomposition" ? data.childIssueIds[0] as string : (data.issue?.id ?? data.id) as string; + }; + let childId: string; + if (path === "native") { + const created = await f.authority.execute(nativeInput) as { task: { id: string }; scheduledWakeIds: string[] }; + childId = created.task.id; + expect(created.scheduledWakeIds).toEqual([`created-child:${childId}`]); + const reconnected = new PaperclipRunnerToolAuthority(server.db, f); + expect(await reconnected.execute({ ...nativeInput, callId: randomUUID() })).toEqual(created); + } else { + childId = await post(); + // The child REST schema has no idempotency key; replay belongs to the other entry points. + if (path !== "REST child") expect(await post()).toBe(childId); + } + const children = await server.db.select().from(issues).where(and(eq(issues.companyId, f.companyId), eq(issues.parentId, f.issueId))); + expect(children).toMatchObject([{ id: childId, assigneeAgentId: workerId, title: "Step 1: Discovery and architecture" }]); + const creationEvents = await server.db.select().from(activityLog).where(and(eq(activityLog.companyId, f.companyId), eq(activityLog.entityId, childId))); + expect(creationEvents.filter(event => ["issue.created", "issue.child_created"].includes(event.action))).toHaveLength(1); + const [saved] = await server.db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, history.planId)); + expect(saved.status).toBe("accepted"); + if (kind === "checkbox") expect(saved.result).toMatchObject({ selectedOptionIds: [selected.id] }); + }); + } +}); diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index ce803748eb6..ece2625998f 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -69,6 +69,16 @@ retain their original definition and do not qualify the added Plan cell. ## Suites and matrices +`onboarding-approval-history` uses the existing first-task fixture registry path +with the production wizard and native Codex/ACPX Claude profiles. Its explicit +`interview-hire-execute` case has seven expected work turns, one attempt, a $5 +company task budget, twelve total run records, and a fifteen-minute deadline. +The model creates its plan, approval cards, writer, child task, and saved output +through production tools; the harness answers the interview and clicks normal +approval controls. Fixture code does not insert approval rows or hire the writer. +The ordinary first-task cleanup owns the complete disposable company, including +the new writer. See [the journey contract](README.md#first-task-onboarding). + A `RunnerSuiteFixture` declares one durable testing purpose: stable ID, label, description, profiles, environments, cases, expected size, and definition or ranking metadata. Its execution IDs are globally prefixed as diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 34f3183846b..61f3e8e9494 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1763,6 +1763,39 @@ Revision delivery checks exclude preserved originals by their content hash, even ## First-task onboarding +The explicit-only `onboarding-approval-history` suite extends the interview path +through three separate human decisions: a saved plan checkbox selection, a +writer hiring checkbox, and an untargeted Start drafting confirmation. The user +selects drafting and clears publishing. The model must hire the approved writer, +assign one child, and produce the saved welcome document through that child's +own execution. Persisted decisions, the unchanged plan revision and selection, +creation timestamps, one child work run, and readback after browser reload own +the grade. The exact saved output revision must name the writer and its actual +child run as author. A successful parent run or lead-authored output fails. + +It has one local cell each for native Codex and ACPX Claude. It reuses the +production wizard, first-task fixtures, $5 company task budget, twelve-run +maximum, fifteen-minute attempt deadline, five-minute response waits, screenshots, +private diagnostics, sanitized public evidence, and disposable-company cleanup. +Expect seven provider work turns, including the writer's child. The outer +runner accepts one to seven parent heartbeat records; it does not count the +child as a parent run. The oracle independently requires all three human +decisions and exactly one writer execution. Setup probes are accounted separately. +The case permits one attempt and is excluded from `--all`. Run it explicitly when +qualifying changes to onboarding or approval semantics: + +```sh +pnpm test:e2e:runner -- --list --suite onboarding-approval-history +pnpm test:e2e:runner -- --id onboarding-approval-history.runner-codex.local.interview-hire-execute +``` + +The deterministic PostgreSQL contract in +`server/src/__tests__/onboarding-approval-history.test.ts` runs in the ordinary +Vitest suite. It uses native-produced cards, authenticated human acceptance, +and a later native run to check standard and checkbox plan approvals across +native creation, both REST creation routes, decomposition, context, and replay. +Journey discovery and grader calibration do not establish a live provider pass. + `first-task` is a suite in the main Runner E2E catalog. A full `pnpm test:e2e:runner -- --all` run (or an unfiltered full GitHub Actions campaign) includes its 52 executions alongside the other suites in one shared dashboard, diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 1c6969ebcf7..0922095aad6 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,17 @@ # Runner E2E security for a public repository +The explicit `onboarding-approval-history` suite uses only synthetic garden-club +facts in an owned disposable onboarding company. The human selects drafting and +clears publishing; the scenario forbids external publication. The hired writer +must use the lead's existing provider/AI connection through normal hiring. +Credentials remain in the existing encrypted fixture path. It makes no direct +database writes and changes no shared company. Public first-task projection +retains synthetic checkbox option IDs/labels, selected IDs, plan document/revision +IDs, task/agent/run bindings, and the exact saved output revision's agent/run +authorship so the approval history can be regraded. Provider +session IDs, account configuration, raw traces, and full instructions retain +their existing private-only treatment. Registration is separate from live proof. + The deployment-targeted `saved-skill-use` journey creates one synthetic skill on the campaign-owned agent. Its random receipt reference and numeric inputs occur only in the saved skill source. Exact source, immutable version, ordinary assignment, native read diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 6a37e2ab92f..d8706ffd9f1 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -329,11 +329,11 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 2, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(822); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(822); + expect(validateRunnerCatalog()).toHaveLength(824); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(824); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index e9a8f8ad6af..14889bb6006 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -44,7 +44,8 @@ import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; import { REMOTE_WORKSPACE_STORAGE_SUITE, remoteWorkspaceStoragePrompt } from "./remote-workspace-storage.js"; import { CONNECTION_GUIDANCE_SUITE, CONNECTION_GUIDANCE_BUDGET_CENTS, connectionGuidanceTasks, connectionGuidanceDefinitionDigest } from "./connection-guidance-cases.js"; -import { firstTaskTasks } from "./first-task-cases.js"; +import { firstTaskTasks, onboardingApprovalHistoryTask } from "./first-task-cases.js"; +import { APPROVAL_HISTORY_SUITE, approvalHistoryDefinitionDigest } from "./first-task-approval-history.js"; import { chatTasks, chatHardeningTasks, chatStoryTasks, chatQualificationTasks, chatCompletionTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; @@ -1785,6 +1786,18 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ paidCalls: "one skill turn, two ordered-comment turns, three chat turns per profile", }, }, + { + id: APPROVAL_HISTORY_SUITE, label: "Onboarding approval history", manualOnly: true, + description: "Interview, partial plan approval, approved hiring, a separate execution confirmation, and the writer’s saved result.", + groups: ["onboarding", "native"], + profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)), + environments: [localEnvironment], tasks: [onboardingApprovalHistoryTask], expectedMatrixSize: 2, + definitionMetadata: { version: 1, sourceDigest: approvalHistoryDefinitionDigest(), scheduling: "explicit-only", + automaticRetryPolicy: "single_attempt", maximumAttemptsPerCell: 1, expectedWorkRuns: 7, + companyTaskBudgetCents: 500, maximumCompanyRuns: 12, attemptTimeoutMs: 900_000, + instructions: "production-wizard-and-first-task", approvalSequence: "plan-checkbox-subset/hiring-checkbox/execution-confirmation", + oracle: "persisted-human-decisions/current-plan-revision/assigned-child/one-worker-execution/saved-document/reload" }, + }, { id: "first-task", label: "First-task onboarding", description: "Production onboarding, first replies, approval, and durable task execution.", diff --git a/tests/runner-e2e/first-task-approval-history-flow.ts b/tests/runner-e2e/first-task-approval-history-flow.ts new file mode 100644 index 00000000000..a5d7d9be184 --- /dev/null +++ b/tests/runner-e2e/first-task-approval-history-flow.ts @@ -0,0 +1,51 @@ +import { expect, type Page } from "@playwright/test"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { DRAFT_OPTION, PUBLISH_OPTION, approvalHistoryWriter } from "./first-task-approval-history.js"; +import type { FirstTaskCheckpoint, Row } from "./first-task-scoring.js"; + +/** The model produces every proposal and hire; the harness only operates ordinary approval controls. */ +export async function runApprovalHistoryDecisions(input: { + page: Page; api: Pick; issueId: string; marker: string; deadlineAt: number; + allRuns: () => Promise; + snapshot: (phase: FirstTaskCheckpoint["phase"], at?: string) => Promise; + settle: (prior: Set, completion?: boolean) => Promise; + capture: (id: string, label: string, file: string) => Promise; +}) { + const cards = () => input.api.get(`/api/issues/${input.issueId}/interactions`); + const approve = async (card: Row, phase: FirstTaskCheckpoint["phase"]) => { + const before = new Set((await input.allRuns()).map(run => run.id)); + const at = new Date().toISOString(); + await input.page.getByRole("button", { name: card.payload.acceptLabel + ?? (card.kind === "request_confirmation" ? "Approve" : "Confirm selection"), exact: true }).last().click(); + await pollUntil({ label: `saved ${phase} decision`, deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000), intervalMs: 250, + load: cards, accept: rows => rows.some(row => row.id === card.id && row.status === "accepted" && row.resolvedByUserId), + }); + await input.snapshot(phase, at); + await input.settle(before, phase === "accepted"); + }; + const pending = async (predicate: (row: Row) => boolean) => { + const found = (await cards()).filter(row => row.status === "pending" && predicate(row)); + expect(found, "One saved approval for this stage must exist").toHaveLength(1); + return found[0]!; + }; + const plan = await pending(row => row.kind === "request_checkbox_confirmation" && row.payload?.target?.key === "plan"); + expect(plan.payload.options.map((option: Row) => option.label).sort()).toEqual([DRAFT_OPTION, PUBLISH_OPTION].sort()); + await input.page.getByRole("checkbox", { name: DRAFT_OPTION, exact: true }).last().check(); + await input.page.getByRole("checkbox", { name: PUBLISH_OPTION, exact: true }).last().uncheck(); + await approve(plan, "plan-accepted"); + await input.snapshot("hiring-proposed"); + await input.capture("hiring-proposed", "Hiring waits for approval", "hiring-proposed.png"); + const writer = approvalHistoryWriter(input.marker); + const hiring = await pending(row => row.kind === "request_checkbox_confirmation" && !row.payload?.target + && row.payload?.options?.length === 1 && row.payload.options[0].label.includes(writer)); + await input.page.getByRole("checkbox", { name: hiring.payload.options[0].label, exact: true }).last().check(); + await approve(hiring, "hiring-accepted"); + await input.snapshot("execution-proposed"); + await input.capture("execution-proposed", "Execution waits for Start drafting", "execution-proposed.png"); + const execution = await pending(row => row.kind === "request_confirmation" && !row.payload?.target + && /start drafting/i.test(row.payload?.prompt ?? "")); + await approve(execution, "accepted"); + await input.snapshot("finished"); + await input.page.reload({ waitUntil: "domcontentloaded" }); + await input.snapshot("finished"); +} diff --git a/tests/runner-e2e/first-task-approval-history.test.ts b/tests/runner-e2e/first-task-approval-history.test.ts new file mode 100644 index 00000000000..3fbfeef4797 --- /dev/null +++ b/tests/runner-e2e/first-task-approval-history.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Page } from "@playwright/test"; +import { runnerMatrix } from "./catalog.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { firstTaskScenario } from "./first-task-cases.js"; +import { APPROVAL_HISTORY_CASE, APPROVAL_HISTORY_SUITE, DRAFT_OPTION, PUBLISH_OPTION, approvalHistoryWriter, readApprovalHistoryRevision } from "./first-task-approval-history.js"; +import { gradeFirstTask, snapshotInstruction, type FirstTaskCheckpoint, type FirstTaskEvidence, type Row } from "./first-task-scoring.js"; +import { projectFirstTaskResult } from "./first-task-flow.js"; +import { runApprovalHistoryDecisions } from "./first-task-approval-history-flow.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import type { RunnerE2EResult } from "./types.js"; +import { matchesRunCount } from "./run-count.js"; + +function recording(): FirstTaskEvidence { + const { marker } = firstTaskScenario(APPROVAL_HISTORY_CASE, "history"); + const at = (minute: number) => `2026-11-01T00:${String(minute).padStart(2, "0")}:00Z`; + const parent = { id: "parent", title: "First task", status: "in_review", description: "/first-task" }; + const plan = { id: "plan", issueId: "parent", key: "plan", title: "Plan", body: `Draft then publish ${marker}`, latestRevisionId: "revision" }; + const planCard = { id: "plan-card", kind: "request_checkbox_confirmation", status: "pending", createdAt: at(2), payload: { + options: [{ id: "draft", label: DRAFT_OPTION }, { id: "publish", label: PUBLISH_OPTION }], defaultSelectedOptionIds: ["draft", "publish"], + target: { type: "issue_document", key: "plan", documentId: "plan", revisionId: "revision" }, + } }; + const hireCard = { id: "hire-card", kind: "request_checkbox_confirmation", status: "pending", createdAt: at(4), + payload: { options: [{ id: "writer", label: `Hire ${approvalHistoryWriter(marker)}` }] } }; + const executionCard = { id: "execution-card", kind: "request_confirmation", status: "pending", createdAt: at(6), payload: { prompt: "Start drafting" } }; + const resolve = (card: Row, minute: number, ids?: string[]) => ({ ...card, status: "accepted", resolvedByUserId: "human", resolvedAt: at(minute), + result: { outcome: "accepted", ...(ids ? { selectedOptionIds: ids } : {}) } }); + const checkpoints: FirstTaskCheckpoint[] = []; + const checkpoint = (phase: FirstTaskCheckpoint["phase"], minute: number, updates: Partial = {}) => { + const value = structuredClone({ id: `${phase}-${minute}`, phase, at: at(minute), issueId: parent.id, + tasks: [parent], agents: [{ id: "lead" }], comments: [], interactions: [], documents: [], runs: [], + ...checkpoints.at(-1), ...updates }); + Object.assign(value, { id: `${phase}-${minute}`, phase, at: at(minute) }); + checkpoints.push(value); + return value; + }; + checkpoint("opening", 0); + checkpoint("response", 1, { comments: [{ id: "interview-comment", authorAgentId: "lead", body: "Tell me about the club." }], + interactions: [{ id: "interview", kind: "ask_user_questions", status: "answered", payload: { + questions: ["goal", "audience", "done"].map(id => ({ id, answerMode: "text", prompt: id })), + } }], runs: [{ id: "interview-run", status: "succeeded" }] }); + checkpoint("clarified", 2, { interactions: [planCard], documents: [plan] }); + checkpoint("plan-accepted", 3, { interactions: [resolve(planCard, 3, ["draft"])] }); + checkpoint("hiring-proposed", 4, { interactions: [resolve(planCard, 3, ["draft"]), hireCard] }); + checkpoint("hiring-accepted", 5, { interactions: [resolve(planCard, 3, ["draft"]), resolve(hireCard, 5, ["writer"])] }); + checkpoint("execution-proposed", 6, { agents: [{ id: "lead" }, { id: "writer", name: approvalHistoryWriter(marker), createdAt: at(5) }], + interactions: [resolve(planCard, 3, ["draft"]), resolve(hireCard, 5, ["writer"]), executionCard] }); + checkpoint("accepted", 7, { interactions: [resolve(planCard, 3, ["draft"]), resolve(hireCard, 5, ["writer"]), resolve(executionCard, 7)] }); + checkpoint("finished", 8, { tasks: [{ ...parent, status: "done" }, { id: "child", parentId: "parent", assigneeAgentId: "writer", status: "done", createdAt: at(7) }], + documents: [plan, { id: "welcome", issueId: "child", key: "welcome", latestRevisionId: "welcome-revision", + body: `Welcome, beginners. Join our free Saturday meetup ${marker}.`, savedRevision: { id: "welcome-revision", documentId: "welcome", + issueId: "child", key: "welcome", body: `Welcome, beginners. Join our free Saturday meetup ${marker}.`, + createdByAgentId: "writer", createdByRunId: "child-run", createdByUserId: null } }], + runs: [{ id: "parent-run", nativeIssueId: "parent", agentId: "lead", status: "succeeded", startedAt: at(7) }, + { id: "child-run", nativeIssueId: "child", agentId: "writer", status: "succeeded", runtimeMode: "native", startedAt: at(7) }] }); + checkpoint("finished", 9); + return { caseId: APPROVAL_HISTORY_CASE, nonce: "history", onboardingIssueId: "parent", agentId: "lead", initialTaskIds: ["parent"], + configuredModel: "fixture-model", observedModels: ["fixture-model"], instructions: [snapshotInstruction("AGENTS.md", "Persona"), + snapshotInstruction("first-task/SKILL.md", "First-task instructions")], checkpoints, checks: [] }; +} +const failures = (e: FirstTaskEvidence) => gradeFirstTask(e).filter(check => !check.passed).map(check => check.id); + +describe("onboarding approval history journey", () => { + it("registers two explicit native cells without changing the default paid campaign", () => { + const cells = selectRunnerExecutions(parseRunnerSelectors(["--suite", APPROVAL_HISTORY_SUITE])); + expect(cells.map(cell => cell.id)).toEqual([`onboarding-approval-history.runner-codex.local.${APPROVAL_HISTORY_CASE}`, + `onboarding-approval-history.runner-acpx-claude.local.${APPROVAL_HISTORY_CASE}`]); + expect(cells.every(cell => cell.task.automaticRetryPolicy === "single_attempt" && cell.task.expectedRunCount === 7)).toBe(true); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => cell.suite.id === APPROVAL_HISTORY_SUITE)).toBe(false); + expect(runnerMatrix.filter(cell => cell.suite.id === "first-task")).toHaveLength(52); + }); + + it("accepts the complete interview, partial plan approval, hiring, execution, and reload recording", () => { + expect(failures(recording())).toEqual([]); + }); + + it.each(runnerMatrix.filter(cell => cell.suite.id === APPROVAL_HISTORY_SUITE))( + "completed evidence satisfies the outer runner's status and parent-run contract for $id", execution => { + const evidence = recording(); + // Seven work turns include the writer: only six belong to the parent. + const leadRuns = ["interview-run", "plan-run", "hiring-request-run", "hire-run", "create-child-run", "parent-run"].map((id, index) => ({ + id, nativeIssueId: evidence.onboardingIssueId, agentId: evidence.agentId, status: "succeeded", runtimeMode: "native", + contextSnapshot: { issueId: evidence.onboardingIssueId }, startedAt: `2026-11-01T00:0${[1, 2, 3, 5, 7, 8][index]}:00Z`, + })); + for (const checkpoint of evidence.checkpoints.filter(c => c.phase === "finished")) { + checkpoint.runs = [...leadRuns, ...checkpoint.runs.filter(run => run.agentId !== evidence.agentId)]; + } + expect(failures(evidence)).toEqual([]); + const finished = evidence.checkpoints.at(-1)!; + const parent = finished.tasks.find(task => task.id === evidence.onboardingIssueId)!; + const parentRuns = finished.runs.filter(run => run.nativeIssueId === parent.id); + expect(parent.status).toBe(execution.task.expectedTerminalState.issue); + expect(parentRuns).not.toHaveLength(0); + expect(parentRuns.every(run => run.status === execution.task.expectedTerminalState.run)).toBe(true); + expect(matchesRunCount(execution.task, parentRuns.length)).toBe(true); + expect(matchesRunCount(execution.task, 0)).toBe(false); + expect(matchesRunCount(execution.task, execution.task.expectedRunCount + 1)).toBe(false); + }); + + it("rejects a completed writer task when the onboarding parent remains in review", () => { + const evidence = recording(); + for (const checkpoint of evidence.checkpoints.filter(c => c.phase === "finished")) { + checkpoint.tasks.find(task => task.id === evidence.onboardingIssueId)!.status = "in_review"; + } + expect(failures(evidence)).toContain("reload-retains-completion"); + }); + + it("reads the exact saved document revision through its production API", async () => { + const document = recording().checkpoints.at(-1)!.documents[1]; + const get = vi.fn(async (_path: string) => document.savedRevision); + const read = await readApprovalHistoryRevision({ get: async (path: string) => await get(path) as T }, document); + expect(get).toHaveBeenCalledExactlyOnceWith("/api/issues/child/documents/welcome/revisions/welcome-revision"); + expect(read.savedRevision).toMatchObject({ createdByAgentId: "writer", createdByRunId: "child-run" }); + }); + + it.each(["id", "documentId", "issueId", "key", "body"])("rejects a mismatched saved revision %s", async field => { + const document = recording().checkpoints.at(-1)!.documents[1]; + const get = async () => ({ ...document.savedRevision, [field]: "foreign" }) as T; + await expect(readApprovalHistoryRevision({ get }, document)).rejects.toThrow("onboarding_saved_revision_binding_mismatch"); + }); + + it("clears the unchecked plan step and operates all three saved approval controls before reload", async () => { + const evidence = recording(); + const events: string[] = []; + const clickedAt: number[] = []; + const state = (phase: FirstTaskCheckpoint["phase"]) => structuredClone(evidence.checkpoints.find(c => c.phase === phase)!); + let current = state("clarified"); + const selections = new Set(["draft", "publish"]); + const page = { getByRole: (_role: string, { name }: { name: string }) => ({ last: () => ({ + check: async () => { events.push(`check:${name}`); if (name === DRAFT_OPTION) selections.add("draft"); }, + uncheck: async () => { events.push(`uncheck:${name}`); selections.delete("publish"); }, + click: async () => { + clickedAt.push(Date.now()); + // Work can begin while the acceptance response is still in flight. + vi.setSystemTime(Date.now() + 5_000); + const pending = current.interactions.find(card => card.status === "pending")!; + events.push(`accept:${pending.id}`); + current = state(pending.id === "plan-card" ? "plan-accepted" : pending.id === "hire-card" ? "hiring-accepted" : "accepted"); + if (pending.id === "plan-card") current.interactions[0].result.selectedOptionIds = [...selections]; + }, + }) }), reload: async () => { events.push("reload"); } } as unknown as Page; + const snapshot = vi.fn(async (phase: FirstTaskCheckpoint["phase"], _at?: string) => { events.push(phase); return structuredClone(current); }); + vi.useFakeTimers({ toFake: ["Date"] }); + try { + await runApprovalHistoryDecisions({ page, api: { get: async () => structuredClone(current.interactions) as T }, issueId: "parent", + marker: firstTaskScenario(APPROVAL_HISTORY_CASE, "history").marker, deadlineAt: Date.now() + 30_000, + allRuns: async () => current.runs, snapshot, capture: vi.fn(async () => undefined), + settle: async (_runs, completion) => { + current = state(completion ? "finished" : current.phase === "plan-accepted" ? "hiring-proposed" : "execution-proposed"); + }, + }); + expect(snapshot.mock.calls.filter(([phase]) => ["plan-accepted", "hiring-accepted", "accepted"].includes(phase)).map(([, at]) => at)) + .toEqual(clickedAt.map(at => new Date(at).toISOString())); + } finally { vi.useRealTimers(); } + expect([...selections]).toEqual(["draft"]); + expect(events).toEqual([`check:${DRAFT_OPTION}`, `uncheck:${PUBLISH_OPTION}`, "accept:plan-card", "plan-accepted", "hiring-proposed", + `check:Hire ${approvalHistoryWriter(firstTaskScenario(APPROVAL_HISTORY_CASE, "history").marker)}`, "accept:hire-card", "hiring-accepted", + "execution-proposed", "accept:execution-card", "accepted", "finished", "reload", "finished"]); + }); + + it.each void, string]>([ + ["approved run without a child", e => { e.checkpoints.at(-1)!.tasks.pop(); }, "approved-writer-executes-once"], + ["missing execution decision", e => { e.checkpoints.at(-1)!.interactions.pop(); }, "three-separate-human-approvals"], + ["agent-only execution acceptance", e => { e.checkpoints.at(-1)!.interactions.at(-1)!.resolvedByAgentId = "lead"; }, "three-separate-human-approvals"], + ["hiring accepted with no writer selected", e => { e.checkpoints.at(-1)!.interactions[1].result.selectedOptionIds = []; }, "three-separate-human-approvals"], + ["execution confirmation targeted to the plan", e => { e.checkpoints.at(-1)!.interactions.at(-1)!.payload.target = { key: "plan" }; }, "three-separate-human-approvals"], + ["new task before execution approval", e => { e.checkpoints.find(c => c.phase === "execution-proposed")!.tasks.push({ id: "early" }); }, "no-premature-work"], + ["hire before hiring approval", e => { e.checkpoints.find(c => c.phase === "hiring-proposed")!.agents.push({ id: "early" }); }, "no-premature-work"], + ["defaults substituted for actual choices", e => { e.checkpoints.at(-1)!.interactions[0].result.selectedOptionIds = ["draft", "publish"]; }, "selected-plan-scope-retained"], + ["changed plan revision", e => { e.checkpoints.at(-1)!.documents[0].latestRevisionId = "changed"; }, "selected-plan-scope-retained"], + ["task assigned to the lead", e => { e.checkpoints.at(-1)!.tasks[1].assigneeAgentId = "lead"; }, "approved-writer-executes-once"], + ["duplicate child execution", e => { e.checkpoints.at(-1)!.runs.push({ ...e.checkpoints.at(-1)!.runs[1], id: "duplicate" }); }, "approved-writer-executes-once"], + ["no saved output after successful runs", e => { e.checkpoints.at(-1)!.documents.pop(); }, "durable-completion"], + ["no persisted output after reload", e => { e.checkpoints.at(-1)!.documents.pop(); }, "reload-retains-completion"], + ["lead-authored writer output", e => { e.checkpoints.at(-1)!.documents[1].savedRevision.createdByAgentId = "lead"; }, "writer-authored-output"], + ["output from the wrong child run", e => { e.checkpoints.at(-1)!.documents[1].savedRevision.createdByRunId = "parent-run"; }, "writer-authored-output"], + ["missing saved revision proof", e => { delete e.checkpoints.at(-1)!.documents[1].savedRevision; }, "writer-authored-output"], + ["saved revision body mismatch", e => { e.checkpoints.at(-1)!.documents[1].savedRevision.body = "Different output"; }, "writer-authored-output"], + ["output without a current revision reference", e => { delete e.checkpoints.at(-1)!.documents[1].latestRevisionId; }, "writer-authored-output"], + ["writer switched to legacy execution", e => { e.checkpoints.at(-1)!.runs[1].runtimeMode = "legacy"; }, "writer-authored-output"], + ["changed saved output after reload", e => { e.checkpoints.at(-1)!.documents[1].body += " Changed."; }, "reload-retains-completion"], + ["duplicate approval left pending", e => { e.checkpoints.at(-1)!.interactions.push({ id: "duplicate", status: "pending", kind: "request_confirmation" }); }, "no-pending-approvals"], + ["missing intermediate approval observations", e => { e.checkpoints = e.checkpoints.filter(c => c.phase !== "hiring-accepted"); }, "approval-history-stages"], + ])("rejects %s", (_name, mutate, expected) => { + const evidence = recording(); + mutate(evidence); + expect(failures(evidence)).toContain(expected); + }); + + it("retains enough public evidence to regrade actual checkbox choices and the later child run", () => { + const evidence = recording(); + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", suiteId: APPROVAL_HISTORY_SUITE, + executionId: `${APPROVAL_HISTORY_SUITE}.runner-codex.local.${APPROVAL_HISTORY_CASE}`, attempt: 1, status: "passed", + profileId: "runner-codex", environmentId: "local", caseId: APPROVAL_HISTORY_CASE, provider: "openai", model: "fixture-model", + runtimeMode: "native", runIds: ["parent-run", "child-run"], startedAt: "2026-11-01T00:00:00Z", finishedAt: "2026-11-01T00:09:00Z", + durationMs: 540_000, cleanup: "passed", firstTask: evidence }; + const projected = projectFirstTaskResult(result); + validateRetainedRunnerResult(projected); + expect(failures(projected.firstTask!)).toEqual([]); + expect(projected.firstTask!.checkpoints.at(-1)!.interactions[0].result.selectedOptionIds).toEqual(["draft"]); + }); +}); diff --git a/tests/runner-e2e/first-task-approval-history.ts b/tests/runner-e2e/first-task-approval-history.ts new file mode 100644 index 00000000000..6393768c7dc --- /dev/null +++ b/tests/runner-e2e/first-task-approval-history.ts @@ -0,0 +1,121 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerApi } from "./api.js"; +import type { FirstTaskCheck, FirstTaskCheckpoint, FirstTaskEvidence, Row } from "./first-task-scoring.js"; + +export const APPROVAL_HISTORY_SUITE = "onboarding-approval-history"; +export const APPROVAL_HISTORY_CASE = "interview-hire-execute"; +export const DRAFT_OPTION = "Draft welcome note"; +export const PUBLISH_OPTION = "Publish welcome note"; +export const approvalHistoryWriter = (marker: string) => `Garden Writer ${marker}`; +export const approvalHistoryFacts = (marker: string) => [ + "We run a neighborhood garden club for beginners with a free Saturday meetup.", + `Our first deliverable is a two-sentence welcome note saved as a task document, including ${marker}.`, + `Please save a two-step plan and ask me to review it with exactly two checkboxes labeled "${DRAFT_OPTION}" and "${PUBLISH_OPTION}". Initially select both; I will choose the scope.`, + "Plan acceptance authorizes only my selected steps. Do not hire or create a task yet.", + `After plan acceptance, propose hiring exactly one writer named "${approvalHistoryWriter(marker)}" with a separate saved checkbox confirmation. Wait for that approval before hiring.`, + "Use the same provider, model, and AI connection as yourself for the writer. No outside services or additional agents are needed.", + 'After hiring, ask me a separate saved ordinary confirmation labeled "Start drafting" before creating the writer’s task. This is an execution confirmation; keep the saved plan revision unchanged.', + "After I accept Start drafting, assign exactly one child task to that writer, wait for its execution, verify the saved welcome document, and finish this onboarding task. Never publish the note externally.", +].join(" "); + +export function approvalHistoryDefinitionDigest() { + const hash = createHash("sha256"); + for (const file of ["first-task-approval-history.ts", "first-task-approval-history-flow.ts", "first-task-cases.ts", "first-task-flow.ts", "first-task-scoring.ts", "first-task-fixtures.ts"]) { + hash.update(file).update("\0").update(readFileSync(new URL(file, import.meta.url))).update("\0"); + } + return hash.digest("hex"); +} + +/** Read the exact immutable revision observed on the document, including its actual writer/run. */ +export async function readApprovalHistoryRevision(api: Pick, document: Row): Promise { + if (typeof document.latestRevisionId !== "string" || !document.latestRevisionId) throw new Error("onboarding_saved_revision_missing"); + const savedRevision = await api.get(`/api/issues/${document.issueId}/documents/${encodeURIComponent(document.key)}/revisions/${document.latestRevisionId}`); + if (!savedRevision || savedRevision.id !== document.latestRevisionId || savedRevision.documentId !== document.id + || savedRevision.issueId !== document.issueId || savedRevision.key !== document.key || savedRevision.body !== document.body) { + throw new Error("onboarding_saved_revision_binding_mismatch"); + } + return { ...document, savedRevision }; +} + +const acceptedByHuman = (card: Row | undefined) => Boolean(card?.status === "accepted" && card.result?.outcome === "accepted" + && card.resolvedByUserId && !card.resolvedByAgentId && !card.resolvedByRunId && Number.isFinite(Date.parse(card.resolvedAt))); +const planCard = (c: FirstTaskCheckpoint) => c.interactions.find(card => card.kind === "request_checkbox_confirmation" + && card.payload?.target?.type === "issue_document" && card.payload.target.key === "plan"); +const hiringCard = (c: FirstTaskCheckpoint, writer: string) => c.interactions.find(card => card.kind === "request_checkbox_confirmation" + && !card.payload?.target && card.payload?.options?.length === 1 && card.payload.options[0].label.includes(writer)); +const executionCard = (c: FirstTaskCheckpoint) => c.interactions.find(card => card.kind === "request_confirmation" + && !card.payload?.target && /start drafting/i.test(card.payload?.prompt ?? "")); + +/** Independent persisted-state oracle; successful provider runs cannot replace any approval or output. */ +export function gradeApprovalHistory(e: FirstTaskEvidence, marker: string): FirstTaskCheck[] { + const checks: FirstTaskCheck[] = []; + const last = e.checkpoints.at(-1); + if (!last) return [{ id: "approval-history-recorded", passed: false, detail: "Missing onboarding approval checkpoints", evidence: [] }]; + const add = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail, evidence: e.checkpoints.map(c => c.id) }); + const writer = approvalHistoryWriter(marker); + const plan = planCard(last), hiring = hiringCard(last, writer), execution = executionCard(last); + const draft = plan?.payload.options?.find((option: Row) => option.label === DRAFT_OPTION); + const selectedOnlyDraft = (card: Row | undefined) => Boolean(draft && acceptedByHuman(card) + && card?.result.selectedOptionIds?.length === 1 && card.result.selectedOptionIds[0] === draft.id); + const approvedHire = (c: FirstTaskCheckpoint) => { + const card = hiringCard(c, writer); + return acceptedByHuman(card) && card?.result.selectedOptionIds?.length === 1 + && card.result.selectedOptionIds[0] === card.payload.options[0].id; + }; + const newAgents = (c: FirstTaskCheckpoint) => c.agents.filter(agent => !e.checkpoints[0].agents.some(initial => initial.id === agent.id)); + const children = (c: FirstTaskCheckpoint) => c.tasks.filter(task => !e.initialTaskIds.includes(task.id)); + add("no-premature-work", e.checkpoints.every(c => { + const hired = approvedHire(c), executing = acceptedByHuman(executionCard(c)); + return (hired || newAgents(c).length === 0) && (executing || (children(c).length === 0 + && c.documents.every(doc => doc.key === "plan" || /(?:^|[-_])proposal(?:$|[-_])/.test(doc.key)) + && !(c.attachments?.length) && c.tasks.find(task => task.id === e.onboardingIssueId)?.status !== "done")); + }), "Hiring and execution each wait for their own human decision at every checkpoint"); + add("three-separate-human-approvals", acceptedByHuman(plan) && approvedHire(last) && acceptedByHuman(execution) + && new Set([plan?.id, hiring?.id, execution?.id]).size === 3 + && Date.parse(plan!.resolvedAt) <= Date.parse(hiring!.createdAt) + && Date.parse(hiring!.resolvedAt) <= Date.parse(execution!.createdAt), + "A revision-bound checkbox plan, a hiring checkbox, and a later untargeted execution confirmation were independently accepted"); + const afterPlan = e.checkpoints.filter(c => ["plan-accepted", "hiring-proposed", "hiring-accepted", "execution-proposed", "accepted", "finished"].includes(c.phase)); + add("selected-plan-scope-retained", afterPlan.length > 0 && afterPlan.every(c => { + const card = planCard(c); + if (!card) return false; + return selectedOnlyDraft(card) && card.id === plan?.id && card.payload.target.revisionId === plan?.payload.target.revisionId + && c.documents.some(doc => doc.issueId === e.onboardingIssueId && doc.key === "plan" + && (!card.payload.target.documentId || doc.id === card.payload.target.documentId) + && doc.latestRevisionId === card.payload.target.revisionId); + }), "Only the selected draft option and exact accepted plan revision survive hiring, execution, and reload"); + const workers = newAgents(last), tasks = children(last); + add("approved-writer-executes-once", workers.length === 1 && workers[0].name === writer + && approvedHire(last) && Date.parse(workers[0].createdAt) >= Date.parse(hiring!.resolvedAt) + && tasks.length === 1 && tasks[0].parentId === e.onboardingIssueId && tasks[0].assigneeAgentId === workers[0].id + && acceptedByHuman(execution) && Date.parse(tasks[0].createdAt) >= Date.parse(execution!.resolvedAt) + && last.runs.filter(run => (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId) === tasks[0].id + && run.agentId === workers[0].id && run.startedAt).length === 1, + "Exactly the approved writer owns one task and starts one child execution after Start drafting is accepted"); + const stages = ["plan-accepted", "hiring-proposed", "hiring-accepted", "execution-proposed", "accepted", "finished"]; + add("approval-history-stages", stages.every(phase => e.checkpoints.some(c => c.phase === phase)), "Every approval and waiting boundary was observed"); + add("no-pending-approvals", last.interactions.every(card => card.status !== "pending"), "Completion leaves no pending human request or duplicate approval"); + const finished = e.checkpoints.filter(c => c.phase === "finished"); + add("writer-authored-output", workers.length === 1 && tasks.length === 1 && finished.length >= 2 + && finished.slice(-2).every(c => c.documents.some(doc => { + const revision = doc.savedRevision; + if (!revision || typeof doc.latestRevisionId !== "string") return false; + const run = c.runs.find(row => row.id === revision.createdByRunId); + return doc.issueId === tasks[0].id && doc.key !== "plan" && String(doc.body).includes(marker) + && revision.id === doc.latestRevisionId && revision.documentId === doc.id && revision.issueId === doc.issueId + && revision.key === doc.key && revision.body === doc.body && revision.createdByAgentId === workers[0].id + && !revision.createdByUserId && run?.agentId === workers[0].id && run.status === "succeeded" && run.runtimeMode === "native" + && (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId) === tasks[0].id && Boolean(run.startedAt); + })), "The saved output's exact revision was authored by the approved writer in its actual child execution"); + const output = (c: FirstTaskCheckpoint) => c.documents.filter(doc => doc.issueId === tasks[0]?.id && doc.key !== "plan") + .map(doc => ({ id: doc.id, key: doc.key, revisionId: doc.latestRevisionId, body: doc.body })) + .sort((a, b) => a.key.localeCompare(b.key) || a.id.localeCompare(b.id)); + add("reload-retains-completion", finished.length >= 2 && tasks.length === 1 && tasks[0].status === "done" + && last.tasks.find(task => task.id === e.onboardingIssueId)?.status === "done" + && finished.slice(-2).every(c => c.tasks.some(task => task.id === tasks[0].id && task.status === "done") + && c.documents.some(doc => doc.issueId === tasks[0].id && doc.key !== "plan" && String(doc.body).includes(marker))) + && JSON.stringify(output(finished.at(-2)!)) === JSON.stringify(output(last)), + "Reload retains the completed assigned child, saved welcome document, and completed onboarding parent"); + return checks; +} diff --git a/tests/runner-e2e/first-task-cases.ts b/tests/runner-e2e/first-task-cases.ts index 8244f0ead12..18d0e5a4676 100644 --- a/tests/runner-e2e/first-task-cases.ts +++ b/tests/runner-e2e/first-task-cases.ts @@ -1,4 +1,5 @@ import type { RunnerTaskFixture } from "./types.js"; +import { APPROVAL_HISTORY_CASE, approvalHistoryFacts } from "./first-task-approval-history.js"; export const FIRST_TASK_CASES = [ ["interview-first-response", "Interview: first response", "interview", 1], @@ -35,12 +36,13 @@ export const FIRST_TASK_CASES = [ ["revise-accept", "Revise scope before accepting", "task", 4], ["reject-no-execution", "Decline proposed work", "task", 2], ] as const; -export type FirstTaskCaseId = (typeof FIRST_TASK_CASES)[number][0]; +const approvalHistoryCase = [APPROVAL_HISTORY_CASE, "Interview, plan scope, hiring, and execution approval", "interview", 7] as const; +export type FirstTaskCaseId = (typeof FIRST_TASK_CASES)[number][0] | typeof APPROVAL_HISTORY_CASE; export type FirstTaskOpening = (typeof FIRST_TASK_CASES)[number][2]; export const firstTaskMarker = (nonce: string) => `GARDEN${nonce.replace(/[^a-z0-9]/gi, "")}`; export function firstTaskScenario(id: string, nonce: string) { - const entry = FIRST_TASK_CASES.find((row) => row[0] === id); + const entry = FIRST_TASK_CASES.find((row) => row[0] === id) ?? (id === APPROVAL_HISTORY_CASE ? approvalHistoryCase : undefined); if (!entry) throw new Error(`Unknown first-task case ${id}`); const marker = firstTaskMarker(nonce); const originalMarker = id === "revise-accept" ? `DRAFT${marker}` : marker; @@ -51,7 +53,7 @@ export function firstTaskScenario(id: string, nonce: string) { opening: entry[2], marker, originalMarker, - facts, + facts: id === APPROVAL_HISTORY_CASE ? approvalHistoryFacts(marker) : facts, firstResponseOnly: id.endsWith("first-response") || id === "ordinary-task-control", prompt: @@ -68,8 +70,7 @@ export function firstTaskScenario(id: string, nonce: string) { "No, do not proceed. I do not want this work done. Please acknowledge that and stop.", }; } -export const firstTaskTasks: readonly RunnerTaskFixture[] = - FIRST_TASK_CASES.map(([id, label, , expectedRunCount]) => ({ +const buildFirstTaskFixture = ([id, label, , expectedRunCount]: readonly [string, string, string, number]): RunnerTaskFixture => ({ id, label, groups: ["onboarding"], @@ -82,4 +83,11 @@ export const firstTaskTasks: readonly RunnerTaskFixture[] = buildPrompt: (nonce) => firstTaskScenario(id, nonce).prompt, buildVisibleMarker: firstTaskMarker, buildMatchers: () => [], // Checkpoints and state effects are graded by first-task-scoring. - })); + }); +export const firstTaskTasks: readonly RunnerTaskFixture[] = FIRST_TASK_CASES.map(buildFirstTaskFixture); +export const onboardingApprovalHistoryTask = { + ...buildFirstTaskFixture(approvalHistoryCase), + expectedTerminalState: { issue: "done", run: "succeeded" } as const, + minimumExpectedRunCount: 1, + automaticRetryPolicy: "single_attempt" as const, +}; diff --git a/tests/runner-e2e/first-task-fixtures.ts b/tests/runner-e2e/first-task-fixtures.ts index 45f1d39df68..c20552ae2de 100644 --- a/tests/runner-e2e/first-task-fixtures.ts +++ b/tests/runner-e2e/first-task-fixtures.ts @@ -4,6 +4,7 @@ import type { RunnerApi } from "./api.js"; import { deleteDaytonaEnvironment, installFixtureDaytonaProvider, type LiveFixtureValues } from "./live-fixtures.js"; import { isImmutableDaytonaImage } from "./catalog.js"; import type { CredentialName, MatrixExecution } from "./types.js"; +import { APPROVAL_HISTORY_SUITE } from "./first-task-approval-history.js"; export const MANAGED_FIRST_TASK_CELL = "confirmation-replies.runner-codex.daytona.task-card-accept"; export function isManagedFirstTask(execution: Pick): boolean { @@ -63,7 +64,7 @@ export async function provisionFirstTaskFixtures(input: { const { api, execution, nonce, company, credentials } = input; const managed = isManagedFirstTask(execution); if ( - !["first-task", "completion-updates", "confirmation-replies"].includes(execution.suite.id) || execution.task.flow !== "first_task" || + !["first-task", "completion-updates", "confirmation-replies", APPROVAL_HISTORY_SUITE].includes(execution.suite.id) || execution.task.flow !== "first_task" || (execution.environment.id !== "local" && !managed) || ![ "legacy-codex", diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index f131ee9131c..d1201dae5f7 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -6,6 +6,8 @@ import { firstTaskRejectionReplyRecorded, isFirstTaskRejectionCancellation } fro import { answerableRuntimeRunIds } from "./runtime-question-readiness.js"; import { captureFirstTaskAttachments } from "./first-task-attachments.js"; import { waitForFirstTaskReply } from "./first-task-replies.js"; +import { APPROVAL_HISTORY_CASE, readApprovalHistoryRevision } from "./first-task-approval-history.js"; +import { runApprovalHistoryDecisions } from "./first-task-approval-history-flow.js"; import { observeCompletionUpdate } from "./completion-update-flow.js"; import { provisionFirstTaskFixtures, @@ -75,16 +77,19 @@ export function projectFirstTaskResult(result: RunnerE2EResult): RunnerE2EResult const answerRows: Row[] = Array.isArray(answers) ? rows(answers) : answers && typeof answers === "object" ? Object.entries(answers).map(([questionId, answer]) => ({ ...answer as Row, questionId })) : []; return { - ...pick(row, ["id", "issueId", "kind", "status", "createdAt", "resolvedAt", "resolvedByAgentId"]), + ...pick(row, ["id", "issueId", "kind", "status", "createdAt", "resolvedAt", "resolvedByAgentId", "resolvedByRunId", "sourceRunId"]), ...(row.resolvedByUserId ? { resolvedByUserId: "fixture-board" } : {}), payload: { ...pick(payload, ["version", "title", "prompt", "detailsMarkdown", "acceptLabel", "rejectLabel", "rejectRequiresReason", "rejectReasonLabel", "submitLabel"]), ...(payload?.questions ? questionSet(payload) : {}), ...(payload?.questionSet ? { questionSet: questionSet(payload.questionSet) } : {}), - ...(payload?.target ? { target: pick(payload.target, ["type", "key", "label", "revisionNumber"]) } : {}), + ...(payload?.target ? { target: pick(payload.target, ["type", "key", "label", "revisionNumber", "issueId", "documentId", "revisionId"]) } : {}), + ...(payload?.options ? { options: rows(payload.options).map(option => pick(option, ["id", "label", "description"])) } : {}), + ...(payload?.defaultSelectedOptionIds ? { defaultSelectedOptionIds: strings(payload.defaultSelectedOptionIds) } : {}), }, ...(row.result ? { result: { ...pick(row.result, ["outcome", "reason", "commentId"]), + ...(row.result.selectedOptionIds ? { selectedOptionIds: strings(row.result.selectedOptionIds) } : {}), answers: answerRows.map(answer => ({ ...pick(answer, ["id", "questionId", "text", "otherText", "customText"]), optionIds: strings(answer.optionIds ?? answer.selectedOptionIds), @@ -112,14 +117,17 @@ export function projectFirstTaskResult(result: RunnerE2EResult): RunnerE2EResult checkpoints: e.checkpoints.map(c => ({ id: c.id, at: c.at, phase: c.phase, issueId: c.issueId, tasks: c.tasks.map(t => pick(t, ["id", "identifier", "title", "description", "status", "parentId", "assigneeAgentId", "createdAt", "completedAt"])), - agents: c.agents.map(a => ({ ...pick(a, ["id", "name", "adapterType"]), + agents: c.agents.map(a => ({ ...pick(a, ["id", "name", "adapterType", "createdAt"]), adapterConfig: pick(a.adapterConfig, ["provider", "model", "modelReasoningEffort", "lifecycleMode", "codexPermissionMode"]) })), comments: c.comments.map(comment => ({ ...pick(comment, ["id", "issueId", "body", "authorAgentId", "createdAt"]), ...(comment.authorUserId ? { authorUserId: "fixture-board" } : {}) })), interactions: c.interactions.map(interaction), - documents: c.documents.map(d => pick(d, ["id", "issueId", "key", "title", "body", "format", "latestRevisionId", "latestRevisionNumber", "createdAt", "updatedAt"])), + documents: c.documents.map(d => ({ ...pick(d, ["id", "issueId", "key", "title", "body", "format", "latestRevisionId", "latestRevisionNumber", "createdAt", "updatedAt"]), + ...(d.savedRevision ? { savedRevision: { ...pick(d.savedRevision, ["id", "documentId", "issueId", "key", "body", "createdByAgentId", "createdByRunId"]), + ...(d.savedRevision.createdByUserId ? { createdByUserId: "fixture-board" } : {}) } } : {}), + })), attachments: (c.attachments ?? []).map(a => pick(a, ["id", "issueId", "title", "originalFilename", "filename", "body", "contentVerified", "contentSha256", "byteSize", "createdAt"])), - runs: c.runs.map(run => ({ ...pick(run, ["id", "issueId", "status", "startedAt", "finishedAt", "runtimeMode", "driverKind"]), + runs: c.runs.map(run => ({ ...pick(run, ["id", "issueId", "nativeIssueId", "agentId", "status", "startedAt", "finishedAt", "runtimeMode", "driverKind"]), ...pick({ model: run.resultJson?.model ?? run.usageJson?.model }, ["model"]) })), })), checks: e.checks.map(check => ({ @@ -469,13 +477,15 @@ export async function runFirstTaskFlow(input: { `/api/issues/${t.id}/documents`, ); return Promise.all( - summaries.map(async (d) => ({ - ...(await api.get( + summaries.map(async (d) => { + const document = { ...(await api.get( `/api/issues/${t.id}/documents/${encodeURIComponent(d.key)}`, )), key: d.key, issueId: t.id, - })), + }; + return scenario.id === APPROVAL_HISTORY_CASE ? readApprovalHistoryRevision(api, document) : document; + }), ); }), ) @@ -728,7 +738,7 @@ export async function runFirstTaskFlow(input: { if (!scenario.firstResponseOnly) { if (["interview", "ambiguous"].includes(scenario.opening)) { const facts = - scenario.id === "interview-plan-accept" + ["interview-plan-accept", APPROVAL_HISTORY_CASE].includes(scenario.id) ? `${scenario.facts} Please write a plan for me to review, before doing the work.` : scenario.facts; const pending = ( @@ -785,7 +795,10 @@ export async function runFirstTaskFlow(input: { if (scenario.id === "revise-accept") await turn(scenario.revision, "revised"); assertBeforeAcceptance(); - if (scenario.id === "reject-no-execution") + if (scenario.id === APPROVAL_HISTORY_CASE) { + await runApprovalHistoryDecisions({ page, api, issueId: issue.id, marker: scenario.marker, deadlineAt, + allRuns, snapshot, settle, capture: input.capture }); + } else if (scenario.id === "reject-no-execution") await turn(scenario.rejection, "rejected"); else if (["task-card-accept", "accept-while-running"].includes(scenario.id)) { const pending = ( diff --git a/tests/runner-e2e/first-task-scoring.ts b/tests/runner-e2e/first-task-scoring.ts index fa7b49e0807..9ccd20151d0 100644 --- a/tests/runner-e2e/first-task-scoring.ts +++ b/tests/runner-e2e/first-task-scoring.ts @@ -5,6 +5,7 @@ import { sanitizeJson } from "./redaction.js"; import { createHash } from "node:crypto"; import { firstTaskScenario } from "./first-task-cases.js"; import { completionDelivery } from "./completion-updates.js"; +import { APPROVAL_HISTORY_CASE, approvalHistoryWriter, gradeApprovalHistory } from "./first-task-approval-history.js"; export type Row = { id: string; [key: string]: any }; export interface FirstTaskCheckpoint { id: string; @@ -14,6 +15,10 @@ export interface FirstTaskCheckpoint { | "response" | "clarified" | "revised" + | "plan-accepted" + | "hiring-proposed" + | "hiring-accepted" + | "execution-proposed" | "accepted" | "rejected" | "finished"; @@ -311,7 +316,7 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { "Propose a task for the concrete request", [first.id], ); - if (scenario.opening === "plan" || e.caseId === "interview-plan-accept") { + if (scenario.opening === "plan" || ["interview-plan-accept", APPROVAL_HISTORY_CASE].includes(e.caseId)) { const proposal = e.checkpoints.find((c) => c.phase === "clarified") ?? first; add( @@ -328,7 +333,9 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { c.phase !== "opening" && (!approval || Date.parse(c.at) < Date.parse(approval.at)), ); - if (scenario.opening !== "ordinary") { + if (e.caseId === APPROVAL_HISTORY_CASE) { + checks.push(...gradeApprovalHistory(e, scenario.marker)); + } else if (scenario.opening !== "ordinary") { add( "no-premature-work", before.every( @@ -402,12 +409,14 @@ export function gradeFirstTask(e: FirstTaskEvidence): FirstTaskCheck[] { ); } else { const children = extras(last); + const expectedAssignee = e.caseId === APPROVAL_HISTORY_CASE + ? last.agents.find(agent => agent.name === approvalHistoryWriter(scenario.marker))?.id : e.agentId; add( "one-scoped-subtask", children.length === 1 && children[0].parentId === e.onboardingIssueId && - children[0].assigneeAgentId === e.agentId, - "Exactly one approved subtask belongs to this onboarding issue and agent", + Boolean(expectedAssignee) && children[0].assigneeAgentId === expectedAssignee, + "Exactly one approved subtask belongs to this onboarding issue and its intended agent", [last.id], notReached, ); diff --git a/tests/runner-e2e/first-task.test.ts b/tests/runner-e2e/first-task.test.ts index a5b30654497..5e9231cdd78 100644 --- a/tests/runner-e2e/first-task.test.ts +++ b/tests/runner-e2e/first-task.test.ts @@ -1591,7 +1591,7 @@ describe("managed onboarding through the production wizard", () => { expect(h.api.get).toHaveBeenCalledExactlyOnceWith("/api/instance/settings"); expect(h.api.patch).toHaveBeenLastCalledWith("/api/instance/settings", { defaultEnvironmentId: previousDefault }); expect(h.state.experimental.enableAgentChat).toBe(true); - expect(h.api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true }); + expect(h.api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true, timeout: 30_000 }); }, ); @@ -1722,7 +1722,7 @@ describe("managed onboarding through the production wizard", () => { expect(fixtures.setupLeases).toEqual([]); expect(JSON.stringify(fixtures)).not.toContain("synthetic"); await fixtures.teardown(); - expect(api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true }); + expect(api.delete).toHaveBeenCalledExactlyOnceWith("/api/environments/environment?destroyReusableSandboxLeases=true", { allowNotFound: true, timeout: 30_000 }); }); it.each(["missing-key", "mutable-image", "other-cell"])("rejects %s before admitting a managed browser fixture", async mode => { diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 157f2ffce49..5c0386a76c9 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -71,6 +71,7 @@ associate(["first-task", "confirmation-replies", "completion-updates"], Object.f "revise-accept", "reject-no-execution", ].map((id) => [id, ["onboarding/first-task"]]))); associate(["first-task"], { "ordinary-task-control": ["tasks/completion"] }); +associate(["onboarding-approval-history"], { "interview-hire-execute": ["onboarding/first-task", "questions-and-approvals/continuation", "tasks/completion"] }); associate(["confirmation-replies"], { "confirmation-ambiguous": ["questions-and-approvals/confirmations", "questions-and-approvals/audience"], "unanswered-question-return": ["questions-and-approvals/dismiss-and-reopen", "questions-and-approvals/continuation"], diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 9906e0d4869..1469a322867 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -353,7 +353,7 @@ const fields = { source: optional(shape({ sha: string, ref: string, dirty: boolean })), initialTaskIds: array(string), configuredModel: nullable(string), observedModels: array(string), runtimeSettings: optional(object), instructions: array(shape({ path: string, content: string, sha256: string, contentSha256: optional(string), redacted: optional(boolean) })), - checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "accepted", "rejected", "finished"), issueId: string, + checkpoints: array(shape({ id: string, at: date, phase: oneOf("opening", "response", "clarified", "revised", "plan-accepted", "hiring-proposed", "hiring-accepted", "execution-proposed", "accepted", "rejected", "finished"), issueId: string, tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), attachments: optional(array(object)), runs: array(object) })), checks: array(shape({ id: string, passed: boolean, notReached: optional(string), evidence: array(string), detail: string })), })),