diff --git a/maps/agent-access-map/onboarding.md b/maps/agent-access-map/onboarding.md index 9c5dded3eff..5160136841c 100644 --- a/maps/agent-access-map/onboarding.md +++ b/maps/agent-access-map/onboarding.md @@ -189,6 +189,16 @@ conversational resolution with a genuine later same-task user message supplies acceptance; an agent-only resolution does not. A plan approval must name the current same-task revision. Newer pending/rejected proposals stop new creation. Already admitted task replay and ordinary delegation retain their existing rules. + +Native `paperclip_finish` does not grant completion authority from an opening +answer. The onboarding parent requires a trusted saved human decision on its +latest proposal. Acceptance permits completion under the existing policy; +rejection permits acknowledgment and closure without authorizing new work. +The native status commit checks this decision again under its existing task/run +locks. This adds no new agent endpoint or cross-company authority. The +[native completion controls](../../server/src/services/native-runtime/paperclip-control-plane-port.test.ts) +check persisted outcomes on disposable PostgreSQL; they are not live Cloud +credential or onboarding qualification. Source: [creation admission](../../server/src/services/onboarding-first-task-admission.ts). Controls: [native/REST PostgreSQL regressions](../../server/src/__tests__/onboarding-first-task-admission.test.ts). These controls do not qualify a live Cloud provider journey. diff --git a/maps/feature-map/onboarding.md b/maps/feature-map/onboarding.md index 347613e3829..d90025b2a31 100644 --- a/maps/feature-map/onboarding.md +++ b/maps/feature-map/onboarding.md @@ -142,6 +142,15 @@ admit new work. [Admission controls](../../server/src/__tests__/onboarding-first exercise native and REST calls against disposable PostgreSQL; live model and Cloud onboarding remain separate evidence. +Native completion also requires a saved human proposal decision. An opening +answer alone must leave the first task in review. Verify that premature native +completion is refused, including a result already saved before finalization. +Accepted proposals and acknowledged human rejections can close the first task; +rejection still does not authorize new tasks. The +[native completion controls](../../server/src/services/native-runtime/paperclip-control-plane-port.test.ts) +exercise these boundaries against disposable PostgreSQL. These controls do not +identify a historical Cloud failure or establish live onboarding success. + For accepted file-producing work, verify exactly one self-assigned child with a self-contained description. The onboarding run must yield through the existing dependency/block disposition so the child's own execution can publish its file. diff --git a/server/src/services/native-runtime/native-completion-feedback.ts b/server/src/services/native-runtime/native-completion-feedback.ts index 9bb94b39c72..668af156fc3 100644 --- a/server/src/services/native-runtime/native-completion-feedback.ts +++ b/server/src/services/native-runtime/native-completion-feedback.ts @@ -1,4 +1,5 @@ import { pendingTaskRepository } from "../task-workspace-tools.js"; +import { onboardingFirstTaskCompletionAllowed } from "../onboarding-first-task-admission.js"; import { eligibleIssueMonitorWait } from "../issue-monitors.js"; import { assertNativeCaseHandoffResult } from "./native-case-handoff.js"; import { activeIssueInteractionCondition, ordinaryQuestionCondition } from "../issue-question-context.js"; @@ -140,6 +141,11 @@ export async function nativeCompletionFeedback( if (issue.executionRunId && issue.executionRunId !== runId) { return "Report accepted; a newer run owns the task. Do not claim this report changed its status."; } + if (result.reportedWorkDisposition === "done" && !await onboardingFirstTaskCompletionAllowed(db, { + companyId: run.companyId, issueId: issue.id, runId, + })) { + throw new Error("First-task proposal requires a persisted human decision before completing onboarding. The opening answer supplies scope, not approval. Present the proposal and keep this task in_review until the human accepts or rejects it; acknowledge a rejection without executing that work."); + } const continuation = run.contextSnapshot?.executionContinuation as { objective?: unknown } | undefined; const objective = typeof continuation?.objective === "string" ? continuation.objective : [issue.title, issue.description].filter(Boolean).join("\n"); diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 4ade80add27..88efc195b86 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -1,4 +1,5 @@ import { pendingNativeGovernance } from "./native-governance.js"; +import { onboardingFirstTaskCompletionAllowed } from "../onboarding-first-task-admission.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { reconcileMuseNativeTerminalRun } from "./muse-terminal-reconciliation.js"; import { assertNativeCaseHandoffResult } from "./native-case-handoff.js"; @@ -1429,7 +1430,7 @@ export async function finalizeNativeRun(input: { authoritativeIssue.executionRunId !== run.id && authoritativeIssue.checkoutRunId !== run.id ? await readNativeMonitorDelivery(input.db, { companyId: run.companyId, issueId: coordinator.issueId, runId: run.id }) : null; - const proposedDecision = resolveNativeFinalizerStatus({ + let proposedDecision = resolveNativeFinalizerStatus({ caseHandoffAuthorized: Boolean(await assertNativeCaseHandoffResult(input.db, run.id, authoritativeIssue.id, result)), monitorWaitDispatched: monitorDelivery !== null, monitorWaitPreserved: preservedMonitorWait !== null, @@ -1471,6 +1472,14 @@ export async function finalizeNativeRun(input: { agentId: run.agentId, priorIssueStatus: authoritativeStatus(authoritativeIssue.status), }); + // Already accepted results still require the server-owned first-task decision. + // Keep the task open without inventing an interaction or rerunning the work. + if (proposedDecision.statusAction === "done" && !await onboardingFirstTaskCompletionAllowed(input.db, { + companyId: run.companyId, issueId: authoritativeIssue.id, runId: run.id, + })) { + proposedDecision = { ...proposedDecision, statusAction: "in_review", toStatus: "in_review", + reasonCode: "onboarding_first_task_decision_required", unblockDescriptor: null, effects: [{ kind: "release_checkout" }] }; + } const decision = conversationNativeDecision({ conversation: isConversation(authoritativeIssue), terminalState, workspaceFinalizeStatus: input.workspaceFinalizeStatus, hasGovernanceGate: !!governanceGate, diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts index f8b9eb1d022..4676979de87 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.test.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.test.ts @@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { asc, eq, sql } from "drizzle-orm"; import { generateKeyPairSync, randomUUID } from "node:crypto"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; +import { ONBOARDING_FIRST_TASK_ORIGIN_KIND } from "@paperclipai/shared"; import { readCompletedAssistantMessageCandidate, resolveHeartbeatRunResponse, selectHeartbeatRunFinalAgentMessage } from "../heartbeat-run-summary.js"; import { activityLog, @@ -46,11 +47,13 @@ import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js"; import { finalizeNativeRun, repairCommittedNativeChatResponse } from "./native-run-finalizer.js"; import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; +import { documentService } from "../documents.js"; import { claimNativeReviewExecutionLock, getNativeReviewAssignment } from "./native-review-participant.js"; import { claimQueuedNativeReviewRun } from "./native-review-dispatch.js"; import { stageNativeRunnerWakeAttachments } from "./native-runner-file-handoff.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; +import { buildOnboardingFirstTaskOpeningQuestion, ONBOARDING_FIRST_TASK_OPENING_QUESTION_ID } from "../onboarding-first-task-assets.js"; import { assertCurrentWakeCommentsRead } from "./current-wake-comments.js"; import { buildExecutionContinuation } from "../execution-continuation.js"; import { childReviewOutcomes } from "./child-review-outcomes.js"; @@ -1709,6 +1712,191 @@ describe("PaperclipControlPlanePort conformance", () => { }); + const allowedFirstTaskCompletionScenarios = [ + "ordinary task", "accepted proposal", "accepted conversational proposal", "accepted current plan", "acknowledged rejection", "acknowledged plan rejection", + ] as const; + const refusedFirstTaskCompletionScenarios = [ + "opening answer only", "agent-resolved without human comment", "quarantined human comment", "newest proposal pending", "stale plan acceptance", + ] as const; + type FirstTaskCompletionScenario = typeof allowedFirstTaskCompletionScenarios[number] + | typeof refusedFirstTaskCompletionScenarios[number]; + + async function prepareFirstTaskCompletion(scenario: FirstTaskCompletionScenario) { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const runId = randomUUID(); + const localContractId = randomUUID(); + const runnerInstanceId = randomUUID(); + const userId = `first-task-board-${randomUUID()}`; + const identity = { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, companyId, agentId, issueId, runId }; + const onboarding = scenario !== "ordinary task"; + const rejected = scenario === "acknowledged rejection" || scenario === "acknowledged plan rejection"; + await db.insert(companies).values({ id: companyId, name: "First-task completion", issuePrefix: `F${companyId.slice(0, 6).toUpperCase()}` }); + await db.insert(authUsers).values({ id: userId, name: "First-task board", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id: agentId, companyId, name: "First-task agent", adapterType: "codex_local", status: "running" }); + await db.insert(issues).values({ id: issueId, companyId, title: onboarding ? "Paperclip onboarding" : "Ordinary low-risk task", + description: onboarding ? "/first-task\nSingle-task proposal mode: confirmation" : "Complete the requested ordinary task. Mentioning /first-task does not grant the server-owned onboarding origin.", + originKind: onboarding ? ONBOARDING_FIRST_TASK_ORIGIN_KIND : "manual", status: "in_progress", assigneeAgentId: agentId, + responsibleUserId: userId, workMode: "standard" }); + await db.insert(completionContracts).values({ id: localContractId, companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v3", risk: "low", completionAuthority: "agent_claim_policy", + incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: { revision: "first-task-native-finish-v1", + objective: onboarding ? "Understand the user request, propose one task, and honor their decision" : "Complete the requested ordinary task", + criteria: [{ id: "objective", requirement: "Complete the requested workflow" }] }, + canonicalSha256: localContractId, createdByActorType: "system", createdByActorId: "test" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: userId, status: "succeeded", runtimeMode: "native", + nativeIssueId: issueId, nativeSessionId: identity.sessionId, runnerInstanceId, completionContractId: localContractId, + completionContractSha256: localContractId, contextSnapshot: { issueId, + // A claimed acceptance copied into continuation context is not the + // persisted human decision; the opening-answer cases must still refuse it. + ...(scenario === "opening answer only" ? { executionContinuation: { humanResponses: [{ id: randomUUID(), + kind: "request_confirmation", status: "accepted", resolvedByUserId: userId, result: { version: 1, outcome: "accepted" } }] } } : {}), + } }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + if (onboarding) { + await db.insert(issueThreadInteractions).values({ companyId, issueId, kind: "ask_user_questions", status: "answered", + createdByAgentId: agentId, payload: await buildOnboardingFirstTaskOpeningQuestion(), + result: { version: 1, answers: [{ questionId: ONBOARDING_FIRST_TASK_OPENING_QUESTION_ID, optionIds: ["task"], otherText: "Help organize my next task." }] }, + resolvedByUserId: userId, resolvedAt: new Date() }); + } + if (onboarding && scenario !== "opening answer only") { + const proposalAt = new Date(Date.now() - 2_000); + const plan = scenario === "accepted current plan" || scenario === "stale plan acceptance" || scenario === "acknowledged plan rejection" + ? (await documentService(db).upsertIssueDocument({ issueId, key: "plan", title: "First task plan", format: "markdown", + body: "Retain this plan for the user's next task.", createdByAgentId: agentId, createdByRunId: runId })).document + : null; + const conversational = scenario === "accepted conversational proposal" || scenario === "quarantined human comment"; + const comment = conversational ? (await db.insert(issueComments).values({ companyId, issueId, + authorUserId: userId, authorType: "user", body: "Yes, proceed with that proposal.", + createdAt: new Date(proposalAt.getTime() + 1_000), + ...(scenario === "quarantined human comment" + ? { sourceTrust: { preset: "low_trust_review" as const, disposition: "quarantined" as const } } : {}), + }).returning())[0]! : null; + const agentResolved = conversational || scenario === "agent-resolved without human comment"; + const outcome = rejected ? "rejected" as const : "accepted" as const; + // Persisted agent interpretations need a real, trusted human reply. The + // missing/quarantined variants simulate an untrusted stored result. + await db.insert(issueThreadInteractions).values({ companyId, issueId, + kind: plan ? "request_checkbox_confirmation" : "request_confirmation", status: outcome, + createdByAgentId: agentId, sourceRunId: runId, createdAt: proposalAt, + payload: plan ? { version: 1, prompt: "May I retain this plan?", + options: [{ id: "retain-plan", label: "Retain the plan" }], defaultSelectedOptionIds: ["retain-plan"], minSelected: 1, + target: { type: "issue_document", key: "plan", issueId, documentId: plan.id, revisionId: plan.latestRevisionId! }, + } : { version: 1, prompt: "May I create one task to organize your agenda?" }, + result: { version: 1, outcome, ...(plan && outcome === "accepted" ? { selectedOptionIds: ["retain-plan"] } : {}), + ...(comment ? { commentId: comment.id } : {}), + ...(outcome === "rejected" ? { reason: "Do not do this work." } : {}) }, + resolvedByUserId: agentResolved ? null : userId, + resolvedByAgentId: agentResolved ? agentId : null, resolvedByRunId: agentResolved ? runId : null, + resolvedAt: new Date(), + }); + if (scenario === "newest proposal pending") { + await db.insert(issueThreadInteractions).values({ companyId, issueId, + kind: "request_checkbox_confirmation", status: "pending", createdByAgentId: agentId, sourceRunId: runId, + createdAt: new Date(proposalAt.getTime() + 1_000), + payload: { version: 1, prompt: "Accept the revised task scope?", options: [{ id: "revised", label: "Use revised scope" }], + defaultSelectedOptionIds: ["revised"], minSelected: 1 }, + }); + } + if (scenario === "stale plan acceptance" || scenario === "acknowledged plan rejection") { + await documentService(db).upsertIssueDocument({ issueId, key: "plan", format: "markdown", + baseRevisionId: plan!.latestRevisionId, body: "A revised scope that has no persisted human decision.", + createdByAgentId: agentId, createdByRunId: runId }); + } + } + const port = new PaperclipControlPlanePort(db, { companyId, issueId, runId, agentId, sessionId: identity.sessionId, + completionContractId: localContractId, completionContractSha256: localContractId, + sourceInstanceId: runnerInstanceId, controlPlaneSourceInstanceId: "first-task-native-finish-control" }); + await port.openRun({ identity, backendKind: "mock", sourceInstanceId: runnerInstanceId }); + const result: PrpStructuredRunResult = { ...structuredClone(CONTROL_PLANE_CONFORMANCE_RESULT), + summary: rejected + ? "You declined the proposal. I will stop here and have not created or performed the proposed task." + : "The requested workflow is complete.", + completionClaim: { contractRevision: "first-task-native-finish-v1", objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, + evidence: [], verification: [{ commandOrCheck: "Check the current task state", status: "passed" }], attentionRequests: [] }; + return { companyId, agentId, issueId, runId, port, result }; + } + + it.each(refusedFirstTaskCompletionScenarios)( + "refuses native completion feedback without a trusted current first-task decision (%s)", async (scenario) => { + const { issueId, runId, result } = await prepareFirstTaskCompletion(scenario); + await expect(nativeCompletionFeedback(db, runId, result)) + .rejects.toThrow("First-task proposal requires a persisted human decision"); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.status).toBe("in_progress"); + expect(await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId))).toEqual([]); + expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, runId))).toEqual([]); + }); + + it.each([...allowedFirstTaskCompletionScenarios, ...refusedFirstTaskCompletionScenarios])( + "honors the first-task human decision during native completion (%s)", async (scenario) => { + const { issueId, runId, port, result } = await prepareFirstTaskCompletion(scenario); + const allowed = allowedFirstTaskCompletionScenarios.some(candidate => candidate === scenario); + if (allowed) await expect(nativeCompletionFeedback(db, runId, result)).resolves.toContain("Completion report accepted"); + // Invalid results bypass only early feedback to exercise the backstop for a + // report already committed by a disconnected or older controller. + await port.completeRun({ result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, callerResultId: "first-task-native-finish-result" }); + await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + + const [completedIssue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId)); + if (!allowed) { + const expectedReason = scenario === "newest proposal pending" ? "governed_gate_pending" : "onboarding_first_task_decision_required"; + expect(completedIssue).toMatchObject({ status: "in_review" }); + expect(decisions).toContainEqual(expect.objectContaining({ toStatus: "in_review", reasonCode: expectedReason })); + expect(decisions).not.toContainEqual(expect.objectContaining({ toStatus: "done" })); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]!.resultJson) + .toMatchObject({ finalizationReasonCode: expectedReason }); + } else { + expect(completedIssue).toMatchObject({ status: "done" }); + expect(decisions).toContainEqual(expect.objectContaining({ toStatus: "done", reasonCode: "completion_claim_policy_accepted" })); + } + }); + + it("rechecks the first-task human decision atomically before native status commit", async () => { + const { companyId, agentId, issueId, runId, port, result } = await prepareFirstTaskCompletion("accepted proposal"); + await expect(nativeCompletionFeedback(db, runId, result)).resolves.toContain("Completion report accepted"); + await port.completeRun({ result, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, callerResultId: "first-task-race-result" }); + expect(await db.select().from(workAssessments).where(eq(workAssessments.runId, runId))).toEqual([]); + + const interactionId = randomUUID(); + const trigger = `first_task_race_${randomUUID().replace(/-/g, "")}`; + // Assessment persistence follows the policy read and precedes status commit. + // Insert a newer proposal at that real database boundary, exactly once. + await db.execute(sql.raw(`create function ${trigger}() returns trigger language plpgsql as $$ + begin + if NEW.issue_id = '${issueId}'::uuid and NEW.run_id = '${runId}'::uuid then + insert into issue_thread_interactions + (id, company_id, issue_id, kind, status, created_by_agent_id, source_run_id, payload, created_at) + values + ('${interactionId}'::uuid, '${companyId}'::uuid, '${issueId}'::uuid, + 'request_confirmation', 'pending', '${agentId}'::uuid, '${runId}'::uuid, + '{"version":1,"prompt":"Accept the revised task scope?"}'::jsonb, clock_timestamp()) + on conflict (id) do nothing; + end if; + return NEW; + end $$`)); + try { + await db.execute(sql.raw(`create trigger ${trigger} before insert on work_assessments + for each row execute function ${trigger}()`)); + await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interactionId)))[0]) + .toMatchObject({ status: "pending", issueId }); + expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]!.status).toBe("in_review"); + const decisions = await db.select().from(statusDecisions).where(eq(statusDecisions.issueId, issueId)); + expect(decisions).toEqual([expect.objectContaining({ toStatus: "in_review", reasonCode: "governed_gate_pending" })]); + expect(decisions).not.toContainEqual(expect.objectContaining({ toStatus: "done" })); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]) + .toMatchObject({ nativePhase: "committed", resultJson: { finalizationReasonCode: "governed_gate_pending" } }); + } finally { + await db.execute(sql.raw(`drop trigger if exists ${trigger} on work_assessments`)); + await db.execute(sql.raw(`drop function if exists ${trigger}()`)); + } + }); + it.each(["none", "queued", "claimed", "deferred_issue_execution", "coalesced", "current_run", "current_intent"])( "preserves completion caveats and pending child delivery (%s)", async (delivery) => { const identity = CONTROL_PLANE_CONFORMANCE_OPEN.identity; diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 0e34904c5b3..f19bb8cc244 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1,4 +1,5 @@ import { scheduledIssueMonitorWait } from "../issue-monitors.js"; +import { onboardingFirstTaskCompletionAllowed } from "../onboarding-first-task-admission.js"; import { hasPublishedGitHubReview, readInheritedGitHubReviewWait } from "../chat-channels/github/review-wait.js"; import { pendingNativeGovernance } from "./native-governance.js"; import { findSupersededExternalReviewRequests, supersedeExternalReviewRequestsInTransaction } from "./superseded-external-review-requests.js"; @@ -1657,6 +1658,11 @@ export async function commitNativeStatusDecision(input: { ) { throw new NativeStatusRaceError(); } + // Recheck under the issue lock, retaining run/proposal/document proof until + // commit. A newer decision cannot race an earlier first-task Done assessment. + if (input.decision.statusAction === "done" && !await onboardingFirstTaskCompletionAllowed(tx as unknown as Db, { + companyId: input.companyId, issueId: input.issueId, runId: input.runId, + })) throw new NativeStatusRaceError(); if (reasonCode === "scheduled_monitor_waiting") { const expected = input.requireMonitorWait; const inReview = input.decision.statusAction === "in_review" && input.decision.toStatus === "in_review"; diff --git a/server/src/services/onboarding-first-task-admission.ts b/server/src/services/onboarding-first-task-admission.ts index e649120bb6b..00182df9e1e 100644 --- a/server/src/services/onboarding-first-task-admission.ts +++ b/server/src/services/onboarding-first-task-admission.ts @@ -59,72 +59,107 @@ export async function assertOnboardingFirstTaskCreationAllowed(tx: Db, input: { )).for("share"); if (!run || runIssueId(run) !== source.id) throw acceptanceRequired(); + if (await readOnboardingFirstTaskDecision(tx, { companyId: input.companyId, issueId: source.id, outcomes: ["accepted"] }) !== "accepted") throw acceptanceRequired(); +} + +/** Shared persisted-decision proof; creation and completion retain different authority. */ +async function readOnboardingFirstTaskDecision(tx: Db, input: { + companyId: string; issueId: string; outcomes: Array<"accepted" | "rejected">; +}): Promise<"accepted" | "rejected" | null> { // Select the newest proposal before checking its outcome: a later pending or // rejected proposal must never fall back to an older accepted scope. Governed // tool/credential actions and custom completion reviews are not proposals. const [proposal] = await tx.select().from(issueThreadInteractions).where(and( - eq(issueThreadInteractions.companyId, input.companyId), eq(issueThreadInteractions.issueId, source.id), + eq(issueThreadInteractions.companyId, input.companyId), eq(issueThreadInteractions.issueId, input.issueId), inArray(issueThreadInteractions.kind, ["request_confirmation", "request_checkbox_confirmation"]), sql`NOT (${issueThreadInteractions.payload} ?| ARRAY['toolAction', 'secretProposal', 'connectionAuthorization'])`, sql`COALESCE(${issueThreadInteractions.payload}->'target'->>'type', '') <> 'custom'`, )).orderBy(desc(issueThreadInteractions.createdAt), desc(issueThreadInteractions.id)).limit(1).for("share"); - if (!proposal || proposal.status !== "accepted" || !proposal.resolvedAt) throw acceptanceRequired(); + if (!proposal || (proposal.status !== "accepted" && proposal.status !== "rejected") || + !input.outcomes.includes(proposal.status) || !proposal.resolvedAt) return null; const checkbox = proposal.kind === "request_checkbox_confirmation"; const payload = checkbox ? requestCheckboxConfirmationPayloadSchema.safeParse(proposal.payload) : requestConfirmationPayloadSchema.safeParse(proposal.payload); const result = checkbox ? requestCheckboxConfirmationResultSchema.safeParse(proposal.result) : requestConfirmationResultSchema.safeParse(proposal.result); - if (!payload.success || !result.success || result.data.outcome !== "accepted") throw acceptanceRequired(); - if (checkbox) { + if (!payload.success || !result.success || result.data.outcome !== proposal.status) return null; + if (checkbox && proposal.status === "accepted") { const selected = object(result.data).selectedOptionIds; - if (!("options" in payload.data) || !Array.isArray(selected) || selected.length === 0) throw acceptanceRequired(); + if (!("options" in payload.data) || !Array.isArray(selected) || selected.length === 0) return null; const allowed = new Set(payload.data.options.map(option => option.id)); if (selected.some(id => typeof id !== "string" || !allowed.has(id)) || selected.length < payload.data.minSelected - || (payload.data.maxSelected != null && selected.length > payload.data.maxSelected)) throw acceptanceRequired(); + || (payload.data.maxSelected != null && selected.length > payload.data.maxSelected)) return null; } if (proposal.sourceRunId) { const [origin] = await tx.select(runOriginFields).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, proposal.sourceRunId), eq(heartbeatRuns.companyId, input.companyId), )).for("share"); - if (!origin || runIssueId(origin) !== source.id || origin.agentId !== proposal.createdByAgentId) throw acceptanceRequired(); + if (!origin || runIssueId(origin) !== input.issueId || origin.agentId !== proposal.createdByAgentId) return null; } else if (proposal.createdByAgentId) { const [creator] = await tx.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, proposal.createdByAgentId)); - if (creator?.companyId !== input.companyId) throw acceptanceRequired(); - } else if (!proposal.createdByUserId) throw acceptanceRequired(); + if (creator?.companyId !== input.companyId) return null; + } else if (!proposal.createdByUserId) return null; const directHumanDecision = Boolean(proposal.resolvedByUserId && !proposal.resolvedByAgentId && !proposal.resolvedByRunId); if (!directHumanDecision) { const commentId = result.data.commentId; - if (!commentId || !proposal.resolvedByAgentId || !proposal.resolvedByRunId) throw acceptanceRequired(); + if (!commentId || !proposal.resolvedByAgentId || !proposal.resolvedByRunId) return null; const [comment] = await tx.select().from(issueComments).where(and( - eq(issueComments.id, commentId), eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, source.id), + eq(issueComments.id, commentId), eq(issueComments.companyId, input.companyId), eq(issueComments.issueId, input.issueId), )).for("share"); if (!comment || comment.deletedAt || comment.authorType !== "user" || !comment.authorUserId || comment.authorAgentId || comment.createdByRunId || comment.derivedAuthorAgentId || comment.derivedCreatedByRunId || comment.sourceTrust?.disposition === "quarantined" || comment.createdAt < proposal.createdAt - || comment.body.trim() === "/new") throw acceptanceRequired(); + || comment.body.trim() === "/new") return null; const [resolver] = await tx.select(runOriginFields).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, proposal.resolvedByRunId), eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.agentId, proposal.resolvedByAgentId), )).for("share"); - if (!resolver || runIssueId(resolver) !== source.id - || (resolver.responsibleUserId && resolver.responsibleUserId !== comment.authorUserId)) throw acceptanceRequired(); + if (!resolver || runIssueId(resolver) !== input.issueId + || (resolver.responsibleUserId && resolver.responsibleUserId !== comment.authorUserId)) return null; } + // Refusal permits acknowledgement and closure, never execution. An agent-only + // plan edit cannot turn that human refusal into permission to execute. + if (proposal.status === "rejected") return "rejected"; + const target = payload.data.target; const [plan] = await tx.select({ documentId: documents.id, revisionId: documents.latestRevisionId }) .from(issueDocuments).innerJoin(documents, eq(documents.id, issueDocuments.documentId)).where(and( eq(issueDocuments.companyId, input.companyId), eq(documents.companyId, input.companyId), - eq(issueDocuments.issueId, source.id), eq(issueDocuments.key, "plan"), + eq(issueDocuments.issueId, input.issueId), eq(issueDocuments.key, "plan"), )).for("share", { of: documents }); // Document writers do not all lock the issue, so hold the document row too. // A saved plan must be accepted by revision; an untargeted card cannot stand // in for approval of a different or subsequently edited document. if (plan || target) { if (!target || target.type !== "issue_document" || target.key !== "plan" - || (target.issueId && target.issueId !== source.id) || !plan || plan.revisionId !== target.revisionId - || (target.documentId && plan.documentId !== target.documentId)) throw acceptanceRequired(); + || (target.issueId && target.issueId !== input.issueId) || !plan || plan.revisionId !== target.revisionId + || (target.documentId && plan.documentId !== target.documentId)) return null; } + return "accepted"; +} + +/** Completion may acknowledge a trusted refusal; it never authorizes rejected work. */ +export async function onboardingFirstTaskCompletionAllowed(tx: Db, input: { + companyId: string; issueId: string; runId: string; +}): Promise { + const [source] = await tx.select({ id: issues.id, originKind: issues.originKind }).from(issues).where(and( + eq(issues.id, input.issueId), eq(issues.companyId, input.companyId), + )); + if (!source) return false; + if (source.originKind !== ONBOARDING_FIRST_TASK_ORIGIN_KIND) return true; + // Only the first-task path takes additional authority locks. At status commit + // these are held until the decision is written, in issue/run/proposal order. + const [lockedSource] = await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.id, source.id), eq(issues.companyId, input.companyId), eq(issues.originKind, ONBOARDING_FIRST_TASK_ORIGIN_KIND), + )).for("update"); + if (!lockedSource) return false; + const [run] = await tx.select({ ...runOriginFields, runtimeMode: heartbeatRuns.runtimeMode }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.companyId, input.companyId), + )).for("share"); + if (!run || run.runtimeMode !== "native" || run.nativeIssueId !== source.id) return false; + return await readOnboardingFirstTaskDecision(tx, { companyId: input.companyId, issueId: source.id, outcomes: ["accepted", "rejected"] }) !== null; } diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 2d23dc214e1..a86949df8e7 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -118,7 +118,8 @@ ownership manifests. An expired lease permits cleanup recovery, not reuse. Managed self-hosted instances are removed after every attempt. Existing Cloud stacks retain the QA account and declared baseline while removing verified run-owned fixtures. Successful disposable stacks are deleted. Failed disposable -stacks are stopped and retained for 24 hours, then swept. Interrupted cleanup +stacks are stopped and retained for up to 24 hours. The deployment provider +declares earlier cleanup eligibility to leave time for its sweeper. Interrupted cleanup remains visible and retains exclusive stack ownership. QA identity and inbox never belong to disposable ownership. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 554b10b32d1..af2ba3758b4 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -261,6 +261,16 @@ status, known adapters, and allowlisted outcome/check/server codes. The case report renders these fields explicitly; older results without the receipt remain valid. This diagnostic does not change the journey grade or qualify a live run. +When the canonical first-task journey fails `onboarding_work_before_acceptance`, +`productJourney.onboardingPreAcceptance` retains the failed checkpoint and every +observed guard violation: unexpected task set, existing attachment, non-planning +document, or first task already Done. The closed version-1 receipt contains only +counts, whether the first task matches the canonical parent, and closed first-task +and canonical-parent statuses. Reports validate and display these observations; +they do not infer a product cause. Only the failing boundary is retained, without +task/document IDs, document keys, text, or bodies. Historical results may omit the +receipt. The original error, failed grade, and dependent blocked outcomes remain. + ## Live provider connection journeys See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 30e2f2bc1a9..c3dde9aef93 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -28,6 +28,16 @@ Unknown codes become `unrecognized`; raw bodies, messages, URLs, headers, and browser errors are excluded. Reports validate the receipt before rendering its explicit fields, and historical results may omit it. +The optional `onboardingPreAcceptance` receipt captures only a failing first-task +acceptance guard. Its closed version-1 fields are a known checkpoint, up to four +distinct guard-violation codes, nonnegative integer task/attachment/document +counts, a canonical-parent match boolean, and closed first-task/canonical-parent +statuses (`missing` and `unknown` remain explicit). It excludes all identifiers, +task text, document keys and bodies, filenames, URLs, credentials, and raw API +responses. Validation rejects extra or inconsistent fields before report use; +older results may omit the receipt. It preserves the original failure grade and +does not attribute a product cause or authorize acceptance. + ## Attended connection authentication See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/case-outcome.ts b/tests/runner-e2e/case-outcome.ts index 489f16f8384..ca463553322 100644 --- a/tests/runner-e2e/case-outcome.ts +++ b/tests/runner-e2e/case-outcome.ts @@ -1,6 +1,7 @@ import type { RunnerE2EResult } from "./types.js"; import { isBlockedConnectionResult } from "./connection-evidence.js"; import { validateOnboardingConnectEvidence } from "./product-journeys/onboarding-connect.js"; +import { validateOnboardingPreAcceptanceEvidence } from "./product-journeys/onboarding-pre-acceptance.js"; const html = (value: unknown) => String(value ?? "").replace( /[&<>"']/g, @@ -31,6 +32,20 @@ function renderOnboardingConnect(result: RunnerE2EResult): string { ${response.truncatedChecks ? "

Only the first 32 validation checks were retained.

" : ""}`; }).join("")}`; } +function renderOnboardingPreAcceptance(result: RunnerE2EResult): string { + const receipt = result.productJourney?.onboardingPreAcceptance; + if (receipt === undefined) return ""; + try { validateOnboardingPreAcceptanceEvidence(receipt); } + catch { return "

Onboarding pre-acceptance evidence unavailable: invalid receipt.

"; } + return `
Onboarding pre-acceptance evidence +

Checkpoint: ${html(receipt.checkpoint)}

+

Observed guard violations:

+
${[ + ["Tasks", receipt.taskCount], ["Attachments", receipt.attachmentCount], ["Documents", receipt.documentCount], + ["Non-planning documents", receipt.nonPlanningDocumentCount], ["First task matches canonical parent", receipt.firstTaskMatchesCanonicalParent], + ["First task status", receipt.firstTaskStatus], ["Canonical parent status", receipt.canonicalParentStatus], + ].map(([label, value]) => `
${html(label)}
${html(value)}
`).join("")}
`; +} /** An interrupted first-task journey is not evidence of failed downstream behavior. */ export function isIncompleteFirstTaskResult( result: RunnerE2EResult, @@ -93,6 +108,7 @@ export function renderCaseOutcome( ${result.productJourney?.operations ? `

Eval operations: ${html(result.productJourney.operations.campaignId)} · ${result.productJourney.operations.requestCount} requests · model cost $${html(result.productJourney.operations.modelCostUsd ?? "unknown")} (setup $${html(result.productJourney.operations.setupCostUsd ?? "unknown")}, counted once). Feedback: ${html(result.productJourney.operations.feedback.collection)} · ${result.productJourney.operations.feedback.complaints} complaints · ${result.productJourney.operations.feedback.suggestions} suggestions.

` : ""} ${result.productJourney?.operations?.traceIds.length ? `
Trace correlation IDs
` : ""} ${renderOnboardingConnect(result)} + ${renderOnboardingPreAcceptance(result)} ${result.providerConnection ? `

Target: ${html(result.providerConnection.target.origin)} · ${html(result.providerConnection.target.commit ?? "revision unavailable")} · ${result.providerConnection.assisted ? "Assisted login" : "Unassisted"} · ${html(result.providerConnection.authFreshness)} browser · ${html(result.providerConnection.entry)} / ${html(result.providerConnection.method)} · stopped at ${html(result.providerConnection.phase)}

` : ""} ${failures.length ? `

Failed checks:

` : checks.length ? `

No behavioral matcher failed.${notReached.length ? " The journey is incomplete; unexercised checks are not passes." : failed ? " The overall failure came from a separate run, cleanup, or evidence check." : ""}

` : ""} ${notReached.length ? `

Not reached:

` : ""} diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts index 8ab17a54a2e..917bcd3d592 100644 --- a/tests/runner-e2e/daytona-image-content.ts +++ b/tests/runner-e2e/daytona-image-content.ts @@ -67,6 +67,7 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [ "packages/paperclip-runner/test/fixtures/copilot-profile-v58-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v61-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v65-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v66-identity.json", "packages/paperclip-runner/scripts/materialize-cursor-distribution.mjs", "packages/paperclip-runner/scripts/cursor-runtime-patch.mjs", "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs", diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index fe8e458c89b..5c49d899045 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -188,8 +188,8 @@ describe("Pi controls catalog admission", () => { // Extended v3 states the strict file oracle's task-wide Bash limit. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 23, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 29, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-content-bound-native-read-both-runs", agentMemoryReadAuthority: "local-withheld-or-exact-remote-agent-run-file", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); - expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 10, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", profileVersion: 29, + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 23, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 30, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-content-bound-native-read-both-runs", agentMemoryReadAuthority: "local-withheld-or-exact-remote-agent-run-file", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); + expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 10, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", profileVersion: 30, remoteProcessIdentity: "observer-pid-startTicks-bootId", controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 6, piFileArtifactTitle: "exact-filename", piFileCommandTransport: "fenced-bash-markdown-paste" }); @@ -199,7 +199,7 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "d7d691088af76d460e353a3982f7732e6a65b2e5a23a66d2ef0d534f9d8dae48", + "pi-native": "04d49c6e098c24091d2939108a97722d1b8353e39192b87e98ee33377f183001", "native-active-stop": "4654a2169bff96cf8377fdd62441f5cabea5c613f447e518830f6b1efd82c051", "rich-acp-warm-continuity": "7e8a0012f2f67fba03cd94403b096a07cfc95cdb7aa568b4c3fd18b852d735f3", "extended-harnesses": "d5bcf5de0ade3a54e4e1560e76feb5d8a787039d56d2379cd501626d85ed7418", diff --git a/tests/runner-e2e/product-journeys/campaign.test.ts b/tests/runner-e2e/product-journeys/campaign.test.ts index 27d42b934f9..e039157c8f1 100644 --- a/tests/runner-e2e/product-journeys/campaign.test.ts +++ b/tests/runner-e2e/product-journeys/campaign.test.ts @@ -5,6 +5,10 @@ import os from "node:os"; import { pathToFileURL } from "node:url"; import { createConnectionProof } from "../connection-evidence.js"; import { retainOnboardingArtifact, validateOnboardingArtifact } from "./artifact-evidence.js"; +import { validateOnboardingPreAcceptanceEvidence, type OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; +const preAcceptanceEvidence: OnboardingPreAcceptanceEvidence = { version: 1, checkpoint: "awaiting-proposal", violations: ["attachment_present"], + taskCount: 1, attachmentCount: 1, documentCount: 0, nonPlanningDocumentCount: 0, + firstTaskMatchesCanonicalParent: true, firstTaskStatus: "in_review", canonicalParentStatus: "in_review" }; const persistenceFault = vi.hoisted(() => ({ kind: null as "write" | "rename" | null, active: false, latestReport: null as any })); vi.mock("node:fs/promises", async importOriginal => { const actual = await importOriginal(); @@ -33,7 +37,7 @@ import { regenerateRunnerDashboard } from "../dashboard-regenerate.js"; const roots: string[]=[]; afterEach(async()=>{vi.useRealTimers(); vi.restoreAllMocks(); persistenceFault.kind=null; persistenceFault.active=false; persistenceFault.latestReport=null; process.exitCode=0; await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));}); type Hook = "collect" | "release" | "stop" | "finalizeEvidence" | "checkpointEvidence"; -async function run(options:{authFailure?:boolean;onboardingFailure?:boolean;missingArtifact?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) { +async function run(options:{authFailure?:boolean;onboardingFailure?:boolean;preAcceptanceFailure?:boolean;missingArtifact?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) { const root=await mkdtemp(path.join(os.tmpdir(),"journey-campaign-test-")); roots.push(root); const module=path.join(root,"provider.mjs"); const proof = createConnectionProof(); @@ -44,7 +48,7 @@ let onboarded=false; const context={identity,company:{id:'company-1',issuePrefix:'QA'},api:{post:async()=>({id:'issue-fixture-1',identifier:'QA-1'})},startingState:'onboarding',capabilities:['company','board-user','task-write'],page:{async screenshot(){} }}; export const provider={version:1, async acquire(input){return { async authenticate(){${options.authFailure ? "throw {outcome:'failed',code:'login_code_expired'};" : ""}}, -async onboard(evidence){${options.onboardingFailure ? "throw {outcome:'failed',code:'onboarding_failed'};" : `${options.missingArtifact ? "" : `evidence.artifact=${JSON.stringify(artifact)};`}evidence.checks.push(...['fresh-workspace','ui-connection-and-native-agent','first-task-complete','downloadable-artifact-verified'${options.missingCheck ? "" : ",'native-trace-verified'"}].map(id=>({id,passed:true})));onboarded=true;return context;`}}, +async onboard(evidence){${options.preAcceptanceFailure ? `evidence.onboardingPreAcceptance=${JSON.stringify(preAcceptanceEvidence)};throw {outcome:'failed',code:'onboarding_work_before_acceptance'};` : options.onboardingFailure ? "throw {outcome:'failed',code:'onboarding_failed'};" : `${options.missingArtifact ? "" : `evidence.artifact=${JSON.stringify(artifact)};`}evidence.checks.push(...['fresh-workspace','ui-connection-and-native-agent','first-task-complete','downloadable-artifact-verified'${options.missingCheck ? "" : ",'native-trace-verified'"}].map(id=>({id,passed:true})));onboarded=true;return context;`}}, async prepare(){throw Error('onboarding receipt was not reused')}, async identity(){return {...identity,commit:'${(options.drift?"b":"a").repeat(40)}'}}, async collect(){${options.collectionFailure ? "throw Error('private collection error');" : options.missingReceipt ? "return [];" : `return onboarded?[{journeyId:'cloud-onboarding',runIds:['run-1'],usage:{runs:[{runId:'run-1',usage:{costUsd:0.01}}]},operations:{campaignId:input.campaignId,attemptId:input.campaignId+':cloud-onboarding:1',modelCostUsd:0.01,setupCostUsd:0.005,requestCount:${options.zeroRequests ? 0 : 2},feedback:{collection:'complete',complaints:0,suggestions:0},traceIds:['a'.repeat(32)]}}]:[];`}}, async own(){}, async release(){${options.cleanupFailure ? "throw Error('private raw error');" : "return 'released';"}}, async stop(){} @@ -109,6 +113,17 @@ describe("product journey campaign boundary outcomes",()=>{ expect(campaign.results.map((r:any)=>r.productJourney.outcome)).toEqual(["failed","blocked"]); expect(campaign.results[0].productJourney.phases.find((p:any)=>p.phase==="journey").outcome).toBe("failed"); }); + it("preserves failed pre-acceptance evidence in normalized results and blocks dependent reuse", async () => { + const campaign = await run({ preAcceptanceFailure: true }); + const [onboarding, dependent] = campaign.results; + expect(campaign.results.map((row:any) => row.productJourney.outcome)).toEqual(["failed", "blocked"]); + expect(onboarding).toMatchObject({ status: "failed", failureClass: "candidate_failure", error: "onboarding_work_before_acceptance" }); + expect(onboarding.productJourney.onboardingPreAcceptance).toEqual(preAcceptanceEvidence); + expect(() => validateOnboardingPreAcceptanceEvidence(onboarding.productJourney.onboardingPreAcceptance)).not.toThrow(); + expect(onboarding.productJourney.phases).toContainEqual({ phase: "journey", outcome: "failed", code: "onboarding_work_before_acceptance" }); + expect(dependent.error).toBe("prerequisite_journey_failed"); + expect(dependent.productJourney.onboardingPreAcceptance).toBeUndefined(); + }); it("reuses a verified onboarding context for the same shared story",async()=>{ const campaign=await run(); expect(campaign.results.map((r:any)=>r.status)).toEqual(["passed","passed"]); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index c018e769aab..6077ff48c36 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -9,7 +9,7 @@ export function preparationPreview(config: TargetConfig) { target: { id: config.id, mode: config.mode, executionEnvironment: config.executionEnvironment }, journeys: config.journeys.map(id => productJourneys.find(journey => journey.id === id)!), configuration: config.baseline, - lifetime: config.mode === "existing-cloud" ? "persistent; clean run-owned resources only" : config.mode === "new-cloud" ? "disposable; delete on pass, retain failures for 24 hours" : "managed instance; remove after every attempt", + lifetime: config.mode === "existing-cloud" ? "persistent; clean run-owned resources only" : config.mode === "new-cloud" ? "disposable; delete on pass, retain failures up to 24 hours" : "managed instance; remove after every attempt", limits: config.limits, authentication: config.mode === "managed-self-hosted" ? "local-trusted" : "normal email-code sign-in", modelProfileId: config.modelProfileId ?? null, maxDisposableStacks: config.mode === "new-cloud" ? 1 : 0, diff --git a/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts b/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts index 0878ade8065..68adbdd2d53 100644 --- a/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts +++ b/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it, vi } from "vitest"; import { cloudOnboarding, type CloudOnboardingInput } from "./cloud-onboarding.js"; import { onboardingFirstTask } from "./onboarding-first-task.js"; import { retainOnboardingArtifact } from "./artifact-evidence.js"; +import { JourneyError } from "./types.js"; +import type { OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; vi.mock("@playwright/test", () => ({ expect: (locator: { isVisible(): Promise }) => ({ toBeVisible: async () => { expect(await locator.isVisible()).toBe(true); }, @@ -118,6 +120,20 @@ function fixture(entry: Entry, options: { intentFailure?: boolean; qualification } describe("Cloud onboarding visible controls", () => { + it("retains pre-acceptance evidence while preserving the failed error and withholding completion markers", async () => { + const f = fixture("portfolio"); + const receipt: OnboardingPreAcceptanceEvidence = { version: 1, checkpoint: "awaiting-proposal", violations: ["attachment_present"], + taskCount: 1, attachmentCount: 1, documentCount: 0, nonPlanningDocumentCount: 0, + firstTaskMatchesCanonicalParent: true, firstTaskStatus: "in_review", canonicalParentStatus: "in_review" }; + vi.mocked(onboardingFirstTask).mockImplementationOnce(async input => { + input.onPreAcceptanceFailure?.(receipt); + throw new JourneyError("failed", "onboarding_work_before_acceptance"); + }); + await expect(cloudOnboarding(f.input)).rejects.toMatchObject({ outcome: "failed", code: "onboarding_work_before_acceptance" }); + expect(JSON.parse(JSON.stringify(f.input.evidence)).onboardingPreAcceptance).toEqual(receipt); + expect(f.input.evidence.checks.map(check => check.id)).toEqual(["fresh-workspace", "ui-connection-and-native-agent"]); + expect(f.events).not.toContain("native:verified"); + }); it.each(["first-run", "portfolio", "create-form"] as const)("creates from %s with intent before dispatch and qualification before paid Connect", async entry => { const f = fixture(entry); const result = await cloudOnboarding(f.input); diff --git a/tests/runner-e2e/product-journeys/cloud-onboarding.ts b/tests/runner-e2e/product-journeys/cloud-onboarding.ts index 6f9a75634a7..215871b411d 100644 --- a/tests/runner-e2e/product-journeys/cloud-onboarding.ts +++ b/tests/runner-e2e/product-journeys/cloud-onboarding.ts @@ -83,7 +83,8 @@ export async function cloudOnboarding(input: CloudOnboardingInput): Promise { evidence.artifact = artifact; } }); + own: input.own, observeRun: input.observeRun, onArtifact: artifact => { evidence.artifact = artifact; }, + onPreAcceptanceFailure: receipt => { evidence.onboardingPreAcceptance = receipt; } }); await input.verifyExecution(context, [completed.artifact.originatingRunId]); evidence.checks.push({ id: "first-task-complete", passed: true }, { id: "downloadable-artifact-verified", passed: true }, { id: "native-trace-verified", passed: true }); diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 410bb7725bf..980822013fd 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -99,7 +99,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); const suiteDefinitionHash = definitionHash.digest("hex"); const expected = config.journeys.map(id => `product-journeys.${config.id}.${id === "cloud-onboarding" ? config.modelProfileId : "browser"}.${config.executionEnvironment}.${id}`); let reportPersisted = false; diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts index ab452f1901c..2e559aa8584 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts @@ -4,6 +4,7 @@ import { createConnectionProof } from "../connection-evidence.js"; import { connectionDelay } from "../connection-target.js"; import { onboardingFirstTask, type OnboardingFirstTaskInput } from "./onboarding-first-task.js"; import { validateOnboardingArtifact } from "./artifact-evidence.js"; +import { validateOnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; vi.mock("../connection-target.js", () => ({ connectionDelay: vi.fn(async () => {}) })); type Row = { id: string; [key: string]: any }; @@ -92,6 +93,90 @@ function fixture(options: Options = {}) { afterEach(() => { vi.restoreAllMocks(); vi.mocked(connectionDelay).mockReset().mockResolvedValue(undefined); }); describe("Cloud canonical onboarding first task", () => { + it.each([ + ["unexpected task set", "unexpected_task_set", (state: Parameters>[1]) => { state.tasks.push({ id: "private-early-task", title: "private task text", status: "todo" }); }, { taskCount: 2 }], + ["attachment", "attachment_present", (state: Parameters>[1]) => { state.attachments.push({ id: "private-attachment", issueId: "parent", originalFilename: "private-file.txt" }); }, { attachmentCount: 1 }], + ["non-planning document", "non_planning_document_present", (state: Parameters>[1]) => { state.documents.push({ id: "private-document", issueId: "parent", key: "private-result-key", body: "private document body" }); }, { documentCount: 1, nonPlanningDocumentCount: 1 }], + ["first task done", "first_task_done", (state: Parameters>[1]) => { state.tasks[0]!.status = "done"; }, { firstTaskStatus: "done", canonicalParentStatus: "done" }], + ] as const)("retains the observed %s violation at the failing acceptance boundary", async (_label, violation, mutate, observed) => { + const f = fixture({ mutate(phase, state) { if (phase === "proposal") mutate(state); } }); + const capture = vi.fn(); + f.input.onPreAcceptanceFailure = capture; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ outcome: "failed", code: "onboarding_work_before_acceptance" }); + expect(capture).toHaveBeenCalledExactlyOnceWith({ version: 1, checkpoint: "awaiting-proposal", violations: [violation], + taskCount: 1, attachmentCount: 0, documentCount: 0, nonPlanningDocumentCount: 0, + firstTaskMatchesCanonicalParent: true, firstTaskStatus: "in_review", canonicalParentStatus: "in_review", ...observed }); + expect(() => validateOnboardingPreAcceptanceEvidence(JSON.parse(JSON.stringify(capture.mock.calls[0]![0])))).not.toThrow(); + const retained = JSON.stringify(capture.mock.calls[0]![0]); + for (const text of ["private-", "parent", "proposal-run", "qa.example.test", "connection", "campaign-1"]) expect(retained).not.toContain(text); + expect(f.events).not.toContain("click:Approve"); + }); + it("retains every simultaneous guard violation from the same failed snapshot", async () => { + const f = fixture({ plan: true, mutate(phase, state) { + if (phase !== "proposal") return; + state.tasks[0]!.status = "done"; + state.tasks.push({ id: "private-early-task" }); + state.attachments.push({ id: "private-attachment", issueId: "parent" }); + state.documents.push({ id: "private-document", issueId: "parent", key: "private-result", body: "private body" }); + } }); + const capture = vi.fn(); + f.input.onPreAcceptanceFailure = capture; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_work_before_acceptance" }); + expect(capture).toHaveBeenCalledExactlyOnceWith({ version: 1, checkpoint: "awaiting-proposal", + violations: ["unexpected_task_set", "attachment_present", "non_planning_document_present", "first_task_done"], + taskCount: 2, attachmentCount: 1, documentCount: 2, nonPlanningDocumentCount: 1, + firstTaskMatchesCanonicalParent: true, firstTaskStatus: "done", canonicalParentStatus: "done" }); + expect(f.events).not.toContain("click:Confirm selection"); + }); + it.each(["missing", "replaced", "reordered", "unknown-status"] as const)("records closed task observations when the snapshot is %s", async mode => { + const f = fixture({ mutate(phase, state) { + if (phase !== "proposal") return; + if (mode === "missing") state.tasks = []; + if (mode === "replaced") state.tasks = [{ id: "private-replacement", status: "done" }]; + if (mode === "reordered") state.tasks.unshift({ id: "private-first", status: "done" }); + if (mode === "unknown-status") { state.tasks[0]!.status = "private future status"; state.attachments.push({ id: "private-attachment", issueId: "parent" }); } + } }); + const capture = vi.fn(); + f.input.onPreAcceptanceFailure = capture; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_work_before_acceptance" }); + expect(capture).toHaveBeenCalledOnce(); + expect(capture.mock.calls[0]![0]).toMatchObject({ + violations: mode === "unknown-status" ? ["attachment_present"] : mode === "missing" ? ["unexpected_task_set"] : ["unexpected_task_set", "first_task_done"], + taskCount: mode === "missing" ? 0 : mode === "reordered" ? 2 : 1, + firstTaskMatchesCanonicalParent: mode === "unknown-status", + firstTaskStatus: mode === "missing" ? "missing" : mode === "unknown-status" ? "unknown" : "done", + canonicalParentStatus: mode === "reordered" ? "in_review" : mode === "unknown-status" ? "unknown" : "missing", + }); + expect(JSON.stringify(capture.mock.calls[0]![0])).not.toContain("private"); + expect(f.events).not.toContain("click:Approve"); + }); + it("retains an opening-boundary failure before the opening answer dispatches work", async () => { + const f = fixture({ mutate(phase, state) { if (phase === "opening") state.tasks[0]!.status = "done"; } }); + const capture = vi.fn(); + f.input.onPreAcceptanceFailure = capture; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_work_before_acceptance" }); + expect(capture).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ checkpoint: "before-opening-answer", violations: ["first_task_done"] })); + expect(f.events).not.toContain("click:Continue"); + }); + it("does not claim the first-slot Done guard when only a later canonical parent is Done", async () => { + const f = fixture({ mutate(phase, state) { + if (phase !== "proposal") return; + state.tasks[0]!.status = "done"; + state.tasks.unshift({ id: "private-first", status: "todo" }); + } }); + const capture = vi.fn(); + f.input.onPreAcceptanceFailure = capture; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_work_before_acceptance" }); + expect(capture).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ violations: ["unexpected_task_set"], + firstTaskMatchesCanonicalParent: false, firstTaskStatus: "todo", canonicalParentStatus: "done" })); + }); + it.each([false, true])("does not retain failure snapshots during clean canonical success, plan mode %s", async plan => { + const f = fixture({ plan }); + const capture = vi.fn(); + f.input.onPreAcceptanceFailure = capture; + await expect(onboardingFirstTask(f.input)).resolves.toMatchObject({ onboardingIssueId: "parent", childIssueId: "child" }); + expect(capture).not.toHaveBeenCalled(); + }); it.each(["reload", "download-link"] as const)("retains the independently verified download when later %s fails", async readbackFailure => { const f = fixture({ readbackFailure }); const capture = vi.fn(); diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.ts index ee50b6466ad..474bd615a84 100644 --- a/tests/runner-e2e/product-journeys/onboarding-first-task.ts +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.ts @@ -6,6 +6,7 @@ import { activeRuns, firstTaskCompletionSettled, type Row } from "../first-task- import { isBlockedUnstartedWake, isTerminalUnstartedWake } from "../non-execution-wake.js"; import { JourneyError, type JourneyContext, type OnboardingArtifactEvidence, type TargetSession } from "./types.js"; import { retainOnboardingArtifact } from "./artifact-evidence.js"; +import { preAcceptanceTaskStatus, preAcceptanceViolations, type OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; export interface OnboardingFirstTaskInput { context: JourneyContext; @@ -18,6 +19,7 @@ export interface OnboardingFirstTaskInput { own: TargetSession["own"]; observeRun(run: Row): Promise; onArtifact?(artifact: OnboardingArtifactEvidence): void; + onPreAcceptanceFailure?(evidence: OnboardingPreAcceptanceEvidence): void; } export interface OnboardingFirstTaskResult { runIds: string[]; @@ -91,9 +93,19 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom fail("onboarding_first_task_failed"); } }; - const assertBeforeAcceptance = (snapshot: Snapshot, parentId: string) => { - if (snapshot.tasks.length !== 1 || snapshot.tasks[0]!.id !== parentId || snapshot.attachments.length || - snapshot.documents.some(document => !isPlanningDocument(document)) || snapshot.tasks[0]!.status === "done") fail("onboarding_work_before_acceptance"); + const assertBeforeAcceptance = (snapshot: Snapshot, parentId: string, checkpoint: OnboardingPreAcceptanceEvidence["checkpoint"]) => { + const observed = { + taskCount: snapshot.tasks.length, attachmentCount: snapshot.attachments.length, + documentCount: snapshot.documents.length, nonPlanningDocumentCount: snapshot.documents.filter(document => !isPlanningDocument(document)).length, + firstTaskMatchesCanonicalParent: snapshot.tasks[0]?.id === parentId, + firstTaskStatus: preAcceptanceTaskStatus(snapshot.tasks[0]), + canonicalParentStatus: preAcceptanceTaskStatus(snapshot.tasks.find(task => task.id === parentId)), + }; + const violations = preAcceptanceViolations(observed); + if (violations.length) { + input.onPreAcceptanceFailure?.({ version: 1, checkpoint, violations, ...observed }); + fail("onboarding_work_before_acceptance"); + } assertRuns(snapshot, [parentId]); }; @@ -105,7 +117,7 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom await input.own({ kind: "issue", name: parent.title, id: parent.id }); snapshot = await load(parent.id); if (seen.size) fail("onboarding_opening_started_model_work"); - assertBeforeAcceptance(snapshot, parent.id); + assertBeforeAcceptance(snapshot, parent.id, "before-opening-answer"); const opening = snapshot.interactions.filter(row => row.kind === "ask_user_questions" && row.status === "pending"); const questions = opening[0]?.payload?.questionSet?.questions ?? opening[0]?.payload?.questions ?? []; const question = questions.find((row: Row) => row.id === "first-task-opening"); @@ -129,7 +141,7 @@ export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Prom let proposal: Row | undefined; while (!proposal) { snapshot = await load(parent.id); - assertBeforeAcceptance(snapshot, parent.id); + assertBeforeAcceptance(snapshot, parent.id, "awaiting-proposal"); const pending = snapshot.interactions.filter(row => row.status === "pending"); const proposals = pending.filter(isProposal); if (proposals.length > 1 || pending.some(row => row.kind === "ask_user_questions" && row.id !== opening[0]!.id)) fail("onboarding_proposal_scope_unclear"); diff --git a/tests/runner-e2e/product-journeys/onboarding-pre-acceptance.test.ts b/tests/runner-e2e/product-journeys/onboarding-pre-acceptance.test.ts new file mode 100644 index 00000000000..84e8ca87aa1 --- /dev/null +++ b/tests/runner-e2e/product-journeys/onboarding-pre-acceptance.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import { renderCaseOutcome } from "../case-outcome.js"; +import type { RunnerE2EResult } from "../types.js"; +import type { OnboardingPreAcceptanceEvidence } from "./onboarding-pre-acceptance.js"; + +function result(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "test", + executionId: "product-journeys.qa.runner-codex.local.cloud-onboarding", profileId: "runner-codex", environmentId: "local", caseId: "cloud-onboarding", + provider: "codex", model: "synthetic-model", runtimeMode: "native", status: "failed", error: "onboarding_work_before_acceptance", cleanup: "passed", attempt: 1, + startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, + productJourney: { outcome: "failed", startingState: "onboarding", phases: [{ phase: "journey", outcome: "failed", code: "onboarding_work_before_acceptance" }], + checks: [], privacy: "private", resourceDisposition: "retained" } }; +} +function preAcceptanceReceipt(): OnboardingPreAcceptanceEvidence { + return { version: 1, checkpoint: "awaiting-proposal", violations: ["attachment_present"], + taskCount: 1, attachmentCount: 1, documentCount: 0, nonPlanningDocumentCount: 0, + firstTaskMatchesCanonicalParent: true, firstTaskStatus: "in_review", canonicalParentStatus: "in_review" }; +} +function passedResult(): RunnerE2EResult { + const retained = result(); + retained.status = "passed"; + delete retained.error; + retained.runIds = ["run-1"]; + Object.assign(retained.productJourney!, { outcome: "passed", campaignId: "campaign", attemptId: "attempt", + phases: [{ phase: "journey", outcome: "passed" }], resourceDisposition: "released", + checks: ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"].map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40), cloudRelease: "release", runtimeImage: "image", configurationHash: "config" }, + operations: { campaignId: "campaign", attemptId: "attempt", modelCostUsd: null, setupCostUsd: null, requestCount: 1, + feedback: { collection: "complete", complaints: 0, suggestions: 0 }, traceIds: ["trace-1"] } }); + return retained; +} +describe("onboarding pre-acceptance retained evidence", () => { + it("accepts historical results without pre-acceptance evidence and preserves optional receipts in reports", () => { + const retained = result(); + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + expect(renderCaseOutcome(retained, true, [])).not.toContain("Onboarding pre-acceptance evidence"); + Object.assign(retained.productJourney!, { onboardingPreAcceptance: preAcceptanceReceipt() }); + const roundTrip = JSON.parse(JSON.stringify(retained)); + expect(() => validateRetainedRunnerResult(roundTrip)).not.toThrow(); + const output = renderCaseOutcome(roundTrip, true, []); + for (const text of ["Onboarding pre-acceptance evidence", "awaiting-proposal", "attachment_present", "First task status", "Canonical parent status", "in_review", "Attachments", "Non-planning documents", "onboarding_work_before_acceptance"]) expect(output).toContain(text); + expect(roundTrip.status).toBe("failed"); + expect(roundTrip.productJourney.outcome).toBe("failed"); + }); + it.each([ + ["a passed result", passedResult], + ["another case", (): RunnerE2EResult => ({ ...result(), caseId: "task-properties", + executionId: "product-journeys.qa.browser.local.task-properties", profileId: "browser", provider: null, model: null, runtimeMode: "none" })], + ["another suite", (): RunnerE2EResult => ({ ...result(), suiteId: "first-task", executionId: "first-task.runner-codex.local.cloud-onboarding" })], + ] as const)("rejects a valid receipt on %s after accepting the same result without it", (_label, makeResult) => { + const retained = makeResult(); + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + retained.productJourney!.onboardingPreAcceptance = preAcceptanceReceipt(); + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.productJourney.onboardingPreAcceptance"); + }); + it.each(["passed", "blocked", "unsupported"] as const)("rejects a valid receipt on a %s journey outcome after accepting the same result without it", outcome => { + const retained = result(); + retained.productJourney!.outcome = outcome; + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + retained.productJourney!.onboardingPreAcceptance = preAcceptanceReceipt(); + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.productJourney.onboardingPreAcceptance"); + }); + it.each(["onboarding_artifact_evidence_missing", undefined])("rejects a valid receipt for an unrelated or absent failure code (%s)", error => { + const retained = result(); + retained.error = error; + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + retained.productJourney!.onboardingPreAcceptance = preAcceptanceReceipt(); + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.productJourney.onboardingPreAcceptance"); + }); + it.each([ + ["raw bodies", (): unknown => ({ ...preAcceptanceReceipt(), body: "private-body-must-not-render" })], + ["task identifiers", (): unknown => ({ ...preAcceptanceReceipt(), taskIds: ["private-task"] })], + ["document keys", (): unknown => ({ ...preAcceptanceReceipt(), documentKeys: ["private-key"] })], + ["unknown version", (): unknown => ({ ...preAcceptanceReceipt(), version: 2 })], + ["unknown checkpoint", (): unknown => ({ ...preAcceptanceReceipt(), checkpoint: "private text" })], + ["unknown violation", (): unknown => ({ ...preAcceptanceReceipt(), violations: ["private text"] })], + ["duplicate violations", (): unknown => ({ ...preAcceptanceReceipt(), violations: ["attachment_present", "attachment_present"] })], + ["empty violations", (): unknown => ({ ...preAcceptanceReceipt(), violations: [] })], + ["inconsistent violations", (): unknown => ({ ...preAcceptanceReceipt(), violations: ["first_task_done"] })], + ["negative counts", (): unknown => ({ ...preAcceptanceReceipt(), attachmentCount: -1 })], + ["fractional counts", (): unknown => ({ ...preAcceptanceReceipt(), taskCount: 1.5 })], + ["unsafe counts", (): unknown => ({ ...preAcceptanceReceipt(), documentCount: Number.MAX_SAFE_INTEGER + 1 })], + ["invalid document counts", (): unknown => ({ ...preAcceptanceReceipt(), nonPlanningDocumentCount: 1 })], + ["raw task status", (): unknown => ({ ...preAcceptanceReceipt(), firstTaskStatus: "private status" })], + ["raw parent status", (): unknown => ({ ...preAcceptanceReceipt(), canonicalParentStatus: "private status" })], + ["nonboolean match", (): unknown => ({ ...preAcceptanceReceipt(), firstTaskMatchesCanonicalParent: "true" })], + ["inconsistent task match", (): unknown => ({ ...preAcceptanceReceipt(), firstTaskMatchesCanonicalParent: false })], + ["inconsistent matched statuses", (): unknown => ({ ...preAcceptanceReceipt(), canonicalParentStatus: "done" })], + ["missing fields", (): unknown => ({ version: 1 })], + ["null", (): unknown => null], + ] as const)("rejects pre-acceptance evidence containing %s", (_label, makeReceipt) => { + const retained = result(); + Object.assign(retained.productJourney!, { onboardingPreAcceptance: makeReceipt() }); + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.productJourney.onboardingPreAcceptance"); + }); + it("renders a fixed fallback for malformed pre-acceptance evidence without exposing its fields", () => { + const retained = result(); + Object.assign(retained.productJourney!, { onboardingPreAcceptance: { ...preAcceptanceReceipt(), body: "private-body-must-not-render" } }); + const output = renderCaseOutcome(retained, false, []); + expect(output).toContain("Onboarding pre-acceptance evidence unavailable: invalid receipt"); + expect(output).not.toContain("private-body-must-not-render"); + expect(output).not.toContain("attachment_present"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/onboarding-pre-acceptance.ts b/tests/runner-e2e/product-journeys/onboarding-pre-acceptance.ts new file mode 100644 index 00000000000..0b0fe7e650e --- /dev/null +++ b/tests/runner-e2e/product-journeys/onboarding-pre-acceptance.ts @@ -0,0 +1,59 @@ +import { ISSUE_STATUSES } from "../../../packages/shared/src/constants.js"; + +const checkpoints = ["before-opening-answer", "awaiting-proposal"] as const; +const violations = ["unexpected_task_set", "attachment_present", "non_planning_document_present", "first_task_done"] as const; +const statuses = [...ISSUE_STATUSES, "missing", "unknown"] as const; + +/** Scalar observations from the first failing pre-acceptance guard; never raw rows. */ +export interface OnboardingPreAcceptanceEvidence { + version: 1; + checkpoint: typeof checkpoints[number]; + violations: Array; + taskCount: number; + attachmentCount: number; + documentCount: number; + nonPlanningDocumentCount: number; + firstTaskMatchesCanonicalParent: boolean; + firstTaskStatus: typeof statuses[number]; + canonicalParentStatus: typeof statuses[number]; +} + +const receiptKeys = { + version: true, checkpoint: true, violations: true, taskCount: true, attachmentCount: true, + documentCount: true, nonPlanningDocumentCount: true, firstTaskMatchesCanonicalParent: true, + firstTaskStatus: true, canonicalParentStatus: true, +} satisfies Record; + +export function preAcceptanceTaskStatus(task: { status?: unknown; [key: string]: unknown } | undefined): OnboardingPreAcceptanceEvidence["firstTaskStatus"] { + if (!task) return "missing"; + return ISSUE_STATUSES.find(status => status === task.status) ?? "unknown"; +} + +export function preAcceptanceViolations(observed: Pick): OnboardingPreAcceptanceEvidence["violations"] { + return [ + ...(observed.taskCount !== 1 || !observed.firstTaskMatchesCanonicalParent ? ["unexpected_task_set" as const] : []), + ...(observed.attachmentCount ? ["attachment_present" as const] : []), + ...(observed.nonPlanningDocumentCount ? ["non_planning_document_present" as const] : []), + ...(observed.firstTaskStatus === "done" ? ["first_task_done" as const] : []), + ]; +} + +/** Closed validation is shared by normalized-result readers and explicit report rendering. */ +export function validateOnboardingPreAcceptanceEvidence(value: unknown): asserts value is OnboardingPreAcceptanceEvidence { + const invalid = (): never => { throw new Error("invalid_onboarding_pre_acceptance_evidence"); }; + const keys = Object.keys(receiptKeys); + if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).length !== keys.length || keys.some(key => !Object.hasOwn(value, key))) invalid(); + const receipt = value as OnboardingPreAcceptanceEvidence; + if (receipt.version !== 1 || !checkpoints.includes(receipt.checkpoint) || + !Array.isArray(receipt.violations) || !receipt.violations.length || receipt.violations.length > violations.length || + receipt.violations.some(code => !violations.includes(code)) || new Set(receipt.violations).size !== receipt.violations.length || + [receipt.taskCount, receipt.attachmentCount, receipt.documentCount, receipt.nonPlanningDocumentCount].some(count => !Number.isSafeInteger(count) || count < 0) || + receipt.nonPlanningDocumentCount > receipt.documentCount || typeof receipt.firstTaskMatchesCanonicalParent !== "boolean" || + !statuses.includes(receipt.firstTaskStatus) || !statuses.includes(receipt.canonicalParentStatus)) invalid(); + if (receipt.taskCount === 0 && (receipt.firstTaskStatus !== "missing" || receipt.canonicalParentStatus !== "missing" || receipt.firstTaskMatchesCanonicalParent) || + receipt.taskCount > 0 && receipt.firstTaskStatus === "missing" || + receipt.firstTaskMatchesCanonicalParent && receipt.firstTaskStatus !== receipt.canonicalParentStatus || + receipt.taskCount === 1 && !receipt.firstTaskMatchesCanonicalParent && receipt.canonicalParentStatus !== "missing" || + preAcceptanceViolations(receipt).join(",") !== receipt.violations.join(",")) invalid(); +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index da0b054a606..00bdeb5c164 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -29,6 +29,7 @@ export interface JourneyEvidence { operations?: JourneyOperations; artifact?: OnboardingArtifactEvidence; onboardingConnect?: import("./onboarding-connect.js").OnboardingConnectEvidence; + onboardingPreAcceptance?: import("./onboarding-pre-acceptance.js").OnboardingPreAcceptanceEvidence; } export interface OnboardingArtifactEvidence { schema: "paperclip.product-journey.artifact/v1"; diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 56f38ed0bec..3d6beece13e 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -6,6 +6,7 @@ import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js"; import { DEPENDABLE_CELLS } from "./dependable-qualification.js"; import { validateOnboardingConnectEvidence } from "./product-journeys/onboarding-connect.js"; import { validateOnboardingArtifact } from "./product-journeys/artifact-evidence.js"; +import { validateOnboardingPreAcceptanceEvidence } from "./product-journeys/onboarding-pre-acceptance.js"; type Rule = (value: unknown, at: string) => void; const invalid = (at: string): never => { @@ -346,6 +347,7 @@ const fields = { targetId: optional(string), campaignId: optional(string), attemptId: optional(string), onboardingConnect: optional((value, at) => { try { validateOnboardingConnectEvidence(value); } catch { invalid(at); } }), + onboardingPreAcceptance: optional((value, at) => { try { validateOnboardingPreAcceptanceEvidence(value); } catch { invalid(at); } }), artifact: optional((value, at) => { try { validateOnboardingArtifact(value); } catch { invalid(at); } }), operations: optional(closedShape({ campaignId: string, attemptId: string, modelCostUsd: nullable(number), setupCostUsd: nullable(number), requestCount: integer, feedback: closedShape({ collection: oneOf("complete", "partial", "unavailable"), complaints: integer, suggestions: integer }), traceIds: array(string) })), @@ -404,6 +406,9 @@ export function validateRetainedRunnerResult( result(value, "result"); const checked = value as RunnerE2EResult; const artifact = checked.productJourney?.artifact; + if (checked.productJourney?.onboardingPreAcceptance && (checked.suiteId !== "product-journeys" || checked.caseId !== "cloud-onboarding" || + checked.status !== "failed" || checked.productJourney.outcome !== "failed" || + checked.error !== "onboarding_work_before_acceptance")) invalid("result.productJourney.onboardingPreAcceptance"); if (artifact && (checked.suiteId !== "product-journeys" || checked.caseId !== "cloud-onboarding" || checked.runIds && !checked.runIds.includes(artifact.originatingRunId))) invalid("result.productJourney.artifact"); if (checked.workerTiming !== undefined) {