diff --git a/doc/evals.md b/doc/evals.md index a5e51d0a7ff..985d358469f 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -6,6 +6,12 @@ one bound continuation, and an exact saved confirmation. Its four development cells use the recorded Codex and Claude models. Registration and calibration do not establish Slack, staging authentication, warm lifecycle, or live qualification. +[Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit +deployment targets and starting-state recipes. The shared task-properties +journey uses no model; Cloud onboarding supplies reusable verified state. Cloud +fleet operations and trusted CI remain private. Local eval gateway qualification +does not qualify Cloud routes, and these selections stay outside `--all`. + The explicit-only [case handoff workflow](../tests/runner-e2e/README.md#persistent-case-handoffs-explicit-only) contains arithmetic and receipt smoke cells for a same-task round trip among three native agents using real case/task APIs, saved documents, handoff receipts, and browser readback. Receipt smoke has diff --git a/maps/feature-map/onboarding.md b/maps/feature-map/onboarding.md index 3910b939c19..c7e2fd0c9de 100644 --- a/maps/feature-map/onboarding.md +++ b/maps/feature-map/onboarding.md @@ -18,8 +18,15 @@ Agent access: [companion](../agent-access-map/onboarding.md). - `company`: `onboarding/company` — create a company with its identity and initial agent. - `adapter`: `onboarding/adapter` — choose an available harness, model, account, and execution environment. - `first-task`: `onboarding/first-task` — leave setup with a task whose execution or missing prerequisite is visible. + - `resume`: `onboarding/resume` — return to an incomplete wizard without losing saved state. +The [Cloud Product onboarding journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) +defines portal workspace creation, API-key connection, native Codex creation, +and a first downloadable artifact with an independent oracle. Its dependent +task-properties journey reuses onboarding state. This is a scripted coverage +recipe, not a claim of a completed Cloud run or strict browser-only discovery. + ## How to get to it (user POV) ### `web-wizard` diff --git a/maps/feature-map/tasks.md b/maps/feature-map/tasks.md index 5058856523c..639bd31c4f3 100644 --- a/maps/feature-map/tasks.md +++ b/maps/feature-map/tasks.md @@ -17,10 +17,16 @@ Agent access: [companion](../agent-access-map/tasks.md). - `create`: `tasks/create` — capture a title/request with the intended agent, project, and attachments. - `organize`: `tasks/organize` — filter, sort, group, and navigate tasks without losing the selected context. - `properties`: `tasks/properties` — change status, priority, assignee, project, and relationships where authorized. + - `discussion`: `tasks/discussion` — persist comments, references, and files on the task. - `completion`: `tasks/completion` — distinguish backlog, active work, review, blocked, done, and cancelled outcomes. - `privacy`: `tasks/privacy` — create private tasks, share or revoke access, and inspect parent/project inheritance before changing visibility. +The [Product task-properties journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) +checks only assignment to the signed-in user and persistence after reload. Its +source is reusable across self-hosted and Cloud; Cloud live qualification is +separate. It does not cover other property operations or agent authorization. + ## How to get to it (user POV) ### `new-task` diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 5122598f5d8..e1a34eee982 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -1,5 +1,10 @@ # Runner E2E fixture authoring +[Deployment-targeted journeys](PRODUCT-JOURNEYS.md) resolve only their requested +fixture graph through `FixtureRegistry.setupAll(requested)`. Omitting the +argument preserves the existing all-fixtures behavior. A verified onboarding +context may seed the graph; independent setup is labeled as recipe provenance. + ## Connection creation fixtures See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md new file mode 100644 index 00000000000..e2320997bb1 --- /dev/null +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -0,0 +1,137 @@ +# Product journeys across deployments + +`product-journeys` is an explicit Product E2E selection. Deployment is separate +from the `local` / `daytona` execution environment. Existing selectors, execution +IDs, and the `--all` matrix keep their historical meaning. These are scripted +browser journeys with API setup and independent assertions; they are not strict +browser-only “use as a user” discovery. + +| Concept | Owner | +| --- | --- | +| Journey | User actions, prerequisites, assertions, feature-map associations | +| Target provider | Authentication, deployed identity, configuration, acquisition, cleanup | +| Setup recipe | Resolve requested company/user/task capabilities through the fixture registry | +| Campaign | Explicit journey order, target, model profile, limits, lifetime | + +Shared code is in `product-journeys/`. Cloud fleet operations, email-code login, +private state, and trusted CI live in the private `paperclip-cloud` repository. +The `TargetProvider` version-1 interface is defined in `product-journeys/types.ts`. +Journeys receive a ready browser, company-scoped API, IDs, and capabilities. They +never start a server or depend on a tenant filesystem. + +## Selections + +| Journey | Required starting state | Observed outcome | +| --- | --- | --- | +| `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | +| `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | + +Supported acquisitions are managed self-hosted (`local`), registered existing +Cloud (`local` or `daytona` when present), and new Cloud. New Cloud must start +with onboarding. Its context can then satisfy the identical task-properties +journey without another setup. A recipe reports `startingState: recipe`; +reusing successful onboarding reports `onboarding`. Preconfigured state is never +counted as onboarding coverage. + +Save this credential-free local target config outside the repository: + +```json +{ + "version": 1, + "id": "local-qa", + "mode": "managed-self-hosted", + "journeys": ["task-properties"], + "executionEnvironment": "local", + "baseline": { "features": {}, "connections": [] }, + "limits": { "durationMs": 600000, "modelBudgetCents": 0, "maxRuns": 0 } +} +``` + +```sh +pnpm test:e2e:runner -- --suite product-journeys --list +pnpm test:e2e:runner -- --target-config /private/target.json --preview +pnpm test:e2e:runner -- --target-config /private/target.json --output /private/new-campaign +``` + +Preview resolves selections and declared changes without loading the private +provider, reading credentials, signing in, provisioning, or making model calls. +Its output includes required state, features/connections, lifetime, and limits. +Output directories must be new; an attempt is never overwritten. Browser-only +results use null provider/model and runtime `none`, including failures before +the journey starts. No artificial model run is introduced for reporting. +New execution IDs have the form +`product-journeys....`; the +report reader continues to accept historical four-part execution IDs. + +Cloud config supplies `adapterModule` and the private provider's reference-only +`adapterConfig`. Use the private CLI and runbook to configure those fields. +Paid onboarding requires positive `modelBudgetCents` and `maxRuns`, a duration +of at most one hour, and a qualified operations adapter. The initial private +Cloud adapter admits at most 30 minutes and $10. Select `runner-codex-mini` +explicitly for its `gpt-5.4-mini` pilot; existing profile defaults remain unchanged. There are no automatic +story retries. One campaign may create at most one disposable workspace. + +## Evidence and lifecycle + +Results use the existing campaign/result schemas and dashboard. Each result +retains campaign/attempt IDs, source revision, definition hash, deployed tenant +commit, Cloud release, runtime identity, configuration hash, setup provenance, +checks, and authentication/setup/journey/collection/cleanup outcomes. A failed +prerequisite remains failed at that boundary; every dependent selection receives +a blocked result. Unsupported target capabilities are distinct from failures. +Deployment/configuration drift during a journey fails its result. + +Collection must finish within the campaign deadline. Cleanup and evidence +publication then share a separate ten-minute grace period, with the last thirty +seconds reserved for a twenty-second stop-only fallback and ten-second browser +close. These are cleanup bounds and do not admit further story or model work. +Provider lifecycle hooks receive a deadline and cancellation signal; they must +bound in-flight requests and fence late resource mutations or lease writes when +the signal aborts. A timed-out release cannot authorize evidence finalization or +lease release. Its manifest stays available for cleanup recovery. + +Every would-be successful result is checked against the retained-result success +contract before successful cleanup. Missing assertions or paid accounting proof +turn the campaign into a failure while preserving valid cost receipts, so an +invalid success cannot trigger disposable-stack deletion. + +Only post-journey screenshots are captured. Browser state, login codes, raw +errors, credentials, request headers, and auth screenshots are excluded. All +Cloud evidence is private; the public history publisher rejects these bundles. +The private provider uses conditional S3 identity/stack leases and intent/receipt +ownership manifests. An expired lease permits cleanup recovery, not reuse. + +Managed self-hosted instances are removed after every attempt. Existing Cloud +stacks retain the QA account and declared baseline while removing verified +run-owned fixtures. Successful disposable stacks are deleted. Failed disposable +stacks are stopped and retained for 24 hours, then swept. Interrupted cleanup +remains visible and retains exclusive stack ownership. QA identity and inbox +never belong to disposable ownership. + +## Eval operations integration and qualification + +The private adapter carries the same campaign/attempt identity through bounded +request admission, setup/agent phase transitions, run observation, cost receipts, +trace IDs, and feedback collection. A normalized operations receipt belongs +only to the onboarding attempt; descendants do not repeat shared setup charges. +Unknown cost remains unknown. Raw feedback, traces, and ledgers stay private. + +The sibling **Operationalize eval runs** implementation currently admits local +cells and binds its gateway to loopback. That is not Cloud reachability or budget +qualification. A Cloud adapter must prove the selected runtime's route, effective +credential binding, budget admission, and exact campaign deadline before any +paid provider action. The normal wizard first saves a custom-gateway connection +and selects its model without inference. Qualification checks this exact saved +route from the actual runtime before the normal Connect probe. Missing qualification blocks onboarding; it must never fall +back to an unmetered direct provider request. + +As of 2026-10-22, the same task-properties story has passed in real browsers on +managed self-hosted and a registered testing Cloud stack, including cleanup and +zero model calls. Normal QA email-code authentication and account admission have +also passed against testing. Cloud onboarding selectors and lifecycle contracts +have offline tests; fresh onboarding, dependent reuse, and disposable cleanup +remain unqualified. Provider and trace secret references are configured. The +separate onboarding gateway product change, trusted CI selection, and real +runtime gateway admission remain live qualification prerequisites. Testing and staging use separate +registries and credentials; only the declared testing control plane is admitted. Defining a story or passing structural checks is +not evidence that its Cloud journey passed. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 1fc34485d5d..df516223c84 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -227,6 +227,11 @@ settlement substitutes for first visible response. The legacy warm-turn accordingly. Report regeneration recomputes interval verdicts, retains changed `recordedVerdict` values and all bounds, and leaves the original files intact. +The explicit [deployment-targeted Product journeys](PRODUCT-JOURNEYS.md) reuse +browser actions across managed self-hosted and Cloud targets. They include a +model-free task-properties story, fresh Cloud onboarding, credential-free +previews, private evidence, and prerequisite reuse; `--all` is unchanged. + ## Live provider connection journeys See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 7cba2b55c14..093f2066130 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -1,5 +1,12 @@ # Runner E2E security for a public repository +[Cloud Product journey bundles](PRODUCT-JOURNEYS.md) are always private and +rejected by public history publication. Authentication uses an isolated browser +context without saved storage state or auth tracing. Only closed error codes +and normalized operations receipts cross the private target-provider boundary. +The private Cloud repository owns operator credentials, conditional S3 leases, +ownership manifests, cleanup recovery, and environment-protected CI dispatch. + ## Attended connection authentication See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/billing.ts b/tests/runner-e2e/billing.ts index 5bdcfe875a4..4681696a7ee 100644 --- a/tests/runner-e2e/billing.ts +++ b/tests/runner-e2e/billing.ts @@ -258,11 +258,12 @@ export function fallbackRuntimeUsage( export function summarizeExecutionBilling( result: Pick< RunnerE2EResult, - "runIds" | "usage" | "environmentId" | "durationMs" | "runtimeUsage" | "firstTaskQuality" | "completionQuality" | "publicMcp" - >, + "runIds" | "usage" | "environmentId" | "durationMs" | "runtimeUsage" | "firstTaskQuality" | "completionQuality" | "publicMcp" | "productJourney" + > & Partial>, ): RunnerE2EBillingSummary { - const requestedRunCount = Math.max(result.runIds?.length ?? 0, 1); - const entries = usageEntries(result.usage, requestedRunCount); + const browserOnly = Boolean(result.productJourney && result.runtimeMode === "none" && !result.runIds?.length); + const requestedRunCount = browserOnly ? 0 : Math.max(result.runIds?.length ?? 0, 1); + const entries = browserOnly ? [] : usageEntries(result.usage, requestedRunCount); let inputTokens = 0; let outputTokens = 0; let cachedInputTokens = 0; @@ -305,13 +306,19 @@ export function summarizeExecutionBilling( : runsWithTokenUsage > 0 ? "unpriced" : "unavailable"; - const runtime = fallbackRuntimeUsage(result); + const runtime = { ...fallbackRuntimeUsage(result) }; + if (browserOnly) runtime.agentRunDurationMs = 0; const estimatedRuntimeCostUsd = runtime.estimatedListCostUsd ?? 0; const judgments = [...(result.firstTaskQuality ? [result.firstTaskQuality] : []), ...(result.completionQuality ?? [])]; const sumKnown = (key: "inputTokens" | "outputTokens" | "estimatedCostUsd") => judgments.some(q => q[key] === null) ? null : judgments.reduce((sum, q) => sum + (q[key] ?? 0), 0); const quality = judgments.length ? { inputTokens: sumKnown("inputTokens"), outputTokens: sumKnown("outputTokens"), estimatedCostUsd: sumKnown("estimatedCostUsd"), reservedCostUsd: judgments.reduce((sum, q) => sum + q.reservedCostUsd, 0) } : undefined; + // The operations receipt covers setup and agent requests for ONE owning attempt. + // Descendants reuse the fixture context without copying its receipt or charges. + const operations = result.productJourney?.operations; + const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd; const complete = (!quality || quality.estimatedCostUsd !== null) && + (!operations || operations.modelCostUsd !== null) && runsWithTokenUsage === runCount && runsWithReportedCost === runCount && runtime.costStatus !== "unavailable"; @@ -332,7 +339,7 @@ export function summarizeExecutionBilling( reportedCostUsd, estimatedRuntimeCostUsd, ...(quality ? { judge: { inputTokens: quality.inputTokens, outputTokens: quality.outputTokens, estimatedCostUsd: quality.estimatedCostUsd, reservedCostUsd: quality.reservedCostUsd } } : {}), - observedAndEstimatedCostUsd: quality?.estimatedCostUsd === null ? null : reportedCostUsd + estimatedRuntimeCostUsd + (quality?.estimatedCostUsd ?? 0) + (result.publicMcp?.estimatedCostUsd ?? 0), + observedAndEstimatedCostUsd: modelSubtotal === null || quality?.estimatedCostUsd === null ? null : modelSubtotal + estimatedRuntimeCostUsd + (quality?.estimatedCostUsd ?? 0) + (result.publicMcp?.estimatedCostUsd ?? 0), complete, }; } diff --git a/tests/runner-e2e/case-outcome.ts b/tests/runner-e2e/case-outcome.ts index 55d1b91c825..03316c81995 100644 --- a/tests/runner-e2e/case-outcome.ts +++ b/tests/runner-e2e/case-outcome.ts @@ -13,6 +13,7 @@ export function isIncompleteFirstTaskResult( result: RunnerE2EResult, errors: readonly string[] = [], ) { + if (result.productJourney && ["blocked", "unsupported"].includes(result.productJourney.outcome)) return result.cleanup === "passed" && errors.length === 0; if (isBlockedConnectionResult(result)) return result.cleanup === "passed" && result.failureClass !== "secret_leak" && errors.length === 0; const checks = result.firstTask?.checks ?? []; return result.status === "failed" && @@ -28,7 +29,7 @@ export function renderCaseOutcome( errors: readonly string[], ) { const checks = - (result.firstTask?.checks.length ? result.firstTask.checks : undefined) ?? + (result.productJourney?.checks.map(c => ({ ...c, detail: c.id.replaceAll("-", " "), notReached: undefined as string | undefined })) ?? (result.firstTask?.checks.length ? result.firstTask.checks : undefined)) ?? (result.matcherResults ?? []).map((m) => ({ id: m.matcher.kind === "json_path" ? m.matcher.path : m.matcher.kind, passed: m.passed, @@ -56,7 +57,18 @@ export function renderCaseOutcome( ), ]; return `
- ${isBlockedConnectionResult(result) ? `Blocked: ${html(result.providerConnection!.outcome.replaceAll("_", " "))}` : incomplete ? "Incomplete journey" : failed ? "Overall failed" : "Overall passed"} · ${checks.length ? `${passed}/${evaluated} behavioral checks passed${notReached.length ? ` · ${notReached.length} not reached` : ""}` : "No behavioral checks recorded"} + ${result.productJourney ? html(result.productJourney.outcome) : isBlockedConnectionResult(result) ? `Blocked: ${html(result.providerConnection!.outcome.replaceAll("_", " "))}` : incomplete ? "Incomplete journey" : failed ? "Overall failed" : "Overall passed"} · ${checks.length ? `${passed}/${evaluated} behavioral checks passed${notReached.length ? ` · ${notReached.length} not reached` : ""}` : "No behavioral checks recorded"} + ${result.productJourney ? `

Deployment: ${html(result.productJourney.deployment?.kind ?? "unavailable")} · ${html(result.productJourney.deployment?.targetId ?? "unavailable")} · ${html(result.productJourney.deployment?.commit ?? "revision unavailable")} · starting state: ${html(result.productJourney.startingState)} · resources: ${html(result.productJourney.resourceDisposition)}

    ${result.productJourney.phases.map(p => `
  • ${html(p.phase)}: ${html(p.outcome)}${p.code ? ` (${html(p.code)})` : ""}
    • `).join("")}
` : ""} + ${result.productJourney ? `
Deployment and setup evidence
${[ + ["Campaign", result.productJourney.campaignId], ["Attempt", result.productJourney.attemptId], + ["Origin", result.productJourney.deployment?.origin], ["Cloud release", result.productJourney.deployment?.cloudRelease], + ["Runtime image", result.productJourney.deployment?.runtimeImage], ["Effective configuration hash", result.productJourney.deployment?.configurationHash], + ["Harness revision", result.source?.sha], ["Journey definition hash", result.suiteDefinitionHash], + ].map(([label, value]) => `
${html(label)}
${html(value ?? "Unavailable")}
`).join("")}
+ ${result.productJourney.deployment?.executionEnvironment ? `

Managed environment: ${html(result.productJourney.deployment.executionEnvironment.id)} · ${html(result.productJourney.deployment.executionEnvironment.driver)} · ${html(result.productJourney.deployment.executionEnvironment.status)}

` : ""} + ${result.productJourney.deployment?.effectiveFeatures ? `

Effective features:

${Object.entries(result.productJourney.deployment.effectiveFeatures).map(([key, value]) => `
${html(key)}
${html(value)}
`).join("")}
` : ""}
` : ""} + ${result.productJourney?.operations ? `

Eval operations: ${html(result.productJourney.operations.campaignId)} · ${result.productJourney.operations.requestCount} requests · model cost $${html(result.productJourney.operations.modelCostUsd ?? "unknown")} (setup $${html(result.productJourney.operations.setupCostUsd ?? "unknown")}, counted once). Feedback: ${html(result.productJourney.operations.feedback.collection)} · ${result.productJourney.operations.feedback.complaints} complaints · ${result.productJourney.operations.feedback.suggestions} suggestions.

` : ""} + ${result.productJourney?.operations?.traceIds.length ? `
Trace correlation IDs
    ${result.productJourney.operations.traceIds.map(id => `
  • ${html(id)}
    • `).join("")}
` : ""} ${result.providerConnection ? `

Target: ${html(result.providerConnection.target.origin)} · ${html(result.providerConnection.target.commit ?? "revision unavailable")} · ${result.providerConnection.assisted ? "Assisted login" : "Unassisted"} · ${html(result.providerConnection.authFreshness)} browser · ${html(result.providerConnection.entry)} / ${html(result.providerConnection.method)} · stopped at ${html(result.providerConnection.phase)}

` : ""} ${failures.length ? `

Failed checks:

    ${failures.map((c) => `
  • ${html(c.id)} — ${html(c.detail)}
    • `).join("")}
` : checks.length ? `

No behavioral matcher failed.${notReached.length ? " The journey is incomplete; unexercised checks are not passes." : failed ? " The overall failure came from a separate run, cleanup, or evidence check." : ""}

` : ""} ${notReached.length ? `

Not reached:

    ${notReached.map((c) => `
  • ${html(c.id)} — ${html(c.notReached)}
    • `).join("")}
` : ""} diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 5ba023cf1b0..b06ccb8b1fd 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -20,6 +20,7 @@ import { daytonaWarmEnvironment, isImmutableDaytonaImage, pendingContextIntegrityProfiles, + productJourneys, suiteDefinitionHash, validateRunnerCatalog, } from "./catalog.js"; @@ -41,7 +42,7 @@ describe("runner E2E catalog", () => { expect(cell.requiredCredentials).toEqual([credential]); expect(cell.task).toMatchObject({ flow: "case_handoff", expectedRunCount: 4, minimumExpectedRunCount: 4, turnTimeoutMs: 90_000, attemptTimeoutMs: { local: 300_000 }, automaticRetryPolicy: "single_attempt" }); - expect(cell.suite.definitionMetadata).toMatchObject({ version: 3, scheduling: "explicit-only", + expect(cell.suite.definitionMetadata).toMatchObject({ version: 4, scheduling: "explicit-only", credentialModes: ["isolated-company-subscription-login", "company-encrypted-api-secret-reference"], cases: { "receipt-smoke": { budgetMonthlyCents: 100, maximumProviderTurns: 4, minimumProviderTurns: 4, runTimeoutMs: 90_000, reasoningEffort: "low", agentDailyRunCaps: { triage: 1, builder: 2, verifier: 1 } }, @@ -57,6 +58,13 @@ describe("runner E2E catalog", () => { ? { ...task, expectedRunCount: 5 } : task) })).not.toBe(cell.suiteDefinitionHash); }); + it("exports deployment journeys separately from the model matrix and default selection", () => { + expect(productJourneys.map(journey => journey.id)).toEqual(["task-properties", "cloud-onboarding"]); + expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); + expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); + }); + it("keeps the single managed onboarding variant explicit-only without changing local onboarding coverage", () => { const id = "confirmation-replies.runner-codex.daytona.task-card-accept"; const suite = runnerSuites.find(candidate => candidate.id === "confirmation-replies")!; @@ -211,10 +219,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, + 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 4, 116, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(760); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(760); + expect(validateRunnerCatalog()).toHaveLength(761); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(761); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 926acf26998..8f33bc2ad3c 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -2042,3 +2042,5 @@ export function runnerExecutionById(id: string): MatrixExecution { if (!execution) throw new Error(`Unknown runner E2E execution id: ${id}`); return execution; } + +export { productJourneys } from "./product-journeys/catalog.js"; diff --git a/tests/runner-e2e/connection-support.test.ts b/tests/runner-e2e/connection-support.test.ts index 77738d80444..0a6127b9a0a 100644 --- a/tests/runner-e2e/connection-support.test.ts +++ b/tests/runner-e2e/connection-support.test.ts @@ -9,9 +9,8 @@ import { connectionHarnesses, connectionScopeGaps, connectionCell, supportsConne import { parseConnectionConfig, targetOrigin, selectedSecret, resolveConnectionSecret, resolveConnectionSettings } from "./connection-config.js"; import { connectionCheckpoints, connectionEvidencePasses, connectionRunDiagnostic, createConnectionProof, verifyConnectionArtifact, verifyConnectionRun, completedConnectionArtifactRun, connectionProbeChecks, type ConnectionEvidence } from "./connection-evidence.js"; import { validateRetainedRunnerResult } from "./result-validation.js"; -import { verifyConnectionTarget } from "./connection-target.js"; +import { ConnectionApi, verifyConnectionTarget } from "./connection-target.js"; import { cleanupConnectionCompany, connectionCreationReceipt, runConnectionCampaign, safeConnectionFailure, withConnectionCampaignCancellation } from "./connection-launch.js"; -import type { ConnectionApi } from "./connection-target.js"; import * as connectionConfig from "./connection-config.js"; import * as connectionTarget from "./connection-target.js"; import * as reportAssets from "./report-assets.js"; @@ -144,6 +143,16 @@ describe("live connection coverage", () => { }); describe("target and selected credential boundaries", () => { + it("keeps scoped requests on tenant APIs and bounds extended probe timeouts", async () => { + const fetch = vi.fn(async (_url: string, _options: unknown) => ({ ok: () => true, json: async () => ({ passed: true }) })); + const api = new ConnectionApi({ fetch } as any, "https://qa.test.paperclip.app"); + await api.post("/api/companies/qa/adapters/paperclip_runner/test-environment", {}, { timeoutMs: 120_000 }); + expect(fetch.mock.calls[0]?.[1]).toMatchObject({ headers: { Origin: "https://qa.test.paperclip.app" }, timeout: 120_000, maxRedirects: 0 }); + fetch.mockClear(); + for (const route of ["https://other.test/api/health", "/api/../v1/admin", "/api/%2e%2e/v1/admin"]) await expect(api.get(route)).rejects.toThrow("invalid_api_route"); + await expect(api.post("/api/health", {}, { timeoutMs: 120_001 })).rejects.toThrow("invalid_api_timeout"); + expect(fetch).not.toHaveBeenCalled(); + }); it("accepts local/staging origins and rejects credential-bearing or redirectable target strings", () => { expect(targetOrigin("http://127.0.0.1:3109/")).toBe("http://127.0.0.1:3109"); expect(targetOrigin("https://staging.example.test")).toBe("https://staging.example.test"); diff --git a/tests/runner-e2e/connection-target.ts b/tests/runner-e2e/connection-target.ts index c0c5c2acbca..455c67d8c9e 100644 --- a/tests/runner-e2e/connection-target.ts +++ b/tests/runner-e2e/connection-target.ts @@ -15,10 +15,13 @@ export const connectionDelay = (ms: number) => new Promise(resolve => setT /** No raw response bodies, callback URLs, headers, or Playwright diagnostics escape. */ export class ConnectionApi { constructor(readonly request: APIRequestContext, readonly origin: string) {} - async response(route: string, method = "GET", data?: unknown) { - if (!route.startsWith("/api/") || new URL(route, this.origin).origin !== this.origin) throw new ConnectionFailure("invalid_api_route"); + async response(route: string, method = "GET", data?: unknown, options: { timeoutMs?: number } = {}) { + const target = new URL(route, this.origin); + if (!route.startsWith("/api/") || target.origin !== this.origin || !target.pathname.startsWith("/api/")) throw new ConnectionFailure("invalid_api_route"); + const timeout = options.timeoutMs ?? 30_000; + if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 120_000) throw new ConnectionFailure("invalid_api_timeout"); try { - const response = await this.request.fetch(`${this.origin}${route}`, { method, ...(data === undefined ? {} : { data }), maxRedirects: 0, timeout: 30_000 }); + const response = await this.request.fetch(`${this.origin}${route}`, { method, headers: { Origin: this.origin }, ...(data === undefined ? {} : { data }), maxRedirects: 0, timeout }); if (!response.ok()) throw new ConnectionFailure(`api_status_${response.status()}`); return response; } catch (error) { @@ -27,7 +30,7 @@ export class ConnectionApi { } } async get(route: string): Promise { return (await this.response(route)).json(); } - async post(route: string, data?: unknown): Promise { return (await this.response(route, "POST", data)).json(); } + async post(route: string, data?: unknown, options?: { timeoutMs?: number }): Promise { return (await this.response(route, "POST", data, options)).json(); } async patch(route: string, data: unknown): Promise { return (await this.response(route, "PATCH", data)).json(); } async delete(route: string) { await this.response(route, "DELETE"); } async bytes(route: string) { return (await this.response(route)).body(); } diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 2875917c056..c949f6a07b1 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -146,6 +146,7 @@ function renderCase( const outcome = entry ? renderCaseOutcome(entry.result, entry.valid, entry.errors) : ""; const passed = state === "passed" && entry?.result.status === "passed"; const firstTaskChecks = entry?.result.firstTask?.checks; + const productChecks = entry?.result.productJourney?.checks; const notReached = firstTaskChecks?.filter((c) => c.notReached).length ?? 0; const matcherResults = firstTaskChecks?.length ? firstTaskChecks.filter((c) => !c.notReached).map((c) => ({ @@ -153,7 +154,10 @@ function renderCase( passed: c.passed, detail: c.detail, })) - : entry?.result.matcherResults ?? []; + : productChecks?.length ? productChecks.map(c => ({ + matcher: { kind: "json_path" as const, path: `productJourney.checks.${c.id}`, expected: true }, + passed: c.passed, detail: c.id.replaceAll("-", " "), + })) : entry?.result.matcherResults ?? []; const passedMatchers = matcherResults.filter( (result) => result.passed, ).length; diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts index cfd6e80df92..4ca6a41d376 100644 --- a/tests/runner-e2e/daytona-image-content.ts +++ b/tests/runner-e2e/daytona-image-content.ts @@ -60,6 +60,7 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [ "packages/paperclip-runner/test/fixtures/copilot-profile-v49-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v54-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v55-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v58-identity.json", "packages/paperclip-runner/scripts/materialize-cursor-distribution.mjs", "packages/paperclip-runner/scripts/cursor-runtime-patch.mjs", "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs", diff --git a/tests/runner-e2e/daytona-image.test.ts b/tests/runner-e2e/daytona-image.test.ts index f3bdd9467a0..80d8ffdf0b1 100644 --- a/tests/runner-e2e/daytona-image.test.ts +++ b/tests/runner-e2e/daytona-image.test.ts @@ -36,6 +36,32 @@ describe("runner E2E Daytona image contract", () => { expect(path.matchesGlob("packages/paperclip-runner/test/fixtures/copilot-live-denial-2026-09-28.json", exception)).toBe(false); }); + it("invalidates the image content ID when the active Copilot identity changes", async () => { + const generator = await readFile( + path.join(repositoryRoot, "packages/paperclip-runner/scripts/copilot-profile-identity.mjs"), + "utf8", + ); + const fixture = generator.match(/const fixture = resolve\(root, "([^"]+)"\)/)?.[1]; + expect(fixture).toBeDefined(); + const fixturePath = `packages/paperclip-runner/${fixture}`; + const root = await mkdtemp(path.join(tmpdir(), "paperclip-daytona-copilot-identity-")); + const options = { + repositoryRoot: root, + inputPaths: DAYTONA_IMAGE_INPUT_PATHS.filter(inputPath => inputPath === fixturePath), + baseImages: [`example.test/base:1@sha256:${"a".repeat(64)}`], + frontendDigest: `sha256:${"c".repeat(64)}`, + } as const; + try { + await mkdir(path.dirname(path.join(root, fixturePath)), { recursive: true }); + await writeFile(path.join(root, fixturePath), '{"commandDigest":"before"}\n'); + const before = await computeDaytonaImageContentId(options); + await writeFile(path.join(root, fixturePath), '{"commandDigest":"after"}\n'); + expect(await computeDaytonaImageContentId(options)).not.toBe(before); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("keeps the qualified native Grok binary separate from the legacy command", async () => { const [dockerfile, packBuilder, runnerPackage] = await Promise.all([ readFile(path.join(repositoryRoot, "docker/daytona-runner/Dockerfile"), "utf8"), diff --git a/tests/runner-e2e/fixture-registry.ts b/tests/runner-e2e/fixture-registry.ts index b9b994fb56f..b2a54fec35c 100644 --- a/tests/runner-e2e/fixture-registry.ts +++ b/tests/runner-e2e/fixture-registry.ts @@ -15,7 +15,7 @@ export class FixtureRegistry { return this; } - async setupAll() { + async setupAll(requested: readonly string[] = [...this.#definitions.keys()]) { const resolved = new Map(); const completed: FixtureDefinition[] = []; const visiting = new Set(); @@ -36,7 +36,7 @@ export class FixtureRegistry { }; try { - for (const id of this.#definitions.keys()) await setup(id); + for (const id of requested) await setup(id); } catch (setupError) { try { await this.#teardown(completed, resolved); diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts index b33458adfd4..80159a5f0a3 100644 --- a/tests/runner-e2e/history-publish.ts +++ b/tests/runner-e2e/history-publish.ts @@ -83,6 +83,7 @@ function publicEvidencePath( } export function publicScreenshotPaths(campaign: RunnerE2ECampaign) { + assertPublicCampaign(campaign); const screenshots = new Set(); for (const result of campaign.results) { const files = @@ -116,6 +117,12 @@ export function publicScreenshotPaths(campaign: RunnerE2ECampaign) { return screenshots; } +export function assertPublicCampaign(campaign: Pick) { + if (campaign.results.some(result => result.productJourney?.privacy === "private")) { + throw new Error("Private Product E2E campaigns cannot enter the public history publisher"); + } +} + export function isHistoricalBundlePathAllowed( relative: string, allowPublicSummary = false, @@ -521,6 +528,7 @@ async function main() { await readFile(path.join(reportRoot, "normalized-results.json"), "utf8"), ) as RunnerE2ECampaign; for (const result of campaign.results ?? []) validateRetainedRunnerResult(result); + assertPublicCampaign(campaign); if (campaign.schema !== "paperclip.runner-e2e.campaign/v2") { throw new Error("Historical publishing requires a v2 normalized campaign"); } diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 15c3648a2d4..092ce425f9e 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -18,6 +18,7 @@ function associate(suites: readonly string[], cases: Record( } async function main() { + const args = process.argv.slice(2); + if (args.includes("--target-config") || args.includes("product-journeys")) { + const { launchProductJourneys } = await import("./product-journeys/launch.js"); + return launchProductJourneys(args, repositoryRoot); + } let options: ReturnType; try { options = parseRunnerSelectors(process.argv.slice(2)); diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index df579dce971..599d8c6fc25 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(760); + expect(validateRunnerCatalog()).toHaveLength(761); for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); diff --git a/tests/runner-e2e/product-journeys/campaign.test.ts b/tests/runner-e2e/product-journeys/campaign.test.ts new file mode 100644 index 00000000000..ff1a0689012 --- /dev/null +++ b/tests/runner-e2e/product-journeys/campaign.test.ts @@ -0,0 +1,264 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { mkdtemp, writeFile, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { pathToFileURL } from "node:url"; +const persistenceFault = vi.hoisted(() => ({ kind: null as "write" | "rename" | null, active: false, latestReport: null as any })); +vi.mock("node:fs/promises", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, + async writeFile(...args: Parameters) { + if (String(args[0]).endsWith("/normalized-results.tmp")) { + persistenceFault.latestReport = JSON.parse(String(args[1])); + if (persistenceFault.active && persistenceFault.kind === "write") throw Error("private disk diagnostic api_key_fixture"); + } + return actual.writeFile(...args); + }, + async rename(...args: Parameters) { + if (String(args[1]).endsWith("/normalized-results.json") && persistenceFault.active && persistenceFault.kind === "rename") throw Error("private rename diagnostic api_key_fixture"); + return actual.rename(...args); + }, + }; +}); +vi.mock("@playwright/test", () => ({ chromium: { launch: async () => ({ newContext: async () => ({setDefaultTimeout(){}, pages:()=>[]}), close:async()=>{} }) }, expect: {} })); +vi.mock("../dashboard-regenerate.js", () => ({ regenerateRunnerDashboard: vi.fn(async (input:{outputDirectory?:string}) => { + if(input.outputDirectory){const {writeFile}=await import("node:fs/promises");await Promise.all(["index.html","dashboard.html"].map(name=>writeFile(path.join(input.outputDirectory!,name),"dashboard")));} +}) })); +vi.mock("./task-properties.js", () => ({taskProperties: async (_context:unknown,_session:unknown,_id:unknown,evidence:any) => {evidence.checks.push({id:"assignment-persisted",passed:true},{id:"no-agent-work",passed:true});}})); +import { chromium } from "@playwright/test"; +import { launchProductJourneys } from "./launch.js"; +import { regenerateRunnerDashboard } from "../dashboard-regenerate.js"; +const roots: string[]=[]; +afterEach(async()=>{vi.useRealTimers(); vi.restoreAllMocks(); persistenceFault.kind=null; persistenceFault.active=false; persistenceFault.latestReport=null; process.exitCode=0; await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));}); +type Hook = "collect" | "release" | "stop" | "finalizeEvidence" | "checkpointEvidence"; +async function run(options:{authFailure?:boolean;onboardingFailure?:boolean;drift?:boolean;cleanupFailure?:boolean;collectionFailure?:boolean;mini?:boolean;missingReceipt?:boolean;zeroRequests?:boolean;missingCheck?:boolean;finalizationFailure?:boolean;persistenceFailure?:"write"|"rename";persistenceAfter?:"collection"|"release";stallHook?:Hook;onHook?:(hook:Hook,signal?:AbortSignal)=>void}={}) { + const root=await mkdtemp(path.join(os.tmpdir(),"journey-campaign-test-")); roots.push(root); + const module=path.join(root,"provider.mjs"); + await writeFile(module, `import {readFile,writeFile} from 'node:fs/promises'; +const identity={kind:'cloud',targetId:'qa',origin:'https://qa.example.test',commit:'${"a".repeat(40)}',cloudRelease:'v1',runtimeImage:'image',configurationHash:'config'}; +let onboarded=false; +const context={identity,company:{id:'company-1',issuePrefix:'QA'},api:{post:async()=>({id:'issue-fixture-1',identifier:'QA-1'})},startingState:'onboarding',capabilities:['company','board-user','task-write'],page:{async screenshot(){} }}; +export const provider={version:1, async acquire(input){return { +async authenticate(){${options.authFailure ? "throw {outcome:'failed',code:'login_code_expired'};" : ""}}, +async onboard(evidence){${options.onboardingFailure ? "throw {outcome:'failed',code:'onboarding_failed'};" : `evidence.checks.push(...['fresh-workspace','ui-connection-and-native-agent','first-task-complete','downloadable-artifact-verified'${options.missingCheck ? "" : ",'native-trace-verified'"}].map(id=>({id,passed:true})));onboarded=true;return context;`}}, +async prepare(){throw Error('onboarding receipt was not reused')}, +async identity(){return {...identity,commit:'${(options.drift?"b":"a").repeat(40)}'}}, +async collect(){${options.collectionFailure ? "throw Error('private collection error');" : options.missingReceipt ? "return [];" : `return onboarded?[{journeyId:'cloud-onboarding',runIds:['run-1'],usage:{runs:[{runId:'run-1',usage:{costUsd:0.01}}]},operations:{campaignId:input.campaignId,attemptId:input.campaignId+':cloud-onboarding:1',modelCostUsd:0.01,setupCostUsd:0.005,requestCount:${options.zeroRequests ? 0 : 2},feedback:{collection:'complete',complaints:0,suggestions:0},traceIds:['a'.repeat(32)]}}]:[];`}}, async own(){}, async release(){${options.cleanupFailure ? "throw Error('private raw error');" : "return 'released';"}}, async stop(){} +, async finalizeEvidence(){ +const report=JSON.parse(await readFile(input.outputDir+'/normalized-results.json','utf8')); +if(report.results.some(r=>r.cleanup==='not_started'))throw Error('final evidence preceded cleanup'); +${options.finalizationFailure ? "throw Error('private evidence upload diagnostic');" : ""} +}, async checkpointEvidence(){ +const report=JSON.parse(await readFile(input.outputDir+'/normalized-results.json','utf8')); +if(report.results.some(r=>r.cleanup!=='failed'))throw Error('failure checkpoint preceded failure persistence'); +await writeFile(input.outputDir+'/failure-checkpoint.json',JSON.stringify(report)); +}}}};`); + const config={version:1,id:"qa",mode:"new-cloud",adapterModule:module,journeys:["cloud-onboarding","task-properties"],executionEnvironment:"daytona",baseline:{features:{},connections:[]},limits:{durationMs:60000,modelBudgetCents:1000,maxRuns:1},...(options.mini?{modelProfileId:"runner-codex-mini"}:{})}; + const file=path.join(root,"target.json"),output=path.join(root,"report"); await writeFile(file,JSON.stringify(config)); + const lifecycle: string[] = []; + { + persistenceFault.kind=options.persistenceFailure ?? null; + const { provider } = await import(pathToFileURL(module).href); + const acquire = provider.acquire; + vi.spyOn(provider,"acquire").mockImplementation(async (...args: unknown[]) => { + const session = await acquire(...args); + return { ...session, + async collect(input?: {signal:AbortSignal}) { options.onHook?.("collect",input?.signal); if(options.stallHook==="collect")await new Promise(()=>{}); try { return await session.collect(); } finally { if (options.persistenceAfter !== "release" && options.persistenceFailure) persistenceFault.active=true; } }, + async release(outcome: string,input?: {signal:AbortSignal}) { lifecycle.push(`release:${outcome}`); options.onHook?.("release",input?.signal); if(options.stallHook==="release")await new Promise(()=>{}); try { return await session.release(outcome); } finally { if (options.persistenceAfter === "release" && options.persistenceFailure) persistenceFault.active=true; } }, + async stop(input?: {signal:AbortSignal}) { lifecycle.push("stop"); options.onHook?.("stop",input?.signal); if(options.stallHook==="stop")await new Promise(()=>{}); }, + async finalizeEvidence(input?: {signal:AbortSignal}) { lifecycle.push("finalizeEvidence"); options.onHook?.("finalizeEvidence",input?.signal); if(options.stallHook==="finalizeEvidence")await new Promise(()=>{}); return session.finalizeEvidence(); }, + async checkpointEvidence(input?: {signal:AbortSignal}) { lifecycle.push("checkpointEvidence"); options.onHook?.("checkpointEvidence",input?.signal); if(options.stallHook==="checkpointEvidence")await new Promise(()=>{}); return session.checkpointEvidence(); }, + }; + }); + } + await launchProductJourneys(["--target-config",file,"--output",output],path.resolve(import.meta.dirname,"../../..")); + return { ...JSON.parse(await readFile(path.join(output,"normalized-results.json"),"utf8")), + lifecycle, outputDir:output, attemptedReport: persistenceFault.latestReport, + failureCheckpoint: await readFile(path.join(output,"failure-checkpoint.json"),"utf8").then(JSON.parse).catch(()=>null) }; +} +describe("product journey campaign boundary outcomes",()=>{ + it("accepts the conventional pnpm separator in a credential-free preview", async () => { + const root=await mkdtemp(path.join(os.tmpdir(),"journey-preview-test-")); roots.push(root); + const file=path.join(root,"target.json"); + await writeFile(file,JSON.stringify({version:1,id:"qa",mode:"existing-cloud",adapterModule:"/not-installed-private-provider.mjs",adapterConfig:{operatorSecret:"never-print"},journeys:["task-properties"],executionEnvironment:"daytona",baseline:{features:{},connections:[]},limits:{durationMs:60000,modelBudgetCents:0,maxRuns:0}})); + const output=vi.spyOn(console,"log").mockImplementation(()=>{}); + const launch=vi.spyOn(chromium,"launch"); + await launchProductJourneys(["--","--target-config",file,"--preview"],path.resolve(import.meta.dirname,"../../..")); + const preview=JSON.parse(String(output.mock.calls[0]?.[0])); + expect(preview).toMatchObject({modelProfileId:null,retries:0,limits:{modelBudgetCents:0,maxRuns:0}}); + expect(JSON.stringify(preview)).not.toContain("never-print"); expect(launch).not.toHaveBeenCalled(); + }); + it("retains browser startup failure and every selected dependent result", async () => { + vi.spyOn(chromium, "launch").mockRejectedValueOnce(new Error("private startup detail")); + const campaign = await run(); + expect(campaign.results.map((r:any)=>r.productJourney.outcome)).toEqual(["failed", "blocked"]); + expect(JSON.stringify(campaign)).not.toContain("private startup detail"); + }); + it("preserves failed auth and reports every dependent journey blocked",async()=>{ + const campaign=await run({authFailure:true}); + expect(campaign.results.map((r:any)=>r.productJourney.outcome)).toEqual(["failed","blocked"]); + expect(campaign.results[0].productJourney.phases[0]).toEqual({phase:"authentication",outcome:"failed",code:"login_code_expired"}); + }); + it("does not turn onboarding failure into preconfigured-state success",async()=>{ + const campaign=await run({onboardingFailure:true}); + expect(campaign.results.map((r:any)=>r.productJourney.outcome)).toEqual(["failed","blocked"]); + expect(campaign.results[0].productJourney.phases.find((p:any)=>p.phase==="journey").outcome).toBe("failed"); + }); + it("reuses a verified onboarding context for the same shared story",async()=>{ + const campaign=await run(); + expect(campaign.results.map((r:any)=>r.status)).toEqual(["passed","passed"]); + expect(campaign.results.map((r:any)=>r.productJourney.startingState)).toEqual(["onboarding","onboarding"]); + expect(campaign.results[1]).toMatchObject({provider:null,model:null,runtimeMode:"none"}); + }); + it("fails deployment drift and blocks descendants",async()=>{ + const campaign=await run({drift:true}); + expect(campaign.results[0].error).toBe("deployment_changed_during_journey"); + expect(campaign.results[1].productJourney.outcome).toBe("blocked"); + }); + it("keeps cleanup failures visible without copying raw errors",async()=>{ + const campaign=await run({cleanupFailure:true}); + expect(campaign.results.every((r:any)=>r.cleanup==="failed"&&r.status==="failed")).toBe(true); + expect(JSON.stringify(campaign)).not.toContain("private raw error"); + }); + it("preserves failed authentication and blocked dependents when finalization also fails",async()=>{ + const campaign=await run({authFailure:true,collectionFailure:true,cleanupFailure:true}); + expect(campaign.results.map((r:any)=>r.productJourney.outcome)).toEqual(["failed","blocked"]); + expect(campaign.results.every((r:any)=>r.status==="failed"&&r.cleanup==="failed")).toBe(true); + expect(campaign.results[1].productJourney.phases).toContainEqual({phase:"collection",outcome:"failed",code:"evidence_collection_failed"}); + }); + it("fails paid success when its provider silently omits accounting evidence",async()=>{ + const campaign=await run({missingReceipt:true}); + expect(campaign.results[0].status).toBe("failed"); + expect(campaign.results[0].error).toBe("paid_journey_receipt_missing"); + expect(campaign.results[0].productJourney.phases).toContainEqual({phase:"collection",outcome:"failed",code:"paid_journey_receipt_missing"}); + }); + it.each(["zeroRequests","missingCheck"] as const)("retains disposable resources when success evidence has %s",async field=>{ + const campaign=await run({[field]:true}); + expect(campaign.results.every((r:any)=>r.status==="failed")).toBe(true); + expect(campaign.lifecycle[0]).toBe("release:failed"); + expect(campaign.results[0].runIds).toEqual(["run-1"]); + expect(campaign.results[0].productJourney.operations.modelCostUsd).toBe(0.01); + expect(campaign.results[0].productJourney.phases).toContainEqual({phase:"collection",outcome:"failed",code:"evidence_collection_failed"}); + }); + it.each(["collect","release","stop","finalizeEvidence","checkpointEvidence"] as const)("bounds a stalled %s hook and still closes the browser",async hook=>{ + vi.useFakeTimers(); + const close=vi.fn(async()=>{}); + vi.spyOn(chromium,"launch").mockResolvedValueOnce({newContext:async()=>({setDefaultTimeout(){},pages:()=>[]}),close} as any); + let notify!:()=>void; + const entered=new Promise(resolve=>{notify=resolve;}); + let signal:AbortSignal|undefined; + const pending=run({stallHook:hook,...(hook==="checkpointEvidence"?{finalizationFailure:true}:{}),...(hook==="stop"?{cleanupFailure:true}:{}),onHook(name,input){if(name===hook){signal=input;notify();}}}); + await entered; + await vi.advanceTimersByTimeAsync(hook==="collect"?60_000:600_000); + const campaign=await pending; + expect(signal?.aborted).toBe(true); + expect(close).toHaveBeenCalledOnce(); + expect(campaign.results.every((r:any)=>r.status==="failed")).toBe(true); + if(hook==="collect")expect(campaign.lifecycle[0]).toBe("release:failed"); + if(hook==="release"){ + expect(campaign.lifecycle).toContain("stop"); + expect(campaign.lifecycle).not.toContain("finalizeEvidence"); + expect(campaign.results.every((r:any)=>r.cleanup==="failed"&&r.productJourney.resourceDisposition==="cleanup-failed")).toBe(true); + } + }); + it("bounds a stalled browser close without skipping final evidence",async()=>{ + vi.useFakeTimers(); + let notify!:()=>void; + const entered=new Promise(resolve=>{notify=resolve;}); + const close=vi.fn(async()=>{notify();await new Promise(()=>{});}); + vi.spyOn(chromium,"launch").mockResolvedValueOnce({newContext:async()=>({setDefaultTimeout(){},pages:()=>[]}),close} as any); + const pending=run(); + await entered; + await vi.advanceTimersByTimeAsync(10_000); + const campaign=await pending; + expect(close).toHaveBeenCalledOnce(); + expect(campaign.results.every((r:any)=>r.cleanup==="failed")).toBe(true); + expect(campaign.results[0].productJourney.phases).toContainEqual({phase:"cleanup",outcome:"failed",code:"browser_cleanup_timeout"}); + expect(campaign.lifecycle).toContain("finalizeEvidence"); + }); + it("rejects a hook that finishes past its deadline before the timeout callback runs",async()=>{ + vi.useFakeTimers(); + let signal:AbortSignal|undefined; + const campaign=await run({onHook(hook,input){if(hook==="collect"){signal=input;vi.setSystemTime(Date.now()+60_000);}}}); + expect(signal?.aborted).toBe(true); + expect(campaign.lifecycle[0]).toBe("release:failed"); + expect(campaign.results[0].productJourney.phases).toContainEqual({phase:"collection",outcome:"failed",code:"evidence_collection_timeout"}); + }); + it("keeps a timed-out renderer's late writes away from retained failure results",async()=>{ + vi.useFakeTimers(); + let entered!:()=>void,release!:()=>void,finished!:()=>void; + const rendering=new Promise(resolve=>{entered=resolve;}); + const continueRendering=new Promise(resolve=>{release=resolve;}); + const lateWrite=new Promise(resolve=>{finished=resolve;}); + vi.mocked(regenerateRunnerDashboard).mockImplementationOnce(async input=>{ + const earlier=await readFile(path.join(input.bundle,"normalized-results.json"),"utf8"); + entered(); + await continueRendering; + await writeFile(path.join(input.outputDirectory??input.bundle,"normalized-results.json"),earlier); + finished(); + }); + const pending=run({onHook(hook){if(hook==="release")vi.advanceTimersByTime(569_000);}}); + await rendering; + await vi.advanceTimersByTimeAsync(1_000); + const campaign=await pending; + expect(campaign.results.every((r:any)=>r.status==="failed")).toBe(true); + release(); + await lateWrite; + const retained=JSON.parse(await readFile(path.join(campaign.outputDir,"normalized-results.json"),"utf8")); + expect(retained.results.every((r:any)=>r.status==="failed")).toBe(true); + expect(retained.results[0].productJourney.phases).toContainEqual({phase:"cleanup",outcome:"failed",code:"report_generation_failed"}); + expect(campaign.lifecycle).not.toContain("finalizeEvidence"); + }); + it("retains an evidence-finalization failure after the cleanup report was written",async()=>{ + const campaign=await run({finalizationFailure:true}); + expect(campaign.results.every((r:any)=>r.status==="failed"&&r.cleanup==="failed")).toBe(true); + expect(campaign.results[0].productJourney.phases).toContainEqual({phase:"cleanup",outcome:"failed",code:"evidence_finalization_failed"}); + expect(campaign.failureCheckpoint.results.every((r:any)=>r.cleanup==="failed")).toBe(true); + expect(JSON.stringify(campaign)).not.toContain("private evidence upload diagnostic"); + }); + it("attempts evidence finalization even when dashboard generation fails",async()=>{ + vi.mocked(regenerateRunnerDashboard).mockRejectedValueOnce(new Error("private render diagnostic")); + const campaign=await run(); + expect(campaign.results.every((r:any)=>r.status==="failed"&&r.cleanup==="failed")).toBe(true); + expect(campaign.results[0].productJourney.phases).toContainEqual({phase:"cleanup",outcome:"failed",code:"report_generation_failed"}); + expect(campaign.results[0].productJourney.phases).not.toContainEqual({phase:"cleanup",outcome:"failed",code:"evidence_finalization_failed"}); + expect(JSON.stringify(campaign)).not.toContain("private render diagnostic"); + }); + it("retains an explicitly selected mini model without adding a model to the browser story",async()=>{ + const campaign=await run({mini:true}); + expect(campaign.results[0]).toMatchObject({profileId:"runner-codex-mini",model:"gpt-5.4-mini"}); + expect(campaign.expected[0]).toBe("product-journeys.qa.runner-codex-mini.daytona.cloud-onboarding"); + expect(campaign.results[1]).toMatchObject({profileId:"browser",model:null,runtimeMode:"none"}); + }); + it.each(["write","rename"] as const)("still stops owned work and closes the browser when evidence %s fails",async kind=>{ + const close=vi.fn(async()=>{}); + vi.spyOn(chromium,"launch").mockResolvedValueOnce({newContext:async()=>({setDefaultTimeout(){},pages:()=>[]}),close} as any); + const errors=vi.spyOn(console,"error").mockImplementation(()=>{}); + const listeners={interrupt:process.listenerCount("SIGINT"),terminate:process.listenerCount("SIGTERM")}; + const campaign=await run({persistenceFailure:kind}); + expect(campaign.lifecycle).toEqual(["release:failed"]); expect(close).toHaveBeenCalledOnce(); + expect(process.listenerCount("SIGINT")).toBe(listeners.interrupt); expect(process.listenerCount("SIGTERM")).toBe(listeners.terminate); + expect(campaign.attemptedReport.results.every((r:any)=>r.status==="failed"&&r.cleanup==="failed")).toBe(true); + expect(campaign.attemptedReport.results[0].productJourney.phases).toContainEqual({phase:"cleanup",outcome:"failed",code:"evidence_persistence_failed"}); + expect(JSON.stringify(campaign)).not.toContain("api_key_fixture"); expect(JSON.stringify(errors.mock.calls)).not.toContain("api_key_fixture"); + expect(errors).toHaveBeenCalledWith("evidence_persistence_failed"); expect(process.exitCode).toBe(1); + }); + it("withholds lease finalization when the report fails to persist after resource cleanup",async()=>{ + const close=vi.fn(async()=>{}); + vi.spyOn(chromium,"launch").mockResolvedValueOnce({newContext:async()=>({setDefaultTimeout(){},pages:()=>[]}),close} as any); + vi.spyOn(console,"error").mockImplementation(()=>{}); + const campaign=await run({persistenceFailure:"rename",persistenceAfter:"release"}); + expect(campaign.lifecycle).toEqual(["release:passed"]); expect(close).toHaveBeenCalledOnce(); + expect(campaign.attemptedReport.results.every((r:any)=>r.cleanup==="failed")).toBe(true); + expect(process.exitCode).toBe(1); + }); + it("preserves failed authentication and blocked dependents through evidence persistence failure",async()=>{ + vi.spyOn(console,"error").mockImplementation(()=>{}); + const campaign=await run({authFailure:true,persistenceFailure:"write"}); + expect(campaign.lifecycle).toEqual(["release:failed"]); + expect(campaign.attemptedReport.results.map((r:any)=>r.productJourney.outcome)).toEqual(["failed","blocked"]); + expect(campaign.attemptedReport.results[0].error).toBe("login_code_expired"); + expect(campaign.attemptedReport.results[1].error).toBe("prerequisite_journey_failed"); + expect(JSON.stringify(campaign.attemptedReport)).not.toContain("api_key_fixture"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts new file mode 100644 index 00000000000..c018e769aab --- /dev/null +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -0,0 +1,44 @@ +import type { JourneyId, TargetConfig } from "./types.js"; +export const productJourneys = [ + { id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false }, + { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, +] as const; + +export function preparationPreview(config: TargetConfig) { + return { + target: { id: config.id, mode: config.mode, executionEnvironment: config.executionEnvironment }, + journeys: config.journeys.map(id => productJourneys.find(journey => journey.id === id)!), + configuration: config.baseline, + lifetime: config.mode === "existing-cloud" ? "persistent; clean run-owned resources only" : config.mode === "new-cloud" ? "disposable; delete on pass, retain failures for 24 hours" : "managed instance; remove after every attempt", + limits: config.limits, authentication: config.mode === "managed-self-hosted" ? "local-trusted" : "normal email-code sign-in", + modelProfileId: config.modelProfileId ?? null, + maxDisposableStacks: config.mode === "new-cloud" ? 1 : 0, + retries: 0, + }; +} + +export function parseTargetConfig(value: unknown): TargetConfig { + const c = value as TargetConfig; + const object = (v: unknown): v is Record => Boolean(v && typeof v === "object" && !Array.isArray(v)); + if (!object(value) || Object.keys(value).some(key => !["version", "id", "mode", "adapterModule", "adapterConfig", "journeys", "executionEnvironment", "baseline", "limits", "headed", "modelProfileId"].includes(key))) throw new Error("invalid_target_configuration_fields"); + if (!c || c.version !== 1 || !/^[a-z][a-z0-9-]{0,63}$/.test(c.id) || + !["managed-self-hosted", "existing-cloud", "new-cloud"].includes(c.mode) || + !["local", "daytona"].includes(c.executionEnvironment) || !Array.isArray(c.journeys) || !c.journeys.length || + new Set(c.journeys).size !== c.journeys.length || c.journeys.some(id => !productJourneys.some(j => j.id === id)) || + !object(c.baseline) || Object.keys(c.baseline).some(k => !["features", "connections"].includes(k)) || !object(c.baseline.features) || Object.entries(c.baseline.features).some(([k, v]) => !/^[a-z][a-zA-Z0-9_.-]{0,80}$/.test(k) || typeof v !== "boolean") || + !Array.isArray(c.baseline.connections) || c.baseline.connections.some(v => typeof v !== "string" || !/^[a-zA-Z0-9_-]{1,128}$/.test(v)) || + !object(c.limits) || Object.keys(c.limits).some(k => !["durationMs", "modelBudgetCents", "maxRuns"].includes(k)) || !Number.isSafeInteger(c.limits.durationMs) || c.limits.durationMs < 1000 || c.limits.durationMs > 3_600_000 || + !Number.isSafeInteger(c.limits.modelBudgetCents) || c.limits.modelBudgetCents < 0 || + !Number.isSafeInteger(c.limits.maxRuns) || c.limits.maxRuns < 0) throw new Error("invalid_target_configuration"); + if (c.headed !== undefined && typeof c.headed !== "boolean") throw new Error("invalid_browser_mode"); + if (c.adapterModule !== undefined && (typeof c.adapterModule !== "string" || !c.adapterModule)) throw new Error("invalid_adapter_module"); + if (c.adapterConfig !== undefined && !object(c.adapterConfig)) throw new Error("invalid_adapter_configuration"); + if (c.modelProfileId !== undefined && !["runner-codex", "runner-codex-mini"].includes(c.modelProfileId)) throw new Error("unsupported_model_profile"); + if (c.mode === "new-cloud" && c.journeys[0] !== "cloud-onboarding") throw new Error("new_cloud_requires_onboarding_first"); + if (c.mode === "new-cloud" && c.baseline.connections.length) throw new Error("onboarding_requires_ui_connection_setup"); + if (c.mode !== "new-cloud" && c.journeys.includes("cloud-onboarding")) throw new Error("onboarding_requires_fresh_cloud_workspace"); + if (c.journeys.includes("cloud-onboarding") && (!c.limits.modelBudgetCents || !c.limits.maxRuns)) throw new Error("paid_journey_requires_limits"); + if (c.mode !== "managed-self-hosted" && !c.adapterModule) throw new Error("cloud_target_requires_private_adapter"); + if (c.mode === "managed-self-hosted" && c.executionEnvironment !== "local") throw new Error("unsupported_self_hosted_environment"); + return { ...c, ...(c.journeys.includes("cloud-onboarding") ? { modelProfileId: c.modelProfileId ?? "runner-codex" as const } : {}), headed: c.headed ?? false, journeys: c.journeys as JourneyId[] }; +} diff --git a/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts b/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts new file mode 100644 index 00000000000..30085b0ab4f --- /dev/null +++ b/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it, vi } from "vitest"; +import { cloudOnboarding, type CloudOnboardingInput } from "./cloud-onboarding.js"; +import { onboardingFirstTask } from "./onboarding-first-task.js"; + +vi.mock("@playwright/test", () => ({ expect: (locator: { isVisible(): Promise }) => ({ + toBeVisible: async () => { expect(await locator.isVisible()).toBe(true); }, + not: { toBeVisible: async () => { expect(await locator.isVisible()).toBe(false); } }, +}) })); +vi.mock("./onboarding-first-task.js", () => ({ onboardingFirstTask: vi.fn(async () => ({ artifact: { originatingRunId: "artifact-run" } })) })); + +type Entry = "first-run" | "portfolio" | "create-form"; +type Control = { role: string; name: string; scope?: string; fill?(value: string): void; click?(): void; select?(value: string): void }; + +/** Visible control contracts only; no browser, Cloud API, or provider is started. */ +function fixture(entry: Entry, options: { intentFailure?: boolean; qualificationFailure?: boolean } = {}) { + let stage: string = entry, name = "", apiMode = false, source = false, advanced = false, runnerOpen = false; + let modelsOpen = false, model = "", runner = "", saved = false, hired = false; + const fields: Record = {}, events: string[] = []; + const gateway = { baseUrl: "https://owned-gateway.example.test/v1", model: "gpt-5.4-mini", apiKey: "synthetic-test-key" }; + const connection = { id: "connection", name: "My owned-gateway.example.test", provider: "openai", method: "api_key", status: "connected" }; + const create = () => { + expect(events).toContain("intent:workspace"); + expect(name).toBe("Journey QA campaign"); + events.push("create:workspace"); stage = "agent"; + }; + const controls = (): Control[] => { + if (stage === "portfolio") return [{ role: "link", name: "Create a new organization", click() { stage = "create-form"; } }]; + if (stage === "first-run" || stage === "create-form") return [ + { role: "textbox", name: stage === "first-run" ? "Name" : "Organization name", fill(value) { name = value; } }, + { role: "button", name: stage === "first-run" ? "Next" : "Create organization", click: create }, + ]; + if (stage === "agent") return [ + { role: "heading", name: "Create your first agent" }, + { role: "textbox", name: "Agent name", fill(value) { name = value; } }, + { role: "button", name: "Next", click() { stage = "connect"; } }, + ]; + if (stage === "started") return []; + if (hired) return [{ role: "button", name: "Get started", click() { stage = "started"; } }]; + const result: Control[] = [ + { role: "button", name: apiMode ? "Use subscription instead" : "Use API key instead", click() { apiMode = !apiMode; } }, + { role: "radio", name: apiMode ? "OpenAI API key" : "OpenAI Subscription", click() { source = true; } }, + ]; + if (source && apiMode) result.push({ role: "button", name: "Use a custom gateway", click() { stage = "gateway"; } }); + if (stage === "gateway") { + result.push({ role: "dialog", name: "Custom gateway connection" }); + for (const field of ["Provider URL", "API key", ...(modelsOpen ? ["Model IDs"] : [])]) result.push({ + role: "textbox", name: field, scope: "gateway", fill(value) { fields[field] = value; }, + }); + result.push({ role: "button", name: "Change", scope: "gateway", click() { modelsOpen = true; } }, + { role: "button", name: "Save connection", scope: "gateway", click() { + expect(fields).toEqual({ "Provider URL": gateway.baseUrl, "API key": gateway.apiKey, "Model IDs": gateway.model }); + expect(events).toContain("intent:connection"); events.push("save:connection"); saved = true; stage = "connect"; + } }); + } + if (saved) result.push({ role: "combobox", name: "Model", select(value) { expect(value).toBe(gateway.model); model = value; } }); + if (source) result.push({ role: "summary", name: "Advanced", click() { advanced = true; } }); + if (advanced) result.push({ role: "button", name: "Runner", click() { runnerOpen = true; } }); + if (runnerOpen) result.push({ role: "option", name: "Paperclip Runner (default)", click() { runner = "paperclip"; runnerOpen = false; } }); + if (saved && model && runner) result.push({ role: "button", name: "Connect", click() { + expect(events).toContain("qualified"); events.push("paid:connect"); hired = true; + } }); + return result; + }; + const match = (value: string, expected: string | RegExp) => typeof expected === "string" ? value === expected : expected.test(value); + const locator = (resolve: () => Control[]): any => ({ + first() { return locator(() => resolve().slice(0, 1)); }, + or(other: { resolve(): Control[] }) { return locator(() => [...resolve(), ...other.resolve()]); }, + resolve, + async isVisible() { return resolve().length > 0; }, + async fill(value: string) { const control = resolve()[0]; if (!control?.fill) throw new Error("No visible text field"); control.fill(value); }, + async click() { const control = resolve()[0]; if (!control?.click) throw new Error("No visible action"); events.push(`click:${control.name}`); control.click(); }, + async selectOption(value: string) { const control = resolve()[0]; if (!control?.select) throw new Error("No visible native model picker"); control.select(value); }, + async waitFor() { expect(resolve().length).toBeGreaterThan(0); }, + filter({ hasText }: { hasText: string }) { return locator(() => resolve().filter(control => control.name.includes(hasText))); }, + getByLabel(label: string) { return locator(() => controls().filter(control => control.scope === "gateway" && control.name === label)); }, + getByRole(role: string, { name: label }: { name: string | RegExp }) { + return locator(() => controls().filter(control => control.scope === "gateway" && control.role === role && match(control.name, label))); + }, + }); + const page = { + getByLabel(label: string) { return locator(() => controls().filter(control => control.name === label && control.role === "textbox")); }, + getByRole(role: string, { name: label }: { name: string | RegExp }) { + return locator(() => controls().filter(control => control.role === role && match(control.name, label))); + }, + locator(selector: string) { expect(selector).toBe("summary"); return locator(() => controls().filter(control => control.role === "summary")); }, + }; + const context = { page, company: { id: "company" }, identity: { kind: "cloud" }, startingState: "recipe", api: { + async get(route: string) { + if (route.endsWith("/ai-connections")) return { connections: saved ? [connection] : [] }; + if (route.endsWith("/agents")) return hired ? [{ id: "agent", name, adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: gateway.model } }] : []; + throw new Error("Unexpected API read"); + }, + } }; + const input = { page, campaignId: "campaign", deadlineAt: Date.now() + 60_000, gateway, maxRuns: 8, evidence: { checks: [] }, + async beforeWorkspace() { if (options.intentFailure) throw new Error("ownership unavailable"); events.push("intent:workspace"); }, + async workspaceCreated() { expect(stage).toBe("agent"); return context; }, + async own(resource: { kind: string }) { events.push(`intent:${resource.kind}`); }, + async phase() {}, async observeRun() {}, + async qualify() { if (options.qualificationFailure) throw new Error("admission denied"); events.push("qualified"); }, + async verifyExecution(_context: unknown, runIds: string[]) { expect(runIds).toEqual(["artifact-run"]); events.push("native:verified"); }, + } as unknown as CloudOnboardingInput; + return { input, events }; +} + +describe("Cloud onboarding visible controls", () => { + it.each(["first-run", "portfolio", "create-form"] as const)("creates from %s with intent before dispatch and qualification before paid Connect", async entry => { + const f = fixture(entry); + const result = await cloudOnboarding(f.input); + expect(result.startingState).toBe("onboarding"); + expect(f.events.indexOf("intent:workspace")).toBeLessThan(f.events.indexOf("create:workspace")); + expect(f.events.indexOf("intent:connection")).toBeLessThan(f.events.indexOf("save:connection")); + expect(f.events.indexOf("qualified")).toBeLessThan(f.events.indexOf("paid:connect")); + expect(f.events).toContain("native:verified"); + }); + it("does not dispatch workspace creation without durable ownership", async () => { + const f = fixture("first-run", { intentFailure: true }); + await expect(cloudOnboarding(f.input)).rejects.toThrow("ownership unavailable"); + expect(f.events).not.toContain("create:workspace"); + }); + it("never presses paid Connect when route admission fails", async () => { + const f = fixture("portfolio", { qualificationFailure: true }); + const previous = vi.mocked(onboardingFirstTask).mock.calls.length; + await expect(cloudOnboarding(f.input)).rejects.toThrow("admission denied"); + expect(f.events).not.toContain("paid:connect"); + expect(vi.mocked(onboardingFirstTask).mock.calls).toHaveLength(previous); + }); +}); diff --git a/tests/runner-e2e/product-journeys/cloud-onboarding.ts b/tests/runner-e2e/product-journeys/cloud-onboarding.ts new file mode 100644 index 00000000000..1d5f2d4bbdd --- /dev/null +++ b/tests/runner-e2e/product-journeys/cloud-onboarding.ts @@ -0,0 +1,89 @@ +import { expect } from "@playwright/test"; +import { createConnectionProof } from "../connection-evidence.js"; +import { onboardingFirstTask } from "./onboarding-first-task.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +export interface CloudOnboardingInput { + page: JourneyContext["page"]; + campaignId: string; + deadlineAt: number; + gateway: { model: string; baseUrl: string; apiKey: string }; + maxRuns: number; + evidence: JourneyEvidence; + own: TargetSession["own"]; + beforeWorkspace(name: string): Promise; + workspaceCreated(): Promise; + /** The saved route must pass a deny-only runtime probe before Connect can spend. */ + qualify(context: JourneyContext, connectionId: string): Promise; + verifyExecution(context: JourneyContext, runIds: string[]): Promise; + phase(phase: "setup" | "agent"): Promise; + observeRun(run: Row): Promise; +} +/** Measured actions use normal Cloud and tenant controls; infrastructure stays in the provider. */ +export async function cloudOnboarding(input: CloudOnboardingInput): Promise { + const { page, evidence, campaignId } = input; + const name = `Journey QA ${campaignId}`; + const firstRunName = page.getByLabel("Name", { exact: true }); + const organizationName = page.getByLabel("Organization name", { exact: true }); + const createLink = page.getByRole("link", { name: /^(Create (?:a new )?organization|\+ New organization)$/ }).first(); + // The first organization uses the signup Name/Next form. Later organizations + // use Organization name/Create organization, reached from the managed portfolio. + await expect(firstRunName.or(organizationName).or(createLink).first()).toBeVisible(); + if (!await firstRunName.isVisible() && !await organizationName.isVisible()) { + await createLink.click(); + await expect(organizationName).toBeVisible(); + } + const firstRun = await firstRunName.isVisible(); + await (firstRun ? firstRunName : organizationName).fill(name); + await input.beforeWorkspace(name); + await page.getByRole("button", { name: firstRun ? "Next" : "Create organization", exact: true }).click(); + const context = await input.workspaceCreated(); + const { api, company } = context; + evidence.deployment = context.identity; + evidence.checks.push({ id: "fresh-workspace", passed: true }); + await input.phase("setup"); + await expect(page.getByRole("heading", { name: "Create your first agent", exact: true })).toBeVisible({ timeout: 180_000 }); + await input.own({ kind: "agent", name }); + await page.getByLabel("Agent name", { exact: true }).fill(name); + await page.getByRole("button", { name: "Next", exact: true }).click(); + await page.getByRole("button", { name: "Use API key instead", exact: true }).click(); + await page.getByRole("radio", { name: /OpenAI.*API/ }).click(); + await page.getByRole("button", { name: "Use a custom gateway", exact: true }).click(); + const dialog = page.getByRole("dialog", { name: "Custom gateway connection", exact: true }); + await dialog.getByLabel("Provider URL", { exact: true }).fill(input.gateway.baseUrl); + await dialog.getByLabel("API key", { exact: true }).fill(input.gateway.apiKey); + await dialog.getByRole("button", { name: "Change", exact: true }).click(); + await dialog.getByLabel("Model IDs", { exact: true }).fill(input.gateway.model); + const connectionName = `My ${new URL(input.gateway.baseUrl).hostname}`; + await input.own({ kind: "connection", name: connectionName }); + await dialog.getByRole("button", { name: "Save connection", exact: true }).click(); + await expect(dialog).not.toBeVisible(); + const connections = (await api.get(`/api/companies/${company.id}/ai-connections`)).connections as Row[]; + const connection = connections.find(c => c.name === connectionName && c.provider === "openai" && c.method === "api_key" && c.status === "connected"); + if (!connection || connections.length !== 1) throw new JourneyError("failed", "onboarding_connection_not_saved"); + await input.own({ kind: "connection", name: connectionName, id: connection.id }); + await page.getByRole("combobox", { name: "Model", exact: true }).selectOption(input.gateway.model); + await page.locator("summary").filter({ hasText: "Advanced" }).click(); + await page.getByRole("button", { name: "Runner", exact: true }).click(); + await page.getByRole("option", { name: /^Paperclip Runner/ }).click(); + await input.qualify(context, connection.id); + // Connect performs the production probe and agent creation. Never press Retry. + await page.getByRole("button", { name: "Connect", exact: true }).click(); + await page.getByRole("button", { name: "Get started", exact: true }).waitFor({ timeout: 180_000 }); + const agents = await api.get(`/api/companies/${company.id}/agents`); + const agent = agents.find(a => a.name === name); + if (!agent || agents.length !== 1 || agent.adapterType !== "paperclip_runner" || agent.adapterConfig?.provider !== "codex" || agent.adapterConfig?.model !== input.gateway.model) throw new JourneyError("failed", "onboarding_agent_configuration_mismatch"); + await input.own({ kind: "agent", name, id: agent.id }); + evidence.checks.push({ id: "ui-connection-and-native-agent", passed: true }); + await input.own({ kind: "issue", name: "Paperclip onboarding" }); + await page.getByRole("button", { name: "Get started", exact: true }).click(); + await input.phase("agent"); + const completed = await onboardingFirstTask({ context, agent: { id: agent.id }, connectionId: connection.id, + proof: createConnectionProof(), campaignId, maxRuns: input.maxRuns, deadlineAt: input.deadlineAt, + own: input.own, observeRun: input.observeRun }); + await input.verifyExecution(context, [completed.artifact.originatingRunId]); + evidence.checks.push({ id: "first-task-complete", passed: true }, + { id: "downloadable-artifact-verified", passed: true }, { id: "native-trace-verified", passed: true }); + return { ...context, startingState: "onboarding" }; +} diff --git a/tests/runner-e2e/product-journeys/collection.ts b/tests/runner-e2e/product-journeys/collection.ts new file mode 100644 index 00000000000..2eb4ca28095 --- /dev/null +++ b/tests/runner-e2e/product-journeys/collection.ts @@ -0,0 +1,20 @@ +import { JourneyError, type JourneyCollection } from "./types.js"; + +/** Reject accidental raw run records or credentials at the private-provider boundary. */ +export function validateJourneyCollection(receipt: JourneyCollection) { + const closed = (value: unknown, keys: string[]) => { + if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(k => !keys.includes(k))) throw new JourneyError("failed", "invalid_collection_receipt"); + }; + const id = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,255}$/.test(value); + closed(receipt, ["journeyId", "runIds", "usage", "operations"]); + if (!Array.isArray(receipt.runIds) || receipt.runIds.some(value => !id(value)) || new Set(receipt.runIds).size !== receipt.runIds.length) throw new JourneyError("failed", "invalid_collection_runs"); + closed(receipt.usage, ["runs"]); + if (!Array.isArray(receipt.usage.runs) || receipt.usage.runs.length !== receipt.runIds.length) throw new JourneyError("failed", "invalid_collection_usage"); + for (const [index, run] of receipt.usage.runs.entries()) { + closed(run, ["runId", "usage"]); + if (run.runId !== receipt.runIds[index]) throw new JourneyError("failed", "invalid_collection_usage"); + closed(run.usage, ["inputTokens", "outputTokens", "cachedInputTokens", "costUsd"]); + if (Object.values(run.usage).some(value => typeof value !== "number" || !Number.isFinite(value) || value < 0)) throw new JourneyError("failed", "invalid_collection_usage"); + } + return receipt; +} diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts new file mode 100644 index 00000000000..82762c533fc --- /dev/null +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -0,0 +1,183 @@ +import { describe, it, expect, vi } from "vitest"; +import { FixtureRegistry } from "../fixture-registry.js"; +import { parseTargetConfig, preparationPreview, productJourneys } from "./catalog.js"; +import { journeyFixtures } from "./fixtures.js"; +import { taskProperties } from "./task-properties.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import { assertPublicCampaign } from "../history-publish.js"; +import { validateJourneyCollection } from "./collection.js"; +import { summarizeExecutionBilling, aggregateCampaignBilling } from "../billing.js"; +import { discoverReportCatalog } from "../report-catalog.js"; +import { isIncompleteFirstTaskResult, renderCaseOutcome } from "../case-outcome.js"; +import type { RunnerE2EResult } from "../types.js"; +import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; +vi.mock("@playwright/test", () => ({ + expect: Object.assign( + (locator: { textContent(): Promise }) => ({ async toContainText(value: string) { expect(await locator.textContent()).toContain(value); } }), + { poll: (read: () => Promise) => ({ async toBe(value: unknown) { expect(await read()).toBe(value); } }) }, + ), +})); +const local = { version: 1, id: "local-qa", mode: "managed-self-hosted", journeys: ["task-properties"], executionEnvironment: "local", baseline: { features: {}, connections: [] }, limits: { durationMs: 10000, modelBudgetCents: 0, maxRuns: 0 } }; +function preparedContext(startingState: JourneyContext["startingState"] = "recipe") { + const api = { origin: "https://qa.example.test", post: vi.fn(async () => ({ id: "task-1", identifier: "QA-1" })) }; + const context = { capabilities: ["company", "board-user", "task-write"], company: { id: "company-1", issuePrefix: "QA" }, user: { id: "user-1", name: "QA" }, api, startingState } as unknown as JourneyContext; + return { context, api }; +} +function result(): RunnerE2EResult { + return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "test", executionId: "product-journeys.local-qa.local.task-properties", profileId: "local-qa", environmentId: "local", caseId: "task-properties", provider: null, model: null, runtimeMode: "none", status: "passed", cleanup: "passed", attempt: 1, startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, + productJourney: { outcome: "passed", startingState: "recipe", phases: [{phase: "journey", outcome: "passed"}], checks: [{id:"assignment-persisted", passed:true},{id:"no-agent-work",passed:true}], privacy: "private", resourceDisposition: "released", deployment: { kind: "self-hosted", targetId: "local-qa", origin: "http://127.0.0.1:3100", commit: "a".repeat(40), cloudRelease: null, runtimeImage: null, configurationHash: "test" } } }; +} +describe("deployment journey contracts", () => { + it("resolves only requested prerequisites, once, and unwinds on failure", async () => { + const calls: string[] = []; + const registry = new FixtureRegistry().register({id:"a", async setup(){calls.push("a"); return 1;}, async teardown(){calls.push("-a");}}) + .register({id:"b",dependencies:["a"],async setup(){calls.push("b");throw new Error("setup");}}) + .register({id:"unselected",async setup(){throw new Error("unselected");}}); + await expect(registry.setupAll(["b"])).rejects.toThrow("setup"); + expect(calls).toEqual(["a","b","-a"]); + }); + it("reuses verified onboarding without claiming recipe setup is onboarding coverage", async () => { + const ready = { capabilities:["company","board-user","task-write"], startingState:"onboarding" } as JourneyContext; + const prepare = vi.fn(async () => ({...ready, startingState:"recipe"})); + const fixtures = journeyFixtures({prepare} as unknown as TargetSession); + fixtures.provide(ready); + expect(await fixtures.resolve(["company","board-user"])).toBe(ready); + expect(prepare).not.toHaveBeenCalled(); + const independent = journeyFixtures({prepare} as unknown as TargetSession); + expect((await independent.resolve(["task-write"])).startingState).toBe("recipe"); + expect(prepare).toHaveBeenCalledTimes(1); + }); + it.each(["recipe", "onboarding"] as const)("prepares one owned backlog task from %s context before the journey", async startingState => { + const { context, api } = preparedContext(startingState); + const calls: string[] = []; + api.post.mockImplementation(async () => { calls.push("create"); return { id: "task-1", identifier: "QA-1" }; }); + const prepare = vi.fn(async () => { calls.push("prepare"); return context; }); + const own = vi.fn(async (resource: { id?: string }) => { calls.push(resource.id ? "receipt" : "intent"); }); + const fixtures = journeyFixtures({ prepare, own } as unknown as TargetSession, "campaign-1"); + if (startingState === "onboarding") fixtures.provide(context); + const required = productJourneys.find(journey => journey.id === "task-properties")!.requires; + for (const resolved of await Promise.all([fixtures.resolve(required), fixtures.resolve(required)])) expect(resolved).toBe(context); + expect(calls).toEqual([...(startingState === "recipe" ? ["prepare"] : []), "intent", "create", "receipt"]); + expect(api.post).toHaveBeenCalledExactlyOnceWith("/api/companies/company-1/issues", { title: "Journey QA campaign-1", description: "Disposable task-property verification", status: "backlog" }); + expect(own.mock.calls).toEqual([[{ kind: "issue", name: "Journey QA campaign-1" }], [{ kind: "issue", name: "Journey QA campaign-1", id: "task-1" }]]); + expect(context.fixtures?.backlogTask).toEqual({ id: "task-1", identifier: "QA-1", title: "Journey QA campaign-1" }); + expect(context.startingState).toBe(startingState); + expect(prepare).toHaveBeenCalledTimes(startingState === "recipe" ? 1 : 0); + }); + it("requires a campaign ID only when a run-owned task is requested", async () => { + const { context, api } = preparedContext(); + const prepare = vi.fn(async () => context); + const own = vi.fn(); + const fixtures = journeyFixtures({ prepare, own } as unknown as TargetSession); + await expect(fixtures.resolve(["backlog-task"])).rejects.toMatchObject({ outcome: "blocked", code: "campaign_id_required_for_backlog_task" }); + expect(prepare).not.toHaveBeenCalled(); + expect(api.post).not.toHaveBeenCalled(); + expect(own).not.toHaveBeenCalled(); + expect(await fixtures.resolve(["company"])).toBe(context); + }); + it.each(["intent", "receipt"])("does not redispatch task creation after an uncertain %s failure", async failure => { + const { context, api } = preparedContext(); + const own = vi.fn(async (resource: { id?: string }) => { if (Boolean(resource.id) === (failure === "receipt")) throw Error("ownership write failed"); }); + const fixtures = journeyFixtures({ prepare: async () => context, own } as unknown as TargetSession, "campaign-1"); + await expect(fixtures.resolve(["backlog-task"])).rejects.toThrow("ownership write failed"); + await expect(fixtures.resolve(["backlog-task"])).rejects.toThrow("ownership write failed"); + expect(api.post).toHaveBeenCalledTimes(failure === "receipt" ? 1 : 0); + expect(own).toHaveBeenCalledTimes(failure === "receipt" ? 2 : 1); + }); + it("uses the supplied task for UI assignment without creating another task", async () => { + const { context, api } = preparedContext(); + const session = { prepare: async () => context, own: vi.fn(async () => {}) } as unknown as TargetSession; + const fixtures = journeyFixtures(session, "campaign-1"); + await fixtures.resolve(["backlog-task"]); + api.post.mockImplementation(async () => { throw Error("task creation inside measured story"); }); + let assigned = false; + const locator = { filter: vi.fn(), first: vi.fn(), locator: vi.fn(), getByRole: vi.fn(), click: vi.fn(), isVisible: vi.fn(async () => false), textContent: vi.fn(async () => assigned ? "You" : "Unassigned") }; + for (const method of [locator.filter, locator.first, locator.locator, locator.getByRole]) method.mockReturnValue(locator); + const assign = { click: vi.fn(async () => { assigned = true; }) }; + const page = { goto: vi.fn(), reload: vi.fn(), locator: vi.fn(() => locator), getByRole: vi.fn((_role: string, options?: { name?: unknown }) => options?.name === "Assign to me" ? assign : locator) }; + const get = vi.fn(async (route: string) => route.endsWith("/runs") ? [] : { assigneeUserId: assigned ? context.user.id : null, assigneeAgentId: null, status: "backlog" }); + Object.assign(context.api, { get }); + context.page = page as unknown as JourneyContext["page"]; + const evidence = { checks: [] } as unknown as JourneyEvidence; + await taskProperties(context, session, "campaign-1", evidence); + expect(api.post).toHaveBeenCalledTimes(1); + expect(session.own).toHaveBeenCalledTimes(2); + expect(page.goto).toHaveBeenCalledExactlyOnceWith("https://qa.example.test/QA/issues"); + expect(locator.filter).toHaveBeenCalledWith({ hasText: "Journey QA campaign-1" }); + expect(assign.click).toHaveBeenCalledOnce(); + expect(page.reload).toHaveBeenCalledOnce(); + expect(get.mock.calls.map(([route]) => route)).toEqual(["/api/issues/task-1", "/api/issues/task-1", "/api/issues/task-1/runs"]); + expect(evidence.checks).toEqual([{ id: "assignment-persisted", passed: true }, { id: "no-agent-work", passed: true }]); + }); + it("blocks a story without the prepared task before using browser or API", async () => { + const { context, api } = preparedContext(); + const own = vi.fn(); + await expect(taskProperties(context, { own } as unknown as TargetSession, "campaign-1", { checks: [] } as unknown as JourneyEvidence)).rejects.toMatchObject({ outcome: "blocked", code: "missing_prepared_backlog_task" }); + expect(api.post).not.toHaveBeenCalled(); + expect(own).not.toHaveBeenCalled(); + }); + it("rejects missing capabilities and cycles", async () => { + const { context, api } = preparedContext(); + context.capabilities = []; + const fixtures = journeyFixtures({ prepare: async () => context } as unknown as TargetSession, "campaign-1"); + await expect(fixtures.resolve(["backlog-task"])).rejects.toMatchObject({outcome:"unsupported"}); + expect(api.post).not.toHaveBeenCalled(); + const graph = new FixtureRegistry().register({id:"a",dependencies:["b"],setup:async()=>null}).register({id:"b",dependencies:["a"],setup:async()=>null}); + await expect(graph.setupAll(["a"])).rejects.toThrow("cycle"); + }); + it("requires explicit paid limits and compatible deployment selections", () => { + expect(preparationPreview(parseTargetConfig(local)).modelProfileId).toBeNull(); + const fresh = {...local,mode:"new-cloud",adapterModule:"/private/provider.mjs",journeys:["cloud-onboarding","task-properties"]}; + expect(()=>parseTargetConfig(fresh)).toThrow("paid_journey_requires_limits"); + expect(()=>parseTargetConfig({...fresh,limits:{...local.limits,modelBudgetCents:1000,maxRuns:1}})).not.toThrow(); + expect(()=>parseTargetConfig({...local,executionEnvironment:"daytona"})).toThrow("unsupported"); + expect(()=>parseTargetConfig({...local,journeys:["cloud-onboarding"]})).toThrow("onboarding_requires"); + expect(()=>parseTargetConfig({...local,apiKey:"secret"})).toThrow("fields"); + }); + it("preview never loads or emits private adapter configuration", () => { + const config = parseTargetConfig({...local,mode:"existing-cloud",adapterModule:"/not-installed.mjs",adapterConfig:{operatorSecret:"never-print"}}); + expect(JSON.stringify(preparationPreview(config))).not.toContain("never-print"); + }); + it("counts shared setup once and rejects raw data in collection receipts", () => { + const onboarding = result(); onboarding.caseId="cloud-onboarding"; onboarding.runtimeMode="native"; + onboarding.runIds=["run-1"]; onboarding.usage={runs:[{runId:"run-1",usage:{costUsd:0.5,inputTokens:10,outputTokens:10}}]}; + onboarding.productJourney!.operations={campaignId:"campaign",attemptId:"attempt",modelCostUsd:0.7,setupCostUsd:0.2,requestCount:2,feedback:{collection:"complete",complaints:0,suggestions:1},traceIds:[]}; + expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7); + expect(()=>validateJourneyCollection({journeyId:"cloud-onboarding",runIds:["run"],usage:{runs:[{runId:"run",usage:{apiKey:"not-retainable"} as any}]}})).toThrow("invalid_collection_receipt"); + onboarding.productJourney!.operations.modelCostUsd=null; + expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull(); + }); + it("requires real pass evidence and forbids model work in browser-only passes", () => { + expect(()=>validateRetainedRunnerResult(result())).not.toThrow(); + const missing = result(); missing.productJourney!.checks=[]; + expect(()=>validateRetainedRunnerResult(missing)).toThrow(); + const paid = result(); paid.runIds=["unexpected"]; + expect(()=>validateRetainedRunnerResult(paid)).toThrow(); + }); + it("keeps browser-only reporting free of invented model calls and rejects public publication", () => { + const value=result(); + expect(summarizeExecutionBilling(value)).toMatchObject({llm:{runCount:0,reportedCostUsd:0},runtime:{agentRunDurationMs:0}}); + expect(discoverReportCatalog({expected:[value.executionId],results:[value],catalog:[]})[0]?.profile).toMatchObject({provider:null,model:null,generation:"none"}); + expect(()=>assertPublicCampaign({results:[value]})).toThrow("Private"); + value.status="failed"; value.productJourney!.outcome="blocked"; + value.productJourney!.checks=[]; + expect(summarizeExecutionBilling(value).llm.runCount).toBe(0); + expect(isIncompleteFirstTaskResult(value)).toBe(true); + value.productJourney!.outcome="failed"; + expect(isIncompleteFirstTaskResult(value)).toBe(false); + }); + it("renders deployment provenance and distinct unsupported outcomes without exposing unknown fields", () => { + const value=result(); value.status="failed"; + value.productJourney!.outcome="unsupported"; + value.productJourney!.campaignId="campaign-1"; value.productJourney!.attemptId="attempt-1"; + Object.assign(value.productJourney!.deployment!, { cloudRelease:"release-1", runtimeImage:"image-1", configurationHash:"config-1", effectiveFeatures:{runner:true}, executionEnvironment:{id:"environment-1",driver:"sandbox",status:"active"} }); + Object.assign(value.productJourney!, { credential:"must-not-render" }); + const output=renderCaseOutcome(value,true,[]); + for (const text of ["unsupported", "Deployment and setup evidence", "campaign-1", "attempt-1", "release-1", "image-1", "config-1", "starting state: recipe", "Effective features", "environment-1", "sandbox"]) expect(output).toContain(text); + expect(output).not.toContain("must-not-render"); + value.productJourney!.deployment!.runtimeImage=""; + expect(renderCaseOutcome(value,true,[])).toContain("<script>"); + value.productJourney!.deployment!.effectiveFeatures={runner:"not-a-boolean"} as any; + expect(()=>validateRetainedRunnerResult(value)).toThrow(); + }); +}); diff --git a/tests/runner-e2e/product-journeys/fixtures.ts b/tests/runner-e2e/product-journeys/fixtures.ts new file mode 100644 index 00000000000..8a13eee0c02 --- /dev/null +++ b/tests/runner-e2e/product-journeys/fixtures.ts @@ -0,0 +1,52 @@ +import { FixtureRegistry } from "../fixture-registry.js"; +import { JourneyError, type JourneyContext, type PreparedBacklogTask, type TargetSession } from "./types.js"; + +/** One registry per campaign: an onboarding receipt may satisfy later prerequisites. */ +export function journeyFixtures(session: TargetSession, campaignId?: string) { + let ready: JourneyContext | undefined; + let preparation: Promise | undefined; + let backlogTask: Promise | undefined; + let taskCompanyId: string | undefined; + const registry = new FixtureRegistry().register({ + id: "ready-context", + async setup() { return ready ??= await (preparation ??= session.prepare()); }, + }); + for (const id of ["company", "board-user", "task-write"]) registry.register({ + id, dependencies: ["ready-context"], + async setup(resolved) { + const context = resolved.get("ready-context") as JourneyContext; + if (!context.capabilities.includes(id)) throw new JourneyError("unsupported", `missing_${id}`); + return context; + }, + }); + registry.register({ + id: "backlog-task", dependencies: ["company", "board-user", "task-write"], + async setup(resolved) { + if (!campaignId?.trim()) throw new JourneyError("blocked", "campaign_id_required_for_backlog_task"); + const context = resolved.get("ready-context") as JourneyContext; + if (backlogTask && taskCompanyId !== context.company.id) throw new JourneyError("failed", "backlog_task_company_changed"); + taskCompanyId = context.company.id; + backlogTask ??= (async () => { + const title = `Journey QA ${campaignId}`; + await session.own({ kind: "issue", name: title }); + const issue = await context.api.post<{ id: string; identifier: string }>(`/api/companies/${context.company.id}/issues`, { + title, description: "Disposable task-property verification", status: "backlog", + }); + if (typeof issue?.id !== "string" || !issue.id) throw new JourneyError("failed", "invalid_backlog_task_receipt"); + await session.own({ kind: "issue", name: title, id: issue.id }); + if (typeof issue.identifier !== "string" || !issue.identifier) throw new JourneyError("failed", "invalid_backlog_task_receipt"); + return { id: issue.id, title, identifier: issue.identifier }; + })(); + context.fixtures = { ...context.fixtures, backlogTask: await backlogTask }; + return context.fixtures.backlogTask; + }, + }); + return { + provide(context: JourneyContext) { ready = context; }, + async resolve(required: readonly string[]) { + if (required.includes("backlog-task") && !campaignId?.trim()) throw new JourneyError("blocked", "campaign_id_required_for_backlog_task"); + const fixtures = await registry.setupAll(required); + return fixtures.values.get("ready-context") as JourneyContext; + }, + }; +} diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts new file mode 100644 index 00000000000..26d8817c0f1 --- /dev/null +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -0,0 +1,282 @@ +import { execFileSync } from "node:child_process"; +import { randomUUID, createHash } from "node:crypto"; +import { mkdir, readFile, writeFile, rename } from "node:fs/promises"; +import { renameSync } from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { chromium, type Browser, type BrowserContext } from "@playwright/test"; +import { runnerSuites } from "../catalog.js"; +import { productJourneys, parseTargetConfig, preparationPreview } from "./catalog.js"; +import { journeyFixtures } from "./fixtures.js"; +import { taskProperties } from "./task-properties.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; +import type { RunnerE2EResult } from "../types.js"; +import { regenerateRunnerDashboard } from "../dashboard-regenerate.js"; +import { resolveRunnerE2ESource } from "../source.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; +import { validateJourneyCollection } from "./collection.js"; + +const cleanupGraceMs = 600_000; +const stopReserveMs = 20_000; +const browserCloseReserveMs = 10_000; + +/** Providers must fence late mutations when this signal is aborted. */ +async function withinLifecycleDeadline(deadlineAt: number, code: string, operation: (input: TargetLifecycleInput) => Promise, parentSignal?: AbortSignal): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + let cancel: () => void = () => {}; + const expiration = new Promise((_resolve, reject) => { + cancel = () => { controller.abort(); reject(new JourneyError("failed", code)); }; + if (deadlineAt <= Date.now() || parentSignal?.aborted) cancel(); + else { + timer = setTimeout(cancel, deadlineAt - Date.now()); + parentSignal?.addEventListener("abort", cancel, { once: true }); + } + }); + try { + const value = await Promise.race([expiration, Promise.resolve().then(() => { + if (controller.signal.aborted) throw new JourneyError("failed", code); + return operation({ signal: controller.signal, deadlineAt }); + })]); + if (controller.signal.aborted || parentSignal?.aborted || Date.now() >= deadlineAt) { + controller.abort(); + throw new JourneyError("failed", code); + } + return value; + } finally { + clearTimeout(timer); + parentSignal?.removeEventListener("abort", cancel); + } +} + +export function journeyFailure(error: unknown): { outcome: Exclude; code: string } { + if (error instanceof JourneyError) return error; + const candidate = error as { outcome?: string; code?: string } | null; + if (candidate && ["failed", "blocked", "unsupported"].includes(candidate.outcome ?? "") && /^[a-z][a-z0-9_-]{0,100}$/.test(candidate.code ?? "")) { + return { outcome: candidate.outcome as Exclude, code: candidate.code! }; + } + // Browser errors may contain keys, codes, URLs and form values. Never retain them. + return { outcome: "failed", code: "journey_operation_failed" }; +} + +export async function launchProductJourneys(args: string[], repositoryRoot: string) { + args = args.filter(arg => arg !== "--"); + if (args.includes("--list") && !args.includes("--target-config")) { console.log(JSON.stringify(productJourneys, null, 2)); return; } + const allowed = new Set(["--target-config", "--output", "--preview", "--list", "--suite"]); + const values = new Map(); + for (let i = 0; i < args.length; i++) { + const flag = args[i]!; + if (!allowed.has(flag)) throw new Error("unknown_product_journey_option"); + if (!["--preview", "--list"].includes(flag)) { + const value = args[++i]; if (!value || value.startsWith("--") || values.has(flag)) throw new Error("invalid_product_journey_option"); + values.set(flag, value); + } + } + const file = values.get("--target-config"); + if (!file) throw new Error("target_config_required"); + const config = parseTargetConfig(JSON.parse(await readFile(file, "utf8"))); + if (args.includes("--preview") || args.includes("--list")) { console.log(JSON.stringify(preparationPreview(config), null, 2)); return; } + const campaignId = `journey-${Date.now()}-${randomUUID().slice(0, 8)}`; + const outputDir = path.resolve(values.get("--output") ?? path.join(repositoryRoot, "tests/runner-e2e/results", campaignId)); + // A campaign directory is exclusive; attempts are never overwritten. + await mkdir(path.dirname(outputDir), { recursive: true, mode: 0o700 }); + await mkdir(outputDir, { recursive: false, mode: 0o700 }); + const deadlineAt = Date.now() + config.limits.durationMs; + const controller = new AbortController(); + let browser: Browser | undefined; + let context: BrowserContext | undefined; + const abort = () => { controller.abort(); for (const page of context?.pages() ?? []) void page.close().catch(() => {}); }; + let session: TargetSession | undefined; + let ready: JourneyContext | undefined; + let fixtures: ReturnType | undefined; + let prerequisiteFailure: ReturnType | undefined; + let releaseSettled = false; + let cleanupDeadlineAt = 0; + let cleanupWorkDeadlineAt = 0; + const results: RunnerE2EResult[] = []; + let sha: string | null = null; + try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } + const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); + const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); + for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-first-task.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + const suiteDefinitionHash = definitionHash.digest("hex"); + const expected = config.journeys.map(id => `product-journeys.${config.id}.${id === "cloud-onboarding" ? config.modelProfileId : "browser"}.${config.executionEnvironment}.${id}`); + let reportPersisted = false; + const failFinalization = (code: string) => { + for (const result of results) { + result.status = "failed"; result.cleanup = "failed"; result.failureClass = "cleanup_failure"; + result.error ??= code; + if (result.productJourney!.outcome === "passed") result.productJourney!.outcome = "failed"; + result.productJourney!.resourceDisposition = "cleanup-failed"; + if (!result.productJourney!.phases.some(phase => phase.phase === "cleanup" && phase.code === code)) { + result.productJourney!.phases.push({ phase: "cleanup", outcome: "failed", code }); + } + } + }; + const persist = async () => { + try { + await writeFile(path.join(outputDir, "normalized-results.tmp"), JSON.stringify({ schema: "paperclip.runner-e2e.campaign/v2", campaignId, generatedAt: new Date().toISOString(), expected, source, results }, null, 2), { mode: 0o600 }); + await rename(path.join(outputDir, "normalized-results.tmp"), path.join(outputDir, "normalized-results.json")); + reportPersisted = true; + } catch { + reportPersisted = false; + failFinalization("evidence_persistence_failed"); + throw new JourneyError("failed", "evidence_persistence_failed"); + } + }; + const persistSafely = async () => { try { await persist(); return true; } catch { return false; } }; + const renderDashboard = async () => { + // An uncooperative late renderer can only write its own private staging area. + // The launcher's normalized results remain authoritative, including failures. + const staging = path.join(outputDir, `.dashboard-${randomUUID()}`); + await withinLifecycleDeadline(cleanupWorkDeadlineAt, "report_generation_timeout", async () => { + await mkdir(staging, { mode: 0o700 }); + await regenerateRunnerDashboard({ bundle: outputDir, historyFile: null, outputDirectory: staging }); + }); + // Publication has no async gap in which a timeout can persist a newer failure. + if (Date.now() >= cleanupWorkDeadlineAt) throw new JourneyError("failed", "report_generation_timeout"); + renameSync(path.join(staging, "index.html"), path.join(outputDir, "index.html")); + renameSync(path.join(staging, "dashboard.html"), path.join(outputDir, "dashboard.html")); + }; + process.once("SIGINT", abort); process.once("SIGTERM", abort); + const timer = setTimeout(abort, config.limits.durationMs); + try { + try { + browser = await chromium.launch({ headless: !config.headed, ...(process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ? { executablePath: process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE } : {}) }); + context = await browser.newContext(); + context.setDefaultTimeout(30_000); + if (controller.signal.aborted) throw new JourneyError("blocked", "campaign_deadline_or_interruption"); + const loaded = config.mode === "managed-self-hosted" ? await import("./local-target.js") : await import(pathToFileURL(path.resolve(config.adapterModule!)).href); + const provider = loaded.provider as TargetProvider; + if (provider?.version !== 1 || typeof provider.acquire !== "function") throw new JourneyError("blocked", "unsupported_target_provider_contract"); + session = await provider.acquire({ config, campaignId, deadlineAt, browser: context, signal: controller.signal, repositoryRoot, outputDir, harness: { revision: source.sha, definitionHash: suiteDefinitionHash } }); + await session.authenticate(); + fixtures = journeyFixtures(session, campaignId); + } catch (error) { prerequisiteFailure = journeyFailure(error); } + for (const [index, id] of config.journeys.entries()) { + const startedAt = new Date().toISOString(); + const evidence: JourneyEvidence = { targetId: config.id, outcome: "blocked", startingState: ready?.startingState ?? "recipe", phases: [], checks: [], resourceDisposition: "cleanup-failed", privacy: "private", campaignId, attemptId: `${campaignId}:${id}:1` }; + const profile = id === "cloud-onboarding" ? runnerSuites.flatMap(suite => suite.profiles).find(p => p.id === config.modelProfileId) : undefined; + const result: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v2", executionId: expected[index]!, suiteId: "product-journeys", suiteDefinitionHash, + source, attempt: 1, status: "failed", profileId: profile?.id ?? "browser", environmentId: config.executionEnvironment, caseId: id, + provider: profile?.provider ?? null, model: profile?.model ?? null, runtimeMode: profile?.generation ?? "none", startedAt, finishedAt: startedAt, durationMs: 0, cleanup: "not_started", productJourney: evidence, + }; + results.push(result); + let phase: JourneyEvidence["phases"][number]["phase"] = "authentication"; + try { + if (prerequisiteFailure) { + if (index === 0) throw new JourneyError(prerequisiteFailure.outcome, prerequisiteFailure.code); + throw new JourneyError("blocked", "prerequisite_journey_failed"); + } + if (controller.signal.aborted) throw new JourneyError("blocked", "campaign_deadline_or_interruption"); + evidence.phases.push({ phase: "authentication", outcome: "passed" }); + await persist(); + phase = id === "cloud-onboarding" ? "journey" : "setup"; + if (id === "cloud-onboarding") { + if (!session!.onboard) throw new JourneyError("unsupported", "target_has_no_onboarding"); + ready = await session!.onboard(evidence); + fixtures!.provide(ready); + } else { + ready = await fixtures!.resolve(productJourneys.find(j => j.id === id)!.requires); + evidence.phases.push({ phase: "setup", outcome: "passed" }); + evidence.deployment = ready.identity; + evidence.startingState = ready.startingState; + await persist(); + phase = "journey"; + await taskProperties(ready, session!, campaignId, evidence); + } + evidence.deployment = ready.identity; + evidence.startingState = ready.startingState; + if (JSON.stringify(await session!.identity()) !== JSON.stringify(ready.identity)) throw new JourneyError("failed", "deployment_changed_during_journey"); + evidence.outcome = "passed"; + evidence.phases.push({ phase: "journey", outcome: "passed" }); + result.status = "passed"; + const evidenceDir = path.join(outputDir, "evidence", result.executionId, "attempt-1"); + await mkdir(evidenceDir, { recursive: true, mode: 0o700 }); + await ready.page.screenshot({ path: path.join(evidenceDir, `${id}.png`) }); + result.screenshots = [{ id, label: productJourneys.find(j => j.id === id)!.label, file: `${id}.png` }]; + } catch (error) { + const failure = journeyFailure(error); + evidence.outcome = failure.outcome; + evidence.phases.push({ phase, ...failure }); + result.status = "failed"; + result.error = failure.code; + result.failureClass = failure.outcome === "failed" ? "candidate_failure" : "permanent_infrastructure"; + if (id === "cloud-onboarding" || !ready) prerequisiteFailure = failure; + } + result.finishedAt = new Date().toISOString(); result.durationMs = Date.parse(result.finishedAt) - Date.parse(startedAt); + if (!await persistSafely()) prerequisiteFailure ??= { outcome: "failed", code: "evidence_persistence_failed" }; + } + } finally { + try { + const collected = await withinLifecycleDeadline(deadlineAt, "evidence_collection_timeout", input => session?.collect(input) ?? Promise.resolve([]), controller.signal) ?? []; + const seen = new Set(); + for (const receipt of collected) { + validateJourneyCollection(receipt); + const result = results.find(r => r.caseId === receipt.journeyId); + if (!result || seen.has(receipt.journeyId) || receipt.operations && (receipt.operations.campaignId !== campaignId || receipt.operations.attemptId !== result.productJourney!.attemptId)) throw new JourneyError("failed", "foreign_or_duplicate_operations_receipt"); + seen.add(receipt.journeyId); + // Validate the normalized boundary before retaining any adapter output. + const candidate = { ...result, status: "failed", runIds: receipt.runIds, usage: receipt.usage, productJourney: { ...result.productJourney!, operations: receipt.operations } }; + validateRetainedRunnerResult(candidate); + result.runIds = receipt.runIds; + result.usage = receipt.usage; + if (receipt.operations) result.productJourney!.operations = receipt.operations; + } + for (const result of results) { + if (result.caseId === "cloud-onboarding" && result.status === "passed" && (!seen.has(result.caseId) || !result.runIds?.length || !result.productJourney?.operations || !result.productJourney.operations.traceIds.length)) throw new JourneyError("failed", "paid_journey_receipt_missing"); + // Success-only checks must run before a disposable target can be deleted. + // Cleanup is provisional here; the observed cleanup result is retained below. + if (result.status === "passed") validateRetainedRunnerResult({ ...result, cleanup: "passed" }); + } + for (const result of results) result.productJourney!.phases.push({ phase: "collection", outcome: "passed" }); + } catch (error) { + const failure = journeyFailure(error); + for (const result of results) { result.status = "failed"; if (result.productJourney!.outcome === "passed") { result.productJourney!.outcome = "failed"; result.error = failure.code; } result.productJourney!.phases.push({ phase: "collection", outcome: "failed", code: ["paid_journey_receipt_missing", "evidence_collection_timeout"].includes(failure.code) ? failure.code : "evidence_collection_failed" }); } + } + clearTimeout(timer); + cleanupDeadlineAt = Date.now() + cleanupGraceMs; + cleanupWorkDeadlineAt = cleanupDeadlineAt - stopReserveMs - browserCloseReserveMs; + // A private provider checkpoints the measured evidence before deleting resources. + await persistSafely(); + try { + const disposition = await withinLifecycleDeadline(cleanupWorkDeadlineAt, "target_cleanup_timeout", input => session?.release(results.every(r => r.status === "passed") ? "passed" : "failed", input) ?? Promise.resolve("released" as const)); + releaseSettled = true; + for (const result of results) { if (result.cleanup !== "failed") result.cleanup = "passed"; result.productJourney!.resourceDisposition = disposition; result.productJourney!.phases.push({ phase: "cleanup", outcome: "passed" }); } + } catch (error) { + failFinalization(error instanceof JourneyError && error.code === "target_cleanup_timeout" ? error.code : "target_cleanup_failed"); + // A timed-out release may still have an in-flight request. Stop-only recovery + // gets its own bounded signal and cannot delete resources or free the lease. + try { await withinLifecycleDeadline(Math.min(cleanupDeadlineAt - browserCloseReserveMs, Date.now() + stopReserveMs), "target_stop_timeout", input => session?.stop?.(input) ?? Promise.resolve()); } + catch { failFinalization("target_stop_failed"); } + } + try { await withinLifecycleDeadline(Math.min(cleanupDeadlineAt, Date.now() + browserCloseReserveMs), "browser_cleanup_timeout", () => browser?.close() ?? Promise.resolve()); } + catch (error) { failFinalization(error instanceof JourneyError && error.code === "browser_cleanup_timeout" ? error.code : "browser_cleanup_failed"); } + process.removeListener("SIGINT", abort); process.removeListener("SIGTERM", abort); + await persistSafely(); + } + // If the final write failed, make one bounded attempt to retain that failure itself. + if (!reportPersisted) await persistSafely(); + if (reportPersisted) { + try { await renderDashboard(); } + catch { failFinalization("report_generation_failed"); await persistSafely(); } + } + try { + if (reportPersisted) await withinLifecycleDeadline(cleanupWorkDeadlineAt, "evidence_finalization_timeout", input => { + // Only a settled release can authorize lease-releasing finalization. + return (releaseSettled ? session?.finalizeEvidence?.(input) : session?.checkpointEvidence?.(input)) ?? Promise.resolve(); + }); + } + catch { + failFinalization("evidence_finalization_failed"); + await persistSafely(); + if (reportPersisted) { + try { await renderDashboard(); } catch { /* Durable provider checkpoint and cleanup failure remain recoverable. */ } + try { await withinLifecycleDeadline(cleanupWorkDeadlineAt, "evidence_checkpoint_timeout", input => session?.checkpointEvidence?.(input) ?? Promise.resolve()); } catch { /* Keep the provider's pending manifest and exclusive leases for recovery. */ } + } + } + if (!reportPersisted) console.error("evidence_persistence_failed"); + console.log(`Product journeys: ${results.filter(r => r.status === "passed").length}/${results.length} passed; ${reportPersisted ? `private report ${outputDir}/dashboard.html` : "evidence pending recovery"}`); + if (!reportPersisted || results.some(r => r.status !== "passed")) process.exitCode = 1; +} diff --git a/tests/runner-e2e/product-journeys/local-target.ts b/tests/runner-e2e/product-journeys/local-target.ts new file mode 100644 index 00000000000..6a70bdc7e6b --- /dev/null +++ b/tests/runner-e2e/product-journeys/local-target.ts @@ -0,0 +1,37 @@ +import { createHash } from "node:crypto"; +import { parseConnectionConfig } from "../connection-config.js"; +import { ConnectionApi, startConnectionTarget } from "../connection-target.js"; +import { JourneyError, type JourneyContext, type TargetProvider } from "./types.js"; + +export const provider: TargetProvider = { + version: 1, + async acquire(input) { + if (input.config.baseline.connections.length) throw new JourneyError("unsupported", "local_connection_baseline_not_supported"); + const target = await startConnectionTarget(parseConnectionConfig({ target: { mode: "managed-local" } }), [], input.repositoryRoot, input.signal); + const api = new ConnectionApi(input.browser.request, target.origin); + const page = await input.browser.newPage().catch(async error => { await target.stop(); throw error; }); + const identity = async () => { + const health = await api.get<{ commit: string; deploymentMode: string }>("/api/health"); + if (!/^[a-f0-9]{40}$/.test(health.commit) || health.deploymentMode !== "local_trusted") throw new JourneyError("blocked", "local_identity_unavailable"); + const settings = await api.get("/api/instance/settings/experimental"); + return { kind: "self-hosted" as const, targetId: input.config.id, origin: target.origin, commit: health.commit, cloudRelease: null, runtimeImage: null, + effectiveFeatures: Object.fromEntries(Object.entries(settings).filter(([, value]) => typeof value === "boolean")) as Record, + configurationHash: createHash("sha256").update(JSON.stringify(settings)).digest("hex") }; + }; + return { + async authenticate() { await api.get("/api/auth/get-session"); }, + async prepare(): Promise { + if (Object.keys(input.config.baseline.features).length) await api.patch("/api/instance/settings/experimental", input.config.baseline.features); + const effective = await api.get>("/api/instance/settings/experimental"); + if (Object.entries(input.config.baseline.features).some(([key, value]) => effective[key] !== value)) throw new JourneyError("failed", "configuration_did_not_converge"); + const company = await api.post<{ id: string; issuePrefix: string }>("/api/companies", { name: `Journey QA ${input.campaignId}` }); + const { user } = await api.get<{ user: { id: string; name: string } }>("/api/auth/get-session"); + return { page, api, company, user, identity: await identity(), capabilities: ["company", "board-user", "task-write"], startingState: "recipe" }; + }, + identity, + async own() { /* Entire managed database belongs to this attempt. */ }, + async collect() {}, + async release() { await target.stop(); return "released" as const; }, + }; + }, +}; diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts new file mode 100644 index 00000000000..d7ab78a64fa --- /dev/null +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.test.ts @@ -0,0 +1,166 @@ +import { createHash } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createConnectionProof } from "../connection-evidence.js"; +import { connectionDelay } from "../connection-target.js"; +import { onboardingFirstTask, type OnboardingFirstTaskInput } from "./onboarding-first-task.js"; + +vi.mock("../connection-target.js", () => ({ connectionDelay: vi.fn(async () => {}) })); +type Row = { id: string; [key: string]: any }; +type Options = { + plan?: boolean; + wrongProof?: boolean; + mutate?(phase: string, state: { tasks: Row[]; runs: Row[]; interactions: Row[]; documents: Row[]; attachments: Row[]; comments: Row[] }): void; +}; + +/** Models the existing REST responses and visible card controls, with no mutating API. */ +function fixture(options: Options = {}) { + let phase = "opening"; + const proof = createConnectionProof(); + const bytes = Buffer.from(JSON.stringify(options.wrongProof ? { ...proof.expected, total: proof.expected.total + 1 } : proof.expected)); + const events: string[] = []; + const title = "Artifact campaign-1"; + const run = (id: string, issueId: string): Row => ({ + id, companyId: "company", agentId: "agent", nativeIssueId: issueId, runtimeMode: "native", status: "succeeded", + startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", usageJson: { inputTokens: 1, outputTokens: 1 }, + contextSnapshot: { issueId, aiConnection: { connectionId: "connection", method: "api_key" }, paperclipEnvironment: { id: "environment" } }, + }); + const state = () => { + const accepted = phase === "accepted"; + const parent = { id: "parent", identifier: "QA-1", title: "Paperclip onboarding", originKind: "onboarding_first_task", description: "/first-task", assigneeAgentId: "agent", status: accepted ? "done" : "in_review" }; + const opening = { id: "opening", kind: "ask_user_questions", status: phase === "opening" ? "pending" : "answered", payload: { submitLabel: "Continue", questions: [{ id: "first-task-opening", options: [{ id: "task", label: "I have a task in mind", freeText: true }] }] } }; + const proposal = { id: "proposal", kind: options.plan ? "request_checkbox_confirmation" : "request_confirmation", status: accepted ? "accepted" : "pending", payload: { prompt: `Create one child task: ${title}`, ...(options.plan ? { options: [{ id: "scope", label: "One artifact task" }], target: { type: "issue_document", key: "plan", revisionId: "revision" } } : {}) } }; + const result = { + tasks: [parent, ...(accepted ? [{ id: "child", identifier: "QA-2", title, parentId: "parent", assigneeAgentId: "agent", status: "done" }] : [])], + runs: phase === "opening" ? [] : [run("proposal-run", "parent"), ...(accepted ? [run("acceptance-run", "parent"), run("child-run", "child"), run("completion-run", "parent")] : [])], + interactions: [opening, ...(phase === "opening" ? [] : [proposal])], + documents: [...(options.plan && phase !== "opening" ? [{ id: "plan", issueId: "parent", key: "plan", latestRevisionId: "revision", body: `One child: ${title}` }] : []), ...(accepted ? [{ id: "result", issueId: "child", key: "connection-proof", updatedByAgentId: "agent", body: `connection-proof.json ${proof.expected.nonce}` }] : [])], + attachments: accepted ? [{ id: "artifact", issueId: "child", originalFilename: "connection-proof.json", createdByAgentId: "agent", originatingRunId: "child-run", byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex") }] : [], + comments: [{ id: "greeting", authorAgentId: "agent", body: "Welcome. What would you like to do?" }, ...(accepted ? [{ id: "completion", authorAgentId: "agent", body: "[QA-2](/QA/issues/child) [connection-proof.json](/api/attachments/artifact/content)" }] : [])], + }; + options.mutate?.(phase, result); + return result; + }; + const api = { + origin: "https://qa.example.test", + async get(route: string) { + const current = state(); + if (route.startsWith("/api/companies/company/issues?")) { + expect(route).toContain("includeAgentCreatedTasks=true"); + return current.tasks; + } + if (route === "/api/companies/company/heartbeat-runs?limit=1000") return current.runs.map(row => ({ id: row.id })); + if (route.startsWith("/api/heartbeat-runs/")) return current.runs.find(row => row.id === route.split("/").at(-1)); + if (route === "/api/issues/parent/interactions") return current.interactions; + if (route === "/api/issues/parent/comments?order=asc") return current.comments; + const match = route.match(/^\/api\/issues\/([^/]+)\/(documents|attachments)(?:\/([^/]+))?$/); + if (match) { + const rows: Row[] = current[match[2] as "documents" | "attachments"].filter(row => row.issueId === match[1]); + return match[3] ? rows.find(row => row.key === decodeURIComponent(match[3]!)) : rows; + } + throw new Error(`Unexpected read: ${route}`); + }, + async bytes(route: string) { expect(route).toBe("/api/attachments/artifact/content"); return bytes; }, + }; + const locator = (name: string): any => ({ + first() { return this; }, last() { return this; }, locator() { return this; }, + async click() { + events.push(`click:${name}`); + if (name === "Continue") phase = "proposal"; + if (name === "Approve" || name === "Confirm selection") phase = "accepted"; + }, + async check() { events.push(`check:${name}`); }, + async fill(text: string) { expect(text).toContain(title); events.push("fill:request"); }, + async waitFor() { events.push("visible:download"); expect(phase).toBe("accepted"); }, + }); + const page = { + getByRole(_role: string, { name }: { name: string }) { return locator(name); }, + getByTestId(name: string) { return locator(name); }, + locator(selector: string) { expect(selector).toContain("/api/attachments/artifact/content"); return locator(selector); }, + async goto(url: string) { expect(url).toBe("https://qa.example.test/QA/issues/QA-1"); events.push("open:parent"); }, + async reload() { events.push("reload:parent"); }, + }; + const own = vi.fn(async (resource: { name: string; id?: string }) => { events.push(`own:${resource.id ?? resource.name}`); }); + const observeRun = vi.fn(async (row: Row) => { events.push(`run:${row.id}`); }); + const input = { context: { api, page, company: { id: "company", issuePrefix: "QA" }, environmentId: "environment" }, + agent: { id: "agent" }, connectionId: "connection", proof, campaignId: "campaign-1", maxRuns: 8, + deadlineAt: Date.now() + 60_000, own, observeRun } as unknown as OnboardingFirstTaskInput; + return { input, own, observeRun, events }; +} + +afterEach(() => { vi.restoreAllMocks(); vi.mocked(connectionDelay).mockReset().mockResolvedValue(undefined); }); + +describe("Cloud canonical onboarding first task", () => { + it.each([false, true])("answers the opening and approves one child, including plan mode %s", async plan => { + const f = fixture({ plan }); + const result = await onboardingFirstTask(f.input); + expect(result).toMatchObject({ onboardingIssueId: "parent", childIssueId: "child", artifact: { id: "artifact", originatingRunId: "child-run" } }); + expect(result.runIds).toEqual(["proposal-run", "acceptance-run", "child-run", "completion-run"]); + expect(f.events.indexOf("own:Artifact campaign-1")).toBeLessThan(f.events.indexOf("click:Continue")); + expect(f.events.indexOf("run:proposal-run")).toBeLessThan(f.events.indexOf(plan ? "click:Confirm selection" : "click:Approve")); + expect(f.events.at(-1)).toBe("visible:download"); + expect(f.own).toHaveBeenCalledWith({ kind: "issue", name: "Artifact campaign-1", id: "child" }); + expect(f.observeRun).toHaveBeenCalledTimes(4); + }); + it("refuses premature child creation without approving it", async () => { + const f = fixture({ mutate(phase, state) { if (phase === "proposal") state.tasks.push({ id: "early", title: "Artifact campaign-1" }); } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_work_before_acceptance" }); + expect(f.events).not.toContain("click:Approve"); + }); + it("does not accept a proposal that omits the intended campaign child", async () => { + const f = fixture({ mutate(phase, state) { if (phase === "proposal") state.interactions[1]!.payload.prompt = "Create an unrelated task"; } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_proposal_child_title_mismatch" }); + expect(f.events).not.toContain("click:Approve"); + }); + it("observes but rejects model work before the opening answer", async () => { + const f = fixture({ mutate(phase, state) { if (phase === "opening") state.runs.push({ id: "unexpected", status: "running" }); } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_opening_started_model_work" }); + expect(f.observeRun).toHaveBeenCalledTimes(1); + expect(f.events).not.toContain("click:Continue"); + }); + it.each(["no-child", "duplicate", "wrong-title"])("rejects an approved scope violation: %s", async mode => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + if (mode === "no-child") state.tasks.pop(); + if (mode === "duplicate") state.tasks.push({ ...state.tasks[1]!, id: "extra" }); + if (mode === "wrong-title") state.tasks[1]!.title = "Unowned child"; + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: mode === "no-child" ? "onboarding_completed_without_child" : "onboarding_approved_child_mismatch" }); + }); + it.each(["parent-run", "wrong-connection", "bad-hash", "missing-document", "missing-parent-links"])("does not pass an unverified deliverable: %s", async mode => { + const f = fixture({ mutate(phase, state) { + if (phase !== "accepted") return; + if (mode === "parent-run") state.attachments[0]!.originatingRunId = "acceptance-run"; + if (mode === "wrong-connection") state.runs[2]!.contextSnapshot.aiConnection.connectionId = "other"; + if (mode === "bad-hash") state.attachments[0]!.sha256 = "0".repeat(64); + if (mode === "missing-document") state.documents = []; + if (mode === "missing-parent-links") state.comments.pop(); + } }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: mode === "bad-hash" ? "artifact_independent_check_failed" : mode === "missing-document" ? "onboarding_child_document_missing" : mode === "missing-parent-links" ? "onboarding_parent_result_links_missing" : "artifact_run_attribution_mismatch" }); + expect(f.events).not.toContain("visible:download"); + }); + it("counts proposal and parent runs toward the campaign cap", async () => { + const f = fixture(); f.input.maxRuns = 3; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "campaign_run_limit_exceeded" }); + expect(f.observeRun).toHaveBeenCalledTimes(4); + }); + it("independently rejects a wrong result even when size and download hash are correct", async () => { + const f = fixture({ wrongProof: true }); + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "artifact_independent_check_failed" }); + expect(f.events).not.toContain("visible:download"); + }); + it("waits for parent completion and stops at the campaign deadline", async () => { + let now = Date.now(); + vi.spyOn(Date, "now").mockImplementation(() => now); + vi.mocked(connectionDelay).mockImplementation(async ms => { now += ms; }); + const f = fixture({ mutate(phase, state) { if (phase === "accepted") state.tasks[0]!.status = "blocked"; } }); + f.input.deadlineAt = now + 2000; + await expect(onboardingFirstTask(f.input)).rejects.toMatchObject({ code: "onboarding_first_task_deadline" }); + expect(f.events).not.toContain("visible:download"); + }); + it("retains an ordinary blocked unstarted wake without mistaking it for paid failure", async () => { + const f = fixture({ mutate(phase, state) { + if (phase === "accepted") state.runs.push({ ...state.runs[0]!, id: "obsolete-wake", status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null }); + } }); + expect((await onboardingFirstTask(f.input)).runIds).toContain("obsolete-wake"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/onboarding-first-task.ts b/tests/runner-e2e/product-journeys/onboarding-first-task.ts new file mode 100644 index 00000000000..9923c81ba42 --- /dev/null +++ b/tests/runner-e2e/product-journeys/onboarding-first-task.ts @@ -0,0 +1,190 @@ +import { createHash } from "node:crypto"; +import { ONBOARDING_FIRST_TASK_ORIGIN_KIND } from "../../../packages/shared/src/constants.js"; +import { type createConnectionProof, verifyConnectionArtifact, verifyConnectionRun } from "../connection-evidence.js"; +import { connectionDelay } from "../connection-target.js"; +import { activeRuns, firstTaskCompletionSettled, type Row } from "../first-task-scoring.js"; +import { isBlockedUnstartedWake, isTerminalUnstartedWake } from "../non-execution-wake.js"; +import { JourneyError, type JourneyContext, type TargetSession } from "./types.js"; + +export interface OnboardingFirstTaskInput { + context: JourneyContext; + agent: { id: string }; + connectionId: string; + proof: ReturnType; + campaignId: string; + maxRuns: number; + deadlineAt: number; + own: TargetSession["own"]; + observeRun(run: Row): Promise; +} +export interface OnboardingFirstTaskResult { + runIds: string[]; + onboardingIssueId: string; + childIssueId: string; + artifact: { id: string; originatingRunId: string; sha256: string }; +} +interface Snapshot { + tasks: Row[]; + runs: Row[]; + interactions: Row[]; + comments: Row[]; + documents: Row[]; + attachments: Row[]; +} + +function fail(code: string): never { throw new JourneyError("failed", code); } +const runIssueId = (run: Row) => run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId; +const isProposal = (row: Row) => ["request_confirmation", "request_checkbox_confirmation"].includes(row.kind); +const isPlanningDocument = (row: Row) => /^(?:plan|proposal)$/.test(row.key); + +/** Exercise the ordinary onboarding conversation; APIs observe, never submit its actions. */ +export async function onboardingFirstTask(input: OnboardingFirstTaskInput): Promise { + const { context: { api, page, company }, agent, campaignId, proof } = input; + if (!Number.isSafeInteger(input.maxRuns) || input.maxRuns < 1) fail("invalid_onboarding_run_limit"); + const title = `Artifact ${campaignId}`; + const seen = new Map(); + const observed = new Map(); + const remaining = () => { + const ms = input.deadlineAt - Date.now(); + if (ms <= 0) fail("onboarding_first_task_deadline"); + return Math.min(ms, 30_000); + }; + const wait = async () => { await connectionDelay(Math.min(1000, remaining())); }; + const load = async (parentId?: string): Promise => { + remaining(); + const [tasks, rows] = await Promise.all([ + api.get(`/api/companies/${company.id}/issues?limit=100&includeAgentCreatedTasks=true`), + api.get(`/api/companies/${company.id}/heartbeat-runs?limit=1000`), + ]); + const runs = await Promise.all(rows.map(row => api.get(`/api/heartbeat-runs/${row.id}`))); + for (const run of runs) { + seen.set(run.id, run); + const revision = JSON.stringify([run.status, run.startedAt, run.finishedAt, run.usageJson, run.costUsd]); + if (observed.get(run.id) !== revision) { + await input.observeRun(run); + observed.set(run.id, revision); + } + } + if (seen.size > input.maxRuns || rows.length === 1000) fail("campaign_run_limit_exceeded"); + if (tasks.length === 100) fail("onboarding_task_scope_overflow"); + if (!parentId) return { tasks, runs, interactions: [], comments: [], documents: [], attachments: [] }; + const [interactions, comments, documents, attachments] = await Promise.all([ + api.get(`/api/issues/${parentId}/interactions`), + api.get(`/api/issues/${parentId}/comments?order=asc`), + Promise.all(tasks.map(async task => { + const summaries = await api.get(`/api/issues/${task.id}/documents`); + return Promise.all(summaries.map(async document => ({ + ...await api.get(`/api/issues/${task.id}/documents/${encodeURIComponent(document.key)}`), + issueId: task.id, key: document.key, + }))); + })).then(rows => rows.flat()), + Promise.all(tasks.map(task => api.get(`/api/issues/${task.id}/attachments`))).then(rows => rows.flat()), + ]); + return { tasks, runs, interactions, comments, documents, attachments }; + }; + const assertRuns = (snapshot: Snapshot, allowedIssues: string[]) => { + for (const run of seen.values()) { + if (run.companyId !== company.id || run.agentId !== agent.id || !allowedIssues.includes(runIssueId(run))) fail("onboarding_run_scope_mismatch"); + if (activeRuns([run]).length || run.status === "succeeded" || isBlockedUnstartedWake(run) || isTerminalUnstartedWake(run, snapshot.tasks)) continue; + fail("onboarding_first_task_failed"); + } + }; + const assertBeforeAcceptance = (snapshot: Snapshot, parentId: string) => { + if (snapshot.tasks.length !== 1 || snapshot.tasks[0]!.id !== parentId || snapshot.attachments.length || + snapshot.documents.some(document => !isPlanningDocument(document)) || snapshot.tasks[0]!.status === "done") fail("onboarding_work_before_acceptance"); + assertRuns(snapshot, [parentId]); + }; + + let snapshot = await load(); + while (!snapshot.tasks.length) { await wait(); snapshot = await load(); } + const parent = snapshot.tasks[0]!; + if (snapshot.tasks.length !== 1 || parent.originKind !== ONBOARDING_FIRST_TASK_ORIGIN_KIND || + parent.assigneeAgentId !== agent.id || !String(parent.description).includes("/first-task")) fail("onboarding_canonical_first_task_missing"); + await input.own({ kind: "issue", name: parent.title, id: parent.id }); + snapshot = await load(parent.id); + if (seen.size) fail("onboarding_opening_started_model_work"); + assertBeforeAcceptance(snapshot, parent.id); + const opening = snapshot.interactions.filter(row => row.kind === "ask_user_questions" && row.status === "pending"); + const questions = opening[0]?.payload?.questionSet?.questions ?? opening[0]?.payload?.questions ?? []; + const question = questions.find((row: Row) => row.id === "first-task-opening"); + const option = question?.options?.find((row: Row) => row.id === "task" && row.freeText === true); + if (opening.length !== 1 || questions.length !== 1 || !option || + !snapshot.comments.some(row => row.authorAgentId === agent.id && String(row.body ?? "").trim())) fail("onboarding_opening_not_presented"); + await page.goto(`${api.origin}/${company.issuePrefix}/issues/${parent.identifier ?? parent.id}`, { timeout: remaining() }); + // Persist the exact intended child before the opening reply can dispatch any model work. + await input.own({ kind: "issue", name: title }); + await page.getByRole("radio", { name: option.label }).last().click({ timeout: remaining() }); + await page.getByTestId("question-other-answer-composer").last().locator('[contenteditable="true"],textarea').first().fill([ + `I have one task in mind. Propose exactly one child task named "${title}" and wait for my approval before creating it or doing its work.`, + "After approval, assign it to yourself and let its own Paperclip execution produce the result. Then verify and link its saved document and downloadable file in this onboarding conversation and complete both tasks.", + "The child's approved scope: decode the following base64 into connection-input.json in its workspace, using actual tools.", + `Exact input bytes: ${Buffer.from(proof.input).toString("base64")}`, + 'Write connection-proof.json containing exactly {"nonce":string,"count":number,"total":number,"sha256":string}. Copy nonce, count the input values, sum them, and compute sha256 of the exact input bytes including the final newline.', + "Publish connection-proof.json on that child using register_deliverable with workspace-relative contentRef and correct byteSize and SHA-256. Save a connection-proof document on the child including the input nonce, result, and file link. Do not hire agents, create additional tasks, or access credentials or unrelated files.", + ].join("\n"), { timeout: remaining() }); + await page.getByRole("button", { name: opening[0]!.payload.submitLabel ?? "Continue", exact: true }).last().click({ timeout: remaining() }); + + let proposal: Row | undefined; + while (!proposal) { + snapshot = await load(parent.id); + assertBeforeAcceptance(snapshot, parent.id); + const pending = snapshot.interactions.filter(row => row.status === "pending"); + const proposals = pending.filter(isProposal); + if (proposals.length > 1 || pending.some(row => row.kind === "ask_user_questions" && row.id !== opening[0]!.id)) fail("onboarding_proposal_scope_unclear"); + // Let the producing turn finish before acceptance; this journey does not test overlap. + if (proposals.length === 1 && seen.size && !activeRuns([...seen.values()]).length) proposal = proposals[0]; + else await wait(); + } + const target = proposal.payload?.target; + const plan = target?.type === "issue_document" ? snapshot.documents.find(document => document.issueId === parent.id && document.key === target.key && + (!target.issueId || target.issueId === parent.id) && document.latestRevisionId === target.revisionId && + (!target.documentId || target.documentId === document.id)) : undefined; + if ((proposal.kind === "request_checkbox_confirmation" && (!plan || !isPlanningDocument(plan))) || + ![proposal.payload?.prompt, proposal.payload?.detailsMarkdown, plan?.body].some(text => typeof text === "string" && text.includes(title))) fail("onboarding_proposal_child_title_mismatch"); + if (proposal.kind === "request_checkbox_confirmation") { + if (!proposal.payload?.options?.length) fail("onboarding_proposal_scope_unclear"); + for (const item of proposal.payload.options) await page.getByRole("checkbox", { name: item.label, exact: true }).last().check({ timeout: remaining() }); + } + await page.getByRole("button", { name: proposal.payload?.acceptLabel ?? (proposal.kind === "request_confirmation" ? "Approve" : "Confirm selection"), exact: true }).last().click({ timeout: remaining() }); + for (;;) { + snapshot = await load(parent.id); + const status = snapshot.interactions.find(row => row.id === proposal!.id)?.status; + if (status === "accepted") break; + if (status !== "pending") fail("onboarding_proposal_not_accepted"); + await wait(); + } + + let childId: string | undefined; + for (;;) { + const currentParent = snapshot.tasks.find(row => row.id === parent.id); + const children = snapshot.tasks.filter(row => row.id !== parent.id); + if (!currentParent || children.length > 1 || children.some(row => row.parentId !== parent.id || row.title !== title || row.assigneeAgentId !== agent.id)) fail("onboarding_approved_child_mismatch"); + const child = children[0]; + if (currentParent.status === "cancelled" || child?.status === "cancelled") fail("onboarding_first_task_failed"); + if (!child && currentParent.status === "done") fail("onboarding_completed_without_child"); + if (child && !childId) { childId = child.id; await input.own({ kind: "issue", name: title, id: child.id }); } + if (childId && child?.id !== childId) fail("onboarding_approved_child_mismatch"); + assertRuns(snapshot, [parent.id, ...(childId ? [childId] : [])]); + if (snapshot.attachments.some(row => row.issueId !== childId)) fail("onboarding_artifact_created_by_parent"); + const settled = child && currentParent.status === "done" && firstTaskCompletionSettled(snapshot.tasks, [parent.id], parent.id) && !activeRuns([...seen.values()]).length; + if (settled) { + const artifacts = snapshot.attachments.filter(row => row.issueId === child.id && row.originalFilename === "connection-proof.json"); + if (artifacts.length !== 1) fail("onboarding_child_artifact_missing"); + const artifact = artifacts[0]!; + const run = seen.get(artifact.originatingRunId); + if (artifact.createdByAgentId !== agent.id || !input.context.environmentId || !run || run.nativeIssueId !== child.id || + run.contextSnapshot?.issueId !== child.id || !verifyConnectionRun(run, { agentId: agent.id, connectionId: input.connectionId, method: "api_key", runtimeMode: "native", environmentId: input.context.environmentId })) fail("artifact_run_attribution_mismatch"); + const bytes = await api.bytes(`/api/attachments/${artifact.id}/content`); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + if (artifact.byteSize !== bytes.byteLength || artifact.sha256 !== sha256 || !verifyConnectionArtifact(bytes, proof.expected)) fail("artifact_independent_check_failed"); + if (!snapshot.documents.some(row => row.issueId === child.id && !isPlanningDocument(row) && row.updatedByAgentId === agent.id && + typeof row.body === "string" && row.body.includes(proof.expected.nonce) && row.body.includes("connection-proof.json"))) fail("onboarding_child_document_missing"); + if (!snapshot.comments.some(row => row.authorAgentId === agent.id && String(row.body).includes(artifact.id) && + (String(row.body).includes(child.id) || String(row.body).includes(child.identifier ?? child.id)))) fail("onboarding_parent_result_links_missing"); + await page.reload({ timeout: remaining() }); + await page.locator(`a[href*="/api/attachments/${artifact.id}/content"]`).first().waitFor({ state: "visible", timeout: remaining() }); + return { runIds: [...seen.keys()], onboardingIssueId: parent.id, childIssueId: child.id, artifact: { id: artifact.id, originatingRunId: run.id, sha256 } }; + } + await wait(); snapshot = await load(parent.id); + } +} diff --git a/tests/runner-e2e/product-journeys/task-properties.ts b/tests/runner-e2e/product-journeys/task-properties.ts new file mode 100644 index 00000000000..ca836af1c8b --- /dev/null +++ b/tests/runner-e2e/product-journeys/task-properties.ts @@ -0,0 +1,27 @@ +import { expect } from "@playwright/test"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +export async function taskProperties(context: JourneyContext, _session: TargetSession, _campaignId: string, evidence: JourneyEvidence) { + for (const capability of ["company", "board-user", "task-write"]) { + if (!context.capabilities.includes(capability)) throw new JourneyError("unsupported", `missing_${capability}`); + } + const { api, page, company, user } = context; + const issue = context.fixtures?.backlogTask; + if (!issue?.id || !issue.title || !issue.identifier) throw new JourneyError("blocked", "missing_prepared_backlog_task"); + // The declared starting point is the task list. Navigation to the task is a user action. + await page.goto(`${api.origin}/${company.issuePrefix}/issues`); + await page.getByRole("link").filter({ hasText: issue.title }).first().click(); + const properties = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).first(); + if (await properties.isVisible()) await properties.click(); + const assignee = page.locator("[data-property-row]").filter({ has: page.locator('[data-property-label="Assignee"]') }); + await assignee.locator("[data-property-value]").getByRole("button").first().click(); + await page.getByRole("button", { name: "Assign to me", exact: true }).click(); + await expect.poll(async () => (await api.get<{ assigneeUserId: string }>(`/api/issues/${issue.id}`)).assigneeUserId).toBe(user.id); + await page.reload(); + await expect(assignee.locator("[data-property-value]")).toContainText("You"); + const persisted = await api.get<{ assigneeUserId: string; assigneeAgentId: string | null; status: string }>(`/api/issues/${issue.id}`); + if (persisted.assigneeUserId !== user.id || persisted.assigneeAgentId !== null || persisted.status !== "backlog") throw new JourneyError("failed", "assignment_persistence_mismatch"); + const runs = await api.get(`/api/issues/${issue.id}/runs`); + if (runs.length) throw new JourneyError("failed", "browser_only_journey_started_agent_work"); + evidence.checks.push({ id: "assignment-persisted", passed: true }, { id: "no-agent-work", passed: true }); +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts new file mode 100644 index 00000000000..6c9b79ac63a --- /dev/null +++ b/tests/runner-e2e/product-journeys/types.ts @@ -0,0 +1,107 @@ +import type { BrowserContext, Page } from "@playwright/test"; +import type { ConnectionApi } from "../connection-target.js"; + +export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; +export type JourneyId = "task-properties" | "cloud-onboarding"; +export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; +export interface DeploymentIdentity { + kind: "self-hosted" | "cloud"; + targetId: string; + origin: string; + commit: string; + cloudRelease: string | null; + runtimeImage: string | null; + configurationHash: string; + effectiveFeatures?: Record; + executionEnvironment?: { id: string; driver: string; status: string }; +} +export interface JourneyEvidence { + targetId?: string; + outcome: JourneyOutcome; + deployment?: DeploymentIdentity; + startingState: "recipe" | "onboarding"; + phases: Array<{ phase: "authentication" | "setup" | "journey" | "collection" | "cleanup"; outcome: JourneyOutcome; code?: string }>; + checks: Array<{ id: string; passed: boolean }>; + resourceDisposition: "released" | "retained" | "cleanup-failed"; + privacy: "private"; + campaignId?: string; + attemptId?: string; + operations?: JourneyOperations; +} +/** Normalized receipt from the private, Cloud-qualified eval-operations adapter. */ +export interface JourneyOperations { + campaignId: string; + attemptId: string; + modelCostUsd: number | null; + setupCostUsd: number | null; + requestCount: number; + feedback: { collection: "complete" | "partial" | "unavailable"; complaints: number; suggestions: number }; + traceIds: string[]; +} +export interface JourneyCollection { + journeyId: JourneyId; + runIds: string[]; + usage: { runs: Array<{ runId: string; usage: { inputTokens?: number; outputTokens?: number; cachedInputTokens?: number; costUsd?: number } }> }; + operations?: JourneyOperations; +} +export interface TargetConfig { + version: 1; + id: string; + mode: TargetMode; + adapterModule?: string; + adapterConfig?: Record; + journeys: JourneyId[]; + executionEnvironment: "local" | "daytona"; + baseline: { features: Record; connections: string[] }; + limits: { durationMs: number; modelBudgetCents: number; maxRuns: number }; + headed: boolean; + modelProfileId?: "runner-codex" | "runner-codex-mini"; +} +export interface JourneyContext { + page: Page; + api: ConnectionApi; + company: { id: string; issuePrefix: string }; + user: { id: string; name: string }; + environmentId?: string; + identity: DeploymentIdentity; + capabilities: string[]; + startingState: "recipe" | "onboarding"; + fixtures?: { backlogTask?: PreparedBacklogTask }; +} +export interface PreparedBacklogTask { + id: string; + title: string; + identifier: string; +} +export interface TargetLifecycleInput { + signal: AbortSignal; + deadlineAt: number; +} +export interface TargetSession { + context?: JourneyContext; + authenticate(): Promise; + prepare(): Promise; + onboard?(evidence: JourneyEvidence): Promise; + identity(): Promise; + collect(input?: TargetLifecycleInput): Promise; + release(outcome: JourneyOutcome, input?: TargetLifecycleInput): Promise<"released" | "retained">; + /** Stop only run-owned work; never delete fixtures or release leases. */ + stop?(input?: TargetLifecycleInput): Promise; + /** Durable final evidence must precede releasing a Cloud target's held leases. */ + finalizeEvidence?(input?: TargetLifecycleInput): Promise; + /** Publish the observed finalization failure without releasing resources or leases. */ + checkpointEvidence?(input?: TargetLifecycleInput): Promise; + /** Durable intent before creation; receipt immediately after creation. */ + own(resource: { kind: "issue" | "agent" | "connection"; name: string; id?: string }): Promise; +} +export interface TargetProvider { + version: 1; + acquire(input: { + config: TargetConfig; campaignId: string; deadlineAt: number; + browser: BrowserContext; signal: AbortSignal; repositoryRoot: string; outputDir: string; + harness: { revision: string | null; definitionHash: string }; + }): Promise; +} +export class JourneyError extends Error { + constructor(readonly outcome: Exclude, readonly code: string) { super(code); } +} diff --git a/tests/runner-e2e/report-catalog.ts b/tests/runner-e2e/report-catalog.ts index 8d6fcb821be..3128832ec17 100644 --- a/tests/runner-e2e/report-catalog.ts +++ b/tests/runner-e2e/report-catalog.ts @@ -10,10 +10,7 @@ export interface ReportExecution { MatrixExecution["suite"], "id" | "label" | "description" | "expectedMatrixSize" >; - profile: Pick< - MatrixExecution["profile"], - "id" | "label" | "generation" | "provider" | "model" | "expectedRuntimeMode" - >; + profile: { id: string; label: string; generation: "legacy" | "native" | "none"; provider: string | null; model: string | null; expectedRuntimeMode: "legacy" | "native" | "none" }; environment: Pick< MatrixExecution["environment"], "id" | "label" | "provider" | "expectedExecutionTarget" @@ -23,9 +20,12 @@ export interface ReportExecution { export function parseReportExecutionId(id: string) { // These identities also become evidence paths. Reject path traversal and any - // syntax other than the four catalog identity segments, even in imported JSON. + // syntax other than catalog identities, even in imported JSON. Deployment + // journeys add one explicit target segment; all historical IDs stay intact. if (typeof id !== "string") throw new Error("Report execution identity must be a string"); + const journey = /^(product-journeys)\.([a-z0-9][a-z0-9_-]*)\.([a-z0-9][a-z0-9_-]*)\.(local|daytona)\.([a-z0-9][a-z0-9_-]*)$/.exec(id); + if (journey && id.length <= 512) return { suiteId: journey[1]!, targetId: journey[2]!, profileId: journey[3]!, environmentId: journey[4]! as "local" | "daytona", caseId: journey[5]! }; const match = /^([a-z0-9][a-z0-9_-]*)\.([a-z0-9][a-z0-9_-]*)\.(local|daytona)\.([a-z0-9][a-z0-9_-]*)$/.exec( id, @@ -39,6 +39,7 @@ export function parseReportExecutionId(id: string) { profileId: match[2]!, environmentId: match[3]! as "local" | "daytona", caseId: match[4]!, + targetId: undefined, }; } @@ -50,6 +51,7 @@ export function validateReportResultIdentity(result: RunnerE2EResult) { result.profileId !== identity.profileId || result.environmentId !== identity.environmentId || result.caseId !== identity.caseId + || identity.targetId !== undefined && result.productJourney?.targetId !== identity.targetId ) { throw new Error( `Result identity does not match execution ID: ${result.executionId}`, @@ -101,7 +103,7 @@ export function discoverReportCatalog(input: { const knownEnvironment = input.catalog.find( (execution) => execution.environment.id === identity.environmentId, )?.environment; - const runtimeMode = result?.runtimeMode === "legacy" ? "legacy" : "native"; + const runtimeMode = result?.runtimeMode ?? "native"; catalog.push({ id, suite, @@ -114,8 +116,8 @@ export function discoverReportCatalog(input: { generation: runtimeMode, expectedRuntimeMode: runtimeMode, provider: - typeof result?.provider === "string" ? result.provider : "unknown", - model: typeof result?.model === "string" ? result.model : "unknown", + result?.runtimeMode === "none" ? null : result?.provider ?? "unknown", + model: result?.runtimeMode === "none" ? null : result?.model ?? "unknown", }, environment: knownEnvironment ?? { id: identity.environmentId, diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index acf4bfd8391..eb06745b108 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -25,6 +25,10 @@ const integer: Rule = (value, at) => { const boolean: Rule = (value, at) => { if (typeof value !== "boolean") invalid(at); }; +const booleanRecord: Rule = (value, at) => { + object(value, at); + for (const [key, entry] of Object.entries(value as object)) boolean(entry, `${at}.${key}`); +}; const object: Rule = (value, at) => { if (!value || typeof value !== "object" || Array.isArray(value)) invalid(at); }; @@ -218,9 +222,9 @@ const fields = { profileId: string, environmentId: oneOf("local", "daytona"), caseId: string, - provider: string, - model: string, - runtimeMode: oneOf("legacy", "native"), + provider: nullable(string), + model: nullable(string), + runtimeMode: oneOf("legacy", "native", "none"), issueId: optional(string), issueIdentifier: optional(nullable(string)), runIds: optional(array(string)), @@ -292,6 +296,19 @@ const fields = { tasks: array(object), agents: array(object), comments: array(object), interactions: array(object), documents: array(object), attachments: optional(array(object)), runs: array(object) })), checks: array(shape({ id: string, passed: boolean, notReached: optional(string), evidence: array(string), detail: string })), })), + productJourney: optional(shape({ + targetId: optional(string), + campaignId: optional(string), attemptId: optional(string), + operations: optional(closedShape({ campaignId: string, attemptId: string, modelCostUsd: nullable(number), setupCostUsd: nullable(number), requestCount: integer, + feedback: closedShape({ collection: oneOf("complete", "partial", "unavailable"), complaints: integer, suggestions: integer }), traceIds: array(string) })), + outcome: oneOf("passed", "failed", "blocked", "unsupported"), + deployment: optional(shape({ kind: oneOf("self-hosted", "cloud"), targetId: string, origin: url, commit: string, cloudRelease: nullable(string), runtimeImage: nullable(string), configurationHash: string, + effectiveFeatures: optional(booleanRecord), executionEnvironment: optional(closedShape({ id: string, driver: string, status: string })) })), + startingState: oneOf("recipe", "onboarding"), privacy: oneOf("private"), + phases: array(shape({ phase: oneOf("authentication", "setup", "journey", "collection", "cleanup"), outcome: oneOf("passed", "failed", "blocked", "unsupported"), code: optional(string) })), + checks: array(shape({ id: string, passed: boolean })), + resourceDisposition: oneOf("released", "retained", "cleanup-failed"), + })), providerConnection: optional(shape({ version: integer, outcome: oneOf("running", "passed", "failed", "awaiting_user", "missing_credential", "blocked_target"), @@ -327,5 +344,13 @@ export function validateRetainedRunnerResult( ): asserts value is RunnerE2EResult { result(value, "result"); const checked = value as RunnerE2EResult; + if (checked.suiteId === "product-journeys" && checked.status === "passed") { + const evidence = checked.productJourney; + if (!evidence) return invalid("result.productJourney"); + const required = checked.caseId === "task-properties" ? ["assignment-persisted", "no-agent-work"] : ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"]; + if (evidence.outcome !== "passed" || !evidence.deployment || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); + if (checked.caseId === "task-properties" && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); + if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); + } if (checked.suiteId === "provider-connections" && checked.status === "passed" && (!checked.providerConnection || !connectionEvidencePasses(checked.providerConnection))) invalid("result.providerConnection"); } diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 61874dfa7ff..df280261e58 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -316,9 +316,9 @@ export interface RunnerE2EResult { profileId: string; environmentId: RunnerEnvironmentId; caseId: string; - provider: string; - model: string; - runtimeMode: RunnerGeneration; + provider: string | null; + model: string | null; + runtimeMode: RunnerGeneration | "none"; issueId?: string; issueIdentifier?: string | null; runIds?: string[]; @@ -354,6 +354,7 @@ export interface RunnerE2EResult { /** Absent in historical results; new captures bind the exact PNG bytes. */ sha256?: string; }>; + productJourney?: import("./product-journeys/types.js").JourneyEvidence; firstTask?: import("./first-task-scoring.js").FirstTaskEvidence; providerConnection?: import("./connection-evidence.js").ConnectionEvidence; completionQuality?: import("./completion-quality.js").CompletionQualityRecord[]; @@ -436,8 +437,8 @@ export interface RunnerE2EHistoryExecution { profileId: string; environmentId: RunnerEnvironmentId; caseId: string; - provider: string; - model: string; + provider: string | null; + model: string | null; status: "passed" | "failed" | "incomplete"; durationMs: number; attempt: number;