diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 47853d3ba74..129f75c4e31 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -237,6 +237,11 @@ The explicit [deployment-targeted Product journeys](PRODUCT-JOURNEYS.md) reuse browser actions across managed self-hosted and Cloud targets. They include a model-free task-properties story, fresh Cloud onboarding, credential-free previews, private evidence, and prerequisite reuse; `--all` is unchanged. +Cloud onboarding may retain a versioned `onboardingConnect` receipt for the +Connect action: its last checkpoint, validation/hire response state and HTTP +status, known adapters, and allowlisted outcome/check/server codes. The case +report renders these fields explicitly; older results without the receipt remain +valid. This diagnostic does not change the journey grade or qualify a live run. ## Live provider connection journeys diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 26a326b9cca..afbcf8dd494 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -14,6 +14,13 @@ their expected four-field result, exact base64 download bytes, checksums, size, and attachment/run IDs. Extra fields and arbitrary agent output are rejected. It contains no credentials, browser state, request headers, or provider text. +The optional `onboardingConnect` diagnostic is also a closed, versioned receipt. +It retains only a known checkpoint, response states, HTTP statuses, known adapter +names, and allowlisted outcome/check/server codes (at most 32 checks per response). +Unknown codes become `unrecognized`; raw bodies, messages, URLs, headers, and +browser errors are excluded. Reports validate the receipt before rendering its +explicit fields, and historical results may omit it. + ## Attended connection authentication See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/case-outcome.ts b/tests/runner-e2e/case-outcome.ts index 03316c81995..489f16f8384 100644 --- a/tests/runner-e2e/case-outcome.ts +++ b/tests/runner-e2e/case-outcome.ts @@ -1,5 +1,6 @@ import type { RunnerE2EResult } from "./types.js"; import { isBlockedConnectionResult } from "./connection-evidence.js"; +import { validateOnboardingConnectEvidence } from "./product-journeys/onboarding-connect.js"; const html = (value: unknown) => String(value ?? "").replace( /[&<>"']/g, @@ -8,6 +9,28 @@ const html = (value: unknown) => c ]!, ); + +function renderOnboardingConnect(result: RunnerE2EResult): string { + const receipt = result.productJourney?.onboardingConnect; + if (receipt === undefined) return ""; + try { + validateOnboardingConnectEvidence(receipt); + } catch { + return "

Onboarding Connect diagnostics unavailable: invalid receipt.

"; + } + return `
Onboarding Connect diagnostics +

Last checkpoint: ${html(receipt.checkpoint)}

+ ${([["validation", "Native validation"], ["hire", "Agent hire"]] as const).map(([key, label]) => { + const response = receipt[key]; + return `

${label}

${[ + ["Response state", response.state], ["HTTP status", response.httpStatus], + ["Test endpoint adapter", response.requestedAdapter], ["Returned adapter", response.adapter], + ["Validation outcome", response.outcome], ["Server code", response.serverCode], + ].map(([name, value]) => `
${name}
${html(value ?? "Unavailable")}
`).join("")}
+ ${response.checks.length ? `

Validation checks:

` : ""} + ${response.truncatedChecks ? "

Only the first 32 validation checks were retained.

" : ""}`; + }).join("")}
`; +} /** An interrupted first-task journey is not evidence of failed downstream behavior. */ export function isIncompleteFirstTaskResult( result: RunnerE2EResult, @@ -69,6 +92,7 @@ export function renderCaseOutcome( ${result.productJourney.deployment?.effectiveFeatures ? `

Effective features:

${Object.entries(result.productJourney.deployment.effectiveFeatures).map(([key, value]) => `
${html(key)}
${html(value)}
`).join("")}
` : ""}` : ""} ${result.productJourney?.operations ? `

Eval operations: ${html(result.productJourney.operations.campaignId)} · ${result.productJourney.operations.requestCount} requests · model cost $${html(result.productJourney.operations.modelCostUsd ?? "unknown")} (setup $${html(result.productJourney.operations.setupCostUsd ?? "unknown")}, counted once). Feedback: ${html(result.productJourney.operations.feedback.collection)} · ${result.productJourney.operations.feedback.complaints} complaints · ${result.productJourney.operations.feedback.suggestions} suggestions.

` : ""} ${result.productJourney?.operations?.traceIds.length ? `
Trace correlation IDs
` : ""} + ${renderOnboardingConnect(result)} ${result.providerConnection ? `

Target: ${html(result.providerConnection.target.origin)} · ${html(result.providerConnection.target.commit ?? "revision unavailable")} · ${result.providerConnection.assisted ? "Assisted login" : "Unassisted"} · ${html(result.providerConnection.authFreshness)} browser · ${html(result.providerConnection.entry)} / ${html(result.providerConnection.method)} · stopped at ${html(result.providerConnection.phase)}

` : ""} ${failures.length ? `

Failed checks:

` : checks.length ? `

No behavioral matcher failed.${notReached.length ? " The journey is incomplete; unexercised checks are not passes." : failed ? " The overall failure came from a separate run, cleanup, or evidence check." : ""}

` : ""} ${notReached.length ? `

Not reached:

` : ""} diff --git a/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts b/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts index 94589ca5069..0878ade8065 100644 --- a/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts +++ b/tests/runner-e2e/product-journeys/cloud-onboarding.test.ts @@ -86,6 +86,8 @@ function fixture(entry: Entry, options: { intentFailure?: boolean; qualification }, }); const page = { + on() { return this; }, + off() { return this; }, getByLabel(label: string) { return locator(() => controls().filter(control => control.name === label && control.role === "textbox")); }, getByRole(role: string, { name: label }: { name: string | RegExp }) { return locator(() => controls().filter(control => control.role === role && match(control.name, label))); @@ -93,6 +95,7 @@ function fixture(entry: Entry, options: { intentFailure?: boolean; qualification locator(selector: string) { expect(selector).toBe("summary"); return locator(() => controls().filter(control => control.role === "summary")); }, }; const context = { page, company: { id: "company" }, identity: { kind: "cloud" }, startingState: "recipe", api: { + origin: "https://tenant.example.test", async get(route: string) { if (route.endsWith("/ai-connections")) return { connections: saved ? [connection] : [] }; if (route.endsWith("/agents")) return hired ? [{ id: "agent", name, adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: gateway.model } }] : []; diff --git a/tests/runner-e2e/product-journeys/cloud-onboarding.ts b/tests/runner-e2e/product-journeys/cloud-onboarding.ts index 3173676c3ad..6f9a75634a7 100644 --- a/tests/runner-e2e/product-journeys/cloud-onboarding.ts +++ b/tests/runner-e2e/product-journeys/cloud-onboarding.ts @@ -1,5 +1,6 @@ import { expect } from "@playwright/test"; import { createConnectionProof } from "../connection-evidence.js"; +import { onboardingConnect } from "./onboarding-connect.js"; import { onboardingFirstTask } from "./onboarding-first-task.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; @@ -69,11 +70,12 @@ export async function cloudOnboarding(input: CloudOnboardingInput): Promise { evidence.onboardingConnect = value; } }); const agents = await api.get(`/api/companies/${company.id}/agents`); const agent = agents.find(a => a.name === name); if (!agent || agents.length !== 1 || agent.adapterType !== "paperclip_runner" || agent.adapterConfig?.provider !== "codex" || agent.adapterConfig?.model !== input.gateway.model) throw new JourneyError("failed", "onboarding_agent_configuration_mismatch"); + connect.checkpoint = "agent-verification"; await input.own({ kind: "agent", name, id: agent.id }); evidence.checks.push({ id: "ui-connection-and-native-agent", passed: true }); await input.own({ kind: "issue", name: "Paperclip onboarding" }); diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index 754d6c7bab9..b02057281a3 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -13,6 +13,7 @@ import type { RunnerE2EResult } from "../types.js"; import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; import { createConnectionProof } from "../connection-evidence.js"; import { retainOnboardingArtifact } from "./artifact-evidence.js"; +import type { OnboardingConnectEvidence } from "./onboarding-connect.js"; vi.mock("@playwright/test", () => ({ expect: Object.assign( (locator: { textContent(): Promise }) => ({ async toContainText(value: string) { expect(await locator.textContent()).toContain(value); } }), @@ -29,7 +30,60 @@ function result(): RunnerE2EResult { return { schema: "paperclip.runner-e2e.result/v2", suiteId: "product-journeys", suiteDefinitionHash: "test", executionId: "product-journeys.local-qa.local.task-properties", profileId: "local-qa", environmentId: "local", caseId: "task-properties", provider: null, model: null, runtimeMode: "none", status: "passed", cleanup: "passed", attempt: 1, startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, productJourney: { outcome: "passed", startingState: "recipe", phases: [{phase: "journey", outcome: "passed"}], checks: [{id:"assignment-persisted", passed:true},{id:"no-agent-work",passed:true}], privacy: "private", resourceDisposition: "released", deployment: { kind: "self-hosted", targetId: "local-qa", origin: "http://127.0.0.1:3100", commit: "a".repeat(40), cloudRelease: null, runtimeImage: null, configurationHash: "test" } } }; } +function connectReceipt(): OnboardingConnectEvidence { + return { + version: 1, checkpoint: "hire", + validation: { state: "json", httpStatus: 200, requestedAdapter: "paperclip_runner", adapter: "paperclip_runner", outcome: "fail", serverCode: null, checks: [{ code: "adapter_auth_missing", level: "error" }], truncatedChecks: false }, + hire: { state: "json", httpStatus: 422, requestedAdapter: null, adapter: null, outcome: null, serverCode: "agent_runner_unavailable", checks: [], truncatedChecks: false }, + }; +} describe("deployment journey contracts", () => { + it("accepts historical results without Connect diagnostics and optional closed receipts", () => { + const retained = result(); + retained.caseId = "cloud-onboarding"; + retained.status = "failed"; + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + expect(renderCaseOutcome(retained, true, [])).not.toContain("Onboarding Connect diagnostics"); + retained.productJourney!.onboardingConnect = connectReceipt(); + expect(() => validateRetainedRunnerResult(JSON.parse(JSON.stringify(retained)))).not.toThrow(); + retained.productJourney!.onboardingConnect.validation = { state: "pending", httpStatus: 200, requestedAdapter: "codex_local", adapter: null, outcome: null, serverCode: null, checks: [], truncatedChecks: false }; + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + }); + it.each([ + ["request headers", () => ({ ...connectReceipt(), requestHeaders: { authorization: "must-not-retain" } })], + ["response bodies", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, body: "must-not-retain" } })], + ["check messages", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, checks: [{ code: "adapter_auth_missing", level: "error", message: "must-not-retain" }] } })], + ["unknown server codes", () => ({ ...connectReceipt(), hire: { ...connectReceipt().hire, serverCode: "raw-provider-error" } })], + ["unknown check codes", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, checks: [{ code: "raw-provider-error", level: "error" }] } })], + ["unknown adapters", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, adapter: "raw-provider-name" } })], + ["unknown requested adapters", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, requestedAdapter: "raw-provider-name" } })], + ["unknown outcomes", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, outcome: "raw-provider-message" } })], + ["unknown response states", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, state: "raw-provider-message" } })], + ["unknown check levels", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, checks: [{ code: "adapter_auth_missing", level: "fatal" }] } })], + ["oversized check arrays", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, checks: Array.from({ length: 33 }, () => ({ code: "adapter_auth_missing", level: "error" })) } })], + ["invalid HTTP statuses", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, httpStatus: 600 } })], + ["non-boolean truncation", () => ({ ...connectReceipt(), validation: { ...connectReceipt().validation, truncatedChecks: "true" } })], + ["missing response fields", () => ({ ...connectReceipt(), hire: { state: "missing" } })], + ["unsupported receipt versions", () => ({ ...connectReceipt(), version: 2 })], + ["unknown checkpoints", () => ({ ...connectReceipt(), checkpoint: "raw-browser-text" })], + ] as const)("rejects Connect receipts containing %s", (_label, makeReceipt) => { + const retained = result(); + retained.productJourney!.onboardingConnect = makeReceipt() as OnboardingConnectEvidence; + expect(() => validateRetainedRunnerResult(retained)).toThrow("result.productJourney.onboardingConnect"); + }); + it("renders only validated Connect fields and marks missing or truncated evidence", () => { + const retained = result(); + retained.productJourney!.onboardingConnect = connectReceipt(); + const output = renderCaseOutcome(retained, true, []); + for (const text of ["Onboarding Connect diagnostics", "Last checkpoint", "Native validation", "Agent hire", "HTTP status", "Test endpoint adapter", "Returned adapter", "200", "422", "paperclip_runner", "adapter_auth_missing", "agent_runner_unavailable", "error", "Unavailable"]) expect(output).toContain(text); + retained.productJourney!.onboardingConnect.validation.truncatedChecks = true; + expect(renderCaseOutcome(retained, true, [])).toContain("Only the first 32 validation checks were retained"); + Object.assign(retained.productJourney!.onboardingConnect.validation, { body: "" }); + const invalid = renderCaseOutcome(retained, false, []); + expect(invalid).toContain("Onboarding Connect diagnostics unavailable: invalid receipt"); + expect(invalid).not.toContain("must-not-render"); + expect(invalid).not.toContain("adapter_auth_missing"); + }); it("validates optional historical artifact evidence without trusting extra fields or a different run", () => { const retained = result(); retained.caseId = "cloud-onboarding"; diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 2d0233bd2e9..410bb7725bf 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -99,7 +99,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-first-task.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); const suiteDefinitionHash = definitionHash.digest("hex"); const expected = config.journeys.map(id => `product-journeys.${config.id}.${id === "cloud-onboarding" ? config.modelProfileId : "browser"}.${config.executionEnvironment}.${id}`); let reportPersisted = false; diff --git a/tests/runner-e2e/product-journeys/onboarding-connect.test.ts b/tests/runner-e2e/product-journeys/onboarding-connect.test.ts new file mode 100644 index 00000000000..be9535cf0c7 --- /dev/null +++ b/tests/runner-e2e/product-journeys/onboarding-connect.test.ts @@ -0,0 +1,182 @@ +import { EventEmitter } from "node:events"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { onboardingConnect, validateOnboardingConnectEvidence, type OnboardingConnectEvidence } from "./onboarding-connect.js"; + +const origin = "https://tenant.example.test", companyId = "company-1"; +const validation = `/api/companies/${companyId}/adapters/paperclip_runner/test-environment`; +const hire = `/api/companies/${companyId}/agent-hires`; +const secret = "PRIVATE_SENTINEL_do_not_retain"; +function fixture(options: { ready?: boolean; durationMs?: number; click?: () => Promise | void; wait?: () => Promise } = {}) { + const events = new EventEmitter(); + let retained!: OnboardingConnectEvidence; + const page = Object.assign(events, { getByRole(_role: string, { name }: { name: string }) { + return { click: vi.fn(async () => { expect(name).toBe("Connect"); await options.click?.(); }), + waitFor: vi.fn(({ timeout }: { timeout: number }) => { + expect(name).toBe("Get started"); + if (options.wait) return options.wait(); + return options.ready ? Promise.resolve() : new Promise((_resolve, reject) => { + setTimeout(() => reject(Object.assign(new Error(secret), { name: "TimeoutError" })), timeout); + }); + }) }; + } }); + const request = (path = validation, method = "POST", requestOrigin = origin) => ({ url: () => requestOrigin + path, method: () => method }); + const respond = (body: unknown, status = 200, path = validation, method = "POST", requestOrigin = origin) => { + const req = request(path, method, requestOrigin); + const response = { request: () => req, status: () => status, body: () => Promise.resolve(Buffer.isBuffer(body) ? body : Buffer.from(JSON.stringify(body))) }; + events.emit("request", req); events.emit("response", response); + return response; + }; + const run = () => onboardingConnect({ page: page as any, origin, companyId, deadlineAt: Date.now() + (options.durationMs ?? 100), retain: value => { retained = value; } }); + return { page, events, request, respond, run, receipt: () => retained }; +} +afterEach(() => { vi.useRealTimers(); }); + +describe("one onboarding Connect diagnostic boundary", () => { + it("does not press Connect after the action deadline already expired", async () => { + const click = vi.fn(); + const f = fixture({ durationMs: 0, click }); + await expect(f.run()).rejects.toMatchObject({ code: "onboarding_connect_timeout" }); + expect(click).not.toHaveBeenCalled(); + expect(f.receipt().checkpoint).toBe("connect"); + expect(f.events.eventNames()).toEqual([]); + }); + + it("rejects a delayed failure body even after Get started becomes visible", async () => { + let resolveBody!: (bytes: Buffer) => void; + const body = new Promise(resolve => { resolveBody = resolve; }); + let reviewSeen!: () => void; + const review = new Promise(resolve => { reviewSeen = resolve; }); + const f = fixture({ wait: async () => { reviewSeen(); }, click: () => { + const req = f.request(); f.events.emit("request", req); + f.events.emit("response", { request: () => req, status: () => 200, body: () => body }); + } }); + const result = expect(f.run()).rejects.toMatchObject({ code: "onboarding_native_validation_failed" }); + await review; + for (let i = 0; i < 10 && f.events.listenerCount("response"); i++) await Promise.resolve(); + expect(f.events.listenerCount("response")).toBe(0); + resolveBody(Buffer.from(JSON.stringify({ status: "fail", checks: [] }))); + await result; + expect(f.receipt().validation.outcome).toBe("fail"); + expect(f.receipt().checkpoint).toBe("validation"); + expect(f.events.eventNames()).toEqual([]); + }); + + it.each(["details", "top-level"])("retains a known %s server rejection without its free text", async location => { + const f = fixture({ click: () => { f.respond({ error: secret, ...(location === "details" ? { details: { code: "agent_runner_config_incompatible", fields: [secret] } } : { code: "agent_runner_unavailable" }) }, 422); } }); + await expect(f.run()).rejects.toMatchObject({ code: "onboarding_native_validation_failed" }); + expect(f.receipt()).toMatchObject({ checkpoint: "validation", validation: { state: "json", httpStatus: 422, serverCode: location === "details" ? "agent_runner_config_incompatible" : "agent_runner_unavailable" }, hire: { state: "missing" } }); + expect(JSON.stringify(f.receipt())).not.toContain(secret); + validateOnboardingConnectEvidence(f.receipt()); + expect(f.events.listenerCount("response")).toBe(0); + }); + + it.each([{ status: "fail", checks: [] }, { status: "pass", checks: [{ code: "adapter_auth_missing", level: "error" }] }])("fails on a 200 validation rejection: %j", async body => { + const f = fixture({ click: () => { f.respond(body); } }); + await expect(f.run()).rejects.toMatchObject({ code: "onboarding_native_validation_failed" }); + expect(f.receipt().validation.httpStatus).toBe(200); + }); + + it("retains the hire boundary rejection and does not wait for Get started", async () => { + const f = fixture({ click: () => { f.respond({ details: { code: "agent_runner_platform_probe_failed" }, error: secret }, 503, hire); } }); + await expect(f.run()).rejects.toMatchObject({ code: "onboarding_agent_hire_failed" }); + expect(f.receipt()).toMatchObject({ checkpoint: "hire", hire: { httpStatus: 503, serverCode: "agent_runner_platform_probe_failed" } }); + }); + + it("projects only enums from malicious nested fields, including unknown codes and bounded checks", async () => { + const f = fixture({ ready: true, click: () => { f.respond({ adapterType: secret, status: secret, code: { credential: secret }, details: { code: secret }, + message: secret, hint: secret, headers: { authorization: secret }, checks: Array.from({ length: 40 }, () => ({ code: secret, level: secret, message: secret, nested: { apiKey: secret } })) }); } }); + const result = await f.run(); + expect(result.validation).toMatchObject({ adapter: "unrecognized", outcome: "unrecognized", serverCode: "unrecognized", truncatedChecks: true }); + expect(result.validation.checks).toHaveLength(32); + expect(result.validation.checks[0]).toEqual({ code: "unrecognized", level: "unrecognized" }); + expect(JSON.stringify(result)).not.toContain(secret); + validateOnboardingConnectEvidence(result); + expect(() => validateOnboardingConnectEvidence({ ...result, validation: { ...result.validation, requestedAdapter: ["codex_local"] } })).toThrow(); + }); + + it("ignores wrong origin, company, method, query, and responses without an in-action request", async () => { + vi.useFakeTimers(); + const f = fixture({ click: () => { + f.respond({}, 500, validation, "POST", "https://foreign.example.test"); + f.respond({}, 500, validation.replace(companyId, "company-2")); + f.respond({}, 500, validation, "GET"); + f.respond({}, 500, validation + "?private=" + secret); + f.events.emit("response", { request: () => f.request(), status: () => 500 }); + } }); + const result = expect(f.run()).rejects.toMatchObject({ code: "onboarding_get_started_timeout" }); + await vi.advanceTimersByTimeAsync(101); await result; + expect(f.receipt()).toMatchObject({ checkpoint: "connect", validation: { state: "missing" }, hire: { state: "missing" } }); + expect(f.events.eventNames()).toEqual([]); + }); + + it.each([Buffer.from("{not-json"), Buffer.alloc(65_537, "x")])("keeps malformed or oversized bodies unknown and still requires Get started", async bytes => { + vi.useFakeTimers(); + const f = fixture({ click: () => { f.respond(bytes); } }); + const result = expect(f.run()).rejects.toMatchObject({ code: "onboarding_get_started_timeout" }); + await vi.advanceTimersByTimeAsync(101); await result; + expect(f.receipt().validation.state).toBe(bytes.byteLength > 65_536 ? "oversized" : "unreadable"); + expect(f.receipt().checkpoint).toBe("validation"); + }); + + it("records missing responses and ignores late responses after timeout", async () => { + vi.useFakeTimers(); + const f = fixture({ click: () => { f.events.emit("request", f.request()); } }); + const result = expect(f.run()).rejects.toMatchObject({ code: "onboarding_get_started_timeout" }); + await vi.advanceTimersByTimeAsync(101); await result; + const sealed = JSON.stringify(f.receipt()); + f.respond({ status: "pass", adapterType: "paperclip_runner", checks: [] }); + expect(JSON.stringify(f.receipt())).toBe(sealed); + expect(f.receipt().validation.state).toBe("missing"); + }); + + it("bounds stalled JSON reads and cannot mutate retained evidence after return or reject unhandled", async () => { + vi.useFakeTimers(); + let rejectBody!: (reason: Error) => void; + const body = new Promise((_resolve, reject) => { rejectBody = reject; }); + const f = fixture({ ready: true, durationMs: 10_000, click: () => { + const req = f.request(); f.events.emit("request", req); + f.events.emit("response", { request: () => req, status: () => 200, body: () => body }); + } }); + const result = f.run(); + await vi.advanceTimersByTimeAsync(5001); + expect((await result).validation.state).toBe("unreadable"); + const sealed = JSON.stringify(f.receipt()); + rejectBody(new Error(secret)); await Promise.resolve(); + expect(JSON.stringify(f.receipt())).toBe(sealed); + expect(f.events.eventNames()).toEqual([]); + }); + + it("preserves the Get started timeout when a previously received body arrives later", async () => { + vi.useFakeTimers(); + let resolveBody!: (bytes: Buffer) => void; + const body = new Promise(resolve => { resolveBody = resolve; }); + const f = fixture(); + const result = expect(f.run()).rejects.toMatchObject({ code: "onboarding_get_started_timeout" }); + await vi.advanceTimersByTimeAsync(10); + const req = f.request(); f.events.emit("request", req); + f.events.emit("response", { request: () => req, status: () => 200, body: () => body }); + await vi.advanceTimersByTimeAsync(90); + resolveBody(Buffer.from(JSON.stringify({ status: "fail" }))); + await vi.advanceTimersByTimeAsync(1); await result; + expect(f.receipt().checkpoint).toBe("validation"); + expect(f.events.eventNames()).toEqual([]); + }); + + it("keeps a browser error at its own boundary without retaining the browser text", async () => { + const f = fixture({ click: () => { throw new Error(secret); } }); + await expect(f.run()).rejects.toMatchObject({ code: "onboarding_connect_failed" }); + expect(f.receipt().checkpoint).toBe("connect"); + expect(JSON.stringify(f.receipt())).not.toContain(secret); + expect(f.events.eventNames()).toEqual([]); + }); + + it("observes both successful responses and requires the visible review step", async () => { + const f = fixture({ ready: true, click: () => { + f.respond({ adapterType: "paperclip_runner", status: "pass", checks: [{ code: "codex_hello_probe_passed", level: "info", message: secret }] }, 200, validation.replace("paperclip_runner", "codex_local")); + f.respond({ agent: { id: secret }, approval: null }, 201, hire); + } }); + expect(await f.run()).toMatchObject({ checkpoint: "review", validation: { requestedAdapter: "codex_local", adapter: "paperclip_runner", outcome: "pass", checks: [{ code: "codex_hello_probe_passed", level: "info" }] }, hire: { state: "json", httpStatus: 201 } }); + expect(JSON.stringify(f.receipt())).not.toContain(secret); + expect(f.events.eventNames()).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/onboarding-connect.ts b/tests/runner-e2e/product-journeys/onboarding-connect.ts new file mode 100644 index 00000000000..4aa6e43becf --- /dev/null +++ b/tests/runner-e2e/product-journeys/onboarding-connect.ts @@ -0,0 +1,131 @@ +import type { Page, Response, Request } from "@playwright/test"; +import { JourneyError } from "./types.js"; + +export const onboardingServerCodes = ["agent_runner_config_incompatible", "agent_runner_unavailable", "agent_runner_platform_probe_failed", "managed_sandbox_unavailable", "environment_company_mismatch"] as const; +export const onboardingCheckCodes = ["adapter_auth_missing", "codex_hello_probe_failed", "codex_hello_probe_passed", "codex_hello_probe_timeout", "codex_hello_probe_auth_required", "environment_lease_acquire_failed", "environment_workspace_realize_failed", "environment_target_failed", "environment_target_unsupported", "environment_not_found", "environment_target_unavailable", "managed_sandbox_unavailable", "environment_env_binding_missing", "environment_env_resolution_failed", "ai_connection_api_key_rejected", "ai_connection_verification_failed", "sandbox_test_identity"] as const; +const checkpoints = ["connect", "validation", "hire", "review", "agent-verification"] as const; +const states = ["missing", "pending", "json", "unreadable", "oversized"] as const; +const requestedAdapters = ["codex_local", "paperclip_runner"] as const; +const adapters = ["codex_local", "paperclip_runner", "unrecognized"] as const; +const outcomes = ["pass", "warn", "fail", "unrecognized"] as const; +const levels = ["info", "warn", "error", "unrecognized"] as const; +type Known = T[number] | "unrecognized"; +export interface OnboardingResponseEvidence { + state: typeof states[number]; httpStatus: number | null; + requestedAdapter: "codex_local" | "paperclip_runner" | null; + adapter: typeof adapters[number] | null; outcome: typeof outcomes[number] | null; + serverCode: Known | null; + checks: Array<{ code: Known; level: typeof levels[number] }>; + truncatedChecks: boolean; +} +export interface OnboardingConnectEvidence { + version: 1; checkpoint: typeof checkpoints[number]; validation: OnboardingResponseEvidence; hire: OnboardingResponseEvidence; +} +const blank = (): OnboardingResponseEvidence => ({ state: "missing", httpStatus: null, requestedAdapter: null, adapter: null, outcome: null, serverCode: null, checks: [], truncatedChecks: false }); +const record = (v: unknown): v is Record => Boolean(v && typeof v === "object" && !Array.isArray(v)); +const member = (values: T, v: unknown): v is T[number] => typeof v === "string" && values.includes(v); +const closed = (v: unknown, keys: string[]) => record(v) && Object.keys(v).length === keys.length && keys.every(key => Object.hasOwn(v, key)); +export function validateOnboardingConnectEvidence(value: unknown): void { + if (!closed(value, ["version", "checkpoint", "validation", "hire"]) || !record(value) || value.version !== 1 || !member(checkpoints, value.checkpoint)) throw new Error("invalid_onboarding_connect_evidence"); + for (const response of [value.validation, value.hire]) { + if (!closed(response, ["state", "httpStatus", "requestedAdapter", "adapter", "outcome", "serverCode", "checks", "truncatedChecks"]) || !record(response) || + !member(states, response.state) || !(response.httpStatus === null || Number.isSafeInteger(response.httpStatus) && Number(response.httpStatus) >= 100 && Number(response.httpStatus) <= 599) || + !(response.requestedAdapter === null || member(requestedAdapters, response.requestedAdapter)) || + !(response.adapter === null || member(adapters, response.adapter)) || !(response.outcome === null || member(outcomes, response.outcome)) || + !(response.serverCode === null || response.serverCode === "unrecognized" || member(onboardingServerCodes, response.serverCode)) || + typeof response.truncatedChecks !== "boolean" || !Array.isArray(response.checks) || response.checks.length > 32 || + response.checks.some(check => !closed(check, ["code", "level"]) || !record(check) || !(check.code === "unrecognized" || member(onboardingCheckCodes, check.code)) || !member(levels, check.level))) throw new Error("invalid_onboarding_connect_evidence"); + } +} + +/** Observe only responses to this Connect action; never retain bodies, text, headers or browser errors. */ +export async function onboardingConnect(input: { page: Page; origin: string; companyId: string; deadlineAt: number; retain(value: OnboardingConnectEvidence): void }): Promise { + const { page } = input, receipt: OnboardingConnectEvidence = { version: 1, checkpoint: "connect", validation: blank(), hire: blank() }; + input.retain(receipt); + let active = true, failure: JourneyError | undefined; + let signalFailure!: (error: JourneyError) => void; + const failed = new Promise(resolve => { signalFailure = resolve; }); + const requests = new Set(), pending = new Set>(); + const deadlineAt = Math.min(input.deadlineAt, Date.now() + 180_000); + const remaining = () => Math.max(1, deadlineAt - Date.now()); + const advance = (checkpoint: OnboardingConnectEvidence["checkpoint"]) => { + if (checkpoints.indexOf(checkpoint) > checkpoints.indexOf(receipt.checkpoint)) receipt.checkpoint = checkpoint; + }; + const reject = (code: string) => { failure ??= new JourneyError("failed", code); signalFailure(failure); }; + const route = (request: Request): { kind: "validation" | "hire"; adapter: "codex_local" | "paperclip_runner" | null } | null => { + try { + const url = new URL(request.url()); + if (request.method() !== "POST" || url.origin !== input.origin || url.search || url.hash) return null; + const prefix = `/api/companies/${encodeURIComponent(input.companyId)}`; + if (url.pathname === `${prefix}/agent-hires`) return { kind: "hire", adapter: null }; + for (const adapter of ["codex_local", "paperclip_runner"] as const) if (url.pathname === `${prefix}/adapters/${adapter}/test-environment`) return { kind: "validation", adapter }; + } catch { /* An unrelated or malformed URL is not this action's evidence. */ } + return null; + }; + const onRequest = (request: Request) => { if (active && route(request)) requests.add(request); }; + const onResponse = (response: Response) => { + const request = response.request(), matched = route(request); + if (!active || !matched || !requests.delete(request)) return; + const observed = receipt[matched.kind]; + // One Connect action can report each boundary once. Later traffic cannot replace its failure. + if (observed.state !== "missing") return; + advance(matched.kind); observed.state = "pending"; + const status = response.status(); + observed.httpStatus = Number.isSafeInteger(status) && status >= 100 && status <= 599 ? status : null; + observed.requestedAdapter = matched.adapter; + if (response.status() >= 400) reject(matched.kind === "validation" ? "onboarding_native_validation_failed" : "onboarding_agent_hire_failed"); + const classify = async () => { + let timer: ReturnType | undefined; + try { + const bytes = await Promise.race([response.body(), new Promise(resolve => { timer = setTimeout(() => resolve(null), Math.min(5000, remaining())); })]); + if (!active) return; + if (!bytes) { observed.state = "unreadable"; return; } + if (bytes.byteLength > 65_536) { observed.state = "oversized"; return; } + let body: unknown; try { body = JSON.parse(bytes.toString("utf8")); } catch { observed.state = "unreadable"; return; } + if (!record(body)) { observed.state = "unreadable"; return; } + observed.state = "json"; + const detailCode = record(body.details) ? body.details.code : undefined; + const code = member(onboardingServerCodes, detailCode) ? detailCode : member(onboardingServerCodes, body.code) ? body.code : detailCode ?? body.code; + if (code !== undefined) observed.serverCode = member(onboardingServerCodes, code) ? code : "unrecognized"; + if (matched.kind === "validation") { + observed.adapter = member(adapters, body.adapterType) ? body.adapterType : "unrecognized"; + observed.outcome = member(outcomes, body.status) ? body.status : "unrecognized"; + if (Array.isArray(body.checks)) { + observed.truncatedChecks = body.checks.length > 32; + observed.checks = body.checks.slice(0, 32).map(check => ({ code: record(check) && member(onboardingCheckCodes, check.code) ? check.code : "unrecognized", level: record(check) && member(levels, check.level) ? check.level : "unrecognized" })); + if (body.status === "fail" || body.checks.some(check => record(check) && check.code === "adapter_auth_missing")) reject("onboarding_native_validation_failed"); + } else if (body.status === "fail") reject("onboarding_native_validation_failed"); + } + } catch { if (active) observed.state = "unreadable"; } + finally { clearTimeout(timer); } + }; + const work = classify(); pending.add(work); void work.then(() => pending.delete(work), () => pending.delete(work)); + }; + page.on("request", onRequest); page.on("response", onResponse); + try { + if (Date.now() >= deadlineAt) throw new JourneyError("failed", "onboarding_connect_timeout"); + const clicked = page.getByRole("button", { name: "Connect", exact: true }).click({ timeout: remaining() }) + .then(() => null, error => new JourneyError("failed", error?.name === "TimeoutError" ? "onboarding_connect_timeout" : "onboarding_connect_failed")); + const clickResult = await Promise.race([failed, clicked]); + if (clickResult) throw clickResult; + if (failure) throw failure; + const result = await Promise.race([failed, page.getByRole("button", { name: "Get started", exact: true }).waitFor({ timeout: remaining() }) + .then(() => null, error => new JourneyError("failed", error?.name === "TimeoutError" ? "onboarding_get_started_timeout" : "onboarding_get_started_failed"))]); + const boundaryFailure = failure; + // Seal the observed boundary before awaiting body reads. A response arriving + // after Get started or its timeout cannot change this action's outcome. + page.off("request", onRequest); page.off("response", onResponse); + if (pending.size) await Promise.all([...pending]); + if (boundaryFailure) throw boundaryFailure; + if (result) throw result; + if (failure) throw failure; + if (Date.now() >= deadlineAt) throw new JourneyError("failed", "onboarding_get_started_timeout"); + advance("review"); + return receipt; + } finally { + // Settle only already observed responses within their bounded read; detach before later traffic. + page.off("request", onRequest); page.off("response", onResponse); + if (pending.size) await Promise.all([...pending]); + active = false; requests.clear(); + } +} diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 65941f30d86..da0b054a606 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -28,6 +28,7 @@ export interface JourneyEvidence { attemptId?: string; operations?: JourneyOperations; artifact?: OnboardingArtifactEvidence; + onboardingConnect?: import("./onboarding-connect.js").OnboardingConnectEvidence; } export interface OnboardingArtifactEvidence { schema: "paperclip.product-journey.artifact/v1"; diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index 883d928c510..56f38ed0bec 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -4,6 +4,7 @@ import { githubJourneyCheckpoints, gradeGitHubReviewJourney } from "./github-rev import { STARTUP_REUSE_LAYERS, STARTUP_REUSE_REASONS } from "../../packages/adapter-utils/src/startup-reuse.js"; import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js"; import { DEPENDABLE_CELLS } from "./dependable-qualification.js"; +import { validateOnboardingConnectEvidence } from "./product-journeys/onboarding-connect.js"; import { validateOnboardingArtifact } from "./product-journeys/artifact-evidence.js"; type Rule = (value: unknown, at: string) => void; @@ -344,6 +345,7 @@ const fields = { productJourney: optional(shape({ targetId: optional(string), campaignId: optional(string), attemptId: optional(string), + onboardingConnect: optional((value, at) => { try { validateOnboardingConnectEvidence(value); } catch { invalid(at); } }), artifact: optional((value, at) => { try { validateOnboardingArtifact(value); } catch { invalid(at); } }), operations: optional(closedShape({ campaignId: string, attemptId: string, modelCostUsd: nullable(number), setupCostUsd: nullable(number), requestCount: integer, feedback: closedShape({ collection: oneOf("complete", "partial", "unavailable"), complaints: integer, suggestions: integer }), traceIds: array(string) })),