diff --git a/.agents/skills/verify-paperclip/SKILL.md b/.agents/skills/verify-paperclip/SKILL.md index 3bdc6e759e2..292fcf67c2a 100644 --- a/.agents/skills/verify-paperclip/SKILL.md +++ b/.agents/skills/verify-paperclip/SKILL.md @@ -54,6 +54,14 @@ npm install. Report only the combinations actually exercised. ## Deployment handoff +For hosted testing accounts, read [QA accounts and credentials](references/hosted-qa-accounts.md). +The testing environment has an AgentMail inbox on the allowed `paperclipbots.com` +domain and a persistent QA workspace at `https://qa.test.paperclip.app`. Use the +dedicated QA identity and normal email-code login. The expected 1Password service +account needs read and item-creation/update access; verify those capabilities +before promising that new credentials have been saved. Keep concrete secret +bindings and account receipts in the private eval configuration. + When a requested revision must first be built or deployed, use the skills in the private `paperclip-cloud` repo: diff --git a/.agents/skills/verify-paperclip/references/hosted-qa-accounts.md b/.agents/skills/verify-paperclip/references/hosted-qa-accounts.md new file mode 100644 index 00000000000..0e5426686ed --- /dev/null +++ b/.agents/skills/verify-paperclip/references/hosted-qa-accounts.md @@ -0,0 +1,128 @@ +# Hosted QA accounts and credentials + +Use this for testing Cloud login and tenant journeys. It is an environment setup +recipe, not a requirement for local or self-hosted verification. + +## Known testing setup + +- `https://test.paperclip.app` is the testing Cloud portal. The persistent tenant + is `https://qa.test.paperclip.app`. Keep testing distinct from staging and + production, including operator origins and secret namespaces. +- AgentMail supplies `qa@paperclipbots.com`. The custom `paperclipbots.com` domain + is admitted for testing sign-up. Verify admission through the actual customer + session when it matters; inbox access alone does not prove admission. +- A dedicated QA account and the persistent workspace already exist. Reuse them + for existing-user journeys. Account authentication, workspace creation, and + product onboarding are separate coverage claims. +- Hosted testing currently supports normal email-code authentication. A mailbox + key is an infrastructure credential; the short-lived sign-in code is not a + durable password. Do not fabricate a password for a passwordless account. +- The private Cloud Product E2E adapter owns the AgentMail and operator secret + references, registered stack/company IDs, leases, and cleanup. Locate its + `scripts/product-e2e/README.md` and reference-only target configuration. Use + that adapter for campaigns rather than inventing a second Cloud provisioner. + +The QA inbox credential can be inbox-scoped/read-only: domain listing or inbox +creation may return 403 while reading the configured QA inbox succeeds. An older +AgentMail key may expose only an `agentmail.to` inbox. That is not the allowed +custom-domain identity; do not silently substitute it. Read the configured inbox +directly. Create a new inbox only when the requested journey needs one and the +available account has that capability. + +The Cloud Product E2E credential is the rotation authority. A host-side 1Password +copy can become stale when that shared key rotates, even while browser cookies +still work. Probe the exact inbox and verify a fresh email-code session when +qualifying authentication. If the copy fails, reconcile it from the authorized +current source and verify read-back before creating another key. Record the +rotation owner and mirror refresh procedure in private setup; preserve the +original failed attempt. Use the declared staging AWS profile/region when a +one-time reconciliation reads the Cloud secret, and keep IAM access outside the +normal 1Password-only login dependency. + +For Dotta's QA journeys, reading the current verification email in the dedicated +test account's mailbox is an authorized part of sign-in, including Ralph's +GitHub device verification through existing AgentMail credentials. Do not ask +again for that same mailbox access on every run. Keep it scoped to the named QA +account, current login attempt, expected sender, and recipient; this does not +authorize reading unrelated mail. Declare the mailbox and its credential binding +as an explicit harness dependency rather than assuming the Paperclip QA inbox +also receives another account's codes. + +Ralph's dedicated verification binding is a separate, single-inbox, read-only +AgentMail key saved in 1Password, expiring **2027-10-23**. Resolve its current +item reference from the private Ponytail account registry; keep provider keys +out of this public reference. Normal GitHub password and current-device-email +verification worked without exporting a passkey. Use the supported ordinary +authentication flow; request assistance only if an additional factor actually +requires it. Expiry is a setup prerequisite and renewal responsibility, not a +reason to read another inbox or create credentials on every eval attempt. + +Cloud's shared GitHub OAuth App, the QA user's saved product connection, and a +journey-created review-bot App are separate resources. A 404 before GitHub consent +can be a registration prerequisite failure: compare the broker's actual client +ID/callback with the intended environment registration, then inspect whether the +App can be authorized by accounts outside its owner. A private App is not the +required public distribution for that setup. Verify the repaired OAuth handoff +and exact QA identity separately; changing visibility alone does not prove a +saved user connection or a working review bot. Keep one-time administrator repair +credentials outside normal per-attempt dependencies. Do not silently substitute +a personal token or another account for the requested connection route. + +## 1Password access and durable records + +The host is expected to supply `OP_SERVICE_ACCOUNT_TOKEN` for a designated QA +vault. It must permit reading required items and creating/updating credential +records when setup includes saving credentials. Successful reads do not prove +write permission. Save the intended account record and verify its returned item +ID; on denial, report the missing capability and retain the setup as incomplete. +Do not mark credentials saved, substitute a personal account, or change vault +permissions yourself. + +On Dotta's macOS host, the bootstrap token is stored in Keychain under service +`codex-1password`, account `$USER`. Read it with `/usr/bin/security +find-generic-password -a "$USER" -s "codex-1password" -w` inside a process that +captures stdout in memory and passes it only to the `op` child. A long-lived +shell's `OP_SERVICE_ACCOUNT_TOKEN` may be stale after rotation. Use the configured +bootstrap source; never run that command in a way that prints its result. CI +should inject its own service-account token through the runner's secret store. + +Use `op://` references in private setup profiles, with stable aliases such as +`paperclip-qa-user`, `agentmail-qa-inbox`, and `github-qa-user`. Bind each alias to +its account, consumer, required scopes, reference/field, and ownership. The +service-account token bootstraps the harness secret resolver; it never belongs +in a browser, product-agent environment, committed file, or evidence report. +Use the CLI's structured stdin input when creating an item. On macOS, `op` +may ignore Node socket-backed stdin; the private setup helper uses an actual +POSIX pipe and verifies saved fields rather than trusting an item ID. Keep secret values +out of shell arguments, transcripts, screenshots, and temporary handoff files. + +For a passwordless account, save the login URL, username, authentication method, +and reference to the approved mailbox credential. Save a real password only if +the product actually creates one. Preserve passkeys in their supported +authenticator and record that attended authentication may be required; a +1Password service account's ability to read a login item does not prove it can +exercise that item's passkey. + +## Authenticate, verify, and retain + +1. Read the exact QA inbox and credential bindings from private setup. Capture + message IDs already present before requesting a code. +2. Use the normal portal UI and dedicated QA identity. For a genuinely new + account, complete its sign-up flow and save its account record. For the + existing QA account, reuse normal sign-in without resetting credentials. +3. Read only the current attempt's verification mail. Match the configured + sender, exact recipient, unseen message ID, time window, and one unexpired + code. Keep the key and code in process memory; exclude authentication pages + and raw email from screenshots, traces, and reports. +4. Verify the authenticated principal and, if workspace creation is relevant, + `capabilities.canCreateOrganizations` through the customer session. Enter the + intended tenant and verify its actual company. A portal login alone does not + prove tenant access. +5. Record the account alias, credential-item reference, target, auth method, + fresh/reused state, and the observed outcome in private evidence. Keep pending + credential storage explicit. Retain the shared QA account, inbox, workspace, + and reusable repository; clean up only attempt-owned resources. + +Use the shared Product Journey target adapter for automated authentication and +ownership. Fixed UI scripts prove the actions they execute; an email login to an +existing account does not establish new-account onboarding coverage. diff --git a/doc/evals.md b/doc/evals.md index 9d89aa297eb..2c0af7e5ec7 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -1,5 +1,12 @@ # Paperclip evaluation guide +The explicit-only [GitHub review bot journey](../tests/runner-e2e/GITHUB-REVIEW-JOURNEY.md) +converts the Ponytail walkthrough into attended App setup, scoped installation, +automatic PR review, and independent two-service checks. It supports attached +self-hosted/Cloud targets and separately selected local/Daytona agents. Account +and credential bindings stay in the private eval repo; registration and oracle +tests do not establish a completed live pass or unattended CI qualification. + The explicit-only [necessary free-text question journey](../tests/runner-e2e/README.md#necessary-free-text-question) checks one essential input, the rendered prompt and context, the real answer, one bound continuation, and an exact saved confirmation. Its four development diff --git a/tests/runner-e2e/GITHUB-REVIEW-JOURNEY.md b/tests/runner-e2e/GITHUB-REVIEW-JOURNEY.md new file mode 100644 index 00000000000..ceeb56a95e0 --- /dev/null +++ b/tests/runner-e2e/GITHUB-REVIEW-JOURNEY.md @@ -0,0 +1,120 @@ +# GitHub review bot journey (explicit, attended) + +The `github-review-journey` suite converts the recorded Ponytail walkthrough into +fixed browser actions and an independent review oracle. It creates a GitHub App, +installs it on one private eval repository, saves automatic review configuration +with a selected skill and 5/5 passing threshold, opens a defective PR without a +bot mention, and checks the matching automatic task, review, GitHub delivery, +failed check, task completion, and persistence. + +The suite is applicable to self-hosted deployments with public GitHub webhook +ingress and to Paperclip Cloud with GitHub App support. Hosting is independent of +the configured agent's `local` or `daytona` execution environment. Both cells are +registered; select exactly one and bind it in the private setup profile. Browser +registration and oracle tests do not establish live qualification of either cell. + +## Dependencies and starting state + +Use a dedicated private browser profile outside the checkout. Supply the intended +company, serving SHA, deployment mode, existing agent/model/provider/runtime, +review skill, agent environment ID, GitHub account handle, App owner, and private +reusable repository. The GitHub login identifier may be an email; the handle is +separate. The agent and its provider connection must already be configured. +A matching existing GitHub user connection is required; fresh user OAuth and +agent hiring are outside this variant. App creation and installation remain +measured UI actions with attended consent/passkey boundaries. + +The browser, harness, and review agent need no external MCP tools. The agent uses +the task-bound GitHub review tools. The preexisting personal GitHub connection +is a separate product identity prerequisite; the product may store/expose it as +an MCP-backed connection. That does not supply MCP access to the harness or +review agent. The browser account controls the App and eval repository; the +agent consumes its own provider and task bindings. Harness access to 1Password +does not supply those product connections. + +Private setup profiles bind credential aliases to environment variable names or +selected `op://vault/item/field` references. A 1Password service-account token +goes only to the resolver's `op` child, and browser processes inherit a limited +environment. Never commit passwords, mailbox keys, passkeys, cookies, auth traces, +or browser profile contents. A private optional `authentication` module and +config file can prepare an ordinary hosted session before measured actions; +the private QA implementation reuses Cloud's email-code authentication helper. + +## Select and run + +```sh +pnpm test:e2e:runner -- --list --suite github-review-journey +pnpm test:e2e:runner -- \ + --id github-review-journey.configured-agent.daytona.setup-and-first-review \ + --github-review-config /private/ponytail-qa.json +``` + +The private definition, reference-only setup example, credential setup utilities, +and source-step mapping live in `paperclip-evals/evals/product-journeys/ponytail`. +Its wrapper supports the host's Keychain bootstrap without putting a token in +shell arguments. The suite is excluded from `--all` and generated CI matrices. +It has no scheduler, public upload, or automatic retry. A full setup needs +attended external authorization. Do not claim it is an unattended CI journey. +In a terminal, each creation/installation pause also accepts the exact displayed +action and unique App name. Inspect that attempt's permissions first. The setup +profile is not authorization: use the current task's actual authority, including +any explicitly granted standing permission for disposable QA resources. Do not +repeat an already covered approval; obey any required host approval boundary. + +## Oracle and ownership + +`cart-total.js` is a newly authored deterministic precision defect, not an +assertion that the source walkthrough had identical fixture bytes. A passing +eval requires a complete assessment of that path and head, a relevant inline +precision finding, a score below 5, and a correctly failed GitHub check. The +oracle rejects a stale/foreign company, endpoint, issue, PR, or head, manual +triggering, missing findings/delivery, and false completion. It does not require +the source's exact 3/5 score. Persisted skill selection proves binding; this +variant does not yet independently verify a runtime skill-read receipt. + +The rendered inline discussion must identify the created bot, `cart-total.js`, +the complete reviewed head, and the relevant precision finding. A summary that +mentions the defect does not substitute for inline publication. The review's +completed state alone is also insufficient: the actual run must succeed and the +associated task must reach Done. A native failure after publication fails the +journey and retains the publication evidence already observed. + +Each attempt records endpoint, App, PR/branch, task, and run ownership. Cleanup +removes the owned endpoint, closes the owned PR/deletes its branch, and asks for +the owned App's removal. App identity must match the created endpoint, company, +agent, exact name, GitHub-confirmed owner/slug, and immutable App ID. Installation +management is recorded separately: uninstalling alone does not prove App deletion. +It retains the shared repository, account, provider/user +connections, and completed task evidence. An unconfirmed cleanup fails the +attempt; retain the resource receipt for recovery. Read the receipt before any +manual cleanup. Never delete by a guessed name or broad account search. + +The configured deadline and observed run limit bound the attempt. They are not +a hard provider-dollar cap or proof that no request outlives cancellation. Use +the existing budgeted Cloud operations adapter before automated paid campaigns; +this attended attached-agent variant does not replace that infrastructure. +Effective company/agent budget policies are recorded when readable. A route +that reports tokens without a dollar cost can stop on `unpricedUsagePolicy: +block` despite a zero legacy spend counter. Any authorized exception is a +distinct tested setup condition; normal fixture setup must not silently grant +it. Restore temporary exceptions after verification and retain their receipts. + +The case declares a ten-minute provider-turn allowance within its fifteen-minute +body deadline. Read back the attached agent's effective timeout separately; +catalog metadata cannot increase a deployed agent's limit. If the target rejects +the required configuration or its lifecycle cannot settle, preserve that setup +failure and repair the fixture before claiming a qualified run. + +Private normalized results and the existing Product E2E dashboard retain actual +model/provider, selected environment, serving SHA, setup ID, source hash, +checkpoints, screenshots, observed usage, partial cost coverage, assistance, and +cleanup. Missing usage remains unknown. Authentication screens are excluded. + +`github-review-report.ts` uses the existing evidence sanitizer and manifest, +stages only declared screenshots, and verifies screenshot hashes and attempt +path ownership. The standard report inputs are `evidence/result.json`, +`evidence/evidence-manifest.json`, and `evidence/snapshots/`. Raw private operator +receipts are outside the manifest. Existing report tooling imports these files; +no second runner or report service is required. A presentation repair must use a +new directory, preserve the original result/hash and failure, and dispatch no +provider work. Rebuilding a dashboard never qualifies a failed live attempt. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 94b9f55ffa1..47853d3ba74 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1,5 +1,9 @@ # Paid runner full-stack E2E +The explicit-only [GitHub review bot journey](GITHUB-REVIEW-JOURNEY.md) tests +attended bot setup and an automatic PR review against an attached deployment. +Private account bindings and setup receipts live in the private eval repository. + ## Dependable-work qualification checkpoint `dependable-qualification.ts` defines the six everyday journeys across Codie diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 8fe1d5957fe..4f36708a091 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -219,10 +219,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, + 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(763); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(763); + expect(validateRunnerCatalog()).toHaveLength(765); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(765); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); @@ -519,13 +519,13 @@ describe("runner E2E catalog", () => { it("uses only declared secret references in generated payloads", () => { expect( - runnerMatrix.filter(entry => entry.task.flow !== "provider_connection").every((entry) => + runnerMatrix.filter(entry => !["provider_connection", "github_review_journey"].includes(entry.task.flow)).every((entry) => entry.requiredCredentials.includes(entry.profile.credential), ), ).toBe(true); expect( runnerMatrix - .filter((entry) => entry.environment.id === "daytona" && entry.task.flow !== "provider_connection") + .filter((entry) => entry.environment.id === "daytona" && !["provider_connection", "github_review_journey"].includes(entry.task.flow)) .every((entry) => entry.requiredCredentials.includes("DAYTONA_API_KEY"), ), diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index c6bc7b4d292..84c733fe035 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -19,6 +19,7 @@ import { NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_BASE_SHA, nativeInstructio import { nativeCompletionProfile, NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; import { chatConfirmationTasks } from "./chat-cases.js"; import { buildConnectionSuite } from "./connection-cases.js"; +import { buildGitHubReviewJourneySuite } from "./github-review-cases.js"; import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDigest } from "./hiring-template-cases.js"; import { copilotProviderDeathTasks } from "./copilot-active-stop-tasks.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; @@ -1800,6 +1801,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ } }, }, buildConnectionSuite(runnerProfiles, runnerEnvironments), + buildGitHubReviewJourneySuite(runnerProfiles, runnerEnvironments), { id: "repository-skills", label: "Repository skill packages", manualOnly: true, description: "Live private Poteto import and byte-for-byte delivery to local and Daytona agent runtimes.", @@ -1875,8 +1877,8 @@ export function buildRunnerMatrix( ]), ], requiredCredentials: [ - ...(task.flow === "provider_connection" ? [] : [profile.credential]), - ...(task.flow !== "provider_connection" && environment.credential ? [environment.credential] : []), + ...(["provider_connection", "github_review_journey"].includes(task.flow) ? [] : [profile.credential]), + ...(!["provider_connection", "github_review_journey"].includes(task.flow) && environment.credential ? [environment.credential] : []), ...(task.requiredCredentials ?? []), ], })) diff --git a/tests/runner-e2e/connection-launch.ts b/tests/runner-e2e/connection-launch.ts index 8781a2b4abd..99eb7a7a196 100644 --- a/tests/runner-e2e/connection-launch.ts +++ b/tests/runner-e2e/connection-launch.ts @@ -66,7 +66,7 @@ export async function cleanupConnectionCompany(input: { else for (const agent of fixtureAgents) await api.delete(`/api/agents/${agent.id}`); } -export async function openConnectionBrowser(config: ConnectionConfig, repositoryRoot: string, origin: string) { +export async function openConnectionBrowser(config: Pick, repositoryRoot: string, origin: string) { const temporary = config.browser.freshness === "signed-out" || !config.browser.profileDir; const profile = temporary ? await mkdtemp(path.join(os.tmpdir(), "paperclip-qa-browser-")) : path.resolve(config.browser.profileDir!); await assertOutsideRepository(profile, repositoryRoot); diff --git a/tests/runner-e2e/connection-target.ts b/tests/runner-e2e/connection-target.ts index 455c67d8c9e..fafecc2c0e4 100644 --- a/tests/runner-e2e/connection-target.ts +++ b/tests/runner-e2e/connection-target.ts @@ -92,11 +92,13 @@ export async function startConnectionTarget(config: ConnectionConfig, executions } catch (error) { await stop(); throw error; } } -export async function verifyConnectionTarget(origin: string, config: ConnectionConfig) { +export async function verifyConnectionTarget(origin: string, config: ConnectionConfig, request?: APIRequestContext) { let health: any; try { - const response = await fetch(`${targetOrigin(origin)}/api/health`, { redirect: "error", signal: AbortSignal.timeout(15_000) }); - if (!response.ok) throw new ConnectionFailure(); + const response = request + ? await request.get(`${targetOrigin(origin)}/api/health`, { maxRedirects: 0, timeout: 15_000 }) + : await fetch(`${targetOrigin(origin)}/api/health`, { redirect: "error", signal: AbortSignal.timeout(15_000) }); + if (!(typeof response.ok === "function" ? response.ok() : response.ok)) throw new ConnectionFailure(); health = await response.json(); } catch { throw new ConnectionBlock("blocked_target", "target_health_unavailable_or_redirected"); } if (typeof health.commit !== "string" || !/^[a-f0-9]{40}$/.test(health.commit)) throw new ConnectionBlock("blocked_target", "target_revision_unavailable"); diff --git a/tests/runner-e2e/github-review-attendance.ts b/tests/runner-e2e/github-review-attendance.ts new file mode 100644 index 00000000000..8bfcd067346 --- /dev/null +++ b/tests/runner-e2e/github-review-attendance.ts @@ -0,0 +1,31 @@ +import { createInterface } from "node:readline"; +import type { Readable } from "node:stream"; +import type { Locator } from "@playwright/test"; +import { ConnectionBlock } from "./connection-config.js"; +import { connectionDelay } from "./connection-target.js"; + +/** An attended operator can use the QA browser or confirm its displayed action + * in this run's terminal. Approval is never saved in the reusable profile. */ +export async function attendGitHubJourneyAction(input: { + control: Locator; action: "create-app" | "install-app"; appName: string; + signal: AbortSignal; timeoutMs: number; +}, operatorInput: Readable & { isTTY?: boolean } = process.stdin) { + const command = `${input.action} ${input.appName}`; + const terminal = operatorInput.isTTY ? createInterface({ input: operatorInput, terminal: false }) : undefined; + let confirmed = false; + terminal?.on("line", line => { if (line.trim() === command) confirmed = true; }); + console.log(`[github-review-journey] Inspect the displayed App permissions. Complete the action in the QA browser${terminal ? `, or type exactly: ${command}` : "."}`); + const deadline = Date.now() + input.timeoutMs; + try { + while (Date.now() < deadline) { + if (input.signal.aborted) throw new ConnectionBlock("blocked_target", "journey_interrupted"); + if (!await input.control.isVisible()) return; + if (confirmed) { + await input.control.click(); + return; + } + await connectionDelay(500); + } + throw new ConnectionBlock("awaiting_user", `${input.action.replaceAll("-", "_")}_consent_required`); + } finally { terminal?.close(); } +} diff --git a/tests/runner-e2e/github-review-browser.ts b/tests/runner-e2e/github-review-browser.ts new file mode 100644 index 00000000000..dffb516bcdd --- /dev/null +++ b/tests/runner-e2e/github-review-browser.ts @@ -0,0 +1,181 @@ +import { expect, type Page } from "@playwright/test"; +import { ConnectionBlock, ConnectionFailure } from "./connection-config.js"; +import { connectionDelay } from "./connection-target.js"; +import type { GitHubReviewJourneyConfig } from "./github-review-config.js"; + +export async function enterGitHubJourneyCompany(page: Page, origin: string, company: { issuePrefix: string }) { + if (!/^[A-Za-z0-9]+$/.test(company.issuePrefix)) throw new ConnectionFailure("invalid_journey_company_prefix"); + const pathname = `/${company.issuePrefix}/dashboard`; + await page.goto(`${origin}${pathname}`); + await page.waitForURL(url => url.origin === origin && url.pathname === pathname); +} + +/** Capture measured pages only, excluding authentication and unrelated GitHub data. */ +export function isGitHubJourneyEvidencePage(value: string, targetOrigin: string, repository: string, appSlug?: string) { + const url = new URL(value); + if ([...url.searchParams.keys()].some(key => /token|secret|code|state|password/i.test(key))) return false; + if (url.origin === targetOrigin) return !/connect|login|auth/.test(url.pathname); + if (url.origin !== "https://github.com") return false; + const prefix = `/${repository}/`; + if (url.pathname.startsWith(prefix) && /^(?:pull\/\d+(?:\/(?:files|commits|checks))?|runs\/\d+|actions\/runs\/\d+(?:\/job\/\d+)?)\/?$/.test(url.pathname.slice(prefix.length))) return true; + return Boolean(appSlug && (url.pathname === `/apps/${appSlug}/installations/new` || url.pathname === `/apps/${appSlug}/installations/new/permissions`)); +} + +export async function awaitJourneyCondition(check: () => Promise, timeoutMs: number, code: string, signal: AbortSignal, boundary: "awaiting_user" | "failed" = "awaiting_user") { + const end = Date.now() + timeoutMs; + while (Date.now() < end) { + if (signal.aborted) throw new ConnectionBlock("blocked_target", "journey_interrupted"); + if (await check()) return; + await connectionDelay(750); + } + if (boundary === "failed") throw new ConnectionFailure(code); + throw new ConnectionBlock("awaiting_user", code); +} + +export async function observeGitHubJourneyAccount(page: Page) { + const menu = page.getByRole("button", { name: /Open user navigation menu|View profile and more/i }); + const dialog = page.getByRole("dialog", { name: "User navigation", exact: true }); + await menu.click(); + try { await dialog.waitFor({ timeout: 3_000 }); } + catch (error) { + // GitHub can replace its server-rendered header during hydration. Retry + // once only when the observed control still explicitly says Open. + if (!await menu.isVisible() || await dialog.isVisible()) throw error; + await menu.click(); + await dialog.waitFor(); + } + const profile = dialog.getByRole("link", { name: "Profile", exact: true }); + const href = await profile.getAttribute("href"); + const url = new URL(href ?? "", "https://github.com"); + const account = url.origin === "https://github.com" ? /^\/([A-Za-z0-9-]+)\/?$/.exec(url.pathname)?.[1] : undefined; + if (!account) throw new ConnectionFailure("github_profile_identity_missing"); + await page.keyboard.press("Escape"); + return account; +} + +/** Prepare one repository grant; submitting Install remains a separate action. */ +export async function selectGitHubJourneyRepository(page: Page, repository: string) { + const selectedOnly = page.getByRole("radio", { name: /Only select repositories/i }); + await selectedOnly.check(); + await page.getByRole("button", { name: /Select repositories/i }).click(); + await page.getByRole("textbox", { name: "Search for a repository", exact: true }).fill(repository.split("/")[1]!); + const name = new RegExp(`^${repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?:\\s|$)`); + // GitHub nests the selectable repository item inside a containing menu item. + await page.getByRole("menuitem", { name }).last().click(); + await page.keyboard.press("Escape"); + await expect(selectedOnly).toBeChecked(); + await expect(page.getByRole("radio", { name: /^All repositories/ })).not.toBeChecked(); + await expect(page.getByRole("main")).toContainText("Selected 1 repository."); + await expect(page.getByRole("button", { name: `Remove ${repository.split("/")[1]}`, exact: true })).toBeVisible(); + await expect(page.getByRole("button", { name: "Install", exact: true })).toBeEnabled(); +} + +/** Login remains ordinary browser UI. Passkeys are never exported or emulated. */ +export async function ensureGitHubJourneyLogin(input: { + page: Page; config: GitHubReviewJourneyConfig; username: string; password?: () => Promise; signal: AbortSignal; + checkpoint(id: string): Promise; +}) { + const { page, config } = input; + const signedIn = () => page.getByRole("button", { name: /Open user navigation menu|View profile and more/i }).isVisible(); + if (!await signedIn()) { + const loginLink = page.getByRole("link", { name: /^Sign in$/ }); + if (await loginLink.isVisible()) await loginLink.click(); + if (config.github.login === "password") { + await page.getByLabel("Username or email address", { exact: true }).fill(input.username); + await page.getByLabel("Password", { exact: true }).fill(await input.password!()); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + } + await input.checkpoint("awaiting_github_sign_in"); + console.log("[github-review-journey] Complete Ralph QA GitHub sign-in/passkey in the dedicated QA browser. No credential needs to be pasted into chat."); + await awaitJourneyCondition(signedIn, config.browser.loginTimeoutMs, "github_login_required", input.signal); + } + // The login identifier can be an email address; verify the GitHub handle. + if ((await observeGitHubJourneyAccount(page)).toLowerCase() !== config.github.account.toLowerCase()) throw new ConnectionBlock("blocked_target", "github_account_mismatch"); +} + +/** Explicit one-time bootstrap. Later attempts reuse this private repository. */ +export async function createPrivateJourneyRepository(page: Page, repository: string) { + const [owner, name] = repository.split("/"); + await page.getByRole("button", { name: /Create new/i }).click(); + await page.getByRole("menuitem", { name: /New repository/i }).click(); + const ownerButton = page.getByRole("button", { name: /Owner|Choose an owner/i }); + await expect(ownerButton).toHaveText(owner!); + await page.getByRole("textbox", { name: /Repository name/i }).fill(name!); + await expect(page.getByText(`${name} is available.`, { exact: true })).toBeVisible(); + const privateChoice = page.getByRole("radio", { name: /Private/ }); + if (await privateChoice.isVisible()) await privateChoice.check(); + else { + await page.getByRole("button", { name: /Choose visibility|Public/i }).click(); + await page.getByRole("menuitemradio", { name: /Private/ }).click(); + } + await expect(page.getByRole("button", { name: "Choose visibility: Private", exact: true })).toBeVisible(); + const readme = page.getByRole("button", { name: "Add README", exact: true }); + if (await readme.getAttribute("aria-pressed") !== "true") await readme.click(); + await expect(readme).toHaveAttribute("aria-pressed", "true"); + await page.getByRole("button", { name: "Create repository", exact: true }).click(); + await page.waitForURL(url => url.origin === "https://github.com" && url.pathname === `/${repository}`); + await expect(page.getByRole("link", { name: name!, exact: true }).first()).toBeVisible(); + await expect(page.getByText("Private", { exact: true }).first()).toBeVisible(); +} + +export async function createGitHubJourneyPullRequest(input: { page: Page; branch: string; title: string; contents: string }) { + const { page } = input; + // The initial file list is server-rendered before GitHub's controls hydrate. + // Its loaded commit receipt provides a visible readiness signal. + await expect(page.getByRole("link", { name: /^Commit [a-f0-9]{7,40}$/ })).toBeVisible(); + await page.getByRole("button", { name: "Add file", exact: true }).click(); + await page.getByRole("menuitem", { name: "Create new file", exact: true }).click(); + await page.getByRole("textbox", { name: /Name your file|File name/i }).fill("cart-total.js"); + await page.getByRole("textbox", { name: /Editing cart-total.js file contents|File contents/i }).fill(input.contents); + await page.getByRole("button", { name: /Commit changes/i }).click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("textbox", { name: /Commit message/i }).fill("Add isolated cart precision review fixture"); + await dialog.getByRole("radio", { name: /Create a new branch/i }).check(); + const branch = dialog.getByRole("textbox", { name: "PR target branch", exact: true }); + await branch.fill(input.branch); + await expect(branch).toHaveValue(input.branch); + await dialog.getByRole("button", { name: /Propose changes|Commit changes/i }).click(); + await page.getByRole("textbox", { name: /Add a title/i }).fill(input.title); + await page.getByRole("textbox", { name: /Add a description|Comment|Description/i }).fill("Disposable automated review fixture. This money-total helper promises exact cent totals. Review only; do not modify or merge the branch. No manual bot mention is used."); + await page.getByRole("button", { name: "Create pull request", exact: true }).click(); + await page.waitForURL(url => url.origin === "https://github.com" && /\/pull\/\d+$/.test(url.pathname)); + const match = /\/pull\/(\d+)$/.exec(new URL(page.url()).pathname); + if (!match) throw new ConnectionFailure("created_pr_url_missing"); + return Number(match[1]); +} + +export async function observeGitHubHead(page: Page) { + const pull = new URL(page.url()); + const repository = /^\/([^/]+\/[^/]+)\/pull\/\d+$/.exec(pull.pathname)?.[1]; + if (pull.origin !== "https://github.com" || !repository) throw new ConnectionFailure("github_pr_evidence_missing"); + const commits = page.getByRole("link", { name: /^Commits(?:\s*\(?\d+\)?)?$/ }); + await expect(commits).toBeVisible(); + const commitsUrl = new URL(await commits.getAttribute("href") ?? "", pull.origin); + if (commitsUrl.origin !== pull.origin || commitsUrl.pathname !== `${pull.pathname}/commits`) throw new ConnectionFailure("github_commits_navigation_missing"); + // Follow the rendered navigation link. A click immediately after PR creation + // can be swallowed when GitHub replaces its server-rendered navigation. + await page.goto(commitsUrl.href); + const commit = page.getByRole("link", { name: /^[a-f0-9]{7,40}$/ }).last(); + await expect(commit).toBeVisible(); + const url = new URL(await commit.getAttribute("href") ?? "", pull.origin); + const sha = /\/(?:commits?|changes)\/([a-f0-9]{40})$/.exec(url.pathname)?.[1]; + if (url.origin !== pull.origin || !sha || ![`/${repository}/commit/${sha}`, `${pull.pathname}/commits/${sha}`, `${pull.pathname}/changes/${sha}`].includes(url.pathname)) throw new ConnectionFailure("github_head_evidence_missing"); + const conversation = page.getByRole("link", { name: /^Conversation(?:\s*\(?\d+\)?)?$/ }); + await expect(conversation).toBeVisible(); + const conversationUrl = new URL(await conversation.getAttribute("href") ?? "", pull.origin); + if (conversationUrl.origin !== pull.origin || conversationUrl.pathname !== pull.pathname) throw new ConnectionFailure("github_conversation_navigation_missing"); + await page.goto(conversationUrl.href); + return sha; +} + +export async function closeOwnedGitHubPullRequest(page: Page) { + const closed = page.getByText("Closed", { exact: true }).first(); + if (!await closed.isVisible()) { + await expect(page.getByRole("button", { name: /^(?:Merge status|Status)$/ })).toBeEnabled(); + await page.getByRole("button", { name: "Close pull request", exact: true }).click(); + } + await expect(closed).toBeVisible({ timeout: 30_000 }); + const restored = page.getByRole("button", { name: "Restore branch", exact: true }); + if (!await restored.isVisible()) await page.getByRole("button", { name: "Delete branch", exact: true }).click(); + await expect(restored).toBeVisible({ timeout: 30_000 }); +} diff --git a/tests/runner-e2e/github-review-cases.ts b/tests/runner-e2e/github-review-cases.ts new file mode 100644 index 00000000000..614d11ba157 --- /dev/null +++ b/tests/runner-e2e/github-review-cases.ts @@ -0,0 +1,23 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { EnvironmentFixture, RunnerProfileFixture, RunnerSuiteFixture } from "./types.js"; + +export function buildGitHubReviewJourneySuite(profiles: readonly RunnerProfileFixture[], environments: readonly EnvironmentFixture[]): RunnerSuiteFixture { + const base = profiles.find(p => p.id === "runner-codex")!; + const profile: RunnerProfileFixture = { ...base, id: "configured-agent", label: "Existing review agent (profile-bound)", groups: [], + model: "configured-in-journey-profile", provider: "configured-in-journey-profile", supportedEnvironments: ["local", "daytona"], + modelQualification: { source: "candidate_runner_profile", qualificationId: "explicit-attached-agent-profile" }, + buildAgent: () => { throw new Error("GitHub review journey uses the explicitly bound existing agent"); } }; + const hash = createHash("sha256"); + for (const file of ["github-review-cases.ts", "github-review-config.ts", "github-review-browser.ts", "github-review-attendance.ts", "github-review-flow.ts", "github-review-launch.ts", "github-review-report.ts", "github-review-evidence.ts", "github-review-ownership.ts", "journey-dependencies.ts"]) hash.update(file).update(readFileSync(new URL(`./${file}`, import.meta.url))); + return { id: "github-review-journey", label: "GitHub review bot setup and first review", groups: [], manualOnly: true, + description: "Attended real GitHub App setup, repository-scoped installation, automatic PR review, independent publication and task checks. Reuses a private QA repository.", + profiles: [profile], environments, expectedMatrixSize: 2, + definitionMetadata: { version: 1, digest: hash.digest("hex"), scheduling: "explicit-only", sourceJourneySha256: "2a504ec0b68c42d597a20fad5fc3bb8d7086ab5c24d34fc75c2a031f8559ab40", + applicability: ["self-hosted-with-public-github-ingress", "paperclip-cloud-with-github-app-support"], mcpConnections: [], + dependencies: [{ alias: "github-user", consumer: "browser", required: true }, { alias: "github-verification-mailbox", consumer: "harness", phase: "setup-profile-dependent" }, { alias: "review-agent", consumer: "product-agent", required: true }, + { alias: "github-user-connection", consumer: "browser-identity-verification", required: true }, { alias: "review-bot-app", consumer: "product-agent", phase: "created-in-journey" }], authentication: "dedicated-browser-with-profile-bound-authentication", providerCredential: "existing-product-agent-binding" }, + tasks: [{ id: "setup-and-first-review", label: "Create App, configure Ponytail skill, open PR, verify automatic review", groups: [], flow: "github_review_journey", workMode: "standard", + expectedRunCount: 3, minimumExpectedRunCount: 1, automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 1_800_000, daytona: 1_800_000 }, turnTimeoutMs: 600_000, + expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: n => `GitHub review journey ${n}`, buildPrompt: () => "Real GitHub opened event; never dispatched by fixture.", buildVisibleMarker: n => n, buildMatchers: () => [] }] }; +} diff --git a/tests/runner-e2e/github-review-config.ts b/tests/runner-e2e/github-review-config.ts new file mode 100644 index 00000000000..abbee09e79f --- /dev/null +++ b/tests/runner-e2e/github-review-config.ts @@ -0,0 +1,69 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { ConnectionBlock, parseConnectionConfig, targetOrigin, type ConnectionConfig } from "./connection-config.js"; +import { parseJourneyCredentialBinding, type JourneyCredentialBinding } from "./journey-dependencies.js"; + +export interface GitHubReviewJourneyConfig { + version: 1; + setupId: string; + hosting: "self-hosted" | "paperclip-cloud"; + executionEnvironment: "local" | "daytona"; + target: Extract & { companyId: string; environmentId: string }; + browser: ConnectionConfig["browser"]; + authentication?: { module: string; configFile: string }; + agent: { id: string; name: string; skill: string; model: string; provider: string; runtimeMode: "native" | "legacy" }; + github: { repository: string; ownerType: "personal" | "organization"; appOwner: string; account: string; login: "passkey" | "password"; username: JourneyCredentialBinding; password?: JourneyCredentialBinding }; + deadlineMs: number; + maximumRuns: number; +} + +const invalid = (): never => { throw new ConnectionBlock("blocked_target", "invalid_github_journey_configuration"); }; +function object(v: unknown, keys: string[]): Record { + if (!v || typeof v !== "object" || Array.isArray(v) || Object.keys(v).some(k => !keys.includes(k))) return invalid(); + return v as Record; +} +function text(v: unknown, pattern?: RegExp): string { + if (typeof v !== "string" || !v.trim() || v.length > 1024 || (pattern && !pattern.test(v))) return invalid(); + return v; +} +export function parseGitHubReviewJourneyConfig(input: unknown): GitHubReviewJourneyConfig { + try { + const v = object(input, ["version", "setupId", "hosting", "executionEnvironment", "target", "browser", "authentication", "agent", "github", "deadlineMs", "maximumRuns"]); + if (v.version !== 1 || !["self-hosted", "paperclip-cloud"].includes(v.hosting)) return invalid(); + if (v.executionEnvironment !== undefined && !["local", "daytona"].includes(v.executionEnvironment)) return invalid(); + const target = object(v.target, ["mode", "baseURL", "expectedCommit", "deploymentMode", "companyId", "environmentId"]); + if (target.mode !== "attach") return invalid(); + const common = parseConnectionConfig({ target, browser: v.browser }); + if (common.target.mode !== "attach" || !common.browser.profileDir) return invalid(); + const agent = object(v.agent, ["id", "name", "skill", "model", "provider", "runtimeMode"]); + const github = object(v.github, ["repository", "ownerType", "appOwner", "account", "login", "username", "password"]); + if (!["personal", "organization"].includes(github.ownerType) || !["passkey", "password"].includes(github.login) + || !["native", "legacy"].includes(agent.runtimeMode) || (github.login === "password" && !github.password)) return invalid(); + const deadlineMs = v.deadlineMs ?? 900_000; const maximumRuns = v.maximumRuns ?? 3; + if (!Number.isInteger(deadlineMs) || deadlineMs < 60_000 || deadlineMs > 1_800_000 + || !Number.isInteger(maximumRuns) || maximumRuns < 1 || maximumRuns > 5) return invalid(); + const uuid = /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/; + const authentication = v.authentication === undefined ? undefined : object(v.authentication, ["module", "configFile"]); + if (authentication && (!path.isAbsolute(text(authentication.module)) || !path.isAbsolute(text(authentication.configFile)))) return invalid(); + return { version: 1, setupId: text(v.setupId, /^[a-z0-9-]{1,80}$/), hosting: v.hosting, executionEnvironment: v.executionEnvironment ?? "local", + target: { ...common.target, baseURL: targetOrigin(common.target.baseURL), companyId: text(target.companyId, uuid), environmentId: text(target.environmentId, uuid) }, browser: common.browser, + ...(authentication ? { authentication: { module: authentication.module, configFile: authentication.configFile } } : {}), + agent: { id: text(agent.id, uuid), name: text(agent.name), skill: text(agent.skill, /^[a-z0-9-]+$/), model: text(agent.model), provider: text(agent.provider), runtimeMode: agent.runtimeMode }, + github: { repository: text(github.repository, /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/), ownerType: github.ownerType, appOwner: text(github.appOwner, /^[A-Za-z0-9-]+$/), + account: text(github.account ?? (github.ownerType === "personal" ? github.appOwner : undefined), /^[A-Za-z0-9-]+$/), login: github.login, + username: parseJourneyCredentialBinding(github.username), ...(github.password ? { password: parseJourneyCredentialBinding(github.password) } : {}) }, deadlineMs, maximumRuns }; + } catch { return invalid(); } +} + +export async function loadGitHubReviewJourneyConfig(file?: string) { + if (!file) throw new ConnectionBlock("blocked_target", "github_journey_config_required"); + try { + const value = JSON.parse(await readFile(file, "utf8")); + if (value.browser?.profileDir?.startsWith("~/")) value.browser.profileDir = path.join(os.homedir(), value.browser.profileDir.slice(2)); + // Profile paths are relative to the profile, not the invoking checkout. + if (value.browser?.profileDir && !path.isAbsolute(value.browser.profileDir)) value.browser.profileDir = path.resolve(path.dirname(file), value.browser.profileDir); + for (const key of ["module", "configFile"]) if (value.authentication?.[key] && !path.isAbsolute(value.authentication[key])) value.authentication[key] = path.resolve(path.dirname(file), value.authentication[key]); + return parseGitHubReviewJourneyConfig(value); + } catch { throw new ConnectionBlock("blocked_target", "invalid_github_journey_configuration"); } +} diff --git a/tests/runner-e2e/github-review-evidence.ts b/tests/runner-e2e/github-review-evidence.ts new file mode 100644 index 00000000000..f2f6cda9588 --- /dev/null +++ b/tests/runner-e2e/github-review-evidence.ts @@ -0,0 +1,104 @@ +import { GITHUB_REVIEW_EVENTS, type GitHubReviewPolicy, type GitHubTaskReview } from "../../packages/shared/src/types/chat-github.js"; +import type { Matcher } from "./types.js"; + +export const githubJourneyCheckpoints = ["target", "dependencies", "agent", "app_creation_consent", "app_created", "repository_scope", "identity", "configuration_saved", "configuration_reloaded", "pr_created", "automatic_task", "skill_binding", "review_completed", "github_delivery", "github_inline_delivery", "check_projection", "task_done", "persisted"] as const; +export interface GitHubReviewJourneyEvidence { + version: 1; + outcome: "running" | "passed" | "failed" | "awaiting_user" | "missing_credential" | "blocked_target"; + phase: string; + setupId: string; + hosting: "self-hosted" | "paperclip-cloud"; + targetCommit: string | null; + assisted: boolean; + dependencies: Array<{ alias: string; consumer: "harness" | "browser" | "product-agent"; status: "ready" | "unverified" | "created-in-journey"; source: "onepassword" | "environment" | "browser-session" | "product-connection" }>; + checkpoints: Partial>; + cleanup: { endpoint: boolean; pullRequest: boolean; branch: boolean; app: boolean }; + definitionHash?: string; + score?: number; + conclusion?: string; + budgetPolicyRead?: "verified" | "unavailable"; + budgetPolicies?: Array<{ scopeType: string; amount: number; hardStopEnabled: boolean; unpricedUsagePolicy: string }>; + execution?: { reviewId: string; issueId: string; runId: string | null; runStatus?: string; issueStatus?: string; runtimeMode?: string; environmentId?: string; provider?: string; model?: string }; + review?: { id: string; repository: string; pullNumber: number; headSha: string; state: string; score?: number; conclusion: string | null; summaryUrl: string | null; checkUrl: string | null }; + inlinePublication?: { url: string; fileUrl: string; path: string; author: string; text: string }; +} + +export function retainGitHubReview(review: GitHubTaskReview): NonNullable { + return { id: review.id, repository: review.repository, pullNumber: review.pullNumber, headSha: review.headSha, + state: review.state, score: review.assessment?.score, conclusion: review.conclusion, summaryUrl: review.summaryUrl, checkUrl: review.checkUrl }; +} + +export function gradeGitHubReviewJourney(evidence: GitHubReviewJourneyEvidence) { + return [...githubJourneyCheckpoints.map(id => ({ matcher: { kind: "json_path", path: `githubReviewJourney.checkpoints.${id}`, expected: true } as Matcher, + passed: evidence.checkpoints[id] === true, detail: `${id}: ${evidence.checkpoints[id] === true ? "verified" : "missing"}` })), + { matcher: { kind: "json_path", path: "githubReviewJourney.cleanup", expected: true } as Matcher, + passed: Object.values(evidence.cleanup).every(Boolean), detail: "All attempt-owned resources reconciled; reusable repository retained." }]; +} + +/** Mentions and comment replies are manual intake, separate from automatic PR events. */ +export function hasOnlyOpenedAutomaticReview(policy: Pick) { + const automatic = policy.events.filter(event => GITHUB_REVIEW_EVENTS.slice(0, 4).includes(event)); + return automatic.length === 1 && automatic[0] === "opened" && policy.issueOpened !== true; +} + +export function matchesGitHubJourneyEnvironment(environment: { id: string; driver: string; config: Record }, id: string, kind: "local" | "daytona") { + return environment.id === id && (kind === "local" ? environment.driver === "local" + : environment.driver === "sandbox" && environment.config.provider === "daytona"); +} + +export function verifyGitHubReviewRun(run: Record | undefined, expected: { + runId: string | null; companyId: string; agentId: string; issueId: string; + runtimeMode: string; environmentId: string; executionEnvironment: "local" | "daytona"; + environment: { id: string; driver: string; config: Record }; provider: string; model: string; +}) { + const provider = run?.runnerProfileJson?.nativeExecutionInput?.provider; + return Boolean(run && run.id === expected.runId && run.companyId === expected.companyId + && run.agentId === expected.agentId && run.issueId === expected.issueId + && run.status === "succeeded" && run.runtimeMode === expected.runtimeMode + && matchesGitHubJourneyEnvironment(expected.environment, expected.environmentId, expected.executionEnvironment) + && run.contextSnapshot?.paperclipEnvironment?.id === expected.environmentId + && run.contextSnapshot?.paperclipEnvironment?.driver === expected.environment.driver + && (expected.runtimeMode !== "native" || (provider?.kind === expected.provider && provider?.model === expected.model))); +} + +function ownedDeliveryUrl(value: string | null, repository: string, pullNumber: number, kind: "summary" | "check") { + if (!value) return false; + try { + const url = new URL(value); + if (url.origin !== "https://github.com" || url.username || url.password) return false; + const prefix = `/${repository}/`; + if (!url.pathname.startsWith(prefix)) return false; + const path = url.pathname.slice(prefix.length); + return kind === "summary" + ? (path === `pull/${pullNumber}` || path === `issues/${pullNumber}`) && /^#(?:issuecomment|pullrequestreview)-\d+$/.test(url.hash) + : /^(?:runs\/\d+|actions\/runs\/\d+(?:\/job\/\d+)?)$/.test(path); + } catch { return false; } +} + +/** Independent identity/cause/output gates; a Done task or success prose cannot satisfy them. */ +export function verifyGitHubReview(input: { + review: GitHubTaskReview; companyId: string; endpointId: string; issueId: string; repository: string; pullNumber: number; headSha: string; + issue: { id: string; companyId: string; assigneeAgentId: string; status: string }; agentId: string; +}) { + const { review: r } = input; const assessment = r.assessment; + return r.companyId === input.companyId && r.endpointId === input.endpointId && r.issueId === input.issueId + && r.repository === input.repository && r.pullNumber === input.pullNumber && r.headSha === input.headSha + && r.event.event === "opened" && r.event.headSha === input.headSha && r.event.repository === input.repository + && r.state === "completed" && assessment?.complete === true && assessment.reviewedCommit === input.headSha + && assessment.coverage.reviewedPaths.includes("cart-total.js") && assessment.findings.some(f => f.path === "cart-total.js" && f.side === "RIGHT" + && /precision|decimal|floating|cents/i.test(f.body) && f.line >= 1 && f.line <= 7) + && assessment.score < 5 && r.conclusion === "failure" + && ownedDeliveryUrl(r.summaryUrl, input.repository, input.pullNumber, "summary") && ownedDeliveryUrl(r.checkUrl, input.repository, input.pullNumber, "check") + && input.issue.id === input.issueId && input.issue.companyId === input.companyId && input.issue.assigneeAgentId === input.agentId && input.issue.status === "done"; +} + +/** New, versioned fixture: price precision is explicitly part of the repository's test contract. */ +export const githubCartFixture = [ + "// Prices are decimal currency units; totals must be exact to a cent.", + "export function calculateCartTotal(items) {", + " return items.reduce((total, item) => {", + " const price = Number(item.price);", + " return total + price * Number(item.quantity);", + " }, 0);", + "}", +].join("\n") + "\n"; diff --git a/tests/runner-e2e/github-review-flow.ts b/tests/runner-e2e/github-review-flow.ts new file mode 100644 index 00000000000..c580ed767fa --- /dev/null +++ b/tests/runner-e2e/github-review-flow.ts @@ -0,0 +1,236 @@ +import { expect, type Page } from "@playwright/test"; +import type { GitHubTaskReview } from "../../packages/shared/src/types/chat-github.js"; +import type { GitHubRepositoryPage } from "../../packages/shared/src/types/chat-channels.js"; +import { ConnectionBlock, ConnectionFailure } from "./connection-config.js"; +import { ConnectionApi } from "./connection-target.js"; +import { awaitJourneyCondition, createGitHubJourneyPullRequest, enterGitHubJourneyCompany, observeGitHubHead, selectGitHubJourneyRepository } from "./github-review-browser.js"; +import { githubCartFixture, hasOnlyOpenedAutomaticReview, matchesGitHubJourneyEnvironment, retainGitHubReview, verifyGitHubReview, verifyGitHubReviewRun, type GitHubReviewJourneyEvidence } from "./github-review-evidence.js"; +import type { GitHubReviewJourneyConfig } from "./github-review-config.js"; +import { captureOwnedGitHubApp, readOwnedGitHubReviewRuns, type GitHubJourneyResources } from "./github-review-ownership.js"; +import { attendGitHubJourneyAction } from "./github-review-attendance.js"; + +export async function runGitHubReviewFlow(input: { + page: Page; github: Page; githubUsername: string; api: ConnectionApi; config: GitHubReviewJourneyConfig; resources: GitHubJourneyResources; + evidence: GitHubReviewJourneyEvidence; signal: AbortSignal; + checkpoint(id: keyof GitHubReviewJourneyEvidence["checkpoints"], page: Page): Promise; + phase(id: string): Promise; + persistResources(): Promise; +}) { + const { page, github, api, config, resources, evidence } = input; + const capabilities = await api.get('/api/instance/settings/experimental'); + if (capabilities.enableGitHubReviewBots !== true) throw new ConnectionBlock("blocked_target", "github_review_capability_unavailable"); + const company = await api.get(`/api/companies/${config.target.companyId}`); + const agent = await api.get(`/api/agents/${config.agent.id}`); + if (company.id !== config.target.companyId || agent.companyId !== company.id || agent.name !== config.agent.name || !["idle", "running"].includes(agent.status)) throw new ConnectionBlock("blocked_target", "configured_review_agent_unavailable"); + const environment = await api.get(`/api/environments/${config.target.environmentId}`); + // Inheritance and task trust policy can select the environment without an + // agent override. Validate the declared target now; the actual run binding + // remains authoritative after dispatch. + if (environment.status !== "active" + || !matchesGitHubJourneyEnvironment(environment, config.target.environmentId, config.executionEnvironment)) throw new ConnectionBlock("blocked_target", "configured_review_environment_mismatch"); + // The profile is explicit about the provider/model it claims to qualify. + if (JSON.stringify(agent.adapterConfig ?? {}).includes(config.agent.model) === false) throw new ConnectionBlock("blocked_target", "configured_agent_model_mismatch"); + // Record effective accounting prerequisites without granting or changing them. + try { + const budgets = await api.get(`/api/companies/${company.id}/budgets/overview`); + evidence.budgetPolicies = budgets.policies.filter((p: any) => p.isActive && (p.scopeType === "company" || (p.scopeType === "agent" && p.scopeId === agent.id))) + .map((p: any) => ({ scopeType: p.scopeType, amount: p.amount, hardStopEnabled: p.hardStopEnabled, unpricedUsagePolicy: p.unpricedUsagePolicy })); + evidence.budgetPolicyRead = "verified"; + } catch { evidence.budgetPolicyRead = "unavailable"; } + await enterGitHubJourneyCompany(page, api.origin, company); + await input.checkpoint("agent", page); + await input.phase("connector_navigation"); + await page.getByRole("link", { name: "Connectors", exact: true }).click(); + const botCard = page.getByRole("listitem").filter({ has: page.getByRole("heading", { name: "GitHub Code Review Bot", exact: true }) }); + await botCard.getByRole("button", { name: /^(?:Connect|Add connection) GitHub Code Review Bot$/ }).click(); + await page.getByRole("button", { name: /Select agent/ }).click(); + await page.getByRole("button", { name: `Select ${config.agent.name}`, exact: true }).click(); + const creation = page.waitForResponse(r => new URL(r.url()).pathname === `/api/companies/${company.id}/chat-endpoints` && r.request().method() === "POST"); + await page.getByRole("button", { name: "Continue", exact: true }).click(); + const response = await creation; + if (!response.ok()) throw new ConnectionFailure("endpoint_creation_failed"); + const endpoint = await response.json(); + if (endpoint.companyId !== company.id || endpoint.assignedAgentId !== agent.id || endpoint.provider !== "github") throw new ConnectionFailure("created_endpoint_binding_mismatch"); + resources.endpointId = endpoint.id; + evidence.cleanup.endpoint = false; + await input.persistResources(); + // Check the declared existing-user prerequisite before creating an external + // App. A missing fixture must not leave an unnecessary GitHub installation. + const existingAccounts = await api.get>(`/api/chat-endpoints/${endpoint.id}/github/personal-connections`); + const expectedAccounts = existingAccounts.filter(a => a.status === "active" && a.enabled && a.login?.toLowerCase() === input.githubUsername.toLowerCase()); + if (expectedAccounts.length !== 1) throw new ConnectionBlock("blocked_target", "dedicated_github_user_connection_required"); + await input.phase("github_app_registration"); + if (config.github.ownerType === "organization") { + await page.getByLabel("GitHub account", { exact: true }).selectOption("organization"); + await page.getByLabel("Organization", { exact: true }).fill(config.github.appOwner); + } + await page.getByLabel("App name", { exact: true }).fill(resources.appName); + await page.getByRole("button", { name: "Continue to GitHub", exact: true }).click(); + // Manifest flow navigates the same tab. External App grants remain attended. + evidence.assisted = true; + await awaitJourneyCondition(async () => new URL(page.url()).hostname === "github.com", 30_000, "github_manifest_handoff_missing", input.signal); + const createApp = page.getByRole("button", { name: new RegExp(`^Create GitHub App(?: for ${config.github.appOwner})?$`) }); + await createApp.waitFor({ timeout: config.browser.loginTimeoutMs }); + await page.bringToFront(); + await input.checkpoint("app_creation_consent", page); + console.log(`[github-review-journey] Authorize creation of only ${resources.appName} in the dedicated QA browser. Install it only on ${config.github.repository}.`); + await attendGitHubJourneyAction({ control: createApp, action: "create-app", appName: resources.appName, signal: input.signal, timeoutMs: config.browser.loginTimeoutMs }); + await input.phase("github_repository_installation"); + await awaitJourneyCondition(async () => await page.getByRole("radio", { name: /Only select repositories/i }).isVisible(), config.browser.loginTimeoutMs, "github_app_creation_consent_required", input.signal); + await awaitJourneyCondition(async () => captureOwnedGitHubApp(resources, await api.get(`/api/chat-endpoints/${endpoint.id}`), config), 30_000, "created_github_app_identity_unavailable", input.signal, "failed"); + await input.checkpoint("app_created", page); + await selectGitHubJourneyRepository(page, config.github.repository); + await input.checkpoint("repository_scope", page); + await page.bringToFront(); + console.log(`[github-review-journey] The installation selects only ${config.github.repository}. Confirm Install in the QA browser.`); + await attendGitHubJourneyAction({ control: page.getByRole("button", { name: "Install", exact: true }), action: "install-app", appName: resources.appName, signal: input.signal, timeoutMs: config.browser.loginTimeoutMs }); + await awaitJourneyCondition(async () => new URL(page.url()).origin === api.origin, config.browser.loginTimeoutMs, "github_installation_consent_required", input.signal); + const savedAccount = page.getByLabel("Existing GitHub connection", { exact: true }); + await input.phase("github_identity_binding"); + // Returning to the tenant can precede signed-webhook verification and the + // asynchronously loaded identity choices. Wait for a usable wizard state. + await awaitJourneyCondition(async () => await savedAccount.isVisible() + || await page.getByRole("button", { name: "Connect my GitHub account", exact: true }).isVisible() + || await page.getByRole("button", { name: "Confirm my account", exact: true }).isVisible() + || await page.getByRole("button", { name: "Done", exact: true }).isVisible(), + 120_000, "github_identity_or_connection_state_missing", input.signal, "failed"); + if (await savedAccount.isVisible()) { + // Select only a single already configured account; ambiguity never picks a stranger's login. + const accounts = await api.get>(`/api/chat-endpoints/${endpoint.id}/github/personal-connections`); + const active = accounts.filter(a => a.status === "active" && a.enabled && a.login?.toLowerCase() === input.githubUsername.toLowerCase()); + if (active.length !== 1) throw new ConnectionBlock("blocked_target", "dedicated_github_user_connection_required"); + await savedAccount.selectOption(active[0]!.connectionId); + await page.getByRole("button", { name: "Check my account", exact: true }).click(); + await expect(page.getByText(`GitHub account: ${input.githubUsername}`, { exact: true })).toBeVisible(); + await page.getByRole("button", { name: "Confirm my account", exact: true }).click(); + } else if (await page.getByRole("button", { name: "Confirm my account", exact: true }).isVisible()) { + await expect(page.getByText(`Connect GitHub account ${input.githubUsername} to your Paperclip account.`, { exact: true })).toBeVisible(); + await page.getByRole("button", { name: "Confirm my account", exact: true }).click(); + } else if (await page.getByRole("button", { name: "Connect my GitHub account", exact: true }).isVisible()) { + throw new ConnectionBlock("blocked_target", "saved_github_user_connection_required"); + } + await page.getByRole("button", { name: "Done", exact: true }).click(); + await input.checkpoint("identity", page); + await input.phase("review_configuration"); + const configuredEndpoint = await api.get(`/api/chat-endpoints/${endpoint.id}`); + const repositories = await api.get(`/api/chat-endpoints/${endpoint.id}/github/repositories?limit=100&offset=0`); + if (configuredEndpoint.status !== "active" || configuredEndpoint.assignedAgentId !== config.agent.id + || repositories.totalCount !== 1 || repositories.enabledCount !== 1 + || repositories.items.length !== 1 || !repositories.items[0]!.enabled + || repositories.items[0]!.label !== config.github.repository) throw new ConnectionFailure("installation_repository_scope_mismatch"); + if (!captureOwnedGitHubApp(resources, configuredEndpoint, config)) throw new ConnectionFailure("created_github_app_identity_unavailable"); + const editor = page.getByRole("textbox", { name: "Agent instructions", exact: true }); + await editor.fill("/"); + await page.getByRole("button", { name: new RegExp(`/${config.agent.skill}.*SKILL`, "i") }).click(); + await editor.press("End"); + await page.keyboard.type("\nReview this pull request using the selected review skill. Post findings and a numeric rating out of 5. Review only; do not modify repository files."); + const automatic = page.getByRole("switch", { name: "Run automatically", exact: true }); + if (await automatic.getAttribute("aria-checked") !== "true") await automatic.click(); + for (const label of ["New pull request", "Updated commits", "Reopened", "Ready for review"]) await page.getByRole("checkbox", { name: label, exact: true }).setChecked(label === "New pull request"); + await page.getByRole("checkbox", { name: "New issues", exact: true }).setChecked(false); + await page.getByLabel("Passing score", { exact: true }).fill("5"); + const savedConfiguration = page.waitForResponse(r => new URL(r.url()).pathname === `/api/chat-endpoints/${endpoint.id}/github/configuration` && r.request().method() === "PUT"); + await page.getByRole("button", { name: "Save changes", exact: true }).click(); + if (!(await savedConfiguration).ok()) throw new ConnectionFailure("review_configuration_save_failed"); + await expect(page.getByText("Changes saved.", { exact: true })).toBeVisible(); + const record = await api.get(`/api/chat-endpoints/${endpoint.id}/github/configuration`); + const policy = record.configuration.defaults; + if (policy.invocation === "mentions_only") throw new ConnectionFailure("saved_review_invocation_mismatch"); + if (!hasOnlyOpenedAutomaticReview(policy)) throw new ConnectionFailure("saved_review_events_mismatch"); + if (policy.ratingThreshold !== 5) throw new ConnectionFailure("saved_review_threshold_mismatch"); + if (!policy.instructions.includes(config.agent.skill)) throw new ConnectionFailure("saved_review_skill_mismatch"); + await input.checkpoint("configuration_saved", page); + await page.reload(); + await expect(editor).toContainText(config.agent.skill); + await input.checkpoint("configuration_reloaded", page); + await input.checkpoint("skill_binding", page); + // A branch can persist even if PR submission or navigation later fails. + evidence.cleanup.branch = false; + await input.persistResources(); + resources.pullNumber = await createGitHubJourneyPullRequest({ page: github, branch: resources.branch, title: `Ponytail precision eval ${resources.branch}`, contents: githubCartFixture }); + evidence.cleanup.pullRequest = false; + await input.persistResources(); + await input.checkpoint("pr_created", github); + const headSha = await observeGitHubHead(github); + await page.getByRole("link", { name: "Reviews", exact: true }).click(); + let review: GitHubTaskReview | undefined; + await awaitJourneyCondition(async () => { + const rows = await api.get(`/api/chat-endpoints/${endpoint.id}/github/reviews`); + const matches = rows.filter(r => r.repository === config.github.repository && r.pullNumber === resources.pullNumber && r.headSha === headSha && r.event.event === "opened"); + if (matches.length > 1) throw new ConnectionFailure("duplicate_automatic_review"); + review = matches[0]; return Boolean(review); + }, config.deadlineMs, "automatic_review_missing", input.signal, "failed"); + resources.issueId = review!.issueId; + await page.getByRole("link").filter({ hasText: `#${resources.pullNumber}` }).click(); + await input.checkpoint("automatic_task", page); + await awaitJourneyCondition(async () => { + review = await api.get(`/api/chat-endpoints/${endpoint.id}/github/reviews/${review!.id}`); + const runs = await readOwnedGitHubReviewRuns(api, { companyId: company.id, agentId: agent.id, issueId: review!.issueId }); + resources.runIds = runs.map(r => r.id); + if (resources.runIds.length > config.maximumRuns) throw new ConnectionFailure("review_run_limit_exceeded"); + if (["error", "incomplete", "superseded", "manual_required"].includes(review!.state)) throw new ConnectionFailure("review_terminal_failure"); + return review!.state === "completed"; + }, config.deadlineMs, "review_completion_timeout", input.signal, "failed"); + await input.phase("review_execution_completion"); + evidence.review = retainGitHubReview(review!); + let issue: any; let completedRun: any; + // Submitting the assessment can precede publication and the agent's final + // task completion. A completed review record is not a settled heartbeat run. + await awaitJourneyCondition(async () => { + review = await api.get(`/api/chat-endpoints/${endpoint.id}/github/reviews/${review!.id}`); + evidence.review = retainGitHubReview(review!); + const runs = await readOwnedGitHubReviewRuns(api, { companyId: company.id, agentId: agent.id, issueId: review!.issueId }); + resources.runIds = runs.map(r => r.id); + if (resources.runIds.length > config.maximumRuns) throw new ConnectionFailure("review_run_limit_exceeded"); + issue = await api.get(`/api/issues/${review!.issueId}`); + completedRun = review!.runId ? await api.get(`/api/heartbeat-runs/${review!.runId}`) : undefined; + evidence.execution = { reviewId: review!.id, issueId: review!.issueId, runId: review!.runId, + runStatus: completedRun?.status, issueStatus: issue.status, runtimeMode: completedRun?.runtimeMode, + environmentId: completedRun?.contextSnapshot?.paperclipEnvironment?.id, + provider: completedRun?.runnerProfileJson?.nativeExecutionInput?.provider?.kind, + model: completedRun?.runnerProfileJson?.nativeExecutionInput?.provider?.model }; + if (completedRun && (completedRun.companyId !== company.id || completedRun.agentId !== agent.id + || completedRun.issueId !== review!.issueId)) throw new ConnectionFailure("review_run_owner_mismatch"); + if (completedRun && ["failed", "timed_out", "cancelled", "interrupted"].includes(completedRun.status)) throw new ConnectionFailure("review_run_terminal_failure"); + return completedRun?.status === "succeeded" && issue.status === "done"; + }, config.deadlineMs, "review_execution_completion_timeout", input.signal, "failed"); + if (!verifyGitHubReviewRun(completedRun, { runId: review!.runId, companyId: company.id, agentId: agent.id, issueId: review!.issueId, + runtimeMode: config.agent.runtimeMode, environmentId: config.target.environmentId, executionEnvironment: config.executionEnvironment, + environment: await api.get(`/api/environments/${config.target.environmentId}`), provider: config.agent.provider, model: config.agent.model })) throw new ConnectionFailure("review_run_binding_mismatch"); + evidence.dependencies.find(d => d.alias === "review-agent-provider")!.status = "ready"; + if (!verifyGitHubReview({ review: review!, issue, companyId: company.id, endpointId: endpoint.id, issueId: review!.issueId, repository: config.github.repository, + pullNumber: resources.pullNumber, headSha, agentId: agent.id })) throw new ConnectionFailure("review_oracle_failed"); + evidence.score = review!.assessment!.score; evidence.conclusion = review!.conclusion!; + await input.checkpoint("review_completed", page); + await github.reload(); + await expect(github.getByText(`${evidence.score}/5`, { exact: false }).first()).toBeVisible(); + await expect(github.getByText(/precision|decimal|floating|cents/i).first()).toBeVisible(); + await input.checkpoint("github_delivery", github); + const inline = github.locator(".review-thread-component") + .filter({ has: github.getByRole("link", { name: "cart-total.js", exact: true }) }) + .filter({ has: github.locator(".review-comment .comment-body").filter({ hasText: /precision|decimal|floating|cents/i }) }).first(); + await expect(inline).toBeVisible(); + await expect(inline.locator(`.review-comment a[href="/apps/${resources.appSlug}"]`).first()).toBeVisible(); + const fileUrl = new URL(await inline.getByRole("link", { name: "cart-total.js", exact: true }).getAttribute("href") ?? "", "https://github.com"); + const commentUrl = new URL(await inline.locator('a[href^="#discussion_r"]').first().getAttribute("href") ?? "", github.url()); + if (fileUrl.origin !== "https://github.com" || fileUrl.pathname !== `/${config.github.repository}/pull/${resources.pullNumber}/files/${headSha}` + || commentUrl.origin !== "https://github.com" || commentUrl.pathname !== `/${config.github.repository}/pull/${resources.pullNumber}` + || !/^#discussion_r\d+$/.test(commentUrl.hash)) throw new ConnectionFailure("inline_publication_binding_mismatch"); + evidence.inlinePublication = { url: commentUrl.href, fileUrl: fileUrl.href, path: "cart-total.js", author: resources.appSlug!, + text: await inline.locator(".review-comment .comment-body").filter({ hasText: /precision|decimal|floating|cents/i }).first().innerText() }; + await inline.scrollIntoViewIfNeeded(); + await input.checkpoint("github_inline_delivery", github); + const checkPopup = page.context().waitForEvent("page"); + await page.getByRole("link", { name: "Check", exact: true }).click(); + const check = await checkPopup; + await expect(check.getByText(`${evidence.score}/5`, { exact: false }).first()).toBeVisible(); + await expect(check.getByText(/failed/i).first()).toBeVisible(); + await input.checkpoint("check_projection", check); + await page.getByRole("link", { name: "Paperclip task", exact: true }).click(); + await expect(page.getByText("Done", { exact: true }).first()).toBeVisible(); + await input.checkpoint("task_done", page); + await page.reload(); await github.reload(); + await expect(page.getByText("Done", { exact: true }).first()).toBeVisible(); + await expect(github.getByText(`${evidence.score}/5`, { exact: false }).first()).toBeVisible(); + await input.checkpoint("persisted", page); +} diff --git a/tests/runner-e2e/github-review-journey.test.ts b/tests/runner-e2e/github-review-journey.test.ts new file mode 100644 index 00000000000..8489b4596b0 --- /dev/null +++ b/tests/runner-e2e/github-review-journey.test.ts @@ -0,0 +1,309 @@ +import { describe, expect, it, vi } from "vitest"; +import { parseJourneyCredentialBinding, resolveJourneyCredential } from "./journey-dependencies.js"; +import { parseGitHubReviewJourneyConfig } from "./github-review-config.js"; +import { githubJourneyCheckpoints, gradeGitHubReviewJourney, hasOnlyOpenedAutomaticReview, verifyGitHubReview, verifyGitHubReviewRun, type GitHubReviewJourneyEvidence } from "./github-review-evidence.js"; +import { runnerMatrix } from "./catalog.js"; +import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import type { GitHubTaskReview } from "../../packages/shared/src/types/chat-github.js"; +import { verifyConnectionTarget } from "./connection-target.js"; +import { parseConnectionConfig } from "./connection-config.js"; +import type { APIRequestContext } from "@playwright/test"; +import { captureOwnedGitHubApp, readOwnedGitHubReviewRuns, type GitHubJourneyResources } from "./github-review-ownership.js"; +import type { ChatEndpoint } from "../../packages/shared/src/types/chat-channels.js"; +import { enterGitHubJourneyCompany, isGitHubJourneyEvidencePage } from "./github-review-browser.js"; +import { attendGitHubJourneyAction } from "./github-review-attendance.js"; +import { PassThrough } from "node:stream"; +import type { Locator, Page } from "@playwright/test"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import { runGitHubReviewCampaign } from "./github-review-launch.js"; +import type { ConnectionApi } from "./connection-target.js"; +import { runGitHubReviewFlow } from "./github-review-flow.js"; + +describe("reused target ownership", () => { + it("admits an inherited environment before setup and still leaves actual-run verification mandatory", async () => { + const setup = parseGitHubReviewJourneyConfig(config()); + const company = { id: setup.target.companyId, issuePrefix: "QA" }; + const rows: Record = { + "/api/instance/settings/experimental": { enableGitHubReviewBots: true }, + [`/api/companies/${company.id}`]: company, + [`/api/agents/${setup.agent.id}`]: { companyId: company.id, name: setup.agent.name, status: "idle", defaultEnvironmentId: null, adapterConfig: { model: setup.agent.model } }, + [`/api/environments/${setup.target.environmentId}`]: { id: setup.target.environmentId, driver: "local", status: "active", config: {} }, + [`/api/companies/${company.id}/budgets/overview`]: { policies: [] }, + }; + const api = { origin: setup.target.baseURL, get: async (route: string) => rows[route] }; + const page = { goto: async () => {}, waitForURL: async () => {} }; + await expect(runGitHubReviewFlow({ api: api as unknown as ConnectionApi, page: page as unknown as Page, github: {} as Page, + githubUsername: "fixture", config: setup, resources: { branch: "owned", appName: "owned", runIds: [] }, + evidence: { version: 1, setupId: "qa", hosting: "paperclip-cloud", targetCommit: null, assisted: false, outcome: "running", phase: "setup", dependencies: [], checkpoints: {}, cleanup: { endpoint: true, app: true, pullRequest: true, branch: true } }, + signal: new AbortController().signal, phase: async () => {}, persistResources: async () => {}, + checkpoint: async id => { if (id === "agent") throw new Error("preflight_admitted"); }, + })).rejects.toThrow("preflight_admitted"); + }); + it("enters the configured company before using its navigation despite remembered browser state", async () => { + let current = new URL("https://qa.example.test/OTHER/dashboard"); + const page = { goto: vi.fn(async (url: string) => { current = new URL(url); }), + waitForURL: vi.fn(async (predicate: (url: URL) => boolean) => { expect(predicate(current)).toBe(true); }) }; + await enterGitHubJourneyCompany(page as unknown as Page, current.origin, { issuePrefix: "QA" }); + expect(current.pathname).toBe("/QA/dashboard"); + }); + it("finds the owned run outside company history windows and excludes related foreign-task activity", async () => { + const get = vi.fn(async (route: string) => { + if (route === "/api/issues/issue/runs") return [{ runId: "old-owned" }, { runId: "related" }, { runId: "old-owned" }]; + if (route === "/api/heartbeat-runs/old-owned") return { id: "old-owned", companyId: "company", agentId: "agent", issueId: "issue" }; + if (route === "/api/heartbeat-runs/related") return { id: "related", companyId: "company", agentId: "agent", issueId: "other-issue" }; + throw new Error("company_history_must_not_be_used"); + }); + const runs = await readOwnedGitHubReviewRuns({ get } as unknown as Pick, { companyId: "company", agentId: "agent", issueId: "issue" }); + expect(runs.map(run => run.id)).toEqual(["old-owned"]); + expect(get).toHaveBeenCalledTimes(3); + }); + it("rejects a foreign company in hydrated run inventory", async () => { + const get = vi.fn(async (route: string) => route.endsWith("/runs") ? [{ runId: "foreign" }] : { id: "foreign", companyId: "other", agentId: "agent", issueId: "issue" }); + await expect(readOwnedGitHubReviewRuns({ get } as unknown as Pick, { companyId: "company", agentId: "agent", issueId: "issue" })).rejects.toThrow("issue_run_company_mismatch"); + }); +}); + +describe("governed campaign admission", () => { + it("rejects operations-managed selection before opening credentials or browsers", async () => { + const execution = runnerMatrix.find(entry => entry.id === "github-review-journey.configured-agent.daytona.setup-and-first-review")!; + vi.stubEnv("PAPERCLIP_E2E_OPERATIONS_MANIFEST", "/private/campaign.json"); + try { + await expect(runGitHubReviewCampaign({ executions: [execution], catalog: runnerMatrix, repositoryRoot: "/does-not-exist" })) + .rejects.toThrow("no budgeted routing qualification"); + } finally { vi.unstubAllEnvs(); } + }); +}); + +describe("automatic review event policy", () => { + it("accepts opened-only automation while ordinary mentions and comment replies remain available", () => { + expect(hasOnlyOpenedAutomaticReview({ events: ["opened", "mention", "comment"], issueOpened: false })).toBe(true); + }); + it("rejects missing opened events, other automatic triggers, and issue intake", () => { + for (const events of [["mention", "comment"], ["opened", "synchronize"], ["opened", "reopened"], ["opened", "ready_for_review"]] as const) { + expect(hasOnlyOpenedAutomaticReview({ events: [...events] })).toBe(false); + } + expect(hasOnlyOpenedAutomaticReview({ events: ["opened"], issueOpened: true })).toBe(false); + }); +}); + +describe("retained journey metadata", () => { + it("accepts actual budget/execution fields while rejecting unrelated private metadata", () => { + const evidence: GitHubReviewJourneyEvidence = { version: 1, outcome: "failed", phase: "review_execution_completion", setupId: "qa", hosting: "paperclip-cloud", targetCommit: "a".repeat(40), assisted: true, + dependencies: [], checkpoints: { automatic_task: true }, cleanup: { endpoint: true, app: true, branch: true, pullRequest: true }, budgetPolicyRead: "verified", + budgetPolicies: [{ scopeType: "agent", amount: 100, hardStopEnabled: true, unpricedUsagePolicy: "allow" }], + execution: { reviewId: "review", issueId: "issue", runId: "run", runStatus: "failed", issueStatus: "in_progress", runtimeMode: "native", environmentId: "sandbox", provider: "opencode", model: "configured-model" } }; + const result = { schema: "paperclip.runner-e2e.result/v2", executionId: "github-review-journey.configured-agent.local.setup-and-first-review", suiteId: "github-review-journey", suiteDefinitionHash: "definition", + attempt: 1, status: "failed", profileId: "configured-agent", environmentId: "local", caseId: "setup-and-first-review", provider: "opencode", model: "configured-model", runtimeMode: "native", + startedAt: "2026-10-24T00:00:00Z", finishedAt: "2026-10-24T00:01:00Z", durationMs: 60000, cleanup: "passed", githubReviewJourney: evidence }; + expect(() => validateRetainedRunnerResult(result)).not.toThrow(); + expect(() => validateRetainedRunnerResult({ ...result, githubReviewJourney: { ...evidence, execution: { ...evidence.execution, credential: "not-allowed" } } })).toThrow(/execution/); + expect(() => validateRetainedRunnerResult({ ...result, githubReviewJourney: { ...evidence, budgetPolicies: [{ ...evidence.budgetPolicies![0], credential: "not-allowed" }] } })).toThrow(/budgetPolicies/); + }); +}); + +describe("review execution attribution", () => { + const expected = { runId: "run", companyId: "company", agentId: "agent", issueId: "issue", runtimeMode: "native", environmentId: "sandbox", executionEnvironment: "daytona" as const, + environment: { id: "sandbox", driver: "sandbox", config: { provider: "daytona" } }, provider: "opencode", model: "configured-model" }; + const run = () => ({ id: "run", companyId: "company", agentId: "agent", issueId: "issue", status: "succeeded", runtimeMode: "native", + contextSnapshot: { paperclipEnvironment: { id: "sandbox", driver: "sandbox" } }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "opencode", model: "configured-model" } } } }); + it("rejects a mismatched environment kind or provider even when the ID matches", () => { + expect(verifyGitHubReviewRun(run(), { ...expected, executionEnvironment: "local" })).toBe(false); + expect(verifyGitHubReviewRun(run(), { ...expected, environment: { ...expected.environment, config: { provider: "fake" } } })).toBe(false); + expect(verifyGitHubReviewRun({ ...run(), contextSnapshot: { paperclipEnvironment: { id: "sandbox", driver: "local" } } }, expected)).toBe(false); + const local = { ...run(), contextSnapshot: { paperclipEnvironment: { id: "local", driver: "local" } } }; + expect(verifyGitHubReviewRun(local, { ...expected, executionEnvironment: "local", environmentId: "local", environment: { id: "local", driver: "local", config: {} } })).toBe(true); + }); + it("requires the actual successful native run, environment, provider, and model", () => { + expect(verifyGitHubReviewRun(run(), expected)).toBe(true); + expect(verifyGitHubReviewRun(undefined, expected)).toBe(false); + expect(verifyGitHubReviewRun({ ...run(), runnerProfileJson: {} }, expected)).toBe(false); + expect(verifyGitHubReviewRun(run(), { ...expected, model: "another-model" })).toBe(false); + expect(verifyGitHubReviewRun(run(), { ...expected, environmentId: "another-sandbox" })).toBe(false); + }); + it.each(["queued", "running", "failed", "cancelled"])("does not accept a %s run merely because a review record completed", status => { + expect(verifyGitHubReviewRun({ ...run(), status }, expected)).toBe(false); + }); + it.each(["runId", "companyId", "agentId", "issueId"] as const)("rejects foreign %s attribution", key => { + expect(verifyGitHubReviewRun(run(), { ...expected, [key]: "foreign" })).toBe(false); + }); +}); + +describe("attended terminal control", () => { + it("ignores another App/action, accepts the current action once, and does not retain consent", async () => { + const terminal = Object.assign(new PassThrough(), { isTTY: true }); + const control = { isVisible: vi.fn(async () => true), click: vi.fn(async () => {}) }; + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.useFakeTimers(); + try { + const first = attendGitHubJourneyAction({ control: control as unknown as Locator, action: "create-app", appName: "Ponytail QA one", signal: new AbortController().signal, timeoutMs: 5_000 }, terminal); + terminal.write("create-app Ponytail QA another\ninstall-app Ponytail QA one\n"); + await vi.advanceTimersByTimeAsync(500); + expect(control.click).not.toHaveBeenCalled(); + terminal.write("create-app Ponytail QA one\n"); + await vi.advanceTimersByTimeAsync(500); + await first; + expect(control.click).toHaveBeenCalledTimes(1); + const abort = new AbortController(); + const second = attendGitHubJourneyAction({ control: control as unknown as Locator, action: "install-app", appName: "Ponytail QA one", signal: abort.signal, timeoutMs: 5_000 }, terminal); + const stopped = expect(second).rejects.toThrow("journey_interrupted"); + abort.abort(); + await vi.advanceTimersByTimeAsync(500); + await stopped; + expect(control.click).toHaveBeenCalledTimes(1); + } finally { terminal.destroy(); vi.useRealTimers(); log.mockRestore(); } + }); + it("never treats piped input as attended confirmation", async () => { + const terminal = Object.assign(new PassThrough(), { isTTY: false }); + const control = { isVisible: vi.fn(async () => true), click: vi.fn(async () => {}) }; + const abort = new AbortController(); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.useFakeTimers(); + try { + const pending = attendGitHubJourneyAction({ control: control as unknown as Locator, action: "create-app", appName: "Ponytail QA one", signal: abort.signal, timeoutMs: 5_000 }, terminal); + const stopped = expect(pending).rejects.toThrow("journey_interrupted"); + terminal.write("create-app Ponytail QA one\n"); + abort.abort(); + await vi.advanceTimersByTimeAsync(500); + await stopped; + expect(control.click).not.toHaveBeenCalled(); + } finally { terminal.destroy(); vi.useRealTimers(); log.mockRestore(); } + }); +}); + +describe("journey screenshot boundaries", () => { + it("captures owned PR, check and installation pages while excluding auth and foreign resources", () => { + const capture = (url: string) => isGitHubJourneyEvidencePage(url, "https://qa.example.test", "fixture/repo", "owned-app"); + for (const url of ["https://qa.example.test/QA/issues/QA-1", "https://github.com/fixture/repo/pull/7", "https://github.com/fixture/repo/runs/123", "https://github.com/apps/owned-app/installations/new"]) expect(capture(url)).toBe(true); + for (const url of ["https://qa.example.test/QA/apps/connect", "https://qa.example.test/callback?code=secret", "https://github.com/login", "https://github.com/apps/shared-app/installations/new", "https://github.com/foreign/repo/pull/7", "https://github.com/fixture/repo/settings/secrets"]) expect(capture(url)).toBe(false); + }); +}); + +describe("durable journey credential references", () => { + it("resolves one selected field through a service-account-only child environment", async () => { + const secret = vi.fn(); + const read = vi.fn(async () => "resolved-value"); + expect(await resolveJourneyCredential({ source: "onepassword", reference: "op://vault/item/username" }, { env: { PATH: "/bin", OP_SERVICE_ACCOUNT_TOKEN: "service-account", GITHUB_TOKEN: "unrelated", OP_SESSION_user: "personal-session", OPENAI_API_KEY: "foreign-provider" }, read, registerSecret: secret })).toBe("resolved-value"); + expect(read).toHaveBeenCalledWith("op://vault/item/username", { PATH: "/bin", OP_SERVICE_ACCOUNT_TOKEN: "service-account" }, undefined); + expect(secret.mock.calls.flat()).toEqual(["service-account", "resolved-value"]); + }); + it("does not fall back to a personal desktop login or another environment credential", async () => { + const read = vi.fn(); + await expect(resolveJourneyCredential({ source: "onepassword", reference: "op://vault/item/password" }, { env: { OP_SESSION_user: "personal" }, read, registerSecret: vi.fn() })).rejects.toThrow("service_account_unavailable"); + expect(read).not.toHaveBeenCalled(); + }); + it.each([{ source: "onepassword", reference: "op://vault/item/password", value: "raw-password" }, { source: "onepassword", reference: "$(cat secrets)" }, { source: "environment", variable: "OP_SERVICE_ACCOUNT_TOKEN" }])("rejects malformed or literal bindings", binding => { + expect(() => parseJourneyCredentialBinding(binding)).toThrow("invalid_dependency_binding"); + }); + it("retains only a closed failure code when op fails with sensitive stderr", async () => { + await expect(resolveJourneyCredential({ source: "onepassword", reference: "op://vault/item/password" }, { env: { OP_SERVICE_ACCOUNT_TOKEN: "secret" }, read: async () => { throw new Error("private token and item path"); }, registerSecret: vi.fn() })).rejects.toThrow(/^credential_unavailable$/); + }); +}); + +const config = () => ({ version: 1, setupId: "qa-test", hosting: "paperclip-cloud", target: { mode: "attach", baseURL: "https://qa.example.test", expectedCommit: "a".repeat(40), deploymentMode: "authenticated", companyId: "11111111-1111-4111-8111-111111111111", environmentId: "22222222-2222-4222-8222-222222222222" }, + browser: { headed: true, channel: "chrome", profileDir: "/private/qa-browser", accountAlias: "ralph-qa", freshness: "signed-in" }, + agent: { id: "22222222-2222-4222-8222-222222222222", name: "Ponytail QA", skill: "ponytail-review", model: "configured-model", provider: "openai", runtimeMode: "native" }, + github: { repository: "fixture/ponytail-evals", ownerType: "personal", appOwner: "fixture", login: "passkey", username: { source: "onepassword", reference: "op://vault/item/username" } } }); + +describe("setup profiles and selection", () => { + it("requires an actual environment binding for every selected cell", () => { + const value = config(); + const { environmentId: _environmentId, ...target } = value.target; + expect(() => parseGitHubReviewJourneyConfig({ ...value, target })).toThrow("invalid_github_journey_configuration"); + }); + it("verifies a protected hosted target through the authenticated session without following redirects", async () => { + const get = vi.fn(async () => ({ ok: () => true, json: async () => ({ commit: "a".repeat(40), deploymentMode: "authenticated" }) })); + await expect(verifyConnectionTarget("https://qa.example.test", parseConnectionConfig({ target: config().target }), { get } as unknown as APIRequestContext)).resolves.toMatchObject({ commit: "a".repeat(40) }); + expect(get).toHaveBeenCalledWith("https://qa.example.test/api/health", { maxRedirects: 0, timeout: 15_000 }); + get.mockResolvedValueOnce({ ok: () => false, json: async () => ({}) } as any); + await expect(verifyConnectionTarget("https://qa.example.test", parseConnectionConfig({ target: config().target }), { get } as unknown as APIRequestContext)).rejects.toThrow("target_health_unavailable_or_redirected"); + }); + it("keeps Cloud hosting separate from agent execution and declares passkey without requiring an exported password", () => { + expect(parseGitHubReviewJourneyConfig(config())).toMatchObject({ hosting: "paperclip-cloud", github: { login: "passkey" } }); + expect(parseGitHubReviewJourneyConfig({ ...config(), hosting: "self-hosted" }).hosting).toBe("self-hosted"); + expect(parseGitHubReviewJourneyConfig({ ...config(), executionEnvironment: "daytona" }).executionEnvironment).toBe("daytona"); + expect(parseGitHubReviewJourneyConfig(config()).github.account).toBe("fixture"); + expect(() => parseGitHubReviewJourneyConfig({ ...config(), github: { ...config().github, ownerType: "organization" } })).toThrow("invalid_github_journey_configuration"); + expect(() => parseGitHubReviewJourneyConfig({ ...config(), github: { ...config().github, password: "literal" } })).toThrow("invalid_github_journey_configuration"); + }); + it("discovers the exact journey and excludes it from all and CI paid matrices", () => { + const cells = runnerMatrix.filter(c => c.suite.id === "github-review-journey"); + expect(cells).toHaveLength(2); + const options = parseRunnerSelectors(["--suite", "github-review-journey", "--environment", "local", "--github-review-config", "/private/profile.json"]); + expect(selectRunnerExecutions(options).map(c => c.id)).toEqual(cells.filter(c => c.environment.id === "local").map(c => c.id)); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "github-review-journey")).toBe(false); + expect(() => buildMatrixJobs(cells)).toThrow(/Attended connection journeys/); + expect(cells[0]!.requiredCredentials).toEqual([]); + }); +}); + +describe("GitHub App cleanup ownership", () => { + const fixture = () => { + const profile = parseGitHubReviewJourneyConfig(config()); + const resources: GitHubJourneyResources = { endpointId: "endpoint", appName: "Ponytail QA nonce", branch: "owned", runIds: [] }; + const endpoint = { id: "endpoint", companyId: profile.target.companyId, provider: "github", assignedAgentId: profile.agent.id, + botExternalId: "12345", botUsername: "ponytail-qa-nonce[bot]", botLabel: resources.appName, + setup: { step: "provider_setup", github: { registrationStatus: "completed", appOwnerType: "personal", appOwnerLogin: "fixture", appSlug: "ponytail-qa-nonce", managementUrl: "https://github.com/settings/installations/67890" } } } as ChatEndpoint; + return { profile, resources, endpoint }; + }; + it("records the verified App registration separately from its installation", () => { + const f = fixture(); + expect(captureOwnedGitHubApp(f.resources, f.endpoint, f.profile)).toBe(true); + expect(f.resources).toMatchObject({ appId: "12345", appSlug: "ponytail-qa-nonce", appSettingsUrl: "https://github.com/settings/apps/ponytail-qa-nonce", installationSettingsUrl: "https://github.com/settings/installations/67890" }); + }); + it("cannot adopt the shared testing App, a foreign endpoint, or a renamed App", () => { + for (const mutate of [(e: ChatEndpoint) => { e.setup.github!.appOwnerLogin = "paperclipai"; }, (e: ChatEndpoint) => { e.id = "foreign"; }, + (e: ChatEndpoint) => { e.botLabel = "[Test] Paperclip"; }, (e: ChatEndpoint) => { e.botUsername = "foreign[bot]"; }]) { + const f = fixture(); mutate(f.endpoint); + expect(() => captureOwnedGitHubApp(f.resources, f.endpoint, f.profile)).toThrow("owned_github_app_binding_mismatch"); + expect(f.resources.appSettingsUrl).toBeUndefined(); + } + }); + it("rejects replacement of an already recorded immutable App ID", () => { + const f = fixture(); captureOwnedGitHubApp(f.resources, f.endpoint, f.profile); f.endpoint.botExternalId = "99999"; + expect(() => captureOwnedGitHubApp(f.resources, f.endpoint, f.profile)).toThrow("owned_github_app_binding_mismatch"); + expect(f.resources.appId).toBe("12345"); + }); + it("does not invent a cleanup URL while registration is incomplete", () => { + const f = fixture(); f.endpoint.setup.github!.registrationStatus = "pending"; + expect(captureOwnedGitHubApp(f.resources, f.endpoint, f.profile)).toBe(false); + expect(f.resources.appSettingsUrl).toBeUndefined(); + }); +}); + +const head = "d".repeat(40); +function oracleInput() { + const review = { id: "review", companyId: "company", endpointId: "endpoint", issueId: "issue", runId: "run", repositoryId: "repo", repository: "fixture/repo", pullNumber: 7, headSha: head, + event: { event: "opened", headSha: head, repository: "fixture/repo" }, state: "completed", conclusion: "failure", summaryUrl: "https://github.com/fixture/repo/pull/7#issuecomment-1", checkUrl: "https://github.com/fixture/repo/runs/1", + assessment: { complete: true, reviewedCommit: head, score: 3, summary: "Decimal prices lose precision.", rationale: "Reproduced currency error", coverage: { reviewedPaths: ["cart-total.js"], omittedPaths: [], limitations: [] }, findings: [{ path: "cart-total.js", side: "RIGHT", line: 5, body: "Decimal price precision fails", severity: "warning", category: "correctness" }] } } as unknown as GitHubTaskReview; + return { review, companyId: "company", endpointId: "endpoint", issueId: "issue", repository: "fixture/repo", pullNumber: 7, headSha: head, agentId: "agent", issue: { id: "issue", companyId: "company", assigneeAgentId: "agent", status: "done" } }; +} + +describe("independent GitHub review oracle", () => { + it("accepts a useful 3/5 review with a correctly failed check", () => expect(verifyGitHubReview(oracleInput())).toBe(true)); + it.each(["headSha", "repository", "issueId", "endpointId", "companyId"] as const)("rejects a stale or foreign %s", key => { + const input = oracleInput(); input.review[key] = "foreign"; + expect(verifyGitHubReview(input)).toBe(false); + }); + it("rejects manual triggering, false completion, missing delivery and a 5/5 score for the known defect", () => { + for (const mutate of [(r: GitHubTaskReview) => { r.event.event = "mention"; }, (r: GitHubTaskReview) => { r.assessment!.complete = false; }, + (r: GitHubTaskReview) => { r.summaryUrl = null; }, (r: GitHubTaskReview) => { r.assessment!.score = 5; }, (r: GitHubTaskReview) => { r.assessment!.findings = []; }]) { + const input = oracleInput(); mutate(input.review); expect(verifyGitHubReview(input)).toBe(false); + } + }); + it("rejects foreign delivery links even when the task and assessment match", () => { + for (const [field, value] of [["summaryUrl", "https://github.com/fixture/repo/pull/8#issuecomment-1"], ["summaryUrl", "https://github.com/foreign/repo/pull/7#issuecomment-1"], + ["checkUrl", "https://example.test/fixture/repo/runs/1"], ["checkUrl", "https://github.com/foreign/repo/runs/1"]] as const) { + const input = oracleInput(); input.review[field] = value; + expect(verifyGitHubReview(input)).toBe(false); + } + }); + it("fails missing checkpoint or cleanup evidence even when every other gate passed", () => { + const evidence: GitHubReviewJourneyEvidence = { version: 1, outcome: "passed", phase: "finished", hosting: "paperclip-cloud", setupId: "qa", targetCommit: head, assisted: true, dependencies: [], + checkpoints: Object.fromEntries(githubJourneyCheckpoints.map(id => [id, true])), cleanup: { endpoint: true, app: true, pullRequest: true, branch: true } }; + expect(gradeGitHubReviewJourney(evidence).every(c => c.passed)).toBe(true); + delete evidence.checkpoints.github_delivery; + expect(gradeGitHubReviewJourney(evidence).every(c => c.passed)).toBe(false); + evidence.checkpoints.github_delivery = true; evidence.cleanup.app = false; + expect(gradeGitHubReviewJourney(evidence).every(c => c.passed)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/github-review-launch.ts b/tests/runner-e2e/github-review-launch.ts new file mode 100644 index 00000000000..a78365f6add --- /dev/null +++ b/tests/runner-e2e/github-review-launch.ts @@ -0,0 +1,271 @@ +import { randomBytes, createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import type { Page } from "@playwright/test"; +import { expect } from "@playwright/test"; +import { ConnectionBlock, ConnectionFailure } from "./connection-config.js"; +import { ConnectionApi, verifyConnectionTarget } from "./connection-target.js"; +import { openConnectionBrowser, withConnectionCampaignCancellation } from "./connection-launch.js"; +import { createPrivateJourneyRepository, ensureGitHubJourneyLogin, closeOwnedGitHubPullRequest, awaitJourneyCondition, observeGitHubJourneyAccount, isGitHubJourneyEvidencePage } from "./github-review-browser.js"; +import { loadGitHubReviewJourneyConfig } from "./github-review-config.js"; +import { gradeGitHubReviewJourney, type GitHubReviewJourneyEvidence } from "./github-review-evidence.js"; +import { runGitHubReviewFlow } from "./github-review-flow.js"; +import { captureOwnedGitHubApp, readOwnedGitHubReviewRuns, type GitHubJourneyResources } from "./github-review-ownership.js"; +import { JourneyDependencyError, resolveJourneyCredential } from "./journey-dependencies.js"; +import { sanitizeJson, assertSecretFree } from "./redaction.js"; +import { writeGitHubReviewReport } from "./github-review-report.js"; +import { summarizeExecutionBilling, buildRuntimeUsage } from "./billing.js"; +import type { MatrixExecution, RunnerE2EResult } from "./types.js"; + +export async function runGitHubReviewCampaign(input: { executions: readonly MatrixExecution[]; catalog: readonly MatrixExecution[]; configFile?: string; repositoryRoot: string; provisionRepository?: boolean }) { + if (input.executions.length !== 1 || input.executions[0]!.task.flow !== "github_review_journey") throw new ConnectionFailure("Select exactly one GitHub review journey cell separately."); + if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST) throw new ConnectionFailure("GitHub review journey has no budgeted routing qualification"); + return withConnectionCampaignCancellation(async campaignSignal => { + const deadline = new AbortController(); + const signal = AbortSignal.any([campaignSignal, deadline.signal]); + const execution = input.executions[0]!; + const startedAt = new Date().toISOString(); + const nonce = randomBytes(5).toString("hex"); + const campaignId = `github-review-${startedAt.replace(/[:.]/g, "-")}-${nonce}`; + const output = path.join(input.repositoryRoot, "tests/runner-e2e/results", campaignId); + await mkdir(output, { recursive: true, mode: 0o700 }); + const secrets: string[] = []; + const save = async (name: string, value: unknown) => { + const text = JSON.stringify(sanitizeJson(value, secrets), null, 2) + "\n"; + assertSecretFree(text, secrets, name); await writeFile(path.join(output, name), text, { mode: 0o600 }); + }; + const evidence: GitHubReviewJourneyEvidence = { version: 1, setupId: "unresolved", hosting: "self-hosted", targetCommit: null, assisted: false, + outcome: "running", phase: "configuration", dependencies: [], checkpoints: {}, cleanup: { endpoint: true, pullRequest: true, branch: true, app: true } }; + const sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: input.repositoryRoot, encoding: "utf8" }).trim(); + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, + source: { sha, ref: "working-tree", workflowRunUrl: null }, attempt: 1, status: "failed", profileId: execution.profile.id, environmentId: execution.environment.id, caseId: execution.task.id, + provider: execution.profile.provider, model: execution.profile.model, runtimeMode: "native", startedAt, finishedAt: startedAt, durationMs: 0, cleanup: "not_started", githubReviewJourney: evidence, screenshots: [] }; + const resources: GitHubJourneyResources = { branch: `codex/ponytail-eval-${nonce}`, appName: `Ponytail QA ${nonce}`, runIds: [] }; + let browser: Awaited> | undefined; + let api: ConnectionApi | undefined; let github: Page | undefined; let page: Page | undefined; + let config: Awaited> | undefined; + const checkpoint = async (id: string) => { evidence.phase = id; await save("progress.json", { executionId: execution.id, ...evidence }); }; + const abort = () => { for (const tab of browser?.context.pages() ?? []) void tab.close().catch(() => {}); }; + signal.addEventListener("abort", abort, { once: true }); + let timer: ReturnType | undefined; + let repositoryCreated = false; + try { + config = await loadGitHubReviewJourneyConfig(input.configFile); + timer = setTimeout(() => deadline.abort(), config.deadlineMs); + const authentication = config.authentication ? await import(pathToFileURL(config.authentication.module).href) : undefined; + if (config.executionEnvironment !== execution.environment.id) throw new ConnectionBlock("blocked_target", "selected_execution_environment_mismatch"); + evidence.setupId = config.setupId; evidence.hosting = config.hosting; + result.provider = config.agent.provider; result.model = config.agent.model; result.runtimeMode = config.agent.runtimeMode; + evidence.definitionHash = createHash("sha256").update(execution.suiteDefinitionHash).update(JSON.stringify({ setup: config.setupId, hosting: config.hosting, agent: config.agent, repository: config.github.repository, login: config.github.login })).digest("hex"); + const common = { ...config, models: {}, routes: {}, credentials: {}, budgetCents: 1, maxRuns: config.maximumRuns, turnTimeoutMs: config.deadlineMs, retainCompany: true }; + const username = await resolveJourneyCredential(config.github.username, { signal, registerSecret: value => secrets.push(value) }); + evidence.dependencies.push({ alias: "github-user", consumer: "browser", status: "ready", source: config.github.username.source }); + evidence.dependencies.push({ alias: "review-agent-provider", consumer: "product-agent", status: "unverified", source: "product-connection" }); + evidence.checkpoints.dependencies = true; + browser = await openConnectionBrowser(common, input.repositoryRoot, config.target.baseURL); + page = await browser.context.newPage(); + github = await browser.context.newPage(); + await checkpoint("github_authentication"); + await github.goto(`https://github.com/${config.github.repository}`); + if (authentication?.authenticateGitHubJourneyUser) { + await authentication.authenticateGitHubJourneyUser({ page: github, configFile: config.authentication!.configFile, signal, + deadlineAt: Date.now() + config.browser.loginTimeoutMs, registerSecret: (value: string) => secrets.push(value), checkpoint: (id: string) => checkpoint(id) }); + } + await ensureGitHubJourneyLogin({ page: github, config, username, signal, checkpoint: async id => { evidence.assisted = true; await checkpoint(id); }, + ...(config.github.password ? { password: () => resolveJourneyCredential(config!.github.password!, { signal, registerSecret: value => secrets.push(value) }) } : {}) }); + if (input.provisionRepository) { + // Bootstrap is intentionally not graded as a setup-and-review attempt. + await createPrivateJourneyRepository(github, config.github.repository); + await save("repository-receipt.json", { version: 1, repository: config.github.repository, visibility: "private", createdAt: new Date().toISOString(), retained: true }); + repositoryCreated = true; + console.log("[github-review-journey] Private reusable QA repository created; receipt saved. Run the journey separately."); + return output; + } + // Authentication adapters can finish at GitHub home. Re-enter the declared + // repository before checking its visibility and preparing the measured PR. + await checkpoint("github_repository"); + await github.goto(`https://github.com/${config.github.repository}`); + await expect(github.getByText("Private", { exact: true }).first()).toBeVisible(); + await checkpoint("paperclip_authentication"); + await page.goto(config.target.baseURL); + if (config.authentication) { + if (typeof authentication?.authenticateGitHubJourney !== "function") throw new ConnectionBlock("blocked_target", "invalid_journey_authentication_adapter"); + await authentication.authenticateGitHubJourney({ page, origin: config.target.baseURL, configFile: config.authentication.configFile, signal, + deadlineAt: Date.now() + config.browser.loginTimeoutMs, registerSecret: (value: string) => secrets.push(value) }); + } + api = new ConnectionApi(browser.context.request, config.target.baseURL); + await checkpoint("awaiting_board_login"); + await awaitJourneyCondition(async () => { try { await api!.get(`/api/companies/${config!.target.companyId}`); return true; } catch { return false; } }, config.browser.loginTimeoutMs, "paperclip_login_required", signal); + const identity = await verifyConnectionTarget(config.target.baseURL, common, browser.context.request); + evidence.targetCommit = identity.commit; evidence.checkpoints.target = true; + await runGitHubReviewFlow({ page, github, githubUsername: config.github.account, api, config, resources, evidence, signal, + phase: checkpoint, persistResources: () => save("owned-resources.json", resources), checkpoint: async (id, tab) => { + evidence.checkpoints[id] = true; + // Authentication pages and forms are deliberately never captured. + const consentUrl = new URL(tab.url()); + const appCreation = id === "app_creation_consent" && consentUrl.origin === "https://github.com" + && await tab.getByRole("button", { name: new RegExp(`^Create GitHub App(?: for ${config!.github.appOwner})?$`) }).isVisible(); + if (id === "app_creation_consent") await save("app-creation-boundary.json", { origin: consentUrl.origin, path: consentUrl.pathname }); + if ((appCreation || isGitHubJourneyEvidencePage(tab.url(), config!.target.baseURL, config!.github.repository, resources.appSlug)) + && !await tab.locator('input[type="password"]:visible,input[autocomplete="one-time-code"]:visible').count()) { + const file = `${id}.png`; const bytes = await tab.screenshot({ path: path.join(output, file) }); + result.screenshots!.push({ id, label: id.replaceAll("_", " "), file, sha256: createHash("sha256").update(bytes).digest("hex") }); + } + await checkpoint(id); + await save("owned-resources.json", resources); + } }); + await verifyConnectionTarget(config.target.baseURL, common, browser.context.request); + result.issueId = resources.issueId; evidence.outcome = "passed"; + result.status = "passed"; + } catch (error) { + evidence.outcome = error instanceof ConnectionBlock ? error.reason : error instanceof JourneyDependencyError ? "missing_credential" : "failed"; + result.error = error instanceof ConnectionBlock ? error.code : error instanceof JourneyDependencyError ? error.code : error instanceof ConnectionFailure ? error.message : `${evidence.phase}_failed`; + result.failureClass = evidence.outcome === "failed" ? "candidate_failure" : "permanent_infrastructure"; + const frame = error instanceof Error ? error.stack?.match(/(?:github-review-(?:browser|flow|launch)|qa-authentication|github-authentication)\.(?:ts|mjs):\d+:\d+/)?.[0] : undefined; + if (frame) result.error += ` (${frame})`; + await save("attempt-error.json", { error: result.error, outcome: evidence.outcome, phase: evidence.phase }); + await save("progress.json", { executionId: execution.id, ...evidence }); + await save("owned-resources.json", resources); + if (github && config && !github.isClosed()) { + const url = new URL(github.url()); + await save("github-boundary.json", { origin: url.origin, path: url.pathname }); + const repositoryPath = config.github.repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const pull = new RegExp(`^/${repositoryPath}/pull/(\\d+)$`).exec(url.pathname); + const title = `Ponytail precision eval ${resources.branch}`.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + if (!resources.pullNumber && pull && url.origin === "https://github.com" + && await github.getByRole("heading", { name: new RegExp(`^${title}(?:\\s|$)`) }).first().isVisible() + && await github.getByRole("link", { name: resources.branch, exact: true }).first().isVisible()) { + resources.pullNumber = Number(pull[1]); evidence.cleanup.pullRequest = false; + await save("owned-resources.json", resources); + } + if (url.origin === "https://github.com" && url.pathname === `/${config.github.repository}` + && !await github.locator('input[type="password"]:visible,input[autocomplete="one-time-code"]:visible').count()) { + await save("github-controls.json", { headings: await github.getByRole("heading").allTextContents(), buttons: await github.getByRole("button").allTextContents() }); + const bytes = await github.screenshot({ path: path.join(output, "github-failure.png") }).catch(() => null); + if (bytes) result.screenshots!.push({ id: "github-failure", label: evidence.phase, file: "github-failure.png", sha256: createHash("sha256").update(bytes).digest("hex") }); + } + } + if (page && config && !page.isClosed() && new URL(page.url()).origin === config.target.baseURL + && !/login|auth/.test(new URL(page.url()).pathname) && !await page.locator('input[type="password"]:visible,input[autocomplete="one-time-code"]:visible').count()) { + const bytes = await page.screenshot({ path: path.join(output, "failure.png") }).catch(() => null); + if (bytes) result.screenshots!.push({ id: "failure", label: evidence.phase, file: "failure.png", sha256: createHash("sha256").update(bytes).digest("hex") }); + await save("failure-controls.json", { phase: evidence.phase, + buttons: await page.getByRole("button").allTextContents().catch(() => []), + headings: await page.getByRole("heading").allTextContents().catch(() => []) }); + } + } finally { + if (timer) clearTimeout(timer); + signal.removeEventListener("abort", abort); + const cleanupSignal = AbortSignal.timeout(Math.min(config?.browser.loginTimeoutMs ?? 180_000, 600_000)); + const cleanupErrors: string[] = []; + if (resources.endpointId && api) { + evidence.cleanup.endpoint = false; + try { + const owned = await api.get(`/api/chat-endpoints/${resources.endpointId}`); + if (owned.companyId !== config!.target.companyId || owned.assignedAgentId !== config!.agent.id) throw new ConnectionFailure("cleanup_endpoint_owner_mismatch"); + if (owned.botExternalId) { + evidence.cleanup.app = false; + try { + if (!captureOwnedGitHubApp(resources, owned, config!)) cleanupErrors.push("owned_app_identity_unconfirmed"); + } catch { cleanupErrors.push("owned_app_binding_mismatch"); } + } + if (resources.issueId) { + const issue = await api.get(`/api/issues/${resources.issueId}`); + if (issue.companyId !== config!.target.companyId || issue.assigneeAgentId !== config!.agent.id) throw new ConnectionFailure("cleanup_issue_owner_mismatch"); + const boundRuns = await readOwnedGitHubReviewRuns(api, { companyId: config!.target.companyId, agentId: config!.agent.id, issueId: resources.issueId }); + resources.runIds = [...new Set([...resources.runIds, ...boundRuns.map(run => run.id)])]; + } + await save("owned-resources.json", resources); + await api.post(`/api/chat-endpoints/${resources.endpointId}/setup`, { action: "remove" }); + for (const runId of resources.runIds) { + const run = await api.get(`/api/heartbeat-runs/${runId}`); + if (run.companyId !== config!.target.companyId || run.agentId !== config!.agent.id) throw new ConnectionFailure("cleanup_run_owner_mismatch"); + if (["queued", "running"].includes(run.status)) { + await api.post(`/api/heartbeat-runs/${runId}/cancel`); + await awaitJourneyCondition(async () => !["queued", "running"].includes((await api!.get(`/api/heartbeat-runs/${runId}`)).status), 30_000, "owned_run_cancellation_unconfirmed", cleanupSignal, "failed"); + } + } + if (resources.issueId && result.status !== "passed") { + const issue = await api.get(`/api/issues/${resources.issueId}`); + if (issue.companyId !== config!.target.companyId || issue.assigneeAgentId !== config!.agent.id) throw new ConnectionFailure("cleanup_issue_owner_mismatch"); + if (!["done", "cancelled"].includes(issue.status)) { + await api.patch(`/api/issues/${issue.id}`, { status: "cancelled" }); + if ((await api.get(`/api/issues/${issue.id}`)).status !== "cancelled") throw new ConnectionFailure("owned_task_settlement_unconfirmed"); + await save("owned-task-cleanup.json", { issueId: issue.id, previousStatus: issue.status, status: "cancelled", reason: "failed_attempt" }); + } + } + evidence.cleanup.endpoint = (await api.get(`/api/chat-endpoints/${resources.endpointId}`)).status === "archived"; + } catch { cleanupErrors.push("endpoint_cleanup_failed"); } + } + if (resources.pullNumber && github && config) { + evidence.cleanup.pullRequest = false; evidence.cleanup.branch = false; + try { + if (github.isClosed() && browser) github = await browser.context.newPage(); + await github.goto(`https://github.com/${config.github.repository}/pull/${resources.pullNumber}`); + if ((await observeGitHubJourneyAccount(github)).toLowerCase() !== config.github.account.toLowerCase()) throw new ConnectionFailure("cleanup_github_account_mismatch"); + await closeOwnedGitHubPullRequest(github); + evidence.cleanup.pullRequest = true; evidence.cleanup.branch = true; + } catch { cleanupErrors.push("pull_request_cleanup_failed"); } + } else if (!evidence.cleanup.branch && github && config) { + try { + if (github.isClosed() && browser) github = await browser.context.newPage(); + await github.goto(`https://github.com/${config.github.repository}`); + const repository = await github.request.get(`https://github.com/${config.github.repository}`, { maxRedirects: 0 }); + const branch = await github.request.get(`https://github.com/${config.github.repository}/tree/${resources.branch}`, { maxRedirects: 0 }); + evidence.cleanup.branch = repository.status() === 200 && branch.status() === 404 + && (await observeGitHubJourneyAccount(github)).toLowerCase() === config.github.account.toLowerCase(); + if (!evidence.cleanup.branch) cleanupErrors.push("owned_branch_cleanup_required"); + } catch { cleanupErrors.push("owned_branch_cleanup_unconfirmed"); } + } + if (evidence.checkpoints.app_created || resources.appId || !evidence.cleanup.app) { + evidence.cleanup.app = false; + // App deletion is an explicit human action; never delete by an inferred name/URL. + if (resources.appSettingsUrl && github && config && new URL(resources.appSettingsUrl).origin === "https://github.com") { + try { + if (github.isClosed() && browser) github = await browser.context.newPage(); + await github.goto(resources.appSettingsUrl); + await github.bringToFront(); + console.log(`[github-review-journey] Remove only the disposable App named ${resources.appName} in the QA browser. The shared repository and existing user/provider credentials must remain.`); + await awaitJourneyCondition(async () => { + const response = await github!.request.get(resources.appSettingsUrl!, { maxRedirects: 0, timeout: 15_000 }); + if (response.status() !== 404) return false; + if ((await observeGitHubJourneyAccount(github!)).toLowerCase() !== config!.github.account.toLowerCase()) throw new ConnectionFailure("cleanup_github_account_mismatch"); + return true; + }, Math.min(config.browser.loginTimeoutMs, 600_000), "owned_app_cleanup_required", cleanupSignal); + evidence.cleanup.app = true; + } catch { cleanupErrors.push("app_cleanup_unconfirmed"); } + } else cleanupErrors.push("app_cleanup_unconfirmed"); + } + result.runIds = resources.runIds; + await save("owned-resources.json", resources); + if (api && resources.issueId && config) { + try { + const runs = await readOwnedGitHubReviewRuns(api, { companyId: config.target.companyId, agentId: config.agent.id, issueId: resources.issueId }); + result.runtimeUsage = buildRuntimeUsage({ environmentId: execution.environment.id, runs, fallbackFinishedAt: new Date().toISOString() }); + result.billing = summarizeExecutionBilling({ ...result, usage: { runs: runs.map(r => ({ usage: r.usageJson })) } }); + } catch { /* Missing billing remains unavailable; never substitute zero. */ } + } + if (result.billing) { result.billing.complete = false; result.billing.llm.costStatus = "partial"; } + await browser?.close().catch(() => cleanupErrors.push("browser_cleanup_failed")); + result.cleanup = cleanupErrors.length || !Object.values(evidence.cleanup).every(Boolean) ? "failed" : "passed"; + if (result.cleanup === "failed") { result.status = "failed"; result.failureClass = "cleanup_failure"; result.error = [result.error, ...cleanupErrors].filter(Boolean).join("; "); } + result.finishedAt = new Date().toISOString(); result.durationMs = Date.parse(result.finishedAt) - Date.parse(startedAt); + result.matcherResults = gradeGitHubReviewJourney(evidence); + if (!input.provisionRepository && result.status === "passed" && !result.matcherResults.every(check => check.passed)) { + result.status = "failed"; evidence.outcome = "failed"; result.failureClass = "candidate_failure"; result.error = "required_journey_evidence_missing"; + } + if (!input.provisionRepository) { + const reported = await writeGitHubReviewReport({ output, campaignId, result, resources, catalog: input.catalog, secrets }); + if (reported.status !== "passed" || reported.cleanup !== "passed") process.exitCode = 1; + } else { + await save("bootstrap-result.json", { created: repositoryCreated, outcome: repositoryCreated ? "passed" : evidence.outcome, error: result.error }); + if (!repositoryCreated) process.exitCode = 1; + } + await save("progress.json", evidence); + console.log(`[github-review-journey] ${evidence.outcome}; report directory: ${output}`); + } + return output; + }); +} diff --git a/tests/runner-e2e/github-review-ownership.ts b/tests/runner-e2e/github-review-ownership.ts new file mode 100644 index 00000000000..f1d15ee4855 --- /dev/null +++ b/tests/runner-e2e/github-review-ownership.ts @@ -0,0 +1,43 @@ +import type { ChatEndpoint } from "../../packages/shared/src/types/chat-channels.js"; +import { ConnectionFailure } from "./connection-config.js"; +import type { GitHubReviewJourneyConfig } from "./github-review-config.js"; +import type { ConnectionApi } from "./connection-target.js"; + +/** The issue run source includes history beyond company list windows. Related + * activity can name a run for another task, so hydrate and check direct ownership. */ +export async function readOwnedGitHubReviewRuns(api: Pick, scope: { companyId: string; agentId: string; issueId: string }) { + const rows = await api.get>(`/api/issues/${scope.issueId}/runs`); + if (!Array.isArray(rows) || rows.some(row => typeof row.runId !== "string" || !row.runId)) throw new ConnectionFailure("invalid_issue_run_inventory"); + const runs = await Promise.all([...new Set(rows.map(row => row.runId))].map(id => api.get>(`/api/heartbeat-runs/${id}`))); + if (runs.some(run => run.companyId !== scope.companyId)) throw new ConnectionFailure("issue_run_company_mismatch"); + return runs.filter(run => run.issueId === scope.issueId && run.agentId === scope.agentId); +} + +export interface GitHubJourneyResources { + endpointId?: string; pullNumber?: number; branch: string; appName: string; + appId?: string; appSlug?: string; appSettingsUrl?: string; installationSettingsUrl?: string; + issueId?: string; runIds: string[]; +} + +/** Bind cleanup to GitHub-confirmed identity on this attempt's owned endpoint. */ +export function captureOwnedGitHubApp(resources: GitHubJourneyResources, endpoint: ChatEndpoint, config: GitHubReviewJourneyConfig) { + if (!endpoint.botExternalId || endpoint.setup.github?.registrationStatus !== "completed") return false; + const app = endpoint.setup.github; + if (endpoint.id !== resources.endpointId || endpoint.companyId !== config.target.companyId || endpoint.provider !== "github" + || endpoint.assignedAgentId !== config.agent.id || endpoint.botLabel !== resources.appName + || !/^[1-9][0-9]*$/.test(endpoint.botExternalId) || !app.appSlug || !/^[a-z0-9-]+$/.test(app.appSlug) + || endpoint.botUsername !== `${app.appSlug}[bot]` || app.appOwnerType !== config.github.ownerType + || app.appOwnerLogin?.toLowerCase() !== config.github.appOwner.toLowerCase() + || (resources.appId && resources.appId !== endpoint.botExternalId) + || (resources.appSlug && resources.appSlug !== app.appSlug)) throw new ConnectionFailure("owned_github_app_binding_mismatch"); + resources.appId = endpoint.botExternalId; + resources.appSlug = app.appSlug; + const owner = encodeURIComponent(app.appOwnerLogin!); + resources.appSettingsUrl = app.appOwnerType === "organization" + ? `https://github.com/organizations/${owner}/settings/apps/${app.appSlug}` + : `https://github.com/settings/apps/${app.appSlug}`; + // Installation management is a different resource. Uninstalling is not proof + // that the owned App registration was deleted. + resources.installationSettingsUrl = app.managementUrl; + return true; +} diff --git a/tests/runner-e2e/github-review-report.test.ts b/tests/runner-e2e/github-review-report.test.ts new file mode 100644 index 00000000000..8840fcf171d --- /dev/null +++ b/tests/runner-e2e/github-review-report.test.ts @@ -0,0 +1,51 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { writeGitHubReviewReport } from "./github-review-report.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import type { RunnerE2EResult } from "./types.js"; + +const temporary: string[] = []; +afterEach(async () => { await Promise.all(temporary.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const pixel = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jBv0AAAAASUVORK5CYII=", "base64"); +function fixture(): RunnerE2EResult { + const cell = runnerMatrix.find(row => row.id === "github-review-journey.configured-agent.local.setup-and-first-review")!; + return { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash, + profileId: cell.profile.id, environmentId: "local", caseId: cell.task.id, provider: "opencode", model: "configured-model", runtimeMode: "native", + attempt: 1, status: "failed", error: "native_session_interrupted", cleanup: "passed", durationMs: 1000, + startedAt: "2026-10-24T00:00:00Z", finishedAt: "2026-10-24T00:00:01Z", + githubReviewJourney: { version: 1, outcome: "failed", phase: "review_execution_completion", setupId: "qa", hosting: "paperclip-cloud", targetCommit: "a".repeat(40), assisted: true, + dependencies: [], checkpoints: { github_delivery: true }, cleanup: { endpoint: true, app: true, branch: true, pullRequest: true }, + budgetPolicyRead: "verified", budgetPolicies: [{ scopeType: "agent", amount: 100, hardStopEnabled: true, unpricedUsagePolicy: "allow" }], + execution: { reviewId: "review", issueId: "issue", runId: "run", runStatus: "failed", runtimeMode: "native", provider: "opencode", model: "configured-model" } }, + screenshots: [{ id: "failure", label: "Failed native completion", file: "failure.png", sha256: createHash("sha256").update(pixel).digest("hex") }] }; +} + +it("packages an honest failed journey for the existing report pipeline without admitting other private files", async () => { + const output = await mkdtemp(path.join(os.tmpdir(), "github-report-test-")); temporary.push(output); + await writeFile(path.join(output, "failure.png"), pixel); + await writeFile(path.join(output, "auth.png"), pixel); + const result = await writeGitHubReviewReport({ output, campaignId: "retained-failure", result: fixture(), resources: { branch: "owned", appName: "owned", runIds: ["run"] }, catalog: runnerMatrix, secrets: [] }); + expect(result.status).toBe("failed"); + expect(() => validateRetainedRunnerResult(result)).not.toThrow(); + const manifest = JSON.parse(await readFile(path.join(output, "evidence/evidence-manifest.json"), "utf8")); + expect(manifest).toMatchObject({ leaks: [], missing: [] }); + expect(manifest.files).toContain("snapshots/failure.png"); + expect(manifest.files).toContain("snapshots/journey-resources.json"); + await expect(stat(path.join(output, "evidence/auth.png"))).rejects.toThrow(); + expect(await readFile(path.join(output, "evidence/snapshots/failure.png"))).toEqual(pixel); + const campaign = JSON.parse(await readFile(path.join(output, "campaign.json"), "utf8")); + expect(campaign).toMatchObject({ passed: 0, failed: 1 }); + const html = await readFile(path.join(output, "dashboard.html"), "utf8"); + expect(html).toContain("evidence/snapshots/failure.png"); +}); + +it("refuses an altered screenshot rather than retaining its claimed digest", async () => { + const output = await mkdtemp(path.join(os.tmpdir(), "github-report-test-")); temporary.push(output); + await writeFile(path.join(output, "failure.png"), "altered"); + await expect(writeGitHubReviewReport({ output, campaignId: "altered", result: fixture(), resources: { branch: "owned", appName: "owned", runIds: [] }, catalog: runnerMatrix, secrets: [] })) + .rejects.toThrow("journey_screenshot_digest_mismatch"); +}); diff --git a/tests/runner-e2e/github-review-report.ts b/tests/runner-e2e/github-review-report.ts new file mode 100644 index 00000000000..6ad2cc05149 --- /dev/null +++ b/tests/runner-e2e/github-review-report.ts @@ -0,0 +1,62 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { packageEvidence } from "./evidence.js"; +import { buildRunnerCampaign } from "./history.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { stageDashboardBrandAssets } from "./report-assets.js"; +import { assertSecretFree, sanitizeJson } from "./redaction.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import type { GitHubJourneyResources } from "./github-review-ownership.js"; +import type { MatrixExecution, RunnerE2EResult } from "./types.js"; + +/** Use the normal sanitizer, artifact manifest, validator, and dashboard. Raw + * operator receipts stay in the private attempt; they are not auto-published. */ +export async function writeGitHubReviewReport(input: { + output: string; campaignId: string; result: RunnerE2EResult; resources: GitHubJourneyResources; + catalog: readonly MatrixExecution[]; secrets: readonly string[]; +}) { + const stage = await mkdtemp(path.join(os.tmpdir(), "github-journey-report-")); + const output = path.resolve(input.output); + const evidenceDir = path.join(output, "evidence"); + const realOutput = await realpath(output); + const save = async (root: string, name: string, value: unknown) => { + const text = JSON.stringify(sanitizeJson(value, input.secrets), null, 2) + "\n"; + assertSecretFree(text, input.secrets, name); + await writeFile(path.join(root, name), text, { mode: 0o600 }); + }; + try { + await mkdir(path.join(stage, "snapshots"), { mode: 0o700 }); + const result = structuredClone(input.result); + for (const shot of result.screenshots ?? []) { + const source = path.resolve(output, shot.file); + if (!source.startsWith(`${output}${path.sep}`) || !/^[a-z0-9_-]+$/.test(shot.id) || path.extname(source) !== ".png") throw new Error("unsafe_journey_screenshot_reference"); + if (!(await realpath(source)).startsWith(`${realOutput}${path.sep}`)) throw new Error("unsafe_journey_screenshot_reference"); + const bytes = await readFile(source); + if (shot.sha256 && createHash("sha256").update(bytes).digest("hex") !== shot.sha256) throw new Error("journey_screenshot_digest_mismatch"); + shot.file = `snapshots/${shot.id}.png`; + await writeFile(path.join(stage, shot.file), bytes, { mode: 0o600 }); + } + validateRetainedRunnerResult(result); + await save(stage, "result.json", result); + await save(stage, "snapshots/journey-resources.json", input.resources); + await save(stage, "snapshots/journey-evidence.json", result.githubReviewJourney); + const manifest = await packageEvidence({ privateDir: stage, uploadDir: evidenceDir, secrets: input.secrets, expectPassScreenshot: false }); + if (manifest.leaks.length || manifest.missing.length) { + result.status = "failed"; result.failureClass = "secret_leak"; result.error = "journey_evidence_packaging_failed"; + if (result.githubReviewJourney) result.githubReviewJourney.outcome = "failed"; + } + await save(evidenceDir, "result.json", result); + await save(evidenceDir, "evidence-manifest.json", manifest); + const campaign = buildRunnerCampaign({ campaignId: input.campaignId, generatedAt: result.finishedAt, expected: [result.executionId], results: [result] }); + await save(output, "campaign.json", campaign); + await save(output, "normalized-results.json", { ...campaign, results: [{ ...result, evidenceValid: !manifest.leaks.length && !manifest.missing.length, evidenceErrors: [] }] }); + await stageDashboardBrandAssets(output); + const html = renderRunnerE2EDashboard({ title: "Ponytail GitHub user journey", generatedAt: result.finishedAt, expected: [result.executionId], catalog: input.catalog, campaign, + entries: [{ result, valid: !manifest.leaks.length && !manifest.missing.length, errors: [], evidenceBaseHref: "evidence" }] }); + assertSecretFree(html, input.secrets, "dashboard.html"); + await writeFile(path.join(output, "dashboard.html"), html, { mode: 0o600 }); + return result; + } finally { await rm(stage, { recursive: true, force: true }); } +} diff --git a/tests/runner-e2e/journey-dependencies.ts b/tests/runner-e2e/journey-dependencies.ts new file mode 100644 index 00000000000..8adadae8fad --- /dev/null +++ b/tests/runner-e2e/journey-dependencies.ts @@ -0,0 +1,62 @@ +import { execFile } from "node:child_process"; + +/** Public requirements use aliases; deployment profiles bind these to private references. */ +export type JourneyCredentialBinding = + | { source: "environment"; variable: string } + | { source: "onepassword"; reference: string }; + +export class JourneyDependencyError extends Error { + constructor(readonly code: "invalid_dependency_binding" | "service_account_unavailable" | "credential_unavailable") { + super(code); + } +} + +export function parseJourneyCredentialBinding(input: unknown): JourneyCredentialBinding { + if (!input || typeof input !== "object" || Array.isArray(input)) throw new JourneyDependencyError("invalid_dependency_binding"); + const value = input as Record; + if (value.source === "environment" && Object.keys(value).sort().join() === "source,variable" + && typeof value.variable === "string" && /^[A-Z][A-Z0-9_]{1,100}$/.test(value.variable) + && value.variable !== "OP_SERVICE_ACCOUNT_TOKEN") return { source: value.source, variable: value.variable }; + // Resolve one selected field, never a vault/item dump or shell expression. + if (value.source === "onepassword" && Object.keys(value).sort().join() === "reference,source" + && typeof value.reference === "string" && value.reference.length < 1024 + && /^op:\/\/[^/?#\s]+\/[^/?#\s]+\/(?:[^/?#\r\n]+\/)?[^/?#\r\n]+(?:\?attribute=otp)?$/.test(value.reference) + && !/[`$\x00-\x1f]/.test(value.reference)) return { source: value.source, reference: value.reference }; + throw new JourneyDependencyError("invalid_dependency_binding"); +} + +export type SecretReader = (reference: string, env: NodeJS.ProcessEnv, signal?: AbortSignal) => Promise; +const readOnePassword: SecretReader = (reference, env, signal) => new Promise((resolve, reject) => { + execFile("op", ["read", "--no-newline", reference], { + env, encoding: "utf8", timeout: 30_000, maxBuffer: 64 * 1024, signal, + }, (error, stdout) => { + // CLI errors can contain private paths, item references or provider responses. + if (error || !stdout.trim()) reject(new JourneyDependencyError("credential_unavailable")); + else resolve(stdout); + }); +}); + +/** The service-account credential goes only to op, never the browser or product agent. */ +export async function resolveJourneyCredential(binding: JourneyCredentialBinding, options: { + env?: NodeJS.ProcessEnv; signal?: AbortSignal; read?: SecretReader; registerSecret(value: string): void; +}): Promise { + const checked = parseJourneyCredentialBinding(binding); + const env = options.env ?? process.env; + if (checked.source === "environment") { + const value = env[checked.variable]; + if (!value?.trim()) throw new JourneyDependencyError("credential_unavailable"); + options.registerSecret(value); + return value; + } + if (!env.OP_SERVICE_ACCOUNT_TOKEN?.trim()) throw new JourneyDependencyError("service_account_unavailable"); + options.registerSecret(env.OP_SERVICE_ACCOUNT_TOKEN); + const isolatedEnv: NodeJS.ProcessEnv = Object.fromEntries(["PATH", "HOME", "USER", "TMPDIR", "LANG", "SSL_CERT_FILE", "NODE_EXTRA_CA_CERTS"] + .flatMap(key => env[key] ? [[key, env[key]!]] : [])); + isolatedEnv.OP_SERVICE_ACCOUNT_TOKEN = env.OP_SERVICE_ACCOUNT_TOKEN; + try { + const value = await (options.read ?? readOnePassword)(checked.reference, isolatedEnv, options.signal); + if (!value.trim()) throw new JourneyDependencyError("credential_unavailable"); + options.registerSecret(value); + return value; + } catch { throw new JourneyDependencyError("credential_unavailable"); } +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 934daa1dc86..69ea18c372f 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1074,6 +1074,13 @@ async function main() { } if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST && executions.some(execution => execution.task.flow === "provider_connection")) throw new Error("Provider connection campaign has no budgeted routing qualification"); + if (executions.some(execution => execution.task.flow === "github_review_journey")) { + if (options.connectionConfig) throw new Error("Use --github-review-config for the GitHub journey."); + const { runGitHubReviewCampaign } = await import("./github-review-launch.js"); + await runGitHubReviewCampaign({ executions, catalog: runnerMatrix, configFile: options.githubReviewConfig, repositoryRoot, provisionRepository: options.provisionGitHubRepository }); + return; + } + if (options.githubReviewConfig || options.provisionGitHubRepository) throw new Error("GitHub journey options require an explicit GitHub review journey selection."); if (executions.some(execution => execution.task.flow === "provider_connection")) { const { runConnectionCampaign } = await import("./connection-launch.js"); await runConnectionCampaign({ executions, catalog: runnerMatrix, configFile: options.connectionConfig, repositoryRoot }); diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index ea99d9a1554..883d928c510 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -1,5 +1,6 @@ import { validateWorkerTiming } from "./worker-timing.js"; import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js"; +import { githubJourneyCheckpoints, gradeGitHubReviewJourney } from "./github-review-evidence.js"; import { STARTUP_REUSE_LAYERS, STARTUP_REUSE_REASONS } from "../../packages/adapter-utils/src/startup-reuse.js"; import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js"; import { DEPENDABLE_CELLS } from "./dependable-qualification.js"; @@ -379,6 +380,17 @@ const fields = { scores: array(shape({ dimension: oneOf("questionRelevance", "useOfFacts", "proposalUsefulness", "clarity", "lowFriction"), score: integer, rationale: string, evidence: array(string) })), inputTokens: nullable(integer), outputTokens: nullable(integer), estimatedCostUsd: nullable(number), reservedCostUsd: number, recordedAt: date, error: optional(string), })), + githubReviewJourney: optional(closedShape({ version: integer, outcome: oneOf("running", "passed", "failed", "awaiting_user", "missing_credential", "blocked_target"), + phase: string, setupId: string, hosting: oneOf("self-hosted", "paperclip-cloud"), targetCommit: nullable(string), assisted: boolean, + dependencies: array(closedShape({ alias: string, consumer: oneOf("harness", "browser", "product-agent"), status: oneOf("ready", "unverified", "created-in-journey"), source: oneOf("onepassword", "environment", "browser-session", "product-connection") })), + checkpoints: closedShape(Object.fromEntries(githubJourneyCheckpoints.map(id => [id, optional(boolean)]))), + cleanup: closedShape({ endpoint: boolean, pullRequest: boolean, branch: boolean, app: boolean }), definitionHash: optional(string), score: optional(integer), conclusion: optional(string), + budgetPolicyRead: optional(oneOf("verified", "unavailable")), + budgetPolicies: optional(array(closedShape({ scopeType: oneOf("agent", "company"), amount: number, hardStopEnabled: boolean, unpricedUsagePolicy: oneOf("allow", "block") }))), + execution: optional(closedShape({ reviewId: string, issueId: string, runId: nullable(string), runStatus: optional(string), issueStatus: optional(string), + runtimeMode: optional(string), environmentId: optional(string), provider: optional(string), model: optional(string) })), + review: optional(closedShape({ id: string, repository: string, pullNumber: integer, headSha: string, state: string, score: optional(integer), conclusion: nullable(string), summaryUrl: nullable(string), checkUrl: nullable(string) })), + inlinePublication: optional(closedShape({ url: string, fileUrl: string, path: string, author: string, text: string })) })), cleanup: oneOf("not_started", "passed", "failed"), } satisfies Record; const result = shape(fields); @@ -404,4 +416,5 @@ export function validateRetainedRunnerResult( if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); } if (checked.suiteId === "provider-connections" && checked.status === "passed" && (!checked.providerConnection || !connectionEvidencePasses(checked.providerConnection))) invalid("result.providerConnection"); + if (checked.suiteId === "github-review-journey" && checked.status === "passed" && (!checked.githubReviewJourney || checked.githubReviewJourney.outcome !== "passed" || !checked.githubReviewJourney.targetCommit || !gradeGitHubReviewJourney(checked.githubReviewJourney).every(check => check.passed))) invalid("result.githubReviewJourney"); } diff --git a/tests/runner-e2e/selectors.ts b/tests/runner-e2e/selectors.ts index 222b3eaf343..e40bf303794 100644 --- a/tests/runner-e2e/selectors.ts +++ b/tests/runner-e2e/selectors.ts @@ -18,6 +18,8 @@ export interface RunnerSelectorOptions { maxParallel: number; maxAutomaticRetries: number; connectionConfig?: string; + githubReviewConfig?: string; + provisionGitHubRepository?: boolean; } export class RunnerSelectorError extends Error {} @@ -62,6 +64,11 @@ export function parseRunnerSelectors( options.connectionConfig = valueFor(args, index, flag); index += 1; } + else if (flag === "--github-review-config") { + options.githubReviewConfig = valueFor(args, index, flag); + index += 1; + } + else if (flag === "--provision-github-repository") options.provisionGitHubRepository = true; else if (flag === "--max-parallel") { const value = valueFor(args, index, flag); index += 1; @@ -207,7 +214,7 @@ export function selectRunnerExecutions( export function buildMatrixJobs( executions: readonly MatrixExecution[], ): MatrixJob[] { - if (executions.some(e => e.task.flow === "provider_connection")) throw new RunnerSelectorError("Provider connection journeys use an explicit local/staging connection configuration; they are not admitted to paid CI matrix jobs yet."); + if (executions.some(e => ["provider_connection", "github_review_journey"].includes(e.task.flow))) throw new RunnerSelectorError("Attended connection journeys require explicit private setup profiles; they are not admitted to paid CI matrix jobs yet."); return executions .map((execution) => ({ executionId: execution.id, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index f54348ef351..6cc59cbabda 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -26,6 +26,7 @@ export type RunnerTaskFlow = | "followup_stop_resume" | "repository_skills" | "provider_connection" + | "github_review_journey" | "plan_task_guidance" | "blocker_guidance" | "public_mcp" @@ -363,6 +364,7 @@ export interface RunnerE2EResult { productJourney?: import("./product-journeys/types.js").JourneyEvidence; firstTask?: import("./first-task-scoring.js").FirstTaskEvidence; providerConnection?: import("./connection-evidence.js").ConnectionEvidence; + githubReviewJourney?: import("./github-review-evidence.js").GitHubReviewJourneyEvidence; completionQuality?: import("./completion-quality.js").CompletionQualityRecord[]; firstTaskQuality?: import("./first-task-quality.js").FirstTaskQuality; /** External assistant API calls, separate from the team's heartbeat executions. */