import { randomBytes, createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import type { Page } from "@playwright/test"; import { expect } from "@playwright/test"; import { ConnectionBlock, ConnectionFailure } from "./connection-config.js"; import { ConnectionApi, verifyConnectionTarget } from "./connection-target.js"; import { openConnectionBrowser, withConnectionCampaignCancellation } from "./connection-launch.js"; import { createPrivateJourneyRepository, ensureGitHubJourneyLogin, closeOwnedGitHubPullRequest, awaitJourneyCondition, observeGitHubJourneyAccount, isGitHubJourneyEvidencePage } from "./github-review-browser.js"; import { loadGitHubReviewJourneyConfig } from "./github-review-config.js"; import { gradeGitHubReviewJourney, type GitHubReviewJourneyEvidence } from "./github-review-evidence.js"; import { runGitHubReviewFlow } from "./github-review-flow.js"; import { captureOwnedGitHubApp, readOwnedGitHubReviewRuns, type GitHubJourneyResources } from "./github-review-ownership.js"; import { JourneyDependencyError, resolveJourneyCredential } from "./journey-dependencies.js"; import { sanitizeJson, assertSecretFree } from "./redaction.js"; import { writeGitHubReviewReport } from "./github-review-report.js"; import { summarizeExecutionBilling, buildRuntimeUsage } from "./billing.js"; import type { MatrixExecution, RunnerE2EResult } from "./types.js"; export async function runGitHubReviewCampaign(input: { executions: readonly MatrixExecution[]; catalog: readonly MatrixExecution[]; configFile?: string; repositoryRoot: string; provisionRepository?: boolean }) { if (input.executions.length !== 1 || input.executions[0]!.task.flow !== "github_review_journey") throw new ConnectionFailure("Select exactly one GitHub review journey cell separately."); if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST) throw new ConnectionFailure("GitHub review journey has no budgeted routing qualification"); return withConnectionCampaignCancellation(async campaignSignal => { const deadline = new AbortController(); const signal = AbortSignal.any([campaignSignal, deadline.signal]); const execution = input.executions[0]!; const startedAt = new Date().toISOString(); const nonce = randomBytes(5).toString("hex"); const campaignId = `github-review-${startedAt.replace(/[:.]/g, "-")}-${nonce}`; const output = path.join(input.repositoryRoot, "tests/runner-e2e/results", campaignId); await mkdir(output, { recursive: true, mode: 0o700 }); const secrets: string[] = []; const save = async (name: string, value: unknown) => { const text = JSON.stringify(sanitizeJson(value, secrets), null, 2) + "\n"; assertSecretFree(text, secrets, name); await writeFile(path.join(output, name), text, { mode: 0o600 }); }; const evidence: GitHubReviewJourneyEvidence = { version: 1, setupId: "unresolved", hosting: "self-hosted", targetCommit: null, assisted: false, outcome: "running", phase: "configuration", dependencies: [], checkpoints: {}, cleanup: { endpoint: true, pullRequest: true, branch: true, app: true } }; const sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: input.repositoryRoot, encoding: "utf8" }).trim(); const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, source: { sha, ref: "working-tree", workflowRunUrl: null }, attempt: 1, status: "failed", profileId: execution.profile.id, environmentId: execution.environment.id, caseId: execution.task.id, provider: execution.profile.provider, model: execution.profile.model, runtimeMode: "native", startedAt, finishedAt: startedAt, durationMs: 0, cleanup: "not_started", githubReviewJourney: evidence, screenshots: [] }; const resources: GitHubJourneyResources = { branch: `codex/ponytail-eval-${nonce}`, appName: `Ponytail QA ${nonce}`, runIds: [] }; let browser: Awaited> | undefined; let api: ConnectionApi | undefined; let github: Page | undefined; let page: Page | undefined; let config: Awaited> | undefined; const checkpoint = async (id: string) => { evidence.phase = id; await save("progress.json", { executionId: execution.id, ...evidence }); }; const abort = () => { for (const tab of browser?.context.pages() ?? []) void tab.close().catch(() => {}); }; signal.addEventListener("abort", abort, { once: true }); let timer: ReturnType | undefined; let repositoryCreated = false; try { config = await loadGitHubReviewJourneyConfig(input.configFile); timer = setTimeout(() => deadline.abort(), config.deadlineMs); const authentication = config.authentication ? await import(pathToFileURL(config.authentication.module).href) : undefined; if (config.executionEnvironment !== execution.environment.id) throw new ConnectionBlock("blocked_target", "selected_execution_environment_mismatch"); evidence.setupId = config.setupId; evidence.hosting = config.hosting; result.provider = config.agent.provider; result.model = config.agent.model; result.runtimeMode = config.agent.runtimeMode; evidence.definitionHash = createHash("sha256").update(execution.suiteDefinitionHash).update(JSON.stringify({ setup: config.setupId, hosting: config.hosting, agent: config.agent, repository: config.github.repository, login: config.github.login })).digest("hex"); const common = { ...config, models: {}, routes: {}, credentials: {}, budgetCents: 1, maxRuns: config.maximumRuns, turnTimeoutMs: config.deadlineMs, retainCompany: true }; const username = await resolveJourneyCredential(config.github.username, { signal, registerSecret: value => secrets.push(value) }); evidence.dependencies.push({ alias: "github-user", consumer: "browser", status: "ready", source: config.github.username.source }); evidence.dependencies.push({ alias: "review-agent-provider", consumer: "product-agent", status: "unverified", source: "product-connection" }); evidence.checkpoints.dependencies = true; browser = await openConnectionBrowser(common, input.repositoryRoot, config.target.baseURL); page = await browser.context.newPage(); github = await browser.context.newPage(); await checkpoint("github_authentication"); await github.goto(`https://github.com/${config.github.repository}`); if (authentication?.authenticateGitHubJourneyUser) { await authentication.authenticateGitHubJourneyUser({ page: github, configFile: config.authentication!.configFile, signal, deadlineAt: Date.now() + config.browser.loginTimeoutMs, registerSecret: (value: string) => secrets.push(value), checkpoint: (id: string) => checkpoint(id) }); } await ensureGitHubJourneyLogin({ page: github, config, username, signal, checkpoint: async id => { evidence.assisted = true; await checkpoint(id); }, ...(config.github.password ? { password: () => resolveJourneyCredential(config!.github.password!, { signal, registerSecret: value => secrets.push(value) }) } : {}) }); if (input.provisionRepository) { // Bootstrap is intentionally not graded as a setup-and-review attempt. await createPrivateJourneyRepository(github, config.github.repository); await save("repository-receipt.json", { version: 1, repository: config.github.repository, visibility: "private", createdAt: new Date().toISOString(), retained: true }); repositoryCreated = true; console.log("[github-review-journey] Private reusable QA repository created; receipt saved. Run the journey separately."); return output; } // Authentication adapters can finish at GitHub home. Re-enter the declared // repository before checking its visibility and preparing the measured PR. await checkpoint("github_repository"); await github.goto(`https://github.com/${config.github.repository}`); await expect(github.getByText("Private", { exact: true }).first()).toBeVisible(); await checkpoint("paperclip_authentication"); await page.goto(config.target.baseURL); if (config.authentication) { if (typeof authentication?.authenticateGitHubJourney !== "function") throw new ConnectionBlock("blocked_target", "invalid_journey_authentication_adapter"); await authentication.authenticateGitHubJourney({ page, origin: config.target.baseURL, configFile: config.authentication.configFile, signal, deadlineAt: Date.now() + config.browser.loginTimeoutMs, registerSecret: (value: string) => secrets.push(value) }); } api = new ConnectionApi(browser.context.request, config.target.baseURL); await checkpoint("awaiting_board_login"); await awaitJourneyCondition(async () => { try { await api!.get(`/api/companies/${config!.target.companyId}`); return true; } catch { return false; } }, config.browser.loginTimeoutMs, "paperclip_login_required", signal); const identity = await verifyConnectionTarget(config.target.baseURL, common, browser.context.request); evidence.targetCommit = identity.commit; evidence.checkpoints.target = true; await runGitHubReviewFlow({ page, github, githubUsername: config.github.account, api, config, resources, evidence, signal, phase: checkpoint, persistResources: () => save("owned-resources.json", resources), checkpoint: async (id, tab) => { evidence.checkpoints[id] = true; // Authentication pages and forms are deliberately never captured. const consentUrl = new URL(tab.url()); const appCreation = id === "app_creation_consent" && consentUrl.origin === "https://github.com" && await tab.getByRole("button", { name: new RegExp(`^Create GitHub App(?: for ${config!.github.appOwner})?$`) }).isVisible(); if (id === "app_creation_consent") await save("app-creation-boundary.json", { origin: consentUrl.origin, path: consentUrl.pathname }); if ((appCreation || isGitHubJourneyEvidencePage(tab.url(), config!.target.baseURL, config!.github.repository, resources.appSlug)) && !await tab.locator('input[type="password"]:visible,input[autocomplete="one-time-code"]:visible').count()) { const file = `${id}.png`; const bytes = await tab.screenshot({ path: path.join(output, file) }); result.screenshots!.push({ id, label: id.replaceAll("_", " "), file, sha256: createHash("sha256").update(bytes).digest("hex") }); } await checkpoint(id); await save("owned-resources.json", resources); } }); await verifyConnectionTarget(config.target.baseURL, common, browser.context.request); result.issueId = resources.issueId; evidence.outcome = "passed"; result.status = "passed"; } catch (error) { evidence.outcome = error instanceof ConnectionBlock ? error.reason : error instanceof JourneyDependencyError ? "missing_credential" : "failed"; result.error = error instanceof ConnectionBlock ? error.code : error instanceof JourneyDependencyError ? error.code : error instanceof ConnectionFailure ? error.message : `${evidence.phase}_failed`; result.failureClass = evidence.outcome === "failed" ? "candidate_failure" : "permanent_infrastructure"; const frame = error instanceof Error ? error.stack?.match(/(?:github-review-(?:browser|flow|launch)|qa-authentication|github-authentication)\.(?:ts|mjs):\d+:\d+/)?.[0] : undefined; if (frame) result.error += ` (${frame})`; await save("attempt-error.json", { error: result.error, outcome: evidence.outcome, phase: evidence.phase }); await save("progress.json", { executionId: execution.id, ...evidence }); await save("owned-resources.json", resources); if (github && config && !github.isClosed()) { const url = new URL(github.url()); await save("github-boundary.json", { origin: url.origin, path: url.pathname }); const repositoryPath = config.github.repository.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const pull = new RegExp(`^/${repositoryPath}/pull/(\\d+)$`).exec(url.pathname); const title = `Ponytail precision eval ${resources.branch}`.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); if (!resources.pullNumber && pull && url.origin === "https://github.com" && await github.getByRole("heading", { name: new RegExp(`^${title}(?:\\s|$)`) }).first().isVisible() && await github.getByRole("link", { name: resources.branch, exact: true }).first().isVisible()) { resources.pullNumber = Number(pull[1]); evidence.cleanup.pullRequest = false; await save("owned-resources.json", resources); } if (url.origin === "https://github.com" && url.pathname === `/${config.github.repository}` && !await github.locator('input[type="password"]:visible,input[autocomplete="one-time-code"]:visible').count()) { await save("github-controls.json", { headings: await github.getByRole("heading").allTextContents(), buttons: await github.getByRole("button").allTextContents() }); const bytes = await github.screenshot({ path: path.join(output, "github-failure.png") }).catch(() => null); if (bytes) result.screenshots!.push({ id: "github-failure", label: evidence.phase, file: "github-failure.png", sha256: createHash("sha256").update(bytes).digest("hex") }); } } if (page && config && !page.isClosed() && new URL(page.url()).origin === config.target.baseURL && !/login|auth/.test(new URL(page.url()).pathname) && !await page.locator('input[type="password"]:visible,input[autocomplete="one-time-code"]:visible').count()) { const bytes = await page.screenshot({ path: path.join(output, "failure.png") }).catch(() => null); if (bytes) result.screenshots!.push({ id: "failure", label: evidence.phase, file: "failure.png", sha256: createHash("sha256").update(bytes).digest("hex") }); await save("failure-controls.json", { phase: evidence.phase, buttons: await page.getByRole("button").allTextContents().catch(() => []), headings: await page.getByRole("heading").allTextContents().catch(() => []) }); } } finally { if (timer) clearTimeout(timer); signal.removeEventListener("abort", abort); const cleanupSignal = AbortSignal.timeout(Math.min(config?.browser.loginTimeoutMs ?? 180_000, 600_000)); const cleanupErrors: string[] = []; if (resources.endpointId && api) { evidence.cleanup.endpoint = false; try { const owned = await api.get(`/api/chat-endpoints/${resources.endpointId}`); if (owned.companyId !== config!.target.companyId || owned.assignedAgentId !== config!.agent.id) throw new ConnectionFailure("cleanup_endpoint_owner_mismatch"); if (owned.botExternalId) { evidence.cleanup.app = false; try { if (!captureOwnedGitHubApp(resources, owned, config!)) cleanupErrors.push("owned_app_identity_unconfirmed"); } catch { cleanupErrors.push("owned_app_binding_mismatch"); } } if (resources.issueId) { const issue = await api.get(`/api/issues/${resources.issueId}`); if (issue.companyId !== config!.target.companyId || issue.assigneeAgentId !== config!.agent.id) throw new ConnectionFailure("cleanup_issue_owner_mismatch"); const boundRuns = await readOwnedGitHubReviewRuns(api, { companyId: config!.target.companyId, agentId: config!.agent.id, issueId: resources.issueId }); resources.runIds = [...new Set([...resources.runIds, ...boundRuns.map(run => run.id)])]; } await save("owned-resources.json", resources); await api.post(`/api/chat-endpoints/${resources.endpointId}/setup`, { action: "remove" }); for (const runId of resources.runIds) { const run = await api.get(`/api/heartbeat-runs/${runId}`); if (run.companyId !== config!.target.companyId || run.agentId !== config!.agent.id) throw new ConnectionFailure("cleanup_run_owner_mismatch"); if (["queued", "running"].includes(run.status)) { await api.post(`/api/heartbeat-runs/${runId}/cancel`); await awaitJourneyCondition(async () => !["queued", "running"].includes((await api!.get(`/api/heartbeat-runs/${runId}`)).status), 30_000, "owned_run_cancellation_unconfirmed", cleanupSignal, "failed"); } } if (resources.issueId && result.status !== "passed") { const issue = await api.get(`/api/issues/${resources.issueId}`); if (issue.companyId !== config!.target.companyId || issue.assigneeAgentId !== config!.agent.id) throw new ConnectionFailure("cleanup_issue_owner_mismatch"); if (!["done", "cancelled"].includes(issue.status)) { await api.patch(`/api/issues/${issue.id}`, { status: "cancelled" }); if ((await api.get(`/api/issues/${issue.id}`)).status !== "cancelled") throw new ConnectionFailure("owned_task_settlement_unconfirmed"); await save("owned-task-cleanup.json", { issueId: issue.id, previousStatus: issue.status, status: "cancelled", reason: "failed_attempt" }); } } evidence.cleanup.endpoint = (await api.get(`/api/chat-endpoints/${resources.endpointId}`)).status === "archived"; } catch { cleanupErrors.push("endpoint_cleanup_failed"); } } if (resources.pullNumber && github && config) { evidence.cleanup.pullRequest = false; evidence.cleanup.branch = false; try { if (github.isClosed() && browser) github = await browser.context.newPage(); await github.goto(`https://github.com/${config.github.repository}/pull/${resources.pullNumber}`); if ((await observeGitHubJourneyAccount(github)).toLowerCase() !== config.github.account.toLowerCase()) throw new ConnectionFailure("cleanup_github_account_mismatch"); await closeOwnedGitHubPullRequest(github); evidence.cleanup.pullRequest = true; evidence.cleanup.branch = true; } catch { cleanupErrors.push("pull_request_cleanup_failed"); } } else if (!evidence.cleanup.branch && github && config) { try { if (github.isClosed() && browser) github = await browser.context.newPage(); await github.goto(`https://github.com/${config.github.repository}`); const repository = await github.request.get(`https://github.com/${config.github.repository}`, { maxRedirects: 0 }); const branch = await github.request.get(`https://github.com/${config.github.repository}/tree/${resources.branch}`, { maxRedirects: 0 }); evidence.cleanup.branch = repository.status() === 200 && branch.status() === 404 && (await observeGitHubJourneyAccount(github)).toLowerCase() === config.github.account.toLowerCase(); if (!evidence.cleanup.branch) cleanupErrors.push("owned_branch_cleanup_required"); } catch { cleanupErrors.push("owned_branch_cleanup_unconfirmed"); } } if (evidence.checkpoints.app_created || resources.appId || !evidence.cleanup.app) { evidence.cleanup.app = false; // App deletion is an explicit human action; never delete by an inferred name/URL. if (resources.appSettingsUrl && github && config && new URL(resources.appSettingsUrl).origin === "https://github.com") { try { if (github.isClosed() && browser) github = await browser.context.newPage(); await github.goto(resources.appSettingsUrl); await github.bringToFront(); console.log(`[github-review-journey] Remove only the disposable App named ${resources.appName} in the QA browser. The shared repository and existing user/provider credentials must remain.`); await awaitJourneyCondition(async () => { const response = await github!.request.get(resources.appSettingsUrl!, { maxRedirects: 0, timeout: 15_000 }); if (response.status() !== 404) return false; if ((await observeGitHubJourneyAccount(github!)).toLowerCase() !== config!.github.account.toLowerCase()) throw new ConnectionFailure("cleanup_github_account_mismatch"); return true; }, Math.min(config.browser.loginTimeoutMs, 600_000), "owned_app_cleanup_required", cleanupSignal); evidence.cleanup.app = true; } catch { cleanupErrors.push("app_cleanup_unconfirmed"); } } else cleanupErrors.push("app_cleanup_unconfirmed"); } result.runIds = resources.runIds; await save("owned-resources.json", resources); if (api && resources.issueId && config) { try { const runs = await readOwnedGitHubReviewRuns(api, { companyId: config.target.companyId, agentId: config.agent.id, issueId: resources.issueId }); result.runtimeUsage = buildRuntimeUsage({ environmentId: execution.environment.id, runs, fallbackFinishedAt: new Date().toISOString() }); result.billing = summarizeExecutionBilling({ ...result, usage: { runs: runs.map(r => ({ usage: r.usageJson })) } }); } catch { /* Missing billing remains unavailable; never substitute zero. */ } } if (result.billing) { result.billing.complete = false; result.billing.llm.costStatus = "partial"; } await browser?.close().catch(() => cleanupErrors.push("browser_cleanup_failed")); result.cleanup = cleanupErrors.length || !Object.values(evidence.cleanup).every(Boolean) ? "failed" : "passed"; if (result.cleanup === "failed") { result.status = "failed"; result.failureClass = "cleanup_failure"; result.error = [result.error, ...cleanupErrors].filter(Boolean).join("; "); } result.finishedAt = new Date().toISOString(); result.durationMs = Date.parse(result.finishedAt) - Date.parse(startedAt); result.matcherResults = gradeGitHubReviewJourney(evidence); if (!input.provisionRepository && result.status === "passed" && !result.matcherResults.every(check => check.passed)) { result.status = "failed"; evidence.outcome = "failed"; result.failureClass = "candidate_failure"; result.error = "required_journey_evidence_missing"; } if (!input.provisionRepository) { const reported = await writeGitHubReviewReport({ output, campaignId, result, resources, catalog: input.catalog, secrets }); if (reported.status !== "passed" || reported.cleanup !== "passed") process.exitCode = 1; } else { await save("bootstrap-result.json", { created: repositoryCreated, outcome: repositoryCreated ? "passed" : evidence.outcome, error: result.error }); if (!repositoryCreated) process.exitCode = 1; } await save("progress.json", evidence); console.log(`[github-review-journey] ${evidence.outcome}; report directory: ${output}`); } return output; }); }