From de14edc606f3d073fcf0342d5c56b47a19bd1478 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 20 Oct 2026 21:04:37 -0500 Subject: [PATCH] test: exercise automatic repository continuation through a real task Use production hire instructions and the existing paid browser harness. Compare delivered bytes with the actual checkout, require retained workspace and context, and reject timer or user-driven continuation. Keep current qualified provider admission free of ambient overrides and retain explicit pending-profile gate controls. Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 17 +++ tests/runner-e2e/automatic-repository-flow.ts | 111 ++++++++++++++++++ tests/runner-e2e/automatic-repository.test.ts | 29 +++++ tests/runner-e2e/automatic-repository.ts | 73 ++++++++++++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 11 ++ tests/runner-e2e/extended-harnesses.test.ts | 9 +- .../runner-e2e/harness-env-admission.test.ts | 27 +++++ tests/runner-e2e/installed-cli.test.ts | 4 +- tests/runner-e2e/pi-controls-evidence.test.ts | 2 +- tests/runner-e2e/runner.spec.ts | 10 ++ tests/runner-e2e/types.ts | 1 + 12 files changed, 288 insertions(+), 12 deletions(-) create mode 100644 tests/runner-e2e/automatic-repository-flow.ts create mode 100644 tests/runner-e2e/automatic-repository.test.ts create mode 100644 tests/runner-e2e/automatic-repository.ts create mode 100644 tests/runner-e2e/harness-env-admission.test.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index f17b530e77b..9da51889668 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1,5 +1,22 @@ # Paid runner full-stack E2E +## Automatic repository continuation + +The explicit-only `automatic-repository` suite is an early repair check for +native Codex and Claude on local and Daytona execution. One browser request +must stage its managed repository, continue without a timer or another user +message, and deliver a registered download. The oracle compares downloaded +JSON with the actual checkout and the original task file. Both admissions +must retain one workspace and commit their native results. The fixture uses +production hire instructions and records the selected catalog model. These four +development cells do not qualify Slack, staging configurations, migration, +finalization restart, or the complete dependable-work milestone. + +Select it explicitly with `--suite automatic-repository`. Every execution has +one attempt, two expected provider admissions, and 1,000-cent company and agent +hard stops. Automatic infrastructure retries are disabled. Existing campaign +reports retain failures, timings, usage, runtime costs, and source identity. + ## Live provider connection journeys See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/automatic-repository-flow.ts b/tests/runner-e2e/automatic-repository-flow.ts new file mode 100644 index 00000000000..a2e28680c2f --- /dev/null +++ b/tests/runner-e2e/automatic-repository-flow.ts @@ -0,0 +1,111 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { realpath } from "node:fs/promises"; +import path from "node:path"; +import { expect, type Page } from "@playwright/test"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { createTaskThroughUi } from "./user-actions.js"; +import { readRegisteredArtifacts } from "./registered-artifact.js"; +import { automaticRepositoryFiles, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; +import type { MatrixExecution } from "./types.js"; + +type Row = Record; +const execFile = promisify(execFileCallback); + +export async function runAutomaticRepositoryFlow(input: { + page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; + temporaryRoot: string; deadlineAt: number; + observe(issue: Row, runs: Row[], checks: ReturnType): void; + capture(id: string, label: string, file: string): Promise; + evidence(name: string, value: unknown): Promise; +}) { + const { page, api, fixtures, execution, nonce } = input; + const files = automaticRepositoryFiles(nonce); + let issue: Row | undefined; + let runs: Row[] = []; + const observation: AutomaticRepositoryObservation = { reference: files.reference, issueDone: false, runCount: 0, + successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false, + savedContext: null, actualReadme: null, actualCommit: null, downloaded: null, downloadVerified: false, + visibleDelivery: false, pendingInteractions: -1, monitorScheduled: false }; + async function refresh() { + if (!issue) return; + issue = await api.get(`/api/issues/${issue.id}`); + const listed = await api.get(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`); + runs = (await Promise.all(listed.map(row => api.get(`/api/heartbeat-runs/${row.id}`)))) + .filter(row => row.agentId === fixtures.agent.id && [row.nativeIssueId, row.issueId, row.contextSnapshot?.issueId, row.contextSnapshot?.taskId].includes(issue!.id)) + .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)) || a.id.localeCompare(b.id)); + observation.issueDone = issue.status === "done"; + observation.runCount = runs.length; + observation.successfulCommittedRuns = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded" && + run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed").length; + observation.sourceContinuation = runs[0]?.resultJson?.nativeResult?.continuation?.kind ?? null; + observation.successorSource = runs[1]?.contextSnapshot?.source ?? null; + observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null); + observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_"))); + } + async function save(phase: string) { + const checks = gradeAutomaticRepository(observation); + if (issue) input.observe(issue, runs, checks); + await input.evidence("automatic-repository.json", { schema: "paperclip.e2e.automatic-repository.v1", phase, observation, checks, + runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source, + finalizationReason: run.resultJson?.finalizationReasonCode, startedAt: run.startedAt, finishedAt: run.finishedAt })) }); + } + try { + if (fixtures.project || execution.profile.generation !== "native") throw new Error("Automatic repository requires a projectless native fixture"); + const agent = await api.get(`/api/agents/${fixtures.agent.id}`); + if (agent.adapterConfig?.cwd) throw new Error("Automatic repository must not inherit an operator cwd"); + await api.patch(`/api/companies/${fixtures.company.id}/budgets`, { budgetMonthlyCents: 1000 }); + await api.patch(`/api/agents/${fixtures.agent.id}/budgets`, { budgetMonthlyCents: 1000 }); + const companyBudget = await api.get(`/api/companies/${fixtures.company.id}`); + const agentBudget = await api.get(`/api/agents/${fixtures.agent.id}`); + if (companyBudget.budgetMonthlyCents !== 1000 || agentBudget.budgetMonthlyCents !== 1000) + throw new Error("Automatic repository hard-stop budgets were not retained"); + const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, + title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", requireExplicitTitle: true }); + issue = await api.get(`/api/issues/${created.issueId}`); + await page.goto(`/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`); + await pollUntil({ label: "repository setup and automatic result delivery", deadlineAt: input.deadlineAt, intervalMs: 750, + load: async () => { await refresh(); return { issue, runs }; }, + accept: () => observation.issueDone && observation.successfulCommittedRuns === 2, + reject: state => state.runs.length > 2 ? "Unexpected extra provider admission" : + state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A provider admission failed" : undefined }); + const view = await api.get(`/api/issues/${issue.id}/workspace`); + const cwd = await realpath(view.workspace.cwd), isolated = await realpath(input.temporaryRoot); + if (!cwd.startsWith(`${isolated}${path.sep}`) || !cwd.endsWith(`/isolated-workspaces/${fixtures.company.id}/${issue.id}`)) + throw new Error("Repository oracle requires the confined task directory"); + const repositories = view.repositories.filter((repo: Row) => repo.state === "ready" && repo.repoUrl === "https://github.com/octocat/Hello-World"); + if (repositories.length !== 1 || !/^\.paperclip-repositories\/task-repo-[a-f0-9]{24}$/.test(repositories[0].relativePath)) + throw new Error("Expected one contained prepared repository"); + const checkout = await realpath(path.join(cwd, repositories[0].relativePath)); + if (!checkout.startsWith(`${cwd}${path.sep}`)) throw new Error("Prepared repository escaped the task directory"); + const git = async (args: string[]) => (await execFile("git", ["-C", checkout, ...args], { + env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" }, + timeout: 20_000, maxBuffer: 262_144, encoding: "utf8", + })).stdout; + observation.actualCommit = (await git(["rev-parse", "HEAD"])).trim(); + observation.actualReadme = await git(["show", "HEAD:README"]); + observation.readyRepository = repositories[0].pinnedCommit === observation.actualCommit; + const context = await api.get(`/api/issues/${issue.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(files.context)}`); + observation.savedContext = context.resource?.workspaceKind === "execution_workspace" && typeof context.content?.data === "string" ? context.content.data : null; + const [artifact] = await readRegisteredArtifacts(api, issue.id, runs[1]!.id, [files.result]); + observation.downloaded = JSON.parse(artifact.content); + observation.downloadVerified = artifact.contentVerified; + observation.pendingInteractions = (await api.get(`/api/issues/${issue.id}/interactions`)).filter(row => row.status === "pending").length; + await page.reload(); + await expect(page.getByTestId("task-chat-thread-header").getByTestId("issue-detail-header")).toBeVisible(); + const delivery = page.locator(`a[href*="${artifact.attachmentId}"]`).first(); + await expect(delivery).toBeVisible({ timeout: 30_000 }); + observation.visibleDelivery = true; + await save("completed"); + await input.capture("final-state", "Automatically resumed repository result", "final-state.png"); + const checks = gradeAutomaticRepository(observation); + const failed = checks.filter(check => !check.passed); + if (failed.length) throw new Error(`Automatic repository oracle failure: ${failed.map(check => check.id).join(", ")}`); + return { issue, runs, checks }; + } catch (error) { + await refresh().catch(() => undefined); + await save("failed"); + throw error; + } +} diff --git a/tests/runner-e2e/automatic-repository.test.ts b/tests/runner-e2e/automatic-repository.test.ts new file mode 100644 index 00000000000..58edb86ed3a --- /dev/null +++ b/tests/runner-e2e/automatic-repository.test.ts @@ -0,0 +1,29 @@ +import { expect, it } from "vitest"; +import { gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; + +const sha = "a".repeat(40); +const valid: AutomaticRepositoryObservation = { reference: "REFERENCE-fresh", issueDone: true, runCount: 2, + successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"], + readyRepository: true, savedContext: "REFERENCE-fresh\n", actualReadme: "Hello World!\n", actualCommit: sha, + downloaded: { reference: "REFERENCE-fresh", readme: "Hello World!\n", commit: sha }, downloadVerified: true, + visibleDelivery: true, pendingInteractions: 0, monitorScheduled: false }; + +it("accepts independent checkout, retained context, and accessible registered output", () => { + expect(gradeAutomaticRepository(valid).every(check => check.passed)).toBe(true); +}); + +it("rejects unexpected result fields rather than trusting a model's extra claims", () => { + const extra = { ...valid, downloaded: { ...valid.downloaded, claimedPass: true } }; + expect(gradeAutomaticRepository(extra).find(check => check.id === "independent-result")?.passed).toBe(false); +}); + +it.each>([ + { runCount: 1 }, { successfulCommittedRuns: 1 }, { sourceContinuation: "monitor" }, { successorSource: "issue.comment" }, + { workspaceIds: ["old", "new"] }, { savedContext: null }, { readyRepository: false }, { downloaded: null }, + { downloaded: { ...valid.downloaded, reference: "REFERENCE-stale" } }, + { downloaded: { ...valid.downloaded, readme: "I inspected it and all checks pass" } }, + { downloaded: { ...valid.downloaded, commit: "b".repeat(40) } }, + { actualCommit: null }, { downloadVerified: false }, { visibleDelivery: false }, { pendingInteractions: 1 }, { monitorScheduled: true }, +])("rejects a plausible incomplete or wrong result %j", fault => { + expect(gradeAutomaticRepository({ ...valid, ...fault }).some(check => !check.passed)).toBe(true); +}); diff --git a/tests/runner-e2e/automatic-repository.ts b/tests/runner-e2e/automatic-repository.ts new file mode 100644 index 00000000000..2b71cc690ac --- /dev/null +++ b/tests/runner-e2e/automatic-repository.ts @@ -0,0 +1,73 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerTaskFixture } from "./types.js"; + +export function automaticRepositoryFiles(nonce: string) { + return { context: `setup-context-${nonce}.txt`, result: `repository-result-${nonce}.json`, reference: `REFERENCE-${nonce}` }; +} + +export const automaticRepositoryTask: RunnerTaskFixture = { + id: "automatic-repository-resume", label: "Repository preparation continues without supervision", groups: [], + flow: "automatic_repository", workMode: "standard", expectedRunCount: 2, + automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 12 * 60_000, daytona: 15 * 60_000 }, + turnTimeoutMs: 5 * 60_000, expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Verify a managed repository ${nonce}`, + buildVisibleMarker: nonce => automaticRepositoryFiles(nonce).result, + buildPrompt(nonce) { + const files = automaticRepositoryFiles(nonce); + return [ + "Verify a repository in this projectless task and deliver the result. Do not create a project or another task.", + `Before repository setup, save ${files.context} in the task directory with exactly ${JSON.stringify(`${files.reference}\n`)}. The final newline is an actual LF.`, + "Use the task's managed repository capability to prepare https://github.com/octocat/Hello-World. Keep the task directory as the root.", + "When the checkout is ready, read its committed README and HEAD commit. Read the saved context file again; preserve its original reference.", + `Write ${files.result} as JSON with exactly these fields: reference (the saved reference without its trailing LF), readme (the complete committed README bytes as text), and commit (the full HEAD SHA).`, + `Register ${files.result} as a downloadable artifact work product with that exact title. Deliver its accessible link and finish the verification. Repository setup alone does not complete this request.`, + ].join("\n"); + }, + buildMatchers: () => [], +}; + +export interface AutomaticRepositoryObservation { + reference: string; + issueDone: boolean; + runCount: number; + successfulCommittedRuns: number; + sourceContinuation: string | null; + successorSource: string | null; + workspaceIds: Array; + readyRepository: boolean; + savedContext: string | null; + actualReadme: string | null; + actualCommit: string | null; + downloaded: { reference?: unknown; readme?: unknown; commit?: unknown } | null; + downloadVerified: boolean; + visibleDelivery: boolean; + pendingInteractions: number; + monitorScheduled: boolean; +} + +export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) { + const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail }); + return [ + check("durable-completion", value.issueDone && value.runCount === 2 && value.successfulCommittedRuns === 2, + "Two committed provider admissions must finish the requested result."), + check("server-owned-continuation", value.sourceContinuation === "same_agent" && value.successorSource === "native_status_decision" && !value.monitorScheduled, + "Repository setup must continue through the saved same-agent wake without a timer or user message."), + check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository, + "Both admissions must retain the same task workspace and its prepared repository."), + check("retained-context", value.savedContext === `${value.reference}\n`, "The original task file must survive setup byte for byte."), + check("independent-result", value.downloadVerified && value.downloaded !== null && + Object.keys(value.downloaded).sort().join(",") === "commit,readme,reference" && value.downloaded.reference === value.reference && + typeof value.actualReadme === "string" && value.actualReadme.length > 0 && value.downloaded?.readme === value.actualReadme && + typeof value.actualCommit === "string" && /^[a-f0-9]{40}$/.test(value.actualCommit) && value.downloaded?.commit === value.actualCommit, + "Downloaded result bytes must agree with the independently inspected checkout and original reference."), + check("accessible-delivery", value.visibleDelivery && value.pendingInteractions === 0, "The originating task must render its result link without an unanswered question."), + ]; +} + +export function automaticRepositoryDigest() { + const hash = createHash("sha256"); + for (const file of ["automatic-repository.ts", "automatic-repository-flow.ts", "registered-artifact.ts", "stock-harness.ts", "runner.spec.ts", "../../skills/paperclip/SKILL.md"]) + hash.update(file).update(readFileSync(new URL(file, import.meta.url))); + return hash.digest("hex"); +} diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index afa81bc6458..0a669011c49 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -211,10 +211,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, + 1, 3, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(736); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(736); + expect(validateRunnerCatalog()).toHaveLength(740); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(740); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f4bd0d09d46..f71b15cabf3 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,6 +1,7 @@ import { caseHandoffTask, caseHandoffDefinitionDigest } from "./case-handoffs-cases.js"; import { caseReceiptTask, caseReceiptDefinitionDigest, CASE_RECEIPT_BUDGET_CENTS, CASE_RECEIPT_MAX_TURNS, CASE_RECEIPT_RUN_TIMEOUT_MS } from "./case-receipt-cases.js"; import { TASK_WORKSPACES_SUITE, taskWorkspaceTask, taskWorkspaceProfile, taskWorkspaceDefinitionDigest } from "./task-workspaces-cases.js"; +import { automaticRepositoryTask, automaticRepositoryDigest } from "./automatic-repository.js"; import { nativeActiveStopTasks } from "./native-active-stop-tasks.js"; import { piControlTasks } from "./pi-controls-cases.js"; import { cursorNativeTasks } from "./cursor-native-cases.js"; @@ -1166,6 +1167,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ definitionMetadata: { version: 1, definitionDigest: taskWorkspaceDefinitionDigest(), scheduling: "explicit-only", providerTurns: 3, attempts: 1, budgetMonthlyCents: 1000, project: "none", cwd: "unconfigured-task-default", oracle: "public-file-download-and-host-nested-git", remoteLifecycle: "per_turn" }, }, + { + id: "automatic-repository", label: "Automatic repository continuation", manualOnly: true, + description: "A projectless user request stages its managed repository, continues automatically, and delivers independently verified downloaded bytes.", + groups: [], profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)) + .map(taskWorkspaceProfile).map(productionDefaultHireProfile), + environments: runnerEnvironments, tasks: [automaticRepositoryTask], expectedMatrixSize: 4, + definitionMetadata: { version: 1, definitionDigest: automaticRepositoryDigest(), scheduling: "explicit-only", + providerTurns: 2, attempts: 1, budgetMonthlyCents: 1000, project: "none", instructions: "production-default", + surface: "paperclip", oracle: "registered-download-and-independent-checkout", remoteLifecycle: "per_turn" }, + }, { id: "pi-controls", label: "Pi active controls", manualOnly: true, description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.", diff --git a/tests/runner-e2e/extended-harnesses.test.ts b/tests/runner-e2e/extended-harnesses.test.ts index f0bbe7114ea..a6077456149 100644 --- a/tests/runner-e2e/extended-harnesses.test.ts +++ b/tests/runner-e2e/extended-harnesses.test.ts @@ -64,8 +64,7 @@ describe("extended ACP harness qualification", () => { const config = cell.profile.buildAgent({ executionId: "warm", environmentId: "env", environmentFixtureId: cell.environment.id, workspacePath: "/tmp/workspace", secretRefs: { [cell.profile.credential]: { type: "secret_ref", secretId: "11111111-1111-4111-8111-111111111111", version: "latest" } } }).adapterConfig; expect(config).toMatchObject({ lifecycleMode: "warm", idleTimeoutMs: 300_000, timeoutSec: 120 }); const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION; - if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined(); - else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]); + expect(admission).toBeUndefined(); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit"); } expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => cell.suite.id === "rich-acp-warm-continuity")).toBe(false); @@ -94,8 +93,7 @@ describe("extended ACP harness qualification", () => { expect(new Set(cells.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"])); for (const cell of cells) { const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION; - if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined(); - else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]); + expect(admission).toBeUndefined(); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: cell.profile.qualificationCandidate === "pi" ? "cursor" : "pi" } }])).toThrow("explicit"); } } @@ -109,8 +107,7 @@ describe("extended ACP harness qualification", () => { for (const cell of cells) { expect(cell.suite.manualOnly).toBe(true); const admission = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION; - if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined(); - else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]); + expect(admission).toBeUndefined(); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit"); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: "pi" } }])).toThrow("explicit"); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, id: "unrelated-manual-suite" } }])).toThrow("explicit"); diff --git a/tests/runner-e2e/harness-env-admission.test.ts b/tests/runner-e2e/harness-env-admission.test.ts new file mode 100644 index 00000000000..29623e28df9 --- /dev/null +++ b/tests/runner-e2e/harness-env-admission.test.ts @@ -0,0 +1,27 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; + +const profilesModule = "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +const copilot = runnerMatrix.find(cell => cell.suite.id === "extended-harnesses" && cell.profile.qualificationCandidate === "copilot")!; +afterEach(() => { vi.doUnmock(profilesModule); vi.resetModules(); }); + +it("uses normal production admission for the current qualified Copilot profile", async () => { + expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined(); + const { buildRunnerE2EProcessEnvironment } = await import("./harness-env.js"); + expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [copilot]) + .PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); +}); + +it("retains exact, explicit admission when a future profile is pending", async () => { + const actual = await vi.importActual(profilesModule); + vi.doMock(profilesModule, () => ({ ...actual, QUALIFIED_ACPX_PROFILES: { ...actual.QUALIFIED_ACPX_PROFILES, + copilot: { ...actual.QUALIFIED_ACPX_PROFILES.copilot, qualificationStatus: "pending" } } })); + vi.resetModules(); + const { buildRunnerE2EProcessEnvironment } = await import("./harness-env.js"); + const env = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [copilot]); + expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "copilot", model: copilot.profile.model }]); + expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...copilot, suite: { ...copilot.suite, manualOnly: false } }])).toThrow("explicit"); + expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...copilot, suite: { ...copilot.suite, id: "unrelated-suite" } }])).toThrow("explicit"); + expect(() => buildRunnerE2EProcessEnvironment({}, [copilot, { ...copilot, profile: { ...copilot.profile, model: "conflicting-model" } }])).toThrow("Conflicting"); +}); diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts index 5929e409ee6..4879f94f2d8 100644 --- a/tests/runner-e2e/installed-cli.test.ts +++ b/tests/runner-e2e/installed-cli.test.ts @@ -60,11 +60,11 @@ it("admits only the 20 agreed Copilot cells through normal installed resolution expect(() => assertInstalledCliSelection(env, [{ ...cell, id: `${cell.id}-unreviewed` }])).toThrow("explicit supported"); } }); -it("qualified providers drop ambient admission while pending Copilot retains its exact gate", () => { +it("current qualified providers drop ambient admission, including explicit Copilot cells", () => { for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); for (const agent of ["copilot"]) { const cell = runnerMatrix.find(e => e.profile.qualificationCandidate === agent)!; - expect(JSON.parse(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent, model: cell.profile.model }]); + expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); } const changed = { ...cells[0]!, profile: { ...cells[0]!.profile, model: "custom/explicit-model" } }; expect(buildRunnerE2EProcessEnvironment({}, [changed]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 35cd507e08e..38c2a0e3716 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(736); + expect(validateRunnerCatalog()).toHaveLength(740); for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 355a29abc45..c2a461bdec1 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,6 +1,7 @@ import { runCaseHandoffsFlow } from "./case-handoffs-flow.js"; import { runCaseReceiptFlow } from "./case-receipt-flow.js"; import { runTaskWorkspacesFlow } from "./task-workspaces-flow.js"; +import { runAutomaticRepositoryFlow } from "./automatic-repository-flow.js"; import { runNativeActiveStopFlow as runCopilotActiveStopFlow } from "./copilot-active-stop-flow.js"; import { nativeActiveStopTasks as copilotActiveStopTasks } from "./copilot-active-stop-tasks.js"; import type { RestartRunnerIdentity } from "./process-tree-owner.js"; @@ -1020,6 +1021,15 @@ for (const execution of executions) { matcherResults, }, secrets); await captureScreenshot("final-state", "Final visible task state", "final-state.png"); + } else if (execution.task.flow === "automatic_repository") { + const journey = await runAutomaticRepositoryFlow({ page, api, fixtures, execution, nonce, temporaryRoot, + deadlineAt: startedAtMs + deadlineMs - 30_000, + observe: (currentIssue, currentRuns, checks) => { + issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; + matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `automaticRepository.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })); + }, capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), + }); + issue = journey.issue as IssueRecord; selectedRuns = journey.runs as RunRecord[]; } else if (execution.task.flow === "task_workspaces") { const journey = await runTaskWorkspacesFlow({ page, api, fixtures, execution, nonce, temporaryRoot, deadlineAt: startedAtMs + deadlineMs - 30_000, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index f5338c138f2..b273caf6f4b 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -20,6 +20,7 @@ export type RunnerTaskWorkMode = "standard" | "planning" | "ask"; export type RunnerTaskFlow = | "case_handoff" | "task_workspaces" + | "automatic_repository" | "repository_skills" | "provider_connection" | "plan_task_guidance"