test: exercise automatic repository continuation through a real task

Use production hire instructions and the existing paid browser harness. Compare delivered bytes with the actual checkout, require retained workspace and context, and reject timer or user-driven continuation. Keep current qualified provider admission free of ambient overrides and retain explicit pending-profile gate controls.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-21 07:18:35 -05:00
1 parent dff63551fe
commit de14edc606
12 files changed
+288 -12

No files matched your search

+17
View File
@@ -1,5 +1,22 @@
# Paid runner full-stack E2E
## Automatic repository continuation
The explicit-only `automatic-repository` suite is an early repair check for
native Codex and Claude on local and Daytona execution. One browser request
must stage its managed repository, continue without a timer or another user
message, and deliver a registered download. The oracle compares downloaded
JSON with the actual checkout and the original task file. Both admissions
must retain one workspace and commit their native results. The fixture uses
production hire instructions and records the selected catalog model. These four
development cells do not qualify Slack, staging configurations, migration,
finalization restart, or the complete dependable-work milestone.
Select it explicitly with `--suite automatic-repository`. Every execution has
one attempt, two expected provider admissions, and 1,000-cent company and agent
hard stops. Automatic infrastructure retries are disabled. Existing campaign
reports retain failures, timings, usage, runtime costs, and source identity.
## Live provider connection journeys
See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only
@@ -0,0 +1,111 @@
import { execFile as execFileCallback } from "node:child_process";
import { promisify } from "node:util";
import { realpath } from "node:fs/promises";
import path from "node:path";
import { expect, type Page } from "@playwright/test";
import { pollUntil, type RunnerApi } from "./api.js";
import { createTaskThroughUi } from "./user-actions.js";
import { readRegisteredArtifacts } from "./registered-artifact.js";
import { automaticRepositoryFiles, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { MatrixExecution } from "./types.js";
type Row = Record<string, any>;
const execFile = promisify(execFileCallback);
export async function runAutomaticRepositoryFlow(input: {
page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string;
temporaryRoot: string; deadlineAt: number;
observe(issue: Row, runs: Row[], checks: ReturnType<typeof gradeAutomaticRepository>): void;
capture(id: string, label: string, file: string): Promise<void>;
evidence(name: string, value: unknown): Promise<void>;
}) {
const { page, api, fixtures, execution, nonce } = input;
const files = automaticRepositoryFiles(nonce);
let issue: Row | undefined;
let runs: Row[] = [];
const observation: AutomaticRepositoryObservation = { reference: files.reference, issueDone: false, runCount: 0,
successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false,
savedContext: null, actualReadme: null, actualCommit: null, downloaded: null, downloadVerified: false,
visibleDelivery: false, pendingInteractions: -1, monitorScheduled: false };
async function refresh() {
if (!issue) return;
issue = await api.get<Row>(`/api/issues/${issue.id}`);
const listed = await api.get<Row[]>(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`);
runs = (await Promise.all(listed.map(row => api.get<Row>(`/api/heartbeat-runs/${row.id}`))))
.filter(row => row.agentId === fixtures.agent.id && [row.nativeIssueId, row.issueId, row.contextSnapshot?.issueId, row.contextSnapshot?.taskId].includes(issue!.id))
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)) || a.id.localeCompare(b.id));
observation.issueDone = issue.status === "done";
observation.runCount = runs.length;
observation.successfulCommittedRuns = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded" &&
run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed").length;
observation.sourceContinuation = runs[0]?.resultJson?.nativeResult?.continuation?.kind ?? null;
observation.successorSource = runs[1]?.contextSnapshot?.source ?? null;
observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null);
observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_")));
}
async function save(phase: string) {
const checks = gradeAutomaticRepository(observation);
if (issue) input.observe(issue, runs, checks);
await input.evidence("automatic-repository.json", { schema: "paperclip.e2e.automatic-repository.v1", phase, observation, checks,
runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source,
finalizationReason: run.resultJson?.finalizationReasonCode, startedAt: run.startedAt, finishedAt: run.finishedAt })) });
}
try {
if (fixtures.project || execution.profile.generation !== "native") throw new Error("Automatic repository requires a projectless native fixture");
const agent = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
if (agent.adapterConfig?.cwd) throw new Error("Automatic repository must not inherit an operator cwd");
await api.patch(`/api/companies/${fixtures.company.id}/budgets`, { budgetMonthlyCents: 1000 });
await api.patch(`/api/agents/${fixtures.agent.id}/budgets`, { budgetMonthlyCents: 1000 });
const companyBudget = await api.get<Row>(`/api/companies/${fixtures.company.id}`);
const agentBudget = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
if (companyBudget.budgetMonthlyCents !== 1000 || agentBudget.budgetMonthlyCents !== 1000)
throw new Error("Automatic repository hard-stop budgets were not retained");
const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name,
title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", requireExplicitTitle: true });
issue = await api.get<Row>(`/api/issues/${created.issueId}`);
await page.goto(`/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`);
await pollUntil({ label: "repository setup and automatic result delivery", deadlineAt: input.deadlineAt, intervalMs: 750,
load: async () => { await refresh(); return { issue, runs }; },
accept: () => observation.issueDone && observation.successfulCommittedRuns === 2,
reject: state => state.runs.length > 2 ? "Unexpected extra provider admission" :
state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A provider admission failed" : undefined });
const view = await api.get<Row>(`/api/issues/${issue.id}/workspace`);
const cwd = await realpath(view.workspace.cwd), isolated = await realpath(input.temporaryRoot);
if (!cwd.startsWith(`${isolated}${path.sep}`) || !cwd.endsWith(`/isolated-workspaces/${fixtures.company.id}/${issue.id}`))
throw new Error("Repository oracle requires the confined task directory");
const repositories = view.repositories.filter((repo: Row) => repo.state === "ready" && repo.repoUrl === "https://github.com/octocat/Hello-World");
if (repositories.length !== 1 || !/^\.paperclip-repositories\/task-repo-[a-f0-9]{24}$/.test(repositories[0].relativePath))
throw new Error("Expected one contained prepared repository");
const checkout = await realpath(path.join(cwd, repositories[0].relativePath));
if (!checkout.startsWith(`${cwd}${path.sep}`)) throw new Error("Prepared repository escaped the task directory");
const git = async (args: string[]) => (await execFile("git", ["-C", checkout, ...args], {
env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" },
timeout: 20_000, maxBuffer: 262_144, encoding: "utf8",
})).stdout;
observation.actualCommit = (await git(["rev-parse", "HEAD"])).trim();
observation.actualReadme = await git(["show", "HEAD:README"]);
observation.readyRepository = repositories[0].pinnedCommit === observation.actualCommit;
const context = await api.get<Row>(`/api/issues/${issue.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(files.context)}`);
observation.savedContext = context.resource?.workspaceKind === "execution_workspace" && typeof context.content?.data === "string" ? context.content.data : null;
const [artifact] = await readRegisteredArtifacts(api, issue.id, runs[1]!.id, [files.result]);
observation.downloaded = JSON.parse(artifact.content);
observation.downloadVerified = artifact.contentVerified;
observation.pendingInteractions = (await api.get<Row[]>(`/api/issues/${issue.id}/interactions`)).filter(row => row.status === "pending").length;
await page.reload();
await expect(page.getByTestId("task-chat-thread-header").getByTestId("issue-detail-header")).toBeVisible();
const delivery = page.locator(`a[href*="${artifact.attachmentId}"]`).first();
await expect(delivery).toBeVisible({ timeout: 30_000 });
observation.visibleDelivery = true;
await save("completed");
await input.capture("final-state", "Automatically resumed repository result", "final-state.png");
const checks = gradeAutomaticRepository(observation);
const failed = checks.filter(check => !check.passed);
if (failed.length) throw new Error(`Automatic repository oracle failure: ${failed.map(check => check.id).join(", ")}`);
return { issue, runs, checks };
} catch (error) {
await refresh().catch(() => undefined);
await save("failed");
throw error;
}
}
@@ -0,0 +1,29 @@
import { expect, it } from "vitest";
import { gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js";
const sha = "a".repeat(40);
const valid: AutomaticRepositoryObservation = { reference: "REFERENCE-fresh", issueDone: true, runCount: 2,
successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"],
readyRepository: true, savedContext: "REFERENCE-fresh\n", actualReadme: "Hello World!\n", actualCommit: sha,
downloaded: { reference: "REFERENCE-fresh", readme: "Hello World!\n", commit: sha }, downloadVerified: true,
visibleDelivery: true, pendingInteractions: 0, monitorScheduled: false };
it("accepts independent checkout, retained context, and accessible registered output", () => {
expect(gradeAutomaticRepository(valid).every(check => check.passed)).toBe(true);
});
it("rejects unexpected result fields rather than trusting a model's extra claims", () => {
const extra = { ...valid, downloaded: { ...valid.downloaded, claimedPass: true } };
expect(gradeAutomaticRepository(extra).find(check => check.id === "independent-result")?.passed).toBe(false);
});
it.each<Partial<AutomaticRepositoryObservation>>([
{ runCount: 1 }, { successfulCommittedRuns: 1 }, { sourceContinuation: "monitor" }, { successorSource: "issue.comment" },
{ workspaceIds: ["old", "new"] }, { savedContext: null }, { readyRepository: false }, { downloaded: null },
{ downloaded: { ...valid.downloaded, reference: "REFERENCE-stale" } },
{ downloaded: { ...valid.downloaded, readme: "I inspected it and all checks pass" } },
{ downloaded: { ...valid.downloaded, commit: "b".repeat(40) } },
{ actualCommit: null }, { downloadVerified: false }, { visibleDelivery: false }, { pendingInteractions: 1 }, { monitorScheduled: true },
])("rejects a plausible incomplete or wrong result %j", fault => {
expect(gradeAutomaticRepository({ ...valid, ...fault }).some(check => !check.passed)).toBe(true);
});
+73
View File
@@ -0,0 +1,73 @@
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import type { RunnerTaskFixture } from "./types.js";
export function automaticRepositoryFiles(nonce: string) {
return { context: `setup-context-${nonce}.txt`, result: `repository-result-${nonce}.json`, reference: `REFERENCE-${nonce}` };
}
export const automaticRepositoryTask: RunnerTaskFixture = {
id: "automatic-repository-resume", label: "Repository preparation continues without supervision", groups: [],
flow: "automatic_repository", workMode: "standard", expectedRunCount: 2,
automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 12 * 60_000, daytona: 15 * 60_000 },
turnTimeoutMs: 5 * 60_000, expectedTerminalState: { issue: "done", run: "succeeded" },
buildTitle: nonce => `Verify a managed repository ${nonce}`,
buildVisibleMarker: nonce => automaticRepositoryFiles(nonce).result,
buildPrompt(nonce) {
const files = automaticRepositoryFiles(nonce);
return [
"Verify a repository in this projectless task and deliver the result. Do not create a project or another task.",
`Before repository setup, save ${files.context} in the task directory with exactly ${JSON.stringify(`${files.reference}\n`)}. The final newline is an actual LF.`,
"Use the task's managed repository capability to prepare https://github.com/octocat/Hello-World. Keep the task directory as the root.",
"When the checkout is ready, read its committed README and HEAD commit. Read the saved context file again; preserve its original reference.",
`Write ${files.result} as JSON with exactly these fields: reference (the saved reference without its trailing LF), readme (the complete committed README bytes as text), and commit (the full HEAD SHA).`,
`Register ${files.result} as a downloadable artifact work product with that exact title. Deliver its accessible link and finish the verification. Repository setup alone does not complete this request.`,
].join("\n");
},
buildMatchers: () => [],
};
export interface AutomaticRepositoryObservation {
reference: string;
issueDone: boolean;
runCount: number;
successfulCommittedRuns: number;
sourceContinuation: string | null;
successorSource: string | null;
workspaceIds: Array<string | null>;
readyRepository: boolean;
savedContext: string | null;
actualReadme: string | null;
actualCommit: string | null;
downloaded: { reference?: unknown; readme?: unknown; commit?: unknown } | null;
downloadVerified: boolean;
visibleDelivery: boolean;
pendingInteractions: number;
monitorScheduled: boolean;
}
export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) {
const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail });
return [
check("durable-completion", value.issueDone && value.runCount === 2 && value.successfulCommittedRuns === 2,
"Two committed provider admissions must finish the requested result."),
check("server-owned-continuation", value.sourceContinuation === "same_agent" && value.successorSource === "native_status_decision" && !value.monitorScheduled,
"Repository setup must continue through the saved same-agent wake without a timer or user message."),
check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository,
"Both admissions must retain the same task workspace and its prepared repository."),
check("retained-context", value.savedContext === `${value.reference}\n`, "The original task file must survive setup byte for byte."),
check("independent-result", value.downloadVerified && value.downloaded !== null &&
Object.keys(value.downloaded).sort().join(",") === "commit,readme,reference" && value.downloaded.reference === value.reference &&
typeof value.actualReadme === "string" && value.actualReadme.length > 0 && value.downloaded?.readme === value.actualReadme &&
typeof value.actualCommit === "string" && /^[a-f0-9]{40}$/.test(value.actualCommit) && value.downloaded?.commit === value.actualCommit,
"Downloaded result bytes must agree with the independently inspected checkout and original reference."),
check("accessible-delivery", value.visibleDelivery && value.pendingInteractions === 0, "The originating task must render its result link without an unanswered question."),
];
}
export function automaticRepositoryDigest() {
const hash = createHash("sha256");
for (const file of ["automatic-repository.ts", "automatic-repository-flow.ts", "registered-artifact.ts", "stock-harness.ts", "runner.spec.ts", "../../skills/paperclip/SKILL.md"])
hash.update(file).update(readFileSync(new URL(file, import.meta.url)));
return hash.digest("hex");
}
+3 -3
View File
@@ -211,10 +211,10 @@ describe("runner E2E catalog", () => {
expect(localIntegrityTasks).toHaveLength(2);
expect(openRouterBreadthTasks).toHaveLength(3);
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
1, 3, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2,
1, 3, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2,
]);
expect(validateRunnerCatalog()).toHaveLength(736);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(736);
expect(validateRunnerCatalog()).toHaveLength(740);
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(740);
expect(
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
).toHaveLength(48);
+11
View File
@@ -1,6 +1,7 @@
import { caseHandoffTask, caseHandoffDefinitionDigest } from "./case-handoffs-cases.js";
import { caseReceiptTask, caseReceiptDefinitionDigest, CASE_RECEIPT_BUDGET_CENTS, CASE_RECEIPT_MAX_TURNS, CASE_RECEIPT_RUN_TIMEOUT_MS } from "./case-receipt-cases.js";
import { TASK_WORKSPACES_SUITE, taskWorkspaceTask, taskWorkspaceProfile, taskWorkspaceDefinitionDigest } from "./task-workspaces-cases.js";
import { automaticRepositoryTask, automaticRepositoryDigest } from "./automatic-repository.js";
import { nativeActiveStopTasks } from "./native-active-stop-tasks.js";
import { piControlTasks } from "./pi-controls-cases.js";
import { cursorNativeTasks } from "./cursor-native-cases.js";
@@ -1166,6 +1167,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
definitionMetadata: { version: 1, definitionDigest: taskWorkspaceDefinitionDigest(), scheduling: "explicit-only", providerTurns: 3,
attempts: 1, budgetMonthlyCents: 1000, project: "none", cwd: "unconfigured-task-default", oracle: "public-file-download-and-host-nested-git", remoteLifecycle: "per_turn" },
},
{
id: "automatic-repository", label: "Automatic repository continuation", manualOnly: true,
description: "A projectless user request stages its managed repository, continues automatically, and delivers independently verified downloaded bytes.",
groups: [], profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id))
.map(taskWorkspaceProfile).map(productionDefaultHireProfile),
environments: runnerEnvironments, tasks: [automaticRepositoryTask], expectedMatrixSize: 4,
definitionMetadata: { version: 1, definitionDigest: automaticRepositoryDigest(), scheduling: "explicit-only",
providerTurns: 2, attempts: 1, budgetMonthlyCents: 1000, project: "none", instructions: "production-default",
surface: "paperclip", oracle: "registered-download-and-independent-checkout", remoteLifecycle: "per_turn" },
},
{
id: "pi-controls", label: "Pi active controls", manualOnly: true,
description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.",
+3 -6
View File
@@ -64,8 +64,7 @@ describe("extended ACP harness qualification", () => {
const config = cell.profile.buildAgent({ executionId: "warm", environmentId: "env", environmentFixtureId: cell.environment.id, workspacePath: "/tmp/workspace", secretRefs: { [cell.profile.credential]: { type: "secret_ref", secretId: "11111111-1111-4111-8111-111111111111", version: "latest" } } }).adapterConfig;
expect(config).toMatchObject({ lifecycleMode: "warm", idleTimeoutMs: 300_000, timeoutSec: 120 });
const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined();
else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]);
expect(admission).toBeUndefined();
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit");
}
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => cell.suite.id === "rich-acp-warm-continuity")).toBe(false);
@@ -94,8 +93,7 @@ describe("extended ACP harness qualification", () => {
expect(new Set(cells.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"]));
for (const cell of cells) {
const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined();
else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]);
expect(admission).toBeUndefined();
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: cell.profile.qualificationCandidate === "pi" ? "cursor" : "pi" } }])).toThrow("explicit");
}
}
@@ -109,8 +107,7 @@ describe("extended ACP harness qualification", () => {
for (const cell of cells) {
expect(cell.suite.manualOnly).toBe(true);
const admission = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined();
else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]);
expect(admission).toBeUndefined();
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit");
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: "pi" } }])).toThrow("explicit");
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, id: "unrelated-manual-suite" } }])).toThrow("explicit");
@@ -0,0 +1,27 @@
import { afterEach, expect, it, vi } from "vitest";
import { runnerMatrix } from "./catalog.js";
import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
const profilesModule = "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
const copilot = runnerMatrix.find(cell => cell.suite.id === "extended-harnesses" && cell.profile.qualificationCandidate === "copilot")!;
afterEach(() => { vi.doUnmock(profilesModule); vi.resetModules(); });
it("uses normal production admission for the current qualified Copilot profile", async () => {
expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined();
const { buildRunnerE2EProcessEnvironment } = await import("./harness-env.js");
expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [copilot])
.PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
});
it("retains exact, explicit admission when a future profile is pending", async () => {
const actual = await vi.importActual<typeof import("../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js")>(profilesModule);
vi.doMock(profilesModule, () => ({ ...actual, QUALIFIED_ACPX_PROFILES: { ...actual.QUALIFIED_ACPX_PROFILES,
copilot: { ...actual.QUALIFIED_ACPX_PROFILES.copilot, qualificationStatus: "pending" } } }));
vi.resetModules();
const { buildRunnerE2EProcessEnvironment } = await import("./harness-env.js");
const env = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [copilot]);
expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "copilot", model: copilot.profile.model }]);
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...copilot, suite: { ...copilot.suite, manualOnly: false } }])).toThrow("explicit");
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...copilot, suite: { ...copilot.suite, id: "unrelated-suite" } }])).toThrow("explicit");
expect(() => buildRunnerE2EProcessEnvironment({}, [copilot, { ...copilot, profile: { ...copilot.profile, model: "conflicting-model" } }])).toThrow("Conflicting");
});
+2 -2
View File
@@ -60,11 +60,11 @@ it("admits only the 20 agreed Copilot cells through normal installed resolution
expect(() => assertInstalledCliSelection(env, [{ ...cell, id: `${cell.id}-unreviewed` }])).toThrow("explicit supported");
}
});
it("qualified providers drop ambient admission while pending Copilot retains its exact gate", () => {
it("current qualified providers drop ambient admission, including explicit Copilot cells", () => {
for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
for (const agent of ["copilot"]) {
const cell = runnerMatrix.find(e => e.profile.qualificationCandidate === agent)!;
expect(JSON.parse(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent, model: cell.profile.model }]);
expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
}
const changed = { ...cells[0]!, profile: { ...cells[0]!.profile, model: "custom/explicit-model" } };
expect(buildRunnerE2EProcessEnvironment({}, [changed]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
@@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => {
expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true);
expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]);
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false);
expect(validateRunnerCatalog()).toHaveLength(736);
expect(validateRunnerCatalog()).toHaveLength(740);
for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow();
});
+10
View File
@@ -1,6 +1,7 @@
import { runCaseHandoffsFlow } from "./case-handoffs-flow.js";
import { runCaseReceiptFlow } from "./case-receipt-flow.js";
import { runTaskWorkspacesFlow } from "./task-workspaces-flow.js";
import { runAutomaticRepositoryFlow } from "./automatic-repository-flow.js";
import { runNativeActiveStopFlow as runCopilotActiveStopFlow } from "./copilot-active-stop-flow.js";
import { nativeActiveStopTasks as copilotActiveStopTasks } from "./copilot-active-stop-tasks.js";
import type { RestartRunnerIdentity } from "./process-tree-owner.js";
@@ -1020,6 +1021,15 @@ for (const execution of executions) {
matcherResults,
}, secrets);
await captureScreenshot("final-state", "Final visible task state", "final-state.png");
} else if (execution.task.flow === "automatic_repository") {
const journey = await runAutomaticRepositoryFlow({ page, api, fixtures, execution, nonce, temporaryRoot,
deadlineAt: startedAtMs + deadlineMs - 30_000,
observe: (currentIssue, currentRuns, checks) => {
issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[];
matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `automaticRepository.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
}, capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
});
issue = journey.issue as IssueRecord; selectedRuns = journey.runs as RunRecord[];
} else if (execution.task.flow === "task_workspaces") {
const journey = await runTaskWorkspacesFlow({ page, api, fixtures, execution, nonce, temporaryRoot,
deadlineAt: startedAtMs + deadlineMs - 30_000,
+1
View File
@@ -20,6 +20,7 @@ export type RunnerTaskWorkMode = "standard" | "planning" | "ask";
export type RunnerTaskFlow =
| "case_handoff"
| "task_workspaces"
| "automatic_repository"
| "repository_skills"
| "provider_connection"
| "plan_task_guidance"