mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 10:13:11 +02:00
test: exercise automatic repository continuation through a real task
Use production hire instructions and the existing paid browser harness. Compare delivered bytes with the actual checkout, require retained workspace and context, and reject timer or user-driven continuation. Keep current qualified provider admission free of ambient overrides and retain explicit pending-profile gate controls. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
dff63551fe
commit
de14edc606
12 files changed
+288
-12
No files matched your search
@@ -1,5 +1,22 @@
|
||||
# Paid runner full-stack E2E
|
||||
|
||||
## Automatic repository continuation
|
||||
|
||||
The explicit-only `automatic-repository` suite is an early repair check for
|
||||
native Codex and Claude on local and Daytona execution. One browser request
|
||||
must stage its managed repository, continue without a timer or another user
|
||||
message, and deliver a registered download. The oracle compares downloaded
|
||||
JSON with the actual checkout and the original task file. Both admissions
|
||||
must retain one workspace and commit their native results. The fixture uses
|
||||
production hire instructions and records the selected catalog model. These four
|
||||
development cells do not qualify Slack, staging configurations, migration,
|
||||
finalization restart, or the complete dependable-work milestone.
|
||||
|
||||
Select it explicitly with `--suite automatic-repository`. Every execution has
|
||||
one attempt, two expected provider admissions, and 1,000-cent company and agent
|
||||
hard stops. Automatic infrastructure retries are disabled. Existing campaign
|
||||
reports retain failures, timings, usage, runtime costs, and source identity.
|
||||
|
||||
## Live provider connection journeys
|
||||
|
||||
See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { realpath } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import { pollUntil, type RunnerApi } from "./api.js";
|
||||
import { createTaskThroughUi } from "./user-actions.js";
|
||||
import { readRegisteredArtifacts } from "./registered-artifact.js";
|
||||
import { automaticRepositoryFiles, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js";
|
||||
import type { LiveFixtureValues } from "./live-fixtures.js";
|
||||
import type { MatrixExecution } from "./types.js";
|
||||
|
||||
type Row = Record<string, any>;
|
||||
const execFile = promisify(execFileCallback);
|
||||
|
||||
export async function runAutomaticRepositoryFlow(input: {
|
||||
page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string;
|
||||
temporaryRoot: string; deadlineAt: number;
|
||||
observe(issue: Row, runs: Row[], checks: ReturnType<typeof gradeAutomaticRepository>): void;
|
||||
capture(id: string, label: string, file: string): Promise<void>;
|
||||
evidence(name: string, value: unknown): Promise<void>;
|
||||
}) {
|
||||
const { page, api, fixtures, execution, nonce } = input;
|
||||
const files = automaticRepositoryFiles(nonce);
|
||||
let issue: Row | undefined;
|
||||
let runs: Row[] = [];
|
||||
const observation: AutomaticRepositoryObservation = { reference: files.reference, issueDone: false, runCount: 0,
|
||||
successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false,
|
||||
savedContext: null, actualReadme: null, actualCommit: null, downloaded: null, downloadVerified: false,
|
||||
visibleDelivery: false, pendingInteractions: -1, monitorScheduled: false };
|
||||
async function refresh() {
|
||||
if (!issue) return;
|
||||
issue = await api.get<Row>(`/api/issues/${issue.id}`);
|
||||
const listed = await api.get<Row[]>(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`);
|
||||
runs = (await Promise.all(listed.map(row => api.get<Row>(`/api/heartbeat-runs/${row.id}`))))
|
||||
.filter(row => row.agentId === fixtures.agent.id && [row.nativeIssueId, row.issueId, row.contextSnapshot?.issueId, row.contextSnapshot?.taskId].includes(issue!.id))
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)) || a.id.localeCompare(b.id));
|
||||
observation.issueDone = issue.status === "done";
|
||||
observation.runCount = runs.length;
|
||||
observation.successfulCommittedRuns = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded" &&
|
||||
run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed").length;
|
||||
observation.sourceContinuation = runs[0]?.resultJson?.nativeResult?.continuation?.kind ?? null;
|
||||
observation.successorSource = runs[1]?.contextSnapshot?.source ?? null;
|
||||
observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null);
|
||||
observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_")));
|
||||
}
|
||||
async function save(phase: string) {
|
||||
const checks = gradeAutomaticRepository(observation);
|
||||
if (issue) input.observe(issue, runs, checks);
|
||||
await input.evidence("automatic-repository.json", { schema: "paperclip.e2e.automatic-repository.v1", phase, observation, checks,
|
||||
runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source,
|
||||
finalizationReason: run.resultJson?.finalizationReasonCode, startedAt: run.startedAt, finishedAt: run.finishedAt })) });
|
||||
}
|
||||
try {
|
||||
if (fixtures.project || execution.profile.generation !== "native") throw new Error("Automatic repository requires a projectless native fixture");
|
||||
const agent = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
|
||||
if (agent.adapterConfig?.cwd) throw new Error("Automatic repository must not inherit an operator cwd");
|
||||
await api.patch(`/api/companies/${fixtures.company.id}/budgets`, { budgetMonthlyCents: 1000 });
|
||||
await api.patch(`/api/agents/${fixtures.agent.id}/budgets`, { budgetMonthlyCents: 1000 });
|
||||
const companyBudget = await api.get<Row>(`/api/companies/${fixtures.company.id}`);
|
||||
const agentBudget = await api.get<Row>(`/api/agents/${fixtures.agent.id}`);
|
||||
if (companyBudget.budgetMonthlyCents !== 1000 || agentBudget.budgetMonthlyCents !== 1000)
|
||||
throw new Error("Automatic repository hard-stop budgets were not retained");
|
||||
const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name,
|
||||
title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", requireExplicitTitle: true });
|
||||
issue = await api.get<Row>(`/api/issues/${created.issueId}`);
|
||||
await page.goto(`/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`);
|
||||
await pollUntil({ label: "repository setup and automatic result delivery", deadlineAt: input.deadlineAt, intervalMs: 750,
|
||||
load: async () => { await refresh(); return { issue, runs }; },
|
||||
accept: () => observation.issueDone && observation.successfulCommittedRuns === 2,
|
||||
reject: state => state.runs.length > 2 ? "Unexpected extra provider admission" :
|
||||
state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A provider admission failed" : undefined });
|
||||
const view = await api.get<Row>(`/api/issues/${issue.id}/workspace`);
|
||||
const cwd = await realpath(view.workspace.cwd), isolated = await realpath(input.temporaryRoot);
|
||||
if (!cwd.startsWith(`${isolated}${path.sep}`) || !cwd.endsWith(`/isolated-workspaces/${fixtures.company.id}/${issue.id}`))
|
||||
throw new Error("Repository oracle requires the confined task directory");
|
||||
const repositories = view.repositories.filter((repo: Row) => repo.state === "ready" && repo.repoUrl === "https://github.com/octocat/Hello-World");
|
||||
if (repositories.length !== 1 || !/^\.paperclip-repositories\/task-repo-[a-f0-9]{24}$/.test(repositories[0].relativePath))
|
||||
throw new Error("Expected one contained prepared repository");
|
||||
const checkout = await realpath(path.join(cwd, repositories[0].relativePath));
|
||||
if (!checkout.startsWith(`${cwd}${path.sep}`)) throw new Error("Prepared repository escaped the task directory");
|
||||
const git = async (args: string[]) => (await execFile("git", ["-C", checkout, ...args], {
|
||||
env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" },
|
||||
timeout: 20_000, maxBuffer: 262_144, encoding: "utf8",
|
||||
})).stdout;
|
||||
observation.actualCommit = (await git(["rev-parse", "HEAD"])).trim();
|
||||
observation.actualReadme = await git(["show", "HEAD:README"]);
|
||||
observation.readyRepository = repositories[0].pinnedCommit === observation.actualCommit;
|
||||
const context = await api.get<Row>(`/api/issues/${issue.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(files.context)}`);
|
||||
observation.savedContext = context.resource?.workspaceKind === "execution_workspace" && typeof context.content?.data === "string" ? context.content.data : null;
|
||||
const [artifact] = await readRegisteredArtifacts(api, issue.id, runs[1]!.id, [files.result]);
|
||||
observation.downloaded = JSON.parse(artifact.content);
|
||||
observation.downloadVerified = artifact.contentVerified;
|
||||
observation.pendingInteractions = (await api.get<Row[]>(`/api/issues/${issue.id}/interactions`)).filter(row => row.status === "pending").length;
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("task-chat-thread-header").getByTestId("issue-detail-header")).toBeVisible();
|
||||
const delivery = page.locator(`a[href*="${artifact.attachmentId}"]`).first();
|
||||
await expect(delivery).toBeVisible({ timeout: 30_000 });
|
||||
observation.visibleDelivery = true;
|
||||
await save("completed");
|
||||
await input.capture("final-state", "Automatically resumed repository result", "final-state.png");
|
||||
const checks = gradeAutomaticRepository(observation);
|
||||
const failed = checks.filter(check => !check.passed);
|
||||
if (failed.length) throw new Error(`Automatic repository oracle failure: ${failed.map(check => check.id).join(", ")}`);
|
||||
return { issue, runs, checks };
|
||||
} catch (error) {
|
||||
await refresh().catch(() => undefined);
|
||||
await save("failed");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js";
|
||||
|
||||
const sha = "a".repeat(40);
|
||||
const valid: AutomaticRepositoryObservation = { reference: "REFERENCE-fresh", issueDone: true, runCount: 2,
|
||||
successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"],
|
||||
readyRepository: true, savedContext: "REFERENCE-fresh\n", actualReadme: "Hello World!\n", actualCommit: sha,
|
||||
downloaded: { reference: "REFERENCE-fresh", readme: "Hello World!\n", commit: sha }, downloadVerified: true,
|
||||
visibleDelivery: true, pendingInteractions: 0, monitorScheduled: false };
|
||||
|
||||
it("accepts independent checkout, retained context, and accessible registered output", () => {
|
||||
expect(gradeAutomaticRepository(valid).every(check => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects unexpected result fields rather than trusting a model's extra claims", () => {
|
||||
const extra = { ...valid, downloaded: { ...valid.downloaded, claimedPass: true } };
|
||||
expect(gradeAutomaticRepository(extra).find(check => check.id === "independent-result")?.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each<Partial<AutomaticRepositoryObservation>>([
|
||||
{ runCount: 1 }, { successfulCommittedRuns: 1 }, { sourceContinuation: "monitor" }, { successorSource: "issue.comment" },
|
||||
{ workspaceIds: ["old", "new"] }, { savedContext: null }, { readyRepository: false }, { downloaded: null },
|
||||
{ downloaded: { ...valid.downloaded, reference: "REFERENCE-stale" } },
|
||||
{ downloaded: { ...valid.downloaded, readme: "I inspected it and all checks pass" } },
|
||||
{ downloaded: { ...valid.downloaded, commit: "b".repeat(40) } },
|
||||
{ actualCommit: null }, { downloadVerified: false }, { visibleDelivery: false }, { pendingInteractions: 1 }, { monitorScheduled: true },
|
||||
])("rejects a plausible incomplete or wrong result %j", fault => {
|
||||
expect(gradeAutomaticRepository({ ...valid, ...fault }).some(check => !check.passed)).toBe(true);
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import type { RunnerTaskFixture } from "./types.js";
|
||||
|
||||
export function automaticRepositoryFiles(nonce: string) {
|
||||
return { context: `setup-context-${nonce}.txt`, result: `repository-result-${nonce}.json`, reference: `REFERENCE-${nonce}` };
|
||||
}
|
||||
|
||||
export const automaticRepositoryTask: RunnerTaskFixture = {
|
||||
id: "automatic-repository-resume", label: "Repository preparation continues without supervision", groups: [],
|
||||
flow: "automatic_repository", workMode: "standard", expectedRunCount: 2,
|
||||
automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 12 * 60_000, daytona: 15 * 60_000 },
|
||||
turnTimeoutMs: 5 * 60_000, expectedTerminalState: { issue: "done", run: "succeeded" },
|
||||
buildTitle: nonce => `Verify a managed repository ${nonce}`,
|
||||
buildVisibleMarker: nonce => automaticRepositoryFiles(nonce).result,
|
||||
buildPrompt(nonce) {
|
||||
const files = automaticRepositoryFiles(nonce);
|
||||
return [
|
||||
"Verify a repository in this projectless task and deliver the result. Do not create a project or another task.",
|
||||
`Before repository setup, save ${files.context} in the task directory with exactly ${JSON.stringify(`${files.reference}\n`)}. The final newline is an actual LF.`,
|
||||
"Use the task's managed repository capability to prepare https://github.com/octocat/Hello-World. Keep the task directory as the root.",
|
||||
"When the checkout is ready, read its committed README and HEAD commit. Read the saved context file again; preserve its original reference.",
|
||||
`Write ${files.result} as JSON with exactly these fields: reference (the saved reference without its trailing LF), readme (the complete committed README bytes as text), and commit (the full HEAD SHA).`,
|
||||
`Register ${files.result} as a downloadable artifact work product with that exact title. Deliver its accessible link and finish the verification. Repository setup alone does not complete this request.`,
|
||||
].join("\n");
|
||||
},
|
||||
buildMatchers: () => [],
|
||||
};
|
||||
|
||||
export interface AutomaticRepositoryObservation {
|
||||
reference: string;
|
||||
issueDone: boolean;
|
||||
runCount: number;
|
||||
successfulCommittedRuns: number;
|
||||
sourceContinuation: string | null;
|
||||
successorSource: string | null;
|
||||
workspaceIds: Array<string | null>;
|
||||
readyRepository: boolean;
|
||||
savedContext: string | null;
|
||||
actualReadme: string | null;
|
||||
actualCommit: string | null;
|
||||
downloaded: { reference?: unknown; readme?: unknown; commit?: unknown } | null;
|
||||
downloadVerified: boolean;
|
||||
visibleDelivery: boolean;
|
||||
pendingInteractions: number;
|
||||
monitorScheduled: boolean;
|
||||
}
|
||||
|
||||
export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) {
|
||||
const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail });
|
||||
return [
|
||||
check("durable-completion", value.issueDone && value.runCount === 2 && value.successfulCommittedRuns === 2,
|
||||
"Two committed provider admissions must finish the requested result."),
|
||||
check("server-owned-continuation", value.sourceContinuation === "same_agent" && value.successorSource === "native_status_decision" && !value.monitorScheduled,
|
||||
"Repository setup must continue through the saved same-agent wake without a timer or user message."),
|
||||
check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository,
|
||||
"Both admissions must retain the same task workspace and its prepared repository."),
|
||||
check("retained-context", value.savedContext === `${value.reference}\n`, "The original task file must survive setup byte for byte."),
|
||||
check("independent-result", value.downloadVerified && value.downloaded !== null &&
|
||||
Object.keys(value.downloaded).sort().join(",") === "commit,readme,reference" && value.downloaded.reference === value.reference &&
|
||||
typeof value.actualReadme === "string" && value.actualReadme.length > 0 && value.downloaded?.readme === value.actualReadme &&
|
||||
typeof value.actualCommit === "string" && /^[a-f0-9]{40}$/.test(value.actualCommit) && value.downloaded?.commit === value.actualCommit,
|
||||
"Downloaded result bytes must agree with the independently inspected checkout and original reference."),
|
||||
check("accessible-delivery", value.visibleDelivery && value.pendingInteractions === 0, "The originating task must render its result link without an unanswered question."),
|
||||
];
|
||||
}
|
||||
|
||||
export function automaticRepositoryDigest() {
|
||||
const hash = createHash("sha256");
|
||||
for (const file of ["automatic-repository.ts", "automatic-repository-flow.ts", "registered-artifact.ts", "stock-harness.ts", "runner.spec.ts", "../../skills/paperclip/SKILL.md"])
|
||||
hash.update(file).update(readFileSync(new URL(file, import.meta.url)));
|
||||
return hash.digest("hex");
|
||||
}
|
||||
@@ -211,10 +211,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
1, 3, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2,
|
||||
1, 3, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(736);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(736);
|
||||
expect(validateRunnerCatalog()).toHaveLength(740);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(740);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(48);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { caseHandoffTask, caseHandoffDefinitionDigest } from "./case-handoffs-cases.js";
|
||||
import { caseReceiptTask, caseReceiptDefinitionDigest, CASE_RECEIPT_BUDGET_CENTS, CASE_RECEIPT_MAX_TURNS, CASE_RECEIPT_RUN_TIMEOUT_MS } from "./case-receipt-cases.js";
|
||||
import { TASK_WORKSPACES_SUITE, taskWorkspaceTask, taskWorkspaceProfile, taskWorkspaceDefinitionDigest } from "./task-workspaces-cases.js";
|
||||
import { automaticRepositoryTask, automaticRepositoryDigest } from "./automatic-repository.js";
|
||||
import { nativeActiveStopTasks } from "./native-active-stop-tasks.js";
|
||||
import { piControlTasks } from "./pi-controls-cases.js";
|
||||
import { cursorNativeTasks } from "./cursor-native-cases.js";
|
||||
@@ -1166,6 +1167,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
definitionMetadata: { version: 1, definitionDigest: taskWorkspaceDefinitionDigest(), scheduling: "explicit-only", providerTurns: 3,
|
||||
attempts: 1, budgetMonthlyCents: 1000, project: "none", cwd: "unconfigured-task-default", oracle: "public-file-download-and-host-nested-git", remoteLifecycle: "per_turn" },
|
||||
},
|
||||
{
|
||||
id: "automatic-repository", label: "Automatic repository continuation", manualOnly: true,
|
||||
description: "A projectless user request stages its managed repository, continues automatically, and delivers independently verified downloaded bytes.",
|
||||
groups: [], profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id))
|
||||
.map(taskWorkspaceProfile).map(productionDefaultHireProfile),
|
||||
environments: runnerEnvironments, tasks: [automaticRepositoryTask], expectedMatrixSize: 4,
|
||||
definitionMetadata: { version: 1, definitionDigest: automaticRepositoryDigest(), scheduling: "explicit-only",
|
||||
providerTurns: 2, attempts: 1, budgetMonthlyCents: 1000, project: "none", instructions: "production-default",
|
||||
surface: "paperclip", oracle: "registered-download-and-independent-checkout", remoteLifecycle: "per_turn" },
|
||||
},
|
||||
{
|
||||
id: "pi-controls", label: "Pi active controls", manualOnly: true,
|
||||
description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.",
|
||||
|
||||
@@ -64,8 +64,7 @@ describe("extended ACP harness qualification", () => {
|
||||
const config = cell.profile.buildAgent({ executionId: "warm", environmentId: "env", environmentFixtureId: cell.environment.id, workspacePath: "/tmp/workspace", secretRefs: { [cell.profile.credential]: { type: "secret_ref", secretId: "11111111-1111-4111-8111-111111111111", version: "latest" } } }).adapterConfig;
|
||||
expect(config).toMatchObject({ lifecycleMode: "warm", idleTimeoutMs: 300_000, timeoutSec: 120 });
|
||||
const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
|
||||
if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined();
|
||||
else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]);
|
||||
expect(admission).toBeUndefined();
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit");
|
||||
}
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(cell => cell.suite.id === "rich-acp-warm-continuity")).toBe(false);
|
||||
@@ -94,8 +93,7 @@ describe("extended ACP harness qualification", () => {
|
||||
expect(new Set(cells.map(cell => cell.environment.id))).toEqual(new Set(["local", "daytona"]));
|
||||
for (const cell of cells) {
|
||||
const admission = buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
|
||||
if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined();
|
||||
else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]);
|
||||
expect(admission).toBeUndefined();
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: cell.profile.qualificationCandidate === "pi" ? "cursor" : "pi" } }])).toThrow("explicit");
|
||||
}
|
||||
}
|
||||
@@ -109,8 +107,7 @@ describe("extended ACP harness qualification", () => {
|
||||
for (const cell of cells) {
|
||||
expect(cell.suite.manualOnly).toBe(true);
|
||||
const admission = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION;
|
||||
if (cell.profile.qualificationCandidate !== "copilot") expect(admission).toBeUndefined();
|
||||
else expect(JSON.parse(admission!)).toEqual([{ agent: "copilot", model: cell.profile.model }]);
|
||||
expect(admission).toBeUndefined();
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, manualOnly: false } }])).toThrow("explicit");
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, profile: { ...cell.profile, qualificationCandidate: "pi" } }])).toThrow("explicit");
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...cell.suite, id: "unrelated-manual-suite" } }])).toThrow("explicit");
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import { runnerMatrix } from "./catalog.js";
|
||||
import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
|
||||
|
||||
const profilesModule = "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
|
||||
const copilot = runnerMatrix.find(cell => cell.suite.id === "extended-harnesses" && cell.profile.qualificationCandidate === "copilot")!;
|
||||
afterEach(() => { vi.doUnmock(profilesModule); vi.resetModules(); });
|
||||
|
||||
it("uses normal production admission for the current qualified Copilot profile", async () => {
|
||||
expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined();
|
||||
const { buildRunnerE2EProcessEnvironment } = await import("./harness-env.js");
|
||||
expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [copilot])
|
||||
.PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains exact, explicit admission when a future profile is pending", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js")>(profilesModule);
|
||||
vi.doMock(profilesModule, () => ({ ...actual, QUALIFIED_ACPX_PROFILES: { ...actual.QUALIFIED_ACPX_PROFILES,
|
||||
copilot: { ...actual.QUALIFIED_ACPX_PROFILES.copilot, qualificationStatus: "pending" } } }));
|
||||
vi.resetModules();
|
||||
const { buildRunnerE2EProcessEnvironment } = await import("./harness-env.js");
|
||||
const env = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [copilot]);
|
||||
expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "copilot", model: copilot.profile.model }]);
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...copilot, suite: { ...copilot.suite, manualOnly: false } }])).toThrow("explicit");
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...copilot, suite: { ...copilot.suite, id: "unrelated-suite" } }])).toThrow("explicit");
|
||||
expect(() => buildRunnerE2EProcessEnvironment({}, [copilot, { ...copilot, profile: { ...copilot.profile, model: "conflicting-model" } }])).toThrow("Conflicting");
|
||||
});
|
||||
@@ -60,11 +60,11 @@ it("admits only the 20 agreed Copilot cells through normal installed resolution
|
||||
expect(() => assertInstalledCliSelection(env, [{ ...cell, id: `${cell.id}-unreviewed` }])).toThrow("explicit supported");
|
||||
}
|
||||
});
|
||||
it("qualified providers drop ambient admission while pending Copilot retains its exact gate", () => {
|
||||
it("current qualified providers drop ambient admission, including explicit Copilot cells", () => {
|
||||
for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
for (const agent of ["copilot"]) {
|
||||
const cell = runnerMatrix.find(e => e.profile.qualificationCandidate === agent)!;
|
||||
expect(JSON.parse(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent, model: cell.profile.model }]);
|
||||
expect(buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
}
|
||||
const changed = { ...cells[0]!, profile: { ...cells[0]!.profile, model: "custom/explicit-model" } };
|
||||
expect(buildRunnerE2EProcessEnvironment({}, [changed]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
|
||||
@@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => {
|
||||
expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true);
|
||||
expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]);
|
||||
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false);
|
||||
expect(validateRunnerCatalog()).toHaveLength(736);
|
||||
expect(validateRunnerCatalog()).toHaveLength(740);
|
||||
for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined();
|
||||
expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow();
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { runCaseHandoffsFlow } from "./case-handoffs-flow.js";
|
||||
import { runCaseReceiptFlow } from "./case-receipt-flow.js";
|
||||
import { runTaskWorkspacesFlow } from "./task-workspaces-flow.js";
|
||||
import { runAutomaticRepositoryFlow } from "./automatic-repository-flow.js";
|
||||
import { runNativeActiveStopFlow as runCopilotActiveStopFlow } from "./copilot-active-stop-flow.js";
|
||||
import { nativeActiveStopTasks as copilotActiveStopTasks } from "./copilot-active-stop-tasks.js";
|
||||
import type { RestartRunnerIdentity } from "./process-tree-owner.js";
|
||||
@@ -1020,6 +1021,15 @@ for (const execution of executions) {
|
||||
matcherResults,
|
||||
}, secrets);
|
||||
await captureScreenshot("final-state", "Final visible task state", "final-state.png");
|
||||
} else if (execution.task.flow === "automatic_repository") {
|
||||
const journey = await runAutomaticRepositoryFlow({ page, api, fixtures, execution, nonce, temporaryRoot,
|
||||
deadlineAt: startedAtMs + deadlineMs - 30_000,
|
||||
observe: (currentIssue, currentRuns, checks) => {
|
||||
issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[];
|
||||
matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `automaticRepository.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
||||
}, capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
||||
});
|
||||
issue = journey.issue as IssueRecord; selectedRuns = journey.runs as RunRecord[];
|
||||
} else if (execution.task.flow === "task_workspaces") {
|
||||
const journey = await runTaskWorkspacesFlow({ page, api, fixtures, execution, nonce, temporaryRoot,
|
||||
deadlineAt: startedAtMs + deadlineMs - 30_000,
|
||||
|
||||
@@ -20,6 +20,7 @@ export type RunnerTaskWorkMode = "standard" | "planning" | "ask";
|
||||
export type RunnerTaskFlow =
|
||||
| "case_handoff"
|
||||
| "task_workspaces"
|
||||
| "automatic_repository"
|
||||
| "repository_skills"
|
||||
| "provider_connection"
|
||||
| "plan_task_guidance"
|
||||
|
||||
Reference in new issue
Block a user