From d20304da703f8599730a88c400824c396b0567f2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 07:30:21 -0500 Subject: [PATCH] test: verify configured repositories after real runner conversion Co-Authored-By: Paperclip --- tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 16 +++ tests/runner-e2e/live-fixtures.ts | 3 +- .../repository-verification-flow.ts | 132 ++++++++++++++++++ .../repository-verification.test.ts | 62 ++++++++ tests/runner-e2e/repository-verification.ts | 101 ++++++++++++++ tests/runner-e2e/runner.spec.ts | 10 ++ tests/runner-e2e/types.ts | 1 + 8 files changed, 327 insertions(+), 4 deletions(-) create mode 100644 tests/runner-e2e/repository-verification-flow.ts create mode 100644 tests/runner-e2e/repository-verification.test.ts create mode 100644 tests/runner-e2e/repository-verification.ts diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 0a669011c49..b25fbe9c483 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -211,10 +211,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, + 1, 3, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(740); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(740); + expect(validateRunnerCatalog()).toHaveLength(748); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(748); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index f71b15cabf3..87e0609c892 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,6 +1,7 @@ import { caseHandoffTask, caseHandoffDefinitionDigest } from "./case-handoffs-cases.js"; import { caseReceiptTask, caseReceiptDefinitionDigest, CASE_RECEIPT_BUDGET_CENTS, CASE_RECEIPT_MAX_TURNS, CASE_RECEIPT_RUN_TIMEOUT_MS } from "./case-receipt-cases.js"; import { TASK_WORKSPACES_SUITE, taskWorkspaceTask, taskWorkspaceProfile, taskWorkspaceDefinitionDigest } from "./task-workspaces-cases.js"; +import { repositoryProofTasks, repositoryProofProfile, repositoryProofDigest } from "./repository-verification.js"; import { automaticRepositoryTask, automaticRepositoryDigest } from "./automatic-repository.js"; import { nativeActiveStopTasks } from "./native-active-stop-tasks.js"; import { piControlTasks } from "./pi-controls-cases.js"; @@ -1167,6 +1168,21 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ definitionMetadata: { version: 1, definitionDigest: taskWorkspaceDefinitionDigest(), scheduling: "explicit-only", providerTurns: 3, attempts: 1, budgetMonthlyCents: 1000, project: "none", cwd: "unconfigured-task-default", oracle: "public-file-download-and-host-nested-git", remoteLifecycle: "per_turn" }, }, + ...([false, true] as const).map(conversion => ({ + id: conversion ? "legacy-conversion" : "configured-repository", + label: conversion ? "Legacy conversion with verified delivery" : "Configured repository with verified delivery", + manualOnly: true, description: "Production instructions, retained configuration, and a downloaded result checked against an independent Git and file baseline.", + groups: [], profiles: [ + { native: nativeProfile({ id: "codie", label: "Codex staging-model diagnostic", provider: "codex", model: "gpt-6.1-sol", credential: "OPENAI_API_KEY", + modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-codie-model-2026-10-20" } }), legacy: runnerProfiles.find(profile => profile.id === "legacy-codex")! }, + { native: nativeProfile({ id: "claude", label: "Claude staging-model diagnostic", provider: "acpx", acpxAgent: "claude", model: "claude-opus-5-5", credential: "ANTHROPIC_API_KEY", + modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-claude-model-2026-10-20" } }), legacy: runnerProfiles.find(profile => profile.id === "legacy-claude")! }, + ].map(({ native, legacy }) => repositoryProofProfile(native, conversion ? legacy : undefined)), + environments: runnerEnvironments, tasks: repositoryProofTasks.filter(task => task.id === (conversion ? "convert-and-deliver" : "repository-output")), expectedMatrixSize: 4, + definitionMetadata: { version: 1, definitionDigest: repositoryProofDigest(), scheduling: "explicit-only", providerTurns: 1, attempts: 1, + budgetMonthlyCents: 1000, instructions: "production-default", surface: "paperclip", oracle: "registered-download-and-independent-configured-checkout", + target: "isolated-development-instance", authentication: "api-key", stagingAuthentication: "subscription", remoteLifecycle: "per_turn", stagingQualification: false }, + })), { id: "automatic-repository", label: "Automatic repository continuation", manualOnly: true, description: "A projectless user request stages its managed repository, continues automatically, and delivers independently verified downloaded bytes.", diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 87bc56b1f19..2e52b8fdbc5 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -389,7 +389,8 @@ export async function setupLiveFixtures(input: { }, }); - if (execution.task.flow === "warm_three_turn" + if (execution.task.flow === "repository_verification" + || execution.task.flow === "warm_three_turn" || (execution.suite.id === "extended-harnesses" && execution.task.id === "file-edit-validate")) { registry.register({ id: "project", diff --git a/tests/runner-e2e/repository-verification-flow.ts b/tests/runner-e2e/repository-verification-flow.ts new file mode 100644 index 00000000000..8dd7ac74a0d --- /dev/null +++ b/tests/runner-e2e/repository-verification-flow.ts @@ -0,0 +1,132 @@ +import { createHash, randomBytes } from "node:crypto"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect, type Page } from "@playwright/test"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { createTaskThroughUi } from "./user-actions.js"; +import { readRegisteredArtifacts } from "./registered-artifact.js"; +import { gradeRepositoryProof, repositoryConfigurationDigest, repositoryProofFiles, type RepositoryProofObservation } from "./repository-verification.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; +import type { MatrixExecution } from "./types.js"; + +type Row = Record; +const execFile = promisify(execFileCallback); + +export async function runRepositoryVerificationFlow(input: { + page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; + workspacePath: string; temporaryRoot: string; deadlineAt: number; + observe(issue: Row, runs: Row[], checks: ReturnType): void; + capture(id: string, label: string, file: string): Promise; + evidence(name: string, value: unknown): Promise; +}) { + const { page, api, fixtures, execution, nonce } = input; + const files = repositoryProofFiles(nonce), conversion = execution.task.id === "convert-and-deliver"; + let issue: Row | undefined, runs: Row[] = []; + const expectedLegacyAdapter = execution.profile.provider === "codex" ? "codex_local" : "claude_local"; + const observation: RepositoryProofObservation = { conversion, expectedLegacyAdapter, originalAdapter: "", finalAdapter: "", + configurationBefore: "", configurationAfter: "", instructionsBefore: "", instructionsAfter: "", permissionsPreserved: false, + expectedModel: execution.profile.model, actualModel: null, expectedProvider: execution.profile.provider === "codex" ? "codex" : "acpx:claude", + actualProvider: null, environmentRetained: false, issueDone: false, runCount: 0, committedRuns: 0, + sourceBytes: "", currentSourceBytes: null, expectedCommit: "", downloaded: null, downloadVerified: false, + visibleDelivery: false, pendingInteractions: -1 }; + async function instructionsDigest() { + const bundle = await api.get<{ entryFile: string; files: Array<{ path: string }> }>(`/api/agents/${fixtures.agent.id}/instructions-bundle`); + const files = await Promise.all([...bundle.files].sort((a, b) => a.path.localeCompare(b.path)).map(async file => ({ + path: file.path, content: (await api.get<{ content: string }>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(file.path)}`)).content, + }))); + if (bundle.entryFile !== "AGENTS.md" || files.length !== 1 || files[0]?.content !== "You are an agent in a Paperclip company.\n") + throw new Error("Repository proof requires the shipped default hire instructions, without a fixture manual"); + return createHash("sha256").update(JSON.stringify({ entryFile: bundle.entryFile, files })).digest("hex"); + } + async function refresh() { + if (!issue) return; + issue = await api.get(`/api/issues/${issue.id}`); + const listed = await api.get(`/api/companies/${fixtures.company.id}/heartbeat-runs?agentId=${fixtures.agent.id}&limit=20`); + runs = (await Promise.all(listed.map(run => api.get(`/api/heartbeat-runs/${run.id}`)))) + .filter(run => [run.nativeIssueId, run.issueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(issue!.id)); + observation.issueDone = issue.status === "done"; + observation.runCount = runs.length; + observation.committedRuns = runs.filter(run => run.status === "succeeded" && run.runtimeMode === "native" && + run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed").length; + const provider = runs[0]?.runnerProfileJson?.nativeExecutionInput?.provider; + observation.actualModel = provider?.model ?? null; + observation.actualProvider = provider ? provider.kind === "acpx" ? `acpx:${provider.agent}` : provider.kind : null; + observation.environmentRetained = runs.length === 1 && runs[0]?.contextSnapshot?.paperclipEnvironment?.id === fixtures.environment.id; + } + async function save(phase: string) { + const checks = gradeRepositoryProof(observation); + if (issue) input.observe(issue, runs, checks); + await input.evidence("repository-verification.json", { schema: "paperclip.e2e.repository-verification.v1", phase, observation, checks, + runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source, + startedAt: run.startedAt, finishedAt: run.finishedAt })) }); + } + try { + if (!fixtures.project || execution.profile.generation !== "native") throw new Error("Repository proof requires its owned configured project"); + const workspace = await realpath(input.workspacePath), root = await realpath(input.temporaryRoot); + if (workspace !== path.join(root, "workspace")) throw new Error("Repository proof cannot seed an unowned source checkout"); + const git = async (args: string[]) => (await execFile("git", ["-C", workspace, ...args], { + env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" }, + timeout: 20_000, maxBuffer: 262_144, encoding: "utf8", + })).stdout; + observation.sourceBytes = `${randomBytes(32).toString("hex")}\n`; + await writeFile(path.join(workspace, files.source), observation.sourceBytes, { flag: "wx" }); + await git(["init", "--initial-branch=verification"]); + await git(["add", "--", files.source]); + await git(["-c", "user.name=Qualification Fixture", "-c", "user.email=qualification@example.test", "-c", "commit.gpgsign=false", "commit", "-m", "Seed independent repository reference"]); + observation.expectedCommit = (await git(["rev-parse", "HEAD"])).trim(); + await api.patch(`/api/companies/${fixtures.company.id}/budgets`, { budgetMonthlyCents: 1000 }); + await api.patch(`/api/agents/${fixtures.agent.id}/budgets`, { budgetMonthlyCents: 1000 }); + const before = await api.get(`/api/agents/${fixtures.agent.id}`); + const company = await api.get(`/api/companies/${fixtures.company.id}`); + if (before.budgetMonthlyCents !== 1000 || company.budgetMonthlyCents !== 1000) throw new Error("Repository proof hard-stop budgets were not retained"); + if (before.adapterType !== (conversion ? expectedLegacyAdapter : "paperclip_runner")) throw new Error("Initial adapter does not match the requested journey"); + observation.originalAdapter = before.adapterType; + observation.configurationBefore = repositoryConfigurationDigest(before); + observation.instructionsBefore = await instructionsDigest(); + if (conversion) await api.patch(`/api/agents/${fixtures.agent.id}`, { runner: "paperclip" }); + const after = await api.get(`/api/agents/${fixtures.agent.id}`); + observation.finalAdapter = after.adapterType; + observation.configurationAfter = repositoryConfigurationDigest(after); + observation.instructionsAfter = await instructionsDigest(); + observation.permissionsPreserved = (conversion ? before.adapterConfig?.dangerouslySkipPermissions === true : true) && + (execution.profile.provider === "codex" ? after.adapterConfig?.codexPermissionMode === "never" : after.adapterConfig?.acpxPermissionMode === "approve-all"); + if (after.defaultEnvironmentId !== fixtures.environment.id || observation.configurationBefore !== observation.configurationAfter || + observation.instructionsBefore !== observation.instructionsAfter || !observation.permissionsPreserved || after.adapterType !== "paperclip_runner") + throw new Error("Production runner selection did not retain the configured agent context"); + if (conversion && execution.environment.id === "daytona" && execution.profile.provider === "codex") { + if (!/^.+@sha256:[a-f0-9]{64}$/i.test(String(fixtures.environment.config?.image ?? "")) || fixtures.environment.config?.snapshot) + throw new Error("Converted Codex requires the immutable fixture image"); + const startedAt = Date.now(); + const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( + `/api/environments/${fixtures.environment.id}/probe?companyId=${encodeURIComponent(fixtures.company.id)}`, {}, { timeout: 330_000 }); + if (!probe.ok || !probe.details?.platformQualificationRecorded || !probe.details.leaseId) throw new Error("Converted Codex platform qualification was not recorded"); + fixtures.setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt }; + } + await save("configured"); + const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, + projectName: fixtures.project.name, title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", requireExplicitTitle: true }); + issue = await api.get(`/api/issues/${created.issueId}`); + await page.goto(`/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`); + await pollUntil({ label: "configured repository result", deadlineAt: input.deadlineAt, intervalMs: 1000, + load: async () => { await refresh(); return { issue, runs }; }, + accept: () => observation.issueDone && observation.committedRuns === 1, + reject: state => state.runs.length > 1 ? "Unexpected extra provider admission" : state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) + ? "Provider admission failed" : state.runs.length && state.runs.every(run => run.status === "succeeded") && ["blocked", "in_review"].includes(state.issue!.status) ? "Task stopped before delivery" : undefined }); + const context = await api.get(`/api/issues/${issue.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(files.source)}`); + observation.currentSourceBytes = context.resource?.workspaceKind === "execution_workspace" && typeof context.content?.data === "string" ? context.content.data : null; + const [artifact] = await readRegisteredArtifacts(api, issue.id, runs[0]!.id, [files.result]); + observation.downloaded = JSON.parse(artifact.content); + observation.downloadVerified = artifact.contentVerified; + observation.pendingInteractions = (await api.get(`/api/issues/${issue.id}/interactions`)).filter(row => row.status === "pending").length; + await page.reload(); + await expect(page.locator(`a[href*="${artifact.attachmentId}"]`).first()).toBeVisible({ timeout: 30_000 }); + observation.visibleDelivery = true; + await save("completed"); + await input.capture("final-state", "Verified configured repository delivery", "final-state.png"); + const checks = gradeRepositoryProof(observation), failed = checks.filter(check => !check.passed); + if (failed.length) throw new Error(`Repository verification failed: ${failed.map(check => check.id).join(", ")}`); + return { issue, runs, checks }; + } catch (error) { await refresh().catch(() => undefined); await save("failed"); throw error; } +} diff --git a/tests/runner-e2e/repository-verification.test.ts b/tests/runner-e2e/repository-verification.test.ts new file mode 100644 index 00000000000..68eada012dc --- /dev/null +++ b/tests/runner-e2e/repository-verification.test.ts @@ -0,0 +1,62 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { gradeRepositoryProof, repositoryConfigurationDigest, repositoryProofTasks, type RepositoryProofObservation } from "./repository-verification.js"; +import { runnerSuites } from "./catalog.js"; + +function valid(): RepositoryProofObservation { + const sourceBytes = `${"a7".repeat(32)}\n`, commit = "b".repeat(40); + return { conversion: true, originalAdapter: "codex_local", expectedLegacyAdapter: "codex_local", finalAdapter: "paperclip_runner", + configurationBefore: "config", configurationAfter: "config", instructionsBefore: "instructions", instructionsAfter: "instructions", + permissionsPreserved: true, expectedModel: "gpt-6.1-sol", actualModel: "gpt-6.1-sol", expectedProvider: "codex", actualProvider: "codex", + environmentRetained: true, issueDone: true, runCount: 1, committedRuns: 1, sourceBytes, currentSourceBytes: sourceBytes, + expectedCommit: commit, downloaded: { reference: sourceBytes.trimEnd(), sourceSha256: createHash("sha256").update(sourceBytes).digest("hex"), commit }, + downloadVerified: true, visibleDelivery: true, pendingInteractions: 0 }; +} +const passes = (value: RepositoryProofObservation) => gradeRepositoryProof(value).every(check => check.passed); + +describe("configured repository and conversion qualification", () => { + it("accepts independently verified actual conversion and ordinary delivery", () => { + expect(passes(valid())).toBe(true); + expect(passes({ ...valid(), conversion: false, originalAdapter: "paperclip_runner" })).toBe(true); + }); + it.each([ + { originalAdapter: "paperclip_runner" }, { finalAdapter: "codex_local" }, + { configurationAfter: "changed environment reference" }, { instructionsAfter: "lost instructions" }, + { permissionsPreserved: false }, { environmentRetained: false }, { actualModel: "gpt-5.6-sol" }, + { actualProvider: "acpx:claude" }, { runCount: 2 }, { committedRuns: 0 }, { issueDone: false }, + { currentSourceBytes: null }, { currentSourceBytes: "recreated" }, { expectedCommit: "" }, + { downloadVerified: false }, { visibleDelivery: false }, { pendingInteractions: 1 }, + ])("rejects missing or contrary evidence %j", change => expect(passes({ ...valid(), ...change })).toBe(false)); + it.each([ + null, "a badge is green", [], {}, { reference: "wrong", sourceSha256: "wrong", commit: "wrong" }, + ])("does not accept provider success or malformed downloaded output %j", downloaded => expect(passes({ ...valid(), downloaded })).toBe(false)); + it("rejects each incorrect result field and extra fields", () => { + const baseline = valid(), result = baseline.downloaded as Record; + for (const field of ["reference", "sourceSha256", "commit"]) expect(passes({ ...baseline, downloaded: { ...result, [field]: "wrong" } })).toBe(false); + expect(passes({ ...baseline, downloaded: { ...result, success: true } })).toBe(false); + }); + it("hashes retained configuration consistently without returning its contents", () => { + const before = { defaultEnvironmentId: "env", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, + adapterConfig: { model: "model", cwd: "/owned", env: { KEY: { type: "secret_ref", secretId: "sensitive-fixture-reference", version: 1 } }, timeoutSec: 240 } }; + const reordered = { ...before, adapterConfig: { ...before.adapterConfig, env: { KEY: { version: 1, secretId: "sensitive-fixture-reference", type: "secret_ref" } } } }; + expect(repositoryConfigurationDigest(before)).toBe(repositoryConfigurationDigest(reordered)); + expect(repositoryConfigurationDigest(before)).toMatch(/^[a-f0-9]{64}$/); + expect(repositoryConfigurationDigest(before)).not.toContain("sensitive-fixture-reference"); + expect(repositoryConfigurationDigest({ ...before, defaultEnvironmentId: "other" })).not.toBe(repositoryConfigurationDigest(before)); + expect(repositoryConfigurationDigest({ ...before, adapterConfig: { ...before.adapterConfig, env: {} } })).not.toBe(repositoryConfigurationDigest(before)); + }); + it("registers explicit development cells with staging models and production instructions", () => { + for (const id of ["configured-repository", "legacy-conversion"]) { + const suite = runnerSuites.find(suite => suite.id === id)!; + expect(suite.manualOnly).toBe(true); + expect(suite.groups).toEqual([]); + expect(suite.expectedMatrixSize).toBe(4); + expect(suite.profiles.map(profile => profile.model)).toEqual(["gpt-6.1-sol", "claude-opus-5-5"]); + expect(suite.definitionMetadata?.stagingQualification).toBe(false); + expect(suite.tasks).toHaveLength(1); + expect(suite.tasks[0]?.flow).toBe("repository_verification"); + expect(suite.tasks[0]?.automaticRetryPolicy).toBe("single_attempt"); + } + expect(repositoryProofTasks.every(task => task.expectedRunCount === 1)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/repository-verification.ts b/tests/runner-e2e/repository-verification.ts new file mode 100644 index 00000000000..60897837595 --- /dev/null +++ b/tests/runner-e2e/repository-verification.ts @@ -0,0 +1,101 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerProfileFixture, RunnerTaskFixture } from "./types.js"; + +export function repositoryProofFiles(nonce: string) { + return { source: `source-reference-${nonce}.txt`, result: `repository-proof-${nonce}.json` }; +} + +export function repositoryProofProfile(native: RunnerProfileFixture, legacy?: RunnerProfileFixture): RunnerProfileFixture { + return { ...native, buildAgent(input) { + const { instructionsBundle: _manual, ...agent } = (legacy ?? native).buildAgent(input); + return { ...agent, ...(legacy ? { runner: "legacy" } : {}), budgetMonthlyCents: 1000, + adapterConfig: { ...(agent.adapterConfig as Record), model: native.model } }; + } }; +} + +export const repositoryProofTasks: readonly RunnerTaskFixture[] = ["repository-output", "convert-and-deliver"].map(id => ({ + id, label: id === "repository-output" ? "Configured repository and accessible output" : "Legacy conversion retains configuration and delivers useful work", + groups: [], flow: "repository_verification", workMode: "standard", expectedRunCount: 1, + automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 10 * 60_000, daytona: 15 * 60_000 }, + turnTimeoutMs: 4 * 60_000, expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Verify repository context ${nonce}`, + buildVisibleMarker: nonce => repositoryProofFiles(nonce).result, + buildPrompt(nonce) { + const files = repositoryProofFiles(nonce); + return [ + "Verify the configured repository and deliver a small proof file. Read the existing checkout; do not replace, clone, or recreate it.", + `Read ${files.source}. It contains a reference followed by one LF. Preserve the source file unchanged. If it is missing, report the failure instead of creating a replacement.`, + "Read the current full Git HEAD SHA and compute the SHA-256 of the exact source file bytes, including its final newline.", + `Write ${files.result} as JSON with exactly these fields: reference (the existing reference without its trailing LF), sourceSha256 (the exact source file hash), and commit (the full Git HEAD SHA).`, + `Verify the result, register ${files.result} as a downloadable artifact work product with that exact title, and deliver its accessible link in this task. Finish after delivery. Do not create other tasks or agents.`, + ].join("\n"); + }, buildMatchers: () => [], +})); + +export interface RepositoryProofObservation { + conversion: boolean; + originalAdapter: string; + expectedLegacyAdapter: string; + finalAdapter: string; + configurationBefore: string; + configurationAfter: string; + instructionsBefore: string; + instructionsAfter: string; + permissionsPreserved: boolean; + expectedModel: string; + actualModel: string | null; + actualProvider: string | null; + expectedProvider: string; + environmentRetained: boolean; + issueDone: boolean; + runCount: number; + committedRuns: number; + sourceBytes: string; + currentSourceBytes: string | null; + expectedCommit: string; + downloaded: unknown; + downloadVerified: boolean; + visibleDelivery: boolean; + pendingInteractions: number; +} + +export function gradeRepositoryProof(value: RepositoryProofObservation) { + const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail }); + const result = value.downloaded && typeof value.downloaded === "object" && !Array.isArray(value.downloaded) + ? value.downloaded as Record : null; + return [ + check("production-conversion", value.finalAdapter === "paperclip_runner" && + value.originalAdapter === (value.conversion ? value.expectedLegacyAdapter : "paperclip_runner"), + "Conversion must start with a real legacy agent and use its production runner transition."), + check("retained-configuration", Boolean(value.configurationBefore) && value.configurationBefore === value.configurationAfter && + Boolean(value.instructionsBefore) && value.instructionsBefore === value.instructionsAfter && value.permissionsPreserved && value.environmentRetained, + "Configured model, source directory, secret references, timeout, runtime settings, environment, instructions, and equivalent permissions must survive."), + check("actual-native-runtime", value.actualModel === value.expectedModel && value.actualProvider === value.expectedProvider && + value.runCount === 1 && value.committedRuns === 1 && value.issueDone, + "One actual native admission on the configured provider and model must commit the task result."), + check("retained-repository", /^[a-f0-9]{64}\n$/.test(value.sourceBytes) && value.sourceBytes === value.currentSourceBytes && /^[a-f0-9]{40}$/.test(value.expectedCommit), + "The independently seeded source reference must survive byte for byte in the configured checkout."), + check("independent-result", value.downloadVerified && result !== null && Object.keys(result).sort().join(",") === "commit,reference,sourceSha256" && + result.reference === value.sourceBytes.trimEnd() && result.sourceSha256 === createHash("sha256").update(value.sourceBytes).digest("hex") && result.commit === value.expectedCommit, + "Downloaded bytes must agree with the independent source-file and Git baseline."), + check("accessible-delivery", value.visibleDelivery && value.pendingInteractions === 0, + "The originating task must render the downloadable result with no unanswered interaction."), + ]; +} + +export function repositoryConfigurationDigest(agent: Record) { + const config = agent.adapterConfig ?? {}; + const canonical = (value: any): any => Array.isArray(value) ? value.map(canonical) : value && typeof value === "object" + ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value; + return createHash("sha256").update(JSON.stringify(canonical({ model: config.model, cwd: config.cwd, + timeoutSec: config.timeoutSec, env: config.env, defaultEnvironmentId: agent.defaultEnvironmentId, + runtimeConfig: agent.runtimeConfig }))).digest("hex"); +} + +export function repositoryProofDigest() { + const hash = createHash("sha256"); + for (const file of ["repository-verification.ts", "repository-verification-flow.ts", "registered-artifact.ts", "live-fixtures.ts", "runner.spec.ts", "catalog.ts", "stock-harness.ts"]) + hash.update(file).update(readFileSync(new URL(file, import.meta.url))); + return hash.digest("hex"); +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index c2a461bdec1..aee02c93a8b 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,6 +1,7 @@ import { runCaseHandoffsFlow } from "./case-handoffs-flow.js"; import { runCaseReceiptFlow } from "./case-receipt-flow.js"; import { runTaskWorkspacesFlow } from "./task-workspaces-flow.js"; +import { runRepositoryVerificationFlow } from "./repository-verification-flow.js"; import { runAutomaticRepositoryFlow } from "./automatic-repository-flow.js"; import { runNativeActiveStopFlow as runCopilotActiveStopFlow } from "./copilot-active-stop-flow.js"; import { nativeActiveStopTasks as copilotActiveStopTasks } from "./copilot-active-stop-tasks.js"; @@ -1021,6 +1022,15 @@ for (const execution of executions) { matcherResults, }, secrets); await captureScreenshot("final-state", "Final visible task state", "final-state.png"); + } else if (execution.task.flow === "repository_verification") { + const journey = await runRepositoryVerificationFlow({ page, api, fixtures, execution, nonce, workspacePath: workspacePath!, temporaryRoot, + deadlineAt: startedAtMs + deadlineMs - 30_000, + observe: (currentIssue, currentRuns, checks) => { + issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; + matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `repositoryVerification.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })); + }, capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), + }); + issue = journey.issue as IssueRecord; selectedRuns = journey.runs as RunRecord[]; } else if (execution.task.flow === "automatic_repository") { const journey = await runAutomaticRepositoryFlow({ page, api, fixtures, execution, nonce, temporaryRoot, deadlineAt: startedAtMs + deadlineMs - 30_000, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index b273caf6f4b..36e12df7c83 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -21,6 +21,7 @@ export type RunnerTaskFlow = | "case_handoff" | "task_workspaces" | "automatic_repository" + | "repository_verification" | "repository_skills" | "provider_connection" | "plan_task_guidance"