diff --git a/doc/plans/2026-10-20-dependable-everyday-work.md b/doc/plans/2026-10-20-dependable-everyday-work.md index 2dd300b3377..0eb582960ff 100644 --- a/doc/plans/2026-10-20-dependable-everyday-work.md +++ b/doc/plans/2026-10-20-dependable-everyday-work.md @@ -113,6 +113,15 @@ repair's Git and authorization cohort passes locally. Fresh provider remeasurement and the monitor-fallback defect remain open. +Two additional Product E2E journeys now cover configured repository delivery and +real legacy-to-native conversion followed by verified work. Each checks a fresh +fixture-owned source file, its hash and Git commit, retained configuration and +instructions, the actual native provider/model, and a registered download visible +after reload. A third explicit selection replays automatic repository setup with +the recorded Codex and Claude models. All twelve new cells declare development +API-key authentication and per-turn lifecycle differences. None counts as staging +subscription, warm lifecycle, Slack, or the complete 24-cell milestone. Fresh +provider measurement remains required; catalog registration is not acceptance. The fifth development attempt uses the recorded Codex model. It has no monitor, but selects a response wake and then reaches productive-continuation recovery. Its next admission still fails at Git setup. The first plain-root regression had diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 7173130bd16..e1777d49225 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -27,6 +27,45 @@ Source-server startup prepares the existing shared/plugin SDK build dependencies before importing native tool authority. Installed-package checks never execute checkout build helpers. +## Configured repository and conversion checks + +The explicit-only `configured-repository` and `legacy-conversion` suites each +have four development cells: Codex `gpt-6.1-sol` and Claude `claude-opus-5-5`, +on local and Daytona environments. Each uses production hire instructions, +one attempt, one expected provider admission, and 1,000-cent company and agent +hard stops. The conversion journey creates a real legacy agent and selects +Paperclip Runner through the public agent update API. It checks retained +configuration, encrypted secret references, environment, instructions, and +corresponding permission policy before submitting work through the browser. +The disposable legacy profile starts without optional CLI-only extra arguments or turn limits; +the journey does not remove settings from an existing user agent. For either +provider, Daytona's public environment probe must save platform qualification +before the same agent selects Runner. Probe leases and uncertain probe accounting +remain included in attempt evidence and cleanup. + +The observer seeds a random reference in the fixture-owned Git checkout before +task creation. It checks the downloaded JSON against the original file bytes, +SHA-256 and commit, then checks the preserved source through the workspace API. +A pass also requires the configured native provider/model, one committed run, +a visible artifact link after reload, Done, and no pending question. No agent +success claim or agent-authored test can replace these checks. + +The explicit-only `automatic-repository-configured-models` suite reuses the +projectless continuation oracle above with those same two recorded staging +models. Its four cells keep the first admission's artifact observation and the +strict requirement for a saved same-agent continuation. A monitor fallback is +still a failure of that requirement. + +These twelve development cells use API-key authentication and per-turn Runner +lifecycle. They do not qualify staging subscription credentials, warm Runner +reuse, Slack delivery, or the complete dependable-work matrix. Run one exact +ID for a bounded check: + +```sh +pnpm test:e2e:runner -- --id legacy-conversion.runner-codex.local.convert-and-deliver --max-automatic-retries 0 +pnpm test:e2e:runner -- --id automatic-repository-configured-models.runner-codex.local.automatic-repository-resume --max-automatic-retries 0 +``` + ## Latency observation bounds The explicit `codie-latency-diagnostic` arms its observation recorder before diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index ca0ec5edd22..7cba2b55c14 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -415,3 +415,17 @@ limits, case/task state and correlated receipt checks enter workflow evidence. Two passive seed cases support browser list assertions; screenshots remain on the exact primary case/task routes and document dialogs preserve that case URL. The public screenshot allowlist is unchanged. + + +## Configured repository and conversion proof + +The repository proof flow seeds only the exact `workspace` child of its resolved +fixture-owned temporary directory. It initializes that disposable Git checkout +with a fresh random reference. It never modifies a contributor checkout. Provider +credentials remain encrypted fixture secret references. Retention evidence keeps +a digest of allowlisted configuration fields rather than raw environment values. +The public API performs legacy conversion; no database mutation or private +Runner hook performs it. Only synthetic source bytes and proof JSON enter the +existing private attempt evidence and registered-artifact verification pipeline. +A failed attempt remains retained. These development checks are not staging or +Slack qualification, and they do not publish trusted archives or credentials. diff --git a/tests/runner-e2e/automatic-repository-flow.ts b/tests/runner-e2e/automatic-repository-flow.ts index c676b7345a8..89e57f4c823 100644 --- a/tests/runner-e2e/automatic-repository-flow.ts +++ b/tests/runner-e2e/automatic-repository-flow.ts @@ -6,7 +6,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { createTaskThroughUi } from "./user-actions.js"; import { readRegisteredArtifacts } from "./registered-artifact.js"; -import { automaticRepositoryFiles, automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; +import { automaticRepositoryApiState, automaticRepositoryFiles, automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; @@ -24,7 +24,8 @@ export async function runAutomaticRepositoryFlow(input: { const files = automaticRepositoryFiles(nonce); let issue: Row | undefined; let runs: Row[] = []; - const observation: AutomaticRepositoryObservation = { reference: "", issueDone: false, runCount: 0, + const observation: AutomaticRepositoryObservation = { expectedModel: execution.profile.model, + expectedProvider: execution.profile.provider === "codex" ? "codex" : "acpx:claude", actualProviders: [], reference: "", issueDone: false, runCount: 0, successfulCommittedRuns: 0, sourceContinuation: null, successorSource: null, workspaceIds: [], readyRepository: false, savedContext: null, firstSavedContext: null, firstContextAttributed: false, firstContextObservedAt: null, successorStartedAt: null, predecessorFinishedAt: null, predecessorRetagged: true, continuationReceipt: null, @@ -56,12 +57,19 @@ export async function runAutomaticRepositoryFlow(input: { observation.firstContextObservedAt = Date.now(); } } + observation.actualProviders = runs.map(run => { + const provider = run.runnerProfileJson?.nativeExecutionInput?.provider; + return { model: provider?.model ?? null, kind: provider ? provider.kind === "acpx" ? `acpx:${provider.agent}` : provider.kind : null }; + }); observation.workspaceIds = runs.map(run => run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId ?? null); observation.monitorScheduled ||= Boolean(issue.monitorNextCheckAt || runs.some(run => String(run.resultJson?.finalizationReasonCode).startsWith("scheduled_monitor_"))); } async function save(phase: string) { const checks = gradeAutomaticRepository(observation); - if (issue) input.observe(issue, runs, checks); + if (issue) { + input.observe(issue, runs, checks); + await input.evidence("api-state.json", automaticRepositoryApiState(phase, issue, runs, checks)); + } await input.evidence("automatic-repository.json", { schema: "paperclip.e2e.automatic-repository.v1", phase, observation, checks, runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source, finalizationReason: run.resultJson?.finalizationReasonCode, startedAt: run.startedAt, finishedAt: run.finishedAt })) }); diff --git a/tests/runner-e2e/automatic-repository.test.ts b/tests/runner-e2e/automatic-repository.test.ts index 4657a5d3598..a55b97624c5 100644 --- a/tests/runner-e2e/automatic-repository.test.ts +++ b/tests/runner-e2e/automatic-repository.test.ts @@ -1,9 +1,10 @@ import { expect, it } from "vitest"; -import { automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; +import { automaticRepositoryApiState, automaticRepositoryStopReason, gradeAutomaticRepository, type AutomaticRepositoryObservation } from "./automatic-repository.js"; const sha = "a".repeat(40); const reference = "c".repeat(64); -const valid: AutomaticRepositoryObservation = { reference, issueDone: true, runCount: 2, +const valid: AutomaticRepositoryObservation = { expectedModel: "gpt-6.1-sol", expectedProvider: "codex", + actualProviders: [{ model: "gpt-6.1-sol", kind: "codex" }, { model: "gpt-6.1-sol", kind: "codex" }], reference, issueDone: true, runCount: 2, successfulCommittedRuns: 2, sourceContinuation: "same_agent", successorSource: "native_status_decision", workspaceIds: ["workspace", "workspace"], readyRepository: true, savedContext: `${reference}\n`, firstSavedContext: `${reference}\n`, firstContextAttributed: true, firstContextObservedAt: 1000, successorStartedAt: 2000, predecessorFinishedAt: 1900, predecessorRetagged: false, @@ -37,6 +38,9 @@ it.each>([ { workspaceIds: ["old", "new"] }, { savedContext: null }, { firstSavedContext: null }, { firstContextAttributed: false }, { firstSavedContext: `${"d".repeat(64)}\n` }, { firstSavedContext: "REFERENCE-fresh\n" }, { firstContextObservedAt: null }, { firstContextObservedAt: 2000 }, { firstContextObservedAt: 3000 }, { successorStartedAt: null }, + { actualProviders: [] }, { actualProviders: [{ model: "gpt-6.1-sol", kind: "codex" }] }, + { actualProviders: [{ model: "gpt-5.6-sol", kind: "codex" }, valid.actualProviders[1]!] }, + { actualProviders: [valid.actualProviders[0]!, { model: "gpt-6.1-sol", kind: "acpx:claude" }] }, { readyRepository: false }, { downloaded: null }, { predecessorFinishedAt: null }, { predecessorFinishedAt: 2001 }, { predecessorRetagged: true }, { continuationReceipt: null }, { continuationReceipt: { ...valid.continuationReceipt!, dispatchId: valid.continuationReceipt!.intentId } }, @@ -48,3 +52,18 @@ it.each>([ ])("rejects a plausible incomplete or wrong result %j", fault => { expect(gradeAutomaticRepository({ ...valid, ...fault }).some(check => !check.passed)).toBe(true); }); + +it("retains required lifecycle evidence while excluding provider content and unrelated task data", () => { + const issue = { id: "task", companyId: "company", status: "done", description: "private description", privateToken: "private token" }; + const run = { id: "successor", companyId: "company", agentId: "agent", nativeIssueId: "task", status: "succeeded", + wakeupRequestId: "dispatch", contextSnapshot: { source: "native_status_decision", nativeStatusWakeIntentId: "intent", paperclipWake: { prompt: "private prompt" } }, + resultJson: { finalizationPhase: "committed", nativeResult: { summary: "private provider text" } }, privateConfig: { secret: "private secret" } }; + for (const phase of ["completed", "failed"]) { + const snapshot = automaticRepositoryApiState(phase, issue, [run], gradeAutomaticRepository(valid)); + expect(snapshot).toMatchObject({ capturePhase: phase, issue: { id: "task", status: "done" }, + runs: [{ id: "successor", wakeupRequestId: "dispatch", source: "native_status_decision", nativeStatusWakeIntentId: "intent", finalizationPhase: "committed" }] }); + expect(JSON.stringify(snapshot)).not.toMatch(/private|description|paperclipWake|nativeResult/); + } + const malformed = automaticRepositoryApiState("failed", issue, [{ ...run, contextSnapshot: { source: { private: true }, nativeStatusWakeIntentId: { private: true } }, resultJson: { finalizationPhase: { private: true } } }], []); + expect(malformed.runs[0]).toMatchObject({ source: null, nativeStatusWakeIntentId: null, finalizationPhase: null }); +}); diff --git a/tests/runner-e2e/automatic-repository.ts b/tests/runner-e2e/automatic-repository.ts index 27140059279..b8cf9168b70 100644 --- a/tests/runner-e2e/automatic-repository.ts +++ b/tests/runner-e2e/automatic-repository.ts @@ -29,6 +29,9 @@ export const automaticRepositoryTask: RunnerTaskFixture = { }; export interface AutomaticRepositoryObservation { + expectedModel: string; + expectedProvider: string; + actualProviders: Array<{ model: string | null; kind: string | null }>; reference: string; issueDone: boolean; runCount: number; @@ -54,6 +57,25 @@ export interface AutomaticRepositoryObservation { monitorScheduled: boolean; } +export function automaticRepositoryApiState(phase: string, issue: Record, runs: readonly { + id?: string; companyId?: string; agentId?: string; issueId?: string | null; nativeIssueId?: string | null; + status?: string; runtimeMode?: string; nativePhase?: string | null; startedAt?: string | null; finishedAt?: string | null; + errorCode?: string | null; wakeupRequestId?: string | null; + contextSnapshot?: { source?: unknown; nativeStatusWakeIntentId?: unknown } | null; + resultJson?: { finalizationPhase?: unknown } | null; +}[], checks: ReturnType) { + return { capturePhase: phase, + issue: { id: issue.id, companyId: issue.companyId, identifier: issue.identifier, status: issue.status, + assigneeAgentId: issue.assigneeAgentId, executionWorkspaceId: issue.executionWorkspaceId, statusVersion: issue.statusVersion }, + runs: runs.map(run => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, issueId: run.issueId, + nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, nativePhase: run.nativePhase, + startedAt: run.startedAt, finishedAt: run.finishedAt, errorCode: run.errorCode, wakeupRequestId: run.wakeupRequestId, + source: typeof run.contextSnapshot?.source === "string" ? run.contextSnapshot.source : null, + nativeStatusWakeIntentId: typeof run.contextSnapshot?.nativeStatusWakeIntentId === "string" ? run.contextSnapshot.nativeStatusWakeIntentId : null, + finalizationPhase: typeof run.resultJson?.finalizationPhase === "string" ? run.resultJson.finalizationPhase : null })), + checks }; +} + export function automaticRepositoryStopReason(issue: { status: string }, runs: readonly { status: string }[]) { if (runs.length > 2) return "Unexpected extra provider admission"; if (runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status))) return "A provider admission failed"; @@ -75,6 +97,9 @@ export function gradeAutomaticRepository(value: AutomaticRepositoryObservation) /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value.continuationReceipt.dispatchId) && value.continuationReceipt.intentId !== value.continuationReceipt.dispatchId, "The saved same-agent intent must bind a distinct successor dispatch after the unmodified predecessor finishes, without a timer or user message."), + check("configured-provider", value.actualProviders.length === 2 && value.actualProviders.every(provider => + provider.model === value.expectedModel && provider.kind === value.expectedProvider), + "Both admitted runs must retain the configured provider and model."), check("retained-workspace", value.workspaceIds.length === 2 && Boolean(value.workspaceIds[0]) && value.workspaceIds.every(id => id === value.workspaceIds[0]) && value.readyRepository, "Both admissions must retain the same task workspace and its prepared repository."), check("retained-context", value.firstContextAttributed && value.firstContextObservedAt !== null && value.successorStartedAt !== null && diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 0a669011c49..4d43493333f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -211,10 +211,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, + 1, 3, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(740); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(740); + expect(validateRunnerCatalog()).toHaveLength(752); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(752); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0a24b4b6353..a211a1185e8 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,6 +1,7 @@ import { caseHandoffTask, caseHandoffDefinitionDigest } from "./case-handoffs-cases.js"; import { caseReceiptTask, caseReceiptDefinitionDigest, CASE_RECEIPT_BUDGET_CENTS, CASE_RECEIPT_MAX_TURNS, CASE_RECEIPT_RUN_TIMEOUT_MS } from "./case-receipt-cases.js"; import { TASK_WORKSPACES_SUITE, taskWorkspaceTask, taskWorkspaceProfile, taskWorkspaceDefinitionDigest } from "./task-workspaces-cases.js"; +import { repositoryProofTasks, repositoryProofProfile, repositoryProofDigest } from "./repository-verification.js"; import { automaticRepositoryTask, automaticRepositoryDigest } from "./automatic-repository.js"; import { nativeActiveStopTasks } from "./native-active-stop-tasks.js"; import { piControlTasks } from "./pi-controls-cases.js"; @@ -1131,6 +1132,23 @@ function copilotFilePrompt(nonce: string): string { ].join("\n"); } +const repositoryModelProfiles = [ + { + native: nativeProfile({ + id: "runner-codex", label: "Codie staging-model diagnostic", provider: "codex", model: "gpt-6.1-sol", credential: "OPENAI_API_KEY", + modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-codie-model-2026-10-20" }, + }), + legacy: runnerProfiles.find(profile => profile.id === "legacy-codex")!, + }, + { + native: nativeProfile({ + id: "runner-acpx-claude", label: "Claude staging-model diagnostic", provider: "acpx", acpxAgent: "claude", model: "claude-opus-5-5", credential: "ANTHROPIC_API_KEY", + modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-claude-model-2026-10-20" }, + }), + legacy: runnerProfiles.find(profile => profile.id === "legacy-claude")!, + }, +]; + export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "codie-latency-diagnostic", @@ -1168,6 +1186,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ definitionMetadata: { version: 1, definitionDigest: taskWorkspaceDefinitionDigest(), scheduling: "explicit-only", providerTurns: 3, attempts: 1, budgetMonthlyCents: 1000, project: "none", cwd: "unconfigured-task-default", oracle: "public-file-download-and-host-nested-git", remoteLifecycle: "per_turn" }, }, + ...([false, true] as const).map(conversion => ({ + id: conversion ? "legacy-conversion" : "configured-repository", + label: conversion ? "Legacy conversion with verified delivery" : "Configured repository with verified delivery", + manualOnly: true, description: "Production instructions, retained configuration, and a downloaded result checked against an independent Git and file baseline.", + groups: [], profiles: repositoryModelProfiles.map(({ native, legacy }) => repositoryProofProfile(native, conversion ? legacy : undefined)), + environments: runnerEnvironments, tasks: repositoryProofTasks.filter(task => task.id === (conversion ? "convert-and-deliver" : "repository-output")), expectedMatrixSize: 4, + definitionMetadata: { version: 1, definitionDigest: repositoryProofDigest(), scheduling: "explicit-only", providerTurns: 1, attempts: 1, + budgetMonthlyCents: 1000, instructions: "production-default", surface: "paperclip", oracle: "registered-download-and-independent-configured-checkout", + target: "isolated-development-instance", authentication: "api-key", stagingAuthentication: "subscription", remoteLifecycle: "per_turn", stagingQualification: false }, + })), { id: "automatic-repository", label: "Automatic repository continuation", manualOnly: true, description: "A projectless user request stages its managed repository, continues automatically, and delivers independently verified downloaded bytes.", @@ -1178,6 +1206,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ providerTurns: 2, attempts: 1, budgetMonthlyCents: 1000, project: "none", instructions: "production-default", surface: "paperclip", oracle: "registered-download-and-independent-checkout", remoteLifecycle: "per_turn" }, }, + { + id: "automatic-repository-configured-models", label: "Automatic repository with recorded staging models", manualOnly: true, + description: "Replay projectless setup with the recorded Codie and Claude models while keeping development authentication and lifecycle differences explicit.", + groups: [], profiles: repositoryModelProfiles.map(({ native }) => productionDefaultHireProfile(taskWorkspaceProfile(native))), + environments: runnerEnvironments, tasks: [automaticRepositoryTask], expectedMatrixSize: 4, + definitionMetadata: { version: 1, definitionDigest: automaticRepositoryDigest(), scheduling: "explicit-only", providerTurns: 2, attempts: 1, + budgetMonthlyCents: 1000, instructions: "production-default", surface: "paperclip", target: "isolated-development-instance", + authentication: "api-key", stagingAuthentication: "subscription", remoteLifecycle: "per_turn", stagingQualification: false }, + }, { id: "pi-controls", label: "Pi active controls", manualOnly: true, description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.", diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 87bc56b1f19..2e52b8fdbc5 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -389,7 +389,8 @@ export async function setupLiveFixtures(input: { }, }); - if (execution.task.flow === "warm_three_turn" + if (execution.task.flow === "repository_verification" + || execution.task.flow === "warm_three_turn" || (execution.suite.id === "extended-harnesses" && execution.task.id === "file-edit-validate")) { registry.register({ id: "project", diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 38c2a0e3716..bc9fd401613 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(740); + expect(validateRunnerCatalog()).toHaveLength(752); for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); diff --git a/tests/runner-e2e/repository-conversion-setup.test.ts b/tests/runner-e2e/repository-conversion-setup.test.ts new file mode 100644 index 00000000000..aad6ce2e91c --- /dev/null +++ b/tests/runner-e2e/repository-conversion-setup.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from "vitest"; +import { runnerSuites } from "./catalog.js"; +import { convertRepositoryProofAgent } from "./repository-conversion-setup.js"; + +function fixtures(driver = "sandbox"): Parameters[1] { + return { agent: { id: "agent", name: "Fixture Agent", companyId: "company" }, company: { id: "company", name: "Fixture Company" }, environment: { id: "environment", driver, + config: { image: `image@sha256:${"a".repeat(64)}` } } }; +} + +describe("real legacy conversion prerequisites", () => { + it("constructs compatible legacy settings before recording a retention baseline", () => { + const suite = runnerSuites.find(suite => suite.id === "legacy-conversion")!; + for (const profile of suite.profiles) { + const agent = profile.buildAgent({ environmentId: "environment", environmentFixtureId: "local", workspacePath: "/owned/workspace", + executionId: "fixture", secretRefs: { OPENAI_API_KEY: { type: "secret_ref", secretId: "codex-reference", version: "latest" }, + ANTHROPIC_API_KEY: { type: "secret_ref", secretId: "claude-reference", version: "latest" } } }); + expect(agent.runner).toBe("legacy"); + expect(agent.adapterType).toBe(profile.provider === "codex" ? "codex_local" : "claude_local"); + expect(agent.adapterConfig).toMatchObject({ model: profile.model, timeoutSec: 360, cwd: "/owned/workspace" }); + expect(agent.adapterConfig).not.toHaveProperty("extraArgs"); + expect(agent.adapterConfig).not.toHaveProperty("maxTurnsPerRun"); + expect((agent.adapterConfig as Record).env).toMatchObject(profile.provider === "codex" + ? { OPENAI_API_KEY: { type: "secret_ref", secretId: "codex-reference", version: "latest" } } + : { ANTHROPIC_API_KEY: { type: "secret_ref", secretId: "claude-reference", version: "latest" } }); + } + }); + it("qualifies the sandbox before selecting Runner for either provider", async () => { + const fixture = fixtures(); let qualified = false; + const api = { post: vi.fn(async () => { qualified = true; return { ok: true, details: { platformQualificationRecorded: true, leaseId: "lease" } }; }), + patch: vi.fn(async () => { if (!qualified) throw new Error("agent_runner_unavailable"); return {}; }) }; + await expect(convertRepositoryProofAgent(api as any, fixture)).resolves.toBeUndefined(); + expect(api.post).toHaveBeenCalledWith("/api/environments/environment/probe?companyId=company", {}, { timeout: 330_000 }); + expect(fixture.setupProbe).toMatchObject({ leaseId: "lease" }); + expect(api.patch).toHaveBeenCalledExactlyOnceWith("/api/agents/agent", { runner: "paperclip" }); + }); + it("does not attempt conversion when platform qualification is absent", async () => { + const fixture = fixtures(); const api = { post: vi.fn(async () => ({ ok: true, details: { leaseId: "lease" } })), patch: vi.fn() }; + await expect(convertRepositoryProofAgent(api as any, fixture)).rejects.toThrow("qualification"); + expect(api.patch).not.toHaveBeenCalled(); + expect(fixture.setupProbe?.leaseId).toBe("lease"); + }); + it("retains uncertain probe accounting when the response is lost", async () => { + const fixture = fixtures(); const api = { post: vi.fn(async () => { throw new Error("connection lost"); }), patch: vi.fn() }; + await expect(convertRepositoryProofAgent(api as any, fixture)).rejects.toThrow("connection lost"); + expect(api.patch).not.toHaveBeenCalled(); + expect(fixture.setupLeaseAccountingIncomplete).toBe(true); + }); + it.each([{ image: "image:latest" }, { image: `image@sha256:${"a".repeat(64)}`, snapshot: "snapshot" }])("refuses an unqualified image before creating a sandbox", async config => { + const fixture = fixtures(); fixture.environment.config = config; + const api = { post: vi.fn(), patch: vi.fn() }; + await expect(convertRepositoryProofAgent(api as any, fixture)).rejects.toThrow("immutable"); + expect(api.post).not.toHaveBeenCalled(); expect(api.patch).not.toHaveBeenCalled(); + }); + it("converts a local agent without creating a remote sandbox", async () => { + const api = { post: vi.fn(), patch: vi.fn(async () => ({})) }; + await convertRepositoryProofAgent(api as any, fixtures("local")); + expect(api.post).not.toHaveBeenCalled(); expect(api.patch).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/runner-e2e/repository-conversion-setup.ts b/tests/runner-e2e/repository-conversion-setup.ts new file mode 100644 index 00000000000..2f35c46a5d3 --- /dev/null +++ b/tests/runner-e2e/repository-conversion-setup.ts @@ -0,0 +1,20 @@ +import type { RunnerApi } from "./api.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; + +export async function convertRepositoryProofAgent(api: Pick, fixtures: Pick) { + if (fixtures.environment.driver === "sandbox") { + if (!/^.+@sha256:[a-f0-9]{64}$/i.test(String(fixtures.environment.config?.image ?? "")) || fixtures.environment.config?.snapshot) + throw new Error("Legacy conversion requires the immutable fixture image"); + const startedAt = Date.now(), previousAccounting = fixtures.setupLeaseAccountingIncomplete; + fixtures.setupLeaseAccountingIncomplete = true; + const probe = await api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( + `/api/environments/${fixtures.environment.id}/probe?companyId=${encodeURIComponent(fixtures.company.id)}`, {}, { timeout: 330_000 }); + if (probe.details?.leaseId) { + fixtures.setupProbe = { leaseId: probe.details.leaseId, durationMs: Date.now() - startedAt }; + fixtures.setupLeaseAccountingIncomplete = previousAccounting; + } + if (!probe.ok || !probe.details?.platformQualificationRecorded || !probe.details.leaseId) + throw new Error("Legacy conversion platform qualification was not recorded"); + } + await api.patch(`/api/agents/${fixtures.agent.id}`, { runner: "paperclip" }); +} diff --git a/tests/runner-e2e/repository-verification-flow.ts b/tests/runner-e2e/repository-verification-flow.ts new file mode 100644 index 00000000000..3aed6e7f69a --- /dev/null +++ b/tests/runner-e2e/repository-verification-flow.ts @@ -0,0 +1,136 @@ +import { createHash, randomBytes } from "node:crypto"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect, type Page } from "@playwright/test"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { createTaskThroughUi } from "./user-actions.js"; +import { convertRepositoryProofAgent } from "./repository-conversion-setup.js"; +import { readRegisteredArtifacts } from "./registered-artifact.js"; +import { gradeRepositoryProof, repositoryConfigurationDigest, repositoryProofFiles, type RepositoryProofObservation } from "./repository-verification.js"; +import type { LiveFixtureValues } from "./live-fixtures.js"; +import type { MatrixExecution } from "./types.js"; + +type Row = Record; +const execFile = promisify(execFileCallback); + +export async function runRepositoryVerificationFlow(input: { + page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; + workspacePath: string; temporaryRoot: string; deadlineAt: number; + observe(issue: Row, runs: Row[], checks: ReturnType): void; + capture(id: string, label: string, file: string): Promise; + evidence(name: string, value: unknown): Promise; +}) { + const { page, api, fixtures, execution, nonce } = input; + const files = repositoryProofFiles(nonce), conversion = execution.task.id === "convert-and-deliver"; + let issue: Row | undefined, runs: Row[] = []; + const expectedLegacyAdapter = execution.profile.provider === "codex" ? "codex_local" : "claude_local"; + const observation: RepositoryProofObservation = { conversion, expectedLegacyAdapter, originalAdapter: "", finalAdapter: "", + configurationBefore: "", configurationAfter: "", instructionsBefore: "", instructionsAfter: "", permissionsPreserved: false, + expectedModel: execution.profile.model, actualModel: null, expectedProvider: execution.profile.provider === "codex" ? "codex" : "acpx:claude", + actualProvider: null, environmentRetained: false, issueDone: false, runCount: 0, committedRuns: 0, + sourceBytes: "", currentSourceBytes: null, expectedCommit: "", downloaded: null, downloadVerified: false, + visibleDelivery: false, pendingInteractions: -1 }; + async function instructionsDigest() { + const bundle = await api.get<{ entryFile: string; files: Array<{ path: string }> }>(`/api/agents/${fixtures.agent.id}/instructions-bundle`); + const files = await Promise.all([...bundle.files].sort((a, b) => a.path.localeCompare(b.path)).map(async file => ({ + path: file.path, content: (await api.get<{ content: string }>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(file.path)}`)).content, + }))); + if (bundle.entryFile !== "AGENTS.md" || files.length !== 1 || files[0]?.content !== "You are an agent in a Paperclip company.\n") + throw new Error("Repository proof requires the shipped default hire instructions, without a fixture manual"); + return createHash("sha256").update(JSON.stringify({ entryFile: bundle.entryFile, files })).digest("hex"); + } + async function refresh() { + if (!issue) return; + issue = await api.get(`/api/issues/${issue.id}`); + const listed = await api.get(`/api/companies/${fixtures.company.id}/heartbeat-runs?agentId=${fixtures.agent.id}&limit=20`); + runs = (await Promise.all(listed.map(run => api.get(`/api/heartbeat-runs/${run.id}`)))) + .filter(run => [run.nativeIssueId, run.issueId, run.contextSnapshot?.issueId, run.contextSnapshot?.taskId].includes(issue!.id)); + observation.issueDone = issue.status === "done"; + observation.runCount = runs.length; + observation.committedRuns = runs.filter(run => run.status === "succeeded" && run.runtimeMode === "native" && + run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed").length; + const provider = runs[0]?.runnerProfileJson?.nativeExecutionInput?.provider; + observation.actualModel = provider?.model ?? null; + observation.actualProvider = provider ? provider.kind === "acpx" ? `acpx:${provider.agent}` : provider.kind : null; + observation.environmentRetained = runs.length === 1 && runs[0]?.contextSnapshot?.paperclipEnvironment?.id === fixtures.environment.id; + } + async function save(phase: string) { + const checks = gradeRepositoryProof(observation); + if (issue) { + input.observe(issue, runs, checks); + await input.evidence("api-state.json", { + capturePhase: phase, + issue: { id: issue.id, companyId: issue.companyId, identifier: issue.identifier, status: issue.status, + assigneeAgentId: issue.assigneeAgentId, executionWorkspaceId: issue.executionWorkspaceId, statusVersion: issue.statusVersion }, + runs: runs.map(run => ({ id: run.id, companyId: run.companyId, agentId: run.agentId, issueId: run.issueId, + nativeIssueId: run.nativeIssueId, status: run.status, runtimeMode: run.runtimeMode, nativePhase: run.nativePhase, + startedAt: run.startedAt, finishedAt: run.finishedAt, errorCode: run.errorCode, + finalizationPhase: run.resultJson?.finalizationPhase })), + checks, + }); + } + await input.evidence("repository-verification.json", { schema: "paperclip.e2e.repository-verification.v1", phase, observation, checks, + runs: runs.map(run => ({ id: run.id, status: run.status, nativePhase: run.nativePhase, source: run.contextSnapshot?.source, + startedAt: run.startedAt, finishedAt: run.finishedAt })) }); + } + try { + if (!fixtures.project || execution.profile.generation !== "native") throw new Error("Repository proof requires its owned configured project"); + const workspace = await realpath(input.workspacePath), root = await realpath(input.temporaryRoot); + if (workspace !== path.join(root, "workspace")) throw new Error("Repository proof cannot seed an unowned source checkout"); + const git = async (args: string[]) => (await execFile("git", ["-C", workspace, ...args], { + env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" }, + timeout: 20_000, maxBuffer: 262_144, encoding: "utf8", + })).stdout; + observation.sourceBytes = `${randomBytes(32).toString("hex")}\n`; + await writeFile(path.join(workspace, files.source), observation.sourceBytes, { flag: "wx" }); + await git(["init", "--initial-branch=verification"]); + await git(["add", "--", files.source]); + await git(["-c", "user.name=Qualification Fixture", "-c", "user.email=qualification@example.test", "-c", "commit.gpgsign=false", "commit", "-m", "Seed independent repository reference"]); + observation.expectedCommit = (await git(["rev-parse", "HEAD"])).trim(); + await api.patch(`/api/companies/${fixtures.company.id}/budgets`, { budgetMonthlyCents: 1000 }); + await api.patch(`/api/agents/${fixtures.agent.id}/budgets`, { budgetMonthlyCents: 1000 }); + const before = await api.get(`/api/agents/${fixtures.agent.id}`); + const company = await api.get(`/api/companies/${fixtures.company.id}`); + if (before.budgetMonthlyCents !== 1000 || company.budgetMonthlyCents !== 1000) throw new Error("Repository proof hard-stop budgets were not retained"); + if (before.adapterType !== (conversion ? expectedLegacyAdapter : "paperclip_runner")) throw new Error("Initial adapter does not match the requested journey"); + observation.originalAdapter = before.adapterType; + observation.configurationBefore = repositoryConfigurationDigest(before); + observation.instructionsBefore = await instructionsDigest(); + if (conversion) await convertRepositoryProofAgent(api, fixtures); + const after = await api.get(`/api/agents/${fixtures.agent.id}`); + observation.finalAdapter = after.adapterType; + observation.configurationAfter = repositoryConfigurationDigest(after); + observation.instructionsAfter = await instructionsDigest(); + observation.permissionsPreserved = (conversion ? before.adapterConfig?.dangerouslySkipPermissions === true : true) && + (execution.profile.provider === "codex" ? after.adapterConfig?.codexPermissionMode === "never" : after.adapterConfig?.acpxPermissionMode === "approve-all"); + if (after.defaultEnvironmentId !== fixtures.environment.id || observation.configurationBefore !== observation.configurationAfter || + observation.instructionsBefore !== observation.instructionsAfter || !observation.permissionsPreserved || after.adapterType !== "paperclip_runner") + throw new Error("Production runner selection did not retain the configured agent context"); + await save("configured"); + const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name, + projectName: fixtures.project.name, title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", requireExplicitTitle: true }); + issue = await api.get(`/api/issues/${created.issueId}`); + await page.goto(`/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`); + await pollUntil({ label: "configured repository result", deadlineAt: input.deadlineAt, intervalMs: 1000, + load: async () => { await refresh(); return { issue, runs }; }, + accept: () => observation.issueDone && observation.committedRuns === 1, + reject: state => state.runs.length > 1 ? "Unexpected extra provider admission" : state.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) + ? "Provider admission failed" : state.runs.length && state.runs.every(run => run.status === "succeeded") && ["blocked", "in_review"].includes(state.issue!.status) ? "Task stopped before delivery" : undefined }); + const context = await api.get(`/api/issues/${issue.id}/file-resources/content?workspace=execution&path=${encodeURIComponent(files.source)}`); + observation.currentSourceBytes = context.resource?.workspaceKind === "execution_workspace" && typeof context.content?.data === "string" ? context.content.data : null; + const [artifact] = await readRegisteredArtifacts(api, issue.id, runs[0]!.id, [files.result]); + observation.downloaded = JSON.parse(artifact.content); + observation.downloadVerified = artifact.contentVerified; + observation.pendingInteractions = (await api.get(`/api/issues/${issue.id}/interactions`)).filter(row => row.status === "pending").length; + await page.reload(); + await expect(page.locator(`a[href*="${artifact.attachmentId}"]`).first()).toBeVisible({ timeout: 30_000 }); + observation.visibleDelivery = true; + await save("completed"); + await input.capture("final-state", "Verified configured repository delivery", "final-state.png"); + const checks = gradeRepositoryProof(observation), failed = checks.filter(check => !check.passed); + if (failed.length) throw new Error(`Repository verification failed: ${failed.map(check => check.id).join(", ")}`); + return { issue, runs, checks }; + } catch (error) { await refresh().catch(() => undefined); await save("failed"); throw error; } +} diff --git a/tests/runner-e2e/repository-verification.test.ts b/tests/runner-e2e/repository-verification.test.ts new file mode 100644 index 00000000000..f5560ef000f --- /dev/null +++ b/tests/runner-e2e/repository-verification.test.ts @@ -0,0 +1,62 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { gradeRepositoryProof, repositoryConfigurationDigest, repositoryProofTasks, type RepositoryProofObservation } from "./repository-verification.js"; +import { runnerSuites } from "./catalog.js"; + +function valid(): RepositoryProofObservation { + const sourceBytes = `${"a7".repeat(32)}\n`, commit = "b".repeat(40); + return { conversion: true, originalAdapter: "codex_local", expectedLegacyAdapter: "codex_local", finalAdapter: "paperclip_runner", + configurationBefore: "config", configurationAfter: "config", instructionsBefore: "instructions", instructionsAfter: "instructions", + permissionsPreserved: true, expectedModel: "gpt-6.1-sol", actualModel: "gpt-6.1-sol", expectedProvider: "codex", actualProvider: "codex", + environmentRetained: true, issueDone: true, runCount: 1, committedRuns: 1, sourceBytes, currentSourceBytes: sourceBytes, + expectedCommit: commit, downloaded: { reference: sourceBytes.trimEnd(), sourceSha256: createHash("sha256").update(sourceBytes).digest("hex"), commit }, + downloadVerified: true, visibleDelivery: true, pendingInteractions: 0 }; +} +const passes = (value: RepositoryProofObservation) => gradeRepositoryProof(value).every(check => check.passed); + +describe("configured repository and conversion qualification", () => { + it("accepts independently verified actual conversion and ordinary delivery", () => { + expect(passes(valid())).toBe(true); + expect(passes({ ...valid(), conversion: false, originalAdapter: "paperclip_runner" })).toBe(true); + }); + it.each([ + { originalAdapter: "paperclip_runner" }, { finalAdapter: "codex_local" }, + { configurationAfter: "changed environment reference" }, { instructionsAfter: "lost instructions" }, + { permissionsPreserved: false }, { environmentRetained: false }, { actualModel: "gpt-5.6-sol" }, + { actualProvider: "acpx:claude" }, { runCount: 2 }, { committedRuns: 0 }, { issueDone: false }, + { currentSourceBytes: null }, { currentSourceBytes: "recreated" }, { expectedCommit: "" }, + { downloadVerified: false }, { visibleDelivery: false }, { pendingInteractions: 1 }, + ])("rejects missing or contrary evidence %j", change => expect(passes({ ...valid(), ...change })).toBe(false)); + it.each([ + null, "a badge is green", [], {}, { reference: "wrong", sourceSha256: "wrong", commit: "wrong" }, + ])("does not accept provider success or malformed downloaded output %j", downloaded => expect(passes({ ...valid(), downloaded })).toBe(false)); + it("rejects each incorrect result field and extra fields", () => { + const baseline = valid(), result = baseline.downloaded as Record; + for (const field of ["reference", "sourceSha256", "commit"]) expect(passes({ ...baseline, downloaded: { ...result, [field]: "wrong" } })).toBe(false); + expect(passes({ ...baseline, downloaded: { ...result, success: true } })).toBe(false); + }); + it("hashes retained configuration consistently without returning its contents", () => { + const before = { defaultEnvironmentId: "env", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } }, + adapterConfig: { model: "model", cwd: "/owned", env: { KEY: { type: "secret_ref", secretId: "sensitive-fixture-reference", version: 1 } }, timeoutSec: 240 } }; + const reordered = { ...before, adapterConfig: { ...before.adapterConfig, env: { KEY: { version: 1, secretId: "sensitive-fixture-reference", type: "secret_ref" } } } }; + expect(repositoryConfigurationDigest(before)).toBe(repositoryConfigurationDigest(reordered)); + expect(repositoryConfigurationDigest(before)).toMatch(/^[a-f0-9]{64}$/); + expect(repositoryConfigurationDigest(before)).not.toContain("sensitive-fixture-reference"); + expect(repositoryConfigurationDigest({ ...before, defaultEnvironmentId: "other" })).not.toBe(repositoryConfigurationDigest(before)); + expect(repositoryConfigurationDigest({ ...before, adapterConfig: { ...before.adapterConfig, env: {} } })).not.toBe(repositoryConfigurationDigest(before)); + }); + it("registers explicit development cells with staging models and production instructions", () => { + for (const id of ["configured-repository", "legacy-conversion", "automatic-repository-configured-models"]) { + const suite = runnerSuites.find(suite => suite.id === id)!; + expect(suite.manualOnly).toBe(true); + expect(suite.groups).toEqual([]); + expect(suite.expectedMatrixSize).toBe(4); + expect(suite.profiles.map(profile => profile.model)).toEqual(["gpt-6.1-sol", "claude-opus-5-5"]); + expect(suite.definitionMetadata?.stagingQualification).toBe(false); + expect(suite.tasks).toHaveLength(1); + expect(suite.tasks[0]?.flow).toBe(id === "automatic-repository-configured-models" ? "automatic_repository" : "repository_verification"); + expect(suite.tasks[0]?.automaticRetryPolicy).toBe("single_attempt"); + } + expect(repositoryProofTasks.every(task => task.expectedRunCount === 1)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/repository-verification.ts b/tests/runner-e2e/repository-verification.ts new file mode 100644 index 00000000000..c6ef5325b1f --- /dev/null +++ b/tests/runner-e2e/repository-verification.ts @@ -0,0 +1,105 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerProfileFixture, RunnerTaskFixture } from "./types.js"; + +export function repositoryProofFiles(nonce: string) { + return { source: `source-reference-${nonce}.txt`, result: `repository-proof-${nonce}.json` }; +} + +export function repositoryProofProfile(native: RunnerProfileFixture, legacy?: RunnerProfileFixture): RunnerProfileFixture { + return { ...native, buildAgent(input) { + const { instructionsBundle: _manual, ...agent } = (legacy ?? native).buildAgent(input); + const config = { ...(agent.adapterConfig as Record), model: native.model }; + if (legacy) { + delete (config as Record).extraArgs; + delete (config as Record).maxTurnsPerRun; + } + return { ...agent, ...(legacy ? { runner: "legacy" } : {}), budgetMonthlyCents: 1000, adapterConfig: config }; + } }; +} + +export const repositoryProofTasks: readonly RunnerTaskFixture[] = ["repository-output", "convert-and-deliver"].map(id => ({ + id, label: id === "repository-output" ? "Configured repository and accessible output" : "Legacy conversion retains configuration and delivers useful work", + groups: [], flow: "repository_verification", workMode: "standard", expectedRunCount: 1, + automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 10 * 60_000, daytona: 15 * 60_000 }, + turnTimeoutMs: 4 * 60_000, expectedTerminalState: { issue: "done", run: "succeeded" }, + buildTitle: nonce => `Verify repository context ${nonce}`, + buildVisibleMarker: nonce => repositoryProofFiles(nonce).result, + buildPrompt(nonce) { + const files = repositoryProofFiles(nonce); + return [ + "Verify the configured repository and deliver a small proof file. Read the existing checkout; do not replace, clone, or recreate it.", + `Read ${files.source}. It contains a reference followed by one LF. Preserve the source file unchanged. If it is missing, report the failure instead of creating a replacement.`, + "Read the current full Git HEAD SHA and compute the SHA-256 of the exact source file bytes, including its final newline.", + `Write ${files.result} as JSON with exactly these fields: reference (the existing reference without its trailing LF), sourceSha256 (the exact source file hash), and commit (the full Git HEAD SHA).`, + `Verify the result, register ${files.result} as a downloadable artifact work product with that exact title, and deliver its accessible link in this task. Finish after delivery. Do not create other tasks or agents.`, + ].join("\n"); + }, buildMatchers: () => [], +})); + +export interface RepositoryProofObservation { + conversion: boolean; + originalAdapter: string; + expectedLegacyAdapter: string; + finalAdapter: string; + configurationBefore: string; + configurationAfter: string; + instructionsBefore: string; + instructionsAfter: string; + permissionsPreserved: boolean; + expectedModel: string; + actualModel: string | null; + actualProvider: string | null; + expectedProvider: string; + environmentRetained: boolean; + issueDone: boolean; + runCount: number; + committedRuns: number; + sourceBytes: string; + currentSourceBytes: string | null; + expectedCommit: string; + downloaded: unknown; + downloadVerified: boolean; + visibleDelivery: boolean; + pendingInteractions: number; +} + +export function gradeRepositoryProof(value: RepositoryProofObservation) { + const check = (id: string, passed: boolean, detail: string) => ({ id, passed, detail }); + const result = value.downloaded && typeof value.downloaded === "object" && !Array.isArray(value.downloaded) + ? value.downloaded as Record : null; + return [ + check("production-conversion", value.finalAdapter === "paperclip_runner" && + value.originalAdapter === (value.conversion ? value.expectedLegacyAdapter : "paperclip_runner"), + "Conversion must start with a real legacy agent and use its production runner transition."), + check("retained-configuration", Boolean(value.configurationBefore) && value.configurationBefore === value.configurationAfter && + Boolean(value.instructionsBefore) && value.instructionsBefore === value.instructionsAfter && value.permissionsPreserved && value.environmentRetained, + "Configured model, source directory, secret references, timeout, runtime settings, environment, instructions, and equivalent permissions must survive."), + check("actual-native-runtime", value.actualModel === value.expectedModel && value.actualProvider === value.expectedProvider && + value.runCount === 1 && value.committedRuns === 1 && value.issueDone, + "One actual native admission on the configured provider and model must commit the task result."), + check("retained-repository", /^[a-f0-9]{64}\n$/.test(value.sourceBytes) && value.sourceBytes === value.currentSourceBytes && /^[a-f0-9]{40}$/.test(value.expectedCommit), + "The independently seeded source reference must survive byte for byte in the configured checkout."), + check("independent-result", value.downloadVerified && result !== null && Object.keys(result).sort().join(",") === "commit,reference,sourceSha256" && + result.reference === value.sourceBytes.trimEnd() && result.sourceSha256 === createHash("sha256").update(value.sourceBytes).digest("hex") && result.commit === value.expectedCommit, + "Downloaded bytes must agree with the independent source-file and Git baseline."), + check("accessible-delivery", value.visibleDelivery && value.pendingInteractions === 0, + "The originating task must render the downloadable result with no unanswered interaction."), + ]; +} + +export function repositoryConfigurationDigest(agent: Record) { + const config = agent.adapterConfig ?? {}; + const canonical = (value: any): any => Array.isArray(value) ? value.map(canonical) : value && typeof value === "object" + ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value; + return createHash("sha256").update(JSON.stringify(canonical({ model: config.model, cwd: config.cwd, + timeoutSec: config.timeoutSec, env: config.env, defaultEnvironmentId: agent.defaultEnvironmentId, + runtimeConfig: agent.runtimeConfig }))).digest("hex"); +} + +export function repositoryProofDigest() { + const hash = createHash("sha256"); + for (const file of ["repository-verification.ts", "repository-verification-flow.ts", "repository-conversion-setup.ts", "registered-artifact.ts", "live-fixtures.ts", "runner.spec.ts", "catalog.ts", "stock-harness.ts"]) + hash.update(file).update(readFileSync(new URL(file, import.meta.url))); + return hash.digest("hex"); +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 935780909b4..fed14dd3d29 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,6 +1,7 @@ import { runCaseHandoffsFlow } from "./case-handoffs-flow.js"; import { runCaseReceiptFlow } from "./case-receipt-flow.js"; import { runTaskWorkspacesFlow } from "./task-workspaces-flow.js"; +import { runRepositoryVerificationFlow } from "./repository-verification-flow.js"; import { runAutomaticRepositoryFlow } from "./automatic-repository-flow.js"; import { runNativeActiveStopFlow as runCopilotActiveStopFlow } from "./copilot-active-stop-flow.js"; import { nativeActiveStopTasks as copilotActiveStopTasks } from "./copilot-active-stop-tasks.js"; @@ -1023,6 +1024,15 @@ for (const execution of executions) { matcherResults, }, secrets); await captureScreenshot("final-state", "Final visible task state", "final-state.png"); + } else if (execution.task.flow === "repository_verification") { + const journey = await runRepositoryVerificationFlow({ page, api, fixtures, execution, nonce, workspacePath: workspacePath!, temporaryRoot, + deadlineAt: startedAtMs + deadlineMs - 30_000, + observe: (currentIssue, currentRuns, checks) => { + issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; + matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `repositoryVerification.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })); + }, capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), + }); + issue = journey.issue as IssueRecord; selectedRuns = journey.runs as RunRecord[]; } else if (execution.task.flow === "automatic_repository") { const journey = await runAutomaticRepositoryFlow({ page, api, fixtures, execution, nonce, temporaryRoot, deadlineAt: startedAtMs + deadlineMs - 30_000, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 295bf1bea59..5f2c20157c8 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -21,6 +21,7 @@ export type RunnerTaskFlow = | "case_handoff" | "task_workspaces" | "automatic_repository" + | "repository_verification" | "repository_skills" | "provider_connection" | "plan_task_guidance"