diff --git a/doc/plans/2026-10-20-dependable-everyday-work.md b/doc/plans/2026-10-20-dependable-everyday-work.md index e1ce750356d..082870f8b13 100644 --- a/doc/plans/2026-10-20-dependable-everyday-work.md +++ b/doc/plans/2026-10-20-dependable-everyday-work.md @@ -111,3 +111,14 @@ Configured checkouts keep strict validation, real Git roots still receive the exclusion, linked metadata stays rejected, and missing Git stays an error. The repair's Git and authorization cohort passes locally. Fresh provider remeasurement and the monitor-fallback defect remain open. + + +Two additional Product E2E journeys now cover configured repository delivery and +real legacy-to-native conversion followed by verified work. Each checks a fresh +fixture-owned source file, its hash and Git commit, retained configuration and +instructions, the actual native provider/model, and a registered download visible +after reload. A third explicit selection replays automatic repository setup with +the recorded Codex and Claude models. All twelve new cells declare development +API-key authentication and per-turn lifecycle differences. None counts as staging +subscription, warm lifecycle, Slack, or the complete 24-cell milestone. Fresh +provider measurement remains required; catalog registration is not acceptance. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 11cb2492839..dd87e0ac11c 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -25,6 +25,40 @@ Source-server startup prepares the existing shared/plugin SDK build dependencies before importing native tool authority. Installed-package checks never execute checkout build helpers. +## Configured repository and conversion checks + +The explicit-only `configured-repository` and `legacy-conversion` suites each +have four development cells: Codex `gpt-6.1-sol` and Claude `claude-opus-5-5`, +on local and Daytona environments. Each uses production hire instructions, +one attempt, one expected provider admission, and 1,000-cent company and agent +hard stops. The conversion journey creates a real legacy agent and selects +Paperclip Runner through the public agent update API. It checks retained +configuration, encrypted secret references, environment, instructions, and +corresponding permission policy before submitting work through the browser. + +The observer seeds a random reference in the fixture-owned Git checkout before +task creation. It checks the downloaded JSON against the original file bytes, +SHA-256 and commit, then checks the preserved source through the workspace API. +A pass also requires the configured native provider/model, one committed run, +a visible artifact link after reload, Done, and no pending question. No agent +success claim or agent-authored test can replace these checks. + +The explicit-only `automatic-repository-configured-models` suite reuses the +projectless continuation oracle above with those same two recorded staging +models. Its four cells keep the first admission's artifact observation and the +strict requirement for a saved same-agent continuation. A monitor fallback is +still a failure of that requirement. + +These twelve development cells use API-key authentication and per-turn Runner +lifecycle. They do not qualify staging subscription credentials, warm Runner +reuse, Slack delivery, or the complete dependable-work matrix. Run one exact +ID for a bounded check: + +```sh +pnpm test:e2e:runner -- --id legacy-conversion.runner-codex.local.convert-and-deliver --max-automatic-retries 0 +pnpm test:e2e:runner -- --id automatic-repository-configured-models.runner-codex.local.automatic-repository-resume --max-automatic-retries 0 +``` + ## Live provider connection journeys See [PROVIDER-CONNECTIONS.md](PROVIDER-CONNECTIONS.md) for the explicit-only diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index ca0ec5edd22..7cba2b55c14 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -415,3 +415,17 @@ limits, case/task state and correlated receipt checks enter workflow evidence. Two passive seed cases support browser list assertions; screenshots remain on the exact primary case/task routes and document dialogs preserve that case URL. The public screenshot allowlist is unchanged. + + +## Configured repository and conversion proof + +The repository proof flow seeds only the exact `workspace` child of its resolved +fixture-owned temporary directory. It initializes that disposable Git checkout +with a fresh random reference. It never modifies a contributor checkout. Provider +credentials remain encrypted fixture secret references. Retention evidence keeps +a digest of allowlisted configuration fields rather than raw environment values. +The public API performs legacy conversion; no database mutation or private +Runner hook performs it. Only synthetic source bytes and proof JSON enter the +existing private attempt evidence and registered-artifact verification pipeline. +A failed attempt remains retained. These development checks are not staging or +Slack qualification, and they do not publish trusted archives or credentials. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index b25fbe9c483..4d43493333f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -211,10 +211,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, + 1, 3, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 3, 116, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(748); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(748); + expect(validateRunnerCatalog()).toHaveLength(752); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(752); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 87e0609c892..66e8461d88a 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1132,6 +1132,23 @@ function copilotFilePrompt(nonce: string): string { ].join("\n"); } +const repositoryModelProfiles = [ + { + native: nativeProfile({ + id: "runner-codex", label: "Codie staging-model diagnostic", provider: "codex", model: "gpt-6.1-sol", credential: "OPENAI_API_KEY", + modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-codie-model-2026-10-20" }, + }), + legacy: runnerProfiles.find(profile => profile.id === "legacy-codex")!, + }, + { + native: nativeProfile({ + id: "runner-acpx-claude", label: "Claude staging-model diagnostic", provider: "acpx", acpxAgent: "claude", model: "claude-opus-5-5", credential: "ANTHROPIC_API_KEY", + modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-claude-model-2026-10-20" }, + }), + legacy: runnerProfiles.find(profile => profile.id === "legacy-claude")!, + }, +]; + export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "codie-latency-diagnostic", @@ -1172,12 +1189,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ id: conversion ? "legacy-conversion" : "configured-repository", label: conversion ? "Legacy conversion with verified delivery" : "Configured repository with verified delivery", manualOnly: true, description: "Production instructions, retained configuration, and a downloaded result checked against an independent Git and file baseline.", - groups: [], profiles: [ - { native: nativeProfile({ id: "codie", label: "Codex staging-model diagnostic", provider: "codex", model: "gpt-6.1-sol", credential: "OPENAI_API_KEY", - modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-codie-model-2026-10-20" } }), legacy: runnerProfiles.find(profile => profile.id === "legacy-codex")! }, - { native: nativeProfile({ id: "claude", label: "Claude staging-model diagnostic", provider: "acpx", acpxAgent: "claude", model: "claude-opus-5-5", credential: "ANTHROPIC_API_KEY", - modelQualification: { source: "candidate_runner_profile", qualificationId: "observed-staging-claude-model-2026-10-20" } }), legacy: runnerProfiles.find(profile => profile.id === "legacy-claude")! }, - ].map(({ native, legacy }) => repositoryProofProfile(native, conversion ? legacy : undefined)), + groups: [], profiles: repositoryModelProfiles.map(({ native, legacy }) => repositoryProofProfile(native, conversion ? legacy : undefined)), environments: runnerEnvironments, tasks: repositoryProofTasks.filter(task => task.id === (conversion ? "convert-and-deliver" : "repository-output")), expectedMatrixSize: 4, definitionMetadata: { version: 1, definitionDigest: repositoryProofDigest(), scheduling: "explicit-only", providerTurns: 1, attempts: 1, budgetMonthlyCents: 1000, instructions: "production-default", surface: "paperclip", oracle: "registered-download-and-independent-configured-checkout", @@ -1193,6 +1205,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ providerTurns: 2, attempts: 1, budgetMonthlyCents: 1000, project: "none", instructions: "production-default", surface: "paperclip", oracle: "registered-download-and-independent-checkout", remoteLifecycle: "per_turn" }, }, + { + id: "automatic-repository-configured-models", label: "Automatic repository with recorded staging models", manualOnly: true, + description: "Replay projectless setup with the recorded Codie and Claude models while keeping development authentication and lifecycle differences explicit.", + groups: [], profiles: repositoryModelProfiles.map(({ native }) => productionDefaultHireProfile(taskWorkspaceProfile(native))), + environments: runnerEnvironments, tasks: [automaticRepositoryTask], expectedMatrixSize: 4, + definitionMetadata: { version: 1, definitionDigest: automaticRepositoryDigest(), scheduling: "explicit-only", providerTurns: 2, attempts: 1, + budgetMonthlyCents: 1000, instructions: "production-default", surface: "paperclip", target: "isolated-development-instance", + authentication: "api-key", stagingAuthentication: "subscription", remoteLifecycle: "per_turn", stagingQualification: false }, + }, { id: "pi-controls", label: "Pi active controls", manualOnly: true, description: "Pending native-write Stop and browser-originated same-turn steering, with exact control receipts and independent retirement/no-effect evidence.", diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 38c2a0e3716..bc9fd401613 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(740); + expect(validateRunnerCatalog()).toHaveLength(752); for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); diff --git a/tests/runner-e2e/repository-verification.test.ts b/tests/runner-e2e/repository-verification.test.ts index 68eada012dc..f5560ef000f 100644 --- a/tests/runner-e2e/repository-verification.test.ts +++ b/tests/runner-e2e/repository-verification.test.ts @@ -46,7 +46,7 @@ describe("configured repository and conversion qualification", () => { expect(repositoryConfigurationDigest({ ...before, adapterConfig: { ...before.adapterConfig, env: {} } })).not.toBe(repositoryConfigurationDigest(before)); }); it("registers explicit development cells with staging models and production instructions", () => { - for (const id of ["configured-repository", "legacy-conversion"]) { + for (const id of ["configured-repository", "legacy-conversion", "automatic-repository-configured-models"]) { const suite = runnerSuites.find(suite => suite.id === id)!; expect(suite.manualOnly).toBe(true); expect(suite.groups).toEqual([]); @@ -54,7 +54,7 @@ describe("configured repository and conversion qualification", () => { expect(suite.profiles.map(profile => profile.model)).toEqual(["gpt-6.1-sol", "claude-opus-5-5"]); expect(suite.definitionMetadata?.stagingQualification).toBe(false); expect(suite.tasks).toHaveLength(1); - expect(suite.tasks[0]?.flow).toBe("repository_verification"); + expect(suite.tasks[0]?.flow).toBe(id === "automatic-repository-configured-models" ? "automatic_repository" : "repository_verification"); expect(suite.tasks[0]?.automaticRetryPolicy).toBe("single_attempt"); } expect(repositoryProofTasks.every(task => task.expectedRunCount === 1)).toBe(true);