From b10746b4e68415e7f38601534c83c313af5069e7 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 7 Oct 2026 16:20:20 -0500 Subject: [PATCH] test(hermes): qualify managed API accounts through the runner Add ten explicit pending API-account browser cells and independently grade the selected account, native harness, and exact model. Keep fixture credential staging on the shared Connections path. Carry candidate metadata into CI so every selected Hermes profile receives verified runtime assets before secrets. Record the private AWS image build and three actual Linux browser passes without promoting Hermes. Product E2E support typecheck and all 102 affected tests pass; the broader support suite passed 1,826 Vitest tests and 128 node assertions before the final oracle refinements. Co-Authored-By: Paperclip --- .github/workflows/runner-full-stack-e2e.yml | 4 +- doc/evals.md | 6 ++ doc/plans/2026-10-06-hermes-native-runner.md | 67 +++++++++++++++++++ tests/runner-e2e/FIXTURES.md | 9 +++ tests/runner-e2e/README.md | 21 ++++++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 28 +++++++- tests/runner-e2e/harness-env.ts | 1 + .../runner-e2e/hermes-api-connections.test.ts | 55 +++++++++++++++ tests/runner-e2e/hermes-api-connections.ts | 43 ++++++++++++ tests/runner-e2e/live-fixtures.test.ts | 8 ++- tests/runner-e2e/live-fixtures.ts | 6 +- tests/runner-e2e/runner.spec.ts | 12 ++++ tests/runner-e2e/selectors.ts | 1 + tests/runner-e2e/types.ts | 1 + tests/runner-e2e/workflow-security.test.ts | 4 +- 16 files changed, 261 insertions(+), 11 deletions(-) create mode 100644 tests/runner-e2e/hermes-api-connections.test.ts create mode 100644 tests/runner-e2e/hermes-api-connections.ts diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index be2489a8e8..b9ec393f1a 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -336,7 +336,7 @@ jobs: echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")" - echo "candidate_providers=$(jq -r 'if any(.include[]; .profileId == "runner-acpx-hermes") then "hermes" else "" end' <<< "$catalog_json")" + echo "candidate_providers=$(jq -r 'if any(.include[]; .qualificationCandidate == "hermes") then "hermes" else "" end' <<< "$catalog_json")" echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" } >> "$GITHUB_OUTPUT" if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then @@ -905,7 +905,7 @@ jobs: run: node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - name: Provision pinned Hermes before the paid local test - if: matrix.environmentId == 'local' && matrix.profileId == 'runner-acpx-hermes' + if: matrix.environmentId == 'local' && matrix.qualificationCandidate == 'hermes' run: | sudo apt-get update sudo apt-get install -y --no-install-recommends python3-venv bubblewrap diff --git a/doc/evals.md b/doc/evals.md index 77b5ac15e2..ed0484532d 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -4,6 +4,12 @@ The explicit-only [live provider connection suite](../tests/runner-e2e/PROVIDER- is a Product E2E workflow for fresh subscription/API-key/gateway connections, with attended login and independent artifact checks against local or staging targets. +The explicit-only [Hermes managed API suite](../tests/runner-e2e/README.md#extended-acp-harnesses-explicit-only) +checks native completion through five selected managed API accounts on local +and Daytona, with independent account/model attribution. Its registered cells +remain pending until their live evidence passes; its API-only scope leaves the +subscription and lifecycle gates separate. + Paperclip has two live eval families with different questions, owners, and evidence. Choose the family before selecting a model, profile, or case. diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index e2f5dbab2b..cdfc5450ec 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -508,3 +508,70 @@ now covers shared Runner sources, Rust and build manifests, dependency patches, and shared workspace inputs. A glob-matching regression verifies those changes trigger the native fixture. Paid workflow authorization is unchanged. This follow-up requires another exact-head review and CI run. + +### Cloud image and live Linux browser proof, 2026-10-07 + +The trigger-coverage fix passed fresh CI and Greptile 5/5 at qualification head +`575d3bb665e7eb848f1607797f27635aba2e27f5`. Routines +`f2c047d07147f710b40b2dffcb097a71676dc8f4` and native integration +`45f591120dc78a080f347f3d3e4ad4fd05a19993` also have green checks and fresh +5/5 reviews with no unresolved threads. No PR has been merged. Further fixture +changes require new exact-head checks and review. + +The candidate image built successfully in AWS CodeBuild from +`8e31afda4ff3b4c0415bc26f8bb35cec17437741`, whose only change from the above +qualification head selects Hermes in the Dockerfile's candidate-pack default. +Its independently verified immutable digest is +`sha256:b8b8a3279e27a58d6cf5269b7e6480914f34bf036d74abb5d1e65b99996ccd6e`. +The build used a checksum-bound resolved lock, frozen dependencies, no provider +credentials, Python 3.12.14, ACP 0.9.0, ACPX 0.13.1 and the pinned Hermes release. +The image remains private and pending qualification. No local Docker build was +used; Docker Desktop was stopped again when another local process restarted it. + +CodeBuild could build the image but its execution filesystem rejected native +bubblewrap with `Can't open source /: Function not implemented`. That failed +credential-free probe is retained. The same image passed its unmodified native +command policy on a disposable EC2 amd64 host, Amazon Linux kernel +`6.1.188-233.386.amzn2023.x86_64`. The qualifier ran as uid 1001 inside an +explicitly privileged cloud container: protected files remained hidden, +assigned skills remained read-only, and allowed workspace writes succeeded. +This establishes this EC2 host's compatibility; it does not qualify Daytona. + +Three real Linux Product E2E cells passed through Chromium, an isolated Paperclip +server/database, Runnerd, ACPX, native Hermes and managed OpenRouter model +`deepseek/deepseek-v4-flash-0731`: + +| Case | Campaign suffix | Duration | Native runs | Cleanup | +| --- | --- | --- | --- | --- | +| `hello-complete` | `71743c4f328f` | 72.351 seconds | 1 | Passed | +| `question-resume-complete` | `703f08761be6` | 133.251 seconds | 2 | Passed | +| `plan-approve-complete` | `8662152371fd` | 140.890 seconds | 2 | Passed | + +All three results bind the exact image-source commit above. The catalog environment +is `local`, with a separately recorded AWS EC2 Linux execution host; these are +not Daytona/remote-target passes. The first case independently verified Done, +one final answer and a successful native run, with a reviewed final screenshot. +The second retained the question and final screenshots and verified the answer +and continuation. The third verified plan approval, continuation and completion, +with two retained screenshots. Runtime cost is not metered by that local catalog; external +EC2 cost is separate. Native token usage was available for the completion run +and one of the two runs in each question/approval workflow. Model cost remains unpriced/incomplete, and +shared-key aggregate budget readings do not establish exact per-run cost. + +The initial cloud browser attempt failed before server bootstrap because the +disposable controller lacked the compiled plugin SDK. It produced zero agent +runs, and its failed result and budget readings are retained. Setup now builds +the SDK and imports the server before model-credential handoff. Each live cell +has one attempt and no automatic retry. Secrets are passed only after verified +setup through a fresh job-bound encrypted exchange, without plaintext local +credential files. Private proof and campaign archives remain in the task's +restricted S3 evidence prefix with bounded retention; public source/docs contain +no credentials or raw provider traces. + +The new explicit-only `hermes-api-connections` matrix declares ten API-account +completion cells with independently graded native account/model attribution. +Authenticated OpenAI, Anthropic, xAI and Google catalog reads succeeded without +inference. Those discoveries and the new fixture calibration do not constitute +live provider qualification. The complete original release gates remain open: +the rest of the live controls, attachments, state/routines, credential lifecycle, +subscriptions/custom protocols/Bedrock and actual Daytona proof are still required. diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 4113257415..3fdd12780f 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -293,6 +293,15 @@ the model's claimed result by reading the actual final bytes, and also exercises remote copy-back. Keep candidate admission scoped to the selected model and the isolated operator environment; ordinary agent configuration must not enable it. +`hermes-api-connections` reuses the one-turn completion journey for five managed +API providers on local and Daytona. Its independent oracle reads the public run +ledger and native checkpoint model metadata, requiring exactly one successful +native run with the fixture's company, agent, task, account, method and +responsible user. Calibration rejects foreign accounts/companies/tasks/users, +wrong providers/methods/models/harnesses, missing evidence and extra runs. The suite's +definition includes the account/runner harness source digest. Its API-only scope +does not substitute for subscription or lifecycle qualification. + ## Persistent agent files The `instruction_persistence` flow uses production managed storage and public file diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index d313241877..a16c223b22 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1512,6 +1512,27 @@ materialized under the runner package, and build the TypeScript sidecar before local execution. Daytona additionally requires that branch's immutable Linux candidate image and the matching controller-owned provider pack described in [`docker/daytona-runner/README.md`](../../docker/daytona-runner/README.md). +The explicit-only `hermes-api-connections` suite adds ten one-turn completion +cells: managed OpenRouter, Anthropic, OpenAI, xAI and Google API accounts on +local and Daytona. Select one exact execution ID. The October 7 authenticated +catalog choices are pending inference qualification, and no production default +changes. Each fixture creates its personal account through the public +Connections API, removes agent credential overrides, and independently checks +the native run's company/task/agent scope, responsible user, selected account, +provider/method, native Hermes harness and requested/effective model. Missing or mismatched metadata +fails even when the answer is correct. The report retains the checks alongside +the ordinary completion, screenshot, token/cost and cleanup evidence. + +```sh +pnpm test:e2e:runner -- --list --suite hermes-api-connections +pnpm test:e2e:runner -- --id hermes-api-connections.runner-acpx-hermes-api-anthropic.local.hello-complete --max-automatic-retries 0 +``` + +These cells prove API-account native completion only. Subscriptions, custom +protocols, Bedrock, refresh/revocation, images and live controls need their own +qualification. The selected candidate metadata drives pre-credential Hermes +provisioning and image/provider-pack selection for every Hermes profile. + The separate Runner Evals `extended-harnesses` campaign lives in the private `paperclip-evals` repository and grades semantic protocol behavior against the mock control plane. Neither suite substitutes for the other. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 5ac24945c5..0940d3ed03 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -146,10 +146,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 63, 12, 6, 8, 2, 2, 2, 40, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, + 63, 12, 6, 8, 2, 2, 2, 40, 10, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, ]); - expect(validateRunnerCatalog()).toHaveLength(708); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(708); + expect(validateRunnerCatalog()).toHaveLength(718); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(718); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 4358aab7d4..02b92cdea4 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -5,6 +5,7 @@ import { nativeCompletionProfile, NATIVE_COMPLETION_BUDGET_CENTS } from "./nativ import { chatConfirmationTasks } from "./chat-cases.js"; import { buildConnectionSuite } from "./connection-cases.js"; import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDigest } from "./hiring-template-cases.js"; +import { hermesApiConnectionChoices, hermesApiConnectionDefinitionDigest } from "./hermes-api-connections.js"; import { nativeActiveStopTasks } from "./native-active-stop-tasks.js"; import { cursorNativeTasks } from "./cursor-native-cases.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; @@ -361,6 +362,19 @@ export const extendedHarnessProfiles: readonly RunnerProfileFixture[] = [ }), ]; +export const hermesApiConnectionProfiles: readonly RunnerProfileFixture[] = [ + extendedHarnessProfiles.find(profile => profile.qualificationCandidate === "hermes")!, + ...hermesApiConnectionChoices.map(choice => nativeProfile({ + id: `runner-acpx-hermes-api-${choice.provider}`, label: `Hermes ${choice.provider} API (candidate)`, + provider: "acpx", acpxAgent: "hermes", qualificationCandidate: "hermes", + credential: choice.credential, model: choice.model, + modelQualification: { + source: "candidate_runner_profile", + qualificationId: `hermes:v2026.9.24:${choice.provider}:api:catalog-2026-10-07:pending`, + }, + })), +]; + /** Narrow legacy ACP lanes used only by the explicit context-integrity matrix. */ export const legacyAcpxProfiles: readonly RunnerProfileFixture[] = [ legacyProfile({ @@ -1188,6 +1202,18 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ expectedMatrixSize: 40, definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" }, }, + { + id: "hermes-api-connections", label: "Hermes managed API connections", manualOnly: true, + description: "One native browser task per exact model using an independently selected managed API account; catalog discovery is not inference proof.", + groups: ["native"], profiles: hermesApiConnectionProfiles, environments: runnerEnvironments, + tasks: [openRouterBreadthTasks.find(task => task.id === "hello-complete")!], + expectedMatrixSize: 10, + definitionMetadata: { + version: 1, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, + coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest, + }, + }, { id: "instruction-persistence", label: "Instruction Persistence", description: "Agent-owned text and binary files round trip through the editor, survive a server restart and fresh task, and synchronize concurrent edits per file with last-sync-wins.", @@ -1664,7 +1690,7 @@ function assertNoRawSecretValues(value: unknown, label: string) { export function validateRunnerCatalog(): MatrixExecution[] { const connectionSuite = runnerSuites.find(suite => suite.id === "provider-connections")!; - const allProfiles = [...connectionSuite.profiles, ...extendedHarnessProfiles, ...runnerProfiles, ...legacyAcpxProfiles, ...pendingContextIntegrityProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))]; + const allProfiles = [...connectionSuite.profiles, ...extendedHarnessProfiles, ...hermesApiConnectionProfiles.filter(p => !extendedHarnessProfiles.some(existing => existing.id === p.id)), ...runnerProfiles, ...legacyAcpxProfiles, ...pendingContextIntegrityProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))]; const allTasks = [ ...connectionSuite.tasks, extendedHarnessFileTask, diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 51a0968404..c2e53b6a87 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -116,6 +116,7 @@ export function buildRunnerE2EProcessEnvironment( const agent = execution.profile.qualificationCandidate; if (!agent || agent === "cursor") continue; const admittedSuite = execution.suite.id === "extended-harnesses" + || (execution.suite.id === "hermes-api-connections" && agent === "hermes") || execution.suite.id === "rich-acp-warm-continuity"; if (!admittedSuite || !execution.suite.manualOnly) { throw new Error("Candidate qualification requires an explicit provider qualification suite"); diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts new file mode 100644 index 0000000000..95e74c97cb --- /dev/null +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; +import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; +import { gradeHermesApiConnection } from "./hermes-api-connections.js"; + +describe("Hermes managed API connection qualification", () => { + const suite = runnerSuites.find(s => s.id === "hermes-api-connections")!; + const cells = runnerMatrix.filter(e => e.suite.id === suite.id); + it("declares ten bounded pending cells without adding scheduled paid work", () => { + expect(cells).toHaveLength(10); + expect(suite.manualOnly).toBe(true); + expect(new Set(cells.map(e => e.environment.id))).toEqual(new Set(["local", "daytona"])); + expect(cells.every(e => e.task.id === "hello-complete" && e.task.expectedRunCount === 1)).toBe(true); + expect(suite.definitionMetadata).toMatchObject({ qualification: "pending", accountMethod: "api_key", accountMode: "responsible_user", coverage: "api-account-native-completion-only" }); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); + }); + it("pins the exact selected candidate and model in the operator admission", () => { + const cell = cells.find(e => e.profile.credential === "XAI_API_KEY")!; + const env = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]); + expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "hermes", model: "grok-4.7" }]); + expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...suite, manualOnly: false } }])).toThrow("explicit"); + expect(cells.every(e => e.profile.modelQualification.source === "candidate_runner_profile")).toBe(true); + expect(buildMatrixJobs(cells).every(job => job.qualificationCandidate === "hermes")).toBe(true); + expect(cells.every(e => e.profile.id.startsWith("runner-acpx-"))).toBe(true); + }); + it("retains account-fixture source provenance in its historical definition", () => { + expect(suite.definitionMetadata?.sourceDigest).toMatch(/^[a-f0-9]{64}$/); + expect(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, sourceDigest: "changed-account-selection" } })).not.toBe(suiteDefinitionHash(suite)); + }); + const valid = { + companyId: "company", agentId: "agent", issueId: "task", connectionId: "account", provider: "xai", model: "grok-4.7", + runs: [{ companyId: "company", agentId: "agent", issueId: "task", status: "succeeded", runtimeMode: "native", responsibleUserId: "user", + contextSnapshot: { aiConnection: { connectionId: "account", provider: "xai", method: "api_key", mode: "responsible_user", responsibleUserId: "user" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "acpx", agent: "hermes", model: "grok-4.7" } }, + sessionCheckpoint: { providerIdentity: { kind: "acpx", requestedModel: "grok-4.7", effectiveModel: "grok-4.7" } } } }], + }; + it("accepts independently observed account/model metadata", () => { + expect(gradeHermesApiConnection(valid).every(check => check.passed)).toBe(true); + }); + it.each(["company", "task", "account", "provider", "method", "user", "model", "harness", "missing", "extra-run"])("rejects %s evidence even with a successful answer", fault => { + const wrong = structuredClone(valid), run = wrong.runs[0]!; + if (fault === "company") run.companyId = "foreign"; + if (fault === "task") run.issueId = "foreign"; + if (fault === "account") run.contextSnapshot.aiConnection.connectionId = "foreign"; + if (fault === "provider") run.contextSnapshot.aiConnection.provider = "openai"; + if (fault === "method") run.contextSnapshot.aiConnection.method = "subscription"; + if (fault === "user") run.contextSnapshot.aiConnection.responsibleUserId = "foreign"; + if (fault === "model") run.runnerProfileJson.sessionCheckpoint.providerIdentity.effectiveModel = "foreign"; + if (fault === "harness") run.runnerProfileJson.nativeExecutionInput.provider.agent = "codex"; + if (fault === "missing") wrong.runs = []; + if (fault === "extra-run") wrong.runs.push(structuredClone(run)); + expect(gradeHermesApiConnection(wrong).some(check => !check.passed)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts new file mode 100644 index 0000000000..60eb7cb57d --- /dev/null +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -0,0 +1,43 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerProfileFixture } from "./types.js"; + +const record = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const present = (value: unknown) => typeof value === "string" && value.trim().length > 0; + +/** Grade public run/account/model metadata; a model's completion claim cannot supply it. */ +export function gradeHermesApiConnection(input: { + companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; + runs: readonly { + companyId: string; agentId: string; status: string; runtimeMode?: string; + issueId?: string | null; responsibleUserId?: string | null; + contextSnapshot?: Record | null; runnerProfileJson?: Record | null; + }[]; +}) { + const run = input.runs[0]; + const context = record(run?.contextSnapshot), account = record(context.aiConnection); + const identity = record(record(record(run?.runnerProfileJson).sessionCheckpoint).providerIdentity); + const provider = record(record(record(run?.runnerProfileJson).nativeExecutionInput).provider); + return [ + { id: "one-successful-native-run", passed: input.runs.length === 1 && run?.status === "succeeded" && run.runtimeMode === "native" }, + { id: "company-agent-task-scope", passed: run?.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId }, + { id: "selected-managed-api-account", passed: account.connectionId === input.connectionId && account.provider === input.provider && account.method === "api_key" && account.mode === "responsible_user" }, + { id: "responsible-user-attribution", passed: present(run?.responsibleUserId) && account.responsibleUserId === run?.responsibleUserId }, + { id: "native-hermes-provider", passed: provider.kind === "acpx" && provider.agent === "hermes" && provider.model === input.model }, + { id: "exact-native-model", passed: identity.kind === "acpx" && identity.requestedModel === input.model && identity.effectiveModel === input.model }, + ]; +} + +/** Explicit authenticated catalog choices; none is a production default or live qualification. */ +export const hermesApiConnectionChoices = [ + { provider: "anthropic", credential: "ANTHROPIC_API_KEY", model: "claude-haiku-4-5-20251001" }, + { provider: "openai", credential: "OPENAI_API_KEY", model: "gpt-5.6-luna" }, + { provider: "xai", credential: "XAI_API_KEY", model: "grok-4.7" }, + { provider: "google", credential: "GEMINI_API_KEY", model: "gemini-2.5-flash" }, +] as const satisfies readonly { provider: string; credential: RunnerProfileFixture["credential"]; model: string }[]; + +export const hermesApiConnectionDefinitionDigest = createHash("sha256").update( + ["hermes-api-connections.ts", "live-fixtures.ts", "harness-env.ts", "runner.spec.ts"] + .map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n"), +).digest("hex"); diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 2bce9d3e01..d977a0310b 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -93,6 +93,10 @@ describe("live runner fixtures", () => { it.each([ ["runner-acpx-hermes", "extended-harnesses", "hello-complete"], + ["runner-acpx-hermes-api-anthropic", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes-api-openai", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes-api-xai", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes-api-google", "hermes-api-connections", "hello-complete"], ["runner-codex", "hiring-templates", "hire-coder-template-reuse"], ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"], ["runner-codex", "everyday-workflows", "hire-reuse"], @@ -111,7 +115,9 @@ describe("live runner fixtures", () => { e.environment.id === "local", )!; const provider = - profile === "runner-acpx-claude" ? "anthropic" : ["runner-opencode", "runner-acpx-hermes"].includes(profile) ? "openrouter" : "openai"; + profile === "runner-acpx-hermes-api-xai" ? "xai" : profile === "runner-acpx-hermes-api-google" ? "google" + : ["runner-acpx-claude", "runner-acpx-hermes-api-anthropic"].includes(profile) ? "anthropic" + : ["runner-opencode", "runner-acpx-hermes"].includes(profile) ? "openrouter" : "openai"; let connected = false; let agentBody: any; const api = { diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index ad22b4165e..6e92451beb 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -40,7 +40,7 @@ interface AgentRecord { interface ManagedAccountFixture { connectionId: string; binding: { - provider: "openai" | "anthropic" | "openrouter"; + provider: "openai" | "anthropic" | "openrouter" | "xai" | "google"; method: "api_key"; mode: "responsible_user"; }; @@ -252,7 +252,9 @@ export async function setupLiveFixtures(input: { const key = execution.profile.credential; const provider = key === "ANTHROPIC_API_KEY" ? "anthropic" : key === "OPENROUTER_API_KEY" ? "openrouter" - : key === "OPENAI_API_KEY" ? "openai" : null; + : key === "OPENAI_API_KEY" ? "openai" + : key === "XAI_API_KEY" ? "xai" + : key === "GEMINI_API_KEY" ? "google" : null; if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`); const apiKey = input.credentials[key]; if (!apiKey) throw new Error(`Missing credential ${key}`); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 88d1c16bca..5ab50a50e3 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,4 +1,5 @@ import { runPlanTaskFlow } from "./plan-task-flow.js"; +import { gradeHermesApiConnection } from "./hermes-api-connections.js"; import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js"; import { assertNativeInstructionSelection, verifyNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_DEFAULT_SHA256 } from "./native-instruction-consolidation.js"; import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js"; @@ -111,6 +112,8 @@ interface RunRecord { id: string; companyId: string; agentId: string; + issueId?: string | null; + responsibleUserId?: string | null; status: string; runtimeMode?: string; continuationAttempt?: number; @@ -2863,6 +2866,15 @@ for (const execution of executions) { } } } + if (execution.suite.id === "hermes-api-connections") { + if (!fixtures?.aiConnection || !issue?.id) throw new Error("Hermes connection qualification is missing its selected account or task"); + const checks = gradeHermesApiConnection({ companyId: fixtures.company.id, agentId: fixtures.agent.id, + issueId: issue.id, connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider, + model: execution.profile.model, runs: selectedRuns }); + matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const, path: `hermesConnection.${check.id}`, expected: true }, passed: check.passed, detail: "Public native run metadata must match the selected managed API account and exact model." }))); + await writeSanitizedJson(snapshotsDir, "hermes-api-connection.json", { checks }, secrets); + expect(checks.every(check => check.passed), "Hermes managed account and native model attribution").toBe(true); + } if (runsCompletionUpdateProbe(execution) && credentials.OPENAI_API_KEY) { const qualification = completionQualityStatus(completionQuality); if (qualification === "unqualified") { diff --git a/tests/runner-e2e/selectors.ts b/tests/runner-e2e/selectors.ts index 1c48ae8d91..14c34680ca 100644 --- a/tests/runner-e2e/selectors.ts +++ b/tests/runner-e2e/selectors.ts @@ -211,6 +211,7 @@ export function buildMatrixJobs( executionId: execution.id, suiteId: execution.suite.id, profileId: execution.profile.id, + ...(execution.profile.qualificationCandidate ? { qualificationCandidate: execution.profile.qualificationCandidate } : {}), credentialName: execution.profile.credential, environmentId: execution.environment.id, caseId: execution.task.id, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 0f0ad2328f..ea2ce460e1 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -200,6 +200,7 @@ export interface RunnerSuiteFixture { } export interface MatrixJob { + qualificationCandidate?: RunnerProfileFixture["qualificationCandidate"]; executionId: string; suiteId: string; profileId: string; diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index ce363aac65..be176bcaa2 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -59,7 +59,7 @@ describe("public repository paid workflow security", () => { it("provisions selected Hermes assets before credentials and uses the same selection for image identity and packs", async () => { const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:")); - expect(catalog).toContain('if any(.include[]; .profileId == "runner-acpx-hermes") then "hermes" else "" end'); + expect(catalog).toContain('if any(.include[]; .qualificationCandidate == "hermes") then "hermes" else "" end'); expect(catalog.indexOf("Validate selectors and emit matrix")).toBeLessThan(catalog.indexOf("Compute Daytona image content ID")); expect(catalog).toContain('"--candidate-providers=$CANDIDATE_PROVIDERS"'); const image = workflow.slice(workflow.indexOf(" daytona_image:"), workflow.indexOf(" build_runner_artifacts:")); @@ -70,7 +70,7 @@ describe("public repository paid workflow security", () => { expect(pack).toContain('"--candidate-providers=$CANDIDATE_PROVIDERS"'); expect(pack).not.toContain("secrets."); const paid = workflow.slice(workflow.indexOf(" test:"), workflow.indexOf(" aggregate:")); - expect(paid).toContain("matrix.environmentId == 'local' && matrix.profileId == 'runner-acpx-hermes'"); + expect(paid).toContain("matrix.environmentId == 'local' && matrix.qualificationCandidate == 'hermes'"); const setup = paid.indexOf("Provision pinned Hermes before the paid local test"); expect(setup).toBeGreaterThan(0); expect(setup).toBeLessThan(paid.indexOf("secrets.OPENROUTER_API_KEY"));