diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index be2489a8e8..b9ec393f1a 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -336,7 +336,7 @@ jobs: echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")" - echo "candidate_providers=$(jq -r 'if any(.include[]; .profileId == "runner-acpx-hermes") then "hermes" else "" end' <<< "$catalog_json")" + echo "candidate_providers=$(jq -r 'if any(.include[]; .qualificationCandidate == "hermes") then "hermes" else "" end' <<< "$catalog_json")" echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" } >> "$GITHUB_OUTPUT" if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then @@ -905,7 +905,7 @@ jobs: run: node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - name: Provision pinned Hermes before the paid local test - if: matrix.environmentId == 'local' && matrix.profileId == 'runner-acpx-hermes' + if: matrix.environmentId == 'local' && matrix.qualificationCandidate == 'hermes' run: | sudo apt-get update sudo apt-get install -y --no-install-recommends python3-venv bubblewrap diff --git a/doc/evals.md b/doc/evals.md index 77b5ac15e2..ed0484532d 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -4,6 +4,12 @@ The explicit-only [live provider connection suite](../tests/runner-e2e/PROVIDER- is a Product E2E workflow for fresh subscription/API-key/gateway connections, with attended login and independent artifact checks against local or staging targets. +The explicit-only [Hermes managed API suite](../tests/runner-e2e/README.md#extended-acp-harnesses-explicit-only) +checks native completion through five selected managed API accounts on local +and Daytona, with independent account/model attribution. Its registered cells +remain pending until their live evidence passes; its API-only scope leaves the +subscription and lifecycle gates separate. + Paperclip has two live eval families with different questions, owners, and evidence. Choose the family before selecting a model, profile, or case. diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index e2f5dbab2b..cdfc5450ec 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -508,3 +508,70 @@ now covers shared Runner sources, Rust and build manifests, dependency patches, and shared workspace inputs. A glob-matching regression verifies those changes trigger the native fixture. Paid workflow authorization is unchanged. This follow-up requires another exact-head review and CI run. + +### Cloud image and live Linux browser proof, 2026-10-07 + +The trigger-coverage fix passed fresh CI and Greptile 5/5 at qualification head +`575d3bb665e7eb848f1607797f27635aba2e27f5`. Routines +`f2c047d07147f710b40b2dffcb097a71676dc8f4` and native integration +`45f591120dc78a080f347f3d3e4ad4fd05a19993` also have green checks and fresh +5/5 reviews with no unresolved threads. No PR has been merged. Further fixture +changes require new exact-head checks and review. + +The candidate image built successfully in AWS CodeBuild from +`8e31afda4ff3b4c0415bc26f8bb35cec17437741`, whose only change from the above +qualification head selects Hermes in the Dockerfile's candidate-pack default. +Its independently verified immutable digest is +`sha256:b8b8a3279e27a58d6cf5269b7e6480914f34bf036d74abb5d1e65b99996ccd6e`. +The build used a checksum-bound resolved lock, frozen dependencies, no provider +credentials, Python 3.12.14, ACP 0.9.0, ACPX 0.13.1 and the pinned Hermes release. +The image remains private and pending qualification. No local Docker build was +used; Docker Desktop was stopped again when another local process restarted it. + +CodeBuild could build the image but its execution filesystem rejected native +bubblewrap with `Can't open source /: Function not implemented`. That failed +credential-free probe is retained. The same image passed its unmodified native +command policy on a disposable EC2 amd64 host, Amazon Linux kernel +`6.1.188-233.386.amzn2023.x86_64`. The qualifier ran as uid 1001 inside an +explicitly privileged cloud container: protected files remained hidden, +assigned skills remained read-only, and allowed workspace writes succeeded. +This establishes this EC2 host's compatibility; it does not qualify Daytona. + +Three real Linux Product E2E cells passed through Chromium, an isolated Paperclip +server/database, Runnerd, ACPX, native Hermes and managed OpenRouter model +`deepseek/deepseek-v4-flash-0731`: + +| Case | Campaign suffix | Duration | Native runs | Cleanup | +| --- | --- | --- | --- | --- | +| `hello-complete` | `71743c4f328f` | 72.351 seconds | 1 | Passed | +| `question-resume-complete` | `703f08761be6` | 133.251 seconds | 2 | Passed | +| `plan-approve-complete` | `8662152371fd` | 140.890 seconds | 2 | Passed | + +All three results bind the exact image-source commit above. The catalog environment +is `local`, with a separately recorded AWS EC2 Linux execution host; these are +not Daytona/remote-target passes. The first case independently verified Done, +one final answer and a successful native run, with a reviewed final screenshot. +The second retained the question and final screenshots and verified the answer +and continuation. The third verified plan approval, continuation and completion, +with two retained screenshots. Runtime cost is not metered by that local catalog; external +EC2 cost is separate. Native token usage was available for the completion run +and one of the two runs in each question/approval workflow. Model cost remains unpriced/incomplete, and +shared-key aggregate budget readings do not establish exact per-run cost. + +The initial cloud browser attempt failed before server bootstrap because the +disposable controller lacked the compiled plugin SDK. It produced zero agent +runs, and its failed result and budget readings are retained. Setup now builds +the SDK and imports the server before model-credential handoff. Each live cell +has one attempt and no automatic retry. Secrets are passed only after verified +setup through a fresh job-bound encrypted exchange, without plaintext local +credential files. Private proof and campaign archives remain in the task's +restricted S3 evidence prefix with bounded retention; public source/docs contain +no credentials or raw provider traces. + +The new explicit-only `hermes-api-connections` matrix declares ten API-account +completion cells with independently graded native account/model attribution. +Authenticated OpenAI, Anthropic, xAI and Google catalog reads succeeded without +inference. Those discoveries and the new fixture calibration do not constitute +live provider qualification. The complete original release gates remain open: +the rest of the live controls, attachments, state/routines, credential lifecycle, +subscriptions/custom protocols/Bedrock and actual Daytona proof are still required. diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 4113257415..3fdd12780f 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -293,6 +293,15 @@ the model's claimed result by reading the actual final bytes, and also exercises remote copy-back. Keep candidate admission scoped to the selected model and the isolated operator environment; ordinary agent configuration must not enable it. +`hermes-api-connections` reuses the one-turn completion journey for five managed +API providers on local and Daytona. Its independent oracle reads the public run +ledger and native checkpoint model metadata, requiring exactly one successful +native run with the fixture's company, agent, task, account, method and +responsible user. Calibration rejects foreign accounts/companies/tasks/users, +wrong providers/methods/models/harnesses, missing evidence and extra runs. The suite's +definition includes the account/runner harness source digest. Its API-only scope +does not substitute for subscription or lifecycle qualification. + ## Persistent agent files The `instruction_persistence` flow uses production managed storage and public file diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index d313241877..a16c223b22 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1512,6 +1512,27 @@ materialized under the runner package, and build the TypeScript sidecar before local execution. Daytona additionally requires that branch's immutable Linux candidate image and the matching controller-owned provider pack described in [`docker/daytona-runner/README.md`](../../docker/daytona-runner/README.md). +The explicit-only `hermes-api-connections` suite adds ten one-turn completion +cells: managed OpenRouter, Anthropic, OpenAI, xAI and Google API accounts on +local and Daytona. Select one exact execution ID. The October 7 authenticated +catalog choices are pending inference qualification, and no production default +changes. Each fixture creates its personal account through the public +Connections API, removes agent credential overrides, and independently checks +the native run's company/task/agent scope, responsible user, selected account, +provider/method, native Hermes harness and requested/effective model. Missing or mismatched metadata +fails even when the answer is correct. The report retains the checks alongside +the ordinary completion, screenshot, token/cost and cleanup evidence. + +```sh +pnpm test:e2e:runner -- --list --suite hermes-api-connections +pnpm test:e2e:runner -- --id hermes-api-connections.runner-acpx-hermes-api-anthropic.local.hello-complete --max-automatic-retries 0 +``` + +These cells prove API-account native completion only. Subscriptions, custom +protocols, Bedrock, refresh/revocation, images and live controls need their own +qualification. The selected candidate metadata drives pre-credential Hermes +provisioning and image/provider-pack selection for every Hermes profile. + The separate Runner Evals `extended-harnesses` campaign lives in the private `paperclip-evals` repository and grades semantic protocol behavior against the mock control plane. Neither suite substitutes for the other. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 5ac24945c5..0940d3ed03 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -146,10 +146,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 63, 12, 6, 8, 2, 2, 2, 40, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, + 63, 12, 6, 8, 2, 2, 2, 40, 10, 3, 16, 16, 2, 6, 8, 46, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, 116, ]); - expect(validateRunnerCatalog()).toHaveLength(708); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(708); + expect(validateRunnerCatalog()).toHaveLength(718); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(718); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 4358aab7d4..02b92cdea4 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -5,6 +5,7 @@ import { nativeCompletionProfile, NATIVE_COMPLETION_BUDGET_CENTS } from "./nativ import { chatConfirmationTasks } from "./chat-cases.js"; import { buildConnectionSuite } from "./connection-cases.js"; import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDigest } from "./hiring-template-cases.js"; +import { hermesApiConnectionChoices, hermesApiConnectionDefinitionDigest } from "./hermes-api-connections.js"; import { nativeActiveStopTasks } from "./native-active-stop-tasks.js"; import { cursorNativeTasks } from "./cursor-native-cases.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; @@ -361,6 +362,19 @@ export const extendedHarnessProfiles: readonly RunnerProfileFixture[] = [ }), ]; +export const hermesApiConnectionProfiles: readonly RunnerProfileFixture[] = [ + extendedHarnessProfiles.find(profile => profile.qualificationCandidate === "hermes")!, + ...hermesApiConnectionChoices.map(choice => nativeProfile({ + id: `runner-acpx-hermes-api-${choice.provider}`, label: `Hermes ${choice.provider} API (candidate)`, + provider: "acpx", acpxAgent: "hermes", qualificationCandidate: "hermes", + credential: choice.credential, model: choice.model, + modelQualification: { + source: "candidate_runner_profile", + qualificationId: `hermes:v2026.9.24:${choice.provider}:api:catalog-2026-10-07:pending`, + }, + })), +]; + /** Narrow legacy ACP lanes used only by the explicit context-integrity matrix. */ export const legacyAcpxProfiles: readonly RunnerProfileFixture[] = [ legacyProfile({ @@ -1188,6 +1202,18 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ expectedMatrixSize: 40, definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28" }, }, + { + id: "hermes-api-connections", label: "Hermes managed API connections", manualOnly: true, + description: "One native browser task per exact model using an independently selected managed API account; catalog discovery is not inference proof.", + groups: ["native"], profiles: hermesApiConnectionProfiles, environments: runnerEnvironments, + tasks: [openRouterBreadthTasks.find(task => task.id === "hello-complete")!], + expectedMatrixSize: 10, + definitionMetadata: { + version: 1, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, + coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest, + }, + }, { id: "instruction-persistence", label: "Instruction Persistence", description: "Agent-owned text and binary files round trip through the editor, survive a server restart and fresh task, and synchronize concurrent edits per file with last-sync-wins.", @@ -1664,7 +1690,7 @@ function assertNoRawSecretValues(value: unknown, label: string) { export function validateRunnerCatalog(): MatrixExecution[] { const connectionSuite = runnerSuites.find(suite => suite.id === "provider-connections")!; - const allProfiles = [...connectionSuite.profiles, ...extendedHarnessProfiles, ...runnerProfiles, ...legacyAcpxProfiles, ...pendingContextIntegrityProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))]; + const allProfiles = [...connectionSuite.profiles, ...extendedHarnessProfiles, ...hermesApiConnectionProfiles.filter(p => !extendedHarnessProfiles.some(existing => existing.id === p.id)), ...runnerProfiles, ...legacyAcpxProfiles, ...pendingContextIntegrityProfiles, ...openRouterBreadthProfiles, ...everydayProfiles.filter(p => !runnerProfiles.some(existing => existing.id === p.id))]; const allTasks = [ ...connectionSuite.tasks, extendedHarnessFileTask, diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 51a0968404..c2e53b6a87 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -116,6 +116,7 @@ export function buildRunnerE2EProcessEnvironment( const agent = execution.profile.qualificationCandidate; if (!agent || agent === "cursor") continue; const admittedSuite = execution.suite.id === "extended-harnesses" + || (execution.suite.id === "hermes-api-connections" && agent === "hermes") || execution.suite.id === "rich-acp-warm-continuity"; if (!admittedSuite || !execution.suite.manualOnly) { throw new Error("Candidate qualification requires an explicit provider qualification suite"); diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts new file mode 100644 index 0000000000..95e74c97cb --- /dev/null +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; +import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; +import { gradeHermesApiConnection } from "./hermes-api-connections.js"; + +describe("Hermes managed API connection qualification", () => { + const suite = runnerSuites.find(s => s.id === "hermes-api-connections")!; + const cells = runnerMatrix.filter(e => e.suite.id === suite.id); + it("declares ten bounded pending cells without adding scheduled paid work", () => { + expect(cells).toHaveLength(10); + expect(suite.manualOnly).toBe(true); + expect(new Set(cells.map(e => e.environment.id))).toEqual(new Set(["local", "daytona"])); + expect(cells.every(e => e.task.id === "hello-complete" && e.task.expectedRunCount === 1)).toBe(true); + expect(suite.definitionMetadata).toMatchObject({ qualification: "pending", accountMethod: "api_key", accountMode: "responsible_user", coverage: "api-account-native-completion-only" }); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); + }); + it("pins the exact selected candidate and model in the operator admission", () => { + const cell = cells.find(e => e.profile.credential === "XAI_API_KEY")!; + const env = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]); + expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "hermes", model: "grok-4.7" }]); + expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...suite, manualOnly: false } }])).toThrow("explicit"); + expect(cells.every(e => e.profile.modelQualification.source === "candidate_runner_profile")).toBe(true); + expect(buildMatrixJobs(cells).every(job => job.qualificationCandidate === "hermes")).toBe(true); + expect(cells.every(e => e.profile.id.startsWith("runner-acpx-"))).toBe(true); + }); + it("retains account-fixture source provenance in its historical definition", () => { + expect(suite.definitionMetadata?.sourceDigest).toMatch(/^[a-f0-9]{64}$/); + expect(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, sourceDigest: "changed-account-selection" } })).not.toBe(suiteDefinitionHash(suite)); + }); + const valid = { + companyId: "company", agentId: "agent", issueId: "task", connectionId: "account", provider: "xai", model: "grok-4.7", + runs: [{ companyId: "company", agentId: "agent", issueId: "task", status: "succeeded", runtimeMode: "native", responsibleUserId: "user", + contextSnapshot: { aiConnection: { connectionId: "account", provider: "xai", method: "api_key", mode: "responsible_user", responsibleUserId: "user" } }, + runnerProfileJson: { nativeExecutionInput: { provider: { kind: "acpx", agent: "hermes", model: "grok-4.7" } }, + sessionCheckpoint: { providerIdentity: { kind: "acpx", requestedModel: "grok-4.7", effectiveModel: "grok-4.7" } } } }], + }; + it("accepts independently observed account/model metadata", () => { + expect(gradeHermesApiConnection(valid).every(check => check.passed)).toBe(true); + }); + it.each(["company", "task", "account", "provider", "method", "user", "model", "harness", "missing", "extra-run"])("rejects %s evidence even with a successful answer", fault => { + const wrong = structuredClone(valid), run = wrong.runs[0]!; + if (fault === "company") run.companyId = "foreign"; + if (fault === "task") run.issueId = "foreign"; + if (fault === "account") run.contextSnapshot.aiConnection.connectionId = "foreign"; + if (fault === "provider") run.contextSnapshot.aiConnection.provider = "openai"; + if (fault === "method") run.contextSnapshot.aiConnection.method = "subscription"; + if (fault === "user") run.contextSnapshot.aiConnection.responsibleUserId = "foreign"; + if (fault === "model") run.runnerProfileJson.sessionCheckpoint.providerIdentity.effectiveModel = "foreign"; + if (fault === "harness") run.runnerProfileJson.nativeExecutionInput.provider.agent = "codex"; + if (fault === "missing") wrong.runs = []; + if (fault === "extra-run") wrong.runs.push(structuredClone(run)); + expect(gradeHermesApiConnection(wrong).some(check => !check.passed)).toBe(true); + }); +}); diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts new file mode 100644 index 0000000000..60eb7cb57d --- /dev/null +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -0,0 +1,43 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerProfileFixture } from "./types.js"; + +const record = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const present = (value: unknown) => typeof value === "string" && value.trim().length > 0; + +/** Grade public run/account/model metadata; a model's completion claim cannot supply it. */ +export function gradeHermesApiConnection(input: { + companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; + runs: readonly { + companyId: string; agentId: string; status: string; runtimeMode?: string; + issueId?: string | null; responsibleUserId?: string | null; + contextSnapshot?: Record | null; runnerProfileJson?: Record | null; + }[]; +}) { + const run = input.runs[0]; + const context = record(run?.contextSnapshot), account = record(context.aiConnection); + const identity = record(record(record(run?.runnerProfileJson).sessionCheckpoint).providerIdentity); + const provider = record(record(record(run?.runnerProfileJson).nativeExecutionInput).provider); + return [ + { id: "one-successful-native-run", passed: input.runs.length === 1 && run?.status === "succeeded" && run.runtimeMode === "native" }, + { id: "company-agent-task-scope", passed: run?.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId }, + { id: "selected-managed-api-account", passed: account.connectionId === input.connectionId && account.provider === input.provider && account.method === "api_key" && account.mode === "responsible_user" }, + { id: "responsible-user-attribution", passed: present(run?.responsibleUserId) && account.responsibleUserId === run?.responsibleUserId }, + { id: "native-hermes-provider", passed: provider.kind === "acpx" && provider.agent === "hermes" && provider.model === input.model }, + { id: "exact-native-model", passed: identity.kind === "acpx" && identity.requestedModel === input.model && identity.effectiveModel === input.model }, + ]; +} + +/** Explicit authenticated catalog choices; none is a production default or live qualification. */ +export const hermesApiConnectionChoices = [ + { provider: "anthropic", credential: "ANTHROPIC_API_KEY", model: "claude-haiku-4-5-20251001" }, + { provider: "openai", credential: "OPENAI_API_KEY", model: "gpt-5.6-luna" }, + { provider: "xai", credential: "XAI_API_KEY", model: "grok-4.7" }, + { provider: "google", credential: "GEMINI_API_KEY", model: "gemini-2.5-flash" }, +] as const satisfies readonly { provider: string; credential: RunnerProfileFixture["credential"]; model: string }[]; + +export const hermesApiConnectionDefinitionDigest = createHash("sha256").update( + ["hermes-api-connections.ts", "live-fixtures.ts", "harness-env.ts", "runner.spec.ts"] + .map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n"), +).digest("hex"); diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 2bce9d3e01..d977a0310b 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -93,6 +93,10 @@ describe("live runner fixtures", () => { it.each([ ["runner-acpx-hermes", "extended-harnesses", "hello-complete"], + ["runner-acpx-hermes-api-anthropic", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes-api-openai", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes-api-xai", "hermes-api-connections", "hello-complete"], + ["runner-acpx-hermes-api-google", "hermes-api-connections", "hello-complete"], ["runner-codex", "hiring-templates", "hire-coder-template-reuse"], ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"], ["runner-codex", "everyday-workflows", "hire-reuse"], @@ -111,7 +115,9 @@ describe("live runner fixtures", () => { e.environment.id === "local", )!; const provider = - profile === "runner-acpx-claude" ? "anthropic" : ["runner-opencode", "runner-acpx-hermes"].includes(profile) ? "openrouter" : "openai"; + profile === "runner-acpx-hermes-api-xai" ? "xai" : profile === "runner-acpx-hermes-api-google" ? "google" + : ["runner-acpx-claude", "runner-acpx-hermes-api-anthropic"].includes(profile) ? "anthropic" + : ["runner-opencode", "runner-acpx-hermes"].includes(profile) ? "openrouter" : "openai"; let connected = false; let agentBody: any; const api = { diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index ad22b4165e..6e92451beb 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -40,7 +40,7 @@ interface AgentRecord { interface ManagedAccountFixture { connectionId: string; binding: { - provider: "openai" | "anthropic" | "openrouter"; + provider: "openai" | "anthropic" | "openrouter" | "xai" | "google"; method: "api_key"; mode: "responsible_user"; }; @@ -252,7 +252,9 @@ export async function setupLiveFixtures(input: { const key = execution.profile.credential; const provider = key === "ANTHROPIC_API_KEY" ? "anthropic" : key === "OPENROUTER_API_KEY" ? "openrouter" - : key === "OPENAI_API_KEY" ? "openai" : null; + : key === "OPENAI_API_KEY" ? "openai" + : key === "XAI_API_KEY" ? "xai" + : key === "GEMINI_API_KEY" ? "google" : null; if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`); const apiKey = input.credentials[key]; if (!apiKey) throw new Error(`Missing credential ${key}`); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 88d1c16bca..5ab50a50e3 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,4 +1,5 @@ import { runPlanTaskFlow } from "./plan-task-flow.js"; +import { gradeHermesApiConnection } from "./hermes-api-connections.js"; import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js"; import { assertNativeInstructionSelection, verifyNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_DEFAULT_SHA256 } from "./native-instruction-consolidation.js"; import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js"; @@ -111,6 +112,8 @@ interface RunRecord { id: string; companyId: string; agentId: string; + issueId?: string | null; + responsibleUserId?: string | null; status: string; runtimeMode?: string; continuationAttempt?: number; @@ -2863,6 +2866,15 @@ for (const execution of executions) { } } } + if (execution.suite.id === "hermes-api-connections") { + if (!fixtures?.aiConnection || !issue?.id) throw new Error("Hermes connection qualification is missing its selected account or task"); + const checks = gradeHermesApiConnection({ companyId: fixtures.company.id, agentId: fixtures.agent.id, + issueId: issue.id, connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider, + model: execution.profile.model, runs: selectedRuns }); + matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const, path: `hermesConnection.${check.id}`, expected: true }, passed: check.passed, detail: "Public native run metadata must match the selected managed API account and exact model." }))); + await writeSanitizedJson(snapshotsDir, "hermes-api-connection.json", { checks }, secrets); + expect(checks.every(check => check.passed), "Hermes managed account and native model attribution").toBe(true); + } if (runsCompletionUpdateProbe(execution) && credentials.OPENAI_API_KEY) { const qualification = completionQualityStatus(completionQuality); if (qualification === "unqualified") { diff --git a/tests/runner-e2e/selectors.ts b/tests/runner-e2e/selectors.ts index 1c48ae8d91..14c34680ca 100644 --- a/tests/runner-e2e/selectors.ts +++ b/tests/runner-e2e/selectors.ts @@ -211,6 +211,7 @@ export function buildMatrixJobs( executionId: execution.id, suiteId: execution.suite.id, profileId: execution.profile.id, + ...(execution.profile.qualificationCandidate ? { qualificationCandidate: execution.profile.qualificationCandidate } : {}), credentialName: execution.profile.credential, environmentId: execution.environment.id, caseId: execution.task.id, diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 0f0ad2328f..ea2ce460e1 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -200,6 +200,7 @@ export interface RunnerSuiteFixture { } export interface MatrixJob { + qualificationCandidate?: RunnerProfileFixture["qualificationCandidate"]; executionId: string; suiteId: string; profileId: string; diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index ce363aac65..be176bcaa2 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -59,7 +59,7 @@ describe("public repository paid workflow security", () => { it("provisions selected Hermes assets before credentials and uses the same selection for image identity and packs", async () => { const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:")); - expect(catalog).toContain('if any(.include[]; .profileId == "runner-acpx-hermes") then "hermes" else "" end'); + expect(catalog).toContain('if any(.include[]; .qualificationCandidate == "hermes") then "hermes" else "" end'); expect(catalog.indexOf("Validate selectors and emit matrix")).toBeLessThan(catalog.indexOf("Compute Daytona image content ID")); expect(catalog).toContain('"--candidate-providers=$CANDIDATE_PROVIDERS"'); const image = workflow.slice(workflow.indexOf(" daytona_image:"), workflow.indexOf(" build_runner_artifacts:")); @@ -70,7 +70,7 @@ describe("public repository paid workflow security", () => { expect(pack).toContain('"--candidate-providers=$CANDIDATE_PROVIDERS"'); expect(pack).not.toContain("secrets."); const paid = workflow.slice(workflow.indexOf(" test:"), workflow.indexOf(" aggregate:")); - expect(paid).toContain("matrix.environmentId == 'local' && matrix.profileId == 'runner-acpx-hermes'"); + expect(paid).toContain("matrix.environmentId == 'local' && matrix.qualificationCandidate == 'hermes'"); const setup = paid.indexOf("Provision pinned Hermes before the paid local test"); expect(setup).toBeGreaterThan(0); expect(setup).toBeLessThan(paid.indexOf("secrets.OPENROUTER_API_KEY"));