diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 8a2ac44a9a1..547067ef689 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -596,3 +596,54 @@ required Gemini key. The key is now mapped only when the selected matrix cell requires it, and the workflow credential-boundary regression covers it. All 15 workflow tests and actionlint pass. Another current-head review and CI run are required after this correction. + +### Managed API account qualification, 2026-10-07 + +The Google credential mapping subsequently passed all 54 current-head CI checks +(two intentional skips) and fresh Greptile 5/5, with no open threads, at +`ffd6bbc1ebb10c1c8ef0db875bb88d733f64649d`. The following paid account cases +use that controller source and the previously verified image source +`8e31afda4ff3b4c0415bc26f8bb35cec17437741`. Independent tracked-source +comparison proves their production runtime and dependency sources identical; +the changes are qualification fixtures, documentation, workflow credential +mapping, and the image's explicit candidate selection. The runtime-source +fingerprint is `9d9e39d5390de779c619004ebbce55cc57d2c1aeefead36099903d7db1d3e26a`. + +| Managed API account | Exact model | Campaign suffix | Result | Duration | +| --- | --- | --- | --- | --- | +| Anthropic | `claude-haiku-4-5-20251001` | `2d55c9944d84` | Passed | 53.780 s | +| OpenAI | `gpt-5.6-luna` | `4953bfa797ea` | Passed | 58.760 s | +| xAI | `grok-4.7` | `83fd6210c933` | Passed | 69.960 s | +| Google | `gemini-2.5-flash` | `ea98e1dd517f` | Failed | 52.511 s | + +Each passing case independently verifies six public native-run account/model +checks, both scoped 200-cent budgets before task creation, one successful native +run, one final answer, and cleanup. Each has a reviewed final browser screenshot. +All run on the compatible EC2 Linux amd64 host with catalog environment `local`; +they do not qualify macOS or Daytona. Token receipts are available, but monetary +cost remains unpriced. The final screenshots show the existing budget policy +pausing the test agents after unpriced usage; reported zeros are not free runs. + +xAI delivered 41 reasoning deltas and 22 assistant-text deltas before its native +terminal event. Its final screenshot renders a thought card above the answer. +These facts prove event delivery and final rendering, not incremental browser +render timing while the run is active. Raw reasoning is not published. + +Google's authenticated catalog listed Gemini 2.5 Flash, but inference returned +404 because the account had not previously used that model. Google's +[access notice](https://ai.google.dev/gemini-api/docs/deprecations) confirms the +restriction. The failed browser attempt, provider error, missing usage, and +successful cleanup remain retained. Hermes made its native HTTP retry attempts +inside that single controller turn; the campaign did not dispatch a new paid +attempt. The fixture now selects `gemini-3.8-flash`, whose authenticated metadata +supports `generateContent`. That metadata is not a live qualification pass. +The model correction passed all 94 affected tests and Product E2E TypeScript; +it requires a new live attempt, fresh CI, and review. + +Private proof and campaign archives remain encrypted in the task's restricted +S3 prefix. Credentials are supplied only through a fresh verified job-bound +encrypted handoff. Obsolete encrypted key transfers are removed after evidence +collection. No local Docker is used. Subscription/credential lifecycle, custom +protocols, Bedrock, the remaining interactions/state/routines, final macOS +campaign, clean distribution, and actual Daytona gates remain open. Hermes +continues to be pending qualification. diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index 98b85787572..2ce4254654a 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -15,10 +15,13 @@ describe("Hermes managed API connection qualification", () => { expect(suite.definitionMetadata).toMatchObject({ qualification: "pending", accountMethod: "api_key", accountMode: "responsible_user", coverage: "api-account-native-completion-only", budgetMonthlyCents: 200, maximumAttemptsPerCell: 1 }); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(e => e.suite.id === suite.id)).toBe(false); }); - it("pins the exact selected candidate and model in the operator admission", () => { - const cell = cells.find(e => e.profile.credential === "XAI_API_KEY")!; + it.each([ + ["XAI_API_KEY", "grok-4.7"], + ["GEMINI_API_KEY", "gemini-3.8-flash"], + ])("pins the %s candidate and model in the operator admission", (credential, model) => { + const cell = cells.find(e => e.profile.credential === credential)!; const env = buildRunnerE2EProcessEnvironment({ PAPERCLIP_RUNNER_ACPX_QUALIFICATION: "ambient" }, [cell]); - expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "hermes", model: "grok-4.7" }]); + expect(JSON.parse(env.PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "hermes", model }]); expect(() => buildRunnerE2EProcessEnvironment({}, [{ ...cell, suite: { ...suite, manualOnly: false } }])).toThrow("explicit"); expect(cells.every(e => e.profile.modelQualification.source === "candidate_runner_profile")).toBe(true); expect(buildMatrixJobs(cells).every(job => job.qualificationCandidate === "hermes")).toBe(true); diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index 5a8a63cf7a7..b7b1427df64 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -49,7 +49,9 @@ export const hermesApiConnectionChoices = [ { provider: "anthropic", credential: "ANTHROPIC_API_KEY", model: "claude-haiku-4-5-20251001" }, { provider: "openai", credential: "OPENAI_API_KEY", model: "gpt-5.6-luna" }, { provider: "xai", credential: "XAI_API_KEY", model: "grok-4.7" }, - { provider: "google", credential: "GEMINI_API_KEY", model: "gemini-2.5-flash" }, + // Google restricts 2.5 models to prior users even when they appear in the catalog. + // https://ai.google.dev/gemini-api/docs/deprecations + { provider: "google", credential: "GEMINI_API_KEY", model: "gemini-3.8-flash" }, ] as const satisfies readonly { provider: string; credential: RunnerProfileFixture["credential"]; model: string }[]; export const hermesApiConnectionDefinitionDigest = createHash("sha256").update(