diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 6e8d764c34a..008798b6086 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -1453,3 +1453,34 @@ scope error is retained as a failed test attempt; it is not qualification proof. No local Docker, Rust build, secret access or paid provider call was used for this repair. A fresh committed-source browser Stop attempt and complete release gates are still required. The PR stack remains draft and Hermes remains pending. + +### 2026-10-08 bounded qualification budget and cleanup verification + +Hermes qualification campaigns now accept a pinned company and agent budget +from 1 to 200 cents through `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS`. The default +remains 200 cents. Fixture creation, public checks before execution, settlement +checks and definition metadata all use the same captured value; malformed or +larger limits fail before credential handoff. Definition versions advance to +6 for native interactions, 2 for API and Bedrock connections, and 3 for extended +harnesses. Behavioral matchers, account scope and single-attempt policy remain +unchanged. The exact local Stop cell is discoverable with a 100-cent limit. + +The earlier `782774e1` launch stopped at its allowance preflight before any model +call: the selected existing key had less than the required $2 available. It did +not start a campaign, modify the reservation ledger or release an unknown-cost +hold. A future 100-cent campaign is a separate measurement with its own pinned +source and definition identity; existing failed attempts retain their grades. + +Credential-free verification passes 2,023 Product E2E support tests with one +platform skip, 128 native source checks and the Product E2E TypeScript check. +The first full support run retained one process-shutdown timing failure; the +same suite passes with four concurrent workers. Both cloud native targets pass +at `ec40b1eb`, and the existing PR checks are green at their recorded heads. +These results do not qualify a live provider, a subscription or Daytona. + +Two earlier review repairs are also verified: native cancellation accepts the +Rust envelope's omitted action while rejecting conflicting responses (121 +database tests), and Hermes releases assigned skill copies after a credential +or state-collection failure only once runtime exit is verified (141 affected +runtime tests). The local TypeScript runtime must be rebuilt from the current +source before the next paid campaign. No local Docker or Rust build is used. diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 497c00d2765..8d8517ef8ea 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -301,7 +301,10 @@ responsible user. Calibration rejects foreign accounts/companies/tasks/users, wrong providers/methods/models/harnesses, missing evidence and extra runs. The suite's definition includes the account/runner harness source digest. Its API-only scope does not substitute for subscription or lifecycle qualification. -The task permits one attempt. Both public 200-cent budgets must match the fixture +The task permits one attempt. Both public budgets must match the fixture's +immutable campaign limit: 200 cents by default, or a validated whole-number +`PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS` from 1 through 200. Catalog metadata +records the same selected limit. Both public budgets must match the fixture scope before task creation; unlimited or foreign budget records fail admission. OpenRouter also requires a settled provider-reported price with an exact USD amount and healthy company/agent budget state before fixture teardown. The diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 3f5ada0e47b..c5184a0794b 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1478,13 +1478,22 @@ need their separate conformance/qualification evidence. Hermes uses an existing managed OpenRouter connection through the ordinary connection fixture. It remains pending qualification; the native transport fixture does not substitute for these browser and remote workflows. -The ten Hermes/OpenRouter cells set 200-cent company and agent budgets and +The ten Hermes/OpenRouter cells default to 200-cent company and agent budgets and verify them through public reads before creating work. Before cleanup, every run, including a run that yields for human input, must have settled reported wire cost and healthy budget state. The per-run receipts are retained in `hermes-openrouter-workflow-settlement.json`; `api-state.json` also retains all selected runs. Extended-harnesses definition version 2 records this stronger oracle. Older behavioral passes do not prove this accounting requirement. +Before launching, `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS` can lower both +budgets to a whole number from 1 through 200 cents. Missing configuration keeps +the 200-cent default; malformed, zero, and higher limits fail before credential +handoff. The campaign captures one immutable value for fixture creation, public +budget readback, settlement health, and catalog definition identity. Every +observed budget must match it exactly. A lower cap does not settle unknown +charges or release a previous attempt's reservation. Definition versions 3 +(extended harnesses), 2 (Hermes API/Bedrock), and 6 (native interactions) record +this bounded option. Before loading local credentials, the launcher records the checked-out controller SHA and ref for every selected Hermes candidate, including API and Bedrock cells. @@ -1546,7 +1555,7 @@ the native run's company/task/agent scope, responsible user, selected account, provider/method, native Hermes harness and requested/effective model. Missing or mismatched metadata fails even when the answer is correct. The report retains the checks alongside the ordinary completion, screenshot, token/cost and cleanup evidence. -Each cell admits one attempt and configures 200-cent company and agent budgets. +Each cell admits one attempt and configures the selected bounded company and agent budgets (200 cents by default). Public budget readback must pass before task creation. Unpriced model usage remains unknown; this configured limit is not an exact billing receipt. The OpenRouter cell additionally waits for its public run's reported wire cost @@ -1579,7 +1588,7 @@ with Bedrock routing and selects its returned grant explicitly; routed accounts do not become responsible-user provider defaults. Public readback before and after the task must match the region, protocol, authentication, model catalog and account owner. The native run must match the selected connection, grant, -owner, Hermes harness and exact inference profile. Both 200-cent budget checks, +owner, Hermes harness and exact inference profile. Both selected bounded budget checks, the single-attempt rule and normal evidence/cleanup contracts also apply. This scope proves Bedrock completion only; it does not qualify credential refresh or make a local AWS run equivalent to Daytona. The paid GitHub workflow @@ -1602,7 +1611,7 @@ original succeeded run without a continuation. The final answer must contain the reviewer's undisclosed free text returned by the native callback; a semantic `request_human_input` call or a guessed completion cannot pass. -This suite uses the same managed OpenRouter account/model, public 200-cent +This suite uses the same managed OpenRouter account/model, public bounded budgets, settled reported billing, pre-credential source admission and cleanup pipeline. It allows one attempt and records its harness source digest. Its browser-reconnect scope does not qualify controller restart, cancellation, @@ -1860,7 +1869,7 @@ rejects a late answer with HTTP 409. One cancelled run must leave the task In Progress. A read-only PID/start-time journal captures the public per-turn owner and descendants before Stop and verifies retirement before and through cleanup. Its selected account/model and reported wire usage must settle with healthy -200-cent company and agent budgets. One attempt, zero retries, one provider turn. +company and agent budgets (200 cents by default). One attempt, zero retries, one provider turn. Remote Stop is deliberately excluded until its remote retirement observer exists; the release requirement remains pending. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 4a54605ade7..dc7854e92f6 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -30,6 +30,13 @@ public structured evidence. Treat changes to the workflow, harness, fixture prompts, evidence packager, and publisher as security-sensitive production changes. +Hermes qualification defaults to 200-cent company and agent hard stops. The +launcher can set `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS` to lower that limit +to a whole number from 1 through 200. Invalid or unlimited values fail before +credential handoff. Public company/agent reads and post-run settlement must +match the captured limit. Provider-key limits, campaign reservations, and +unknown-cost holds remain separate spending controls. + ## Short-lived Bedrock qualification The explicit-only `hermes-bedrock-connections` suite accepts a short-lived, diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index bf568c8ec62..8c021b08538 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1210,8 +1210,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], expectedMatrixSize: 40, - definitionMetadata: { version: 2, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", - hermesBilling: "per-run-reported-cost-and-budget-health", hermesBudgetMonthlyCents: 200, hermesSettlementSourceDigest: hermesApiConnectionDefinitionDigest }, + definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", + hermesBilling: "per-run-reported-cost-and-budget-health", hermesBudgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, hermesSettlementSourceDigest: hermesApiConnectionDefinitionDigest }, }, { id: HERMES_NATIVE_INTERACTION_SUITE, label: "Hermes native interactions", manualOnly: true, @@ -1220,7 +1220,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: runnerEnvironments, tasks: [hermesNativeQuestionTask, hermesNativeQuestionStopTask], excludedExecutionIds: [`${HERMES_NATIVE_INTERACTION_SUITE}.runner-acpx-hermes.daytona.native-question-batch-stop`], expectedMatrixSize: 3, definitionMetadata: { - version: 5, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", + version: 6, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1, objectiveAdmission: "production-delivery-guard-question-only", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health", @@ -1236,7 +1236,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }], expectedMatrixSize: 10, definitionMetadata: { - version: 1, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + version: 2, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest, @@ -1249,7 +1249,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }], expectedMatrixSize: 2, definitionMetadata: { - version: 1, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + version: 2, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", accountMethod: "api_key", accountMode: "delegated", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", routingSource: "public-selected-account-before-and-after-task", credentialScope: "ephemeral-region-bound-bearer", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index b09d530fc03..da1f4adcd71 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -1,9 +1,9 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js"; -import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText, hasHermesNativeQuestionStop } from "./hermes-api-connections.js"; +import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText, hasHermesNativeQuestionStop, resolveHermesQualificationBudgetCents } from "./hermes-api-connections.js"; describe("Hermes native browser questions", () => { const suite = runnerSuites.find(s => s.id === HERMES_NATIVE_INTERACTION_SUITE)!; @@ -187,6 +187,50 @@ describe("Hermes native browser questions", () => { }); describe("Hermes managed API connection qualification", () => { + it.each([[undefined, 200], ["1", 1], ["100", 100], ["200", 200]] as const)("admits the bounded campaign limit %s", (raw, expected) => { + expect(resolveHermesQualificationBudgetCents(raw)).toBe(expected); + }); + it.each(["", "0", "201", "999", "1000", "-100", "+100", "100.0", "1e2", "0100", " 100", "100 ", "NaN", "Infinity"])("rejects an unlimited or malformed campaign limit %s", raw => { + expect(() => resolveHermesQualificationBudgetCents(raw)).toThrow("integer from 1 to 200 cents"); + }); + it("pins a lowered campaign budget in catalog metadata and both public budget checks", async () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", "100"); + vi.resetModules(); + try { + const bounded = await import("./hermes-api-connections.js"); + const catalog = await import("./catalog.js"); + expect(bounded.HERMES_API_CONNECTION_BUDGET_CENTS).toBe(100); + const suite = catalog.runnerSuites.find(suite => suite.id === HERMES_NATIVE_INTERACTION_SUITE)!; + expect(suite.definitionMetadata).toMatchObject({ budgetMonthlyCents: 100, maximumAttemptsPerCell: 1 }); + expect(catalog.suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash(runnerSuites.find(suite => suite.id === HERMES_NATIVE_INTERACTION_SUITE)!)); + for (const observed of [100, 200, 0]) { + const receipt = await bounded.captureHermesApiBudgets({ companyId: "company", agentId: "agent", api: { + async get(path: string) { return (path === "/api/companies/company" + ? { id: "company", budgetMonthlyCents: observed } + : { id: "agent", companyId: "company", budgetMonthlyCents: observed }) as T; }, + } }); + expect(receipt.budgetMonthlyCents).toBe(100); + expect(receipt.checks.every(check => check.passed)).toBe(observed === 100); + const company = { id: "company", status: "active", budgetMonthlyCents: observed }; + const agent = { id: "agent", companyId: "company", status: "idle", pauseReason: null, budgetMonthlyCents: observed }; + const run = { id: "run", companyId: "company", agentId: "agent", issueId: "task", status: "cancelled", + costAccountingPending: false, costAccountedAt: "2026-10-08T03:00:00Z", usageJson: { + biller: "openrouter", billingType: "metered_api", costStatus: "reported", costUsd: 0.0042, costUsdExact: "0.004200000", + inputTokens: 40, outputTokens: 10, accountingReceiptReady: true, + pricingProvenance: { source: "provider_reported", version: "hermes-openrouter-wire/v1" }, + } }; + const settled = await bounded.captureHermesOpenRouterSettlement({ companyId: "company", agentId: "agent", issueId: "task", + runId: "run", expectedRunStatus: "cancelled", api: { async get(path: string) { + return (path === "/api/companies/company" ? company : path === "/api/agents/agent" ? agent : run) as T; + } } }); + expect(settled.checks.every(check => check.passed)).toBe(observed === 100); + } + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + } + }); + const suite = runnerSuites.find(s => s.id === "hermes-api-connections")!; const cells = runnerMatrix.filter(e => e.suite.id === suite.id); it("declares ten bounded pending cells without adding scheduled paid work", () => { diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index 9310d3925f9..5fdae712736 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -14,7 +14,19 @@ import type { MatrixExecution, RunnerProfileFixture, RunnerTaskFixture } from ". const record = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; const present = (value: unknown) => typeof value === "string" && value.trim().length > 0; -export const HERMES_API_CONNECTION_BUDGET_CENTS = 200; +export const HERMES_API_CONNECTION_MAX_BUDGET_CENTS = 200; +export function resolveHermesQualificationBudgetCents(raw?: string): number { + if (raw === undefined) return HERMES_API_CONNECTION_MAX_BUDGET_CENTS; + if (!/^[1-9][0-9]{0,2}$/.test(raw) || Number(raw) > HERMES_API_CONNECTION_MAX_BUDGET_CENTS) { + throw new Error("Hermes qualification budget must be an integer from 1 to 200 cents"); + } + return Number(raw); +} +// Capture one immutable campaign limit before any credential handoff. Fixture +// creation, public readback, settlement and definition identity use this value. +export const HERMES_API_CONNECTION_BUDGET_CENTS = resolveHermesQualificationBudgetCents( + process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS, +); export const HERMES_NATIVE_INTERACTION_SUITE = "hermes-native-interactions"; export const isHermesConnectionSuite = (suiteId: string) => suiteId === "hermes-api-connections" || suiteId === "hermes-bedrock-connections" || suiteId === HERMES_NATIVE_INTERACTION_SUITE;