diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 008798b6086..0d5d6d7e02d 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -1484,3 +1484,46 @@ database tests), and Hermes releases assigned skill copies after a credential or state-collection failure only once runtime exit is verified (141 affected runtime tests). The local TypeScript runtime must be rebuilt from the current source before the next paid campaign. No local Docker or Rust build is used. + +The `36125adb4e` 100-cent Stop campaign retains its failed/cleanup-failed grade +after 300.437 seconds. The browser Stop is acknowledged, one native input and +its turn cancel in order, the question becomes expired, and the task stays in +progress. All nine observed PID/start identities retire. Billing remains +unpriced despite complete reported token accounting (31,665 input and 218 +output tokens), so the independent settlement gate correctly fails. The private +recovery state and unknown-cost hold remain retained. This is progress on native +cancellation, not a live Stop qualification pass. + +Qualification review identified two independent evidence gaps. Six negative +runner/session identity cases and a local-settings catalog case first failed. +Answer delivery now requires the created request and resolved receipt to name +the same runner and normalized session. The public budget setting is read and +validated before catalog construction and pinned into the child environment; +credentials and other file settings still load at the normal later boundary. +Definitions advance to native version 7, API/Bedrock version 3 and extended +version 4. Earlier measurements retain their source, definition and grade. + +Source inspection finds that pinned ACPX also drops extension notifications +when its native elicitation controller is aborted, before Paperclip's outer +cancelled-receipt boundary. Hermes usage now enters through ACPX's public +inbound-message observer, with the same exact native session and negotiated +turn-token checks. The ordinary extension callback excludes that one method +to avoid duplicate delivery. Other notifications and requests retain their +existing admission paths. No ACPX version or dependency lock changes. + +The strengthened production transport fixture checks the actual durable PRP +carrier: one owned Hermes usage-provenance notice precedes the cancelled turn, +and both retain their runner, session, run and turn identities. The same fixture +fails with the previous adapter because its receipt is missing, then passes with +the repaired adapter and the verified cloud-built Mac daemon. The comparison +restores the repaired source and bundle byte for byte. An earlier fixture +attempt inspected the normalized lifecycle stream, which does not carry these +informational notices; that failed attempt is retained and is not the regression +proof. This fixture has no provider credentials and proves transport, not paid +OpenRouter charge settlement. + +Final local checks pass 184 affected ACP tests, both Runner TypeScript checks, +and 2,035 Product E2E support tests with one platform skip. Product E2E TypeScript +also passes. A fresh committed-source browser Stop, current-head cloud checks, +and the remaining release matrix are still required. No local Docker or Rust +build was used, and no unknown-cost hold was released. diff --git a/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs b/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs index 28ad1d2fedf..d17448f012f 100644 --- a/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs +++ b/packages/paperclip-runner/scripts/qualify-hermes-runnerd.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { once } from 'node:events'; -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -215,6 +215,24 @@ for (const { interactionKind, nativeQuestionAction } of cases) test(nativeQuesti input: usage.payload.usage.runDelta.inputTokens, output: usage.payload.usage.runDelta.outputTokens, complete: usage.payload.usage.runDeltaComplete, }, { input: 10, output: 5, complete: true }, 'Stopped clarification lost final native prompt usage'); + // Informational notices enter durable PRP evidence rather than the + // normalized NativeSession lifecycle stream. Assert the actual carrier. + const controlPlane = JSON.parse(await readFile(join(root, 'runner/control-plane/control-plane-state.json'), 'utf8')); + const committed = controlPlane.committedEvents.map(event => event.envelope.payload); + const receipts = committed.filter(event => event.eventType === 'provider.notice.recorded' + && event.payload.category === 'hermes_usage_provenance'); + assert.equal(receipts.length, 1, 'Stopped clarification lost its owned Hermes usage extension'); + assert.equal(receipts[0].schema, 'paperclip.prp.event.v1'); + assert.equal(receipts[0].sourceKind, 'runner'); + assert.equal(receipts[0].sourceInstanceId, 'hermes-prp-runner'); + assert.equal(receipts[0].normalizedSessionId, identity.sessionId); + assert.equal(receipts[0].runId, identity.runId); + assert.equal(receipts[0].turnId, 'hermes-prp-turn'); + assert.equal(receipts[0].payload.details.find(detail => detail.name === 'Token usage')?.value, 'reported'); + const committedTerminals = committed.filter(event => event.eventType === 'turn.cancelled'); + assert.equal(committedTerminals.length, 1); + assert.equal(committedTerminals[0].turnId, receipts[0].turnId); + assert.ok(receipts[0].sourceSeq < committedTerminals[0].sourceSeq, 'Usage provenance arrived after cancelled settlement'); await session.close({ reason: 'native clarification stopped' }); assert.equal((await session.snapshot()).semanticResult, null, 'Stop invented task completion'); return; diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts index 5836e1a1504..50656bdfb30 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts @@ -43,31 +43,67 @@ describe("Codex ACPX runtime adapter", () => { const turn = port.startTurn({ text: "Work", requestId: "turn-1", onExtensionRequest: request, onExtensionNotification: notification }); const identity = { version: 1, sessionId: "backend-1", turnToken: "00000000-0000-0000-0000-000000000001" }; created.onExtensionNotification!("_hermes/turn_started", identity); + const wireUsage = (params: Record) => created.onAcpMessage?.("inbound", { jsonrpc: "2.0", method: "_hermes/usage", params }); await turn.cancel(); created.onExtensionNotification!("_hermes/delegation", identity); await expect(created.onExtensionRequest!("_hermes/ask_questions", identity, { requestId: 0, signal: new AbortController().signal })).rejects.toThrow("admitted active turn"); for (const foreign of [{ ...identity, sessionId: "foreign" }, { ...identity, sessionId: "agent-1" }, { ...identity, turnToken: "00000000-0000-0000-0000-000000000002" }]) { - expect(() => created.onExtensionNotification!("_hermes/usage", foreign)).not.toThrow(); + expect(() => wireUsage(foreign)).not.toThrow(); } expect(notification).not.toHaveBeenCalled(); const receipt = { ...identity, tokens: "reported", cost: "unavailable", billing: { schema: "paperclip.usage.billing/v1", source: "provider_reported", biller: "openrouter", currency: "USD", complete: true, requestCount: 1, reportedRequestCount: 1, amountUsd: 0.0042, amountUsdExact: "0.004200000", } }; - created.onExtensionNotification!("_hermes/usage", receipt); + const wire = { jsonrpc: "2.0", method: "_hermes/usage", params: receipt }; + created.onAcpMessage?.("outbound", wire); + for (const invalid of [{ ...wire, id: 0 }, { ...wire, id: undefined }, { ...wire, jsonrpc: "1.0" }, + { ...wire, method: "_hermes/delegation" }, { ...wire, params: null }, { ...wire, params: [] }]) { + created.onAcpMessage?.("inbound", invalid); + } + expect(notification).not.toHaveBeenCalled(); + wireUsage(receipt); expect(notification).toHaveBeenCalledExactlyOnceWith("_hermes/usage", receipt); + // ACPX's ordinary extension callback must not deliver the same wire + // notification twice if its cancellation gate has not closed yet. + created.onExtensionNotification!("_hermes/usage", receipt); + expect(notification).toHaveBeenCalledTimes(1); expect(request).not.toHaveBeenCalled(); await turn.closeStream(); await turn.cancel(); - created.onExtensionNotification!("_hermes/usage", receipt); + wireUsage(receipt); expect(notification).toHaveBeenCalledTimes(1); pending.settle(); await turn.result; - created.onExtensionNotification!("_hermes/usage", receipt); + wireUsage(receipt); expect(notification).toHaveBeenCalledTimes(1); await port.close({ reason: "terminal billing boundary verified" }); }); + it("delivers active Hermes usage once through the same wire path and seals it at prompt settlement", async () => { + const pending = pendingExtensionTurn("turn-1"); + const runtime = fakeRuntime(); vi.mocked(runtime.startTurn).mockReturnValue(pending.turn); + let created!: AcpRuntimeOptions; + const options = openOptions(fakeCommand()); options.profile = { ...options.profile, agent: "hermes" }; + const port = await openCodexAcpxRuntime(options, { + createRegistry: () => registry(), createStore: () => store(), createRuntime: value => { created = value; return runtime; }, + }); + const notification = vi.fn(); + const identity = { version: 1, sessionId: "backend-1", turnToken: "00000000-0000-0000-0000-000000000001" }; + const wire = { jsonrpc: "2.0", method: "_hermes/usage", params: { ...identity, tokens: "reported", cost: "unavailable" } }; + created.onAcpMessage?.("inbound", wire); + expect(notification).not.toHaveBeenCalled(); + const turn = port.startTurn({ text: "Work", requestId: "turn-1", onExtensionNotification: notification }); + created.onExtensionNotification!("_hermes/turn_started", identity); + created.onAcpMessage?.("inbound", wire); + created.onExtensionNotification!("_hermes/usage", wire.params); + expect(notification).toHaveBeenCalledExactlyOnceWith("_hermes/usage", wire.params); + pending.settle(); await turn.result; + created.onAcpMessage?.("inbound", wire); + expect(notification).toHaveBeenCalledTimes(1); + await port.close({ reason: "active receipt settled" }); + }); + it("rejects forged permission session identifiers before delegating or applying full-auto policy", async () => { const pending = pendingExtensionTurn("turn-1"); const runtime = fakeRuntime(); vi.mocked(runtime.startTurn).mockReturnValue(pending.turn); diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts index c1b22411645..9c75b7f5313 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts @@ -310,6 +310,24 @@ export async function openQualifiedAcpxRuntime( const cursorInstructions = options.profile.agent === "cursor" ? createCursorInstructionAdmission(options.systemInstructions) : null; + const acceptExtensionNotification: NonNullable = (method, params) => { + const active = extensionBoundary.active; + if (options.profile.agent === "hermes" && method === "_hermes/turn_started") { + if (!active || !ownsExtensionTurn(active, params) || params.version !== 1 || typeof params.turnToken !== "string" || !/^[0-9a-f-]{36}$/.test(params.turnToken) || active.nativeTurnToken) throw new Error("Invalid Hermes turn binding"); + active.nativeTurnToken = params.turnToken; + return; + } + // Stop revokes requests and activity immediately. Only the admitted native + // prompt's exact final usage may cross its cancelled boundary until result + // settlement. Stream/process closure retires this receipt window. + const cancelledUsage = options.profile.agent === "hermes" && method === "_hermes/usage" + && active !== null && extensionBoundary.cancelledReceiptOwner === active && extensionBoundary.active === active + && params.sessionId === active.sessionId && params.version === 1 + && !!active.nativeTurnToken && params.turnToken === active.nativeTurnToken; + if (!extensionNotifications.has(method) || !active?.onNotification || (!ownsExtensionTurn(active, params) && !cancelledUsage)) return; + if (options.profile.agent === "hermes" && (params.version !== 1 || !active.nativeTurnToken || params.turnToken !== active.nativeTurnToken)) throw new Error("Hermes activity belongs to a stale turn"); + active.onNotification(method, params.sessionId === undefined ? { ...params, sessionId: active.sessionId } : params); + }; const runtimeOptions: GoalAwareAcpRuntimeOptions = { cwd: options.cwd, ...(cursorInstructions || modeBinding ? { protocolGuardFactory: () => { @@ -351,24 +369,20 @@ export async function openQualifiedAcpxRuntime( return response; }, onExtensionNotification: (method, params) => { - const active = extensionBoundary.active; - if (options.profile.agent === "hermes" && method === "_hermes/turn_started") { - if (!active || !ownsExtensionTurn(active, params) || params.version !== 1 || typeof params.turnToken !== "string" || !/^[0-9a-f-]{36}$/.test(params.turnToken) || active.nativeTurnToken) throw new Error("Invalid Hermes turn binding"); - active.nativeTurnToken = params.turnToken; - return; - } - // Stop revokes requests and activity immediately, but the native prompt - // still settles its owned work and billing. Admit only its final usage - // notification, with the exact negotiated session/token, until result - // settlement. Stream/process closure never opens this receipt window. - const cancelledUsage = options.profile.agent === "hermes" && method === "_hermes/usage" - && active !== null && extensionBoundary.cancelledReceiptOwner === active && extensionBoundary.active === active - && params.sessionId === active.sessionId && params.version === 1 - && !!active.nativeTurnToken && params.turnToken === active.nativeTurnToken; - if (!extensionNotifications.has(method) || !active?.onNotification || (!ownsExtensionTurn(active, params) && !cancelledUsage)) return; - if (options.profile.agent === "hermes" && (params.version !== 1 || !active.nativeTurnToken || params.turnToken !== active.nativeTurnToken)) throw new Error("Hermes activity belongs to a stale turn"); - active.onNotification(method, params.sessionId === undefined ? { ...params, sessionId: active.sessionId } : params); + // Hermes usage has one wire-observer path, including before cancellation. + // Do not deliver it twice through ACPX's ordinary extension callback. + if (options.profile.agent === "hermes" && method === "_hermes/usage") return; + acceptExtensionNotification(method, params); }, + ...(options.profile.agent === "hermes" ? { onAcpMessage: (direction: "inbound" | "outbound", message: unknown) => { + // Pinned ACPX 0.13.1 aborts its elicitation controller before Hermes emits + // final usage, and drops extensions at that inner gate. Its public wire + // observer still sees the notification. Reapply our exact turn/session + // admission here; this grants no request or activity authority after Stop. + const wire = objectRecord(message); + if (direction !== "inbound" || wire.jsonrpc !== "2.0" || "id" in wire || wire.method !== "_hermes/usage") return; + acceptExtensionNotification("_hermes/usage", objectRecord(wire.params)); + } } : {}), nonInteractivePermissions: "fail", permissionPolicy: { ...options.permissionPolicy, diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index c5184a0794b..47c693ecdef 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1488,12 +1488,23 @@ oracle. Older behavioral passes do not prove this accounting requirement. Before launching, `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS` can lower both budgets to a whole number from 1 through 200 cents. Missing configuration keeps the 200-cent default; malformed, zero, and higher limits fail before credential -handoff. The campaign captures one immutable value for fixture creation, public -budget readback, settlement health, and catalog definition identity. Every +handoff. The public setting can also come from `.env.runner-e2e.local`. It is +validated before catalog construction and pinned into the child environment; +explicit environment configuration takes precedence. This early step loads no +credentials or other file settings. The campaign captures one immutable value +for fixture creation, public budget readback, settlement health, and catalog +definition identity. Every observed budget must match it exactly. A lower cap does not settle unknown -charges or release a previous attempt's reservation. Definition versions 3 -(extended harnesses), 2 (Hermes API/Bedrock), and 6 (native interactions) record -this bounded option. +charges or release a previous attempt's reservation. Definition versions 4 +(extended harnesses), 3 (Hermes API/Bedrock), and 7 (native interactions) record +this bounded option and consistent settings capture. Native answer receipts +must also retain the original runner and normalized session identities. + +The credential-free native Stop fixture checks usage provenance on its durable +PRP carrier before the cancelled terminal, including runner, session, run and +turn identity. Informational notices are not part of the normalized lifecycle +stream. This transport proof does not replace a paid browser campaign's charge +settlement or process-cleanup checks. Before loading local credentials, the launcher records the checked-out controller SHA and ref for every selected Hermes candidate, including API and Bedrock cells. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index dc7854e92f6..be2ccb8c02e 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -36,6 +36,9 @@ to a whole number from 1 through 200. Invalid or unlimited values fail before credential handoff. Public company/agent reads and post-run settlement must match the captured limit. Provider-key limits, campaign reservations, and unknown-cost holds remain separate spending controls. +The public budget setting is resolved from the environment or local settings +file before catalog construction, then pinned for children. Other file settings +and credentials still load only at the normal later admission boundary. ## Short-lived Bedrock qualification @@ -345,7 +348,7 @@ grading receipts alongside the existing sanitized run evidence. No private runner hook supplies the answer. The late duplicate probe uses the existing untraced public POST helper and requires its actual HTTP 409 status; response bodies are withheld. Source admission precedes credential handoff, both public -budgets are 200 cents, OpenRouter cost must settle, and automatic retries remain +budgets match the captured bounded limit, OpenRouter cost must settle, and automatic retries remain disabled. Registered local/Daytona cells are pending qualification, not permission to launch a paid campaign or a claim of remote coverage. @@ -356,5 +359,5 @@ those PIDs. Cancellation goes through the normal browser Stop control and public board API. No question answer is delivered; the only response attempt is a late fixture answer that must be rejected. Retained evidence uses the existing sanitizer and screenshot publication gates. The cell reuses the selected key -allocation and 200-cent budgets, with zero automatic retries. It neither creates +allocation and captured bounded budgets, with zero automatic retries. It neither creates remote resources nor grants permission to use another secret. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 8c021b08538..7f41f36c120 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1210,7 +1210,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], expectedMatrixSize: 40, - definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", + definitionMetadata: { version: 4, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", hermesBilling: "per-run-reported-cost-and-budget-health", hermesBudgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, hermesSettlementSourceDigest: hermesApiConnectionDefinitionDigest }, }, { @@ -1220,7 +1220,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: runnerEnvironments, tasks: [hermesNativeQuestionTask, hermesNativeQuestionStopTask], excludedExecutionIds: [`${HERMES_NATIVE_INTERACTION_SUITE}.runner-acpx-hermes.daytona.native-question-batch-stop`], expectedMatrixSize: 3, definitionMetadata: { - version: 6, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", + version: 7, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1, objectiveAdmission: "production-delivery-guard-question-only", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health", @@ -1236,7 +1236,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }], expectedMatrixSize: 10, definitionMetadata: { - version: 2, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + version: 3, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest, @@ -1249,7 +1249,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }], expectedMatrixSize: 2, definitionMetadata: { - version: 2, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + version: 3, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", accountMethod: "api_key", accountMode: "delegated", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", routingSource: "public-selected-account-before-and-after-task", credentialScope: "ephemeral-region-bound-bearer", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index da1f4adcd71..a767f5f0d0f 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -5,6 +5,14 @@ import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js"; import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText, hasHermesNativeQuestionStop, resolveHermesQualificationBudgetCents } from "./hermes-api-connections.js"; +const settings = vi.hoisted(() => ({ contents: undefined as string | undefined })); +vi.mock("node:fs", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, readFileSync: ((file: Parameters[0], options: unknown) => + settings.contents !== undefined && String(file).endsWith("/.env.runner-e2e.local") + ? settings.contents : actual.readFileSync(file, options as never)) }; +}); + describe("Hermes native browser questions", () => { const suite = runnerSuites.find(s => s.id === HERMES_NATIVE_INTERACTION_SUITE)!; const cells = runnerMatrix.filter(e => e.suite.id === suite.id); @@ -134,7 +142,8 @@ describe("Hermes native browser questions", () => { }); it.each(["missing", "no-created", "no-outcome", "duplicate-created", "duplicate-outcome", "second-request-other-id", "cancelled", "expired", "wrong-action", "wrong-answer", "wrong-answer-order", "wrong-input", "wrong-provider", "semantic-tool", "wrong-adapter", "wrong-type", "wrong-status", "wrong-request-turn", - "wrong-event-turn", "wrong-outcome-turn", "foreign-row", "foreign-event", "wrong-schema", "wrong-version", "wrong-protocol", "wrong-order", "missing-sequence"])("rejects %s evidence", fault => { + "wrong-event-turn", "wrong-outcome-turn", "foreign-row", "foreign-event", "wrong-schema", "wrong-version", "wrong-protocol", "wrong-order", "missing-sequence", + "foreign-source", "foreign-session", "not-runner", "created-not-runner", "missing-source", "missing-session"])("rejects %s evidence", fault => { const rows = structuredClone(native()) as ReturnType; const created = rows[0]!.payload.prpEvent as Record, resolved = rows[1]!.payload.prpEvent as Record; const request = created.payload.request; @@ -169,6 +178,12 @@ describe("Hermes native browser questions", () => { if (fault === "wrong-protocol") rows[1]!.protocolSchemaVersion = 2; if (fault === "wrong-order") resolved.sourceSeq = 1; if (fault === "missing-sequence") delete resolved.sourceSeq; + if (fault === "foreign-source") resolved.sourceInstanceId = "another-runner"; + if (fault === "foreign-session") resolved.normalizedSessionId = "another-session"; + if (fault === "not-runner") resolved.sourceKind = "controller"; + if (fault === "created-not-runner") created.sourceKind = "controller"; + if (fault === "missing-source") delete created.sourceInstanceId; + if (fault === "missing-session") delete created.normalizedSessionId; expect(grade(rows)).toBe(false); }); it.each(["question-count", "mode", "id", "prompt", "required", "choices", "option-id", "option-label", "custom-disabled"])("rejects a changed native form: %s", fault => { @@ -187,6 +202,61 @@ describe("Hermes native browser questions", () => { }); describe("Hermes managed API connection qualification", () => { + it.each(["0", "201", "100.0", ""])("rejects invalid local settings budget %s before catalog construction", async raw => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", undefined); + settings.contents = `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=${raw}\n`; + vi.resetModules(); + try { + await expect(import("./catalog.js")).rejects.toThrow("integer from 1 to 200 cents"); + expect(process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBeUndefined(); + } finally { + settings.contents = undefined; + vi.unstubAllEnvs(); + vi.resetModules(); + } + }); + it("preserves an explicit environment budget ahead of local settings", async () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", "100"); + settings.contents = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=0\n"; + vi.resetModules(); + try { + const bounded = await import("./hermes-api-connections.js"); + expect(bounded.HERMES_API_CONNECTION_BUDGET_CENTS).toBe(100); + expect(process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBe("100"); + } finally { + settings.contents = undefined; + vi.unstubAllEnvs(); + vi.resetModules(); + } + }); + it("captures the local settings budget before catalog construction and pins it for child processes", async () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", undefined); + const existingCredential = process.env.OPENROUTER_API_KEY; + settings.contents = "# Fixture-only public configuration\nexport PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS='100'\nOPENROUTER_API_KEY=never-load-this-fixture-value\n"; + vi.resetModules(); + try { + const catalog = await import("./catalog.js"); + const bounded = await import("./hermes-api-connections.js"); + const cell = catalog.runnerMatrix.find(cell => cell.id === "hermes-native-interactions.runner-acpx-hermes.local.native-question-batch-stop")!; + expect(bounded.HERMES_API_CONNECTION_BUDGET_CENTS).toBe(100); + expect(cell.suite.definitionMetadata).toMatchObject({ budgetMonthlyCents: 100 }); + expect(process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBe("100"); + const child = buildRunnerE2EProcessEnvironment(process.env, [cell]); + expect(child.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBe("100"); + expect(process.env.OPENROUTER_API_KEY).toBe(existingCredential); + // The late full settings loader preserves an already captured value. + settings.contents = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=200\n"; + vi.resetModules(); + const reloaded = await import("./catalog.js"); + const next = reloaded.runnerMatrix.find(next => next.id === cell.id)!; + expect(next.suiteDefinitionHash).toBe(cell.suiteDefinitionHash); + expect(next.suite.definitionMetadata).toMatchObject({ budgetMonthlyCents: 100 }); + } finally { + settings.contents = undefined; + vi.unstubAllEnvs(); + vi.resetModules(); + } + }); it.each([[undefined, 200], ["1", 1], ["100", 100], ["200", 200]] as const)("admits the bounded campaign limit %s", (raw, expected) => { expect(resolveHermesQualificationBudgetCents(raw)).toBe(expected); }); diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index 5fdae712736..91e80c36606 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -22,11 +22,31 @@ export function resolveHermesQualificationBudgetCents(raw?: string): number { } return Number(raw); } -// Capture one immutable campaign limit before any credential handoff. Fixture -// creation, public readback, settlement and definition identity use this value. -export const HERMES_API_CONNECTION_BUDGET_CENTS = resolveHermesQualificationBudgetCents( - process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS, -); +function captureHermesQualificationBudgetCents(): number { + const key = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS"; + let raw = process.env[key]; + if (raw === undefined) { + let contents: string | undefined; + try { contents = readFileSync(new URL("../../.env.runner-e2e.local", import.meta.url), "utf8"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + // Read only this public setting. Credentials still load after source and + // runtime admission in launch.ts. Use the full loader's first-value and + // quoting rules, without importing other settings into the environment. + for (const line of contents?.split(/\r?\n/) ?? []) { + const match = /^(?:export\s+)?PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=(.*)$/.exec(line.trim()); + if (!match) continue; + raw = match[1]!.trim(); + if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) raw = raw.slice(1, -1); + break; + } + } + const budget = resolveHermesQualificationBudgetCents(raw); + // The late settings loader preserves this captured value. Every child sees + // the same validated limit that already identifies the launcher's catalog. + process.env[key] = String(budget); + return budget; +} +export const HERMES_API_CONNECTION_BUDGET_CENTS = captureHermesQualificationBudgetCents(); export const HERMES_NATIVE_INTERACTION_SUITE = "hermes-native-interactions"; export const isHermesConnectionSuite = (suiteId: string) => suiteId === "hermes-api-connections" || suiteId === "hermes-bedrock-connections" || suiteId === HERMES_NATIVE_INTERACTION_SUITE; @@ -122,7 +142,11 @@ export function hasExactHermesNativeQuestionResponse(input: { const outcomes = relevant.filter(({ event }) => event.eventType !== "runtime_request.created"); if (created.length !== 1 || outcomes.length !== 1) return false; const request = record(record(created[0]!.event.payload).request), outcome = outcomes[0]!.event; - return hasAcpxNativeOrigin(request.origin, "hermes", "_hermes/ask_questions") + const origin = created[0]!.event; + return origin.sourceKind === "runner" && [origin.sourceInstanceId, origin.normalizedSessionId].every(present) + && outcome.sourceKind === "runner" && outcome.sourceInstanceId === origin.sourceInstanceId + && outcome.normalizedSessionId === origin.normalizedSessionId + && hasAcpxNativeOrigin(request.origin, "hermes", "_hermes/ask_questions") && request.type === "input" && request.status === "pending" && request.turnId === input.turnId && isDeepStrictEqual(request.input, input.questionSet) && outcome.eventType === "runtime_request.resolved" && record(outcome.payload).action === "submit"