diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 0d5d6d7e02d..bfe2901450a 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -1527,3 +1527,35 @@ and 2,035 Product E2E support tests with one platform skip. Product E2E TypeScri also passes. A fresh committed-source browser Stop, current-head cloud checks, and the remaining release matrix are still required. No local Docker or Rust build was used, and no unknown-cost hold was released. + +### 2026-10-08 live billing and persisted cancellation receipt + +The current-source `a14a752ec8` local Stop attempt retains its original +failed/cleanup-failed grade after 301.128 seconds. The wire repair now delivers +complete provider-reported billing: $0.000973314, 31,673 input tokens and 315 +output tokens. Public read-only observations confirm the cancelled run, exact +audited Stop acknowledgement, expired question and unfinished task. All seven +observed owned process identities retire. Evidence has no leaks or missing +entries, and the failure screenshot was inspected. Earlier unknown-cost holds +remain reserved; this measurement is recorded with its known charge. + +The remaining wait is an oracle error: the browser fixture requires the expired +card's result to be absent, but native cancellation correctly persists a v1 +cancellation receipt with zero answers. Offline analysis of the captured public +state passes the exact native cancellation predicate and rejects only that old +card predicate. The failed campaign is not regraded or replayed as a live pass. + +The corrected card oracle requires the original payload, company, task and run +binding; the same card must expire under the owning run, retain no human or +agent answer actor, and contain the exact cancellation receipt with no answers. +The regression first fails the real receipt and incorrectly accepts a missing +receipt. The corrected source passes all 2,057 Product E2E support tests with one +platform skip, 128 native source checks and Product E2E TypeScript. The restricted +full-suite attempt retains its IPC/process-permission failures; the host-access +run passes. Post-hoc comparison passes the corrected oracle on the captured state +without changing the original grade or making any provider call. + +Affected definitions advance to native version 8, API/Bedrock version 4 and +extended version 5. Production runtime, dependency locks and execution controls +do not change for this oracle repair. Fresh committed-source browser acceptance +and current-head cloud checks remain required. Hermes is still pending. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 47c693ecdef..e055ed660a3 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1495,9 +1495,10 @@ credentials or other file settings. The campaign captures one immutable value for fixture creation, public budget readback, settlement health, and catalog definition identity. Every observed budget must match it exactly. A lower cap does not settle unknown -charges or release a previous attempt's reservation. Definition versions 4 -(extended harnesses), 3 (Hermes API/Bedrock), and 7 (native interactions) record -this bounded option and consistent settings capture. Native answer receipts +charges or release a previous attempt's reservation. Definition versions 5 +(extended harnesses), 4 (Hermes API/Bedrock), and 8 (native interactions) record +this bounded option, consistent settings capture and the Stop card receipt. +Native answer receipts must also retain the original runner and normalized session identities. The credential-free native Stop fixture checks usage provenance on its durable @@ -1506,6 +1507,12 @@ turn identity. Informational notices are not part of the normalized lifecycle stream. This transport proof does not replace a paid browser campaign's charge settlement or process-cleanup checks. +Native Stop leaves a versioned cancellation result on the expired question, +with zero submitted answers. The browser fixture requires that exact receipt, +the original card payload and company/task/run identity, and native cancellation +evidence. A missing result or a result containing answers cannot pass. Historical +failed campaigns keep their original machine grades after this oracle repair. + Before loading local credentials, the launcher records the checked-out controller SHA and ref for every selected Hermes candidate, including API and Bedrock cells. It rejects tracked/untracked source changes or an explicit diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index be2ccb8c02e..f0472967d73 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -361,3 +361,8 @@ fixture answer that must be rejected. Retained evidence uses the existing sanitizer and screenshot publication gates. The cell reuses the selected key allocation and captured bounded budgets, with zero automatic retries. It neither creates remote resources nor grants permission to use another secret. + +The expired native question must retain its versioned cancellation receipt with +zero answers, bound to the original payload, card, company, task and source run. +A missing result, answer actor, changed payload or submitted answer fails the +oracle. Correcting this assertion does not regrade historical failed campaigns. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 7f41f36c120..33cf0b70909 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1210,7 +1210,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments, tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask], expectedMatrixSize: 40, - definitionMetadata: { version: 4, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", + definitionMetadata: { version: 5, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28", hermesBilling: "per-run-reported-cost-and-budget-health", hermesBudgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, hermesSettlementSourceDigest: hermesApiConnectionDefinitionDigest }, }, { @@ -1220,7 +1220,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: runnerEnvironments, tasks: [hermesNativeQuestionTask, hermesNativeQuestionStopTask], excludedExecutionIds: [`${HERMES_NATIVE_INTERACTION_SUITE}.runner-acpx-hermes.daytona.native-question-batch-stop`], expectedMatrixSize: 3, definitionMetadata: { - version: 7, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", + version: 8, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1, objectiveAdmission: "production-delivery-guard-question-only", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health", @@ -1236,7 +1236,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }], expectedMatrixSize: 10, definitionMetadata: { - version: 3, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + version: 4, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest, @@ -1249,7 +1249,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }], expectedMatrixSize: 2, definitionMetadata: { - version: 3, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", + version: 4, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", accountMethod: "api_key", accountMode: "delegated", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", routingSource: "public-selected-account-before-and-after-task", credentialScope: "ephemeral-region-bound-bearer", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index a767f5f0d0f..f8fe9112d43 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -3,7 +3,7 @@ import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js"; -import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText, hasHermesNativeQuestionStop, resolveHermesQualificationBudgetCents } from "./hermes-api-connections.js"; +import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText, hasHermesNativeQuestionStop, hasHermesNativeQuestionStopCard, resolveHermesQualificationBudgetCents } from "./hermes-api-connections.js"; const settings = vi.hoisted(() => ({ contents: undefined as string | undefined })); vi.mock("node:fs", async importOriginal => { @@ -85,6 +85,43 @@ describe("Hermes native browser questions", () => { it("requires the native callback cancellation and exact browser-owned Stop acknowledgement", () => { expect(hasHermesNativeQuestionStop(stopped())).toBe(true); }); + const stoppedCard = () => { + const originalCard = { id: "card", companyId: "company", issueId: "task", sourceRunId: "run", kind: "ask_user_questions", + continuationPolicy: "none", status: "pending", result: null, payload: { version: 1, runtimeRequestId: "request", questionSet } }; + return { originalCard, runId: "run", card: { ...structuredClone(originalCard), status: "expired", resolvedByRunId: "run", + resolvedByUserId: null, resolvedByAgentId: null, + result: { version: 1, cancelled: true, cancellationReason: "Native question cancelled", answers: [], summaryMarkdown: null } } }; + }; + it("accepts the persisted native cancellation receipt with no submitted answers", () => { + expect(hasHermesNativeQuestionStopCard(stoppedCard())).toBe(true); + }); + it.each(["missing-receipt", "answered", "not-cancelled", "wrong-version", "wrong-reason", "summary", "foreign-card", "foreign-company", + "foreign-task", "foreign-source-run", "foreign-resolver-run", "resolved-by-user", "resolved-by-agent", "changed-payload", "wrong-status", + "wrong-kind", "continuation", "original-answered", "original-expired", "missing-request", "wrong-run"])("rejects stopped card fault: %s", fault => { + const value = stoppedCard() as Record; + if (fault === "missing-receipt") value.card.result = null; + if (fault === "answered") value.card.result.answers = [{ questionId: "q0", optionIds: ["o0"] }]; + if (fault === "not-cancelled") value.card.result.cancelled = false; + if (fault === "wrong-version") value.card.result.version = 2; + if (fault === "wrong-reason") value.card.result.cancellationReason = "Another cancellation"; + if (fault === "summary") value.card.result.summaryMarkdown = "An answer"; + if (fault === "foreign-card") value.card.id = "other-card"; + if (fault === "foreign-company") value.card.companyId = "other-company"; + if (fault === "foreign-task") value.card.issueId = "other-task"; + if (fault === "foreign-source-run") value.card.sourceRunId = "other-run"; + if (fault === "foreign-resolver-run") value.card.resolvedByRunId = "other-run"; + if (fault === "resolved-by-user") value.card.resolvedByUserId = "human"; + if (fault === "resolved-by-agent") value.card.resolvedByAgentId = "agent"; + if (fault === "changed-payload") value.card.payload.runtimeRequestId = "other-request"; + if (fault === "wrong-status") value.card.status = "answered"; + if (fault === "wrong-kind") value.card.kind = "request_confirmation"; + if (fault === "continuation") value.card.continuationPolicy = "resume_task"; + if (fault === "original-answered") value.originalCard.result = { answers: [{ questionId: "q0", optionIds: ["o0"] }] }; + if (fault === "original-expired") value.originalCard.status = "expired"; + if (fault === "missing-request") { delete value.originalCard.payload.runtimeRequestId; delete value.card.payload.runtimeRequestId; } + if (fault === "wrong-run") value.runId = "other-run"; + expect(hasHermesNativeQuestionStopCard(value as Parameters[0])).toBe(false); + }); it.each(["missing-created", "missing-closure", "missing-terminal", "duplicate-created", "duplicate-closure", "duplicate-terminal", "second-request", "completed", "failed", "interrupted", "resolved", "expired", "answer", "replay", "wrong-action", "wrong-order", "wrong-schema", "wrong-version", "wrong-protocol", "foreign-row", "foreign-event", "foreign-turn", "foreign-session", "foreign-source", "not-runner", "changed-pending", "changed-input", "wrong-provider", "semantic-tool", diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index 91e80c36606..e4dbf2c9e21 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -201,6 +201,21 @@ export function hasHermesNativeQuestionStop(input: { && browser.id === run.id && record(record(browser.resultJson).nativeCancellation).intentId === stop.intentId; } +/** Bind the terminal question card to the retained unanswered native request. */ +export function hasHermesNativeQuestionStopCard(input: { card: unknown; originalCard: unknown; runId: string }): boolean { + const card = record(input.card), original = record(input.originalCard); + return [input.runId, original.id, original.companyId, original.issueId].every(present) + && original.sourceRunId === input.runId && original.status === "pending" && !original.result + && original.kind === "ask_user_questions" && original.continuationPolicy === "none" + && ["id", "companyId", "issueId", "sourceRunId", "kind", "continuationPolicy"].every(key => card[key] === original[key]) + && card.status === "expired" && card.resolvedByRunId === input.runId + && card.resolvedByUserId === null && card.resolvedByAgentId === null + && present(record(original.payload).runtimeRequestId) && isDeepStrictEqual(card.payload, original.payload) + && isDeepStrictEqual(card.result, { + version: 1, cancelled: true, cancellationReason: "Native question cancelled", answers: [], summaryMarkdown: null, + }); +} + /** The browser owns Stop; public receipts and read-only OS observations grade it. */ export async function runHermesNativeQuestionStop(input: { page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string; @@ -242,7 +257,7 @@ export async function runHermesNativeQuestionStop(input: { return { run, issue, events, interactions }; }; let identity: Omit[0], "events" | "run" | "issue"> | undefined; - let interactionId: string | undefined; + let originalCard: unknown; const retired = () => processes.captured && processes.journal.length >= 2 && processes.live.length === 0 && !processIdentityChanged; input.registerCleanupAssertion(async () => { try { @@ -252,7 +267,7 @@ export async function runHermesNativeQuestionStop(input: { load: async () => { sample(); return processes; }, accept: retired }); } const passed = complete && retired() && runs.length === 1 && state.interactions.length === 1 - && state.interactions[0]?.id === interactionId && state.interactions[0]?.status === "expired" + && hasHermesNativeQuestionStopCard({ card: state.interactions[0], originalCard, runId: identity?.runId ?? "" }) && Boolean(identity && hasHermesNativeQuestionStop({ ...identity, ...state })); await input.evidence("hermes-native-stop-cleanup.json", { complete, processes, processIdentityChanged, passed }); return [{ id: "hermes-native-stop-retired-through-cleanup", passed, detail: "The exact per-turn owner and observed descendants remain retired; no late answer or follow-up appears." }]; @@ -280,7 +295,7 @@ export async function runHermesNativeQuestionStop(input: { && isDeepStrictEqual(request.input, questionSet) && request.requestId === card.payload?.runtimeRequestId && card.sourceRunId === pending.run.id && card.continuationPolicy === "none" && !card.result && !record(pending.run.resultJson).nativeCancellation, "One complete native callback remains unanswered in its original active turn before Stop."); - interactionId = card.id; + originalCard = card; await input.evidence("hermes-native-stop-pending.json", { card, event: pendingEvent, processes, checks }); await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`); await page.getByRole("radio", { name: questionSet.questions[0].options[0].label, exact: true }).last().waitFor({ state: "visible" }); @@ -303,8 +318,9 @@ export async function runHermesNativeQuestionStop(input: { await input.evidence("hermes-native-stop-browser-response.json", { identity, checks }); const stopped = await pollUntil({ label: "Hermes native question cancellation and owned Stop acknowledgement", deadlineAt, intervalMs: 200, load, reject: state => ["succeeded", "failed", "timed_out"].includes(state.run.status) ? "Stop did not produce a cancelled native turn" : undefined, - accept: state => state.interactions.length === 1 && state.interactions[0]?.id === card.id && state.interactions[0]?.status === "expired" - && !state.interactions[0]?.result && hasHermesNativeQuestionStop({ ...identity!, ...state }), + accept: state => state.interactions.length === 1 + && hasHermesNativeQuestionStopCard({ card: state.interactions[0], originalCard, runId: identity!.runId }) + && hasHermesNativeQuestionStop({ ...identity!, ...state }), }); check("exact-native-cancellation", hasHermesNativeQuestionStop({ ...identity, ...stopped }), "One ordered cancellation closes the retained native request and turn, with the browser's exact audited run Stop acknowledgement."); @@ -329,7 +345,7 @@ export async function runHermesNativeQuestionStop(input: { "The loaded cancelled turn offers no native question answer control."); const final = await load(); check("one-unfinished-cancelled-run", runs.length === 1 && final.run.status === "cancelled" && final.issue.status === "in_progress" - && final.interactions.length === 1 && final.interactions[0]?.status === "expired" && !final.interactions[0]?.result + && final.interactions.length === 1 && hasHermesNativeQuestionStopCard({ card: final.interactions[0], originalCard, runId: identity.runId }) && hasHermesNativeQuestionStop({ ...identity, ...final }), "Stop leaves one unfinished task and one cancelled run, without an answer or follow-up."); const connectionChecks = gradeHermesApiConnection({ companyId: fixtures.company.id, agentId: fixtures.agent.id, issueId: issue.id, connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider,