From b0bbaffec6eada22f5eb17e2157de8e1ff947e7f Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 8 Oct 2026 08:33:45 -0500 Subject: [PATCH] Clarify the Hermes native question fixture objective Co-Authored-By: Paperclip --- doc/plans/2026-10-06-hermes-native-runner.md | 19 +++++++++++++++++++ tests/runner-e2e/README.md | 7 +++++++ tests/runner-e2e/catalog.ts | 3 ++- .../runner-e2e/hermes-api-connections.test.ts | 8 ++++++++ tests/runner-e2e/hermes-api-connections.ts | 4 ++-- 5 files changed, 38 insertions(+), 3 deletions(-) diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 7247f07671d..128d890025b 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -1291,3 +1291,22 @@ typecheck and all 1,953 unit tests pass, with one existing skip; discovery lists exactly the two explicit cells. Native browser acceptance is still pending its live run. Controller restart, native cancellation/Stop, subscriptions, Bedrock, actual Daytona and the remaining original release matrix remain separate gates. + +The first live native-question browser campaign at +`d35d986aab52d3c132f5dd243d39684b33d8c613` passes eight native card, reconnect, +exact-delivery and duplicate-rejection checks. The native `clarify` tool returns +the submitted batch. Its full attempt still fails with `candidate_failure`, +cleanup passed, after 144.541 seconds. The production delivery guard treats the +fixture's comma-separated negative clause as a file-output request, rejects +`paperclip_finish`, and the provider attempts publication before the configured +120-second native timeout. The original machine grade and unpriced per-run +usage remain retained. A provider-key aggregate read shows $0.172703466 more +usage; this is not a per-run settlement receipt and no budget hold is released. + +A credential-free reproduction confirms the guard's interpretation and that +standalone prohibitions admit the same question-only task while preserving +positive file-output requirements. Definition version 2 uses those explicit +prohibitions and the current completion claim shape. An admission test calls the +actual production file/document guard. Product E2E typecheck, all 1,954 Vitest +tests with one existing skip, and exact discovery pass for this correction. +The production guard is unchanged. A fresh live attempt remains required. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index b510b2f30f2..a4708f7f3d6 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1607,6 +1607,13 @@ budgets, settled reported billing, pre-credential source admission and cleanup pipeline. It allows one attempt and records its harness source digest. Its browser-reconnect scope does not qualify controller restart, cancellation, Stop, subscriptions, or Daytona until each corresponding live check passes. +Definition version 2 checks the question-only objective against the production +file/document delivery guard before any live execution. The first version's +comma-separated prohibition triggered a file-output requirement: its native +callback/reconnect checks passed, but completion rejected and the run timed out. +That failed attempt is retained. Standalone prohibitions clarify the same +question-only task; the production delivery guard and positive file requirements +remain in force. ```sh pnpm test:e2e:runner -- --list --suite hermes-native-interactions diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 62cdeb5f71f..a89de106d04 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1219,8 +1219,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: [extendedHarnessProfiles.find(profile => profile.qualificationCandidate === "hermes")!], environments: runnerEnvironments, tasks: [hermesNativeQuestionTask], expectedMatrixSize: 2, definitionMetadata: { - version: 1, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", + version: 2, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1, + objectiveAdmission: "production-delivery-guard-question-only", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health", coverage: "native-question-batch-browser-reconnect-exact-delivery", sourceDigest: hermesApiConnectionDefinitionDigest, }, diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index 97e877b59ae..8868f31f174 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; +import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js"; import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText } from "./hermes-api-connections.js"; describe("Hermes native browser questions", () => { @@ -48,6 +49,13 @@ describe("Hermes native browser questions", () => { expect(cells[0]!.task.buildPrompt("fixture-1")).not.toContain(answer); expect(cells[0]!.task.buildVisibleMarker("fixture-1")).toContain(answer); }); + it("admits a question-only objective through the production delivery guard without suppressing file requirements", () => { + const prompt = cells[0]!.task.buildPrompt("fixture-guard"); + expect(explicitlyRequestsFileOutput(prompt)).toBe(false); + expect(explicitlyRequestsTaskDocumentOutput(prompt)).toBe(false); + expect(explicitlyRequestsFileOutput("Write a JSON file with the returned answers.")).toBe(true); + expect(explicitlyRequestsTaskDocumentOutput("Write a document on this task with the returned answers.")).toBe(true); + }); it.each(["acpx-runtime", "acpx-runtime-sidecar"])("accepts a complete native delivery from %s", adapter => { expect(hasHermesNativeQuestionBatch(questionSet)).toBe(true); expect(grade(native(adapter))).toBe(true); diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index d34b855bfc7..fe5b7017a3c 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -40,9 +40,9 @@ export const hermesNativeQuestionTask: RunnerTaskFixture = { { id: "targets", question: "Choose the fixture targets", choices: ["Linux", "Mac"], multi_select: true }, { id: "notes", question: "Describe the fixture constraint" }, ]), - "Wait for the actual native callback. Do not answer your own questions, guess the reviewer's selections, substitute request_human_input, ask through prose, or create a plan or files. If clarify is unavailable, report that limitation without claiming completion.", + "Wait for the actual native callback. Do not answer your own questions. Do not guess the reviewer's selections. Do not substitute request_human_input. Do not ask through prose. Do not create files or plans. If clarify is unavailable, report that limitation without claiming completion.", `After the actual native result arrives, form your summary as HERMES-NATIVE-${nonce}-. The notes are supplied by the reviewer; copy that returned text verbatim.`, - "Call paperclip_finish once with the current completion contract revision, satisfied objective, this computed summary and no remaining work. Wait for acceptance, then emit only that exact summary as your final answer.", + "Use the current turn completion revision and criterion IDs. Call paperclip_finish exactly once with {reportedWorkDisposition:'done',summary:COMPUTED_SUMMARY,completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:'objective',status:'satisfied',evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. COMPUTED_SUMMARY and CURRENT_TURN_REVISION are placeholders for the actual returned-text summary and current revision. Wait for acceptance, then emit only that exact summary as your final answer.", ].join("\n"), buildVisibleMarker: nonce => `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`, buildMatchers: (nonce, execution) => [