Clarify the Hermes native question fixture objective

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-08 12:28:13 -05:00
1 parent 3c7e0ef3a1
commit b0bbaffec6
5 files changed
+38 -3

No files matched your search

+7
View File
@@ -1607,6 +1607,13 @@ budgets, settled reported billing, pre-credential source admission and cleanup
pipeline. It allows one attempt and records its harness source digest. Its
browser-reconnect scope does not qualify controller restart, cancellation,
Stop, subscriptions, or Daytona until each corresponding live check passes.
Definition version 2 checks the question-only objective against the production
file/document delivery guard before any live execution. The first version's
comma-separated prohibition triggered a file-output requirement: its native
callback/reconnect checks passed, but completion rejected and the run timed out.
That failed attempt is retained. Standalone prohibitions clarify the same
question-only task; the production delivery guard and positive file requirements
remain in force.
```sh
pnpm test:e2e:runner -- --list --suite hermes-native-interactions
+2 -1
View File
@@ -1219,8 +1219,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
groups: ["native"], profiles: [extendedHarnessProfiles.find(profile => profile.qualificationCandidate === "hermes")!],
environments: runnerEnvironments, tasks: [hermesNativeQuestionTask], expectedMatrixSize: 2,
definitionMetadata: {
version: 1, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user",
version: 2, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user",
providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1,
objectiveAdmission: "production-delivery-guard-question-only",
budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health",
coverage: "native-question-batch-browser-reconnect-exact-delivery", sourceDigest: hermesApiConnectionDefinitionDigest,
},
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js";
import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js";
import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText } from "./hermes-api-connections.js";
describe("Hermes native browser questions", () => {
@@ -48,6 +49,13 @@ describe("Hermes native browser questions", () => {
expect(cells[0]!.task.buildPrompt("fixture-1")).not.toContain(answer);
expect(cells[0]!.task.buildVisibleMarker("fixture-1")).toContain(answer);
});
it("admits a question-only objective through the production delivery guard without suppressing file requirements", () => {
const prompt = cells[0]!.task.buildPrompt("fixture-guard");
expect(explicitlyRequestsFileOutput(prompt)).toBe(false);
expect(explicitlyRequestsTaskDocumentOutput(prompt)).toBe(false);
expect(explicitlyRequestsFileOutput("Write a JSON file with the returned answers.")).toBe(true);
expect(explicitlyRequestsTaskDocumentOutput("Write a document on this task with the returned answers.")).toBe(true);
});
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("accepts a complete native delivery from %s", adapter => {
expect(hasHermesNativeQuestionBatch(questionSet)).toBe(true);
expect(grade(native(adapter))).toBe(true);
+2 -2
View File
@@ -40,9 +40,9 @@ export const hermesNativeQuestionTask: RunnerTaskFixture = {
{ id: "targets", question: "Choose the fixture targets", choices: ["Linux", "Mac"], multi_select: true },
{ id: "notes", question: "Describe the fixture constraint" },
]),
"Wait for the actual native callback. Do not answer your own questions, guess the reviewer's selections, substitute request_human_input, ask through prose, or create a plan or files. If clarify is unavailable, report that limitation without claiming completion.",
"Wait for the actual native callback. Do not answer your own questions. Do not guess the reviewer's selections. Do not substitute request_human_input. Do not ask through prose. Do not create files or plans. If clarify is unavailable, report that limitation without claiming completion.",
`After the actual native result arrives, form your summary as HERMES-NATIVE-${nonce}-<exact returned notes user_response>. The notes are supplied by the reviewer; copy that returned text verbatim.`,
"Call paperclip_finish once with the current completion contract revision, satisfied objective, this computed summary and no remaining work. Wait for acceptance, then emit only that exact summary as your final answer.",
"Use the current turn completion revision and criterion IDs. Call paperclip_finish exactly once with {reportedWorkDisposition:'done',summary:COMPUTED_SUMMARY,completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:'objective',status:'satisfied',evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. COMPUTED_SUMMARY and CURRENT_TURN_REVISION are placeholders for the actual returned-text summary and current revision. Wait for acceptance, then emit only that exact summary as your final answer.",
].join("\n"),
buildVisibleMarker: nonce => `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`,
buildMatchers: (nonce, execution) => [