mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-30 21:49:18 +01:00
Clarify the Hermes native question fixture objective
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
3c7e0ef3a1
commit
b0bbaffec6
5 files changed
+38
-3
No files matched your search
@@ -1607,6 +1607,13 @@ budgets, settled reported billing, pre-credential source admission and cleanup
|
||||
pipeline. It allows one attempt and records its harness source digest. Its
|
||||
browser-reconnect scope does not qualify controller restart, cancellation,
|
||||
Stop, subscriptions, or Daytona until each corresponding live check passes.
|
||||
Definition version 2 checks the question-only objective against the production
|
||||
file/document delivery guard before any live execution. The first version's
|
||||
comma-separated prohibition triggered a file-output requirement: its native
|
||||
callback/reconnect checks passed, but completion rejected and the run timed out.
|
||||
That failed attempt is retained. Standalone prohibitions clarify the same
|
||||
question-only task; the production delivery guard and positive file requirements
|
||||
remain in force.
|
||||
|
||||
```sh
|
||||
pnpm test:e2e:runner -- --list --suite hermes-native-interactions
|
||||
|
||||
@@ -1219,8 +1219,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
groups: ["native"], profiles: [extendedHarnessProfiles.find(profile => profile.qualificationCandidate === "hermes")!],
|
||||
environments: runnerEnvironments, tasks: [hermesNativeQuestionTask], expectedMatrixSize: 2,
|
||||
definitionMetadata: {
|
||||
version: 1, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user",
|
||||
version: 2, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user",
|
||||
providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1,
|
||||
objectiveAdmission: "production-delivery-guard-question-only",
|
||||
budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health",
|
||||
coverage: "native-question-batch-browser-reconnect-exact-delivery", sourceDigest: hermesApiConnectionDefinitionDigest,
|
||||
},
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js";
|
||||
import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
|
||||
import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
|
||||
import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js";
|
||||
import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText } from "./hermes-api-connections.js";
|
||||
|
||||
describe("Hermes native browser questions", () => {
|
||||
@@ -48,6 +49,13 @@ describe("Hermes native browser questions", () => {
|
||||
expect(cells[0]!.task.buildPrompt("fixture-1")).not.toContain(answer);
|
||||
expect(cells[0]!.task.buildVisibleMarker("fixture-1")).toContain(answer);
|
||||
});
|
||||
it("admits a question-only objective through the production delivery guard without suppressing file requirements", () => {
|
||||
const prompt = cells[0]!.task.buildPrompt("fixture-guard");
|
||||
expect(explicitlyRequestsFileOutput(prompt)).toBe(false);
|
||||
expect(explicitlyRequestsTaskDocumentOutput(prompt)).toBe(false);
|
||||
expect(explicitlyRequestsFileOutput("Write a JSON file with the returned answers.")).toBe(true);
|
||||
expect(explicitlyRequestsTaskDocumentOutput("Write a document on this task with the returned answers.")).toBe(true);
|
||||
});
|
||||
it.each(["acpx-runtime", "acpx-runtime-sidecar"])("accepts a complete native delivery from %s", adapter => {
|
||||
expect(hasHermesNativeQuestionBatch(questionSet)).toBe(true);
|
||||
expect(grade(native(adapter))).toBe(true);
|
||||
|
||||
@@ -40,9 +40,9 @@ export const hermesNativeQuestionTask: RunnerTaskFixture = {
|
||||
{ id: "targets", question: "Choose the fixture targets", choices: ["Linux", "Mac"], multi_select: true },
|
||||
{ id: "notes", question: "Describe the fixture constraint" },
|
||||
]),
|
||||
"Wait for the actual native callback. Do not answer your own questions, guess the reviewer's selections, substitute request_human_input, ask through prose, or create a plan or files. If clarify is unavailable, report that limitation without claiming completion.",
|
||||
"Wait for the actual native callback. Do not answer your own questions. Do not guess the reviewer's selections. Do not substitute request_human_input. Do not ask through prose. Do not create files or plans. If clarify is unavailable, report that limitation without claiming completion.",
|
||||
`After the actual native result arrives, form your summary as HERMES-NATIVE-${nonce}-<exact returned notes user_response>. The notes are supplied by the reviewer; copy that returned text verbatim.`,
|
||||
"Call paperclip_finish once with the current completion contract revision, satisfied objective, this computed summary and no remaining work. Wait for acceptance, then emit only that exact summary as your final answer.",
|
||||
"Use the current turn completion revision and criterion IDs. Call paperclip_finish exactly once with {reportedWorkDisposition:'done',summary:COMPUTED_SUMMARY,completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:'objective',status:'satisfied',evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. COMPUTED_SUMMARY and CURRENT_TURN_REVISION are placeholders for the actual returned-text summary and current revision. Wait for acceptance, then emit only that exact summary as your final answer.",
|
||||
].join("\n"),
|
||||
buildVisibleMarker: nonce => `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`,
|
||||
buildMatchers: (nonce, execution) => [
|
||||
|
||||
Reference in new issue
Block a user