fix(hermes): preserve cancelled usage at the ACPX wire boundary

Bind usage to the admitted native session and turn before cancelled prompt settlement. Check its durable PRP carrier and harden qualification receipt identity and budget capture.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-08 13:37:58 -05:00
1 parent 36125adb4e
commit a14a752ec8
9 files changed
+259 -40

No files matched your search

@@ -1484,3 +1484,46 @@ database tests), and Hermes releases assigned skill copies after a credential
or state-collection failure only once runtime exit is verified (141 affected
runtime tests). The local TypeScript runtime must be rebuilt from the current
source before the next paid campaign. No local Docker or Rust build is used.
The `36125adb4e` 100-cent Stop campaign retains its failed/cleanup-failed grade
after 300.437 seconds. The browser Stop is acknowledged, one native input and
its turn cancel in order, the question becomes expired, and the task stays in
progress. All nine observed PID/start identities retire. Billing remains
unpriced despite complete reported token accounting (31,665 input and 218
output tokens), so the independent settlement gate correctly fails. The private
recovery state and unknown-cost hold remain retained. This is progress on native
cancellation, not a live Stop qualification pass.
Qualification review identified two independent evidence gaps. Six negative
runner/session identity cases and a local-settings catalog case first failed.
Answer delivery now requires the created request and resolved receipt to name
the same runner and normalized session. The public budget setting is read and
validated before catalog construction and pinned into the child environment;
credentials and other file settings still load at the normal later boundary.
Definitions advance to native version 7, API/Bedrock version 3 and extended
version 4. Earlier measurements retain their source, definition and grade.
Source inspection finds that pinned ACPX also drops extension notifications
when its native elicitation controller is aborted, before Paperclip's outer
cancelled-receipt boundary. Hermes usage now enters through ACPX's public
inbound-message observer, with the same exact native session and negotiated
turn-token checks. The ordinary extension callback excludes that one method
to avoid duplicate delivery. Other notifications and requests retain their
existing admission paths. No ACPX version or dependency lock changes.
The strengthened production transport fixture checks the actual durable PRP
carrier: one owned Hermes usage-provenance notice precedes the cancelled turn,
and both retain their runner, session, run and turn identities. The same fixture
fails with the previous adapter because its receipt is missing, then passes with
the repaired adapter and the verified cloud-built Mac daemon. The comparison
restores the repaired source and bundle byte for byte. An earlier fixture
attempt inspected the normalized lifecycle stream, which does not carry these
informational notices; that failed attempt is retained and is not the regression
proof. This fixture has no provider credentials and proves transport, not paid
OpenRouter charge settlement.
Final local checks pass 184 affected ACP tests, both Runner TypeScript checks,
and 2,035 Product E2E support tests with one platform skip. Product E2E TypeScript
also passes. A fresh committed-source browser Stop, current-head cloud checks,
and the remaining release matrix are still required. No local Docker or Rust
build was used, and no unknown-cost hold was released.
@@ -3,7 +3,7 @@
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
import { once } from 'node:events';
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { createServer } from 'node:http';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -215,6 +215,24 @@ for (const { interactionKind, nativeQuestionAction } of cases) test(nativeQuesti
input: usage.payload.usage.runDelta.inputTokens, output: usage.payload.usage.runDelta.outputTokens,
complete: usage.payload.usage.runDeltaComplete,
}, { input: 10, output: 5, complete: true }, 'Stopped clarification lost final native prompt usage');
// Informational notices enter durable PRP evidence rather than the
// normalized NativeSession lifecycle stream. Assert the actual carrier.
const controlPlane = JSON.parse(await readFile(join(root, 'runner/control-plane/control-plane-state.json'), 'utf8'));
const committed = controlPlane.committedEvents.map(event => event.envelope.payload);
const receipts = committed.filter(event => event.eventType === 'provider.notice.recorded'
&& event.payload.category === 'hermes_usage_provenance');
assert.equal(receipts.length, 1, 'Stopped clarification lost its owned Hermes usage extension');
assert.equal(receipts[0].schema, 'paperclip.prp.event.v1');
assert.equal(receipts[0].sourceKind, 'runner');
assert.equal(receipts[0].sourceInstanceId, 'hermes-prp-runner');
assert.equal(receipts[0].normalizedSessionId, identity.sessionId);
assert.equal(receipts[0].runId, identity.runId);
assert.equal(receipts[0].turnId, 'hermes-prp-turn');
assert.equal(receipts[0].payload.details.find(detail => detail.name === 'Token usage')?.value, 'reported');
const committedTerminals = committed.filter(event => event.eventType === 'turn.cancelled');
assert.equal(committedTerminals.length, 1);
assert.equal(committedTerminals[0].turnId, receipts[0].turnId);
assert.ok(receipts[0].sourceSeq < committedTerminals[0].sourceSeq, 'Usage provenance arrived after cancelled settlement');
await session.close({ reason: 'native clarification stopped' });
assert.equal((await session.snapshot()).semanticResult, null, 'Stop invented task completion');
return;
@@ -43,31 +43,67 @@ describe("Codex ACPX runtime adapter", () => {
const turn = port.startTurn({ text: "Work", requestId: "turn-1", onExtensionRequest: request, onExtensionNotification: notification });
const identity = { version: 1, sessionId: "backend-1", turnToken: "00000000-0000-0000-0000-000000000001" };
created.onExtensionNotification!("_hermes/turn_started", identity);
const wireUsage = (params: Record<string, unknown>) => created.onAcpMessage?.("inbound", { jsonrpc: "2.0", method: "_hermes/usage", params });
await turn.cancel();
created.onExtensionNotification!("_hermes/delegation", identity);
await expect(created.onExtensionRequest!("_hermes/ask_questions", identity,
{ requestId: 0, signal: new AbortController().signal })).rejects.toThrow("admitted active turn");
for (const foreign of [{ ...identity, sessionId: "foreign" }, { ...identity, sessionId: "agent-1" }, { ...identity, turnToken: "00000000-0000-0000-0000-000000000002" }]) {
expect(() => created.onExtensionNotification!("_hermes/usage", foreign)).not.toThrow();
expect(() => wireUsage(foreign)).not.toThrow();
}
expect(notification).not.toHaveBeenCalled();
const receipt = { ...identity, tokens: "reported", cost: "unavailable", billing: {
schema: "paperclip.usage.billing/v1", source: "provider_reported", biller: "openrouter", currency: "USD",
complete: true, requestCount: 1, reportedRequestCount: 1, amountUsd: 0.0042, amountUsdExact: "0.004200000",
} };
created.onExtensionNotification!("_hermes/usage", receipt);
const wire = { jsonrpc: "2.0", method: "_hermes/usage", params: receipt };
created.onAcpMessage?.("outbound", wire);
for (const invalid of [{ ...wire, id: 0 }, { ...wire, id: undefined }, { ...wire, jsonrpc: "1.0" },
{ ...wire, method: "_hermes/delegation" }, { ...wire, params: null }, { ...wire, params: [] }]) {
created.onAcpMessage?.("inbound", invalid);
}
expect(notification).not.toHaveBeenCalled();
wireUsage(receipt);
expect(notification).toHaveBeenCalledExactlyOnceWith("_hermes/usage", receipt);
// ACPX's ordinary extension callback must not deliver the same wire
// notification twice if its cancellation gate has not closed yet.
created.onExtensionNotification!("_hermes/usage", receipt);
expect(notification).toHaveBeenCalledTimes(1);
expect(request).not.toHaveBeenCalled();
await turn.closeStream();
await turn.cancel();
created.onExtensionNotification!("_hermes/usage", receipt);
wireUsage(receipt);
expect(notification).toHaveBeenCalledTimes(1);
pending.settle(); await turn.result;
created.onExtensionNotification!("_hermes/usage", receipt);
wireUsage(receipt);
expect(notification).toHaveBeenCalledTimes(1);
await port.close({ reason: "terminal billing boundary verified" });
});
it("delivers active Hermes usage once through the same wire path and seals it at prompt settlement", async () => {
const pending = pendingExtensionTurn("turn-1");
const runtime = fakeRuntime(); vi.mocked(runtime.startTurn).mockReturnValue(pending.turn);
let created!: AcpRuntimeOptions;
const options = openOptions(fakeCommand()); options.profile = { ...options.profile, agent: "hermes" };
const port = await openCodexAcpxRuntime(options, {
createRegistry: () => registry(), createStore: () => store(), createRuntime: value => { created = value; return runtime; },
});
const notification = vi.fn();
const identity = { version: 1, sessionId: "backend-1", turnToken: "00000000-0000-0000-0000-000000000001" };
const wire = { jsonrpc: "2.0", method: "_hermes/usage", params: { ...identity, tokens: "reported", cost: "unavailable" } };
created.onAcpMessage?.("inbound", wire);
expect(notification).not.toHaveBeenCalled();
const turn = port.startTurn({ text: "Work", requestId: "turn-1", onExtensionNotification: notification });
created.onExtensionNotification!("_hermes/turn_started", identity);
created.onAcpMessage?.("inbound", wire);
created.onExtensionNotification!("_hermes/usage", wire.params);
expect(notification).toHaveBeenCalledExactlyOnceWith("_hermes/usage", wire.params);
pending.settle(); await turn.result;
created.onAcpMessage?.("inbound", wire);
expect(notification).toHaveBeenCalledTimes(1);
await port.close({ reason: "active receipt settled" });
});
it("rejects forged permission session identifiers before delegating or applying full-auto policy", async () => {
const pending = pendingExtensionTurn("turn-1");
const runtime = fakeRuntime(); vi.mocked(runtime.startTurn).mockReturnValue(pending.turn);
@@ -310,6 +310,24 @@ export async function openQualifiedAcpxRuntime(
const cursorInstructions = options.profile.agent === "cursor"
? createCursorInstructionAdmission(options.systemInstructions)
: null;
const acceptExtensionNotification: NonNullable<AcpRuntimeOptions["onExtensionNotification"]> = (method, params) => {
const active = extensionBoundary.active;
if (options.profile.agent === "hermes" && method === "_hermes/turn_started") {
if (!active || !ownsExtensionTurn(active, params) || params.version !== 1 || typeof params.turnToken !== "string" || !/^[0-9a-f-]{36}$/.test(params.turnToken) || active.nativeTurnToken) throw new Error("Invalid Hermes turn binding");
active.nativeTurnToken = params.turnToken;
return;
}
// Stop revokes requests and activity immediately. Only the admitted native
// prompt's exact final usage may cross its cancelled boundary until result
// settlement. Stream/process closure retires this receipt window.
const cancelledUsage = options.profile.agent === "hermes" && method === "_hermes/usage"
&& active !== null && extensionBoundary.cancelledReceiptOwner === active && extensionBoundary.active === active
&& params.sessionId === active.sessionId && params.version === 1
&& !!active.nativeTurnToken && params.turnToken === active.nativeTurnToken;
if (!extensionNotifications.has(method) || !active?.onNotification || (!ownsExtensionTurn(active, params) && !cancelledUsage)) return;
if (options.profile.agent === "hermes" && (params.version !== 1 || !active.nativeTurnToken || params.turnToken !== active.nativeTurnToken)) throw new Error("Hermes activity belongs to a stale turn");
active.onNotification(method, params.sessionId === undefined ? { ...params, sessionId: active.sessionId } : params);
};
const runtimeOptions: GoalAwareAcpRuntimeOptions = {
cwd: options.cwd,
...(cursorInstructions || modeBinding ? { protocolGuardFactory: () => {
@@ -351,24 +369,20 @@ export async function openQualifiedAcpxRuntime(
return response;
},
onExtensionNotification: (method, params) => {
const active = extensionBoundary.active;
if (options.profile.agent === "hermes" && method === "_hermes/turn_started") {
if (!active || !ownsExtensionTurn(active, params) || params.version !== 1 || typeof params.turnToken !== "string" || !/^[0-9a-f-]{36}$/.test(params.turnToken) || active.nativeTurnToken) throw new Error("Invalid Hermes turn binding");
active.nativeTurnToken = params.turnToken;
return;
}
// Stop revokes requests and activity immediately, but the native prompt
// still settles its owned work and billing. Admit only its final usage
// notification, with the exact negotiated session/token, until result
// settlement. Stream/process closure never opens this receipt window.
const cancelledUsage = options.profile.agent === "hermes" && method === "_hermes/usage"
&& active !== null && extensionBoundary.cancelledReceiptOwner === active && extensionBoundary.active === active
&& params.sessionId === active.sessionId && params.version === 1
&& !!active.nativeTurnToken && params.turnToken === active.nativeTurnToken;
if (!extensionNotifications.has(method) || !active?.onNotification || (!ownsExtensionTurn(active, params) && !cancelledUsage)) return;
if (options.profile.agent === "hermes" && (params.version !== 1 || !active.nativeTurnToken || params.turnToken !== active.nativeTurnToken)) throw new Error("Hermes activity belongs to a stale turn");
active.onNotification(method, params.sessionId === undefined ? { ...params, sessionId: active.sessionId } : params);
// Hermes usage has one wire-observer path, including before cancellation.
// Do not deliver it twice through ACPX's ordinary extension callback.
if (options.profile.agent === "hermes" && method === "_hermes/usage") return;
acceptExtensionNotification(method, params);
},
...(options.profile.agent === "hermes" ? { onAcpMessage: (direction: "inbound" | "outbound", message: unknown) => {
// Pinned ACPX 0.13.1 aborts its elicitation controller before Hermes emits
// final usage, and drops extensions at that inner gate. Its public wire
// observer still sees the notification. Reapply our exact turn/session
// admission here; this grants no request or activity authority after Stop.
const wire = objectRecord(message);
if (direction !== "inbound" || wire.jsonrpc !== "2.0" || "id" in wire || wire.method !== "_hermes/usage") return;
acceptExtensionNotification("_hermes/usage", objectRecord(wire.params));
} } : {}),
nonInteractivePermissions: "fail",
permissionPolicy: {
...options.permissionPolicy,
+16 -5
View File
@@ -1488,12 +1488,23 @@ oracle. Older behavioral passes do not prove this accounting requirement.
Before launching, `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS` can lower both
budgets to a whole number from 1 through 200 cents. Missing configuration keeps
the 200-cent default; malformed, zero, and higher limits fail before credential
handoff. The campaign captures one immutable value for fixture creation, public
budget readback, settlement health, and catalog definition identity. Every
handoff. The public setting can also come from `.env.runner-e2e.local`. It is
validated before catalog construction and pinned into the child environment;
explicit environment configuration takes precedence. This early step loads no
credentials or other file settings. The campaign captures one immutable value
for fixture creation, public budget readback, settlement health, and catalog
definition identity. Every
observed budget must match it exactly. A lower cap does not settle unknown
charges or release a previous attempt's reservation. Definition versions 3
(extended harnesses), 2 (Hermes API/Bedrock), and 6 (native interactions) record
this bounded option.
charges or release a previous attempt's reservation. Definition versions 4
(extended harnesses), 3 (Hermes API/Bedrock), and 7 (native interactions) record
this bounded option and consistent settings capture. Native answer receipts
must also retain the original runner and normalized session identities.
The credential-free native Stop fixture checks usage provenance on its durable
PRP carrier before the cancelled terminal, including runner, session, run and
turn identity. Informational notices are not part of the normalized lifecycle
stream. This transport proof does not replace a paid browser campaign's charge
settlement or process-cleanup checks.
Before loading local credentials, the launcher records the checked-out controller
SHA and ref for every selected Hermes candidate, including API and Bedrock cells.
+5 -2
View File
@@ -36,6 +36,9 @@ to a whole number from 1 through 200. Invalid or unlimited values fail before
credential handoff. Public company/agent reads and post-run settlement must
match the captured limit. Provider-key limits, campaign reservations, and
unknown-cost holds remain separate spending controls.
The public budget setting is resolved from the environment or local settings
file before catalog construction, then pinned for children. Other file settings
and credentials still load only at the normal later admission boundary.
## Short-lived Bedrock qualification
@@ -345,7 +348,7 @@ grading receipts alongside the existing sanitized run evidence. No private
runner hook supplies the answer. The late duplicate probe uses the existing
untraced public POST helper and requires its actual HTTP 409 status; response
bodies are withheld. Source admission precedes credential handoff, both public
budgets are 200 cents, OpenRouter cost must settle, and automatic retries remain
budgets match the captured bounded limit, OpenRouter cost must settle, and automatic retries remain
disabled. Registered local/Daytona cells are pending qualification, not permission
to launch a paid campaign or a claim of remote coverage.
@@ -356,5 +359,5 @@ those PIDs. Cancellation goes through the normal browser Stop control and public
board API. No question answer is delivered; the only response attempt is a late
fixture answer that must be rejected. Retained evidence uses the existing
sanitizer and screenshot publication gates. The cell reuses the selected key
allocation and 200-cent budgets, with zero automatic retries. It neither creates
allocation and captured bounded budgets, with zero automatic retries. It neither creates
remote resources nor grants permission to use another secret.
+4 -4
View File
@@ -1210,7 +1210,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
groups: ["native"], profiles: extendedHarnessProfiles, environments: runnerEnvironments,
tasks: [...openRouterBreadthTasks, localIntegrityTasks[1]!, extendedHarnessFileTask],
expectedMatrixSize: 40,
definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28",
definitionMetadata: { version: 4, qualification: "pending", scheduling: "explicit-only", admission: "host-exact-candidate-and-model", authenticatedDiscoveryDate: "2026-09-28",
hermesBilling: "per-run-reported-cost-and-budget-health", hermesBudgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, hermesSettlementSourceDigest: hermesApiConnectionDefinitionDigest },
},
{
@@ -1220,7 +1220,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
environments: runnerEnvironments, tasks: [hermesNativeQuestionTask, hermesNativeQuestionStopTask],
excludedExecutionIds: [`${HERMES_NATIVE_INTERACTION_SUITE}.runner-acpx-hermes.daytona.native-question-batch-stop`], expectedMatrixSize: 3,
definitionMetadata: {
version: 6, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user",
version: 7, qualification: "pending", scheduling: "explicit-only", accountMethod: "api_key", accountMode: "responsible_user",
providerTurns: 1, lifecycle: "per-turn", nativeMethod: "_hermes/ask_questions", maximumAttemptsPerCell: 1,
objectiveAdmission: "production-delivery-guard-question-only",
budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, billing: "reported-cost-and-budget-health",
@@ -1236,7 +1236,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }],
expectedMatrixSize: 10,
definitionMetadata: {
version: 2, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07",
version: 3, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07",
accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, expectedUserSource: "public-account-owner-before-task",
budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1,
coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest,
@@ -1249,7 +1249,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
tasks: [{ ...openRouterBreadthTasks.find(task => task.id === "hello-complete")!, automaticRetryPolicy: "single_attempt" }],
expectedMatrixSize: 2,
definitionMetadata: {
version: 2, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07",
version: 3, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07",
accountMethod: "api_key", accountMode: "delegated", providerTurns: 1, expectedUserSource: "public-account-owner-before-task",
routingSource: "public-selected-account-before-and-after-task", credentialScope: "ephemeral-region-bound-bearer",
budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1,
@@ -5,6 +5,14 @@ import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
import { explicitlyRequestsFileOutput, explicitlyRequestsTaskDocumentOutput } from "../../server/src/services/native-runtime/native-deliverable-feedback.js";
import { captureHermesApiAccountOwner, captureHermesApiBudgets, captureHermesOpenRouterSettlement, gradeHermesApiConnection, isHermesOpenRouterWorkflow, isHermesConnectionSuite, HERMES_NATIVE_INTERACTION_SUITE, hasExactHermesNativeQuestionResponse, hasHermesNativeQuestionBatch, hermesNativeAnswerText, hasHermesNativeQuestionStop, resolveHermesQualificationBudgetCents } from "./hermes-api-connections.js";
const settings = vi.hoisted(() => ({ contents: undefined as string | undefined }));
vi.mock("node:fs", async importOriginal => {
const actual = await importOriginal<typeof import("node:fs")>();
return { ...actual, readFileSync: ((file: Parameters<typeof actual.readFileSync>[0], options: unknown) =>
settings.contents !== undefined && String(file).endsWith("/.env.runner-e2e.local")
? settings.contents : actual.readFileSync(file, options as never)) };
});
describe("Hermes native browser questions", () => {
const suite = runnerSuites.find(s => s.id === HERMES_NATIVE_INTERACTION_SUITE)!;
const cells = runnerMatrix.filter(e => e.suite.id === suite.id);
@@ -134,7 +142,8 @@ describe("Hermes native browser questions", () => {
});
it.each(["missing", "no-created", "no-outcome", "duplicate-created", "duplicate-outcome", "second-request-other-id", "cancelled", "expired", "wrong-action", "wrong-answer",
"wrong-answer-order", "wrong-input", "wrong-provider", "semantic-tool", "wrong-adapter", "wrong-type", "wrong-status", "wrong-request-turn",
"wrong-event-turn", "wrong-outcome-turn", "foreign-row", "foreign-event", "wrong-schema", "wrong-version", "wrong-protocol", "wrong-order", "missing-sequence"])("rejects %s evidence", fault => {
"wrong-event-turn", "wrong-outcome-turn", "foreign-row", "foreign-event", "wrong-schema", "wrong-version", "wrong-protocol", "wrong-order", "missing-sequence",
"foreign-source", "foreign-session", "not-runner", "created-not-runner", "missing-source", "missing-session"])("rejects %s evidence", fault => {
const rows = structuredClone(native()) as ReturnType<typeof native>;
const created = rows[0]!.payload.prpEvent as Record<string, any>, resolved = rows[1]!.payload.prpEvent as Record<string, any>;
const request = created.payload.request;
@@ -169,6 +178,12 @@ describe("Hermes native browser questions", () => {
if (fault === "wrong-protocol") rows[1]!.protocolSchemaVersion = 2;
if (fault === "wrong-order") resolved.sourceSeq = 1;
if (fault === "missing-sequence") delete resolved.sourceSeq;
if (fault === "foreign-source") resolved.sourceInstanceId = "another-runner";
if (fault === "foreign-session") resolved.normalizedSessionId = "another-session";
if (fault === "not-runner") resolved.sourceKind = "controller";
if (fault === "created-not-runner") created.sourceKind = "controller";
if (fault === "missing-source") delete created.sourceInstanceId;
if (fault === "missing-session") delete created.normalizedSessionId;
expect(grade(rows)).toBe(false);
});
it.each(["question-count", "mode", "id", "prompt", "required", "choices", "option-id", "option-label", "custom-disabled"])("rejects a changed native form: %s", fault => {
@@ -187,6 +202,61 @@ describe("Hermes native browser questions", () => {
});
describe("Hermes managed API connection qualification", () => {
it.each(["0", "201", "100.0", ""])("rejects invalid local settings budget %s before catalog construction", async raw => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", undefined);
settings.contents = `PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=${raw}\n`;
vi.resetModules();
try {
await expect(import("./catalog.js")).rejects.toThrow("integer from 1 to 200 cents");
expect(process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBeUndefined();
} finally {
settings.contents = undefined;
vi.unstubAllEnvs();
vi.resetModules();
}
});
it("preserves an explicit environment budget ahead of local settings", async () => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", "100");
settings.contents = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=0\n";
vi.resetModules();
try {
const bounded = await import("./hermes-api-connections.js");
expect(bounded.HERMES_API_CONNECTION_BUDGET_CENTS).toBe(100);
expect(process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBe("100");
} finally {
settings.contents = undefined;
vi.unstubAllEnvs();
vi.resetModules();
}
});
it("captures the local settings budget before catalog construction and pins it for child processes", async () => {
vi.stubEnv("PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS", undefined);
const existingCredential = process.env.OPENROUTER_API_KEY;
settings.contents = "# Fixture-only public configuration\nexport PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS='100'\nOPENROUTER_API_KEY=never-load-this-fixture-value\n";
vi.resetModules();
try {
const catalog = await import("./catalog.js");
const bounded = await import("./hermes-api-connections.js");
const cell = catalog.runnerMatrix.find(cell => cell.id === "hermes-native-interactions.runner-acpx-hermes.local.native-question-batch-stop")!;
expect(bounded.HERMES_API_CONNECTION_BUDGET_CENTS).toBe(100);
expect(cell.suite.definitionMetadata).toMatchObject({ budgetMonthlyCents: 100 });
expect(process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBe("100");
const child = buildRunnerE2EProcessEnvironment(process.env, [cell]);
expect(child.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS).toBe("100");
expect(process.env.OPENROUTER_API_KEY).toBe(existingCredential);
// The late full settings loader preserves an already captured value.
settings.contents = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=200\n";
vi.resetModules();
const reloaded = await import("./catalog.js");
const next = reloaded.runnerMatrix.find(next => next.id === cell.id)!;
expect(next.suiteDefinitionHash).toBe(cell.suiteDefinitionHash);
expect(next.suite.definitionMetadata).toMatchObject({ budgetMonthlyCents: 100 });
} finally {
settings.contents = undefined;
vi.unstubAllEnvs();
vi.resetModules();
}
});
it.each([[undefined, 200], ["1", 1], ["100", 100], ["200", 200]] as const)("admits the bounded campaign limit %s", (raw, expected) => {
expect(resolveHermesQualificationBudgetCents(raw)).toBe(expected);
});
+30 -6
View File
@@ -22,11 +22,31 @@ export function resolveHermesQualificationBudgetCents(raw?: string): number {
}
return Number(raw);
}
// Capture one immutable campaign limit before any credential handoff. Fixture
// creation, public readback, settlement and definition identity use this value.
export const HERMES_API_CONNECTION_BUDGET_CENTS = resolveHermesQualificationBudgetCents(
process.env.PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS,
);
function captureHermesQualificationBudgetCents(): number {
const key = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS";
let raw = process.env[key];
if (raw === undefined) {
let contents: string | undefined;
try { contents = readFileSync(new URL("../../.env.runner-e2e.local", import.meta.url), "utf8"); }
catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
// Read only this public setting. Credentials still load after source and
// runtime admission in launch.ts. Use the full loader's first-value and
// quoting rules, without importing other settings into the environment.
for (const line of contents?.split(/\r?\n/) ?? []) {
const match = /^(?:export\s+)?PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=(.*)$/.exec(line.trim());
if (!match) continue;
raw = match[1]!.trim();
if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) raw = raw.slice(1, -1);
break;
}
}
const budget = resolveHermesQualificationBudgetCents(raw);
// The late settings loader preserves this captured value. Every child sees
// the same validated limit that already identifies the launcher's catalog.
process.env[key] = String(budget);
return budget;
}
export const HERMES_API_CONNECTION_BUDGET_CENTS = captureHermesQualificationBudgetCents();
export const HERMES_NATIVE_INTERACTION_SUITE = "hermes-native-interactions";
export const isHermesConnectionSuite = (suiteId: string) =>
suiteId === "hermes-api-connections" || suiteId === "hermes-bedrock-connections" || suiteId === HERMES_NATIVE_INTERACTION_SUITE;
@@ -122,7 +142,11 @@ export function hasExactHermesNativeQuestionResponse(input: {
const outcomes = relevant.filter(({ event }) => event.eventType !== "runtime_request.created");
if (created.length !== 1 || outcomes.length !== 1) return false;
const request = record(record(created[0]!.event.payload).request), outcome = outcomes[0]!.event;
return hasAcpxNativeOrigin(request.origin, "hermes", "_hermes/ask_questions")
const origin = created[0]!.event;
return origin.sourceKind === "runner" && [origin.sourceInstanceId, origin.normalizedSessionId].every(present)
&& outcome.sourceKind === "runner" && outcome.sourceInstanceId === origin.sourceInstanceId
&& outcome.normalizedSessionId === origin.normalizedSessionId
&& hasAcpxNativeOrigin(request.origin, "hermes", "_hermes/ask_questions")
&& request.type === "input" && request.status === "pending" && request.turnId === input.turnId
&& isDeepStrictEqual(request.input, input.questionSet)
&& outcome.eventType === "runtime_request.resolved" && record(outcome.payload).action === "submit"