PaperClipAI/tests/runner-e2e/hermes-api-connections.ts

import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { isDeepStrictEqual } from "node:util";
import { deflateSync } from "node:zlib";
import type { AiProviderRouting } from "../../packages/shared/src/ai-provider-routing.js";
import { hasAcpxNativeOrigin } from "./acpx-native-origin.js";
import type { Page } from "@playwright/test";
import { pollUntil, type RunnerApi } from "./api.js";
import { collectRunEvents } from "./run-observations.js";
import { createTaskThroughUi } from "./user-actions.js";
import { observeRunProcesses, type RunProcessAuthority } from "./native-local-fixtures.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { MatrixExecution, RunnerProfileFixture, RunnerTaskFixture } from "./types.js";

const record = (value: unknown): Record<string, unknown> =>
  value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
const present = (value: unknown) => typeof value === "string" && value.trim().length > 0;
export const HERMES_API_CONNECTION_MAX_BUDGET_CENTS = 200;
export function resolveHermesQualificationBudgetCents(raw?: string): number {
  if (raw === undefined) return HERMES_API_CONNECTION_MAX_BUDGET_CENTS;
  if (!/^[1-9][0-9]{0,2}$/.test(raw) || Number(raw) > HERMES_API_CONNECTION_MAX_BUDGET_CENTS) {
    throw new Error("Hermes qualification budget must be an integer from 1 to 200 cents");
  }
  return Number(raw);
}
function captureHermesQualificationBudgetCents(): number {
  const key = "PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS";
  let raw = process.env[key];
  if (raw === undefined) {
    let contents: string | undefined;
    try { contents = readFileSync(new URL("../../.env.runner-e2e.local", import.meta.url), "utf8"); }
    catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; }
    // Read only this public setting. Credentials still load after source and
    // runtime admission in launch.ts. Use the full loader's first-value and
    // quoting rules, without importing other settings into the environment.
    for (const line of contents?.split(/\r?\n/) ?? []) {
      const match = /^(?:export\s+)?PAPERCLIP_RUNNER_E2E_HERMES_BUDGET_CENTS=(.*)$/.exec(line.trim());
      if (!match) continue;
      raw = match[1]!.trim();
      if ((raw.startsWith('"') && raw.endsWith('"')) || (raw.startsWith("'") && raw.endsWith("'"))) raw = raw.slice(1, -1);
      break;
    }
  }
  const budget = resolveHermesQualificationBudgetCents(raw);
  // The late settings loader preserves this captured value. Every child sees
  // the same validated limit that already identifies the launcher's catalog.
  process.env[key] = String(budget);
  return budget;
}
export const HERMES_API_CONNECTION_BUDGET_CENTS = captureHermesQualificationBudgetCents();
export const HERMES_NATIVE_INTERACTION_SUITE = "hermes-native-interactions";
export const HERMES_IMAGE_INPUT_SUITE = "hermes-image-input";
export const HERMES_IMAGE_INPUT_MODEL = "google/gemini-2.5-flash-lite";
export const isHermesConnectionSuite = (suiteId: string) =>
  suiteId === "hermes-api-connections" || suiteId === "hermes-bedrock-connections"
  || suiteId === HERMES_NATIVE_INTERACTION_SUITE || suiteId === HERMES_IMAGE_INPUT_SUITE;
export const isHermesOpenRouterWorkflow = (execution: {
  suite: { id: string }; profile: { qualificationCandidate?: string; credential: string };
}) => execution.suite.id === "extended-harnesses"
  && execution.profile.qualificationCandidate === "hermes" && execution.profile.credential === "OPENROUTER_API_KEY";
export const hermesBedrockConnectionChoice = {
  credential: "AWS_BEARER_TOKEN_BEDROCK",
  model: "us.anthropic.claude-haiku-4-5-20251001-v1:0",
  routing: { kind: "bedrock", protocol: "bedrock", auth: "bearer", region: "us-east-1",
    models: [{ id: "us.anthropic.claude-haiku-4-5-20251001-v1:0" }] },
} as const satisfies { credential: RunnerProfileFixture["credential"]; model: string; routing: AiProviderRouting };

/** The answer is generated by the reviewer and is absent from the model prompt. */
export const hermesNativeAnswerText = (nonce: string) =>
  `constraint-${createHash("sha256").update(`hermes-native-answer:${nonce}`).digest("hex").slice(0, 20)}`;

export const hermesNativeQuestionTask: RunnerTaskFixture = {
  id: "native-question-batch-reconnect", label: "Native question batch survives browser reconnect",
  groups: ["native"], workMode: "standard", flow: "native_question_completion", expectedRunCount: 1,
  automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 300_000, daytona: 420_000 },
  expectedTerminalState: { issue: "done", run: "succeeded" },
  buildTitle: nonce => `Hermes native questions ${nonce}`,
  buildPrompt: nonce => [
    "Call Hermes's native clarify tool exactly once with a questions batch containing these three questions, in this order:",
    JSON.stringify([
      { id: "color", question: "Choose the fixture color", choices: ["Cobalt", "Amber"] },
      { id: "targets", question: "Choose the fixture targets", choices: ["Linux", "Mac"], multi_select: true },
      { id: "notes", question: "Describe the fixture constraint" },
    ]),
    "Wait for the actual native callback. Do not answer your own questions. Do not guess the reviewer's selections. Do not substitute request_human_input. Do not ask through prose. Do not create files or plans. If clarify is unavailable, report that limitation without claiming completion.",
    `After the actual native result arrives, form your summary as HERMES-NATIVE-${nonce}-<exact returned notes user_response>. The notes are supplied by the reviewer; copy that returned text verbatim.`,
    "Use the current turn completion revision and criterion IDs. Call paperclip_finish exactly once with {reportedWorkDisposition:'done',summary:COMPUTED_SUMMARY,completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:'objective',status:'satisfied',evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. COMPUTED_SUMMARY and CURRENT_TURN_REVISION are placeholders for the actual returned-text summary and current revision. Wait for acceptance, then emit only that exact summary as your final answer.",
  ].join("\n"),
  buildVisibleMarker: nonce => `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`,
  buildMatchers: (nonce, execution) => [
    { kind: "message_exact", expected: `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}` },
    { kind: "message_occurrences", expected: `HERMES-NATIVE-${nonce}-${hermesNativeAnswerText(nonce)}`, count: 1 },
    { kind: "issue_status", expected: "done" }, { kind: "run_status", expected: "succeeded" },
    { kind: "runtime_mode", expected: "native" }, { kind: "environment", expected: execution.environment.id },
  ],
};

export const hermesNativeQuestionStopTask: RunnerTaskFixture = {
  ...hermesNativeQuestionTask,
  id: "native-question-batch-stop", label: "Stop an unanswered native question batch",
  flow: "native_question_stop", expectedTerminalState: { issue: "in_progress", run: "cancelled" },
  buildTitle: nonce => `Stop Hermes native questions ${nonce}`,
  buildMatchers: (_nonce, execution) => [
    { kind: "issue_status", expected: "in_progress" }, { kind: "run_status", expected: "cancelled" },
    { kind: "runtime_mode", expected: "native" }, { kind: "environment", expected: execution.environment.id },
  ],
};

/** An undisclosed code exists only in PNG pixels, never in the filename or prompt. */
export function hermesImageChallenge(nonce: string) {
  const code = createHash("sha256").update(`hermes-image-pixels:${nonce}`).digest("hex").slice(0, 8).toUpperCase();
  const glyphs: Record<string, readonly string[]> = {
    "0": ["01110", "10001", "10011", "10101", "11001", "10001", "01110"],
    "1": ["00100", "01100", "00100", "00100", "00100", "00100", "01110"],
    "2": ["01110", "10001", "00001", "00010", "00100", "01000", "11111"],
    "3": ["11110", "00001", "00001", "01110", "00001", "00001", "11110"],
    "4": ["00010", "00110", "01010", "10010", "11111", "00010", "00010"],
    "5": ["11111", "10000", "10000", "11110", "00001", "00001", "11110"],
    "6": ["01110", "10000", "10000", "11110", "10001", "10001", "01110"],
    "7": ["11111", "00001", "00010", "00100", "01000", "01000", "01000"],
    "8": ["01110", "10001", "10001", "01110", "10001", "10001", "01110"],
    "9": ["01110", "10001", "10001", "01111", "00001", "00001", "01110"],
    A: ["01110", "10001", "10001", "11111", "10001", "10001", "10001"],
    B: ["11110", "10001", "10001", "11110", "10001", "10001", "11110"],
    C: ["01111", "10000", "10000", "10000", "10000", "10000", "01111"],
    D: ["11110", "10001", "10001", "10001", "10001", "10001", "11110"],
    E: ["11111", "10000", "10000", "11110", "10000", "10000", "11111"],
    F: ["11111", "10000", "10000", "11110", "10000", "10000", "10000"],
  };
  const width = 640, height = 180, scale = 12;
  const pixels = Buffer.alloc((width * 3 + 1) * height, 255);
  for (let y = 0; y < height; y++) pixels[y * (width * 3 + 1)] = 0;
  [...code].forEach((character, index) => glyphs[character]!.forEach((row, y) => {
    [...row].forEach((value, x) => {
      if (value !== "1") return;
      for (let dy = 0; dy < scale; dy++) for (let dx = 0; dx < scale; dx++) {
        const offset = (48 + y * scale + dy) * (width * 3 + 1) + 1 + (38 + index * 6 * scale + x * scale + dx) * 3;
        pixels.fill(0, offset, offset + 3);
      }
    });
  }));
  const chunk = (type: string, body: Buffer) => {
    const payload = Buffer.concat([Buffer.from(type), body]);
    let crc = 0xffffffff;
    for (const byte of payload) {
      crc ^= byte;
      for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ ((crc & 1) ? 0xedb88320 : 0);
    }
    const length = Buffer.alloc(4), checksum = Buffer.alloc(4);
    length.writeUInt32BE(body.length); checksum.writeUInt32BE((crc ^ 0xffffffff) >>> 0);
    return Buffer.concat([length, payload, checksum]);
  };
  const header = Buffer.alloc(13); header.writeUInt32BE(width); header.writeUInt32BE(height, 4);
  header[8] = 8; header[9] = 2;
  const bytes = Buffer.concat([Buffer.from("89504e470d0a1a0a", "hex"), chunk("IHDR", header), chunk("IDAT", deflateSync(pixels)), chunk("IEND", Buffer.alloc(0))]);
  return { code, bytes, filename: `hermes-image-${nonce}.png`, sha256: createHash("sha256").update(bytes).digest("hex") };
}

export const hermesImageInputTask: RunnerTaskFixture = {
  id: "image-code-complete", label: "Read an uploaded image through native prompt content",
  groups: ["native"], workMode: "standard", flow: "single_turn", expectedRunCount: 1,
  automaticRetryPolicy: "single_attempt", attemptTimeoutMs: { local: 240_000, daytona: 360_000 },
  expectedTerminalState: { issue: "done", run: "succeeded" },
  buildTitle: nonce => `Read the attached Hermes image ${nonce}`,
  buildPrompt: nonce => [
    "Read the eight uppercase hexadecimal characters pictured in the attached image directly from the image input.",
    "Do not read, decode or analyze the image with a tool. Do not guess the characters. Do not create files or plans. If you cannot see the image, report that limitation without claiming completion.",
    `Form the completion summary as HERMES-IMAGE-${nonce}-<exact pictured characters>. The pictured code is not provided in this prompt.`,
    "Call paperclip_finish with reportedWorkDisposition done, that computed summary, the current completion contract revision and satisfied objective criterion, no remaining work, and verification describing the image reading. Wait for acceptance, then emit only that exact summary as your final answer.",
  ].join("\n"),
  buildVisibleMarker: nonce => `HERMES-IMAGE-${nonce}-${hermesImageChallenge(nonce).code}`,
  buildMatchers: (nonce, execution) => [
    { kind: "message_exact", expected: `HERMES-IMAGE-${nonce}-${hermesImageChallenge(nonce).code}` },
    { kind: "message_occurrences", expected: `HERMES-IMAGE-${nonce}-${hermesImageChallenge(nonce).code}`, count: 1 },
    { kind: "issue_status", expected: "done" }, { kind: "run_status", expected: "succeeded" },
    { kind: "runtime_mode", expected: "native" }, { kind: "environment", expected: execution.environment.id },
  ],
};

/** Wrong scope/bytes or a tool-based substitute cannot qualify native image input. */
export function gradeHermesImageInput(input: {
  nonce: string; companyId: string; issueId: string; runId: string;
  attachments: readonly unknown[]; downloadedBytes: Buffer; events: readonly unknown[];
}) {
  const expected = hermesImageChallenge(input.nonce), attachment = record(input.attachments[0]);
  const tools = input.events.map(record).filter(row => row.eventType === "tool.execution.started")
    .map(row => ({ row, event: record(record(row.payload).prpEvent) }));
  const allowed = new Set(["paperclip_finish", "get_task_context", "set_task_title", "report_progress"]);
  return [
    { id: "one-authorized-image", passed: present(input.companyId) && present(input.issueId)
      && input.attachments.length === 1 && present(attachment.id) && attachment.companyId === input.companyId
      && attachment.issueId === input.issueId && attachment.originalFilename === expected.filename
      && attachment.contentType === "image/png" && attachment.byteSize === expected.bytes.length && attachment.sha256 === expected.sha256 },
    { id: "uploaded-image-bytes", passed: input.downloadedBytes.equals(expected.bytes)
      && createHash("sha256").update(input.downloadedBytes).digest("hex") === expected.sha256 },
    { id: "native-image-without-file-tool-substitution", passed: present(input.runId) && tools.length > 0
      && tools.some(({ event }) => record(event.payload).name === "paperclip_finish")
      && tools.every(({ row, event }) => row.runId === input.runId && event.schema === "paperclip.prp.event.v1"
        && event.runId === input.runId && event.sourceKind === "runner" && present(event.sourceInstanceId)
        && present(event.normalizedSessionId) && record(event.payload).namespace === "paperclip"
        && allowed.has(String(record(event.payload).name))) },
  ];
}

/** Preserve the pinned native form; semantic tools cannot satisfy this oracle. */
export function hasHermesNativeQuestionBatch(value: unknown): boolean {
  const set = record(value), questions = Array.isArray(set.questions) ? set.questions.map(record) : [];
  const expected = [
    ["q0", "Choose the fixture color", "single_select", ["Cobalt", "Amber"]],
    ["q1", "Choose the fixture targets", "multi_select", ["Linux", "Mac"]],
    ["q2", "Describe the fixture constraint", "text", []],
  ] as const;
  return set.schema === "paperclip.question_set.v1" && questions.length === expected.length
    && questions.every((question, index) => {
      const [id, prompt, mode, labels] = expected[index]!;
      const options = Array.isArray(question.options) ? question.options.map(record) : [];
      return question.id === id && question.prompt === prompt && question.answerMode === mode && question.required === true
        && options.length === labels.length && options.every((option, optionIndex) => option.id === `o${optionIndex}`
          && [labels[optionIndex], `${labels[optionIndex]} (Recommended)`].includes(String(option.label)))
        && (labels.length === 0 || record(question.customAnswer).enabled === true);
    });
}

/** Require one exact ordered, post-write delivery receipt for this native callback. */
export function hasExactHermesNativeQuestionResponse(input: {
  events: readonly unknown[]; runId: string; turnId: string; requestId: string; questionSet: unknown; response: unknown;
}): boolean {
  if (![input.runId, input.turnId, input.requestId].every(present) || !hasHermesNativeQuestionBatch(input.questionSet)
    || record(input.response).schema !== "paperclip.question_response.v1") return false;
  const rows = input.events.map(record).map(row => ({ row, event: record(record(row.payload).prpEvent) }));
  if (rows.filter(({ row, event }) => row.runId === input.runId && event.eventType === "runtime_request.created"
    && hasAcpxNativeOrigin(record(record(event.payload).request).origin, "hermes", "_hermes/ask_questions")).length !== 1) return false;
  const relevant = rows
    .filter(({ event }) => event.eventType === "runtime_request.created"
      ? record(record(event.payload).request).requestId === input.requestId
      : ["runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled"].includes(String(event.eventType))
        && record(event.payload).requestId === input.requestId);
  if (relevant.some(({ row, event }) => row.runId !== input.runId || event.runId !== input.runId
    || event.turnId !== input.turnId || event.schema !== "paperclip.prp.event.v1"
    || event.schemaVersion !== 1 || row.protocolSchemaVersion !== 1)) return false;
  const created = relevant.filter(({ event }) => event.eventType === "runtime_request.created");
  const outcomes = relevant.filter(({ event }) => event.eventType !== "runtime_request.created");
  if (created.length !== 1 || outcomes.length !== 1) return false;
  const request = record(record(created[0]!.event.payload).request), outcome = outcomes[0]!.event;
  const origin = created[0]!.event;
  return origin.sourceKind === "runner" && [origin.sourceInstanceId, origin.normalizedSessionId].every(present)
    && outcome.sourceKind === "runner" && outcome.sourceInstanceId === origin.sourceInstanceId
    && outcome.normalizedSessionId === origin.normalizedSessionId
    && hasAcpxNativeOrigin(request.origin, "hermes", "_hermes/ask_questions")
    && request.type === "input" && request.status === "pending" && request.turnId === input.turnId
    && isDeepStrictEqual(request.input, input.questionSet)
    && outcome.eventType === "runtime_request.resolved" && record(outcome.payload).action === "submit"
    && record(outcome.payload).turnId === input.turnId && isDeepStrictEqual(record(outcome.payload).response, input.response)
    && Number.isSafeInteger(created[0]!.event.sourceSeq) && Number.isSafeInteger(outcome.sourceSeq)
    && (created[0]!.event.sourceSeq as number) >= 0 && (outcome.sourceSeq as number) > (created[0]!.event.sourceSeq as number);
}

/** A provider cancellation must close the retained unanswered callback, not complete it. */
export function hasHermesNativeQuestionStop(input: {
  events: readonly unknown[]; runId: string; turnId: string; requestId: string; questionSet: unknown;
  pendingEvent: unknown; run: unknown; issue: unknown; companyId: string; callerUserId: string;
  browserResponse: unknown;
}): boolean {
  if (![input.runId, input.turnId, input.requestId, input.companyId, input.callerUserId].every(present)
    || !hasHermesNativeQuestionBatch(input.questionSet)) return false;
  const pending = record(input.pendingEvent), run = record(input.run), issue = record(input.issue);
  const rows = input.events.map(record).map(row => ({ row, event: record(record(row.payload).prpEvent) }));
  const created = rows.filter(({ event }) => event.eventType === "runtime_request.created"
    && hasAcpxNativeOrigin(record(record(event.payload).request).origin, "hermes", "_hermes/ask_questions"));
  const outcomes = rows.filter(({ event }) => ["runtime_request.resolved", "runtime_request.expired", "runtime_request.cancelled"].includes(String(event.eventType))
    && record(event.payload).requestId === input.requestId);
  const terminals = rows.filter(({ event }) => ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(String(event.eventType)));
  if (created.length !== 1 || outcomes.length !== 1 || terminals.length !== 1) return false;
  const start = created[0]!, closed = outcomes[0]!, terminal = terminals[0]!;
  const request = record(record(start.event.payload).request), closure = record(closed.event.payload), end = record(terminal.event.payload);
  if (!isDeepStrictEqual(start.event, pending) || !isDeepStrictEqual(request.input, input.questionSet)
    || request.requestId !== input.requestId || request.turnId !== input.turnId || request.type !== "input" || request.status !== "pending"
    || !present(request.itemId) || !present(request.requestKind)
    || ![pending.sourceInstanceId, pending.normalizedSessionId].every(present)) return false;
  if ([start, closed, terminal].some(({ row, event }) => row.runId !== input.runId || row.protocolSchemaVersion !== 1
    || event.runId !== input.runId || event.turnId !== input.turnId || event.schema !== "paperclip.prp.event.v1" || event.schemaVersion !== 1
    || event.sourceKind !== "runner" || event.sourceInstanceId !== pending.sourceInstanceId || event.normalizedSessionId !== pending.normalizedSessionId
    || !Number.isSafeInteger(event.sourceSeq) || (event.sourceSeq as number) < 0)) return false;
  if (closed.event.eventType !== "runtime_request.cancelled" || closure.turnId !== input.turnId || closure.itemId !== request.itemId
    || closure.requestKind !== request.requestKind || closure.response !== undefined || closure.replayAllowed === true
    || (closure.action !== undefined && closure.action !== "cancel") || !present(closure.reason)
    || terminal.event.eventType !== "turn.cancelled" || end.status !== "cancelled" || end.error !== null
    || (closed.event.sourceSeq as number) <= (start.event.sourceSeq as number)
    || (terminal.event.sourceSeq as number) <= (closed.event.sourceSeq as number)) return false;
  if (rows.some(({ event }) => event.sourceKind === "runner" && event.sourceInstanceId === pending.sourceInstanceId
    && typeof event.sourceSeq === "number" && event.sourceSeq > (terminal.event.sourceSeq as number)
    && (String(event.eventType).startsWith("tool.execution.") || event.eventType === "runtime_request.created"))) return false;
  const result = record(run.resultJson), stop = record(result.nativeCancellation), browser = record(input.browserResponse);
  return run.id === input.runId && run.companyId === input.companyId && run.issueId === issue.id && run.runtimeMode === "native" && run.status === "cancelled"
    && issue.companyId === input.companyId && issue.status === "in_progress"
    && result.cancelledByActorType === "user" && result.cancelledByUserId === input.callerUserId
    && stop.schema === "paperclip.native-cancellation.v1" && stop.companyId === input.companyId && stop.runId === run.id && stop.issueId === issue.id
    && stop.scope === "run" && stop.dispatched === true && stop.dispatchState === "acknowledged" && stop.reasonCode === "cancellation_run_only"
    && isDeepStrictEqual(stop.effects, ["release_run_resources"]) && present(stop.intentId)
    && present(stop.intentAuditId) && present(stop.acknowledgementAuditId) && stop.intentAuditId !== stop.acknowledgementAuditId
    && browser.id === run.id && record(record(browser.resultJson).nativeCancellation).intentId === stop.intentId;
}

/** Bind the terminal question card to the retained unanswered native request. */
export function hasHermesNativeQuestionStopCard(input: { card: unknown; originalCard: unknown; runId: string }): boolean {
  const card = record(input.card), original = record(input.originalCard);
  return [input.runId, original.id, original.companyId, original.issueId].every(present)
    && original.sourceRunId === input.runId && original.status === "pending" && !original.result
    && original.kind === "ask_user_questions" && original.continuationPolicy === "none"
    && ["id", "companyId", "issueId", "sourceRunId", "kind", "continuationPolicy"].every(key => card[key] === original[key])
    && card.status === "expired" && card.resolvedByRunId === input.runId
    && card.resolvedByUserId === null && card.resolvedByAgentId === null
    && present(record(original.payload).runtimeRequestId) && isDeepStrictEqual(card.payload, original.payload)
    && isDeepStrictEqual(card.result, {
      version: 1, cancelled: true, cancellationReason: "Native question cancelled", answers: [], summaryMarkdown: null,
    });
}

/** The browser owns Stop; public receipts and read-only OS observations grade it. */
export async function runHermesNativeQuestionStop(input: {
  page: Page; api: RunnerApi; fixtures: LiveFixtureValues; execution: MatrixExecution; nonce: string;
  workspacePath: string; deadlineAt: number; callerUserId: string;
  observe(issue: Record<string, any>, runs: Record<string, any>[]): void;
  capture(id: string, label: string, file: string): Promise<void>;
  evidence(name: string, value: unknown): Promise<void>;
  registerCleanupAssertion(callback: () => Promise<Array<{ id: string; passed: boolean; detail: string }>>): void;
}) {
  const { api, page, fixtures, execution, nonce, deadlineAt } = input;
  if (execution.environment.id !== "local" || execution.suite.id !== HERMES_NATIVE_INTERACTION_SUITE
    || execution.profile.qualificationCandidate !== "hermes" || execution.task.id !== hermesNativeQuestionStopTask.id
    || !fixtures.aiConnection || !present(input.callerUserId)) throw new Error("Unadmitted Hermes native question Stop cell");
  const checks: Array<{ id: string; passed: boolean; detail: string }> = [];
  const check = (id: string, passed: boolean, detail: string) => {
    checks.push({ id, passed, detail }); if (!passed) throw new Error(`Hermes native question Stop: ${detail}`);
  };
  type StopRun = Parameters<typeof gradeHermesApiConnection>[0]["runs"][number] & Record<string, any>;
  let issue: Record<string, any> = {}, runs: StopRun[] = [];
  const observer = observeRunProcesses();
  let authority: RunProcessAuthority | undefined, processIdentityChanged = false;
  let processes = observer.sample(), complete = false;
  const sample = () => { processes = observer.sample(authority); };
  const timer = setInterval(() => { try { sample(); } catch { processIdentityChanged = true; } }, 100);
  const load = async () => {
    if (issue.id) issue = await api.get(`/api/issues/${issue.id}`);
    const list = await api.get<Array<{ id: string }>>(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`);
    runs = await Promise.all(list.map(run => api.get<StopRun>(`/api/heartbeat-runs/${run.id}`)));
    if (runs.length > 1) throw new Error("Hermes native question Stop dispatched an extra run");
    const run: Record<string, any> = runs[0] ?? {};
    if (run.processPid && run.processGroupId && run.processStartedAt) {
      const current = { pid: run.processPid, groupId: run.processGroupId, startedAt: run.processStartedAt, runId: run.id };
      if (authority && !isDeepStrictEqual(authority, current)) processIdentityChanged = true;
      authority ??= current;
    }
    sample(); input.observe(issue, runs);
    const events = run.id ? await collectRunEvents<Record<string, any>>((afterSeq, limit) => api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`)) : [];
    const interactions = issue.id ? await api.get<Record<string, any>[]>(`/api/issues/${issue.id}/interactions`) : [];
    return { run, issue, events, interactions };
  };
  let identity: Omit<Parameters<typeof hasHermesNativeQuestionStop>[0], "events" | "run" | "issue"> | undefined;
  let originalCard: unknown;
  const retired = () => processes.captured && processes.journal.length >= 2 && processes.live.length === 0 && !processIdentityChanged;
  input.registerCleanupAssertion(async () => {
    try {
      const state = await load();
      if (processes.captured && processes.live.length > 0) {
        await pollUntil({ label: "Hermes native Stop cleanup retirement", deadlineAt: Date.now() + 5_000, intervalMs: 100,
          load: async () => { sample(); return processes; }, accept: retired });
      }
      const passed = complete && retired() && runs.length === 1 && state.interactions.length === 1
        && hasHermesNativeQuestionStopCard({ card: state.interactions[0], originalCard, runId: identity?.runId ?? "" })
        && Boolean(identity && hasHermesNativeQuestionStop({ ...identity, ...state }));
      await input.evidence("hermes-native-stop-cleanup.json", { complete, processes, processIdentityChanged, passed });
      return [{ id: "hermes-native-stop-retired-through-cleanup", passed, detail: "The exact per-turn owner and observed descendants remain retired; no late answer or follow-up appears." }];
    } finally { clearInterval(timer); }
  });
  const project = await api.post<Record<string, any>>(`/api/companies/${fixtures.company.id}/projects`, {
    name: `Hermes native Stop ${nonce}`, executionWorkspacePolicy: { enabled: true, defaultMode: "shared_workspace", sharedWorkspaceConcurrency: "serialize",
      allowIssueOverride: false, environmentId: fixtures.environment.id, workspaceStrategy: { type: "project_primary" } },
    workspace: { name: "Primary", sourceType: "local_path", cwd: input.workspacePath, isPrimary: true },
  });
  const created = await createTaskThroughUi({ page, issuePrefix: fixtures.company.issuePrefix!, agentName: fixtures.agent.name,
    title: execution.task.buildTitle(nonce), prompt: execution.task.buildPrompt(nonce), workMode: "standard", projectName: project.name, requireExplicitTitle: true });
  issue = await api.get(`/api/issues/${created.issueId}`);
  const pending = await pollUntil({ label: "unanswered native Hermes question before browser Stop", deadlineAt, intervalMs: 200, load,
    reject: state => ["succeeded", "failed", "timed_out", "cancelled"].includes(state.run.status) ? "Hermes ended before the pending native Stop boundary" : undefined,
    accept: state => state.run.status === "running" && state.interactions.length === 1 && state.interactions[0]?.status === "pending"
      && state.events.some(row => record(record(row.payload).prpEvent).eventType === "runtime_request.created"
        && hasAcpxNativeOrigin(record(record(record(record(row.payload).prpEvent).payload).request).origin, "hermes", "_hermes/ask_questions")),
  });
  const card = pending.interactions[0]!, questionSet = card.payload?.questionSet;
  const pendingEvents = pending.events.map(row => record(record(row.payload).prpEvent)).filter(event => event.eventType === "runtime_request.created"
    && hasAcpxNativeOrigin(record(record(event.payload).request).origin, "hermes", "_hermes/ask_questions"));
  const pendingEvent = pendingEvents[0]!, request = record(record(pendingEvent.payload).request);
  check("retained-unanswered-native-callback", pendingEvents.length === 1 && hasHermesNativeQuestionBatch(questionSet)
    && isDeepStrictEqual(request.input, questionSet) && request.requestId === card.payload?.runtimeRequestId
    && card.sourceRunId === pending.run.id && card.continuationPolicy === "none" && !card.result
    && !record(pending.run.resultJson).nativeCancellation, "One complete native callback remains unanswered in its original active turn before Stop.");
  originalCard = card;
  await input.evidence("hermes-native-stop-pending.json", { card, event: pendingEvent, processes, checks });
  await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`);
  await page.getByRole("radio", { name: questionSet.questions[0].options[0].label, exact: true }).last().waitFor({ state: "visible" });
  await input.capture("hermes-native-stop-pending", "Unanswered native Hermes question before browser Stop", "hermes-native-stop-pending.png");
  check("owned-live-process-tree", processes.captured && processes.journal.length >= 2 && processes.live.length >= 2 && !processIdentityChanged,
    "The public per-turn process identity binds a live owner and descendants before Stop.");
  const fresh = await load();
  check("fresh-pending-boundary", fresh.run.status === "running" && fresh.interactions.length === 1 && fresh.interactions[0]?.id === card.id
    && fresh.interactions[0]?.status === "pending" && !fresh.interactions[0]?.result
    && isDeepStrictEqual(fresh.interactions[0]?.payload, card.payload), "The same unanswered callback is still pending immediately before the browser click.");
  const cancelPath = `/api/heartbeat-runs/${pending.run.id}/cancel`;
  const responsePromise = page.waitForResponse(response => new URL(response.url()).pathname === cancelPath && response.request().method() === "POST",
    { timeout: Math.max(1, deadlineAt - Date.now()) });
  await page.getByRole("button", { name: "Stop", exact: true }).last().click();
  const response = await responsePromise;
  check("browser-stop-response", response.status() === 200, "The browser Stop control calls the exact active run's public cancellation API.");
  const browserResponse = await response.json();
  identity = { runId: pending.run.id, turnId: pendingEvent.turnId as string, requestId: request.requestId as string,
    questionSet, pendingEvent, companyId: fixtures.company.id, callerUserId: input.callerUserId, browserResponse };
  await input.evidence("hermes-native-stop-browser-response.json", { identity, checks });
  const stopped = await pollUntil({ label: "Hermes native question cancellation and owned Stop acknowledgement", deadlineAt, intervalMs: 200, load,
    reject: state => ["succeeded", "failed", "timed_out"].includes(state.run.status) ? "Stop did not produce a cancelled native turn" : undefined,
    accept: state => state.interactions.length === 1
      && hasHermesNativeQuestionStopCard({ card: state.interactions[0], originalCard, runId: identity!.runId })
      && hasHermesNativeQuestionStop({ ...identity!, ...state }),
  });
  check("exact-native-cancellation", hasHermesNativeQuestionStop({ ...identity, ...stopped }),
    "One ordered cancellation closes the retained native request and turn, with the browser's exact audited run Stop acknowledgement.");
  let staleRejected = false;
  try {
    await api.postSensitive(`/api/issues/${issue.id}/interactions/${card.id}/respond`, { answers: [
      { questionId: "q0", optionIds: ["o0"] }, { questionId: "q1", optionIds: ["o0"] }, { questionId: "q2", optionIds: [], otherText: "Late answer" },
    ] });
  } catch (error) {
    staleRejected = error instanceof Error && error.message === `Sensitive POST /api/issues/${issue.id}/interactions/${card.id}/respond returned 409; response body withheld`;
    if (!staleRejected) throw error;
  }
  check("stale-answer-refused", staleRejected, "The public response API rejects a later answer with HTTP 409.");
  await pollUntil({ label: "Hermes Stop owner and descendant retirement", deadlineAt: Math.min(deadlineAt, Date.now() + 10_000), intervalMs: 100,
    load: async () => { sample(); return processes; }, accept: retired });
  check("owned-process-tree-retired", retired(), "The exact owner and observed descendants retire before cleanup starts.");
  await page.reload();
  await page.getByTestId("issue-detail-header").getByRole("button", { name: "Change status (current: In Progress)", exact: true }).waitFor({ state: "visible" });
  await page.getByTestId("task-chat-thread").getByTestId("task-chat-collapsible-marker").filter({ has: page.getByText("Run cancelled", { exact: true }) }).waitFor({ state: "visible" });
  await page.getByTestId("task-chat-history-loading").waitFor({ state: "hidden" });
  check("question-unanswerable-after-reload", await page.getByRole("radio", { name: questionSet.questions[0].options[0].label, exact: true }).count() === 0,
    "The loaded cancelled turn offers no native question answer control.");
  const final = await load();
  check("one-unfinished-cancelled-run", runs.length === 1 && final.run.status === "cancelled" && final.issue.status === "in_progress"
    && final.interactions.length === 1 && hasHermesNativeQuestionStopCard({ card: final.interactions[0], originalCard, runId: identity.runId })
    && hasHermesNativeQuestionStop({ ...identity, ...final }), "Stop leaves one unfinished task and one cancelled run, without an answer or follow-up.");
  const connectionChecks = gradeHermesApiConnection({ companyId: fixtures.company.id, agentId: fixtures.agent.id,
    issueId: issue.id, connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider,
    expectedResponsibleUserId: input.callerUserId, model: execution.profile.model, runs, expectedRunStatus: "cancelled" });
  connectionChecks.forEach(value => check(value.id, value.passed, "Public native model and account attribution remain correct after Stop."));
  let billing: Awaited<ReturnType<typeof captureHermesOpenRouterSettlement>> | undefined;
  await pollUntil({ label: "cancelled Hermes run billing settlement", deadlineAt: Math.min(deadlineAt, Date.now() + 30_000), intervalMs: 200,
    load: async () => billing = await captureHermesOpenRouterSettlement({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id,
      issueId: issue.id, runId: final.run.id, expectedRunStatus: "cancelled" }), accept: receipt => receipt.checks.every(value => value.passed) });
  billing!.checks.forEach(value => check(value.id, value.passed, "Cancelled native usage settles reported cost while the company and agent retain healthy budgets."));
  await input.evidence("hermes-openrouter-settlement.json", billing);
  await input.capture("final-state", "Cancelled native question remains unanswerable", "final-state.png");
  await input.evidence("api-state.json", { ...final, runs, run: final.run, checks, processes, nativeStop: identity });
  complete = true;
  return { issue, runs, checks };
}

/** Establish the intended personal-account user independently, before a paid task. */
export async function captureHermesApiAccountOwner(input: {
  api: { get<T>(path: string): Promise<T> }; companyId: string; connectionId: string; provider: string;
  expectedRouting?: AiProviderRouting; expectedGrantId?: string;
}) {
  const listed = await input.api.get<{ currentUserId: string; connections: {
    id: string; companyId: string; provider: string; method: string; ownership: string; ownerUserId?: string; status: string;
    routing?: AiProviderRouting; grantId?: string;
  }[] }>(`/api/companies/${input.companyId}/ai-connections`);
  const matches = listed.connections.filter(connection => connection.id === input.connectionId);
  const selected = matches[0];
  return { expectedResponsibleUserId: selected?.ownerUserId ?? null,
    ...(input.expectedRouting ? { routing: { expected: input.expectedRouting, observed: selected?.routing ?? null } } : {}),
    checks: [
    { id: "selected-account-public-readback", passed: matches.length === 1 && selected?.companyId === input.companyId && selected.provider === input.provider && selected.method === "api_key" && selected.ownership === "personal" && selected.status === "connected" },
    { id: "authenticated-caller-known", passed: present(listed.currentUserId) },
    { id: "selected-account-owned-by-caller", passed: present(selected?.ownerUserId) && selected?.ownerUserId === listed.currentUserId },
    ...(input.expectedRouting ? [{ id: "selected-account-routing", passed: isDeepStrictEqual(selected?.routing, input.expectedRouting) }] : []),
    ...(input.expectedGrantId !== undefined ? [{ id: "selected-account-grant", passed: present(input.expectedGrantId) && selected?.grantId === input.expectedGrantId }] : []),
  ] };
}

/** Read both public budgets before creating the paid task. Unknown usage stays unknown. */
export async function captureHermesApiBudgets(input: {
  api: { get<T>(path: string): Promise<T> }; companyId: string; agentId: string;
}) {
  const [company, agent] = await Promise.all([
    input.api.get<{ id: string; budgetMonthlyCents: unknown }>(`/api/companies/${input.companyId}`),
    input.api.get<{ id: string; companyId: string; budgetMonthlyCents: unknown }>(`/api/agents/${input.agentId}`),
  ]);
  return { budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, checks: [
    { id: "company-budget", passed: company.id === input.companyId && company.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS },
    { id: "agent-budget", passed: agent.id === input.agentId && agent.companyId === input.companyId && agent.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS },
  ] };
}

/** Observe settled billing and budget health before fixture cleanup pauses the agent. */
export async function captureHermesOpenRouterSettlement(input: {
  api: { get<T>(path: string): Promise<T> }; companyId: string; agentId: string; issueId: string; runId: string;
  expectedRunStatus?: "succeeded" | "cancelled";
}) {
  const [company, agent, run] = await Promise.all([
    input.api.get<Record<string, unknown>>(`/api/companies/${input.companyId}`),
    input.api.get<Record<string, unknown>>(`/api/agents/${input.agentId}`),
    input.api.get<Record<string, unknown>>(`/api/heartbeat-runs/${input.runId}`),
  ]);
  const usage = record(run.usageJson), provenance = record(usage.pricingProvenance);
  const cost = usage.costUsd, exact = usage.costUsdExact;
  return { observation: {
    company: { id: company.id, status: company.status, pauseReason: company.pauseReason, budgetMonthlyCents: company.budgetMonthlyCents },
    agent: { id: agent.id, companyId: agent.companyId, status: agent.status, pauseReason: agent.pauseReason, budgetMonthlyCents: agent.budgetMonthlyCents },
    run: { id: run.id, companyId: run.companyId, agentId: run.agentId, issueId: run.issueId, status: run.status,
      costAccountingPending: run.costAccountingPending, costAccountedAt: run.costAccountedAt,
      usage: Object.fromEntries(["provider", "biller", "billingType", "costStatus", "costUsd", "costUsdExact", "inputTokens", "outputTokens", "accountingReceiptReady", "pricingProvenance"].map(key => [key, usage[key]])) },
  }, checks: [
    { id: "billing-observation-scope", passed: company.id === input.companyId && agent.id === input.agentId && agent.companyId === input.companyId
      && run.id === input.runId && run.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId },
    { id: "settled-openrouter-reported-cost", passed: run.status === (input.expectedRunStatus ?? "succeeded") && run.costAccountingPending === false && present(run.costAccountedAt)
      && usage.accountingReceiptReady === true && usage.biller === "openrouter" && usage.billingType === "metered_api" && usage.costStatus === "reported"
      && provenance.source === "provider_reported" && provenance.version === "hermes-openrouter-wire/v1"
      && typeof cost === "number" && Number.isFinite(cost) && cost >= 0 && typeof exact === "string" && /^(0|[1-9][0-9]{0,6})\.[0-9]{9}$/.test(exact) && Number(exact) === cost
      && typeof usage.inputTokens === "number" && Number.isSafeInteger(usage.inputTokens) && usage.inputTokens >= 0
      && typeof usage.outputTokens === "number" && Number.isSafeInteger(usage.outputTokens) && usage.outputTokens >= 0 && usage.inputTokens + usage.outputTokens > 0 },
    { id: "budget-health-after-settlement", passed: company.status === "active" && agent.status === "idle" && agent.pauseReason === null
      && company.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS && agent.budgetMonthlyCents === HERMES_API_CONNECTION_BUDGET_CENTS },
  ] };
}

/** Grade public run/account/model metadata; a model's completion claim cannot supply it. */
export function gradeHermesApiConnection(input: {
  companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; expectedResponsibleUserId: string;
  accountMode?: "responsible_user" | "delegated"; expectedGrantId?: string;
  expectedRunStatus?: "succeeded" | "cancelled";
  runs: readonly {
    companyId: string; agentId: string; status: string; runtimeMode?: string;
    issueId?: string | null; responsibleUserId?: string | null;
    contextSnapshot?: Record<string, unknown> | null; runnerProfileJson?: Record<string, unknown> | null;
  }[];
}) {
  const run = input.runs[0];
  const context = record(run?.contextSnapshot), account = record(context.aiConnection);
  const identity = record(record(record(run?.runnerProfileJson).sessionCheckpoint).providerIdentity);
  const provider = record(record(record(run?.runnerProfileJson).nativeExecutionInput).provider);
  return [
    { id: input.expectedRunStatus === "cancelled" ? "one-cancelled-native-run" : "one-successful-native-run",
      passed: input.runs.length === 1 && run?.status === (input.expectedRunStatus ?? "succeeded") && run.runtimeMode === "native" },
    { id: "company-agent-task-scope", passed: run?.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId },
    { id: "selected-managed-api-account", passed: account.connectionId === input.connectionId && account.provider === input.provider && account.method === "api_key" && account.mode === (input.accountMode ?? "responsible_user") },
    ...(input.expectedGrantId !== undefined ? [{ id: "selected-managed-account-grant", passed: present(input.expectedGrantId) && account.grantId === input.expectedGrantId }] : []),
    { id: "responsible-user-attribution", passed: present(input.expectedResponsibleUserId) && run?.responsibleUserId === input.expectedResponsibleUserId && account.responsibleUserId === input.expectedResponsibleUserId },
    { id: "native-hermes-provider", passed: provider.kind === "acpx" && provider.agent === "hermes" && provider.model === input.model },
    { id: "exact-native-model", passed: identity.kind === "acpx" && identity.requestedModel === input.model && identity.effectiveModel === input.model },
  ];
}

/** Explicit authenticated catalog choices; none is a production default or live qualification. */
export const hermesApiConnectionChoices = [
  { provider: "anthropic", credential: "ANTHROPIC_API_KEY", model: "claude-haiku-4-5-20251001" },
  { provider: "openai", credential: "OPENAI_API_KEY", model: "gpt-5.6-luna" },
  { provider: "xai", credential: "XAI_API_KEY", model: "grok-4.7" },
  // Google restricts 2.5 models to prior users even when they appear in the catalog.
  // https://ai.google.dev/gemini-api/docs/deprecations
  { provider: "google", credential: "GEMINI_API_KEY", model: "gemini-3.8-flash" },
] as const satisfies readonly { provider: string; credential: RunnerProfileFixture["credential"]; model: string }[];

export const hermesApiConnectionDefinitionDigest = createHash("sha256").update(
  ["hermes-api-connections.ts", "acpx-native-origin.ts", "native-local-fixtures.ts", "user-actions.ts", "live-fixtures.ts", "harness-env.ts", "runner.spec.ts"]
    .map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n"),
).digest("hex");