test(hermes): verify the expected account owner independently

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/plans/2026-10-06-hermes-native-runner.md

@@ -687,3 +687,25 @@ verification. The earlier image digest and live results retain their original
 sources and closure; they are not results for the corrected runtime. The stacked
 qualification commits were replayed without changing their patches. Both updated
 PR heads require fresh checks and review. Hermes remains pending qualification.
+
+The subsequent qualification review found an independent-user evidence gap:
+the original responsible-user check compared two run fields with each other.
+That permits a consistent foreign user to pass. New fixture admission reads the
+selected account and authenticated caller through the public Connections API
+before creating a paid task, verifies company/account/provider/method/ownership
+and connected status, and retains the expected owner. Both run fields must equal
+that owner; missing, duplicate, foreign-owner and consistent-wrong-run-user cases
+are calibrated as failures. The historical API results keep their original
+grader provenance; they do not acquire this pre-turn owner receipt retroactively.
+An additional assessment of the retained public task records independently
+checks the immutable task creator, task responsible user and both run attribution
+fields. All five successful API cases pass that comparison with zero provider
+calls. This retained-task assessment remains separate from the new fixture's
+pre-turn account-owner admission. The correction passed all 106 affected tests,
+Product E2E TypeScript and diff checks.
+
+The fresh Linux native CI attempt at the replayed head failed before provisioning
+because GitHub returned HTTP 429 for the pinned source archive. No native fixture
+or provider execution occurred. This is retained as a download infrastructure
+failure; it is not evidence of a closure mismatch. The separate credential-free
+AWS build continues against its recorded immutable source.

tests/runner-e2e/README.md

@@ -1525,6 +1525,11 @@ the ordinary completion, screenshot, token/cost and cleanup evidence.
 Each cell admits one attempt and configures 200-cent company and agent budgets.
 Public budget readback must pass before task creation. Unpriced model usage
 remains unknown; this configured limit is not an exact billing receipt.
+Before creating the task, public account readback must identify the selected
+connected personal account, its company and owner, and the authenticated caller.
+Both run attribution fields must match that independently recorded owner.
+The pre-turn owner receipt is retained; agreement between two run fields alone
+cannot establish the expected user.
 
 ```sh
 pnpm test:e2e:runner -- --list --suite hermes-api-connections

tests/runner-e2e/catalog.ts

@@ -1210,7 +1210,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     expectedMatrixSize: 10,
     definitionMetadata: {
       version: 1, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07",
-      accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1,
+      accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, expectedUserSource: "public-account-owner-before-task",
       budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1,
       coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest,
     },

tests/runner-e2e/hermes-api-connections.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest";
 import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js";
 import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
 import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
-import { captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js";
+import { captureHermesApiAccountOwner, captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js";
 
 describe("Hermes managed API connection qualification", () => {
   const suite = runnerSuites.find(s => s.id === "hermes-api-connections")!;
@@ -32,7 +32,7 @@ describe("Hermes managed API connection qualification", () => {
     expect(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, sourceDigest: "changed-account-selection" } })).not.toBe(suiteDefinitionHash(suite));
   });
   const valid = {
-    companyId: "company", agentId: "agent", issueId: "task", connectionId: "account", provider: "xai", model: "grok-4.7",
+    companyId: "company", agentId: "agent", issueId: "task", connectionId: "account", provider: "xai", model: "grok-4.7", expectedResponsibleUserId: "user",
     runs: [{ companyId: "company", agentId: "agent", issueId: "task", status: "succeeded", runtimeMode: "native", responsibleUserId: "user",
       contextSnapshot: { aiConnection: { connectionId: "account", provider: "xai", method: "api_key", mode: "responsible_user", responsibleUserId: "user" } },
       runnerProfileJson: { nativeExecutionInput: { provider: { kind: "acpx", agent: "hermes", model: "grok-4.7" } },
@@ -41,6 +41,23 @@ describe("Hermes managed API connection qualification", () => {
   it("accepts independently observed account/model metadata", () => {
     expect(gradeHermesApiConnection(valid).every(check => check.passed)).toBe(true);
   });
+  it.each(["valid", "missing", "duplicate", "company", "provider", "method", "ownership", "status", "owner", "caller"])("establishes the expected user from public account readback: %s", async fault => {
+    const account = { id: "account", companyId: "company", provider: "xai", method: "api_key", ownership: "personal", ownerUserId: "user", status: "connected" };
+    if (fault === "company") account.companyId = "foreign";
+    if (fault === "provider") account.provider = "foreign";
+    if (fault === "method") account.method = "foreign";
+    if (fault === "ownership") account.ownership = "foreign";
+    if (fault === "status") account.status = "foreign";
+    if (fault === "owner") account.ownerUserId = "foreign";
+    const receipt = await captureHermesApiAccountOwner({ companyId: "company", connectionId: "account", provider: "xai", api: {
+      async get<T>(url: string) {
+        expect(url).toBe("/api/companies/company/ai-connections");
+        return { currentUserId: fault === "caller" ? "" : "user", connections: fault === "missing" ? [] : fault === "duplicate" ? [account, account] : [account] } as T;
+      },
+    } });
+    expect(receipt.checks.every(check => check.passed)).toBe(fault === "valid");
+    if (fault === "valid") expect(receipt.expectedResponsibleUserId).toBe("user");
+  });
   it.each(["valid", "company-budget", "agent-budget", "company-scope", "agent-scope", "agent-company"])("checks %s through public budget readback before a paid task", async fault => {
     const company = { id: "company", budgetMonthlyCents: fault === "company-budget" ? 0 : 200 };
     const agent = { id: "agent", companyId: "company", budgetMonthlyCents: fault === "agent-budget" ? 0 : 200 };
@@ -54,7 +71,7 @@ describe("Hermes managed API connection qualification", () => {
     expect(new Set(paths)).toEqual(new Set(["/api/companies/company", "/api/agents/agent"]));
     expect(receipt.checks.every(check => check.passed)).toBe(fault === "valid");
   });
-  it.each(["company", "task", "account", "provider", "method", "user", "model", "harness", "missing", "extra-run"])("rejects %s evidence even with a successful answer", fault => {
+  it.each(["company", "task", "account", "provider", "method", "user", "consistent-foreign-user", "missing-expected-user", "model", "harness", "missing", "extra-run"])("rejects %s evidence even with a successful answer", fault => {
     const wrong = structuredClone(valid), run = wrong.runs[0]!;
     if (fault === "company") run.companyId = "foreign";
     if (fault === "task") run.issueId = "foreign";
@@ -62,6 +79,8 @@ describe("Hermes managed API connection qualification", () => {
     if (fault === "provider") run.contextSnapshot.aiConnection.provider = "openai";
     if (fault === "method") run.contextSnapshot.aiConnection.method = "subscription";
     if (fault === "user") run.contextSnapshot.aiConnection.responsibleUserId = "foreign";
+    if (fault === "consistent-foreign-user") { run.responsibleUserId = "foreign"; run.contextSnapshot.aiConnection.responsibleUserId = "foreign"; }
+    if (fault === "missing-expected-user") wrong.expectedResponsibleUserId = "";
     if (fault === "model") run.runnerProfileJson.sessionCheckpoint.providerIdentity.effectiveModel = "foreign";
     if (fault === "harness") run.runnerProfileJson.nativeExecutionInput.provider.agent = "codex";
     if (fault === "missing") wrong.runs = [];

tests/runner-e2e/hermes-api-connections.ts

@@ -7,6 +7,22 @@ const record = (value: unknown): Record<string, unknown> =>
 const present = (value: unknown) => typeof value === "string" && value.trim().length > 0;
 export const HERMES_API_CONNECTION_BUDGET_CENTS = 200;
 
+/** Establish the intended personal-account user independently, before a paid task. */
+export async function captureHermesApiAccountOwner(input: {
+  api: { get<T>(path: string): Promise<T> }; companyId: string; connectionId: string; provider: string;
+}) {
+  const listed = await input.api.get<{ currentUserId: string; connections: {
+    id: string; companyId: string; provider: string; method: string; ownership: string; ownerUserId?: string; status: string;
+  }[] }>(`/api/companies/${input.companyId}/ai-connections`);
+  const matches = listed.connections.filter(connection => connection.id === input.connectionId);
+  const selected = matches[0];
+  return { expectedResponsibleUserId: selected?.ownerUserId ?? null, checks: [
+    { id: "selected-account-public-readback", passed: matches.length === 1 && selected?.companyId === input.companyId && selected.provider === input.provider && selected.method === "api_key" && selected.ownership === "personal" && selected.status === "connected" },
+    { id: "authenticated-caller-known", passed: present(listed.currentUserId) },
+    { id: "selected-account-owned-by-caller", passed: present(selected?.ownerUserId) && selected?.ownerUserId === listed.currentUserId },
+  ] };
+}
+
 /** Read both public budgets before creating the paid task. Unknown usage stays unknown. */
 export async function captureHermesApiBudgets(input: {
   api: { get<T>(path: string): Promise<T> }; companyId: string; agentId: string;
@@ -23,7 +39,7 @@ export async function captureHermesApiBudgets(input: {
 
 /** Grade public run/account/model metadata; a model's completion claim cannot supply it. */
 export function gradeHermesApiConnection(input: {
-  companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string;
+  companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; expectedResponsibleUserId: string;
   runs: readonly {
     companyId: string; agentId: string; status: string; runtimeMode?: string;
     issueId?: string | null; responsibleUserId?: string | null;
@@ -38,7 +54,7 @@ export function gradeHermesApiConnection(input: {
     { id: "one-successful-native-run", passed: input.runs.length === 1 && run?.status === "succeeded" && run.runtimeMode === "native" },
     { id: "company-agent-task-scope", passed: run?.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId },
     { id: "selected-managed-api-account", passed: account.connectionId === input.connectionId && account.provider === input.provider && account.method === "api_key" && account.mode === "responsible_user" },
-    { id: "responsible-user-attribution", passed: present(run?.responsibleUserId) && account.responsibleUserId === run?.responsibleUserId },
+    { id: "responsible-user-attribution", passed: present(input.expectedResponsibleUserId) && run?.responsibleUserId === input.expectedResponsibleUserId && account.responsibleUserId === input.expectedResponsibleUserId },
     { id: "native-hermes-provider", passed: provider.kind === "acpx" && provider.agent === "hermes" && provider.model === input.model },
     { id: "exact-native-model", passed: identity.kind === "acpx" && identity.requestedModel === input.model && identity.effectiveModel === input.model },
   ];

tests/runner-e2e/runner.spec.ts

@@ -1,5 +1,5 @@
 import { runPlanTaskFlow } from "./plan-task-flow.js";
-import { captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js";
+import { captureHermesApiAccountOwner, captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js";
 import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js";
 import { assertNativeInstructionSelection, verifyNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_DEFAULT_SHA256 } from "./native-instruction-consolidation.js";
 import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js";
@@ -598,6 +598,7 @@ for (const execution of executions) {
     const nativeWorkspaceDigest = () => execution.task.id === "native-blocked-report"
       ? nativeCompletionWorkspaceDigest(workspacePath!) : Promise.resolve(null);
     let fixtures: LiveFixtureValues | undefined;
+    let hermesApiAccountOwner: Awaited<ReturnType<typeof captureHermesApiAccountOwner>> | undefined;
     let reviewProvider: Awaited<ReturnType<typeof setupConnectionReview>> | undefined;
     let issue: IssueRecord | undefined;
     let selectedRuns: RunRecord[] = [];
@@ -895,6 +896,11 @@ for (const execution of executions) {
       });
 
       if (execution.suite.id === "hermes-api-connections") {
+        if (!fixtures.aiConnection) throw new Error("Hermes API account is missing before task creation");
+        hermesApiAccountOwner = await captureHermesApiAccountOwner({ api, companyId: fixtures.company.id,
+          connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider });
+        await writeSanitizedJson(snapshotsDir, "hermes-api-owner-before-execution.json", hermesApiAccountOwner, secrets);
+        if (!hermesApiAccountOwner.checks.every(check => check.passed)) throw new Error("Hermes API owner failed admission before task creation");
         const receipt = await captureHermesApiBudgets({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id });
         await writeSanitizedJson(snapshotsDir, "hermes-api-budgets-before-execution.json", receipt, secrets);
         if (!receipt.checks.every(check => check.passed)) throw new Error("Hermes API budgets failed admission before task creation");
@@ -2872,10 +2878,10 @@ for (const execution of executions) {
         }
       }
       if (execution.suite.id === "hermes-api-connections") {
-        if (!fixtures?.aiConnection || !issue?.id) throw new Error("Hermes connection qualification is missing its selected account or task");
+        if (!fixtures?.aiConnection || !issue?.id || !hermesApiAccountOwner?.expectedResponsibleUserId) throw new Error("Hermes connection qualification is missing its selected account, expected user or task");
         const checks = gradeHermesApiConnection({ companyId: fixtures.company.id, agentId: fixtures.agent.id,
           issueId: issue.id, connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider,
-          model: execution.profile.model, runs: selectedRuns });
+          expectedResponsibleUserId: hermesApiAccountOwner.expectedResponsibleUserId, model: execution.profile.model, runs: selectedRuns });
         matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const, path: `hermesConnection.${check.id}`, expected: true }, passed: check.passed, detail: "Public native run metadata must match the selected managed API account and exact model." })));
         await writeSanitizedJson(snapshotsDir, "hermes-api-connection.json", { checks }, secrets);
         expect(checks.every(check => check.passed), "Hermes managed account and native model attribution").toBe(true);