From 08d9803b8f018d573ea60ca6f08ba94044a715e2 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 7 Oct 2026 18:32:31 -0500 Subject: [PATCH] test(hermes): verify the expected account owner independently Co-Authored-By: Paperclip --- doc/plans/2026-10-06-hermes-native-runner.md | 22 ++++++++++++++++ tests/runner-e2e/README.md | 5 ++++ tests/runner-e2e/catalog.ts | 2 +- .../runner-e2e/hermes-api-connections.test.ts | 25 ++++++++++++++++--- tests/runner-e2e/hermes-api-connections.ts | 20 +++++++++++++-- tests/runner-e2e/runner.spec.ts | 12 ++++++--- 6 files changed, 77 insertions(+), 9 deletions(-) diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index dca47d1deab..af06d6a1c49 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -687,3 +687,25 @@ verification. The earlier image digest and live results retain their original sources and closure; they are not results for the corrected runtime. The stacked qualification commits were replayed without changing their patches. Both updated PR heads require fresh checks and review. Hermes remains pending qualification. + +The subsequent qualification review found an independent-user evidence gap: +the original responsible-user check compared two run fields with each other. +That permits a consistent foreign user to pass. New fixture admission reads the +selected account and authenticated caller through the public Connections API +before creating a paid task, verifies company/account/provider/method/ownership +and connected status, and retains the expected owner. Both run fields must equal +that owner; missing, duplicate, foreign-owner and consistent-wrong-run-user cases +are calibrated as failures. The historical API results keep their original +grader provenance; they do not acquire this pre-turn owner receipt retroactively. +An additional assessment of the retained public task records independently +checks the immutable task creator, task responsible user and both run attribution +fields. All five successful API cases pass that comparison with zero provider +calls. This retained-task assessment remains separate from the new fixture's +pre-turn account-owner admission. The correction passed all 106 affected tests, +Product E2E TypeScript and diff checks. + +The fresh Linux native CI attempt at the replayed head failed before provisioning +because GitHub returned HTTP 429 for the pinned source archive. No native fixture +or provider execution occurred. This is retained as a download infrastructure +failure; it is not evidence of a closure mismatch. The separate credential-free +AWS build continues against its recorded immutable source. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index fc9d26edc6a..8c799882d14 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1525,6 +1525,11 @@ the ordinary completion, screenshot, token/cost and cleanup evidence. Each cell admits one attempt and configures 200-cent company and agent budgets. Public budget readback must pass before task creation. Unpriced model usage remains unknown; this configured limit is not an exact billing receipt. +Before creating the task, public account readback must identify the selected +connected personal account, its company and owner, and the authenticated caller. +Both run attribution fields must match that independently recorded owner. +The pre-turn owner receipt is retained; agreement between two run fields alone +cannot establish the expected user. ```sh pnpm test:e2e:runner -- --list --suite hermes-api-connections diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 46253b96c9d..d94a6b276c8 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1210,7 +1210,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ expectedMatrixSize: 10, definitionMetadata: { version: 1, qualification: "pending", scheduling: "explicit-only", authenticatedDiscoveryDate: "2026-10-07", - accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, + accountMethod: "api_key", accountMode: "responsible_user", providerTurns: 1, expectedUserSource: "public-account-owner-before-task", budgetMonthlyCents: HERMES_API_CONNECTION_BUDGET_CENTS, maximumAttemptsPerCell: 1, coverage: "api-account-native-completion-only", sourceDigest: hermesApiConnectionDefinitionDigest, }, diff --git a/tests/runner-e2e/hermes-api-connections.test.ts b/tests/runner-e2e/hermes-api-connections.test.ts index 2ce4254654a..d77b0f646c5 100644 --- a/tests/runner-e2e/hermes-api-connections.test.ts +++ b/tests/runner-e2e/hermes-api-connections.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { runnerMatrix, runnerSuites, suiteDefinitionHash } from "./catalog.js"; import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; -import { captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js"; +import { captureHermesApiAccountOwner, captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js"; describe("Hermes managed API connection qualification", () => { const suite = runnerSuites.find(s => s.id === "hermes-api-connections")!; @@ -32,7 +32,7 @@ describe("Hermes managed API connection qualification", () => { expect(suiteDefinitionHash({ ...suite, definitionMetadata: { ...suite.definitionMetadata, sourceDigest: "changed-account-selection" } })).not.toBe(suiteDefinitionHash(suite)); }); const valid = { - companyId: "company", agentId: "agent", issueId: "task", connectionId: "account", provider: "xai", model: "grok-4.7", + companyId: "company", agentId: "agent", issueId: "task", connectionId: "account", provider: "xai", model: "grok-4.7", expectedResponsibleUserId: "user", runs: [{ companyId: "company", agentId: "agent", issueId: "task", status: "succeeded", runtimeMode: "native", responsibleUserId: "user", contextSnapshot: { aiConnection: { connectionId: "account", provider: "xai", method: "api_key", mode: "responsible_user", responsibleUserId: "user" } }, runnerProfileJson: { nativeExecutionInput: { provider: { kind: "acpx", agent: "hermes", model: "grok-4.7" } }, @@ -41,6 +41,23 @@ describe("Hermes managed API connection qualification", () => { it("accepts independently observed account/model metadata", () => { expect(gradeHermesApiConnection(valid).every(check => check.passed)).toBe(true); }); + it.each(["valid", "missing", "duplicate", "company", "provider", "method", "ownership", "status", "owner", "caller"])("establishes the expected user from public account readback: %s", async fault => { + const account = { id: "account", companyId: "company", provider: "xai", method: "api_key", ownership: "personal", ownerUserId: "user", status: "connected" }; + if (fault === "company") account.companyId = "foreign"; + if (fault === "provider") account.provider = "foreign"; + if (fault === "method") account.method = "foreign"; + if (fault === "ownership") account.ownership = "foreign"; + if (fault === "status") account.status = "foreign"; + if (fault === "owner") account.ownerUserId = "foreign"; + const receipt = await captureHermesApiAccountOwner({ companyId: "company", connectionId: "account", provider: "xai", api: { + async get(url: string) { + expect(url).toBe("/api/companies/company/ai-connections"); + return { currentUserId: fault === "caller" ? "" : "user", connections: fault === "missing" ? [] : fault === "duplicate" ? [account, account] : [account] } as T; + }, + } }); + expect(receipt.checks.every(check => check.passed)).toBe(fault === "valid"); + if (fault === "valid") expect(receipt.expectedResponsibleUserId).toBe("user"); + }); it.each(["valid", "company-budget", "agent-budget", "company-scope", "agent-scope", "agent-company"])("checks %s through public budget readback before a paid task", async fault => { const company = { id: "company", budgetMonthlyCents: fault === "company-budget" ? 0 : 200 }; const agent = { id: "agent", companyId: "company", budgetMonthlyCents: fault === "agent-budget" ? 0 : 200 }; @@ -54,7 +71,7 @@ describe("Hermes managed API connection qualification", () => { expect(new Set(paths)).toEqual(new Set(["/api/companies/company", "/api/agents/agent"])); expect(receipt.checks.every(check => check.passed)).toBe(fault === "valid"); }); - it.each(["company", "task", "account", "provider", "method", "user", "model", "harness", "missing", "extra-run"])("rejects %s evidence even with a successful answer", fault => { + it.each(["company", "task", "account", "provider", "method", "user", "consistent-foreign-user", "missing-expected-user", "model", "harness", "missing", "extra-run"])("rejects %s evidence even with a successful answer", fault => { const wrong = structuredClone(valid), run = wrong.runs[0]!; if (fault === "company") run.companyId = "foreign"; if (fault === "task") run.issueId = "foreign"; @@ -62,6 +79,8 @@ describe("Hermes managed API connection qualification", () => { if (fault === "provider") run.contextSnapshot.aiConnection.provider = "openai"; if (fault === "method") run.contextSnapshot.aiConnection.method = "subscription"; if (fault === "user") run.contextSnapshot.aiConnection.responsibleUserId = "foreign"; + if (fault === "consistent-foreign-user") { run.responsibleUserId = "foreign"; run.contextSnapshot.aiConnection.responsibleUserId = "foreign"; } + if (fault === "missing-expected-user") wrong.expectedResponsibleUserId = ""; if (fault === "model") run.runnerProfileJson.sessionCheckpoint.providerIdentity.effectiveModel = "foreign"; if (fault === "harness") run.runnerProfileJson.nativeExecutionInput.provider.agent = "codex"; if (fault === "missing") wrong.runs = []; diff --git a/tests/runner-e2e/hermes-api-connections.ts b/tests/runner-e2e/hermes-api-connections.ts index b7b1427df64..1518e358b92 100644 --- a/tests/runner-e2e/hermes-api-connections.ts +++ b/tests/runner-e2e/hermes-api-connections.ts @@ -7,6 +7,22 @@ const record = (value: unknown): Record => const present = (value: unknown) => typeof value === "string" && value.trim().length > 0; export const HERMES_API_CONNECTION_BUDGET_CENTS = 200; +/** Establish the intended personal-account user independently, before a paid task. */ +export async function captureHermesApiAccountOwner(input: { + api: { get(path: string): Promise }; companyId: string; connectionId: string; provider: string; +}) { + const listed = await input.api.get<{ currentUserId: string; connections: { + id: string; companyId: string; provider: string; method: string; ownership: string; ownerUserId?: string; status: string; + }[] }>(`/api/companies/${input.companyId}/ai-connections`); + const matches = listed.connections.filter(connection => connection.id === input.connectionId); + const selected = matches[0]; + return { expectedResponsibleUserId: selected?.ownerUserId ?? null, checks: [ + { id: "selected-account-public-readback", passed: matches.length === 1 && selected?.companyId === input.companyId && selected.provider === input.provider && selected.method === "api_key" && selected.ownership === "personal" && selected.status === "connected" }, + { id: "authenticated-caller-known", passed: present(listed.currentUserId) }, + { id: "selected-account-owned-by-caller", passed: present(selected?.ownerUserId) && selected?.ownerUserId === listed.currentUserId }, + ] }; +} + /** Read both public budgets before creating the paid task. Unknown usage stays unknown. */ export async function captureHermesApiBudgets(input: { api: { get(path: string): Promise }; companyId: string; agentId: string; @@ -23,7 +39,7 @@ export async function captureHermesApiBudgets(input: { /** Grade public run/account/model metadata; a model's completion claim cannot supply it. */ export function gradeHermesApiConnection(input: { - companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; + companyId: string; agentId: string; issueId: string; connectionId: string; provider: string; model: string; expectedResponsibleUserId: string; runs: readonly { companyId: string; agentId: string; status: string; runtimeMode?: string; issueId?: string | null; responsibleUserId?: string | null; @@ -38,7 +54,7 @@ export function gradeHermesApiConnection(input: { { id: "one-successful-native-run", passed: input.runs.length === 1 && run?.status === "succeeded" && run.runtimeMode === "native" }, { id: "company-agent-task-scope", passed: run?.companyId === input.companyId && run.agentId === input.agentId && run.issueId === input.issueId }, { id: "selected-managed-api-account", passed: account.connectionId === input.connectionId && account.provider === input.provider && account.method === "api_key" && account.mode === "responsible_user" }, - { id: "responsible-user-attribution", passed: present(run?.responsibleUserId) && account.responsibleUserId === run?.responsibleUserId }, + { id: "responsible-user-attribution", passed: present(input.expectedResponsibleUserId) && run?.responsibleUserId === input.expectedResponsibleUserId && account.responsibleUserId === input.expectedResponsibleUserId }, { id: "native-hermes-provider", passed: provider.kind === "acpx" && provider.agent === "hermes" && provider.model === input.model }, { id: "exact-native-model", passed: identity.kind === "acpx" && identity.requestedModel === input.model && identity.effectiveModel === input.model }, ]; diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 4a67aad8e74..bad6b48143c 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,5 +1,5 @@ import { runPlanTaskFlow } from "./plan-task-flow.js"; -import { captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js"; +import { captureHermesApiAccountOwner, captureHermesApiBudgets, gradeHermesApiConnection } from "./hermes-api-connections.js"; import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js"; import { assertNativeInstructionSelection, verifyNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE, NATIVE_INSTRUCTION_DEFAULT_SHA256 } from "./native-instruction-consolidation.js"; import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js"; @@ -598,6 +598,7 @@ for (const execution of executions) { const nativeWorkspaceDigest = () => execution.task.id === "native-blocked-report" ? nativeCompletionWorkspaceDigest(workspacePath!) : Promise.resolve(null); let fixtures: LiveFixtureValues | undefined; + let hermesApiAccountOwner: Awaited> | undefined; let reviewProvider: Awaited> | undefined; let issue: IssueRecord | undefined; let selectedRuns: RunRecord[] = []; @@ -895,6 +896,11 @@ for (const execution of executions) { }); if (execution.suite.id === "hermes-api-connections") { + if (!fixtures.aiConnection) throw new Error("Hermes API account is missing before task creation"); + hermesApiAccountOwner = await captureHermesApiAccountOwner({ api, companyId: fixtures.company.id, + connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider }); + await writeSanitizedJson(snapshotsDir, "hermes-api-owner-before-execution.json", hermesApiAccountOwner, secrets); + if (!hermesApiAccountOwner.checks.every(check => check.passed)) throw new Error("Hermes API owner failed admission before task creation"); const receipt = await captureHermesApiBudgets({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id }); await writeSanitizedJson(snapshotsDir, "hermes-api-budgets-before-execution.json", receipt, secrets); if (!receipt.checks.every(check => check.passed)) throw new Error("Hermes API budgets failed admission before task creation"); @@ -2872,10 +2878,10 @@ for (const execution of executions) { } } if (execution.suite.id === "hermes-api-connections") { - if (!fixtures?.aiConnection || !issue?.id) throw new Error("Hermes connection qualification is missing its selected account or task"); + if (!fixtures?.aiConnection || !issue?.id || !hermesApiAccountOwner?.expectedResponsibleUserId) throw new Error("Hermes connection qualification is missing its selected account, expected user or task"); const checks = gradeHermesApiConnection({ companyId: fixtures.company.id, agentId: fixtures.agent.id, issueId: issue.id, connectionId: fixtures.aiConnection.connectionId, provider: fixtures.aiConnection.binding.provider, - model: execution.profile.model, runs: selectedRuns }); + expectedResponsibleUserId: hermesApiAccountOwner.expectedResponsibleUserId, model: execution.profile.model, runs: selectedRuns }); matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const, path: `hermesConnection.${check.id}`, expected: true }, passed: check.passed, detail: "Public native run metadata must match the selected managed API account and exact model." }))); await writeSanitizedJson(snapshotsDir, "hermes-api-connection.json", { checks }, secrets); expect(checks.every(check => check.passed), "Hermes managed account and native model attribution").toBe(true);