From 12098ba3fdd1feb00ebbc8155fefebf7ce5b8fe6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 7 Oct 2026 21:37:25 -0500 Subject: [PATCH] fix(hermes): record checked-out qualification source before credentials Co-Authored-By: Paperclip --- doc/evals.md | 3 + doc/plans/2026-10-06-hermes-native-runner.md | 43 +++++++++++++ tests/runner-e2e/README.md | 11 ++++ tests/runner-e2e/hermes-source.test.ts | 64 ++++++++++++++++++++ tests/runner-e2e/hermes-source.ts | 46 ++++++++++++++ tests/runner-e2e/launch.ts | 4 ++ 6 files changed, 171 insertions(+) create mode 100644 tests/runner-e2e/hermes-source.test.ts create mode 100644 tests/runner-e2e/hermes-source.ts diff --git a/doc/evals.md b/doc/evals.md index 409003efeca..0baf3b22c8e 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -9,6 +9,9 @@ checks native completion through five selected managed API accounts on local and Daytona, with independent account/model attribution. Its registered cells remain pending until their live evidence passes; its API-only scope leaves the subscription and lifecycle gates separate. +The launcher records clean Git source before credential loading for Hermes +candidates and rejects a mismatched requested SHA. Its source receipt and normal +result metadata identify the controller; they do not qualify runtime behavior. The same guide registers the separate explicit-only `hermes-bedrock-connections` suite for one exact `us-east-1` inference profile on local and Daytona. Its two diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index 6603da40df4..546981c3315 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -812,3 +812,46 @@ and Linux CI remain failures. Corrected fresh installation, public server/CLI consumer execution and new-head CI/review are still required; no release qualification or successful clean-consumer run is claimed by these corrections. + +### 2026-10-07 source recording and current live evidence + +At `f884b7806665e6cbc18422937c687597e1767c7b`, corrected fresh setup +from the read-only packed Runner artifact completed on Mac arm64 in 30.354 +seconds, with the reviewed closure, 19,248 files and 445,379,309 bytes. Existing +cache verification and independent production-factory admission also passed. +This still leaves the complete public server/CLI installation lifecycle open. +All three stacked PRs had passing checks and fresh 5/5 reviews at their recorded +heads; the qualification head passed 54 checks with two intentional skips. +The current Linux native artifact identifies merge checkout `179f0dd4`, whose +parents independently match native head `de39c85a` and qualification head `f884b780`. +These are credential-free fixtures, not actual Daytona or paid browser proof. + +A real Mac OpenRouter completion cell passed in 49.650 seconds at that head, +including twelve independent matchers and cleanup. Its final screenshot shows +one final answer and Done. It also shows a budget pause before fixture teardown. +The settled run reports 61,035 input and 591 output tokens with unpriced cost; +the pause is consistent with Paperclip's existing unpriced-usage hard stop. +Continued use and billing coverage remain unqualified. OpenRouter documents +per-response `usage.cost`; safely collecting it must include retry, compaction +and delegated-call coverage instead of promoting Hermes's displayed estimate. + +The Mac result producer omitted standard source metadata. Separate preflight +and launch receipts identify its checkout, and a post-run assessment confirms +clean `f884b780` source, but the original null fields and machine grade remain +unchanged. The launcher now derives the controller SHA/ref from clean Git before +credential loading, rejects explicit source mismatches, and retains a campaign +receipt. Read-only discovery remains available in a dirty checkout. This closes +future source-recording omissions; it does not rewrite or upgrade historical +evidence. Runtime identity still requires separate verification. No Docker, +provider inference, or additional spending is required for this correction. +All 60 affected source/API/Bedrock tests and Product E2E TypeScript passed. +The real launcher rejects a dirty checkout before credentials and still permits +read-only discovery. A separate credential-free localhost probe using the pinned +SDK and Hermes's native stream assembly preserved the synthetic response's +reported cost and upstream-cost breakdown. This proves those fields are available +before the bridge boundary; it does not prove complete agent accounting or live +billing. The initial sandbox socket denial is a separate infrastructure outcome. + +Hermes remains pending the full original Mac/Daytona, subscription, connection, +attachment/control/state/routine and public-consumer gates. The ledger still +retains $23 against the $25 cap; unknown charges are not counted as free. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 99fe8cce4cb..b42f76672c6 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1479,6 +1479,17 @@ Hermes uses an existing managed OpenRouter connection through the ordinary connection fixture. It remains pending qualification; the native transport fixture does not substitute for these browser and remote workflows. +Before loading local credentials, the launcher records the checked-out controller +SHA and ref for every selected Hermes candidate, including API and Bedrock cells. +It rejects tracked/untracked source changes or an explicit +`PAPERCLIP_RUNNER_E2E_SOURCE_SHA` that differs from Git HEAD. The public +`hermes-source-before-execution.json` receipt is retained in the campaign root, +and the same source values reach result writers and prerequisite children. +Detached checkouts record `HEAD` unless an explicitly matched target SHA supplies +its ref. Read-only `--list` and `--matrix-json` discovery require no source +admission or credentials. This receipt identifies the controller; runtime/image +provenance and live behavior still require their independent checks. + When a Hermes cell is selected, the trusted workflow provisions its pinned Python closure on Linux before exposing credentials, and selects the same candidate assets for the image content identity, Docker build and remote diff --git a/tests/runner-e2e/hermes-source.test.ts b/tests/runner-e2e/hermes-source.test.ts new file mode 100644 index 00000000000..19a12f0c2f5 --- /dev/null +++ b/tests/runner-e2e/hermes-source.test.ts @@ -0,0 +1,64 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { devNull, tmpdir } from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { prepareHermesQualificationSource } from "./hermes-source.js"; +import { resolveRunnerE2ESource } from "./source.js"; +import type { MatrixExecution } from "./types.js"; + +const selected = [{ profile: { qualificationCandidate: "hermes" } }] as MatrixExecution[]; +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function checkout() { + const root = realpathSync(mkdtempSync(join(tmpdir(), "hermes-source-test-"))); roots.push(root); + const git = (...args: string[]) => execFileSync("git", args, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], + env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: devNull, GIT_CONFIG_NOSYSTEM: "1" } }).trim(); + git("init", "-b", "qualification"); + writeFileSync(join(root, "source.txt"), "original\n"); + git("add", "source.txt"); + git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "-c", "commit.gpgsign=false", "-c", `core.hooksPath=${devNull}`, "commit", "-m", "fixture"); + return { root, git, sha: git("rev-parse", "HEAD") }; +} + +describe("Hermes qualification controller source admission", () => { + it("records the real checkout automatically in child/result provenance instead of a workflow SHA", () => { + const f = checkout(); const env = { PATH: process.env.PATH, GITHUB_SHA: "b".repeat(40), GITHUB_REF: "refs/heads/master" }; + expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({ sha: f.sha, ref: "refs/heads/qualification", workingTreeClean: true }); + expect(resolveRunnerE2ESource(null, env)).toMatchObject({ sha: f.sha, ref: "refs/heads/qualification" }); + }); + it("preserves a requested target ref only after independently matching its SHA", () => { + const f = checkout(); const env = { PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_SOURCE_SHA: f.sha, PAPERCLIP_RUNNER_E2E_SOURCE_REF: "refs/pull/123/merge" }; + expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({ sha: f.sha, ref: "refs/pull/123/merge" }); + expect(() => prepareHermesQualificationSource(selected, f.root, { ...env, PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40) })).toThrow("differs"); + }); + it("records detached HEAD instead of borrowing another workflow ref", () => { + const f = checkout(); f.git("checkout", "--detach", f.sha); + const env = { PATH: process.env.PATH, GITHUB_REF: "refs/heads/master" }; + expect(prepareHermesQualificationSource(selected, f.root, env)).toMatchObject({ sha: f.sha, ref: "HEAD" }); + }); + it.each(["tracked", "untracked"])("rejects %s source changes without returning a source receipt", kind => { + const f = checkout(); writeFileSync(join(f.root, kind === "tracked" ? "source.txt" : "extra.txt"), "changed\n"); + const env: NodeJS.ProcessEnv = { PATH: process.env.PATH }; + expect(() => prepareHermesQualificationSource(selected, f.root, env)).toThrow("clean checkout before credentials"); + expect(env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA).toBeUndefined(); + }); + it("does not run Git or alter provenance for other providers or missing selections", () => { + const env = { PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "existing" }; + expect(prepareHermesQualificationSource([], "/does-not-exist", env)).toBeNull(); + expect(prepareHermesQualificationSource([{ profile: { qualificationCandidate: "cursor" } }] as MatrixExecution[], "/does-not-exist", env)).toBeNull(); + expect(env).toEqual({ PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "existing" }); + }); + it("ignores ambient Git redirection and keeps credentials out of the source receipt", () => { + const f = checkout(); + const env = { PATH: process.env.PATH, GIT_DIR: "/does-not-exist", GIT_WORK_TREE: "/does-not-exist", + OPENROUTER_API_KEY: "fixture-private-token", FUTURE_PROVIDER_API_KEY: "fixture-private-token" }; + const receipt = prepareHermesQualificationSource(selected, f.root, env); + expect(receipt).toMatchObject({ sha: f.sha }); + expect(JSON.stringify(receipt)).not.toContain("fixture-private-token"); + }); + it("rejects unreadable source without accepting an operator-supplied SHA", () => { + const env = { PATH: process.env.PATH, PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40) }; + expect(() => prepareHermesQualificationSource(selected, "/does-not-exist", env)).toThrow("readable Git source before credentials"); + }); +}); diff --git a/tests/runner-e2e/hermes-source.ts b/tests/runner-e2e/hermes-source.ts new file mode 100644 index 00000000000..3e35f75eb26 --- /dev/null +++ b/tests/runner-e2e/hermes-source.ts @@ -0,0 +1,46 @@ +import { execFileSync } from "node:child_process"; +import { devNull } from "node:os"; +import type { MatrixExecution } from "./types.js"; +import { resolveRunnerE2ESource } from "./source.js"; + +/** The checked-out controller is authoritative, independently of workflow context. */ +export function prepareHermesQualificationSource( + executions: readonly MatrixExecution[], + repositoryRoot: string, + environment: NodeJS.ProcessEnv, +) { + if (!executions.some(execution => execution.profile.qualificationCandidate === "hermes")) return null; + const gitEnvironment: NodeJS.ProcessEnv = { + GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: devNull, + ...Object.fromEntries(["PATH", "SYSTEMROOT", "LANG"].flatMap(name => + environment[name] === undefined ? [] : [[name, environment[name]]])), + }; + const git = (args: string[]) => { + try { + return execFileSync("git", ["-c", "core.fsmonitor=false", ...args], { + cwd: repositoryRoot, env: gitEnvironment, encoding: "utf8", + timeout: 10_000, maxBuffer: 1024 * 1024, stdio: ["ignore", "pipe", "pipe"], + }).trim(); + } catch { + throw new Error("Hermes qualification needs readable Git source before credentials; use a clean checkout."); + } + }; + const sha = git(["rev-parse", "HEAD"]); + if (!/^[a-f0-9]{40}$/.test(sha)) throw new Error("Hermes qualification source SHA is invalid."); + if (git(["status", "--porcelain", "--untracked-files=normal"])) { + throw new Error("Hermes qualification needs a clean checkout before credentials; commit or remove source changes."); + } + const requestedSha = environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA?.trim(); + if (requestedSha && requestedSha !== sha) { + throw new Error("Hermes qualification source SHA differs from the checked-out controller before credentials."); + } + const ref = git(["rev-parse", "--symbolic-full-name", "HEAD"]); + if (git(["rev-parse", "HEAD"]) !== sha) throw new Error("Hermes qualification source changed during admission."); + // These explicit values survive local-env loading and reach every child and + // synthetic failure. A workflow's own GITHUB_SHA may identify another commit. + environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA = sha; + environment.PAPERCLIP_RUNNER_E2E_SOURCE_REF = + (requestedSha && environment.PAPERCLIP_RUNNER_E2E_SOURCE_REF?.trim()) || ref; + return { schema: "paperclip.hermes.e2e-source/v1", workingTreeClean: true, + ...resolveRunnerE2ESource(null, environment) }; +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 0ad54f51ab1..534039c14e2 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -47,6 +47,7 @@ import { selectRunnerExecutions, } from "./selectors.js"; import { resolveRunnerE2ESource } from "./source.js"; +import { prepareHermesQualificationSource } from "./hermes-source.js"; import { CREDENTIAL_NAMES, type MatrixExecution, @@ -970,12 +971,15 @@ async function main() { assertRunnerE2EPrerequisites(executions); assertNativeCompletionSelection(executions); assertNativeInstructionSelection(executions); + const hermesSource = prepareHermesQualificationSource(executions, repositoryRoot, process.env); const campaignId = cleanId( process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `local-${new Date().toISOString().replace(/[:.]/g, "-")}`, ); const summaryDir = path.join(resultsRoot, campaignId); await mkdir(summaryDir, { recursive: true }); + if (hermesSource) await writeFile(path.join(summaryDir, "hermes-source-before-execution.json"), + `${JSON.stringify(hermesSource, null, 2)}\n`, "utf8"); if (executions.some(execution => execution.suite.id === "native-completion")) { process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(summaryDir); }