test(evals): check remote OpenRouter key before setup hello

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/CHAT-FEATURES.md

@@ -237,3 +237,11 @@ Merged production now also maps `runs-adapters/model-accounts`; that capability
 is outside this frozen catalog and remains uncovered and unqualified by it.
 The setup-health cell is diagnostic only and is excluded from those 244 scenarios
 and from `--all`, as is the explicit-only chat catalog.
+
+The explicit setup-health diagnostic first makes one bounded, read-only OpenRouter
+current-key request on the trusted remote Linux job. Unrecognized credentials stop
+before agent creation or any model call. A recognized key permits the existing
+single ordinary lifecycle hello. The closed receipt records only HTTP status,
+recognition, expiry and key-budget availability; unlimited/unknown key budgets do
+not establish account credit or model access. Response bodies, labels, amounts,
+account identities and key values are never retained. There is no retry.

tests/runner-e2e/chat-setup-credential.test.ts

@@ -0,0 +1,62 @@
+import { readFileSync } from "node:fs";
+import { expect, it, vi } from "vitest";
+import { checkSetupHealthCredential } from "./chat-setup-credential.js";
+import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
+const key = "synthetic-credential-never-retained";
+const input = () => ({ executionId: "chat-setup-health.runner-opencode.local.setup-only", provider: "opencode", model: CHAT_OPENCODE_MODEL,
+  credential: key, platform: "linux", secrets: [] as string[], now: Date.parse("2026-10-27T00:00:00Z"), env: {
+    GITHUB_ACTIONS: "true", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master",
+    GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master" } });
+const valid = { data: { is_free_tier: false, usage: 123.45, limit: 150, limit_remaining: 26.55, expires_at: null,
+  label: key, creator_user_id: "private-person-id", workspace_id: "private-workspace-id" } };
+it("registers before exactly one fixed read-only request and retains closed recognition facts", async () => {
+  const i = input();
+  const fetch = vi.fn(async (url, options) => {
+    expect(i.secrets).toContain(key); expect(url).toBe("https://openrouter.ai/api/v1/key");
+    expect(options).toMatchObject({ method: "GET", redirect: "error", headers: { Authorization: `Bearer ${key}` } });
+    expect(options.signal).toBeInstanceOf(AbortSignal); return Response.json(valid);
+  });
+  const receipt = await checkSetupHealthCredential({ ...i, fetch });
+  expect(fetch).toHaveBeenCalledTimes(1);
+  expect(receipt).toEqual({ schema: "paperclip.setup-health-credential/v1", code: "recognized", httpStatus: 200, recognized: true, expired: false, budgetAvailable: true });
+  expect(JSON.stringify(receipt)).not.toMatch(/synthetic|private-|123\.45|26\.55|150|label|workspace/);
+});
+it.each([401, 403, 429, 500])("does not parse or retain HTTP%s body and never retries", async status => {
+  const fetch = vi.fn(async () => new Response(JSON.stringify({ error: { message: key, account: "private-person-id" } }), { status }));
+  const receipt = await checkSetupHealthCredential({ ...input(), fetch });
+  expect(receipt.recognized).toBe(false); expect(receipt.httpStatus).toBe(status); expect(fetch).toHaveBeenCalledTimes(1);
+  expect(JSON.stringify(receipt)).not.toMatch(/synthetic|private-|message|account/);
+});
+it.each([{}, { data: { label: key } }, { ...valid, data: { ...valid.data, usage: "123" } }])("refuses malformed HTTP200 recognition evidence", async body => {
+  const receipt = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => Response.json(body)) });
+  expect(receipt).toMatchObject({ code: "invalid-response", recognized: false, httpStatus: 200 });
+});
+it("reports actual expiry and zero/unlimited budget distinctly without amounts", async () => {
+  const expired = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => Response.json({ data: { ...valid.data, expires_at: "2026-10-01T00:00:00Z", limit_remaining: 0 } })) });
+  expect(expired).toMatchObject({ recognized: true, expired: true, budgetAvailable: false });
+  const unlimited = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => Response.json({ data: { ...valid.data, limit: null, limit_remaining: null } })) });
+  expect(unlimited).toMatchObject({ recognized: true, budgetAvailable: null });
+});
+it("withholds thrown network/timeout details and bounds response bodies", async () => {
+  const fetch = vi.fn(async () => { throw new Error(`Authorization ${key} private-error`); });
+  expect(await checkSetupHealthCredential({ ...input(), fetch })).toMatchObject({ code: "request-failed", recognized: false });
+  expect(fetch).toHaveBeenCalledTimes(1);
+  const oversized = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => new Response("x".repeat(32769))) });
+  expect(oversized).toMatchObject({ code: "invalid-response", recognized: false });
+});
+it("never requests outside exact trusted remote health/profile prerequisites", async () => {
+  const fetch = vi.fn();
+  for (const override of [{ executionId: "chat-features.runner-opencode.local.create-backlog" }, { platform: "darwin" },
+    { provider: "codex" }, { model: "another/model" }, { credential: "" }, { env: {} }]) {
+    expect(await checkSetupHealthCredential({ ...input(), ...override, fetch })).toMatchObject({ code: "prerequisite-missing", recognized: false });
+  }
+  expect(fetch).not.toHaveBeenCalled();
+});
+it("gates ordinary fixture creation on recognition and fingerprints the diagnostic source", () => {
+  const source = readFileSync(new URL("./runner.spec.ts", import.meta.url), "utf8");
+  const start = source.indexOf('if (execution.task.flow === "setup_health") {');
+  const guard = source.indexOf('if (!credential.recognized) throw', start);
+  const fixtures = source.indexOf('fixtures = execution.task.flow === "first_task"', start);
+  expect(start).toBeGreaterThan(0); expect(guard).toBeGreaterThan(start); expect(fixtures).toBeGreaterThan(guard);
+  expect(readFileSync(new URL("./chat-setup-health.ts", import.meta.url), "utf8")).toContain('"chat-setup-credential.ts"');
+});

tests/runner-e2e/chat-setup-credential.ts

@@ -0,0 +1,56 @@
+import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
+const healthId = "chat-setup-health.runner-opencode.local.setup-only";
+const endpoint = "https://openrouter.ai/api/v1/key";
+type Code = "recognized" | "auth-rejected" | "provider-unavailable" | "http-rejected" | "invalid-response" | "request-failed" | "prerequisite-missing";
+export interface SetupCredentialReceipt {
+  schema: "paperclip.setup-health-credential/v1"; code: Code; httpStatus: number | null;
+  recognized: boolean; expired: boolean | null; budgetAvailable: boolean | null;
+}
+/** A single Node fetch outside Playwright request tracing; raw response/errors never leave this function. */
+export async function checkSetupHealthCredential(input: {
+  executionId: string; provider: string; model: string | undefined; credential: string | undefined;
+  env: NodeJS.ProcessEnv; platform?: string; secrets: string[]; fetch?: typeof fetch; now?: number;
+}): Promise<SetupCredentialReceipt> {
+  const receipt: SetupCredentialReceipt = { schema: "paperclip.setup-health-credential/v1", code: "prerequisite-missing", httpStatus: null, recognized: false, expired: null, budgetAvailable: null };
+  if (input.executionId !== healthId || input.provider !== "opencode" || input.model !== CHAT_OPENCODE_MODEL
+    || (input.platform ?? process.platform) !== "linux" || input.env.GITHUB_ACTIONS !== "true"
+    || input.env.GITHUB_REPOSITORY !== "paperclipai/paperclip" || input.env.GITHUB_REF !== "refs/heads/master"
+    || input.env.GITHUB_WORKFLOW_REF !== "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master"
+    || !input.credential?.trim()) return receipt;
+  // The normal tracked sink durably registers this before fetch or evidence writes.
+  if (!input.secrets.includes(input.credential)) input.secrets.push(input.credential);
+  try {
+    const response = await (input.fetch ?? globalThis.fetch)(endpoint, { method: "GET", redirect: "error",
+      headers: { Authorization: `Bearer ${input.credential}` }, signal: AbortSignal.timeout(8_000) });
+    receipt.httpStatus = response.status;
+    if (response.status !== 200) {
+      await response.body?.cancel();
+      receipt.code = [401, 403].includes(response.status) ? "auth-rejected" : response.status >= 500 ? "provider-unavailable" : "http-rejected";
+      return receipt;
+    }
+    // Bound reads without persisting or reporting any bytes from the response.
+    const reader = response.body?.getReader(); if (!reader) { receipt.code = "invalid-response"; return receipt; }
+    let bytes = new Uint8Array();
+    try {
+      for (;;) {
+        const chunk = await reader.read(); if (chunk.done) break;
+        if (bytes.length + chunk.value.length > 32_768) { await reader.cancel(); receipt.code = "invalid-response"; return receipt; }
+        bytes = new Uint8Array([...bytes, ...chunk.value]);
+      }
+    } finally { reader.releaseLock(); }
+    let body: any; try { body = JSON.parse(new TextDecoder().decode(bytes)); } catch { receipt.code = "invalid-response"; return receipt; }
+    const data = body?.data;
+    const amount = (value: unknown) => typeof value === "number" && Number.isFinite(value);
+    if (!data || typeof data !== "object" || Array.isArray(data) || typeof data.is_free_tier !== "boolean"
+      || !amount(data.usage) || data.usage < 0 || !(data.limit === null || amount(data.limit))
+      || !(data.limit_remaining === null || amount(data.limit_remaining))
+      || !(data.expires_at === undefined || data.expires_at === null || typeof data.expires_at === "string" && Number.isFinite(Date.parse(data.expires_at)))) {
+      receipt.code = "invalid-response"; return receipt;
+    }
+    receipt.recognized = true; receipt.code = "recognized";
+    receipt.expired = data.expires_at == null ? data.expires_at === null ? false : null : Date.parse(data.expires_at) <= (input.now ?? Date.now());
+    // This reports only the API's key limit, not unknown account balance/model access.
+    receipt.budgetAvailable = amount(data.limit_remaining) ? data.limit_remaining > 0 : null;
+    return receipt;
+  } catch { receipt.code = "request-failed"; return receipt; }
+}

tests/runner-e2e/chat-setup-health.ts

@@ -41,7 +41,7 @@ export const chatSetupHealthTask: RunnerTaskFixture = {
   buildMatchers: () => [],
 };
 export const chatSetupHealthDigest = createHash("sha256").update([
-  "chat-setup-health.ts", "chat-readiness-observer.ts", "server-entry.ts", "runner.spec.ts", "live-fixtures.ts", "chat-remote-host.ts", "server.ts",
+  "chat-setup-health.ts", "chat-setup-credential.ts", "chat-readiness-observer.ts", "server-entry.ts", "runner.spec.ts", "live-fixtures.ts", "chat-remote-host.ts", "server.ts",
 ].map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n")).digest("hex");
 interface HealthAgent { id: string; companyId: string; lifecycleState?: string; lifecycleError?: string | null; status: string;
   defaultEnvironmentId?: string | null; adapterType: string; adapterConfig?: Record<string, unknown> }

tests/runner-e2e/runner.spec.ts

@@ -1,3 +1,4 @@
+import { checkSetupHealthCredential } from "./chat-setup-credential.js";
 import { SetupHealthAbort, runChatSetupHealth } from "./chat-setup-health.js";
 import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js";
 import { verifyInvitationSourceProof } from "./chat-catalog-domains/identity-invite-source-proof.js";
@@ -989,6 +990,12 @@ for (const execution of executions) {
         const receipt = verifyStockHarnessPreflight(process.env[STOCK_PREFLIGHT_ENV]);
         await writeSanitizedJson(snapshotsDir, "stock-harness-preflight.json", receipt, secrets);
       }
+      if (execution.task.flow === "setup_health") {
+        const credential = await checkSetupHealthCredential({ executionId: execution.id, provider: execution.profile.provider,
+          model: execution.profile.model, credential: credentials.OPENROUTER_API_KEY, env: process.env, secrets });
+        await writeSanitizedJson(snapshotsDir, "chat-setup-credential.json", credential, secrets);
+        if (!credential.recognized) throw new Error(`Setup-health credential prerequisite failed: ${credential.code}`);
+      }
       const nativeRolloutEnabled = execution.profile.provider !== "codex";
       const experimental = await api.patch<{
         enableNativeRunner: boolean;