diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 10babdba4bc..86f675b2340 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -237,3 +237,11 @@ Merged production now also maps `runs-adapters/model-accounts`; that capability is outside this frozen catalog and remains uncovered and unqualified by it. The setup-health cell is diagnostic only and is excluded from those 244 scenarios and from `--all`, as is the explicit-only chat catalog. + +The explicit setup-health diagnostic first makes one bounded, read-only OpenRouter +current-key request on the trusted remote Linux job. Unrecognized credentials stop +before agent creation or any model call. A recognized key permits the existing +single ordinary lifecycle hello. The closed receipt records only HTTP status, +recognition, expiry and key-budget availability; unlimited/unknown key budgets do +not establish account credit or model access. Response bodies, labels, amounts, +account identities and key values are never retained. There is no retry. diff --git a/tests/runner-e2e/chat-setup-credential.test.ts b/tests/runner-e2e/chat-setup-credential.test.ts new file mode 100644 index 00000000000..4cace041d5e --- /dev/null +++ b/tests/runner-e2e/chat-setup-credential.test.ts @@ -0,0 +1,62 @@ +import { readFileSync } from "node:fs"; +import { expect, it, vi } from "vitest"; +import { checkSetupHealthCredential } from "./chat-setup-credential.js"; +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; +const key = "synthetic-credential-never-retained"; +const input = () => ({ executionId: "chat-setup-health.runner-opencode.local.setup-only", provider: "opencode", model: CHAT_OPENCODE_MODEL, + credential: key, platform: "linux", secrets: [] as string[], now: Date.parse("2026-10-27T00:00:00Z"), env: { + GITHUB_ACTIONS: "true", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", + GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master" } }); +const valid = { data: { is_free_tier: false, usage: 123.45, limit: 150, limit_remaining: 26.55, expires_at: null, + label: key, creator_user_id: "private-person-id", workspace_id: "private-workspace-id" } }; +it("registers before exactly one fixed read-only request and retains closed recognition facts", async () => { + const i = input(); + const fetch = vi.fn(async (url, options) => { + expect(i.secrets).toContain(key); expect(url).toBe("https://openrouter.ai/api/v1/key"); + expect(options).toMatchObject({ method: "GET", redirect: "error", headers: { Authorization: `Bearer ${key}` } }); + expect(options.signal).toBeInstanceOf(AbortSignal); return Response.json(valid); + }); + const receipt = await checkSetupHealthCredential({ ...i, fetch }); + expect(fetch).toHaveBeenCalledTimes(1); + expect(receipt).toEqual({ schema: "paperclip.setup-health-credential/v1", code: "recognized", httpStatus: 200, recognized: true, expired: false, budgetAvailable: true }); + expect(JSON.stringify(receipt)).not.toMatch(/synthetic|private-|123\.45|26\.55|150|label|workspace/); +}); +it.each([401, 403, 429, 500])("does not parse or retain HTTP%s body and never retries", async status => { + const fetch = vi.fn(async () => new Response(JSON.stringify({ error: { message: key, account: "private-person-id" } }), { status })); + const receipt = await checkSetupHealthCredential({ ...input(), fetch }); + expect(receipt.recognized).toBe(false); expect(receipt.httpStatus).toBe(status); expect(fetch).toHaveBeenCalledTimes(1); + expect(JSON.stringify(receipt)).not.toMatch(/synthetic|private-|message|account/); +}); +it.each([{}, { data: { label: key } }, { ...valid, data: { ...valid.data, usage: "123" } }])("refuses malformed HTTP200 recognition evidence", async body => { + const receipt = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => Response.json(body)) }); + expect(receipt).toMatchObject({ code: "invalid-response", recognized: false, httpStatus: 200 }); +}); +it("reports actual expiry and zero/unlimited budget distinctly without amounts", async () => { + const expired = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => Response.json({ data: { ...valid.data, expires_at: "2026-10-01T00:00:00Z", limit_remaining: 0 } })) }); + expect(expired).toMatchObject({ recognized: true, expired: true, budgetAvailable: false }); + const unlimited = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => Response.json({ data: { ...valid.data, limit: null, limit_remaining: null } })) }); + expect(unlimited).toMatchObject({ recognized: true, budgetAvailable: null }); +}); +it("withholds thrown network/timeout details and bounds response bodies", async () => { + const fetch = vi.fn(async () => { throw new Error(`Authorization ${key} private-error`); }); + expect(await checkSetupHealthCredential({ ...input(), fetch })).toMatchObject({ code: "request-failed", recognized: false }); + expect(fetch).toHaveBeenCalledTimes(1); + const oversized = await checkSetupHealthCredential({ ...input(), fetch: vi.fn(async () => new Response("x".repeat(32769))) }); + expect(oversized).toMatchObject({ code: "invalid-response", recognized: false }); +}); +it("never requests outside exact trusted remote health/profile prerequisites", async () => { + const fetch = vi.fn(); + for (const override of [{ executionId: "chat-features.runner-opencode.local.create-backlog" }, { platform: "darwin" }, + { provider: "codex" }, { model: "another/model" }, { credential: "" }, { env: {} }]) { + expect(await checkSetupHealthCredential({ ...input(), ...override, fetch })).toMatchObject({ code: "prerequisite-missing", recognized: false }); + } + expect(fetch).not.toHaveBeenCalled(); +}); +it("gates ordinary fixture creation on recognition and fingerprints the diagnostic source", () => { + const source = readFileSync(new URL("./runner.spec.ts", import.meta.url), "utf8"); + const start = source.indexOf('if (execution.task.flow === "setup_health") {'); + const guard = source.indexOf('if (!credential.recognized) throw', start); + const fixtures = source.indexOf('fixtures = execution.task.flow === "first_task"', start); + expect(start).toBeGreaterThan(0); expect(guard).toBeGreaterThan(start); expect(fixtures).toBeGreaterThan(guard); + expect(readFileSync(new URL("./chat-setup-health.ts", import.meta.url), "utf8")).toContain('"chat-setup-credential.ts"'); +}); diff --git a/tests/runner-e2e/chat-setup-credential.ts b/tests/runner-e2e/chat-setup-credential.ts new file mode 100644 index 00000000000..e5729859fad --- /dev/null +++ b/tests/runner-e2e/chat-setup-credential.ts @@ -0,0 +1,56 @@ +import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; +const healthId = "chat-setup-health.runner-opencode.local.setup-only"; +const endpoint = "https://openrouter.ai/api/v1/key"; +type Code = "recognized" | "auth-rejected" | "provider-unavailable" | "http-rejected" | "invalid-response" | "request-failed" | "prerequisite-missing"; +export interface SetupCredentialReceipt { + schema: "paperclip.setup-health-credential/v1"; code: Code; httpStatus: number | null; + recognized: boolean; expired: boolean | null; budgetAvailable: boolean | null; +} +/** A single Node fetch outside Playwright request tracing; raw response/errors never leave this function. */ +export async function checkSetupHealthCredential(input: { + executionId: string; provider: string; model: string | undefined; credential: string | undefined; + env: NodeJS.ProcessEnv; platform?: string; secrets: string[]; fetch?: typeof fetch; now?: number; +}): Promise { + const receipt: SetupCredentialReceipt = { schema: "paperclip.setup-health-credential/v1", code: "prerequisite-missing", httpStatus: null, recognized: false, expired: null, budgetAvailable: null }; + if (input.executionId !== healthId || input.provider !== "opencode" || input.model !== CHAT_OPENCODE_MODEL + || (input.platform ?? process.platform) !== "linux" || input.env.GITHUB_ACTIONS !== "true" + || input.env.GITHUB_REPOSITORY !== "paperclipai/paperclip" || input.env.GITHUB_REF !== "refs/heads/master" + || input.env.GITHUB_WORKFLOW_REF !== "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master" + || !input.credential?.trim()) return receipt; + // The normal tracked sink durably registers this before fetch or evidence writes. + if (!input.secrets.includes(input.credential)) input.secrets.push(input.credential); + try { + const response = await (input.fetch ?? globalThis.fetch)(endpoint, { method: "GET", redirect: "error", + headers: { Authorization: `Bearer ${input.credential}` }, signal: AbortSignal.timeout(8_000) }); + receipt.httpStatus = response.status; + if (response.status !== 200) { + await response.body?.cancel(); + receipt.code = [401, 403].includes(response.status) ? "auth-rejected" : response.status >= 500 ? "provider-unavailable" : "http-rejected"; + return receipt; + } + // Bound reads without persisting or reporting any bytes from the response. + const reader = response.body?.getReader(); if (!reader) { receipt.code = "invalid-response"; return receipt; } + let bytes = new Uint8Array(); + try { + for (;;) { + const chunk = await reader.read(); if (chunk.done) break; + if (bytes.length + chunk.value.length > 32_768) { await reader.cancel(); receipt.code = "invalid-response"; return receipt; } + bytes = new Uint8Array([...bytes, ...chunk.value]); + } + } finally { reader.releaseLock(); } + let body: any; try { body = JSON.parse(new TextDecoder().decode(bytes)); } catch { receipt.code = "invalid-response"; return receipt; } + const data = body?.data; + const amount = (value: unknown) => typeof value === "number" && Number.isFinite(value); + if (!data || typeof data !== "object" || Array.isArray(data) || typeof data.is_free_tier !== "boolean" + || !amount(data.usage) || data.usage < 0 || !(data.limit === null || amount(data.limit)) + || !(data.limit_remaining === null || amount(data.limit_remaining)) + || !(data.expires_at === undefined || data.expires_at === null || typeof data.expires_at === "string" && Number.isFinite(Date.parse(data.expires_at)))) { + receipt.code = "invalid-response"; return receipt; + } + receipt.recognized = true; receipt.code = "recognized"; + receipt.expired = data.expires_at == null ? data.expires_at === null ? false : null : Date.parse(data.expires_at) <= (input.now ?? Date.now()); + // This reports only the API's key limit, not unknown account balance/model access. + receipt.budgetAvailable = amount(data.limit_remaining) ? data.limit_remaining > 0 : null; + return receipt; + } catch { receipt.code = "request-failed"; return receipt; } +} diff --git a/tests/runner-e2e/chat-setup-health.ts b/tests/runner-e2e/chat-setup-health.ts index 838b989acea..6393ad904b5 100644 --- a/tests/runner-e2e/chat-setup-health.ts +++ b/tests/runner-e2e/chat-setup-health.ts @@ -41,7 +41,7 @@ export const chatSetupHealthTask: RunnerTaskFixture = { buildMatchers: () => [], }; export const chatSetupHealthDigest = createHash("sha256").update([ - "chat-setup-health.ts", "chat-readiness-observer.ts", "server-entry.ts", "runner.spec.ts", "live-fixtures.ts", "chat-remote-host.ts", "server.ts", + "chat-setup-health.ts", "chat-setup-credential.ts", "chat-readiness-observer.ts", "server-entry.ts", "runner.spec.ts", "live-fixtures.ts", "chat-remote-host.ts", "server.ts", ].map(file => readFileSync(new URL(`./${file}`, import.meta.url), "utf8")).join("\n")).digest("hex"); interface HealthAgent { id: string; companyId: string; lifecycleState?: string; lifecycleError?: string | null; status: string; defaultEnvironmentId?: string | null; adapterType: string; adapterConfig?: Record } diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 106586ec829..0d1177b1846 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,4 @@ +import { checkSetupHealthCredential } from "./chat-setup-credential.js"; import { SetupHealthAbort, runChatSetupHealth } from "./chat-setup-health.js"; import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js"; import { verifyInvitationSourceProof } from "./chat-catalog-domains/identity-invite-source-proof.js"; @@ -989,6 +990,12 @@ for (const execution of executions) { const receipt = verifyStockHarnessPreflight(process.env[STOCK_PREFLIGHT_ENV]); await writeSanitizedJson(snapshotsDir, "stock-harness-preflight.json", receipt, secrets); } + if (execution.task.flow === "setup_health") { + const credential = await checkSetupHealthCredential({ executionId: execution.id, provider: execution.profile.provider, + model: execution.profile.model, credential: credentials.OPENROUTER_API_KEY, env: process.env, secrets }); + await writeSanitizedJson(snapshotsDir, "chat-setup-credential.json", credential, secrets); + if (!credential.recognized) throw new Error(`Setup-health credential prerequisite failed: ${credential.code}`); + } const nativeRolloutEnabled = execution.profile.provider !== "codex"; const experimental = await api.patch<{ enableNativeRunner: boolean;