diff --git a/.env.runner-e2e.example b/.env.runner-e2e.example index 992d56f820a..1e208f879fb 100644 --- a/.env.runner-e2e.example +++ b/.env.runner-e2e.example @@ -9,3 +9,6 @@ DAYTONA_API_KEY= # Required only for Daytona cells. Use an immutable, anonymously pullable # digest from the Runner Full-Stack E2E image job or a locally published image. PAPERCLIP_E2E_DAYTONA_IMAGE=ghcr.io/paperclipai/paperclip-daytona-runner@sha256:REPLACE_ME +# Optional, explicit subscription qualification only. Supply privately at runtime; +# never commit an actual auth.json payload. API-key profiles do not use this value. +# GROK_AUTH_JSON= diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 6d7c4ae6255..fcaad860c27 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -9,6 +9,10 @@ on: description: "Branch in paperclipai/paperclip to test; the trusted workflow still runs from master" type: string required: false + max_parallel: + description: "Optional lower concurrency for this campaign (cannot exceed the configured limit)" + type: string + required: false all: description: "Run the complete paid matrix when no narrower selector is supplied" type: boolean @@ -286,6 +290,7 @@ jobs: SELECT_ID: ${{ inputs.id }} MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }} MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }} + REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }} run: | set -euo pipefail args=(--matrix-json) @@ -346,12 +351,20 @@ jobs: echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 1 through $MAX_PARALLEL_LIMIT for the selected runner." >&2 exit 1 fi + if [ -n "${REQUESTED_MAX_PARALLEL:-}" ]; then + if ! [[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ]; then + echo "max_parallel must be an integer from 1 through the configured campaign limit." >&2 + exit 1 + fi + MAX_PARALLEL="$REQUESTED_MAX_PARALLEL" + fi echo "max_parallel=$MAX_PARALLEL" >> "$GITHUB_OUTPUT" daytona_image: name: Publish verified Daytona image needs: [authorize, target_lock, catalog] - runs-on: ubuntu-latest + # Keep image builds on the same authorized fleet as campaign execution. + runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: 45 permissions: contents: read @@ -874,6 +887,10 @@ jobs: if: matrix.environmentId == 'local' && (matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth') run: node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs + - name: Install checksum-verified Grok executable + if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-acpx-grok' || matrix.profileId == 'runner-acpx-grok-subscription') + run: node packages/grok-acp/install.mjs + - name: Download immutable campaign outputs if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -1024,7 +1041,7 @@ jobs: NODE - name: Prepare pinned Python artifact oracle image - if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect') + if: (matrix.suiteId == 'everyday-workflows' || matrix.suiteId == 'grok-qualification' || matrix.suiteId == 'grok-subscription-qualification') && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect') run: | set -euo pipefail oracle_image='python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285' @@ -1038,6 +1055,7 @@ jobs: ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }} OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }} XAI_API_KEY: ${{ matrix.credentialName == 'XAI_API_KEY' && secrets.XAI_API_KEY || '' }} + GROK_AUTH_JSON: ${{ matrix.credentialName == 'GROK_AUTH_JSON' && secrets.GROK_AUTH_JSON || '' }} DAYTONA_API_KEY: ${{ matrix.environmentId == 'daytona' && secrets.DAYTONA_API_KEY || '' }} PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }} PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack diff --git a/doc/evals.md b/doc/evals.md index b1e2ee306cf..7d453e8e665 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -72,6 +72,12 @@ fixture-driven file wait and persisted-plan oracle are documented in the [Product E2E guide](../tests/runner-e2e/README.md). It does not qualify the native onboarding wizard or change the native API-tool rollout defaults. +The explicit-only `grok-qualification` and `grok-subscription-qualification` +Product suites exercise Grok Build with API and company subscription +authentication respectively. Keep their results separate; the subscription +fixture seeds an explicitly supplied login and does not qualify interactive +login. See the [Grok fixture contract](../tests/runner-e2e/README.md#grok-build-qualification). + ## Validation ladder Start with credential-free checks and a catalog listing. For Product E2E: diff --git a/doc/grok-native-runner.md b/doc/grok-native-runner.md index 584f149ac26..5fa4282eb13 100644 --- a/doc/grok-native-runner.md +++ b/doc/grok-native-runner.md @@ -53,6 +53,13 @@ and continuation after controller restart in both environments. Run with `--suite grok-qualification`; it is excluded from scheduled `--all`. Use the canonical Product E2E dashboard and Evalbook reports. +The separate `grok-subscription-qualification` suite covers the same workflows +with the explicit `GROK_AUTH_JSON` fixture credential. It seeds only the disposable +company's private login home and supplies no API key. It does not exercise the +interactive sign-in UI. Authentication mode remains part of the profile identity. +After a subscription upgrade, a fresh `grok login` may be needed if the existing +login still reports the previous entitlement through ACP. + `packages/paperclip-runner/scripts/grok-native-smoke.mjs` records explicit auth, model/binary identity, MCP outcome, durable resume and restrictive permissions. Pass `--auth subscription --auth-file /private/path/auth.json --output /private/report.json` diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index cedca439ad5..58a30d2930d 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -46,6 +46,14 @@ Agent `adapterConfig.env` values must be `{type:"secret_ref", secretId, version:"latest"}` objects supplied to the factory. A fixture source containing a raw secret-looking value is rejected by catalog validation. +The manual Grok subscription profile uses `GROK_AUTH_JSON` as an explicit login +fixture. It does not put this credential in agent configuration or substitute an +API key. Setup seeds a new company-scoped Grok home inside the disposable instance +with mode 0700 and an exclusive mode-0600 auth file. Setup rejects redirected, +occupied, or nonisolated homes. Production runner discovery and refresh operate on +that company login; teardown destroys it after the remote environment is removed. +This fixture tests subscription execution, not the interactive browser login flow. + ## Environments An `EnvironmentFixture` declares driver/provider, credential requirements, diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 4d61852f42d..0293e2c9343 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1007,7 +1007,13 @@ failure evidence, so a future blank page is distinguishable from a loaded task. `runner-acpx-grok` uses native ACPX with Grok Build 1.0.13 and `grok-4.7`. Set an explicit `XAI_API_KEY` for this API-key profile. The manual `grok-qualification` suite runs the core and restart workflows on local and -Daytona runtimes. Missing subscription coverage remains a qualification blocker; +Daytona runtimes. The separate manual `grok-subscription-qualification` suite +uses `runner-acpx-grok-subscription` and explicit `GROK_AUTH_JSON`. Its fixture +stages that login in a private, disposable company home; this is credential +setup, not a test of interactive browser login. The server never inherits the +JSON secret or an API key from the harness. See [FIXTURES.md](FIXTURES.md) and +[SECURITY.md](SECURITY.md) for staging, cleanup, and redaction requirements. +Missing subscription coverage remains a qualification blocker; an API-key pass must not be reported as subscription evidence. See [`doc/grok-native-runner.md`](../../doc/grok-native-runner.md) for installation, credential boundaries and repetition requirements. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 57ed632d78f..1e84d6484ed 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -63,6 +63,14 @@ rejects mutable tag or branch references. ## Secrets and protected environments +Explicit Grok subscription campaigns may select `GROK_AUTH_JSON` from the same +protected paid environment. Deliver it only to that subscription profile's paid +test step. Prefer a dedicated test-account login and refresh it before a campaign. +The fixture stores only the selected login in its disposable company home, rejects +ambient/redirected homes, and removes the login during teardown. Token fragments +and account identifiers inside its JSON join the evidence redaction set. The +subscription suite remains manual-only and never falls back to `XAI_API_KEY`. + Create `runner-e2e-paid`, restrict deployments to the default branch, and put only `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, and `DAYTONA_API_KEY` in it. Do not duplicate these credentials as repository- or diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 8fbe7c01426..1198cb045f9 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -71,10 +71,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 16, 23, 38, 52, 28, 18, 6, 6, 48, 16, 10, 2, + 16, 16, 23, 38, 52, 28, 18, 6, 6, 48, 16, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(263); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(263); + expect(validateRunnerCatalog()).toHaveLength(279); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(279); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); @@ -560,6 +560,21 @@ describe("runner E2E selectors", () => { } }); + it("keeps subscription qualification separate and never injects an API key", () => { + const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "grok-subscription-qualification"])); + expect(selected).toHaveLength(16); + const all = selectRunnerExecutions(parseRunnerSelectors(["--all"])); + for (const execution of selected) { + expect(all.some(candidate => candidate.id === execution.id)).toBe(false); + expect(execution.requiredCredentials).toEqual(execution.environment.id === "daytona" + ? ["GROK_AUTH_JSON", "DAYTONA_API_KEY"] : ["GROK_AUTH_JSON"]); + expect(execution.profile.buildAgent({ + environmentId: "env", environmentFixtureId: execution.environment.id, + workspacePath: "/tmp/workspace", secretRefs: {}, executionId: "subscription", + })).toMatchObject({ adapterConfig: { provider: "acpx", acpxAgent: "grok", env: {} } }); + } + }); + it("selects a suite without exploding its environment matrix", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "openrouter-model-breadth"]), diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 70269d88043..20709372ae8 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -170,7 +170,9 @@ function nativeProfile(input: { provider: input.provider, }, buildAgent(buildInput) { - const credentialRef = requiredSecret(buildInput, input.credential); + const credentialRef = input.credential === "GROK_AUTH_JSON" + ? null + : requiredSecret(buildInput, input.credential); const permissionConfig = input.provider === "codex" ? { codexPermissionMode: "never" } @@ -184,7 +186,7 @@ function nativeProfile(input: { idleTimeoutMs: 300_000, ...permissionConfig, env: { - [input.credential]: credentialRef, + ...(credentialRef ? { [input.credential]: credentialRef } : {}), // Codex's supported automation credential is CODEX_API_KEY. Keep // OPENAI_API_KEY as the operator-facing fixture secret name and bind // the same encrypted reference to the runtime-specific alias. @@ -917,6 +919,24 @@ const everydayProfiles = [ ].map(productionStoryProfile); export const runnerSuites: readonly RunnerSuiteFixture[] = [ + { + id: "grok-subscription-qualification", label: "Grok Build Subscription Qualification", manualOnly: true, + description: "Explicit company subscription login across Grok browser workflows in local and Daytona environments.", + groups: ["native"], + profiles: [nativeProfile({ + id: "runner-acpx-grok-subscription", label: "Grok Build Subscription", + provider: "acpx", acpxAgent: "grok", + model: QUALIFIED_ACPX_PROFILES.grok.qualificationModel, + credential: "GROK_AUTH_JSON", + })], + environments: [localEnvironment, daytonaWarmEnvironment], + tasks: [ + ...runnerTasks, ...localIntegrityTasks, + ...everydayTasks.filter(task => task.id === "build-revise"), + ...chatHardeningTasks.filter(task => ["stop-new-resume", "continuity-restart"].includes(task.id)), + ], expectedMatrixSize: 16, + definitionMetadata: { version: 1, authentication: "company-subscription", binary: "1.0.13", model: "grok-4.7", scheduling: "explicit-only", repetitionsRequired: 3, artifactOracle: "independent-python-contract", stopBoundary: "provider-turn-started" }, + }, { id: "grok-qualification", label: "Grok Build Qualification", manualOnly: true, description: "Grok replies, planning approval, questions, downloadable artifacts, stop/resume and controller restart in local and Daytona environments.", diff --git a/tests/runner-e2e/grok-subscription-fixture.test.ts b/tests/runner-e2e/grok-subscription-fixture.test.ts new file mode 100644 index 00000000000..a14d0f693e8 --- /dev/null +++ b/tests/runner-e2e/grok-subscription-fixture.test.ts @@ -0,0 +1,54 @@ +import { mkdtemp, mkdir, readFile, realpath, stat, symlink, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { resolveManagedGrokHomeDir } from "../../packages/adapters/grok-local/src/server/grok-home.js"; +import { stageGrokSubscriptionFixture } from "./grok-subscription-fixture.js"; +import { findSecretLeak, normalizedSecrets, sanitizeJson } from "./redaction.js"; +import { buildPaperclipServerEnvironment } from "./harness-env.js"; + +const companyId = "11111111-2222-4333-8444-555555555555"; +const raw = JSON.stringify({ [`https://auth.x.ai::${companyId}`]: { + key: "fixture-access-credential", refresh_token: "fixture-refresh-credential", email: "fixture@example.invalid", +} }); +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function setup() { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "grok-fixture-"))); + roots.push(root); + const home = path.join(root, "paperclip-home"); + await mkdir(home); + return { PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root, PAPERCLIP_HOME: home, PAPERCLIP_INSTANCE_ID: "runner-e2e-test" }; +} + +it("stages only the explicit company credential privately and cleans refreshed state", async () => { + const environment = await setup(); + const remove = await stageGrokSubscriptionFixture({ raw, companyId, environment }); + const home = resolveManagedGrokHomeDir(environment, companyId); + expect(await readFile(path.join(home, "auth.json"), "utf8")).toBe(raw); + expect((await stat(home)).mode & 0o777).toBe(0o700); + expect((await stat(path.join(home, "auth.json"))).mode & 0o777).toBe(0o600); + await expect(stageGrokSubscriptionFixture({ raw, companyId, environment })).rejects.toThrow(); + await remove(); + await expect(stat(home)).rejects.toThrow(); +}); + +it("refuses malformed credentials and nonisolated or redirected homes", async () => { + const environment = await setup(); + await expect(stageGrokSubscriptionFixture({ raw: "{}", companyId, environment })).rejects.toThrow("malformed"); + await expect(stageGrokSubscriptionFixture({ raw, companyId: "../outside", environment })).rejects.toThrow("isolated"); + await expect(stageGrokSubscriptionFixture({ raw, companyId, environment: { ...environment, PAPERCLIP_HOME: tmpdir() } })).rejects.toThrow("outside"); + const home = resolveManagedGrokHomeDir(environment, companyId); + await mkdir(path.dirname(home), { recursive: true }); + await symlink(tmpdir(), home); + await expect(stageGrokSubscriptionFixture({ raw, companyId, environment })).rejects.toThrow("redirected"); +}); + +it("strips subscription input from the server environment and masks token fragments", () => { + const env = buildPaperclipServerEnvironment({ GROK_AUTH_JSON: raw, XAI_API_KEY: "other-key" }); + expect(env.GROK_AUTH_JSON).toBeUndefined(); + expect(env.XAI_API_KEY).toBeUndefined(); + const secrets = normalizedSecrets([raw]); + expect(findSecretLeak("fixture-refresh-credential", secrets)).toBeTruthy(); + expect(sanitizeJson({ message: "fixture-access-credential fixture@example.invalid" }, secrets)).toEqual({ message: "[REDACTED] [REDACTED]" }); +}); diff --git a/tests/runner-e2e/grok-subscription-fixture.ts b/tests/runner-e2e/grok-subscription-fixture.ts new file mode 100644 index 00000000000..26524ea592d --- /dev/null +++ b/tests/runner-e2e/grok-subscription-fixture.ts @@ -0,0 +1,42 @@ +import { mkdir, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { hasUsableGrokAuthValue, parseGrokAuthPayload, resolveManagedGrokHomeDir } from "../../packages/adapters/grok-local/src/server/grok-home.js"; + +function credential(raw: string) { + if (Buffer.byteLength(raw) > 256 * 1024) throw new Error("Grok fixture credential exceeds its bound"); + let parsed: unknown; + try { parsed = JSON.parse(raw); } catch { throw new Error("Grok fixture credential is malformed"); } + const result = parseGrokAuthPayload(parsed); + if (!result || !hasUsableGrokAuthValue(result.value)) throw new Error("Grok fixture credential is malformed"); + return result; +} + +/** Test setup only: seed the explicit login in a new disposable company slot. + * Production credential discovery, process fencing, and refresh remain unchanged. + */ +export async function stageGrokSubscriptionFixture(input: { + raw: string; + companyId: string; + environment: NodeJS.ProcessEnv; +}): Promise<() => Promise> { + credential(input.raw); + const env = input.environment; + const root = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!root || !/^[0-9a-f-]{36}$/i.test(input.companyId) || + !/^runner-e2e-[a-z0-9-]+$/.test(env.PAPERCLIP_INSTANCE_ID ?? "")) { + throw new Error("Grok fixture requires an isolated test company and instance"); + } + const canonicalRoot = await realpath(root); + const home = env.PAPERCLIP_HOME; + if (!home || await realpath(home) !== path.join(canonicalRoot, "paperclip-home")) { + throw new Error("Grok fixture refuses a home outside its isolated test root"); + } + const companyHome = resolveManagedGrokHomeDir(env, input.companyId); + await mkdir(companyHome, { recursive: true, mode: 0o700 }); + if (!(await realpath(companyHome)).startsWith(`${canonicalRoot}${path.sep}`)) { + throw new Error("Grok fixture refuses a redirected company home"); + } + // Never replace an existing company's login, including through a symlink. + await writeFile(path.join(companyHome, "auth.json"), input.raw, { flag: "wx", mode: 0o600 }); + return () => rm(companyHome, { recursive: true, force: true }); +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index afab17b6ead..6be582baf18 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -129,7 +129,8 @@ export function buildRunnerE2EProcessEnvironment( /** * Build the environment inherited by the Paperclip server. Paid credentials * deliberately stay in the launcher/Playwright process and cross the server - * boundary only once, in the encrypted company-secrets API request. + * boundary through encrypted company secrets. Explicit subscription fixtures + * stage their login in the disposable company's private credential home. */ export function buildPaperclipServerEnvironment( source: NodeJS.ProcessEnv, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index edb960b851b..7346ceb6dd0 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { isManagedHiringCase } from "./chat-cases.js"; import { FixtureRegistry } from "./fixture-registry.js"; +import { stageGrokSubscriptionFixture } from "./grok-subscription-fixture.js"; import type { RunnerApi } from "./api.js"; import type { CredentialName, @@ -150,6 +151,7 @@ export async function setupLiveFixtures(input: { const company = value(resolved, "company"); const refs: SecretReferenceMap = {}; for (const credentialName of execution.requiredCredentials) { + if (credentialName === "GROK_AUTH_JSON") continue; const rawValue = input.credentials[credentialName]; if (!rawValue) throw new Error(`Missing credential ${credentialName}`); const secret = await api.postSensitive( @@ -171,11 +173,29 @@ export async function setupLiveFixtures(input: { }, }); + const grokSubscription = execution.profile.credential === "GROK_AUTH_JSON"; + if (grokSubscription) { + registry.register<() => Promise>({ + id: "subscription-login", + dependencies: ["company"], + async setup(resolved) { + const raw = input.credentials.GROK_AUTH_JSON; + if (!raw) throw new Error("Missing credential GROK_AUTH_JSON"); + return stageGrokSubscriptionFixture({ + raw, companyId: value(resolved, "company").id, + environment: process.env, + }); + }, + async teardown(remove) { await remove(); }, + }); + } + registry.register({ id: "environment", dependencies: [ "company", "secrets", + ...(grokSubscription ? ["subscription-login"] : []), ...(execution.environment.id === "daytona" ? ["sandbox-provider"] : []), ], async setup(resolved) { @@ -252,6 +272,7 @@ export async function setupLiveFixtures(input: { "company", "secrets", "environment", + ...(grokSubscription ? ["subscription-login"] : []), ...(managedHiring ? ["ai-connection"] : []), ], async setup(resolved) { diff --git a/tests/runner-e2e/redaction.ts b/tests/runner-e2e/redaction.ts index d6316a6f591..ec9df5fd5c9 100644 --- a/tests/runner-e2e/redaction.ts +++ b/tests/runner-e2e/redaction.ts @@ -14,11 +14,27 @@ const SENSITIVE_JSON_KEY = export function normalizedSecrets(values: readonly (string | undefined)[]) { return [ - ...new Set( - values - .map((value) => value?.trim()) - .filter((value): value is string => Boolean(value)), - ), + ...new Set(values.flatMap((value) => { + const trimmed = value?.trim(); + if (!trimmed) return []; + if (trimmed.startsWith("{")) { + try { + const parsed = JSON.parse(trimmed) as unknown; + // Subscription credentials are structured JSON. A log may contain an + // individual token or identity rather than the whole serialized file. + const leaves = (value: unknown, depth = 0): string[] => { + if (depth > 8) return []; + if (typeof value === "string") return value.length >= 8 ? [value] : []; + if (!value || typeof value !== "object") return []; + return Object.entries(value).flatMap(([key, child]) => [ + ...(key.includes("::") ? [key] : []), ...leaves(child, depth + 1), + ]); + }; + return [trimmed, ...leaves(parsed)]; + } catch { /* Preserve ordinary non-JSON secrets. */ } + } + return [trimmed]; + })), ].sort((left, right) => right.length - left.length); } diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 8bb7f0df3a1..41726d2a6f0 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -3,6 +3,7 @@ export const CREDENTIAL_NAMES = [ "ANTHROPIC_API_KEY", "OPENROUTER_API_KEY", "XAI_API_KEY", + "GROK_AUTH_JSON", "DAYTONA_API_KEY", ] as const; diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index f02dfdd90be..03f937131b8 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -314,7 +314,7 @@ describe("public repository paid workflow security", () => { paidExecution, ); expect(everydayOracleStep).toContain( - "if: matrix.suiteId == 'everyday-workflows' && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", + "if: (matrix.suiteId == 'everyday-workflows' || matrix.suiteId == 'grok-qualification' || matrix.suiteId == 'grok-subscription-qualification') && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", ); expect(everydayOracleStep).toContain( `oracle_image='${everydayOracleImage}'`, @@ -527,6 +527,7 @@ describe("public repository paid workflow security", () => { ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'", XAI_API_KEY: "matrix.credentialName == 'XAI_API_KEY'", + GROK_AUTH_JSON: "matrix.credentialName == 'GROK_AUTH_JSON'", DAYTONA_API_KEY: "matrix.environmentId == 'daytona'", })) { expect(fullStack).toContain(