diff --git a/tests/runner-e2e/controller-outage.test.ts b/tests/runner-e2e/controller-outage.test.ts index edb1c9a5718..d366c3f1096 100644 --- a/tests/runner-e2e/controller-outage.test.ts +++ b/tests/runner-e2e/controller-outage.test.ts @@ -20,9 +20,10 @@ function fixture() { } describe("controller outage qualification", () => { - it("keeps original-execution fault evidence enabled for both dedicated continuity cells", async () => { + it("keeps original-execution fault evidence enabled for every dedicated continuity cell", async () => { const executions = selectRunnerExecutions(parseRunnerSelectors(["--suite", "daytona-restart-continuity"])); - expect(executions).toHaveLength(2); + // Only providers with production managed warm sessions belong in this lane (#18994). + expect(executions.map(execution => execution.profile.id)).toEqual(["runner-codex"]); for (const execution of executions) { const outage = controllerOutageForExecution(execution, {}); expect(outage).toEqual({ signal: "SIGKILL", holdMs: 300_000 }); diff --git a/tests/runner-e2e/operating-receipts.test.ts b/tests/runner-e2e/operating-receipts.test.ts index 4ce448949fd..cb23d7d4546 100644 --- a/tests/runner-e2e/operating-receipts.test.ts +++ b/tests/runner-e2e/operating-receipts.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/ import * as fs from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; -import { expect, it, vi } from "vitest"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js"; import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js"; import { importOperatingReceipt, operatingReceiptObservation, type EvalOperatingReceipt } from "./operating-receipts.js"; @@ -22,6 +22,10 @@ const receipt = (fields: Partial = {}): EvalOperatingRecei windowStartedAt: "2026-10-01T00:00:00Z", windowEndedAt: "2026-10-02T00:00:00Z", coverage: "partial", source: "infrastructure", provider: "fixture", classification: "estimated", costUsd: "0.2", pricingVersion: "fixture-pricing", ...fields }); const now = Date.parse("2026-10-03T00:00:00Z"); +// Paid admission reads the clock against the 31-day fixture allowance. Pin +// only Date, so filesystem and timer behavior stay real. +beforeEach(() => { vi.useFakeTimers({ toFake: ["Date"], now }); }); +afterEach(() => { vi.useRealTimers(); }); it("binds operating observations to the campaign, allowance, evidence and closed observation window", () => { expect(operatingReceiptObservation(receipt(), "fixture", overhead, evidence, now)).toMatchObject({ requestId: "overhead:artifact_storage", final: false }); diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 2726922acff..05d4ba9a38f 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -185,11 +185,12 @@ describe("Pi controls catalog admission", () => { }); it("pins the Pi 1 profile and versioned coverage while retaining active Stop identity", () => { // Current profile identity and native/control coverage retain their own fingerprints. + // Pi profile 37 is the deliberate one-time reset when release attestation landed (#18325). // Extended v3 states the strict file oracle's task-wide Bash limit. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 23, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 36, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-content-bound-native-read-both-runs", agentMemoryReadAuthority: "local-withheld-or-exact-remote-agent-run-file", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); - expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 10, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", profileVersion: 36, + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 23, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 37, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-content-bound-native-read-both-runs", agentMemoryReadAuthority: "local-withheld-or-exact-remote-agent-run-file", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); + expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 10, remoteProcExit: "separately-confirmed-absence-after-read-failure", taskCreation: "explicit-title-and-creation-response-id", profileVersion: 37, remoteProcessIdentity: "observer-pid-startTicks-bootId", controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 6, piFileArtifactTitle: "exact-filename", piFileCommandTransport: "fenced-bash-markdown-paste" }); @@ -199,7 +200,7 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "b8774bd124dd39a694f3b6f26c8c9c5be9278c213b34ffba08d9fe62c21e82b7", + "pi-native": "f204250f5ed9a8b255f1c9619186b957a9353aff97f592ad87e64256a2747959", "native-active-stop": "4654a2169bff96cf8377fdd62441f5cabea5c613f447e518830f6b1efd82c051", "rich-acp-warm-continuity": "7e8a0012f2f67fba03cd94403b096a07cfc95cdb7aa568b4c3fd18b852d735f3", "extended-harnesses": "d5bcf5de0ade3a54e4e1560e76feb5d8a787039d56d2379cd501626d85ed7418", diff --git a/tests/runner-e2e/pi-file-evidence.test.ts b/tests/runner-e2e/pi-file-evidence.test.ts index a66111a3786..ac40992aaeb 100644 --- a/tests/runner-e2e/pi-file-evidence.test.ts +++ b/tests/runner-e2e/pi-file-evidence.test.ts @@ -18,7 +18,7 @@ function fixture() { const tool = (seq: number, executionId: string, name: string, operation: string, phase: "start" | "progress" | "end", target: string | null, text = "", rawOutput: unknown = undefined) => { // Sanitized shape of the retained Pi stream: missing opening location, - // later resolved edit arguments, and stable bash name with command progress. + // later resolved edit arguments, and a bash opening name refined to the command. // This exercises TS normalization/projection, not the Rust implementation. const event = normalize({ type: "tool_call", toolCallId: executionId, tag: phase === "start" ? "tool_call" : "tool_call_update", title: name, kind: operation, @@ -83,7 +83,9 @@ describe("Pi edit, validation and public artifact oracle", () => { expect(result.verification).toMatchObject({ commandEvidence: "correlated_native_progress", exitEvidence: "native_structured_output_receipt", nativeStructuredExitCode: 0, typedExitCode: null }); expect(fixture().events.map(e => e.payload.prpEvent.payload.target)).toEqual([null, "extended-fixture.txt", "extended-fixture.txt", null, null, null]); - expect(fixture().events.slice(3).every(e => e.payload.prpEvent.payload.name === "bash")).toBe(true); + // The runner keeps the latest concrete ACP title (#19265): bash, then the exact command. + expect(fixture().events.slice(3).map(e => e.payload.prpEvent.payload.name)).toEqual(["bash", + piFileContract("fixture").validationCommand, piFileContract("fixture").validationCommand]); expect(result.verification.nativeFileAttribution).toBe("workspace_relative_display_target"); }); it.each(["before", "after"])("rejects an extra metadata bash call %s validation despite correct file and download bytes", placement => { @@ -91,6 +93,8 @@ describe("Pi edit, validation and public artifact oracle", () => { const metadata = structuredClone(f.events.slice(3)); for (const row of metadata) row.payload.prpEvent.payload.executionId = "metadata"; metadata[1]!.payload.prpEvent.payload.progress = "wc -c extended-fixture.txt (in_progress): wc -c extended-fixture.txt"; + // A refined title names the extra call by its own command; it still counts. + metadata[1]!.payload.prpEvent.payload.name = metadata[2]!.payload.prpEvent.payload.name = "wc -c extended-fixture.txt"; metadata[2]!.payload.prpEvent.payload.output = JSON.stringify({ content: [{ type: "text", text: "17 extended-fixture.txt\n" }], structuredContent: { output: "17 extended-fixture.txt\n", truncated: false, exit_code: 0 }, @@ -110,6 +114,8 @@ describe("Pi edit, validation and public artifact oracle", () => { ["missing edit", f => { f.events = f.events.slice(3); }], ["missing validation", f => { f.events = f.events.slice(0, 3); }], ["echo-only validation", f => { f.events[3]!.payload.prpEvent.payload.name = `echo ${piFileContract("fixture").validationMarker}`; f.events[5]!.payload.prpEvent.payload.name = f.events[3]!.payload.prpEvent.payload.name; }], + ["validation name regresses to bash", f => { f.events[5]!.payload.prpEvent.payload.name = "bash"; }], + ["validation renamed to another command", f => { f.events[4]!.payload.prpEvent.payload.name = f.events[5]!.payload.prpEvent.payload.name = "node -e 'console.log(1)'"; }], ["failed validation", f => { f.events[5]!.payload.prpEvent.payload.status = "failed"; }], ["unfinished validation", f => { f.events.pop(); }], ["unrelated validation turn", f => { f.events[3]!.payload.prpEvent.turnId = "other"; f.events[5]!.payload.prpEvent.turnId = "other"; }], diff --git a/tests/runner-e2e/pi-file-evidence.ts b/tests/runner-e2e/pi-file-evidence.ts index 0dbd7c0283f..61114816c23 100644 --- a/tests/runner-e2e/pi-file-evidence.ts +++ b/tests/runner-e2e/pi-file-evidence.ts @@ -95,7 +95,9 @@ export function gradePiFileEvidence(input: { requireEvidence(edits.length === 1, "one native edit required; final-only writes cannot qualify"); const edit = edits[0]!; requireEvidence(edit.p.name === "edit" && edit.p.target === c.filename, "exact native edit and relative target required"); - function lifecycle(end: typeof edit) { + // `names[0]` is the opening name. Later names may only refine it once, to + // the final name (the runner keeps the latest concrete ACP title, #19265). + function lifecycle(end: typeof edit, names: readonly unknown[] = [end.p.name]) { const matches = tools.filter(x => x.p.executionId === end.p.executionId); const starts = matches.filter(x => x.e.eventType === "tool.execution.started"); requireEvidence(typeof end.p.executionId === "string" && end.p.executionId.length > 0 @@ -105,7 +107,7 @@ export function gradePiFileEvidence(input: { && typeof end.e.normalizedSessionId === "string" && end.e.normalizedSessionId.length > 0 && starts.length === 1 && starts[0]!.row.seq < end.row.seq && matches.filter(x => x.e.eventType === "tool.execution.completed").length === 1 - && matches.every(x => x.p.schema === end.p.schema && x.p.name === end.p.name && x.p.operation === end.p.operation + && matches.every(x => x.p.schema === end.p.schema && names.includes(x.p.name) && x.p.operation === end.p.operation && x.row.seq >= starts[0]!.row.seq && x.row.seq <= end.row.seq && x.e.sourceKind === "runner" && x.e.turnId === end.e.turnId && x.e.normalizedSessionId === end.e.normalizedSessionId && x.e.sourceInstanceId === end.e.sourceInstanceId @@ -118,19 +120,27 @@ export function gradePiFileEvidence(input: { requireEvidence(x.p.target === end.p.target || (!resolved && x.p.target === null), "conflicting native tool target"); if (x.p.target !== null) resolved = true; } + let refined = false; + for (const x of matches) { + requireEvidence(!refined || x.p.name === end.p.name && x.p.name !== names[0], "conflicting or regressed native tool name"); + if (x.p.name !== names[0]) refined = true; + } return { start: starts[0]!, matches }; } lifecycle(edit); - const validations = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "execute" - && x.p.name === "bash"); + // Count every completed shell execution, whatever its name, so a refined + // title cannot hide a second command from the one-bash-call rule. + const validations = tools.filter(x => x.e.eventType === "tool.execution.completed" && x.p.operation === "execute"); requireEvidence(validations.length === 1, "one observed provider validation execution"); - const validation = validations[0]!, { start, matches } = lifecycle(validation); + const validationNames = ["bash", c.validationCommand] as const; + requireEvidence(validationNames.includes(validations[0]!.p.name as never), "exact native validation command required"); + const validation = validations[0]!, { start, matches } = lifecycle(validation, validationNames); requireEvidence(start.row.seq > edit.row.seq && validation.e.turnId === edit.e.turnId && validation.e.normalizedSessionId === edit.e.normalizedSessionId && validation.e.sourceInstanceId === edit.e.sourceInstanceId && validation.p.outputTruncated === false, "validation must follow the edit in the same native turn"); - // The common lifecycle preserves the opening name (bash). ACPX's resolved - // command is retained as progress, rather than replacing that stable name. + // The lifecycle opens as bash. The resolved command is retained as + // progress, and may also become the name once the runner sees it. const commandProgress = `${c.validationCommand} (in_progress): ${c.validationCommand}`; const allowedProgress = new Set([`bash (pending): [terminal] ${validation.p.executionId}`, `${c.validationCommand} (pending): ${c.validationCommand}`, commandProgress]); diff --git a/tests/runner-e2e/private-framework-report.test.ts b/tests/runner-e2e/private-framework-report.test.ts index d908e6a2277..2b70e822fd8 100644 --- a/tests/runner-e2e/private-framework-report.test.ts +++ b/tests/runner-e2e/private-framework-report.test.ts @@ -65,6 +65,8 @@ it("uses the real official Playwright JSON producer with failed errors, user ste expect(result.status).toBe("failed"); expect(result.errors.length).toBeGreaterThan(0); expect(result.steps[0].title).toBe("owned failed step"); expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true); }); +// This writes and hashes a 256MiB attachment. Alone it takes about 10s; with the +// whole suite running in parallel it can pass 30s, so it has its own budget. it("keeps the unchanged 256MiB packaging/archive caps with one canonical large attachment", async () => { const { packageEvidence } = await import("./evidence.js"); const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js"); @@ -138,7 +140,7 @@ it("keeps the unchanged 256MiB packaging/archive caps with one canonical large a const allure = await exportPilotAllure(projection, path.join(f.root, "allure"), binding, { schema: "paperclip.chat-completion-testops-selection/v1", ...binding, projectId: 1, planId: 10, jobId: 11, launchId: 12, allureJobRunId: "77", cases: [{ ...descriptor, caseId: 100 }] }); expect(allure[0]!.status).toBe("failed"); expect(JSON.parse(await readFile(path.join(projection, "normalized-results.json"), "utf8")).results[0].status).toBe("failed"); -}, 30000); +}, 120_000); it.each(["literal", "escaped", "base64-body", "escaped-json-body"])("deep-scans registered immutable raw JSON text: %s", async kind => { const f = await fixture(), canary = "synthetic-private-credential"; diff --git a/tests/runner-e2e/private-trust-workflow-security.test.ts b/tests/runner-e2e/private-trust-workflow-security.test.ts new file mode 100644 index 00000000000..a573370ef29 --- /dev/null +++ b/tests/runner-e2e/private-trust-workflow-security.test.ts @@ -0,0 +1,97 @@ +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +// Private archive and TestOps trust boundaries for paid Runner workflows. +// Split from workflow-security.test.ts, which owns the rest of the policy. +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +const workflows = path.join(repositoryRoot, ".github/workflows"); +const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"]; + +describe("private archive and TestOps workflow trust", () => { + it("isolates private archive trust from workflows that execute target-branch code", async () => { + const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8"); + const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8"); + const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]); + expect(subjects).toEqual(environments); + const recurring = source.split("\n test_recurring:")[1]!.split("\n report:")[0]!; + const report = source.split("\n report:")[1]!.split("\n publish_history:")[0]!; + for (const [index, job] of [recurring, report].entries()) { + expect(job).toContain(`name: ${environments[index]}`); + expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"); + } + expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); + for (const filename of await readdir(workflows)) { + if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; + const other = await readFile(path.join(workflows, filename), "utf8"); + // Checked against the reviewed #18516 rules below. + if (filename === "chat-catalog-testops-case.yml") continue; + + for (const environment of environments) expect(other, filename).not.toContain(environment); + } + const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8"); + expect(protocol).toContain("'runner-e2e-private-paid' || 'runner-e2e-paid'"); + const admission = protocol.split(" - name: Validate accounting mode and retry envelope")[1]!.split(" - name:")[0]!; + expect(admission).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"'); + const worker = protocol.split(" steps: &direct_eval_steps")[1]!.split(" - uses:")[0]!; + expect(worker).toContain("test '${{ needs.authorize.outputs.target_sha }}' = \"$GITHUB_SHA\""); + expect(worker).toContain('test "$GITHUB_RUN_ATTEMPT" = 1'); + }); + + it("admits the chat catalog cell to private trust and provider keys only through the guarded dispatcher", async () => { + const other = await readFile(path.join(workflows, "chat-catalog-testops-case.yml"), "utf8"); + // The reviewed #18516 pilot rules, carried to the full catalog's + // reusable cell. Trusted master code admits the dispatch and selection + // before any key; the pinned candidate never sees the TestOps token. + expect(other).toContain(" workflow_call:"); + expect(other).not.toMatch(/^\s+(?:workflow_dispatch|push|pull_request(?:_target)?|schedule):/mu); + expect(other).toContain("\n node tests/runner-e2e/chat-completion-testops-cli.mjs guard\n"); + expect(other).toContain("ref: ${{ github.sha }}"); + expect(other).toContain("TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); + const chat = other.split("\n chat:")[1]!.split("\n report:")[0]!; + expect(chat).toContain("environment: runner-e2e-private-paid"); + expect(chat).toContain(" - &guard"); + expect(chat).toContain("verify-selection"); + expect(chat).not.toContain("ALLURE_TOKEN"); + expect(chat.indexOf("chat-completion-testops-cli.mjs guard")).toBeLessThan(chat.indexOf("ref: ${{ inputs.target_sha }}")); + expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.OPENROUTER_API_KEY")); + expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.ANTHROPIC_API_KEY")); + const report = other.split("\n report:")[1]!.split("\n upload:")[0]!; + expect(report).toContain("environment: runner-e2e-private-history"); + expect(report).toContain("manual-private-transfer.ts report-session"); + expect(report).not.toContain("ALLURE_TOKEN"); + expect(report).not.toContain("ref: ${{ inputs.target_sha }}"); + const upload = other.split("\n upload:")[1]!; + for (const environment of environments) expect(upload).not.toContain(environment); + expect(upload).not.toMatch(/secrets\.(?!ALLURE_TOKEN\b)/u); + // Only the master-only catalog dispatcher calls this cell, with its own + // guard's pinned source and admitted jobs. + const callers = (await Promise.all((await readdir(workflows)).filter(name => /\.ya?ml$/u.test(name)) + .map(async name => [name, await readFile(path.join(workflows, name), "utf8")] as const))) + .filter(([, contents]) => contents.includes("uses: ./.github/workflows/chat-catalog-testops-case.yml")); + expect(callers.map(([name]) => name)).toEqual(["chat-catalog-testops.yml"]); + const dispatcher = callers[0]![1]; + expect(dispatcher).toMatch(/^on:\n {2}workflow_dispatch:/mu); + expect(dispatcher).toContain("github.workflow_ref == 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master'"); + expect(dispatcher.split("\n authorize:")[1]!.split("\n select:")[0]).toContain("run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard"); + const cellJobs = dispatcher.split(/\n(?= {2}[a-z_]+:\n)/u).filter(job => job.includes("uses: ./.github/workflows/chat-catalog-testops-case.yml")); + expect(cellJobs).toHaveLength(2); + for (const job of cellJobs) { + expect(job).toContain("needs: [authorize, select, build]"); + expect(job).toContain("target_sha: ${{ needs.authorize.outputs.target_sha }}"); + } + const keyed = dispatcher.split(/\n(?= {2}[a-z_]+:\n)/u).filter(job => /secrets\.(?:OPENROUTER|ANTHROPIC)_API_KEY/u.test(job)); + expect(keyed).toEqual(cellJobs); + }); + + it("keeps the full-stack TestOps selection and publication jobs on reviewed code", async () => { + const fullStack = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8"); + for (const [job, next] of [["testops_selection", "catalog"], ["publish_testops", "publish_history"]]) { + const trusted = fullStack.split(`\n ${job}:\n`)[1]!.split(`\n ${next}:\n`)[0]!; + expect(trusted, job).toContain("ref: ${{ github.sha }}"); + expect(trusted, job).not.toContain("ref: ${{ needs.authorize.outputs.target_sha }}"); + expect(trusted, job).not.toContain("Download resolved target lockfile"); + expect(trusted, job).not.toMatch(/pnpm install|secrets\.(?!ALLURE_TOKEN\b)/u); + } + }); +}); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index a9770f296cf..f3760eb3cca 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -19,7 +19,9 @@ delete env.NODE_PATH; delete env.NODE_OPTIONS; for (const [name, command] of [ - ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], + // Server modules under test import the built plugin SDK (and shared) package. + ["Plugin build dependencies", ["scripts/ensure-plugin-build-deps.mjs"]], + ["Vitest",[vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], ["Node tests", ["--test", "tests/runner-e2e/product-regression-testops.test.mjs", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs", diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index c61b2c68b13..3a6f8545f93 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -83,9 +83,10 @@ describe("public repository paid workflow security", () => { }); // PR #18348 added the ninth pr-trusted bootstrap, in the Vitest selection job. + // #19533 added the tenth, for the sandbox provider tests. const trustedPnpmBootstraps = { "runner-full-stack-e2e.yml": 8, - "pr-trusted.yml": 9, + "pr-trusted.yml": 10, } as const; it("keeps pnpm bootstrap registry telemetry out of trusted workflow setup", async () => { @@ -544,8 +545,10 @@ describe("public repository paid workflow security", () => { /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g, ), ).toHaveLength(6); - expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(4); - expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(10); + // PR #18983 added two trusted TestOps jobs. private-trust-workflow-security.test.ts + // checks that they check out only reviewed code. + expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(6); + expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(12); expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}"); expect(fullStack).toContain( "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}", @@ -609,6 +612,11 @@ describe("public repository paid workflow security", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", + // PR #18516 admitted guarded, pinned-source chat TestOps cells. The full + // catalog (#18542, #18647) replaced the pilot workflows that #18625 removed. + // private-trust-workflow-security.test.ts checks how these two hold the keys. + "chat-catalog-testops.yml", + "chat-catalog-testops-case.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -877,35 +885,6 @@ describe("public repository paid workflow security", () => { expect(marker).not.toMatch(/BUCKET|ROLE_ARN|private-report-reference/); }); - it("isolates private archive trust from workflows that execute target-branch code", async () => { - const workflows = path.join(repositoryRoot, ".github/workflows"); - const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8"); - const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8"); - const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"]; - const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]); - expect(subjects).toEqual(environments); - const recurring = source.split("\n test_recurring:")[1]!.split("\n report:")[0]!; - const report = source.split("\n report:")[1]!.split("\n publish_history:")[0]!; - for (const [index, job] of [recurring, report].entries()) { - expect(job).toContain(`name: ${environments[index]}`); - expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"); - } - expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); - for (const filename of await readdir(workflows)) { - if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; - const other = await readFile(path.join(workflows, filename), "utf8"); - - for (const environment of environments) expect(other, filename).not.toContain(environment); - } - const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8"); - expect(protocol).toContain("'runner-e2e-private-paid' || 'runner-e2e-paid'"); - const admission = protocol.split(" - name: Validate accounting mode and retry envelope")[1]!.split(" - name:")[0]!; - expect(admission).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"'); - const worker = protocol.split(" steps: &direct_eval_steps")[1]!.split(" - uses:")[0]!; - expect(worker).toContain("test '${{ needs.authorize.outputs.target_sha }}' = \"$GITHUB_SHA\""); - expect(worker).toContain('test "$GITHUB_RUN_ATTEMPT" = 1'); - }); - it("uses environment-scoped OIDC for a no-delete history publisher", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),