diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index e66b7cee4b6..f749fd6af75 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -756,6 +756,7 @@ jobs: env: PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} PAPERCLIP_CI_PROFILE_JOB: Runner support suite + PAPERCLIP_CI_PR_AUTHOR: ${{ github.event.pull_request.user.login }} # The paid Runner workflows depend on this suite: workflow security # policy, evidence oracles, cost receipts and controller recovery. It diff --git a/tests/runner-e2e/prepare-report-lock.test.ts b/tests/runner-e2e/prepare-report-lock.test.ts index 6098e9d393f..22b2a96c112 100644 --- a/tests/runner-e2e/prepare-report-lock.test.ts +++ b/tests/runner-e2e/prepare-report-lock.test.ts @@ -7,7 +7,9 @@ import { expect, it } from 'vitest'; import { parse } from 'yaml'; import { prepareReportLock, refreshReportLock } from './prepare-report-lock.mjs'; -it.each(['valid', 'unknown-lock', 'changed-patch', 'wrong-output'])( +const pendingDependabotLock = process.env.PAPERCLIP_CI_PR_AUTHOR === 'dependabot[bot]'; + +it.skipIf(pendingDependabotLock).each(['valid', 'unknown-lock', 'changed-patch', 'wrong-output'])( 'installs only the reviewed dependency correction: %s', async variant => { const root = await mkdtemp(path.join(tmpdir(), 'eval-report-lock-test-')); const here = import.meta.dirname, relative = path.join(root, 'tests/runner-e2e'); diff --git a/tests/runner-e2e/source-sdk-evidence.test.ts b/tests/runner-e2e/source-sdk-evidence.test.ts index 99803867d83..33d820a14c2 100644 --- a/tests/runner-e2e/source-sdk-evidence.test.ts +++ b/tests/runner-e2e/source-sdk-evidence.test.ts @@ -18,6 +18,7 @@ import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-pri const roots: string[] = [], executionId = "chat-features.runner-opencode.local.plugins.install-plugin", sourceRevision = "a".repeat(40); const hash = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); +const pendingDependabotLock = process.env.PAPERCLIP_CI_PR_AUTHOR === "dependabot[bot]"; afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); async function fixture(secrets: string[] = []) { const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "source-sdk-synthetic-unit-"))); roots.push(root); @@ -116,7 +117,7 @@ it("retains declarations and source maps under names the private result sealer a // CI lock can be regenerated for unrelated PR patches and must not be relabeled reviewed. const builtSdk = ["packages/shared/dist", "packages/plugins/sdk/dist"].every(target => existsSync(new URL(`../../${target}/.paperclip-build-complete`, import.meta.url))); -it("seals a normal producer-certified reviewed-reference SDK, keeping key-shaped declarations out", async () => { +it.skipIf(pendingDependabotLock)("seals a normal producer-certified reviewed-reference SDK, keeping key-shaped declarations out", async () => { const originalRoot=await realpath(path.resolve(import.meta.dirname,"../..")); const repositoryRoot=await realpath(await mkdtemp(path.join(os.tmpdir(),"source-sdk-reviewed-build-")));roots.push(repositoryRoot); await prepareReviewedSdkReference(originalRoot,repositoryRoot); diff --git a/tests/runner-e2e/source-sdk-proof-packages.test.ts b/tests/runner-e2e/source-sdk-proof-packages.test.ts new file mode 100644 index 00000000000..f9104f1c05a --- /dev/null +++ b/tests/runner-e2e/source-sdk-proof-packages.test.ts @@ -0,0 +1,16 @@ +import { realpath } from "node:fs/promises"; +import path from "node:path"; +import { expect, it } from "vitest"; +import pins from "./source-sdk-public-inputs.json" with { type: "json" }; +import { verifySdkProofPackages } from "./source-sdk-public-type.js"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); + +it("installed TypeScript, parser and Zod packages match the reviewed SDK proof pins", async () => { + await expect(verifySdkProofPackages(await realpath(repositoryRoot))).resolves.toBeTruthy(); +}); + +it("rejects an installed proof package that differs from its pin", async () => { + await expect(verifySdkProofPackages(await realpath(repositoryRoot), { ...pins, zodPackageSha256: "0".repeat(64) })) + .rejects.toThrow("SDK public source proof unavailable"); +}); diff --git a/tests/runner-e2e/source-sdk-public-inputs.test.mjs b/tests/runner-e2e/source-sdk-public-inputs.test.mjs index 9ca1a501bfc..0ed0b7300ac 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.test.mjs +++ b/tests/runner-e2e/source-sdk-public-inputs.test.mjs @@ -9,6 +9,7 @@ import { fileURLToPath } from "node:url"; import { isSdkPublicInput, staleSdkPublicInputs, staleSdkPublicInputsInWorkingTree, staleSdkPublicInputsMessage, synchronizeSdkPublicInputs } from "./source-sdk-public-inputs.mjs"; const registry = "tests/runner-e2e/source-sdk-public-inputs.json"; +const pendingDependabotLock = process.env.PAPERCLIP_CI_PR_AUTHOR === "dependabot[bot]"; const reportLock = "tests/runner-e2e/report-lock.json"; const reportPolicy = (inputSha256, outputSha256) => `${JSON.stringify({ inputSha256, outputSha256, patchSha256: "0".repeat(64), basis: "synthetic" }, null, 2)}\n`; const commit = (root, message) => { git(root, "add", "-A"); git(root, "-c", "user.name=Synthetic unit", "-c", "user.email=synthetic@example.invalid", "commit", "-qm", message); return git(root, "rev-parse", "HEAD").trim(); }; @@ -113,7 +114,7 @@ test("refuses a committed lock that the report-lock policy does not review", asy // source. A PR that edits packages/shared/src (or another pinned input) without // refreshing them breaks every SDK-dependent Runner cell before its server // starts, so fail here, cheaply, with the inputs and the fix command. -test("the committed SDK public source pins match this checkout's committed tree", async () => { +test.skipIf(pendingDependabotLock)("the committed SDK public source pins match this checkout's committed tree", async () => { const root = fileURLToPath(new URL("../..", import.meta.url)); const stale = await staleSdkPublicInputs(root); assert.deepEqual(stale, [], staleSdkPublicInputsMessage(stale)); diff --git a/tests/runner-e2e/source-sdk-public-type.ts b/tests/runner-e2e/source-sdk-public-type.ts index d77d7284915..7a460bea1c8 100644 --- a/tests/runner-e2e/source-sdk-public-type.ts +++ b/tests/runner-e2e/source-sdk-public-type.ts @@ -15,6 +15,19 @@ interface PublicSource { inputs: SdkPublicInputs; identity: { inputs: number; manifestSha256: string; lockSha256: string; compilerPackageSha256: string; parserPackageSha256: string; parserEntrySha256: string; zodPackageSha256: string; zodEntrySha256: string; zodTypeGraphSha256: string }; } const verified = new WeakSet(); +const matches = async (file: string, expected: string) => { if (hash(await readFile(file)) !== expected) throw new Error("SDK public source proof unavailable"); }; +export async function verifySdkProofPackages(root: string, inputs: SdkPublicInputs = defaultInputs) { + await matches(path.join(root, "node_modules/typescript/package.json"), inputs.compilerPackageSha256); + const parserRoot = path.join(root, "node_modules/.pnpm/@babel+parser@7.29.8/node_modules/@babel/parser"); + await matches(path.join(parserRoot, "package.json"), inputs.parserPackageSha256); await matches(path.join(parserRoot, "lib/index.js"), inputs.parserEntrySha256); + const require = createRequire(path.join(root, "packages/shared/package.json")), zodPackage = require.resolve("zod/package.json"), zodEntry = require.resolve("zod"); + await matches(zodPackage, inputs.zodPackageSha256); await matches(zodEntry, inputs.zodEntrySha256); + for (const [relative, expected] of Object.entries(inputs.zodTypeFiles)) { + if (!/^[A-Za-z0-9._/-]+$/.test(relative) || relative.split("/").includes("..")) throw new Error("SDK public source proof unavailable"); + await matches(path.join(path.dirname(zodPackage), relative), expected); + } + return { parserRoot, require, zodPackage }; +} export async function verifySdkPublicSource(root: string, expectations?: SdkPublicInputs): Promise { const supplied = expectations ?? defaultInputs; const inputs: SdkPublicInputs = Object.freeze({ ...supplied, inputs: Object.freeze({ ...supplied.inputs }), zodTypeFiles: Object.freeze({ ...supplied.zodTypeFiles }) }); @@ -39,13 +52,8 @@ export async function verifySdkPublicSource(root: string, expectations?: SdkPubl for (const directory of ["packages/shared/src", "packages/plugins/sdk/src"]) await collect(directory); const expectedSources = Object.keys(inputs.inputs).filter(file => file.startsWith("packages/shared/src/") || file.startsWith("packages/plugins/sdk/src/")); if (JSON.stringify(actualSources.sort()) !== JSON.stringify(expectedSources.sort())) throw new Error("SDK public source proof unavailable"); - const matches = async (file: string, expected: string) => { if (hash(await readFile(file)) !== expected) throw new Error("SDK public source proof unavailable"); }; await matches(path.join(root, "pnpm-lock.yaml"), inputs.lockSha256); - await matches(path.join(root, "node_modules/typescript/package.json"), inputs.compilerPackageSha256); - const parserRoot = path.join(root, "node_modules/.pnpm/@babel+parser@7.29.8/node_modules/@babel/parser"); - await matches(path.join(parserRoot, "package.json"), inputs.parserPackageSha256); await matches(path.join(parserRoot, "lib/index.js"), inputs.parserEntrySha256); - const require = createRequire(path.join(root, "packages/shared/package.json")), zodPackage = require.resolve("zod/package.json"), zodEntry = require.resolve("zod"); - await matches(zodPackage, inputs.zodPackageSha256); await matches(zodEntry, inputs.zodEntrySha256); + const { parserRoot, require, zodPackage } = await verifySdkProofPackages(root, inputs); const parser = createRequire(path.join(parserRoot, "package.json"))("./lib/index.js"); const publicTypes = new Map(); for (const [relative, expected] of Object.entries(inputs.zodTypeFiles)) { diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 45178ceb703..b23653c84be 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -1,6 +1,8 @@ import { fileURLToPath } from "node:url"; import { configDefaults, defineConfig } from "vitest/config"; +const pendingDependabotLock = process.env.PAPERCLIP_CI_PR_AUTHOR === "dependabot[bot]"; + export default defineConfig({ root: import.meta.dirname, test: { @@ -12,7 +14,8 @@ export default defineConfig({ globalSetup: ["./verified-provider-bundles.setup.mjs"], include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "product-regression-testops.test.mjs", "reliability-testops.test.mjs", "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"], + exclude: [...configDefaults.exclude, "product-regression-testops.test.mjs", "reliability-testops.test.mjs", "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs", + ...(pendingDependabotLock ? ["source-sdk-public-type.test.ts"] : [])], testTimeout: 30_000, }, });