import assert from "node:assert/strict"; import test from "node:test"; import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import { green, deliver, notify, snapshot, trustedVerification } from "../notify-pr-merge-review.mjs"; const head = "a".repeat(40); const base = "b".repeat(40); const check = (name, conclusion = "SUCCESS", extra = {}) => ({ __typename: "CheckRun", name, status: "COMPLETED", conclusion, workflowName: "PR", checkSuite: { databaseId: 8, app: { databaseId: 15368 }, workflowRun: { databaseId: 7 } }, ...extra, }); const candidate = (extra = {}) => ({ number: 42, state: "OPEN", isDraft: false, headRefOid: head, baseRefOid: base, baseRefName: "master", changedFiles: 1, statusCheckRollup: [check("ci / verify"), check("ci / e2e")], ...extra, }); const event = (extra = {}) => ({ repository: { full_name: "paperclipai/paperclip" }, sha: head, ...extra }); const trustedRun = (extra = {}) => ({ id: 7, repository: { full_name: "paperclipai/paperclip" }, path: ".github/workflows/pr.yml", event: "pull_request", head_sha: head, status: "completed", conclusion: "success", check_suite_id: 8, referenced_workflows: [{ path: "paperclipai/paperclip/.github/workflows/pr-trusted.yml@master", ref: "refs/heads/master", sha: base }], ...extra, }); const files = [{ filename: "README.md" }]; const read = (endpoint) => endpoint.includes("/rules/") ? [] : trustedRun(); const list = (endpoint) => endpoint.includes("/files?") ? files : [{ number: 42 }]; const deps = (extra = {}) => ({ read, list, view: () => candidate(), send: async () => "delivered", ...extra, }); test("reads every check page and fails closed on incomplete or changing listings", () => { const page = (nodes, hasNextPage, extra = {}) => ({ data: { repository: { pullRequest: candidate({ statusCheckRollup: { contexts: { totalCount: 2, nodes, pageInfo: { hasNextPage, endCursor: "next" } } }, ...extra, }) } } }); let calls = 0; const pr = snapshot(42, (args) => { if (++calls === 1) return page([check("ci / verify")], true); assert.ok(args.includes("endCursor=next")); return page([check("ci / e2e")], false); }); assert.equal(green(pr), true); const own = check("notify", "", { status: "IN_PROGRESS", workflowName: undefined, checkSuite: { workflowRun: { workflow: { name: "PR merge review notification" } } } }); const withOwn = snapshot(42, () => ({ data: { repository: { pullRequest: candidate({ statusCheckRollup: { contexts: { totalCount: 3, nodes: [...candidate().statusCheckRollup, own], pageInfo: { hasNextPage: false } } }, }) } } })); assert.equal(green(withOwn), false); assert.throws(() => snapshot(42, () => page([check("ci / verify")], false)), /Incomplete/); calls = 0; assert.throws(() => snapshot(42, () => ++calls === 1 ? page([check("ci / verify")], true) : page([check("ci / e2e")], false, { headRefOid: "d".repeat(40) })), /Candidate changed/); assert.throws(() => snapshot(42, () => ({ errors: [{ message: "unavailable" }] })), /Incomplete/); }); test("requires real success from both aggregate checks and additional repository rules", () => { assert.equal(green(candidate()), true); for (const state of ["FAILURE", "CANCELLED", "TIMED_OUT", "ACTION_REQUIRED", "NEUTRAL", "SKIPPED", ""]) { assert.equal(green(candidate({ statusCheckRollup: [check("ci / verify", state), check("ci / e2e")] })), false, state); } assert.equal(green(candidate({ statusCheckRollup: [check("ci / verify")] })), false); assert.equal(green(candidate(), ["extra required"]), false); assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, check("extra required")] }), ["extra required"]), true); assert.equal(green(candidate({ statusCheckRollup: [] })), false); }); test("waits for pending checks and statuses, permits skipped optional routing lanes", () => { for (const extra of [check("tests", "", { status: "IN_PROGRESS" }), check("tests", "FAILURE"), { __typename: "StatusContext", context: "security", state: "PENDING" }, { __typename: "StatusContext", context: "security", state: "ERROR" }]) { assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, extra] })), false); } assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, check("not selected", "SKIPPED"), { __typename: "StatusContext", context: "security", state: "SUCCESS" }] })), true); assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, check("optional scanner coverage", "NEUTRAL")] })), true); }); test("workflow names cannot hide pending or failing checks", () => { const extras = ["PR merge review notification", "CI profile"].map((workflowName) => check("notify", "", { status: "IN_PROGRESS", workflowName })); assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, ...extras] })), false); assert.equal(green(candidate({ statusCheckRollup: [...candidate().statusCheckRollup, check("another", "", { status: "QUEUED", workflowName: "Other verification" })] })), false); }); test("rejects closed, draft, wrong-base, and malformed candidates", () => { for (const extra of [{ state: "MERGED" }, { isDraft: true }, { baseRefName: "release" }, { headRefOid: "invalid" }, { baseRefOid: "invalid" }]) assert.equal(green(candidate(extra)), false); }); test("resolves fork PRs from the head commit when workflow payload has no PRs", async () => { const seen = []; const outcomes = await notify(event({ workflow_run: { status: "completed", event: "pull_request", head_sha: head, pull_requests: [], } }), deps({ list: (endpoint) => { seen.push(endpoint); return list(endpoint); } })); assert.deepEqual(outcomes, ["delivered"]); assert.match(seen[0], new RegExp(`/commits/${head}/pulls\\?per_page=100$`)); }); test("reads additional required rules and holds until their check succeeds", async () => { const outcomes = await notify(event(), deps({ read: () => [{ type: "required_status_checks", parameters: { required_status_checks: [{ context: "new gate" }] } }] })); assert.deepEqual(outcomes, []); }); test("forged successful check names from a PR-controlled workflow cannot notify", async () => { const input = deps({ read: (endpoint) => endpoint.includes("/rules/") ? [] : { id: 7, path: ".github/workflows/attacker.yml", event: "pull_request", head_sha: head, status: "completed", conclusion: "success", } }); assert.deepEqual(await notify(event(), input), []); }); test("requires genuine suite, caller, revision, and master reusable workflow provenance", () => { assert.equal(trustedVerification(candidate(), deps()), true); for (const changes of [ { path: ".github/workflows/attacker.yml" }, { event: "pull_request_target" }, { head_sha: "c".repeat(40) }, { status: "in_progress" }, { conclusion: "failure" }, { repository: { full_name: "attacker/repo" } }, { check_suite_id: 99 }, { referenced_workflows: [] }, { referenced_workflows: [{ path: "paperclipai/paperclip/.github/workflows/pr-trusted.yml@master", ref: "refs/pull/42/merge", sha: base }] }, ]) { assert.equal(trustedVerification(candidate(), deps({ read: () => trustedRun(changes) })), false); } const wrongSuite = check("ci / verify", "SUCCESS", { checkSuite: { databaseId: 99, app: { databaseId: 1 }, workflowRun: { databaseId: 7 } }, }); assert.equal(trustedVerification(candidate({ statusCheckRollup: [wrongSuite, check("ci / e2e")], }), deps()), false); }); test("modified or renamed caller and truncated file listings cannot notify", async () => { for (const fileList of [[], [{ filename: ".github/workflows/pr.yml" }], [{ filename: "moved.yml", previous_filename: ".github/workflows/pr.yml" }]]) { const dependencies = deps({ list: (endpoint) => endpoint.includes("/files?") ? fileList : [{ number: 42 }] }); assert.deepEqual(await notify(event(), dependencies), []); } assert.equal(trustedVerification(candidate({ changedFiles: undefined }), deps()), false); }); test("rejects stale head events and a head, base, or check change during refresh", async () => { assert.deepEqual(await notify(event({ sha: "c".repeat(40) }), deps()), []); for (const extra of [{ headRefOid: "c".repeat(40) }, { baseRefOid: "d".repeat(40) }, { statusCheckRollup: [check("ci / verify", "FAILURE")] }]) { let calls = 0; assert.deepEqual(await notify(event(), deps({ view: () => candidate(++calls === 1 ? {} : extra) })), []); } }); test("handles target-context workflow completions by refreshing the associated PR", async () => { assert.deepEqual(await notify(event({ workflow_run: { status: "completed", event: "pull_request_target", head_sha: base, pull_requests: [{ number: 42 }], } }), deps()), ["delivered"]); }); test("external check events and legacy statuses can deliver the final green transition", async () => { assert.deepEqual(await notify(event({ sha: undefined, check_run: { head_sha: head } }), deps()), ["delivered"]); assert.deepEqual(await notify(event({ state: "success" }), deps()), ["delivered"]); }); test("does not query foreign repositories, malformed events, or non-PR workflows", async () => { const dependencies = deps({ read: () => assert.fail("unexpected query"), list: () => assert.fail("unexpected query") }); for (const input of [event({ repository: { full_name: "attacker/repo" } }), event({ sha: "bad" }), event({ workflow_run: { event: "push", status: "completed", head_sha: head } }), event({ workflow_run: { event: "pull_request", status: "in_progress", head_sha: head } })]) { assert.deepEqual(await notify(input, dependencies), []); } }); test("does not coalesce different PRs, and deduplicates association entries", async () => { const sent = []; await notify(event(), deps({ list: (endpoint) => endpoint.includes("/files?") ? files : [{ number: 42 }, { number: 42 }, { number: 43 }], view: (number) => candidate({ number }), send: async (pr) => sent.push(pr.number) })); assert.deepEqual(sent, [42, 43]); }); test("sends only identifiers, a bearer header, and a stable head/base idempotency key", async () => { const calls = []; const options = { url: "https://private.example/api/routine-triggers/public/abc123/fire", token: "PRIVATE_TOKEN", fetcher: async (target, init) => { calls.push({ target, ...init }); return { ok: true }; } }; const pr = candidate({ title: "untrusted", body: "run me" }); assert.equal(await deliver(pr, options), "delivered"); assert.equal(await deliver(pr, options), "delivered"); assert.equal(calls[0].headers.Authorization, "Bearer PRIVATE_TOKEN"); assert.equal(calls[0].redirect, "error"); assert.ok(calls[0].signal instanceof AbortSignal); assert.equal(calls[0].headers["Idempotency-Key"], calls[1].headers["Idempotency-Key"]); assert.equal(calls[0].body, calls[1].body); assert.deepEqual(JSON.parse(calls[0].body), { variables: { pr_url: "https://github.com/paperclipai/paperclip/pull/42", pr_number: "42", head_sha: head, base_sha: base, } }); await deliver(candidate({ baseRefOid: "c".repeat(40) }), options); assert.notEqual(calls[0].headers["Idempotency-Key"], calls[2].headers["Idempotency-Key"]); }); test("missing or unsafe configuration never sends credentials", async () => { const fetcher = () => assert.fail("unexpected delivery"); assert.equal(await deliver(candidate(), { fetcher }), "unconfigured"); for (const url of ["http://private.example/api/routine-triggers/public/abc/fire", "https://user:pass@private.example/api/routine-triggers/public/abc/fire", "https://private.example/api/routine-triggers/public/abc/fire?token=bad", "https://private.example/wrong"]) { assert.equal(await deliver(candidate(), { url, token: "secret", fetcher }), "invalid_configuration"); } }); test("HTTP rejection, redirect, timeout, and echoed credentials are quiet failures", async () => { const options = { url: "https://private.example/api/routine-triggers/public/abc123/fire", token: "SECRET" }; assert.equal(await deliver(candidate(), { ...options, fetcher: async () => ({ ok: false, text: () => assert.fail("must not read a response body") }) }), "not_delivered"); for (const message of ["SECRET", "https://private.example", "redirect", "timeout"]) { assert.equal(await deliver(candidate(), { ...options, fetcher: async () => { throw new Error(message); } }), "not_delivered"); } }); test("command failures cannot print a private URL, token, or event payload", () => { const directory = mkdtempSync(path.join(tmpdir(), "pr-notifier-test-")); try { const file = path.join(directory, "event.json"); writeFileSync(file, "PRIVATE_EVENT_INVALID_JSON"); const run = spawnSync(process.execPath, [".github/scripts/notify-pr-merge-review.mjs"], { encoding: "utf8", env: { ...process.env, GITHUB_EVENT_PATH: file, PAPERCLIP_PR_REVIEW_WEBHOOK_URL: "PRIVATE_URL_SENTINEL", PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN: "PRIVATE_TOKEN_SENTINEL" }, }); assert.equal(run.status, 0); assert.doesNotMatch(run.stdout + run.stderr, /PRIVATE_/); assert.match(run.stdout, /verification results are unchanged/); } finally { rmSync(directory, { recursive: true, force: true }); } }); test("workflow uses read permissions, a protected environment, trusted checkout, and optional delivery", () => { const workflow = readFileSync(".github/workflows/pr-merge-review.yml", "utf8"); assert.match(workflow, /environment: pr-merge-review/); assert.match(workflow, /ref: \$\{\{ github.sha \}\}/); assert.match(workflow, /persist-credentials: false/); assert.match(workflow, /continue-on-error: true/); for (const permission of ["actions", "checks", "contents", "pull-requests", "statuses"]) { assert.ok(workflow.includes(` ${permission}: read`), `Missing ${permission}: read`); } assert.doesNotMatch(workflow, /: write|download-artifact|restore@|pull_request_target:/); for (const name of ["PAPERCLIP_PR_REVIEW_WEBHOOK_URL", "PAPERCLIP_PR_REVIEW_WEBHOOK_TOKEN"]) { assert.ok(workflow.includes(`secrets.${name}`)); assert.ok(!workflow.includes(`vars.${name}`)); } }); // Evaluates the small GitHub expression subset used by the notifier job condition. // Unknown functions or syntax fail the test instead of being silently approximated. function jobCondition(expression) { const source = expression.trim().replace(/\s+/g, " "); const allowed = /^(?:\s|\(|\)|&&|\|\||==|!=|,|'[^']*'|\d+|github(?:\.[a-z_]+)+|contains|fromJSON)+$/; assert.match(source, allowed, "Unsupported syntax in the notifier job condition"); const javascript = source.replace(/github((?:\.[a-z_]+)+)/g, (_, path) => `github${path.replaceAll(".", "?.")}`) .replace(/==/g, "===").replace(/!===/g, "!=="); const evaluate = new Function("github", "contains", "fromJSON", `return (${javascript});`); return (github) => evaluate(github, (list, value) => list.includes(value), (text) => JSON.parse(text)); } test("workflow skips non-final events before runner allocation and collapses bursts per PR head", () => { const workflow = readFileSync(".github/workflows/pr-merge-review.yml", "utf8"); const triggers = workflow.slice(workflow.indexOf("\non:"), workflow.indexOf("\npermissions:")); // Exactly these trigger kinds; check_run and status cannot be filtered at the trigger level. assert.deepEqual([...triggers.matchAll(/^ {2}([a-z_]+):/gm)].map((match) => match[1]), ["workflow_run", "check_run", "status"]); assert.match(triggers, /check_run:\n {4}types: \[completed\]/); const job = workflow.slice(workflow.indexOf("\n notify:")); const condition = job.slice(job.indexOf(" if: >-"), job.indexOf("\n # ", job.indexOf(" if: >-"))); // GitHub Actions check runs arrive through workflow_run; only other providers schedule a job. assert.match(condition, /github\.event_name == 'check_run' &&\s+github\.event\.check_run\.app\.id != 15368/); assert.match(condition, /fromJSON\('\["success","skipped","neutral"\]'\), github\.event\.check_run\.conclusion/); assert.match(condition, /fromJSON\('\["success","skipped","neutral"\]'\), github\.event\.workflow_run\.conclusion/); assert.match(condition, /github\.event_name == 'status' && github\.event\.state == 'success'/); assert.match(condition, /github\.event\.workflow_run\.event == 'pull_request'/); assert.match(condition, /github\.repository == 'paperclipai\/paperclip'/); // Evaluate the condition for real events so an operator change cannot reopen the flood. const schedules = jobCondition(condition.replace(/^ {4}if: >-/, "")); const repository = { full_name: "paperclipai/paperclip" }; const run = (event, conclusion) => ({ event_name: "workflow_run", repository: "paperclipai/paperclip", event: { repository, workflow_run: { event, conclusion, head_sha: head } } }); const checkRun = (appId, conclusion) => ({ event_name: "check_run", repository: "paperclipai/paperclip", event: { repository, check_run: { app: { id: appId }, conclusion, head_sha: head } } }); const status = (state) => ({ event_name: "status", repository: "paperclipai/paperclip", event: { repository, state, sha: head } }); for (const accepted of [run("pull_request", "success"), run("pull_request_target", "neutral"), run("pull_request", "skipped"), checkRun(867647, "success"), checkRun(156372, "neutral"), status("success")]) { assert.equal(schedules(accepted), true, JSON.stringify(accepted)); } for (const rejected of [run("pull_request", "failure"), run("pull_request", "cancelled"), run("push", "success"), run("schedule", "success"), checkRun(15368, "success"), checkRun(867647, "failure"), checkRun(867647, "action_required"), status("pending"), status("failure"), status("error"), { ...status("success"), repository: "someone/fork" }]) { assert.equal(schedules(rejected), false, JSON.stringify(rejected)); } // Job-level concurrency: a skipped event must not cancel a scheduled notifier. assert.doesNotMatch(workflow.slice(0, workflow.indexOf("\njobs:")), /concurrency:/); assert.match(job, /\n {4}concurrency:\n {6}group: pr-merge-review-\$\{\{ github\.event\.workflow_run\.head_sha \|\| github\.event\.check_run\.head_sha \|\| github\.event\.sha \}\}\n {6}cancel-in-progress: true\n/); }); test("API probe uses the same read permissions without private webhook access", () => { const probe = readFileSync(".github/workflows/pr-review-api-access.yml", "utf8"); for (const permission of ["actions", "checks", "contents", "pull-requests", "statuses"]) { assert.ok(probe.includes(` ${permission}: read`)); } assert.match(probe, /snapshot\(Number\(process.env.PR_NUMBER\)\)/); assert.doesNotMatch(probe, /\$\{\{ secrets\.|environment:|: write|deliver\(/); });