diff --git a/doc/runner-image-contract.md b/doc/runner-image-contract.md index 621ed998bd9..a6ab0fea197 100644 --- a/doc/runner-image-contract.md +++ b/doc/runner-image-contract.md @@ -226,16 +226,55 @@ the committed files, before any step that rewrites `pnpm-lock.yaml`. - `build_contexts`: one `--build-context` value on each line, which replace the `runner-artifacts` and `runner-artifacts-slim` stages; - `labels`: `io.paperclip.runner.image` and - `io.paperclip.runner.inputs-digest`, for the image that copies the Runner. + `io.paperclip.runner.inputs-digest`, for the image that copies the Runner; +- `build_args`: `PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST=`, the build + argument with which a build checks the Runner it is given. `published` needs a complete publication whose full index names these inputs -and both platforms. Every other answer is `inline`, with the last four values +and both platforms. Every other answer is `inline`, with the last five values empty: the lockfile is behind the Runner's manifests (`lock-behind`), the image does not exist (`absent`), a publication is `incomplete` or `broken`, the full index does not match (`mismatch`), or the registry refused, failed -or could not be reached (`unknown`). An inline build runs the Runner stages -itself and carries neither label. The command does not wait for a publication -and does not verify the attestation of the image. +or could not be reached (`unknown`). A tree whose `Dockerfile` declares no +`PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST` argument is `inline` too +(`unsupported`): its build could not refuse a Runner of other inputs, so +`inputs` runs no build for it. An inline build runs the Runner stages itself +and carries neither label. For an `inline` answer `resolve` prints a notice +with the reason. The command does not verify the attestation of the image: +the workflow that uses the answer does. + +### Waiting for a publication + +A `master` push starts its own run of `.github/workflows/runner-image.yml`, +which may still be publishing when an image build of the same commit asks. +`resolve --dir "$dir" --wait` is for that build. When the first answer is +`absent` or `incomplete`, it lists the runs of that workflow for the push of +`GITHUB_SHA` to `master` (the Actions API, with `GITHUB_TOKEN` and +`actions: read`) and looks again when they end. Only a run of the `push` +event on `master` in this repository counts: a run of the same commit for a +pull request, for the weekly schedule or from a manual start publishes +nothing and is never waited for. Every other first answer is final and starts +no wait. + +- **A run, not a time.** When every run of the commit has completed, the + command looks up the image once more, and that answer is final. A run that + failed, was cancelled, or published nothing because the lockfile is behind + ends the wait as soon as it ends. +- **Published before the run ends.** While a run is in progress the command + looks up the image every 30 seconds. The wait ends when the image is + published, not when the last job of the run ends. +- **Not listed yet.** A run can be missing from the list just after the push. + The command looks for it for two minutes, measured from the start of the + wait. A commit with no run by then is not the head of a `master` push. +- **Limit.** The wait ends 40 minutes after the push + (`RUNNER_IMAGE_WAIT_MINUTES` in the script). The push is the creation time + of the oldest run of the commit, so a build that starts late waits less, and + one that starts after the limit looks once and does not wait. +- **Actions API.** Three failed requests in a row end the wait. Without a + token or a full commit SHA there is no wait. +- **Never an error.** After every ending the command looks up the image one + last time. Without an image the answer is `inline`, with the reason of that + lookup and what the wait saw. ## Publication diff --git a/scripts/runner-image-resolve.mjs b/scripts/runner-image-resolve.mjs index 889dfc285e1..f5d88e7ad1c 100644 --- a/scripts/runner-image-resolve.mjs +++ b/scripts/runner-image-resolve.mjs @@ -29,9 +29,21 @@ import { export const RUNNER_IMAGE_LABEL = "io.paperclip.runner.image"; export const RUNNER_INPUTS_DIGEST_LABEL = "io.paperclip.runner.inputs-digest"; +export const RUNNER_EXPECT_INPUTS_DIGEST_ARGUMENT = "PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST"; +export const RUNNER_IMAGE_WAIT_MINUTES = 40; +export const RUNNER_IMAGE_WAIT = Object.freeze({ + limitMs: RUNNER_IMAGE_WAIT_MINUTES * 60_000, listingMs: 2 * 60_000, pollMs: 30_000, failures: 3, +}); +export const RUNNER_IMAGE_WORKFLOW_PATH = ".github/workflows/runner-image.yml"; +const REPOSITORY = "paperclipai/paperclip"; +const REPOSITORY_ID = 1170821064; +const WAITS_FOR = Object.freeze(["absent", "incomplete"]); const INPUTS_FILE = "runner-image-inputs.json"; const HEX_64 = /^[0-9a-f]{64}$/; const DIGEST = /^sha256:[0-9a-f]{64}$/; +const REVISION = /^[0-9a-f]{40}$/; +const EXPECTS_INPUTS_DIGEST = new RegExp(`^ARG ${RUNNER_EXPECT_INPUTS_DIGEST_ARGUMENT}(?:=.*)?$`, "m"); +const message = (error) => (error instanceof Error ? error.message : String(error)); /** * @param {{ @@ -40,11 +52,21 @@ const DIGEST = /^sha256:[0-9a-f]{64}$/; * run?: (command: string, args: string[], options: object) => { status: number | null, error?: Error }, * lockState?: typeof runnerLockState, * }} input - * @returns {{ lock: "current" | "behind", detail: string, inputsDigest: string | null }} + * @returns {{ consumable: boolean, lock: "current" | "behind" | null, detail: string, inputsDigest: string | null }} */ export function readRunnerInputs({ repoRoot, directory, run = spawnSync, lockState = runnerLockState }) { assert.ok(!existsSync(directory) || readdirSync(directory).length === 0, `${directory} is not empty`); mkdirSync(directory, { recursive: true }); + const write = (inputs) => { + writeFileSync(path.join(directory, INPUTS_FILE), `${JSON.stringify(inputs, null, 2)}\n`); + return inputs; + }; + if (!EXPECTS_INPUTS_DIGEST.test(readFileSync(path.join(repoRoot, "Dockerfile"), "utf8"))) { + return write({ + consumable: false, lock: null, inputsDigest: null, + detail: `the Dockerfile declares no ${RUNNER_EXPECT_INPUTS_DIGEST_ARGUMENT} argument, so its build cannot check a Runner it is given`, + }); + } const build = (target, name) => { const destination = path.join(directory, name); // Build output goes to stderr: stdout carries only this command's result. @@ -61,9 +83,7 @@ export function readRunnerInputs({ repoRoot, directory, run = spawnSync, lockSta inputsDigest = JSON.parse(readFileSync(identity, "utf8")).digest; assert.match(inputsDigest ?? "", HEX_64, "The build computed no inputs digest"); } - const inputs = { lock: lock.state, detail: lock.detail, inputsDigest }; - writeFileSync(path.join(directory, INPUTS_FILE), `${JSON.stringify(inputs, null, 2)}\n`); - return inputs; + return write({ consumable: true, lock: lock.state, detail: lock.detail, inputsDigest }); } function indexProblem(bytes, inputsDigest) { @@ -80,14 +100,15 @@ function indexProblem(bytes, inputsDigest) { } /** - * @param {{ lock: "current" | "behind", inputsDigest: string | null }} inputs + * @param {{ consumable: boolean, lock: "current" | "behind" | null, detail?: string, inputsDigest: string | null }} inputs * @param {{ read?: typeof readRunnerImageManifest, credentials?: { username: string, password: string } | null }} [options] */ -export async function resolveRunnerImage({ lock, inputsDigest }, { read = readRunnerImageManifest, credentials = null } = {}) { - assert.ok(lock === "current" || lock === "behind", "The lockfile state must be current or behind"); +export async function resolveRunnerImage({ consumable, lock, detail: inputsDetail, inputsDigest }, { read = readRunnerImageManifest, credentials = null } = {}) { const inline = (reason, detail) => ({ - mode: "inline", reason, detail, inputsDigest: inputsDigest ?? "", image: "", slimImage: "", buildContexts: [], labels: [], + mode: "inline", reason, detail, inputsDigest: inputsDigest ?? "", image: "", slimImage: "", buildContexts: [], labels: [], buildArgs: [], }); + if (consumable !== true) return inline("unsupported", inputsDetail ?? "the build of this tree cannot check a Runner it is given"); + assert.ok(lock === "current" || lock === "behind", "The lockfile state must be current or behind"); if (lock === "behind") return inline("lock-behind", "pnpm-lock.yaml is behind the Runner's manifests, so no image is published for this tree"); assert.match(inputsDigest ?? "", HEX_64, "A tree with a current lockfile needs an inputs digest"); try { @@ -107,26 +128,96 @@ export async function resolveRunnerImage({ lock, inputsDigest }, { read = readRu mode: "published", reason: "published", detail: publication.detail, inputsDigest, image, slimImage, buildContexts: [`runner-artifacts=docker-image://${image}`, `runner-artifacts-slim=docker-image://${slimImage}`], labels: [`${RUNNER_IMAGE_LABEL}=${image}`, `${RUNNER_INPUTS_DIGEST_LABEL}=${inputsDigest}`], + buildArgs: [`${RUNNER_EXPECT_INPUTS_DIGEST_ARGUMENT}=${inputsDigest}`], }; } catch (error) { - return inline("unknown", `the lookup failed: ${error instanceof Error ? error.message : String(error)}`); + return inline("unknown", `the lookup failed: ${message(error)}`); } } +export function runnerImageRuns(runs, sha) { + return runs.filter((run) => run?.head_sha === sha && run.event === "push" && run.head_branch === "master" + && run.path === RUNNER_IMAGE_WORKFLOW_PATH && run.repository?.id === REPOSITORY_ID && run.head_repository?.id === REPOSITORY_ID); +} + +export async function listRunnerImageRuns(sha, { token = "", fetchImpl = fetch } = {}) { + assert.match(sha ?? "", REVISION, "The commit must be a full SHA"); + assert.ok(token, "A token is needed to list workflow runs"); + const response = await fetchImpl( + `https://api.github.com/repos/${REPOSITORY}/actions/workflows/runner-image.yml/runs?head_sha=${sha}&event=push&branch=master&per_page=30`, + { redirect: "error", headers: { authorization: `Bearer ${token}`, accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }, + ); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`GitHub API: HTTP ${response.status}`); + } + const body = await response.json(); + assert.ok(Array.isArray(body?.workflow_runs), "GitHub API returned no list of workflow runs"); + return body.workflow_runs; +} + +export async function waitForRunnerImage({ + sha, listRuns, resolve, now = Date.now, sleep = (ms) => new Promise((done) => setTimeout(done, ms)), limits = RUNNER_IMAGE_WAIT, +}) { + const started = now(); + const waited = () => `${Math.round((now() - started) / 60_000)} min`; + let failures = 0; + for (;;) { + let runs = null; + try { + runs = runnerImageRuns(await listRuns(sha), sha); + failures = 0; + } catch (error) { + failures += 1; + if (failures >= limits.failures) { + return { outcome: "unknown", detail: `could not list the Runner image runs of ${sha} (${message(error)})` }; + } + } + if (runs?.length === 0 && now() - started >= limits.listingMs) { + return { outcome: "not-listed", detail: `no Runner image run of a master push of ${sha} was listed in ${waited()}` }; + } + if (runs?.length > 0) { + const named = runs.map((run) => `${run.html_url ?? run.id} (${run.status === "completed" ? run.conclusion : run.status})`).join(", "); + if (runs.every((run) => run.status === "completed")) { + return { outcome: "completed", detail: `waited ${waited()} for the Runner image run ${named}` }; + } + const resolved = await resolve(); + if (resolved.mode === "published") return { outcome: "published", resolved, detail: `waited ${waited()} for the Runner image run ${named}` }; + const pushed = Math.min(started, ...runs.map((run) => Date.parse(run.created_at)).filter(Number.isFinite)); + if (now() - pushed >= limits.limitMs) { + return { + outcome: "limit", + detail: `the Runner image run ${named} did not finish within ${Math.round(limits.limitMs / 60_000)} min of the push, after a wait of ${waited()}`, + }; + } + } + await sleep(limits.pollMs); + } +} + +export async function resolveRunnerImageAfterWait({ resolve, wait }) { + const first = await resolve(); + if (!WAITS_FOR.includes(first.reason)) return first; + const waited = await wait(resolve); + const last = waited.resolved ?? await resolve(); + return { ...last, detail: `${last.detail}; ${waited.detail}` }; +} + /** @param {string[]} argv */ export function parseResolveArguments(argv) { const [command, flag, directory, ...rest] = argv; - if (!["inputs", "resolve"].includes(command) || flag !== "--dir" || !directory || directory.startsWith("--") || rest.length > 0) { - throw new Error("Usage: runner-image-resolve.mjs inputs|resolve --dir "); + const wait = command === "resolve" && rest.length === 1 && rest[0] === "--wait"; + if (!["inputs", "resolve"].includes(command) || flag !== "--dir" || !directory || directory.startsWith("--") || (rest.length > 0 && !wait)) { + throw new Error("Usage: runner-image-resolve.mjs inputs --dir | resolve --dir [--wait]"); } - return { command, directory: path.resolve(directory) }; + return { command, directory: path.resolve(directory), wait }; } /** * @param {string[]} argv * @param {{ * env?: NodeJS.ProcessEnv, repoRoot?: string, read?: typeof readRunnerImageManifest, readInputs?: typeof readRunnerInputs, - * log?: (line: string) => void, + * log?: (line: string) => void, waitFor?: typeof waitForRunnerImage, fetchImpl?: typeof fetch, * }} [options] * @returns {Promise>} */ @@ -136,15 +227,35 @@ export async function runResolveCommand(argv, { read = readRunnerImageManifest, readInputs = readRunnerInputs, log = console.error, + waitFor = waitForRunnerImage, + fetchImpl = fetch, } = {}) { - const { command, directory } = parseResolveArguments(argv); + const { command, directory, wait } = parseResolveArguments(argv); if (command === "inputs") { const inputs = readInputs({ repoRoot, directory }); - return { lock: inputs.lock, inputs_digest: inputs.inputsDigest ?? "" }; + return { lock: inputs.lock ?? "", inputs_digest: inputs.inputsDigest ?? "" }; } const inputs = JSON.parse(readFileSync(path.join(directory, INPUTS_FILE), "utf8")); - const resolved = await resolveRunnerImage(inputs, { read, credentials: environmentCredentials(env) }); + const resolve = () => resolveRunnerImage(inputs, { read, credentials: environmentCredentials(env) }); + const sha = env.GITHUB_SHA; + const token = env.GITHUB_TOKEN; + const resolved = wait + ? await resolveRunnerImageAfterWait({ + resolve, + wait: async () => { + if (!REVISION.test(sha ?? "") || !token) return { outcome: "unknown", detail: "no commit or no token to find its Runner image run with" }; + try { + return await waitFor({ sha, resolve, listRuns: (commit) => listRunnerImageRuns(commit, { token, fetchImpl }) }); + } catch (error) { + return { outcome: "unknown", detail: `the wait failed: ${message(error)}` }; + } + }, + }) + : await resolve(); log(`Runner image: ${resolved.mode} (${resolved.reason}: ${resolved.detail})`); + if (resolved.mode === "inline") { + log(`::notice title=Runner built inline::This image build makes its own Runner (${resolved.reason}: ${resolved.detail}).`); + } if (env.GITHUB_STEP_SUMMARY) { appendFileSync(env.GITHUB_STEP_SUMMARY, [ `Runner for an image build of this tree: **${resolved.mode}** (\`${resolved.reason}\`: ${resolved.detail})`, @@ -161,6 +272,7 @@ export async function runResolveCommand(argv, { slim_image: resolved.slimImage, build_contexts: resolved.buildContexts.join("\n"), labels: resolved.labels.join("\n"), + build_args: resolved.buildArgs.join("\n"), }; } diff --git a/server/src/__tests__/runner-image-resolve.test.ts b/server/src/__tests__/runner-image-resolve.test.ts index 752f3b116df..1bea4281180 100644 --- a/server/src/__tests__/runner-image-resolve.test.ts +++ b/server/src/__tests__/runner-image-resolve.test.ts @@ -6,10 +6,15 @@ import { afterAll, describe, expect, it } from "vitest"; import { readRunnerImageManifest, runnerImageTags, SLIM_DIGEST_ANNOTATION } from "../../../scripts/runner-image-publish.mjs"; import { formatOutputs, + listRunnerImageRuns, parseResolveArguments, readRunnerInputs, resolveRunnerImage, + resolveRunnerImageAfterWait, + RUNNER_IMAGE_WAIT, + RUNNER_IMAGE_WAIT_MINUTES, runResolveCommand, + waitForRunnerImage, } from "../../../scripts/runner-image-resolve.mjs"; const INPUTS = "1".repeat(64); @@ -33,8 +38,15 @@ const fullIndex = (annotations: Record, manifests = platforms("a present({ schemaVersion: 2, manifests, annotations }); const full = fullIndex({ "io.paperclip.runner.inputs-digest": INPUTS, [SLIM_DIGEST_ANNOTATION]: slim.digest }); const registry = (tags: Record) => async (reference: string): Promise => tags[reference] ?? { state: "absent" }; -const current = { lock: "current" as const, inputsDigest: INPUTS }; -const NOTHING = { image: "", slimImage: "", buildContexts: [], labels: [] }; +const current = { consumable: true, lock: "current" as const, inputsDigest: INPUTS }; +const NOTHING = { image: "", slimImage: "", buildContexts: [], labels: [], buildArgs: [] }; +const ASSERTING_DOCKERFILE = "FROM base AS build\nARG PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST=\"\"\n"; +function checkout(dockerfile = ASSERTING_DOCKERFILE) { + const root = directory(); + mkdirSync(root, { recursive: true }); + writeFileSync(path.join(root, "Dockerfile"), dockerfile); + return root; +} describe("the Runner an image build uses", () => { it("is the published image, pinned by digest, with the slim image its full index names", async () => { @@ -49,8 +61,9 @@ describe("the Runner an image build uses", () => { `runner-artifacts-slim=docker-image://${IMAGE}@${slim.digest}`, ], labels: [`io.paperclip.runner.image=${IMAGE}@${full.digest}`, `io.paperclip.runner.inputs-digest=${INPUTS}`], + buildArgs: [`PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST=${INPUTS}`], }); - expect(Object.keys(resolved).sort()).toEqual(["buildContexts", "detail", "image", "inputsDigest", "labels", "mode", "reason", "slimImage"]); + expect(Object.keys(resolved).sort()).toEqual(["buildArgs", "buildContexts", "detail", "image", "inputsDigest", "labels", "mode", "reason", "slimImage"]); }); it.each<[string, string, Record]>([ @@ -89,15 +102,24 @@ describe("the Runner an image build uses", () => { .toMatchObject({ mode: "inline", reason: "unknown", detail: expect.stringContaining("does not match the manifest bytes"), ...NOTHING }); let reads = 0; const counting = async (): Promise => { reads += 1; return full; }; - expect(await resolveRunnerImage({ lock: "behind", inputsDigest: null }, { read: counting })) + expect(await resolveRunnerImage({ consumable: true, lock: "behind", inputsDigest: null }, { read: counting })) .toMatchObject({ mode: "inline", reason: "lock-behind", inputsDigest: "", ...NOTHING }); expect(reads).toBe(0); }); + it("is inline, without a lookup, for a tree whose build cannot check the Runner it is given", async () => { + let reads = 0; + const counting = async (): Promise => { reads += 1; return full; }; + for (const inputs of [{ ...current, consumable: false }, { lock: "current" as const, inputsDigest: INPUTS } as typeof current]) { + expect(await resolveRunnerImage(inputs, { read: counting })).toMatchObject({ mode: "inline", reason: "unsupported", ...NOTHING }); + } + expect(reads).toBe(0); + }); + it("refuses inputs it cannot name instead of resolving them", async () => { const read = registry({ [TAGS.full]: full, [TAGS.slim]: slim }); - await expect(resolveRunnerImage({ lock: "current", inputsDigest: null }, { read })).rejects.toThrow(/needs an inputs digest/); - await expect(resolveRunnerImage({ lock: "stale" as "current", inputsDigest: INPUTS }, { read })).rejects.toThrow(/current or behind/); + await expect(resolveRunnerImage({ consumable: true, lock: "current", inputsDigest: null }, { read })).rejects.toThrow(/needs an inputs digest/); + await expect(resolveRunnerImage({ consumable: true, lock: "stale" as "current", inputsDigest: INPUTS }, { read })).rejects.toThrow(/current or behind/); }); }); @@ -119,15 +141,16 @@ describe("the inputs of a tree", () => { it("exports the build context, checks its lockfile, then computes the digest in the build", () => { const calls: Call[] = []; const out = directory(); + const repo = checkout(); const checked: string[] = []; const inputs = readRunnerInputs({ - repoRoot: "/repo", directory: out, run: docker(calls), + repoRoot: repo, directory: out, run: docker(calls), lockState: (context: string) => { checked.push(context); return lock("current")(); }, }); - expect(inputs).toEqual({ lock: "current", detail: "lockfile current", inputsDigest: INPUTS }); + expect(inputs).toEqual({ consumable: true, lock: "current", detail: "lockfile current", inputsDigest: INPUTS }); expect(calls.map((call) => [call.command, ...call.args])).toEqual([ - ["docker", "buildx", "build", "--target", "runner-artifacts-context", "--output", `type=local,dest=${path.join(out, "context")}`, "/repo"], - ["docker", "buildx", "build", "--target", "runner-artifacts-identity", "--output", `type=local,dest=${path.join(out, "identity")}`, "/repo"], + ["docker", "buildx", "build", "--target", "runner-artifacts-context", "--output", `type=local,dest=${path.join(out, "context")}`, repo], + ["docker", "buildx", "build", "--target", "runner-artifacts-identity", "--output", `type=local,dest=${path.join(out, "identity")}`, repo], ]); expect(checked).toEqual([path.join(out, "context")]); expect(JSON.parse(readFileSync(path.join(out, "runner-image-inputs.json"), "utf8"))).toEqual(inputs); @@ -135,15 +158,27 @@ describe("the inputs of a tree", () => { it("computes no digest for a lockfile that is behind: no image is published for that tree", () => { const calls: Call[] = []; - const inputs = readRunnerInputs({ repoRoot: "/repo", directory: directory(), run: docker(calls), lockState: lock("behind") }); - expect(inputs).toEqual({ lock: "behind", detail: "lockfile behind", inputsDigest: null }); + const inputs = readRunnerInputs({ repoRoot: checkout(), directory: directory(), run: docker(calls), lockState: lock("behind") }); + expect(inputs).toEqual({ consumable: true, lock: "behind", detail: "lockfile behind", inputsDigest: null }); expect(calls).toHaveLength(1); }); + it.each<[string, string]>([ + ["has no such argument", "FROM base AS build\nARG PAPERCLIP_BUILD_COMMIT=\"\"\n"], + ["only names the argument in other text", "FROM base AS build\nRUN echo ARG PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST\nARG PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST_OLD=1\n"], + ])("runs no build for a Dockerfile that %s: its build could not refuse another Runner", (_label, dockerfile) => { + const calls: Call[] = []; + const out = directory(); + const inputs = readRunnerInputs({ repoRoot: checkout(dockerfile), directory: out, run: docker(calls), lockState: lock("current") }); + expect(inputs).toMatchObject({ consumable: false, lock: null, inputsDigest: null }); + expect(calls).toEqual([]); + expect(JSON.parse(readFileSync(path.join(out, "runner-image-inputs.json"), "utf8"))).toEqual(inputs); + }); + it("fails, and leaves no result, when a build fails or reports no digest", () => { for (const options of [{ fail: "runner-artifacts-context" }, { fail: "runner-artifacts-identity" }, { digest: "latest" }]) { const out = directory(); - expect(() => readRunnerInputs({ repoRoot: "/repo", directory: out, run: docker([], options), lockState: lock("current") })) + expect(() => readRunnerInputs({ repoRoot: checkout(), directory: out, run: docker([], options), lockState: lock("current") })) .toThrow(/failed \(exit 1\)|no inputs digest/); expect(existsSync(path.join(out, "runner-image-inputs.json"))).toBe(false); } @@ -154,7 +189,7 @@ describe("the inputs of a tree", () => { mkdirSync(out); writeFileSync(path.join(out, "runner-image-inputs.json"), JSON.stringify(current)); const calls: Call[] = []; - expect(() => readRunnerInputs({ repoRoot: "/repo", directory: out, run: docker(calls), lockState: lock("current") })).toThrow(/is not empty/); + expect(() => readRunnerInputs({ repoRoot: checkout(), directory: out, run: docker(calls), lockState: lock("current") })).toThrow(/is not empty/); expect(calls).toEqual([]); }); }); @@ -167,7 +202,7 @@ describe("command line", () => { const stub = ({ directory: target }: { directory: string }) => { mkdirSync(target, { recursive: true }); writeFileSync(path.join(target, "runner-image-inputs.json"), JSON.stringify(current)); - return { lock: "current" as const, detail: "", inputsDigest: INPUTS }; + return { consumable: true, lock: "current" as const, detail: "", inputsDigest: INPUTS }; }; await expect(runResolveCommand(["inputs", "--dir", out], { ...quiet, env: {}, readInputs: stub })).resolves.toEqual({ lock: "current", inputs_digest: INPUTS }); @@ -187,12 +222,15 @@ describe("command line", () => { slim_image: `${IMAGE}@${slim.digest}`, build_contexts: `runner-artifacts=docker-image://${IMAGE}@${full.digest}\nrunner-artifacts-slim=docker-image://${IMAGE}@${slim.digest}`, labels: `io.paperclip.runner.image=${IMAGE}@${full.digest}\nio.paperclip.runner.inputs-digest=${INPUTS}`, + build_args: `PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST=${INPUTS}`, }); expect(seen).toEqual([{ username: "u", password: "p" }, { username: "u", password: "p" }]); expect(readFileSync(summary, "utf8")).toContain(`**published**`); const inline = await runResolveCommand(["resolve", "--dir", out], { ...quiet, env: {}, read: registry({}) }); - expect(inline).toEqual({ mode: "inline", reason: "absent", inputs_digest: INPUTS, image: "", slim_image: "", build_contexts: "", labels: "" }); + expect(inline).toEqual({ + mode: "inline", reason: "absent", inputs_digest: INPUTS, image: "", slim_image: "", build_contexts: "", labels: "", build_args: "", + }); }); it("writes a value of several lines in the form a workflow output takes", () => { @@ -205,8 +243,290 @@ describe("command line", () => { ["an unknown command", ["wait", "--dir", "d"]], ["a missing directory", ["resolve", "--dir"]], ["another flag", ["resolve", "--inputs-digest", INPUTS]], - ["an extra argument", ["inputs", "--dir", "d", "--wait"]], + ["a wait for the command that builds", ["inputs", "--dir", "d", "--wait"]], + ["an extra argument", ["resolve", "--dir", "d", "--wait", "--wait"]], + ["another flag in place of the wait", ["resolve", "--dir", "d", "--strict"]], ])("rejects %s", (_label, argv) => { expect(() => parseResolveArguments(argv)).toThrow(/Usage/); }); }); + +describe("the wait for the Runner image of a master push", () => { + const SHA = "a".repeat(40); + const CANONICAL = { id: 1170821064 }; + type Run = Record; + const run = (overrides: Run = {}): Run => ({ + id: 7, head_sha: SHA, event: "push", head_branch: "master", path: ".github/workflows/runner-image.yml", + repository: CANONICAL, head_repository: CANONICAL, status: "in_progress", conclusion: null, + created_at: new Date(0).toISOString(), html_url: "https://github.com/paperclipai/paperclip/actions/runs/7", ...overrides, + }); + const completed = (conclusion: string, overrides: Run = {}) => run({ status: "completed", conclusion, ...overrides }); + const absent = { mode: "inline", reason: "absent", detail: "the tag does not exist" }; + const published = { mode: "published", reason: "published", detail: "both tags exist" }; + const MINUTE = 60_000; + const { limitMs, listingMs, pollMs } = RUNNER_IMAGE_WAIT; + + function scenario({ lists, answers = [absent], startedAt = 0 }: { lists: Array, answers?: object[], startedAt?: number }) { + let time = startedAt; + const sleeps: number[] = []; + let listed = 0; + let resolved = 0; + const next = (values: T[], index: number) => values[Math.min(index, values.length - 1)]; + return { + sleeps, + counts: () => ({ listed, resolved }), + elapsed: () => time - startedAt, + options: { + sha: SHA, + now: () => time, + sleep: async (ms: number) => { time += ms; sleeps.push(ms); }, + listRuns: async () => { + const answer = next(lists, listed); + listed += 1; + if (answer instanceof Error) throw answer; + return answer; + }, + resolve: async () => { + resolved += 1; + return next(answers, resolved - 1); + }, + }, + }; + } + + it("has one limit, in minutes, that every duration of the wait is read from", () => { + expect(RUNNER_IMAGE_WAIT_MINUTES).toBe(40); + expect(limitMs).toBe(RUNNER_IMAGE_WAIT_MINUTES * MINUTE); + expect(listingMs).toBeLessThan(limitMs); + expect(pollMs).toBeLessThan(listingMs); + }); + + it("ends as soon as the image is published, while the run that published it is still working", async () => { + const wait = scenario({ lists: [[run()]], answers: [absent, absent, published] }); + const result = await waitForRunnerImage(wait.options); + expect(result).toMatchObject({ outcome: "published", resolved: published }); + expect(wait.sleeps).toEqual([pollMs, pollMs]); + expect(wait.counts()).toEqual({ listed: 3, resolved: 3 }); + }); + + it.each(["success", "failure", "cancelled", "skipped", "timed_out"])("ends when the run completes with %s, and leaves the answer to the next lookup", async (conclusion) => { + const wait = scenario({ lists: [[run()], [run()], [completed(conclusion)]] }); + const result = await waitForRunnerImage(wait.options); + expect(result.outcome).toBe("completed"); + expect(result).not.toHaveProperty("resolved"); + expect(result.detail).toContain(`actions/runs/7 (${conclusion})`); + expect(result.detail).toContain("waited 1 min"); + expect(wait.counts()).toEqual({ listed: 3, resolved: 2 }); + }); + + it("retries while the run is not listed yet, and gives up when none is listed in time", async () => { + const late = scenario({ lists: [[], [], [run()], [completed("success")]] }); + expect((await waitForRunnerImage(late.options)).outcome).toBe("completed"); + expect(late.sleeps).toHaveLength(3); + + const never = scenario({ lists: [[]] }); + const result = await waitForRunnerImage(never.options); + expect(result.outcome).toBe("not-listed"); + expect(never.elapsed()).toBe(listingMs); + expect(never.counts().resolved).toBe(0); + }); + + it("waits for every run of the commit, not for the first one that completes", async () => { + const second = { id: 9, html_url: "https://github.com/paperclipai/paperclip/actions/runs/9" }; + const wait = scenario({ lists: [[completed("failure"), run(second)], [completed("failure"), run(second)], [completed("failure"), completed("success", second)]] }); + const result = await waitForRunnerImage(wait.options); + expect(result.outcome).toBe("completed"); + expect(result.detail).toContain("actions/runs/7 (failure)"); + expect(result.detail).toContain("actions/runs/9 (success)"); + expect(wait.sleeps).toHaveLength(2); + }); + + it.each<[string, Run]>([ + ["of another commit", { head_sha: "b".repeat(40) }], + ["of a pull request", { event: "pull_request" }], + ["of the weekly schedule", { event: "schedule" }], + ["started by hand", { event: "workflow_dispatch" }], + ["of another branch", { head_branch: "release" }], + ["of another workflow", { path: ".github/workflows/docker.yml" }], + ["in another repository", { repository: { id: 1 } }], + ["from a fork", { head_repository: { id: 1 } }], + ])("does not wait for a run %s", async (_label, overrides) => { + const wait = scenario({ lists: [[completed("success", overrides), run(overrides)]] }); + expect((await waitForRunnerImage(wait.options)).outcome).toBe("not-listed"); + expect(wait.counts().resolved).toBe(0); + }); + + it("measures the limit from the push, so a build that started late waits less or not at all", async () => { + const late = scenario({ lists: [[run()]], startedAt: limitMs - MINUTE }); + const result = await waitForRunnerImage(late.options); + expect(result.outcome).toBe("limit"); + expect(late.elapsed()).toBe(MINUTE); + expect(result.detail).toContain(`within ${RUNNER_IMAGE_WAIT_MINUTES} min of the push`); + + const expired = scenario({ lists: [[run()]], startedAt: limitMs + 10 * MINUTE }); + expect((await waitForRunnerImage(expired.options)).outcome).toBe("limit"); + expect(expired.sleeps).toEqual([]); + expect(expired.counts()).toEqual({ listed: 1, resolved: 1 }); + + const onTime = scenario({ lists: [[run()]], startedAt: MINUTE }); + expect((await waitForRunnerImage(onTime.options)).outcome).toBe("limit"); + expect(onTime.elapsed()).toBe(limitMs - MINUTE); + }); + + it("measures the limit from the oldest run of the commit, and from its own start when no run names a time", async () => { + const rerun = scenario({ lists: [[completed("failure"), run({ id: 9, created_at: new Date(30 * MINUTE).toISOString() })]], startedAt: 39 * MINUTE }); + expect((await waitForRunnerImage(rerun.options)).outcome).toBe("limit"); + expect(rerun.elapsed()).toBe(MINUTE); + + const undated = scenario({ lists: [[run({ created_at: "soon" })]], startedAt: 5 * limitMs }); + expect((await waitForRunnerImage(undated.options)).outcome).toBe("limit"); + expect(undated.elapsed()).toBe(limitMs); + }); + + it("stops when the Actions API keeps failing, and goes on when it recovers", async () => { + const down = scenario({ lists: [new Error("GitHub API: HTTP 502")] }); + const result = await waitForRunnerImage(down.options); + expect(result).toMatchObject({ outcome: "unknown", detail: expect.stringContaining("HTTP 502") }); + expect(down.counts()).toEqual({ listed: RUNNER_IMAGE_WAIT.failures, resolved: 0 }); + + const flaky = scenario({ lists: [new Error("fetch failed"), new Error("fetch failed"), [run()], new Error("fetch failed"), new Error("fetch failed"), [completed("success")]] }); + expect((await waitForRunnerImage(flaky.options)).outcome).toBe("completed"); + expect(flaky.counts().listed).toBe(6); + }); + + it.each(["absent", "incomplete"])("waits when the image is %s, then looks once more", async (reason) => { + const answers = [{ mode: "inline", reason, detail: "not published" }, published]; + let lookups = 0; + let waits = 0; + const resolved = await resolveRunnerImageAfterWait({ + resolve: async () => answers[lookups++], + wait: async () => { waits += 1; return { outcome: "completed", detail: "waited 12 min" }; }, + }); + expect(resolved).toEqual({ ...published, detail: "both tags exist; waited 12 min" }); + expect({ lookups, waits }).toEqual({ lookups: 2, waits: 1 }); + }); + + it.each(["published", "lock-behind", "unsupported", "broken", "mismatch", "unknown"])("does not wait when the first answer is %s", async (reason) => { + const first = { mode: reason === "published" ? "published" : "inline", reason, detail: "first" }; + let waits = 0; + const resolved = await resolveRunnerImageAfterWait({ resolve: async () => first, wait: async () => { waits += 1; return { outcome: "completed", detail: "" }; } }); + expect(resolved).toBe(first); + expect(waits).toBe(0); + }); + + it.each(["completed", "not-listed", "limit", "unknown"])("is inline, with the reason of the last lookup, when the wait ends as %s without an image", async (outcome) => { + let lookups = 0; + const resolved = await resolveRunnerImageAfterWait({ + resolve: async () => { lookups += 1; return absent; }, + wait: async () => ({ outcome, detail: `the wait ended: ${outcome}` }), + }); + expect(resolved).toEqual({ ...absent, detail: `the tag does not exist; the wait ended: ${outcome}` }); + expect(lookups).toBe(2); + }); + + it("uses the image the wait found without another lookup", async () => { + let lookups = 0; + const resolved = await resolveRunnerImageAfterWait({ + resolve: async () => { lookups += 1; return absent; }, + wait: async () => ({ outcome: "published", resolved: published, detail: "waited 3 min" }), + }); + expect(resolved).toMatchObject({ mode: "published", detail: "both tags exist; waited 3 min" }); + expect(lookups).toBe(1); + }); + + describe("on the command line", () => { + const TOKEN = "token-for-the-test"; + type Fetch = (url: string, init: { headers: Record }) => Promise; + function prepared() { + const out = directory(); + mkdirSync(out, { recursive: true }); + writeFileSync(path.join(out, "runner-image-inputs.json"), JSON.stringify(current)); + return out; + } + const instant: typeof waitForRunnerImage = (options) => { + let time = 0; + return waitForRunnerImage({ ...options, now: () => time, sleep: async (ms: number) => { time += ms; } }); + }; + const actions = (pages: Run[][], seen: Array<{ url: string, authorization: string }> = []): Fetch => async (url, init) => { + seen.push({ url, authorization: init.headers.authorization }); + return new Response(JSON.stringify({ workflow_runs: pages[Math.min(seen.length - 1, pages.length - 1)] })); + }; + + it("waits for the run of GITHUB_SHA with the token of the environment, then reports the image it published", async () => { + const out = prepared(); + const seen: Array<{ url: string, authorization: string }> = []; + const tags: Record = {}; + const fetchImpl = actions([[run()], [completed("success")]], seen); + const lines: string[] = []; + const read = async (reference: string) => { + if (seen.length === 2) Object.assign(tags, { [TAGS.full]: full, [TAGS.slim]: slim }); + return registry(tags)(reference); + }; + const values = await runResolveCommand(["resolve", "--dir", out, "--wait"], { + env: { GITHUB_SHA: SHA, GITHUB_TOKEN: TOKEN }, read, fetchImpl: fetchImpl as typeof fetch, waitFor: instant, log: (line) => lines.push(line), + }); + expect(values).toMatchObject({ mode: "published", image: `${IMAGE}@${full.digest}`, build_args: `PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST=${INPUTS}` }); + expect(seen).toHaveLength(2); + for (const request of seen) { + expect(request.url).toBe(`https://api.github.com/repos/paperclipai/paperclip/actions/workflows/runner-image.yml/runs?head_sha=${SHA}&event=push&branch=master&per_page=30`); + expect(request.authorization).toBe(`Bearer ${TOKEN}`); + } + expect(lines.join("\n")).toContain("actions/runs/7 (success)"); + expect(lines.join("\n")).not.toContain("::notice"); + expect(lines.join("\n")).not.toContain(TOKEN); + }); + + it.each<[string, Run[][], RegExp]>([ + ["fails", [[run()], [completed("failure")]], /actions\/runs\/7 \(failure\)/], + ["is cancelled", [[completed("cancelled")]], /actions\/runs\/7 \(cancelled\)/], + ["publishes nothing because the lockfile is behind", [[completed("success")]], /actions\/runs\/7 \(success\)/], + ["is never listed", [[]], /no Runner image run of a master push/], + ])("is inline, says why in a notice and does not fail when the run %s", async (_label, pages, detail) => { + const lines: string[] = []; + const values = await runResolveCommand(["resolve", "--dir", prepared(), "--wait"], { + env: { GITHUB_SHA: SHA, GITHUB_TOKEN: TOKEN }, read: registry({}), fetchImpl: actions(pages) as typeof fetch, waitFor: instant, log: (line) => lines.push(line), + }); + expect(values).toMatchObject({ mode: "inline", reason: "absent", image: "", build_contexts: "", labels: "", build_args: "" }); + const notice = lines.filter((line) => line.startsWith("::notice title=Runner built inline::")); + expect(notice).toHaveLength(1); + expect(notice[0]).toMatch(detail); + }); + + it.each<[string, NodeJS.ProcessEnv, Fetch]>([ + ["there is no token", { GITHUB_SHA: SHA }, async () => { throw new Error("no request is expected"); }], + ["the commit is not a full SHA", { GITHUB_SHA: "master", GITHUB_TOKEN: TOKEN }, async () => { throw new Error("no request is expected"); }], + ["the Actions API refuses", { GITHUB_SHA: SHA, GITHUB_TOKEN: TOKEN }, async () => new Response("", { status: 403 })], + ["the Actions API cannot be reached", { GITHUB_SHA: SHA, GITHUB_TOKEN: TOKEN }, async () => { throw new TypeError("fetch failed"); }], + ["the Actions API answers with something else", { GITHUB_SHA: SHA, GITHUB_TOKEN: TOKEN }, async () => new Response(JSON.stringify({ message: "moved" }))], + ])("is inline without waiting when %s", async (_label, env, fetchImpl) => { + const values = await runResolveCommand(["resolve", "--dir", prepared(), "--wait"], { + env, read: registry({}), fetchImpl: fetchImpl as typeof fetch, waitFor: instant, log: () => {}, + }); + expect(values).toMatchObject({ mode: "inline", reason: "absent" }); + }); + + it("is inline when the wait itself throws, and asks the Actions API nothing when the image exists or no wait is asked for", async () => { + const failing = async (): Promise => { throw new Error("clock failure"); }; + const refusing: Fetch = async () => { throw new Error("no request is expected"); }; + const env = { GITHUB_SHA: SHA, GITHUB_TOKEN: TOKEN }; + const lines: string[] = []; + await expect(runResolveCommand(["resolve", "--dir", prepared(), "--wait"], { env, read: registry({}), waitFor: failing, log: (line) => lines.push(line) })) + .resolves.toMatchObject({ mode: "inline", reason: "absent" }); + expect(lines.join("\n")).toContain("the wait failed: clock failure"); + const exists = registry({ [TAGS.full]: full, [TAGS.slim]: slim }); + await expect(runResolveCommand(["resolve", "--dir", prepared(), "--wait"], { env, read: exists, fetchImpl: refusing as typeof fetch, waitFor: failing, log: () => {} })) + .resolves.toMatchObject({ mode: "published" }); + await expect(runResolveCommand(["resolve", "--dir", prepared()], { env, read: registry({}), fetchImpl: refusing as typeof fetch, waitFor: failing, log: () => {} })) + .resolves.toMatchObject({ mode: "inline", reason: "absent" }); + }); + + it("refuses to list runs without a token or a full commit, and an answer that is not a list of runs", async () => { + const ok: Fetch = async () => new Response(JSON.stringify({ workflow_runs: [run()] })); + await expect(listRunnerImageRuns(SHA, { token: TOKEN, fetchImpl: ok as typeof fetch })).resolves.toHaveLength(1); + await expect(listRunnerImageRuns(SHA, { fetchImpl: ok as typeof fetch })).rejects.toThrow(/token/); + await expect(listRunnerImageRuns("a".repeat(7), { token: TOKEN, fetchImpl: ok as typeof fetch })).rejects.toThrow(/full SHA/); + await expect(listRunnerImageRuns(SHA, { token: TOKEN, fetchImpl: (async () => new Response("", { status: 500 })) as typeof fetch })).rejects.toThrow(/HTTP 500/); + await expect(listRunnerImageRuns(SHA, { token: TOKEN, fetchImpl: (async () => new Response("{}")) as typeof fetch })).rejects.toThrow(/no list of workflow runs/); + }); + }); +});