mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 12:29:37 +01:00
Merge latest Codex qualification and master fixes
Co-Authored-By: Paperclip <noreply@paperclip.ing> * codex/claude-runner-default: (25 commits) fix: pin ARM qualification helper to workflow source perf(ci): cache the Runner's release crate artifacts keyed by their inputs (#15932) perf(build): reuse a current Runner build in the server's prepare:runner-vendor step (#15926) fix(test): make the vitest home teardown remove read-only runtime-context bundles (#15931) fix(test): stop the Sentry timeout assertion matching a random event id (#15929) test: qualify Linux ARM with stock sandbox prerequisites docs(release): align 2026.1006.0-beta.0 stable notes header with v2026.1012.0 (#16000) fix: use Hermes adapter API URL env in prompt (#9425) fix(docker): skip the full-tree ownership walk on routine boots (#15992) refactor(chat): isolate Telegram, Slack, GitHub and Discord workflows (#15976) Preserve repository dependencies when importing GitHub skills (#15731) refactor(heartbeat): extract instruction working-copy lifecycle (#15984) test: retire redundant suites and consolidate owner coverage (#15978) refactor(heartbeat): extract runtime service coordination (#15980) refactor(heartbeat): extract execution configuration (#15973) refactor(heartbeat): extract run output recording (#15969) refactor(chat): extract failed-run retry coordination (#15845) refactor(chat): extract publication replacement decisions (#15844) refactor(chat): extract identity linking (#15836) refactor(heartbeat): extract session preparation (#15966) ... # Conflicts: # packages/paperclip-runner/package.json
This commit is contained in:
Dotta committed
commit
adf258ede2
210 files changed
+25068
-25979
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -961,6 +962,108 @@ test("premerge native qualification consumes the same Linux-packed npm bytes wit
|
||||
assert.match(qualification, /pnpm install --ignore-scripts --no-frozen-lockfile/);
|
||||
});
|
||||
|
||||
test("native ARM qualification prepares its scoped distro sandbox before the existing consumer", () => {
|
||||
const smoke = readWorkflow("release-smoke.yml");
|
||||
const native = smoke.split(" qualification_native:\n")[1];
|
||||
const preparation = native.split(" - name: Prepare Linux ARM sandbox prerequisites\n")[1].split("\n - ")[0];
|
||||
assert.match(preparation, /if: \$\{\{ matrix\.target == 'linux-arm64' \}\}/);
|
||||
assert.match(preparation, /timeout-minutes: 5/);
|
||||
assert.match(preparation, /bash \.qualification-tooling\/scripts\/prepare-linux-arm64-qualification\.sh/);
|
||||
const tooling = native.split(" - name: Checkout immutable ARM qualification tooling\n")[1].split("\n - ")[0];
|
||||
assert.match(tooling, /if: \$\{\{ matrix\.target == 'linux-arm64' \}\}/);
|
||||
assert.match(tooling, /repository: \$\{\{ job\.workflow_repository \}\}/);
|
||||
assert.match(tooling, /ref: \$\{\{ job\.workflow_sha \}\}/);
|
||||
assert.match(tooling, /sparse-checkout: scripts\/prepare-linux-arm64-qualification\.sh/);
|
||||
assert.match(tooling, /sparse-checkout-cone-mode: false/);
|
||||
assert.match(tooling, /persist-credentials: false/);
|
||||
assert.doesNotMatch(tooling + preparation, /github\.workflow_sha|github\.sha|ref: (?:master|main)|ref: \$\{\{ inputs\.qualification_source_sha/);
|
||||
assert.ok(native.indexOf("Checkout immutable ARM qualification tooling") < native.indexOf("Prepare Linux ARM sandbox prerequisites"));
|
||||
assert.ok(native.indexOf("Prepare Linux ARM sandbox prerequisites") < native.indexOf("Verify host identity and actual clean npm consumer"));
|
||||
assert.match(native, /path: \|\n\s+\$\{\{ runner\.temp \}\}\/native-public-install\.json\n\s+\$\{\{ runner\.temp \}\}\/paperclip-arm-bwrap-prerequisite\//);
|
||||
assert.doesNotMatch(smoke.split(" qualification_daemons:\n")[0], /prepare-linux-arm64-qualification/,
|
||||
"ordinary service and Docker smoke keep their existing setup");
|
||||
});
|
||||
|
||||
test("ARM qualification binds separate product and reusable-workflow revisions even when the product lacks the helper", { skip: process.platform === "win32" }, () => {
|
||||
const native = readWorkflow("release-smoke.yml").split(" qualification_native:\n")[1];
|
||||
const step = native.split(" - name: Prepare Linux ARM sandbox prerequisites\n")[1].split("\n - ")[0];
|
||||
const script = step.split(" run: |\n")[1].split("\n").map(line => line.startsWith(" ") ? line.slice(10) : line).join("\n");
|
||||
const root = mkdtempSync(path.join(tmpdir(), "arm-separate-tooling-"));
|
||||
try {
|
||||
const tools = path.join(root, "tools"); mkdirSync(tools);
|
||||
const tooling = path.join(root, ".qualification-tooling", "scripts"); mkdirSync(tooling, { recursive: true });
|
||||
const helper = 'printf "%s\\n" "$TOOLING_SHA" > "$FIXTURE_PREPARED"\n';
|
||||
writeFileSync(path.join(tooling, "prepare-linux-arm64-qualification.sh"), helper);
|
||||
writeFileSync(path.join(tools, "git"), '#!/bin/bash\nif [[ "$1" == -C ]]; then printf "%s\\n" "$FIXTURE_TOOLING_HEAD"; else printf "%s\\n" "$FIXTURE_PRODUCT_HEAD"; fi\n', { mode: 0o755 });
|
||||
const productSha = "a".repeat(40), toolingSha = "b".repeat(40);
|
||||
for (const [productHead, toolingHead, expectedStatus] of [[productSha, toolingSha, 0], ["c".repeat(40), toolingSha, 1], [productSha, "c".repeat(40), 1], [productSha, "invalid", 1]]) {
|
||||
const report = path.join(root, "report"); mkdirSync(report);
|
||||
const prepared = path.join(root, "prepared");
|
||||
const result = spawnSync("/bin/bash", ["-c", script], { cwd: root, encoding: "utf8", env: {
|
||||
PATH: `${tools}:${path.dirname(process.execPath)}:/usr/bin:/bin`, RUNNER_TEMP: report,
|
||||
SOURCE_SHA: productSha, TOOLING_SHA: toolingSha, TOOLING_REPOSITORY: "paperclipai/paperclip",
|
||||
TOOLING_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/release-smoke.yml@refs/heads/qualification",
|
||||
TOOLING_WORKFLOW_PATH: ".github/workflows/release-smoke.yml",
|
||||
FIXTURE_PRODUCT_HEAD: productHead, FIXTURE_TOOLING_HEAD: toolingHead, FIXTURE_PREPARED: prepared,
|
||||
// Reusable calls retain the caller context; it must not select tooling.
|
||||
GITHUB_REPOSITORY: "different/caller", GITHUB_SHA: "d".repeat(40),
|
||||
} });
|
||||
assert.equal(result.status, expectedStatus, result.stderr);
|
||||
assert.equal(existsSync(path.join(root, "scripts", "prepare-linux-arm64-qualification.sh")), false);
|
||||
assert.equal(existsSync(prepared), expectedStatus === 0);
|
||||
if (expectedStatus === 0) {
|
||||
assert.equal(readFileSync(prepared, "utf8").trim(), toolingSha);
|
||||
const identity = JSON.parse(readFileSync(path.join(report, "paperclip-arm-bwrap-prerequisite", "qualification-identity.json"), "utf8"));
|
||||
assert.equal(identity.productSourceSha, productSha);
|
||||
assert.equal(identity.toolingSourceSha, toolingSha);
|
||||
assert.equal(identity.toolingRepository, "paperclipai/paperclip");
|
||||
assert.equal(identity.toolingWorkflowPath, ".github/workflows/release-smoke.yml");
|
||||
assert.equal(identity.helperSha256, createHash("sha256").update(helper).digest("hex"));
|
||||
}
|
||||
rmSync(report, { recursive: true, force: true });
|
||||
rmSync(prepared, { force: true });
|
||||
}
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("Linux ARM prerequisite setup rejects other hosts before invoking tools and preserves scoped security", { skip: process.platform === "win32" }, () => {
|
||||
const helper = path.join(repoRoot, "scripts/prepare-linux-arm64-qualification.sh");
|
||||
const source = readFileSync(helper, "utf8");
|
||||
const root = mkdtempSync(path.join(tmpdir(), "arm-prerequisite-guard-"));
|
||||
try {
|
||||
const tools = path.join(root, "tools"); mkdirSync(tools);
|
||||
const sentinel = path.join(root, "invoked");
|
||||
for (const command of ["dpkg", "uname", "mkdir", "mktemp", "sysctl", "sudo"]) {
|
||||
writeFileSync(path.join(tools, command), '#!/bin/bash\nprintf "%s\\n" "$0" >> "$PREREQUISITE_SENTINEL"\nexit 85\n', { mode: 0o755 });
|
||||
}
|
||||
for (const [actions, runnerOs] of [["", "Linux"], ["false", "Linux"], ["true", "macOS"], ["true", "Windows"]]) {
|
||||
const result = spawnSync("/bin/bash", [helper], { encoding: "utf8", env: {
|
||||
PATH: tools, GITHUB_ACTIONS: actions, RUNNER_OS: runnerOs, RUNNER_TEMP: root, PREREQUISITE_SENTINEL: sentinel,
|
||||
} });
|
||||
assert.equal(result.status, 1);
|
||||
assert.match(result.stderr, /requires a GitHub Actions Linux runner/);
|
||||
assert.equal(existsSync(sentinel), false, "host guard must precede every external tool");
|
||||
assert.equal(existsSync(path.join(root, "paperclip-arm-bwrap-prerequisite")), false);
|
||||
}
|
||||
assert.ok(source.indexOf('"${GITHUB_ACTIONS:-}" != true') < source.indexOf(". /etc/os-release"));
|
||||
assert.match(source, /-o Acquire::AllowInsecureRepositories=false/);
|
||||
assert.match(source, /download "\$package_name=\$package_version"/);
|
||||
assert.match(source, /printf '%s %s\\n' "\$package_sha256" "\$package_file" \| sha256sum -c -/);
|
||||
assert.match(source, /dpkg-deb --fsys-tarfile "\$package_file"/);
|
||||
assert.match(source, /tar -xO \.\/usr\/bin\/bwrap/);
|
||||
assert.match(source, /11d39094f044f0cda0febb3ad517b830301da6b2ce929664af09ee9e4dd264f9/);
|
||||
assert.ok(source.indexOf('test ! -L "$target"') < source.indexOf('sudo install -o root -g root -m 0644 "$stock_profile" "$target"'));
|
||||
assert.match(source, /timeout 15s sudo "\$parser" -r "\$target"/);
|
||||
for (const setting of ["userns", "apparmor", "clone"]) assert.ok(source.includes(`test "$after_${setting}" = "$before_${setting}"`));
|
||||
const aptCommands = source.replace(/\\\n\s*/g, " ").split("\n").filter(line => /\bapt-get\b/.test(line));
|
||||
assert.equal(aptCommands.length, 2, "only signed-index update and download may invoke APT");
|
||||
assert.match(aptCommands[0], /\bupdate\b/);
|
||||
assert.match(aptCommands[1], /\bdownload\b/);
|
||||
for (const command of aptCommands) assert.doesNotMatch(command, /\b(?:install|upgrade|remove|purge)\b/);
|
||||
assert.doesNotMatch(source, /apt-get[^\n]* install|\bdpkg(?:-deb)?\s+(?:-i|--install|--unpack)|sysctl\s+(?:-w|--system)|\baa-disable\b|\bsetcap\b|systemctl|--no-sandbox/);
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test("exact-source daemon matrices preserve historical three-target sources and add native ARM only for new sources", () => {
|
||||
const workflow = readWorkflow("release-verify.yml");
|
||||
const step = workflow.split(' - name: Preserve the immutable source\'s platform contract')[1].split('\n runner_binaries:')[0];
|
||||
|
||||
@@ -0,0 +1,558 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { chmodSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { CACHED_PATHS, HASHED_INPUTS, REQUIRED, RUNNER_PACKAGE, computeScope, markCurrent } from "../runner-release-crate-cache.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const script = path.join(repoRoot, "scripts/runner-release-crate-cache.mjs");
|
||||
const readWorkflow = (name) => readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8");
|
||||
const prWorkflow = readWorkflow("pr-trusted.yml");
|
||||
const releaseWorkflow = readWorkflow("release-verify.yml");
|
||||
|
||||
// Slice one job out of a workflow, from its two-space-indented key to the next.
|
||||
function job(workflow, name) {
|
||||
const start = workflow.indexOf(`\n ${name}:\n`);
|
||||
assert.ok(start >= 0, `workflow is missing the ${name} job`);
|
||||
const body = workflow.slice(start + 1);
|
||||
const header = ` ${name}:\n`.length;
|
||||
const next = body.slice(header).search(/\n [a-z0-9_-]+:\n/);
|
||||
return next === -1 ? body : body.slice(0, header + next + 1);
|
||||
}
|
||||
|
||||
// Slice one step out of a job by its name, up to the next step.
|
||||
function step(body, name) {
|
||||
const marker = ` - name: ${name}\n`;
|
||||
const start = body.indexOf(marker);
|
||||
assert.ok(start >= 0, `missing step: ${name}`);
|
||||
const rest = body.slice(start + marker.length);
|
||||
const next = rest.search(/\n - name: /);
|
||||
return marker + (next === -1 ? rest : rest.slice(0, next + 1));
|
||||
}
|
||||
|
||||
const RELEASE_DIR = `${RUNNER_PACKAGE}/runner/target/release`;
|
||||
const KEY_STEP = "Compute the Runner release crate cache key";
|
||||
const RESTORE_STEP = "Restore the Runner release crate (read only)";
|
||||
const MARK_STEP = "Mark the restored Runner release crate current";
|
||||
const LOOKUP_STEP = "Check for this push's Runner release crate";
|
||||
const BUILD_STEP = "Build the Runner release crate for its cache";
|
||||
const SAVE_STEP = "Save the Runner release crate";
|
||||
const RUST_RESTORE_STEP = "Restore Runner Rust dependencies (read only)";
|
||||
|
||||
// Every PR job that builds the Runner release binary, and the command each
|
||||
// one runs cargo through. The crate must be restored and marked before it.
|
||||
const READERS = [
|
||||
["typecheck_release_registry", "run: pnpm run typecheck:build-gaps"],
|
||||
["verify_paperclip_runner", "run: pnpm --filter @paperclipai/paperclip-runner ${{ matrix.command }}"],
|
||||
["build", "run: pnpm build"],
|
||||
["canary_dry_run", "- name: Release canary dry run via release.sh internal build"],
|
||||
];
|
||||
|
||||
const WRITER_GATE = "matrix.lane == 'rust' && github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha";
|
||||
|
||||
const expectedPathBlock = ` path: |\n${CACHED_PATHS.map((entry) => ` ${entry}\n`).join("")}`;
|
||||
const expectedKey = `key: runner-release-crate-v1-\${{ steps.runner_release_crate_key.outputs.scope }}-\${{ hashFiles(${HASHED_INPUTS.map((entry) => `'${entry}'`).join(", ")}) }}`;
|
||||
|
||||
function assertCacheStep(body, name, { lookupOnly = false } = {}) {
|
||||
assert.match(body, /uses: actions\/cache\/(restore|save)@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6\.1\.0/, name);
|
||||
assert.ok(body.includes(expectedPathBlock), `${name}: the cached path list must be exactly the script's CACHED_PATHS`);
|
||||
assert.ok(body.includes(expectedKey), `${name}: the key must be built from the script's scope and HASHED_INPUTS`);
|
||||
// An exact match or nothing: a prefix match would restore another
|
||||
// revision's artifacts and the mark step would make cargo keep them.
|
||||
assert.doesNotMatch(body, /restore-keys:/, name);
|
||||
assert.equal(/lookup-only: true/.test(body), lookupOnly, name);
|
||||
assert.doesNotMatch(body, /enableCrossOsArchive|fail-on-cache-miss: true/, name);
|
||||
}
|
||||
|
||||
test("the script's path list and key inputs cover what the release build of the crate reads and writes", () => {
|
||||
assert.deepEqual(CACHED_PATHS, [
|
||||
`${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*`,
|
||||
`${RELEASE_DIR}/deps/libpaperclip_runner_core-*`,
|
||||
`${RELEASE_DIR}/deps/paperclip_runner_core-*.d`,
|
||||
`${RELEASE_DIR}/deps/paperclip_runnerd-*`,
|
||||
`${RELEASE_DIR}/paperclip-runnerd`,
|
||||
`${RELEASE_DIR}/paperclip-runnerd.d`,
|
||||
]);
|
||||
assert.deepEqual(HASHED_INPUTS, [
|
||||
`${RUNNER_PACKAGE}/rust-toolchain.toml`,
|
||||
`${RUNNER_PACKAGE}/runner/Cargo.toml`,
|
||||
`${RUNNER_PACKAGE}/runner/Cargo.lock`,
|
||||
`${RUNNER_PACKAGE}/runner/crates/**`,
|
||||
`${RUNNER_PACKAGE}/protocol/**`,
|
||||
`${RUNNER_PACKAGE}/test-fixtures/**`,
|
||||
`${RUNNER_PACKAGE}/test/fixtures/**`,
|
||||
"tests/runner-e2e/fixtures/**",
|
||||
]);
|
||||
for (const input of HASHED_INPUTS) {
|
||||
const fixed = input.replace(/\/\*\*$/, "");
|
||||
assert.ok(statSync(path.join(repoRoot, fixed)), `${input} must exist in the repository`);
|
||||
}
|
||||
// The stage step copies the uplifted binary; it has to be in the set.
|
||||
const stage = readFileSync(path.join(repoRoot, RUNNER_PACKAGE, "scripts/stage-runner-binary.mjs"), "utf8");
|
||||
assert.match(stage, /"runner", "target", "release", executable/);
|
||||
assert.ok(CACHED_PATHS.includes(`${RELEASE_DIR}/paperclip-runnerd`));
|
||||
});
|
||||
|
||||
// Every `include_str!` / `include_bytes!` in one Rust source, with the path
|
||||
// each one names. Three argument forms are resolved: a string literal
|
||||
// (relative to the source file), and `concat!(...)` of string literals,
|
||||
// `env!("CARGO_MANIFEST_DIR")` (the directory of the nearest Cargo.toml) and
|
||||
// macro metavariables (`$name`, which make the resolved path a directory
|
||||
// prefix rather than a file). Anything else is reported as unsupported, so
|
||||
// a new include form cannot slip past the guard by not matching a regex.
|
||||
function crateIncludes(source, text) {
|
||||
const includes = [];
|
||||
for (const match of text.matchAll(/include_(?:str|bytes)!\s*\(/g)) {
|
||||
const start = match.index + match[0].length;
|
||||
let depth = 1;
|
||||
let inString = false;
|
||||
let end = start;
|
||||
for (; end < text.length && depth > 0; end += 1) {
|
||||
const ch = text[end];
|
||||
if (inString) {
|
||||
if (ch === "\\") end += 1;
|
||||
else if (ch === '"') inString = false;
|
||||
} else if (ch === '"') inString = true;
|
||||
else if (ch === "(") depth += 1;
|
||||
else if (ch === ")") depth -= 1;
|
||||
}
|
||||
assert.equal(depth, 0, `${source}: unbalanced include at offset ${match.index}`);
|
||||
const argument = text.slice(start, end - 1).trim();
|
||||
includes.push({ argument, ...resolveIncludeArgument(source, argument) });
|
||||
}
|
||||
return includes;
|
||||
}
|
||||
|
||||
function manifestDir(source) {
|
||||
let dir = path.dirname(source);
|
||||
while (!statSync(path.join(dir, "Cargo.toml"), { throwIfNoEntry: false })?.isFile()) {
|
||||
const parent = path.dirname(dir);
|
||||
assert.notEqual(parent, dir, `${source}: no Cargo.toml above the source`);
|
||||
dir = parent;
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
function resolveIncludeArgument(source, argument) {
|
||||
const literal = argument.match(/^"((?:[^"\\]|\\.)*)"$/);
|
||||
if (literal) return { form: "literal", resolved: path.resolve(path.dirname(source), literal[1]), exact: true };
|
||||
|
||||
const concat = argument.match(/^concat!\s*\(([\s\S]*)\)$/);
|
||||
assert.ok(concat, `${source}: unsupported include form: ${argument}`);
|
||||
const parts = splitTopLevel(concat[1]);
|
||||
// A relative path is joined to the source's directory; one that starts
|
||||
// with the manifest directory is one string, since its next literal
|
||||
// starts with the "/" that separates the two.
|
||||
let base = `${path.dirname(source)}/`;
|
||||
let text = "";
|
||||
let exact = true;
|
||||
let form = "concat";
|
||||
for (const [index, part] of parts.entries()) {
|
||||
const string = part.match(/^"((?:[^"\\]|\\.)*)"$/);
|
||||
if (string) {
|
||||
assert.ok(exact || !/(^|\/)\.\.(\/|$)/.test(string[1]), `${source}: a path segment after a metavariable must not climb out: ${argument}`);
|
||||
if (exact) text += string[1];
|
||||
continue;
|
||||
}
|
||||
if (part === 'env!("CARGO_MANIFEST_DIR")') {
|
||||
assert.equal(index, 0, `${source}: env!("CARGO_MANIFEST_DIR") must be the first part of concat!: ${argument}`);
|
||||
base = manifestDir(source);
|
||||
form = "concat-manifest-dir";
|
||||
continue;
|
||||
}
|
||||
if (/^\$[A-Za-z_][A-Za-z0-9_]*$/.test(part)) {
|
||||
// The fixed prefix before the first metavariable is the directory the
|
||||
// macro's callers can reach; the suffix cannot climb out of it.
|
||||
exact = false;
|
||||
form = form === "concat-manifest-dir" ? form : "concat-metavariable";
|
||||
continue;
|
||||
}
|
||||
assert.fail(`${source}: unsupported concat! part in include: ${part}`);
|
||||
}
|
||||
const joined = base + text;
|
||||
const resolved = path.resolve(exact || text.endsWith("/") ? joined : path.dirname(joined));
|
||||
return { form, resolved, exact };
|
||||
}
|
||||
|
||||
// Split concat!'s arguments on the commas outside string literals.
|
||||
function splitTopLevel(text) {
|
||||
const parts = [];
|
||||
let current = "";
|
||||
let inString = false;
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const ch = text[index];
|
||||
if (inString) {
|
||||
current += ch;
|
||||
if (ch === "\\") current += text[++index] ?? "";
|
||||
else if (ch === '"') inString = false;
|
||||
} else if (ch === '"') {
|
||||
inString = true;
|
||||
current += ch;
|
||||
} else if (ch === ",") {
|
||||
parts.push(current);
|
||||
current = "";
|
||||
} else current += ch;
|
||||
}
|
||||
parts.push(current);
|
||||
return parts.map((part) => part.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
const underHashedInputs = (resolved) =>
|
||||
HASHED_INPUTS.map((input) => path.join(repoRoot, input.replace(/\/\*\*$/, ""))).some((root) => resolved === root || resolved.startsWith(`${root}${path.sep}`));
|
||||
|
||||
test("the include parser resolves literals, concat! with the manifest directory and with metavariables, and rejects other forms", () => {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-includes-"));
|
||||
try {
|
||||
const crate = path.join(root, "crates/demo");
|
||||
mkdirSync(path.join(crate, "src/nested"), { recursive: true });
|
||||
mkdirSync(path.join(root, "protocol/fixtures"), { recursive: true });
|
||||
writeFileSync(path.join(crate, "Cargo.toml"), '[package]\nname = "demo"\n');
|
||||
writeFileSync(path.join(root, "protocol/schema.json"), "{}");
|
||||
const source = path.join(crate, "src/nested/mod.rs");
|
||||
const text = [
|
||||
'let a = include_str!("../../../../protocol/schema.json");',
|
||||
'let b = include_bytes!(\n "../../../../protocol/schema.json"\n);',
|
||||
'let c = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../protocol/", "schema.json"));',
|
||||
'let d = include_str!(concat!("../../../../protocol/fixtures/", $fixture, ".json"));',
|
||||
'let e = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../protocol/fixtures/", $golden, ".summary.json"));',
|
||||
'let f = include_str!("a (paren) in the name.json");',
|
||||
].join("\n");
|
||||
const includes = crateIncludes(source, text);
|
||||
assert.deepEqual(
|
||||
includes.map(({ form, resolved, exact }) => [form, path.relative(root, resolved), exact]),
|
||||
[
|
||||
["literal", "protocol/schema.json", true],
|
||||
["literal", "protocol/schema.json", true],
|
||||
["concat-manifest-dir", "protocol/schema.json", true],
|
||||
["concat-metavariable", "protocol/fixtures", false],
|
||||
["concat-manifest-dir", "protocol/fixtures", false],
|
||||
["literal", "crates/demo/src/nested/a (paren) in the name.json", true],
|
||||
],
|
||||
);
|
||||
|
||||
const rejected = [
|
||||
'include_str!(concat!("../protocol/", $x, "/../../../outside.json"))',
|
||||
'include_str!(concat!("../protocol/", env!("CARGO_MANIFEST_DIR")))',
|
||||
'include_str!(concat!(env!("OTHER_DIR"), "/schema.json"))',
|
||||
'include_str!(r"../protocol/schema.json")',
|
||||
'include_str!(path_from_a_function())',
|
||||
'include_str!(concat!("../protocol/", some_macro!("x")))',
|
||||
];
|
||||
for (const argument of rejected) {
|
||||
assert.throws(() => crateIncludes(source, argument), /unsupported|must not climb out|must be the first part/, argument);
|
||||
}
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("every file the crate includes by path sits under a hashed input, so an edit to it changes the key", () => {
|
||||
const crateRoot = path.join(repoRoot, RUNNER_PACKAGE, "runner/crates");
|
||||
const sources = [];
|
||||
(function walk(dir) {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(full);
|
||||
else if (entry.name.endsWith(".rs")) sources.push(full);
|
||||
}
|
||||
})(crateRoot);
|
||||
assert.ok(sources.length > 10, "the crate sources must be found");
|
||||
const forms = new Set();
|
||||
let includes = 0;
|
||||
for (const source of sources) {
|
||||
for (const { argument, form, resolved, exact } of crateIncludes(source, readFileSync(source, "utf8"))) {
|
||||
includes += 1;
|
||||
forms.add(form);
|
||||
const where = `${path.relative(repoRoot, source)} includes ${argument.replace(/\s+/g, " ")}`;
|
||||
assert.ok(underHashedInputs(resolved), `${where}, which resolves to ${path.relative(repoRoot, resolved)}, outside every HASHED_INPUTS pattern`);
|
||||
const info = statSync(resolved, { throwIfNoEntry: false });
|
||||
assert.ok(exact ? info?.isFile() : info?.isDirectory(), `${where}: ${path.relative(repoRoot, resolved)} must exist`);
|
||||
}
|
||||
}
|
||||
assert.ok(includes > 0, "the crate includes protocol files by path; the guard must see them");
|
||||
// The crate uses every supported form today, so none of the parser's
|
||||
// branches is exercised only by the synthetic test above.
|
||||
assert.deepEqual([...forms].sort(), ["concat-manifest-dir", "concat-metavariable", "literal"]);
|
||||
});
|
||||
|
||||
test("every PR reader restores the crate after the dependency cache and marks it before building", () => {
|
||||
for (const [name, buildCommand] of READERS) {
|
||||
const body = job(prWorkflow, name);
|
||||
const positions = [RUST_RESTORE_STEP, KEY_STEP, RESTORE_STEP, MARK_STEP].map((stepName) => body.indexOf(` - name: ${stepName}\n`));
|
||||
assert.ok(positions.every((position) => position >= 0), `${name}: a crate cache step is missing`);
|
||||
assert.deepEqual([...positions].sort((a, b) => a - b), positions, `${name}: the crate steps must follow the dependency restore in order`);
|
||||
const build = body.indexOf(buildCommand);
|
||||
assert.ok(build > positions.at(-1), `${name}: the crate must be marked before ${buildCommand}`);
|
||||
|
||||
const key = step(body, KEY_STEP);
|
||||
assert.match(key, /id: runner_release_crate_key\n/, name);
|
||||
assert.match(key, /run: echo "scope=\$\(node scripts\/runner-release-crate-cache\.mjs scope\)" >> "\$GITHUB_OUTPUT"/, name);
|
||||
|
||||
const restore = step(body, RESTORE_STEP);
|
||||
assert.match(restore, /id: runner_release_crate\n/, name);
|
||||
assertCacheStep(restore, `${name} restore`);
|
||||
assert.doesNotMatch(restore, /actions\/cache\/save/, name);
|
||||
|
||||
const mark = step(body, MARK_STEP);
|
||||
assert.match(mark, /if: \$\{\{ steps\.runner_release_crate\.outputs\.cache-hit == 'true' \}\}/, name);
|
||||
assert.match(mark, /run: node scripts\/runner-release-crate-cache\.mjs mark-current/, name);
|
||||
}
|
||||
// No other job restores or saves the crate.
|
||||
const total = prWorkflow.match(/- name: Restore the Runner release crate \(read only\)\n/g)?.length ?? 0;
|
||||
assert.equal(total, READERS.length);
|
||||
assert.doesNotMatch(prWorkflow, /actions\/cache\/save@/);
|
||||
});
|
||||
|
||||
test("the Verify Paperclip Runner job restores the crate only in the lanes that build it", () => {
|
||||
const body = job(prWorkflow, "verify_paperclip_runner");
|
||||
const lanes = [...body.matchAll(/- lane_label: (.+)\n\s+command: (.+)\n(\s+release_crate: (true)\n)?/g)].map(([, label, , , flag]) => [label, flag === "true"]);
|
||||
assert.deepEqual(lanes, [
|
||||
["static checks", false],
|
||||
["rust", true],
|
||||
["vitest 1/2", false],
|
||||
["vitest 2/2", true],
|
||||
]);
|
||||
for (const stepName of [KEY_STEP, RESTORE_STEP]) {
|
||||
assert.match(step(body, stepName), /if: \$\{\{ matrix\.release_crate == true \}\}/, stepName);
|
||||
}
|
||||
// A lane that skipped the restore has no cache-hit output, so the mark
|
||||
// step's own condition is enough there.
|
||||
assert.match(step(body, MARK_STEP), /if: \$\{\{ steps\.runner_release_crate\.outputs\.cache-hit == 'true' \}\}/);
|
||||
for (const [name] of READERS.filter(([name]) => name !== "verify_paperclip_runner")) {
|
||||
for (const stepName of [KEY_STEP, RESTORE_STEP]) {
|
||||
assert.doesNotMatch(step(job(prWorkflow, name), stepName), /^\s+if:/m, `${name}: ${stepName} is unconditional`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("only the master Rust lane writes the crate, after a lookup, with the readers' exact path list and key", () => {
|
||||
const body = job(releaseWorkflow, "verify_paperclip_runner");
|
||||
const positions = ["Verify Paperclip Runner", "Warm debug dependencies for the shared Runner cache", KEY_STEP, LOOKUP_STEP, BUILD_STEP, SAVE_STEP].map((stepName) => body.indexOf(` - name: ${stepName}\n`));
|
||||
assert.ok(positions.every((position) => position >= 0), "a writer step is missing");
|
||||
assert.deepEqual([...positions].sort((a, b) => a - b), positions, "the writer steps must follow the checks and the debug warm-up, in order");
|
||||
|
||||
for (const stepName of [KEY_STEP, LOOKUP_STEP]) {
|
||||
assert.match(step(body, stepName), new RegExp(`if: \\$\\{\\{ ${WRITER_GATE.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\}\\}`), stepName);
|
||||
}
|
||||
for (const stepName of [BUILD_STEP, SAVE_STEP]) {
|
||||
assert.match(
|
||||
step(body, stepName),
|
||||
new RegExp(`if: \\$\\{\\{ ${WRITER_GATE.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && steps\\.runner_release_crate\\.outputs\\.cache-hit != 'true' \\}\\}`),
|
||||
stepName,
|
||||
);
|
||||
}
|
||||
assert.match(step(body, KEY_STEP), /run: echo "scope=\$\(node scripts\/runner-release-crate-cache\.mjs scope\)" >> "\$GITHUB_OUTPUT"/);
|
||||
|
||||
const lookup = step(body, LOOKUP_STEP);
|
||||
assert.match(lookup, /id: runner_release_crate\n/);
|
||||
assert.match(lookup, /uses: actions\/cache\/restore@/);
|
||||
assertCacheStep(lookup, "writer lookup", { lookupOnly: true });
|
||||
|
||||
// The saved artifacts come from the readers' exact command, on a crate
|
||||
// cleaned of the test-profile variants the lane's checks built.
|
||||
const build = step(body, BUILD_STEP);
|
||||
assert.match(build, /working-directory: packages\/paperclip-runner\n/);
|
||||
assert.match(build, /set -euo pipefail\n\s+cargo clean --release --manifest-path runner\/Cargo\.toml -p paperclip-runner-core\n\s+pnpm build:binary\n/);
|
||||
const runnerScripts = JSON.parse(readFileSync(path.join(repoRoot, RUNNER_PACKAGE, "package.json"), "utf8")).scripts;
|
||||
assert.match(runnerScripts["build:binary"], /^cargo build --release --manifest-path runner\/Cargo\.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && /);
|
||||
|
||||
const save = step(body, SAVE_STEP);
|
||||
assert.match(save, /uses: actions\/cache\/save@/);
|
||||
assertCacheStep(save, "writer save");
|
||||
|
||||
// The dependency cache's own writer gate is the same expression.
|
||||
assert.match(body, new RegExp(`save-if: \\$\\{\\{ ${WRITER_GATE.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} \\}\\}`));
|
||||
assert.equal((releaseWorkflow.match(/actions\/cache\/save@/g) ?? []).length, 1);
|
||||
});
|
||||
|
||||
test("the scope names the platform, the compiler and the checkout root, and survives a missing rustc", () => {
|
||||
const a = computeScope({ workspace: "/home/runner/_work/paperclip/paperclip", rustcVersion: "1.97.1-abcdef123-x86_64-unknown-linux-gnu" });
|
||||
const b = computeScope({ workspace: "/home/runner/work/paperclip/paperclip", rustcVersion: "1.97.1-abcdef123-x86_64-unknown-linux-gnu" });
|
||||
const c = computeScope({ workspace: "/home/runner/_work/paperclip/paperclip", rustcVersion: "1.97.2-0123456ab-x86_64-unknown-linux-gnu" });
|
||||
assert.notEqual(a, b, "a different checkout root must select a different entry");
|
||||
assert.notEqual(a, c, "a different compiler must select a different entry");
|
||||
assert.equal(a, computeScope({ workspace: "/home/runner/_work/paperclip/paperclip/", rustcVersion: "1.97.1-abcdef123-x86_64-unknown-linux-gnu" }));
|
||||
assert.match(a, new RegExp(`^${os.platform()}-${os.arch()}-1\\.97\\.1-abcdef123-x86_64-unknown-linux-gnu-[0-9a-f]{12}$`));
|
||||
assert.doesNotMatch(computeScope({ rustcVersion: "weird version/with spaces" }), /[^A-Za-z0-9._-]/);
|
||||
|
||||
// The CLI reads rustc from PATH; a stand-in proves which fields it reads,
|
||||
// and an empty PATH proves a missing compiler still yields a usable key.
|
||||
const bin = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-rustc-"));
|
||||
try {
|
||||
const stub = path.join(bin, "rustc");
|
||||
writeFileSync(stub, '#!/bin/sh\nprintf "rustc 1.97.1 (8bab26f4f 2026-06-30)\\nbinary: rustc\\ncommit-hash: 8bab26f4f0123456789abcdef\\ncommit-date: 2026-06-30\\nhost: x86_64-unknown-linux-gnu\\nrelease: 1.97.1\\nLLVM version: 21.1.2\\n"\n');
|
||||
chmodSync(stub, 0o755);
|
||||
const withStub = execFileSync(process.execPath, [script, "scope"], { encoding: "utf8", env: { ...process.env, PATH: bin, GITHUB_WORKSPACE: "/home/runner/_work/paperclip/paperclip" } }).trim();
|
||||
assert.equal(withStub, computeScope({ workspace: "/home/runner/_work/paperclip/paperclip", rustcVersion: "1.97.1-8bab26f4f-x86_64-unknown-linux-gnu" }));
|
||||
const without = execFileSync(process.execPath, [script, "scope"], { encoding: "utf8", env: { ...process.env, PATH: mkdtempSync(path.join(os.tmpdir(), "empty-path-")), GITHUB_WORKSPACE: "/home/runner/_work/paperclip/paperclip" } }).trim();
|
||||
assert.match(without, /-no-rustc-/);
|
||||
} finally {
|
||||
rmSync(bin, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// The tree cargo leaves behind for the crate's release lib and bin units,
|
||||
// with the mtimes GitHub's cache restores: older than a fresh checkout.
|
||||
function layRestoredCrate(root) {
|
||||
const release = path.join(root, RELEASE_DIR);
|
||||
const lib = path.join(release, ".fingerprint/paperclip-runner-core-0b6c4654c2710743");
|
||||
const bin = path.join(release, ".fingerprint/paperclip-runner-core-46cce78dbc4840c2");
|
||||
mkdirSync(path.join(release, "deps"), { recursive: true });
|
||||
mkdirSync(lib, { recursive: true });
|
||||
mkdirSync(bin, { recursive: true });
|
||||
const files = {
|
||||
[path.join(lib, "lib-paperclip_runner_core")]: "0123456789abcdef",
|
||||
[path.join(lib, "lib-paperclip_runner_core.json")]: "{}",
|
||||
[path.join(lib, "dep-lib-paperclip_runner_core")]: "dep",
|
||||
[path.join(lib, "invoked.timestamp")]: "t",
|
||||
[path.join(bin, "bin-paperclip-runnerd")]: "fedcba9876543210",
|
||||
[path.join(bin, "bin-paperclip-runnerd.json")]: "{}",
|
||||
[path.join(bin, "dep-bin-paperclip-runnerd")]: "dep",
|
||||
[path.join(release, "deps/libpaperclip_runner_core-0b6c4654c2710743.rlib")]: "rlib",
|
||||
[path.join(release, "deps/libpaperclip_runner_core-0b6c4654c2710743.rmeta")]: "rmeta",
|
||||
[path.join(release, "deps/paperclip_runner_core-0b6c4654c2710743.d")]: "d",
|
||||
[path.join(release, "deps/paperclip_runnerd-46cce78dbc4840c2")]: "ELF",
|
||||
[path.join(release, "deps/paperclip_runnerd-46cce78dbc4840c2.d")]: "d",
|
||||
[path.join(release, "paperclip-runnerd")]: "ELF",
|
||||
[path.join(release, "paperclip-runnerd.d")]: "d",
|
||||
// A dependency's artifacts sit beside the crate's and must not be touched.
|
||||
[path.join(release, "deps/libserde-1234567890abcdef.rlib")]: "dep rlib",
|
||||
};
|
||||
const old = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||
for (const [file, content] of Object.entries(files)) {
|
||||
writeFileSync(file, content);
|
||||
utimesSync(file, old, old);
|
||||
}
|
||||
for (const dir of [lib, bin, path.join(release, "deps"), path.join(release, ".fingerprint"), release]) utimesSync(dir, old, old);
|
||||
return { release, files: Object.keys(files), old };
|
||||
}
|
||||
|
||||
test("mark-current makes every restored artifact newer than the checkout and leaves dependencies alone", () => {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-"));
|
||||
try {
|
||||
const { release, files, old } = layRestoredCrate(root);
|
||||
const checkout = path.join(root, RUNNER_PACKAGE, "runner/crates/runner-core/src/lib.rs");
|
||||
mkdirSync(path.dirname(checkout), { recursive: true });
|
||||
writeFileSync(checkout, "// checked out after the artifacts were built\n");
|
||||
const checkedOutAt = statSync(checkout).mtimeMs;
|
||||
|
||||
const result = markCurrent({ root, now: new Date(checkedOutAt + 5000) });
|
||||
assert.equal(result.action, "marked");
|
||||
assert.equal(result.files, 14);
|
||||
assert.equal(result.bytes, files.filter((file) => !file.includes("libserde")).reduce((sum, file) => sum + statSync(file).size, 0));
|
||||
|
||||
for (const file of files) {
|
||||
const mtime = statSync(file).mtimeMs;
|
||||
if (file.includes("libserde")) assert.equal(Math.floor(mtime / 1000), Math.floor(old.getTime() / 1000), "a dependency artifact must keep its mtime");
|
||||
else assert.ok(mtime > checkedOutAt, `${path.relative(release, file)} must be newer than the checkout`);
|
||||
}
|
||||
for (const dir of [".fingerprint/paperclip-runner-core-0b6c4654c2710743", ".fingerprint/paperclip-runner-core-46cce78dbc4840c2"]) {
|
||||
assert.ok(statSync(path.join(release, dir)).mtimeMs > checkedOutAt, `${dir} must be newer than the checkout`);
|
||||
}
|
||||
assert.ok(statSync(checkout).mtimeMs === checkedOutAt, "sources are never touched");
|
||||
|
||||
// The CLI prints the same result as one line of JSON for the job log.
|
||||
const printed = JSON.parse(execFileSync(process.execPath, [script, "mark-current"], { encoding: "utf8", env: { ...process.env, GITHUB_WORKSPACE: root } }));
|
||||
assert.equal(printed.schema, "paperclip.runner-release-crate.v1");
|
||||
assert.equal(printed.action, "marked");
|
||||
assert.equal(printed.files, 14);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// One file per REQUIRED entry, as cargo 1.97.1 lays them out for the crate's
|
||||
// release lib and bin units. Measured: without any one of these cargo
|
||||
// recompiles the unit (the whole crate for a lib file, a relink for a bin
|
||||
// file); without any of the files in NOT_NEEDED it reports the crate fresh
|
||||
// and recreates the uplifted binary from the one under deps.
|
||||
const REQUIRED_FILES = {
|
||||
"the lib fingerprint hash": ".fingerprint/paperclip-runner-core-0b6c4654c2710743/lib-paperclip_runner_core",
|
||||
"the lib dep-info": ".fingerprint/paperclip-runner-core-0b6c4654c2710743/dep-lib-paperclip_runner_core",
|
||||
"the lib rlib": "deps/libpaperclip_runner_core-0b6c4654c2710743.rlib",
|
||||
"the lib rmeta": "deps/libpaperclip_runner_core-0b6c4654c2710743.rmeta",
|
||||
"the bin fingerprint hash": ".fingerprint/paperclip-runner-core-46cce78dbc4840c2/bin-paperclip-runnerd",
|
||||
"the bin dep-info": ".fingerprint/paperclip-runner-core-46cce78dbc4840c2/dep-bin-paperclip-runnerd",
|
||||
"the bin under deps": "deps/paperclip_runnerd-46cce78dbc4840c2",
|
||||
};
|
||||
const NOT_NEEDED = [
|
||||
".fingerprint/paperclip-runner-core-0b6c4654c2710743/lib-paperclip_runner_core.json",
|
||||
".fingerprint/paperclip-runner-core-0b6c4654c2710743/invoked.timestamp",
|
||||
".fingerprint/paperclip-runner-core-46cce78dbc4840c2/bin-paperclip-runnerd.json",
|
||||
"deps/paperclip_runner_core-0b6c4654c2710743.d",
|
||||
"deps/paperclip_runnerd-46cce78dbc4840c2.d",
|
||||
"paperclip-runnerd",
|
||||
"paperclip-runnerd.d",
|
||||
];
|
||||
|
||||
test("mark-current discards a restore that lacks any file cargo needs for either unit, naming the file, so cargo sees a plain miss", () => {
|
||||
assert.deepEqual(REQUIRED.map(({ label }) => label), Object.keys(REQUIRED_FILES), "every REQUIRED entry has a missing-file case");
|
||||
for (const [label, file] of Object.entries(REQUIRED_FILES)) {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-"));
|
||||
try {
|
||||
const { release } = layRestoredCrate(root);
|
||||
rmSync(path.join(release, file));
|
||||
const result = markCurrent({ root });
|
||||
assert.equal(result.action, "discarded", `${file} missing must discard the restore`);
|
||||
assert.equal(result.reason, `incomplete: ${label} is missing`);
|
||||
assert.ok(result.removed > 0, label);
|
||||
assert.deepEqual(readdirSync(path.join(release, ".fingerprint")), [], label);
|
||||
assert.deepEqual(readdirSync(path.join(release, "deps")), ["libserde-1234567890abcdef.rlib"], `${label}: only the crate's own artifacts are removed`);
|
||||
assert.deepEqual(readdirSync(release).sort(), [".fingerprint", "deps"], label);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// The bin's dep-info alone does not stand in for the bin: the glob that
|
||||
// finds the binary under deps must not be satisfied by the `.d` beside it.
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-"));
|
||||
try {
|
||||
const { release } = layRestoredCrate(root);
|
||||
rmSync(path.join(release, "deps/paperclip_runnerd-46cce78dbc4840c2"));
|
||||
assert.ok(statSync(path.join(release, "deps/paperclip_runnerd-46cce78dbc4840c2.d")).isFile());
|
||||
assert.equal(markCurrent({ root }).reason, "incomplete: the bin under deps is missing");
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// A file cargo neither reads nor checks for may be absent without costing
|
||||
// a rebuild, so its absence must not cost one here either.
|
||||
for (const file of NOT_NEEDED) {
|
||||
const intact = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-"));
|
||||
try {
|
||||
const { release } = layRestoredCrate(intact);
|
||||
rmSync(path.join(release, file));
|
||||
const result = markCurrent({ root: intact });
|
||||
assert.equal(result.action, "marked", `${file} missing must not discard the restore`);
|
||||
assert.equal(result.files, 13, file);
|
||||
} finally {
|
||||
rmSync(intact, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing restored at all is reported the same way and exits cleanly.
|
||||
const empty = mkdtempSync(path.join(os.tmpdir(), "runner-release-crate-empty-"));
|
||||
try {
|
||||
const printed = JSON.parse(execFileSync(process.execPath, [script, "mark-current"], { encoding: "utf8", env: { ...process.env, GITHUB_WORKSPACE: empty } }));
|
||||
assert.equal(printed.action, "discarded");
|
||||
assert.equal(printed.reason, "incomplete: the lib fingerprint hash is missing");
|
||||
assert.equal(printed.removed, 0);
|
||||
} finally {
|
||||
rmSync(empty, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("the CLI prints the lists the workflows embed", () => {
|
||||
const paths = execFileSync(process.execPath, [script, "paths"], { encoding: "utf8" }).trim().split("\n");
|
||||
assert.deepEqual(paths, CACHED_PATHS);
|
||||
const inputs = execFileSync(process.execPath, [script, "hashed-inputs"], { encoding: "utf8" }).trim().split("\n");
|
||||
assert.deepEqual(inputs, HASHED_INPUTS);
|
||||
const usage = spawnSync(process.execPath, [script, "nonsense"], { encoding: "utf8" });
|
||||
assert.equal(usage.status, 2);
|
||||
assert.match(usage.stderr, /usage: runner-release-crate-cache\.mjs/);
|
||||
});
|
||||
@@ -1,6 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# Timestamped progress lines make the pre-start phase (container start to
|
||||
# the first app log line) measurable from the container log alone.
|
||||
log() {
|
||||
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) docker-entrypoint.sh: $*"
|
||||
}
|
||||
|
||||
log "starting"
|
||||
|
||||
# Capture runtime UID/GID from environment variables, defaulting to 1000
|
||||
PUID=${USER_UID:-1000}
|
||||
PGID=${USER_GID:-1000}
|
||||
@@ -15,6 +23,7 @@ if [ "$(id -u)" -ne 0 ]; then
|
||||
if [ "$(id -u)" -ne "$PUID" ] || [ "$(id -g)" -ne "$PGID" ]; then
|
||||
echo "docker-entrypoint.sh: running unprivileged as $(id -u):$(id -g); cannot remap to requested ${PUID}:${PGID}" >&2
|
||||
fi
|
||||
log "starting app as $(id -u):$(id -g)"
|
||||
exec "$@"
|
||||
fi
|
||||
|
||||
@@ -35,14 +44,70 @@ fi
|
||||
# (Docker named volume, Railway volume, Kubernetes PV) arrives root-owned
|
||||
# and shadows the image's build-time chown, so with the default UID the old
|
||||
# remap-only condition dropped privileges onto an unwritable home and the
|
||||
# server crashed on its first mkdir. The probe is a first-mismatch find
|
||||
# over the WHOLE tree (uid and gid): a root-owned mount or descendant
|
||||
# (init containers, backup restores, files written before a remap) is
|
||||
# found immediately and repaired recursively, a GID-only remap is caught,
|
||||
# and a fully-correct tree costs one metadata-only walk with no chown.
|
||||
# server crashed on its first mkdir.
|
||||
#
|
||||
# A full-tree probe costs one stat per file. On a home with many agent
|
||||
# worktrees and node_modules trees that is minutes of boot time, so the
|
||||
# probe is tiered:
|
||||
# 1. A shallow probe (the home and its first few levels) runs on every
|
||||
# boot. It catches a fresh root-owned mount and a UID/GID remap, because
|
||||
# both change the owner of the home directory itself.
|
||||
# 2. A full-tree probe runs only when the marker file below is missing or
|
||||
# records a different node UID/GID. It catches a mismatch deeper in the
|
||||
# tree (init containers, backup restores, files written before a remap,
|
||||
# the first boot after an upgrade from an older image).
|
||||
# Any mismatch repairs the whole tree. After a clean probe or a repair, the
|
||||
# marker records the verified node UID/GID, so later boots skip the full
|
||||
# walk. Delete the marker to force a full probe on the next boot.
|
||||
home_dir="${PAPERCLIP_HOME:-/paperclip}"
|
||||
if [ -d "$home_dir" ] && [ -n "$(find "$home_dir" \( ! -user node -o ! -group node \) -print -quit 2>/dev/null)" ]; then
|
||||
chown -R node:node "$home_dir"
|
||||
ownership_marker="$home_dir/.paperclip-ownership-verified"
|
||||
ownership_shallow_depth=3
|
||||
|
||||
# Sets needs_repair=1 when any entry is not node:node, and probe_failed=1
|
||||
# when find exits non-zero (for example, an I/O error), so an incomplete
|
||||
# walk is never recorded as verified. The arguments bound the walk (empty
|
||||
# for the full tree); -quit stops at the first mismatch.
|
||||
probe_ownership() {
|
||||
if ! first_mismatch="$(find "$home_dir" "$@" \( ! -user node -o ! -group node \) -print -quit 2>/dev/null)"; then
|
||||
probe_failed=1
|
||||
log "ownership probe of $home_dir failed; it will run again on the next boot"
|
||||
fi
|
||||
if [ -n "$first_mismatch" ]; then
|
||||
needs_repair=1
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -d "$home_dir" ]; then
|
||||
expected_marker="v1 $(id -u node):$(id -g node)"
|
||||
current_marker="$(cat "$ownership_marker" 2>/dev/null || true)"
|
||||
needs_repair=0
|
||||
probe_failed=0
|
||||
probe_ownership -maxdepth "$ownership_shallow_depth"
|
||||
if [ "$needs_repair" -eq 0 ] && [ "$current_marker" != "$expected_marker" ]; then
|
||||
log "full ownership probe of $home_dir (no current marker)"
|
||||
probe_started=$(date +%s)
|
||||
probe_ownership
|
||||
log "full ownership probe finished in $(( $(date +%s) - probe_started ))s"
|
||||
fi
|
||||
|
||||
if [ "$needs_repair" -eq 1 ]; then
|
||||
log "repairing ownership of $home_dir to node:node"
|
||||
repair_started=$(date +%s)
|
||||
chown -R node:node "$home_dir"
|
||||
log "ownership repair finished in $(( $(date +%s) - repair_started ))s"
|
||||
fi
|
||||
|
||||
# Record the marker only when the tree is known good: a successful
|
||||
# recursive chown (set -e stops the script if it fails), or probes that
|
||||
# all completed without a mismatch.
|
||||
if [ "$current_marker" != "$expected_marker" ] && { [ "$needs_repair" -eq 1 ] || [ "$probe_failed" -eq 0 ]; }; then
|
||||
# Write as node so the marker itself never trips the shallow probe
|
||||
# (root-squashed network mounts included). Best effort: a read-only
|
||||
# home only costs a full probe on the next boot.
|
||||
# shellcheck disable=SC2016 # $1/$2 expand in the inner shell
|
||||
gosu node sh -c 'printf "%s\n" "$1" > "$2"' sh "$expected_marker" "$ownership_marker" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
log "starting app as node"
|
||||
exec gosu node "$@"
|
||||
Executable
+142
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
# Prepare the disposable GitHub ARM qualification host using stock distro files.
|
||||
# Extraction skips package maintainer scripts and their global policy changes.
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${GITHUB_ACTIONS:-}" != true || "${RUNNER_OS:-}" != Linux ]]; then
|
||||
printf 'Linux ARM qualification preparation requires a GitHub Actions Linux runner.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
. /etc/os-release
|
||||
test "$ID" = ubuntu
|
||||
test "$VERSION_ID" = 24.04
|
||||
test "$(dpkg --print-architecture)" = arm64
|
||||
test "$(uname -m)" = aarch64
|
||||
prerequisite_report_dir=${RUNNER_TEMP:?}/paperclip-arm-bwrap-prerequisite
|
||||
mkdir -p "$prerequisite_report_dir"
|
||||
prerequisite_download_dir=$(mktemp -d "${RUNNER_TEMP}/paperclip-arm-stock-packages.XXXXXX")
|
||||
before_userns=$(sysctl -n kernel.apparmor_restrict_unprivileged_userns)
|
||||
before_apparmor=$(cat /sys/module/apparmor/parameters/enabled)
|
||||
before_clone=$(sysctl -n kernel.unprivileged_userns_clone)
|
||||
test "$before_userns" = 1
|
||||
test "$before_apparmor" = Y
|
||||
test "$before_clone" = 1
|
||||
printf 'restriction=%s apparmor=%s clone=%s\n' \
|
||||
"$before_userns" "$before_apparmor" "$before_clone" | \
|
||||
tee "$prerequisite_report_dir/before.txt"
|
||||
|
||||
parser=$(readlink -f "$(command -v apparmor_parser)")
|
||||
test "$parser" = /usr/sbin/apparmor_parser
|
||||
test "$(stat -c '%u' "$parser")" = 0
|
||||
dpkg-query -W -f='${Package} ${Version} ${Architecture}\n' \
|
||||
apparmor libc6 libcap2 libselinux1 | \
|
||||
tee "$prerequisite_report_dir/existing-package-versions.txt"
|
||||
dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libc6)" ge 2.38
|
||||
dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libcap2)" ge 1:2.10
|
||||
dpkg --compare-versions "$(dpkg-query -W -f='${Version}' libselinux1)" ge 3.1~
|
||||
|
||||
timeout 120s sudo apt-get \
|
||||
-o Acquire::AllowInsecureRepositories=false \
|
||||
-o APT::Get::AllowUnauthenticated=false update | \
|
||||
tee "$prerequisite_report_dir/apt-update.txt"
|
||||
apt-cache policy bubblewrap apparmor-profiles | \
|
||||
tee "$prerequisite_report_dir/apt-policy.txt"
|
||||
for package_name in bubblewrap apparmor-profiles; do
|
||||
package_version=$(apt-cache policy "$package_name" | awk '/Candidate:/ {print $2}')
|
||||
test -n "$package_version"
|
||||
test "$package_version" != '(none)'
|
||||
apt-cache show "$package_name=$package_version" > \
|
||||
"$prerequisite_report_dir/$package_name-index.txt"
|
||||
package_sha256=$(awk '/^SHA256:/ {print $2}' \
|
||||
"$prerequisite_report_dir/$package_name-index.txt")
|
||||
test "${#package_sha256}" = 64
|
||||
printf '%s\n' "$package_sha256" | grep -E '^[0-9a-f]{64}$' >/dev/null
|
||||
(
|
||||
cd "$prerequisite_download_dir"
|
||||
timeout 60s apt-get -o APT::Get::AllowUnauthenticated=false \
|
||||
download "$package_name=$package_version"
|
||||
)
|
||||
package_files=("$prerequisite_download_dir"/"$package_name"_*.deb)
|
||||
test "${#package_files[@]}" = 1
|
||||
package_file=${package_files[0]}
|
||||
test -f "$package_file"
|
||||
printf '%s %s\n' "$package_sha256" "$package_file" | sha256sum -c -
|
||||
test "$(dpkg-deb -f "$package_file" Package)" = "$package_name"
|
||||
test "$(dpkg-deb -f "$package_file" Version)" = "$package_version"
|
||||
dpkg-deb -f "$package_file" Package Version Architecture | \
|
||||
tee "$prerequisite_report_dir/$package_name-downloaded-version.txt"
|
||||
if test "$package_name" = bubblewrap; then
|
||||
test "$(dpkg-deb -f "$package_file" Architecture)" = arm64
|
||||
dpkg-deb --fsys-tarfile "$package_file" | \
|
||||
tar -xO ./usr/bin/bwrap > "$prerequisite_download_dir/bwrap"
|
||||
else
|
||||
test "$(dpkg-deb -f "$package_file" Architecture)" = all
|
||||
dpkg-deb --fsys-tarfile "$package_file" | \
|
||||
tar -xO ./usr/share/apparmor/extra-profiles/bwrap-userns-restrict > \
|
||||
"$prerequisite_download_dir/bwrap-userns-restrict"
|
||||
fi
|
||||
done
|
||||
|
||||
stock_binary=$prerequisite_download_dir/bwrap
|
||||
stock_profile=$prerequisite_download_dir/bwrap-userns-restrict
|
||||
printf '11d39094f044f0cda0febb3ad517b830301da6b2ce929664af09ee9e4dd264f9 %s\n' \
|
||||
"$stock_profile" | sha256sum -c -
|
||||
cat "$stock_profile" | tee "$prerequisite_report_dir/stock-profile.txt"
|
||||
test "$(od -An -tx1 -N4 "$stock_binary" | tr -d ' \n')" = 7f454c46
|
||||
readelf -h "$stock_binary" | tee "$prerequisite_report_dir/bwrap-elf-header.txt"
|
||||
grep -E 'Machine:.*AArch64' "$prerequisite_report_dir/bwrap-elf-header.txt"
|
||||
|
||||
# Reject every target symlink, including dangling links, before privileged copy.
|
||||
test ! -L /usr/bin/bwrap
|
||||
if test -e /usr/bin/bwrap; then
|
||||
test -f /usr/bin/bwrap
|
||||
cmp "$stock_binary" /usr/bin/bwrap
|
||||
else
|
||||
sudo install -o root -g root -m 0755 "$stock_binary" /usr/bin/bwrap
|
||||
fi
|
||||
test "$(readlink -f "$(command -v bwrap)")" = /usr/bin/bwrap
|
||||
test "$(stat -c '%u:%g:%a' /usr/bin/bwrap)" = 0:0:755
|
||||
command -v getcap >/dev/null
|
||||
test -z "$(getcap /usr/bin/bwrap)"
|
||||
sha256sum "$stock_binary" /usr/bin/bwrap | \
|
||||
tee "$prerequisite_report_dir/bwrap-sha256.txt"
|
||||
timeout 5s /usr/bin/bwrap --version | tee "$prerequisite_report_dir/bwrap-version.txt"
|
||||
help=$(timeout 5s /usr/bin/bwrap --help)
|
||||
printf '%s\n' "$help" > "$prerequisite_report_dir/bwrap-help.txt"
|
||||
printf '%s\n' "$help" | grep -F -- --as-pid-1 >/dev/null
|
||||
printf '%s\n' "$help" | grep -F -- --perms >/dev/null
|
||||
|
||||
target=/etc/apparmor.d/bwrap-userns-restrict
|
||||
test ! -L "$target"
|
||||
test ! -L /etc/apparmor.d/local/bwrap-userns-restrict
|
||||
test ! -L /etc/apparmor.d/local/unpriv_bwrap
|
||||
test ! -s /etc/apparmor.d/local/bwrap-userns-restrict
|
||||
test ! -s /etc/apparmor.d/local/unpriv_bwrap
|
||||
if test -e "$target"; then
|
||||
test -f "$target"
|
||||
cmp "$stock_profile" "$target"
|
||||
else
|
||||
sudo install -o root -g root -m 0644 "$stock_profile" "$target"
|
||||
fi
|
||||
timeout 15s sudo "$parser" -r "$target"
|
||||
sha256sum "$stock_profile" "$target" | \
|
||||
tee "$prerequisite_report_dir/profile-sha256.txt"
|
||||
loaded_profiles=$(sudo cat /sys/kernel/security/apparmor/profiles)
|
||||
for required_profile in 'bwrap (enforce)' 'unpriv_bwrap (enforce)'; do
|
||||
printf '%s\n' "$loaded_profiles" | grep -Fx "$required_profile" | \
|
||||
tee -a "$prerequisite_report_dir/loaded-profiles.txt"
|
||||
done
|
||||
after_userns=$(sysctl -n kernel.apparmor_restrict_unprivileged_userns)
|
||||
after_apparmor=$(cat /sys/module/apparmor/parameters/enabled)
|
||||
after_clone=$(sysctl -n kernel.unprivileged_userns_clone)
|
||||
test "$after_userns" = "$before_userns"
|
||||
test "$after_apparmor" = "$before_apparmor"
|
||||
test "$after_clone" = "$before_clone"
|
||||
printf 'restriction=%s apparmor=%s clone=%s\n' \
|
||||
"$after_userns" "$after_apparmor" "$after_clone" | \
|
||||
tee "$prerequisite_report_dir/after.txt"
|
||||
timeout 5s /usr/bin/bwrap --die-with-parent --unshare-user --unshare-net \
|
||||
--ro-bind / / --cap-drop ALL /bin/true
|
||||
printf 'stock files materialized without dpkg registration or maintainer scripts; namespace probe passed; no provider processes or turns\n' | \
|
||||
tee "$prerequisite_report_dir/namespace-probe.txt"
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env node
|
||||
// The Runner's own crate is the one thing the shared Rust dependency cache
|
||||
// leaves out: rust-cache drops workspace crates when it saves, and a fresh
|
||||
// checkout would invalidate cargo's mtime fingerprints for them anyway. So
|
||||
// every CI build compiled `paperclip-runner-core` in release mode from
|
||||
// scratch (28-49s per run on 2026-10-11) although 221 of the 239 master
|
||||
// commits of the preceding week left its inputs untouched.
|
||||
//
|
||||
// This script backs a second, content-addressed cache that holds only that
|
||||
// crate's release artifacts (about 70 MB). The master writer saves them
|
||||
// under a key made from the compiler, the checkout root and a hash of every
|
||||
// input the build reads; a pull request whose merge tree carries
|
||||
// byte-identical inputs restores them. Cargo judges a workspace crate fresh
|
||||
// only while no source is newer than its dep-info file, and GitHub's cache
|
||||
// keeps the artifacts' original mtimes, so `mark-current` touches the
|
||||
// restored set after the checkout. Any input change selects a different key,
|
||||
// the restore misses, and cargo compiles exactly as it did before.
|
||||
//
|
||||
// Subcommands:
|
||||
// paths print the cached paths, one per line (repo-relative globs)
|
||||
// hashed-inputs print the hashFiles() patterns that make up the key
|
||||
// scope print the compiler/checkout part of the key
|
||||
// mark-current make a restored artifact set newer than the checkout, or
|
||||
// discard an incomplete one so cargo rebuilds instead
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readdirSync, rmSync, statSync, utimesSync, lstatSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const RUNNER_PACKAGE = "packages/paperclip-runner";
|
||||
const RELEASE_DIR = `${RUNNER_PACKAGE}/runner/target/release`;
|
||||
|
||||
// Everything cargo writes for the crate's release lib and bin units:
|
||||
// fingerprints (both units share the package-named directory prefix), the
|
||||
// lib's rlib/rmeta and dep-info, the bin and its dep-info under deps, and the
|
||||
// uplifted executable the staging step copies into dist/bin.
|
||||
export const CACHED_PATHS = [
|
||||
`${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*`,
|
||||
`${RELEASE_DIR}/deps/libpaperclip_runner_core-*`,
|
||||
`${RELEASE_DIR}/deps/paperclip_runner_core-*.d`,
|
||||
`${RELEASE_DIR}/deps/paperclip_runnerd-*`,
|
||||
`${RELEASE_DIR}/paperclip-runnerd`,
|
||||
`${RELEASE_DIR}/paperclip-runnerd.d`,
|
||||
];
|
||||
|
||||
// Every file the release build of the crate reads, as hashFiles() patterns:
|
||||
// the toolchain pin, the manifests and lockfile, the crate sources, and the
|
||||
// protocol schemas and fixtures the sources include with include_str!. The
|
||||
// three fixture trees are read only by test builds, but hashing them costs
|
||||
// nothing and keeps the include_str! guard test simple: every include in the
|
||||
// crate, whatever its form, must land under one of these patterns.
|
||||
export const HASHED_INPUTS = [
|
||||
`${RUNNER_PACKAGE}/rust-toolchain.toml`,
|
||||
`${RUNNER_PACKAGE}/runner/Cargo.toml`,
|
||||
`${RUNNER_PACKAGE}/runner/Cargo.lock`,
|
||||
`${RUNNER_PACKAGE}/runner/crates/**`,
|
||||
`${RUNNER_PACKAGE}/protocol/**`,
|
||||
`${RUNNER_PACKAGE}/test-fixtures/**`,
|
||||
`${RUNNER_PACKAGE}/test/fixtures/**`,
|
||||
"tests/runner-e2e/fixtures/**",
|
||||
];
|
||||
|
||||
// A restored set is usable only when both units are complete: everything
|
||||
// cargo's freshness check reads for a unit (the fingerprint hash and the
|
||||
// dep-info it compares source mtimes against) and every output it checks for
|
||||
// (the lib's rlib and rmeta, the bin under deps). Measured with cargo 1.97.1:
|
||||
// without any one of these it recompiles the unit; without the `.json`
|
||||
// fingerprint records, rustc's `.d` files or the uplifted binary and its `.d`
|
||||
// it reports the crate fresh and recreates the uplifted copy from deps.
|
||||
// Anything less than this set is discarded, with the missing file named in
|
||||
// the job log, so cargo sees a plain miss rather than a half-restored crate.
|
||||
export const REQUIRED = [
|
||||
{ label: "the lib fingerprint hash", glob: `${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*/lib-paperclip_runner_core` },
|
||||
{ label: "the lib dep-info", glob: `${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*/dep-lib-paperclip_runner_core` },
|
||||
{ label: "the lib rlib", glob: `${RELEASE_DIR}/deps/libpaperclip_runner_core-*.rlib` },
|
||||
{ label: "the lib rmeta", glob: `${RELEASE_DIR}/deps/libpaperclip_runner_core-*.rmeta` },
|
||||
{ label: "the bin fingerprint hash", glob: `${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*/bin-paperclip-runnerd` },
|
||||
{ label: "the bin dep-info", glob: `${RELEASE_DIR}/.fingerprint/paperclip-runner-core-*/dep-bin-paperclip-runnerd` },
|
||||
{ label: "the bin under deps", glob: `${RELEASE_DIR}/deps/paperclip_runnerd-*`, exclude: /\.d$/ },
|
||||
];
|
||||
|
||||
const SCHEMA = "paperclip.runner-release-crate.v1";
|
||||
|
||||
function keySafe(value) {
|
||||
return String(value).replace(/[^A-Za-z0-9._-]+/g, "_");
|
||||
}
|
||||
|
||||
/**
|
||||
* The compiler and checkout part of the cache key. The compiler decides the
|
||||
* artifacts' metadata hashes; the checkout root is baked into cargo's
|
||||
* dep-info for the protocol files the sources include by relative path, so an
|
||||
* entry written under one root is useless under another (the RunsOn fleets
|
||||
* check out under /home/runner/_work, GitHub-hosted runners under
|
||||
* /home/runner/work). Hashing the root into the key turns that into a plain
|
||||
* miss instead of a restore cargo then throws away.
|
||||
*/
|
||||
export function computeScope({ workspace = process.env.GITHUB_WORKSPACE ?? process.cwd(), rustcVersion = readRustcVersion() } = {}) {
|
||||
const root = createHash("sha256").update(path.resolve(workspace)).digest("hex").slice(0, 12);
|
||||
return `${keySafe(os.platform())}-${keySafe(os.arch())}-${keySafe(rustcVersion)}-${root}`;
|
||||
}
|
||||
|
||||
function readRustcVersion(env = process.env) {
|
||||
let output;
|
||||
try {
|
||||
output = execFileSync("rustc", ["--version", "--verbose"], { encoding: "utf8", env });
|
||||
} catch {
|
||||
return "no-rustc";
|
||||
}
|
||||
const field = (name) => output.match(new RegExp(`^${name}: (.+)$`, "m"))?.[1]?.trim() ?? "unknown";
|
||||
return `${field("release")}-${field("commit-hash").slice(0, 9)}-${field("host")}`;
|
||||
}
|
||||
|
||||
function expandGlob(root, pattern) {
|
||||
// Patterns only ever use `*` inside a single path segment.
|
||||
const segments = pattern.split("/");
|
||||
let current = [root];
|
||||
for (const segment of segments) {
|
||||
const next = [];
|
||||
for (const base of current) {
|
||||
if (!segment.includes("*")) {
|
||||
const candidate = path.join(base, segment);
|
||||
if (existsSync(candidate)) next.push(candidate);
|
||||
continue;
|
||||
}
|
||||
let entries;
|
||||
try {
|
||||
entries = readdirSync(base);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const matcher = new RegExp(`^${segment.split("*").map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join(".*")}$`);
|
||||
for (const entry of entries) {
|
||||
if (matcher.test(entry)) next.push(path.join(base, entry));
|
||||
}
|
||||
}
|
||||
current = next;
|
||||
}
|
||||
return current.sort();
|
||||
}
|
||||
|
||||
function walk(target, visit) {
|
||||
const info = lstatSync(target);
|
||||
visit(target, info);
|
||||
if (info.isDirectory()) {
|
||||
for (const entry of readdirSync(target)) walk(path.join(target, entry), visit);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Touch every restored artifact so cargo's freshness check, which compares
|
||||
* each source's mtime against the unit's dep-info, sees the checkout as
|
||||
* older than the build. When the set is incomplete, remove whatever was
|
||||
* restored and report it: cargo then compiles the crate as on a miss.
|
||||
*/
|
||||
export function markCurrent({ root = process.env.GITHUB_WORKSPACE ?? process.cwd(), now = new Date() } = {}) {
|
||||
const missing = REQUIRED.find(({ glob, exclude }) => expandGlob(root, glob).filter((entry) => !exclude?.test(entry)).length === 0);
|
||||
const matched = CACHED_PATHS.flatMap((pattern) => expandGlob(root, pattern));
|
||||
if (missing) {
|
||||
for (const target of matched) rmSync(target, { recursive: true, force: true });
|
||||
return { schema: SCHEMA, action: "discarded", reason: `incomplete: ${missing.label} is missing`, removed: matched.length };
|
||||
}
|
||||
let files = 0;
|
||||
let bytes = 0;
|
||||
const seen = new Set();
|
||||
for (const target of matched) {
|
||||
walk(target, (entry, info) => {
|
||||
if (info.isSymbolicLink()) return;
|
||||
utimesSync(entry, now, now);
|
||||
if (info.isFile()) {
|
||||
files += 1;
|
||||
// The uplifted binary is a hard link to the one under deps.
|
||||
const { ino, size } = statSync(entry);
|
||||
if (!seen.has(ino)) {
|
||||
seen.add(ino);
|
||||
bytes += size;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return { schema: SCHEMA, action: "marked", files, bytes };
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
const [command] = argv;
|
||||
switch (command) {
|
||||
case "paths":
|
||||
process.stdout.write(`${CACHED_PATHS.join("\n")}\n`);
|
||||
return 0;
|
||||
case "hashed-inputs":
|
||||
process.stdout.write(`${HASHED_INPUTS.join("\n")}\n`);
|
||||
return 0;
|
||||
case "scope":
|
||||
process.stdout.write(`${computeScope()}\n`);
|
||||
return 0;
|
||||
case "mark-current": {
|
||||
const result = markCurrent();
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
return 0;
|
||||
}
|
||||
default:
|
||||
process.stderr.write("usage: runner-release-crate-cache.mjs <paths|hashed-inputs|scope|mark-current>\n");
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
process.exitCode = main(process.argv.slice(2));
|
||||
}
|
||||
Reference in new issue
Block a user