import { execFile } from "node:child_process"; import { createReadStream } from "node:fs"; import { createHash } from "node:crypto"; import { lstat, mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { pathToFileURL } from "node:url"; import { BoundedHistoryPublisher } from "./history-budget.js"; const execFileAsync = promisify(execFile); export const PRIVATE_HISTORY_PREFIX = "runner-e2e-private/bounded-v1"; export const PRIVATE_ARCHIVE_BYTES = 264 * 1024 * 1024; export function privateArtifactKey(kind: string, runId: string) { if (!["evidence", "report"].includes(kind) || !/^[1-9][0-9]{0,19}$/.test(runId)) { throw new Error("Private history needs an evidence/report kind and numeric workflow run ID"); } return `${PRIVATE_HISTORY_PREFIX}/campaigns/gha-${runId}-1/${kind}.zip`; } export async function archivePrivateEvidence(operation: "pack" | "unpack", source: string, destination: string) { const result = await execFileAsync("python3", [path.join(import.meta.dirname, "private-history-archive.py"), operation, source, destination], { timeout: 60_000, maxBuffer: 64 * 1024, }); return JSON.parse(result.stdout) as { files: number; sourceBytes: number; archiveBytes?: number }; } export async function verifyPrivateHistory(publisher: BoundedHistoryPublisher, bucket: string) { if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) throw new Error("Invalid private history bucket"); await publisher.verifyRetention(bucket, PRIVATE_HISTORY_PREFIX); const configuration = async (operation: string) => JSON.parse((await publisher.run(["s3api", operation, "--bucket", bucket])).stdout); const block = (await configuration("get-public-access-block")).PublicAccessBlockConfiguration; if (!["BlockPublicAcls", "IgnorePublicAcls", "BlockPublicPolicy", "RestrictPublicBuckets"].every(key => block?.[key] === true)) { throw new Error("Private history requires all bucket public-access blocks"); } const ownership = (await configuration("get-bucket-ownership-controls")).OwnershipControls?.Rules; if (!Array.isArray(ownership) || ownership.length !== 1 || ownership[0]?.ObjectOwnership !== "BucketOwnerEnforced") { throw new Error("Private history requires disabled object ACLs"); } const policy = JSON.parse((await configuration("get-bucket-policy")).Policy); if (!Array.isArray(policy.Statement) || policy.Statement.some((statement: { Effect?: string }) => statement.Effect !== "Deny")) { throw new Error("Private history bucket policy must not grant access to external readers or a CDN"); } } type TransferOptions = { operation: "upload" | "download"; kind: "evidence" | "report"; runId: string; directory: string; bucket: string; publisher: BoundedHistoryPublisher; }; export async function transferPrivateArtifact(options: TransferOptions) { privateArtifactKey(options.kind, options.runId); await verifyPrivateHistory(options.publisher, options.bucket); return transferVerifiedArtifact(options); } async function transferVerifiedArtifact(options: TransferOptions) { const key = privateArtifactKey(options.kind, options.runId); const temporary = await mkdtemp(path.join(os.tmpdir(), "paperclip-private-history-")); const archive = path.join(temporary, "archive.zip"); try { if (options.operation === "upload") { const metadata = await archivePrivateEvidence("pack", options.directory, archive); const hash = createHash("sha256"); for await (const chunk of createReadStream(archive)) hash.update(chunk); const result = await options.publisher.upload(options.bucket, key, archive, "application/zip", "private, no-store", true); const response = JSON.parse(result.stdout) as { VersionId?: string }; if (!response.VersionId) throw new Error("Private archive upload did not return a version identity"); return { private: true, ...metadata, uri: `s3://${options.bucket}/${key}`, versionId: response.VersionId, sha256: hash.digest("hex") }; } await options.publisher.run(["s3api", "get-object", "--bucket", options.bucket, "--key", key, "--range", `bytes=0-${PRIVATE_ARCHIVE_BYTES}`, archive]); if ((await lstat(archive)).size > PRIVATE_ARCHIVE_BYTES) throw new Error("Private archive exceeds transfer allowance"); const metadata = await archivePrivateEvidence("unpack", archive, options.directory); return { ...metadata, uri: `s3://${options.bucket}/${key}` }; } finally { await rm(temporary, { recursive: true, force: true }); } } export async function retainPrivateTransfer(options: TransferOptions, receipt: string, receiptPublisher?: () => BoundedHistoryPublisher) { const key = privateArtifactKey(options.kind, options.runId); await verifyPrivateHistory(options.publisher, options.bucket); let result: Awaited> | undefined; let failure: unknown; try { result = await transferVerifiedArtifact(options); } catch (error) { failure = error; } if (receiptPublisher) { await options.publisher.save(); const snapshot = JSON.stringify({ ...JSON.parse(await readFile(receipt, "utf8")), transfer: result ? { status: "complete", ...result } : { status: "failed_or_unknown" }, receiptUpload: { status: "pending_at_snapshot", maxBytes: 1024 * 1024 } }); if (Buffer.byteLength(snapshot) > 1024 * 1024) throw new Error("Private transfer receipt exceeds allowance"); await writeFile(`${receipt}.transfer.json`, snapshot, { mode: 0o600, flag: "wx" }); try { await receiptPublisher().upload(options.bucket, `${key}.${options.operation}-receipt.json`, `${receipt}.transfer.json`, "application/json", "private, no-store", true); } catch (retentionError) { throw new AggregateError(failure ? [failure, retentionError] : [retentionError], "Private transfer receipt retention failed; local receipts remain authoritative"); } } if (failure) throw failure; return result!; } async function main() { const [operation, kind, runId, directory] = process.argv.slice(2); if (!["verify", "upload", "download"].includes(operation ?? "") || !kind || !runId || !directory) { throw new Error("Usage: private-history.ts verify|upload|download evidence|report RUN_ID DIRECTORY"); } privateArtifactKey(kind, runId); if (process.env.GITHUB_ACTIONS === "true" && (process.env.GITHUB_RUN_ATTEMPT !== "1" || process.env.GITHUB_RUN_ID !== runId)) { throw new Error("Private bounded history accepts only the initial attempt of this workflow run"); } const bucket = process.env.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET ?? ""; const receipt = path.resolve(`private-history-${operation}-${kind}-${runId}.json`); await (await open(receipt, "wx", 0o600)).close(); const publisher = new BoundedHistoryPublisher(receipt); if (operation === "verify") { await verifyPrivateHistory(publisher, bucket); console.log("Private retention and access checks passed"); return; } const receiptPublisher = operation === "upload" || process.env.GITHUB_ACTIONS === "true" ? () => new BoundedHistoryPublisher(`${receipt}.retention.json`) : undefined; const result = await retainPrivateTransfer({ operation: operation as "upload" | "download", kind: kind as "evidence" | "report", runId, directory, bucket, publisher }, receipt, receiptPublisher); console.log(JSON.stringify(process.env.GITHUB_ACTIONS === "true" ? { version: 1, privateTransferCompleted: true } : result)); } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { main().catch(error => { console.error(process.env.GITHUB_ACTIONS === "true" ? "Private history transfer failed; inspect private retained evidence." : error instanceof Error ? error.message : String(error)); process.exitCode = 1; }); }