diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index fa5e9e74e4..67f4840032 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -31,6 +31,10 @@ on: description: "Verify frozen Pi 1.0 closed startup and SDK contracts on native Intel macOS (no provider prompts)" type: boolean default: false + diagnose_pi_snapshot_streaming: + description: "Compare retained B9 baseline and unpublished streaming snapshots in four ABBA children (diagnostic only)" + type: boolean + default: false diagnose_pi_snapshot_pool: description: "Measure exact retained B9 snapshots in four process-start pool4/16 ABBA children (diagnostic only)" type: boolean @@ -74,7 +78,7 @@ permissions: {} concurrency: # Different immutable manual sources retain independent evidence; the same # source/mode still supersedes itself. Ordinary PR grouping is unchanged. - group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.diagnose_pi_snapshot_pool && 'pi-snapshot-pool' || inputs.diagnose_pi_startup && 'pi-startup-diagnostic' || inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }}${{ github.event_name == 'workflow_dispatch' && inputs.expected_source_sha && format('-{0}', inputs.expected_source_sha) || '' }} + group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.diagnose_pi_snapshot_streaming && 'pi-snapshot-streaming' || inputs.diagnose_pi_snapshot_pool && 'pi-snapshot-pool' || inputs.diagnose_pi_startup && 'pi-startup-diagnostic' || inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }}${{ github.event_name == 'workflow_dispatch' && inputs.expected_source_sha && format('-{0}', inputs.expected_source_sha) || '' }} cancel-in-progress: true jobs: @@ -129,10 +133,11 @@ jobs: SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }} IMAGE_MODE: ${{ inputs.publish_eval_image || inputs.verify_public_install || inputs.build_eval_viewer }} VERIFY_PI_INTEL: ${{ inputs.verify_pi_intel }} + DIAGNOSE_PI_SNAPSHOT_STREAMING: ${{ inputs.diagnose_pi_snapshot_streaming }} DIAGNOSE_PI_SNAPSHOT_POOL: ${{ inputs.diagnose_pi_snapshot_pool }} DIAGNOSE_PI_STARTUP: ${{ inputs.diagnose_pi_startup }} DIAGNOSE_AJV: ${{ inputs.diagnose_ajv_pack }} - OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel || inputs.diagnose_pi_startup || inputs.diagnose_pi_snapshot_pool || inputs.source_e2e_support_only }} + OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel || inputs.diagnose_pi_startup || inputs.diagnose_pi_snapshot_pool || inputs.diagnose_pi_snapshot_streaming || inputs.source_e2e_support_only }} run: | set -euo pipefail test "$REPOSITORY" = paperclipai/paperclip @@ -176,6 +181,7 @@ jobs: test "$EXPECTED_LOCK_SHA256" = 5ae57d1ddd475691dac1f1cfbd4836e54f7da1956b47e6e705b218ae3070ae2e fi if [ "$VERIFY_PI_INTEL" = true ]; then + test "$DIAGNOSE_PI_SNAPSHOT_STREAMING" != true test "$DIAGNOSE_PI_SNAPSHOT_POOL" != true test "$DIAGNOSE_PI_STARTUP" != true test "$VERIFY_RUNNER" != true @@ -190,6 +196,7 @@ jobs: test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public fi if [ "$DIAGNOSE_PI_STARTUP" = true ]; then + test "$DIAGNOSE_PI_SNAPSHOT_STREAMING" != true test "$DIAGNOSE_PI_SNAPSHOT_POOL" != true test "$VERIFY_PI_INTEL" != true test "$VERIFY_RUNNER" != true @@ -203,6 +210,21 @@ jobs: test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public fi if [ "$DIAGNOSE_PI_SNAPSHOT_POOL" = true ]; then + test "$DIAGNOSE_PI_SNAPSHOT_STREAMING" != true + test "$DIAGNOSE_PI_STARTUP" != true + test "$VERIFY_PI_INTEL" != true + test "$VERIFY_RUNNER" != true + test "$VERIFY_SOURCE" != true + test "$SOURCE_ROOT_TESTS_ONLY" != true + test "$SOURCE_E2E_SUPPORT_ONLY" != true + test "$DIAGNOSE_AJV" != true + test "$IMAGE_MODE" != true + test "$EXPECTED_SOURCE_SHA" = b9e5d6ecdb05ab7244c90976e07c950f8d09b15b + test "$EXPECTED_LOCK_SHA256" = 38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba + test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public + fi + if [ "$DIAGNOSE_PI_SNAPSHOT_STREAMING" = true ]; then + test "$DIAGNOSE_PI_SNAPSHOT_POOL" != true test "$DIAGNOSE_PI_STARTUP" != true test "$VERIFY_PI_INTEL" != true test "$VERIFY_RUNNER" != true @@ -378,6 +400,57 @@ jobs: retention-days: 7 if-no-files-found: error + manual_pi_snapshot_streaming: + name: Retained B9 native Intel baseline/streaming ABBA (not qualification) + if: github.event_name == 'workflow_dispatch' && inputs.diagnose_pi_snapshot_streaming + needs: authorize_manual + runs-on: macos-15-intel + timeout-minutes: 25 + permissions: + contents: read + actions: read + steps: + - name: Checkout trusted diagnostic helpers only + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.workflow_sha }} + path: trusted-ci + persist-credentials: false + - name: Download exact failed qualification artifacts + timeout-minutes: 5 + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api repos/paperclipai/paperclip/actions/artifacts/11208570718/zip \ + > "$RUNNER_TEMP/pi-startup-36959948724.zip" + printf '%s %s\n' b82cf6c843006781e2e6d5f0a06e0ff39497ffc457b94a511f8bdda66628aa43 \ + "$RUNNER_TEMP/pi-startup-36959948724.zip" | shasum -a 256 --check + - name: Measure four baseline/candidate snapshots with identical process-start pool4 + timeout-minutes: 18 + run: | + set -euo pipefail + python3 -B trusted-ci/scripts/ci/pi-intel/snapshot_streaming.py \ + --archive "$RUNNER_TEMP/pi-startup-36959948724.zip" \ + --output "$GITHUB_WORKSPACE/pi-snapshot-streaming-evidence" + - name: Admit complete diagnostic evidence within the storage bound + if: always() + id: diagnostic_evidence + timeout-minutes: 2 + run: | + python3 -B trusted-ci/scripts/ci/pi-intel/admit_diagnostic_evidence.py \ + --evidence "$GITHUB_WORKSPACE/pi-snapshot-streaming-evidence" \ + --github-output "$GITHUB_OUTPUT" + - name: Retain original identities, four snapshot measurements and cleanup + if: always() && steps.diagnostic_evidence.outputs.admitted == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pi-snapshot-streaming-${{ github.run_id }} + include-hidden-files: true + path: pi-snapshot-streaming-evidence/ + retention-days: 7 + if-no-files-found: error + manual_runner_verify: name: Full offline Runner verification on GitHub-hosted Ubuntu if: github.event_name == 'workflow_dispatch' && inputs.verify_runner @@ -808,7 +881,7 @@ jobs: manual_image: name: Build and verify on EC2 - if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel && !inputs.diagnose_pi_startup && !inputs.diagnose_pi_snapshot_pool && !inputs.source_e2e_support_only + if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel && !inputs.diagnose_pi_startup && !inputs.diagnose_pi_snapshot_pool && !inputs.diagnose_pi_snapshot_streaming && !inputs.source_e2e_support_only needs: authorize_manual runs-on: runs-on/fleet=paperclip-public-pr-x64/env=public-ci timeout-minutes: 90 diff --git a/scripts/ci/pi-intel/snapshot-streaming-candidate.mjs b/scripts/ci/pi-intel/snapshot-streaming-candidate.mjs new file mode 100644 index 0000000000..62ae47c86d --- /dev/null +++ b/scripts/ci/pi-intel/snapshot-streaming-candidate.mjs @@ -0,0 +1,307 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { chmod, lstat, mkdir, mkdtemp, open, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +const MAX_NATIVE_TREE_BYTES = 1024 * 1024 * 1024; +const MAX_NATIVE_FILE_BYTES = 384 * 1024 * 1024; +const MAX_NATIVE_MANIFEST_BYTES = 4 * 1024 * 1024; +// Bound active copy workers and their retained buffers. Large files stream +// through one chunk instead of excluding other copies or retaining whole files. +const NATIVE_COPY_CONCURRENCY = 32; +const NATIVE_DIRECTORY_CONCURRENCY = 32; +const NATIVE_COPY_BUFFER_BYTES = 32 * 1024 * 1024; +const NATIVE_COPY_CHUNK_BYTES = 1024 * 1024; +const BOOTSTRAP = ".paperclip-native-entry.cjs"; +const GUARD = ".paperclip-native-module-guard.cjs"; +const COPILOT_ENTRY = ".paperclip-copilot-entry"; +export const NATIVE_ACPX_BOOTSTRAP_NAME = BOOTSTRAP; +const sha256 = (value) => createHash("sha256").update(value).digest("hex"); +const validPath = (value) => typeof value === "string" && value.length > 0 && value.length <= 4_096 && !isAbsolute(value) && !/[\u0000-\u001f\u007f\\]/.test(value) && value.split("/").every(part => part.length > 0 && part !== "." && part !== ".."); +/** Parse a closed, canonical execution inventory before opening any file in it. */ +export function parseNativeAcpxDistributionEntries(value, expectedSha256) { + if (!/^[a-f0-9]{64}$/.test(expectedSha256)) + throw new Error("Native ACPX closure pin is invalid"); + const entries = value !== null && typeof value === "object" ? value.entries : undefined; + if (!Array.isArray(entries) || entries.length === 0 || entries.length > 30_000) + throw new Error("Native ACPX closure inventory is invalid"); + let total = 0; + let previous = ""; + const parsed = entries.map(raw => { + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) + throw new Error("Native ACPX closure entry is invalid"); + const entry = raw; + if (Object.keys(entry).some(key => !["path", "sha256", "size", "executable"].includes(key)) || !validPath(entry.path) || entry.path <= previous || entry.path === BOOTSTRAP || entry.path === GUARD || entry.path === "manifest.json" || entry.path.startsWith(".paperclip-")) + throw new Error("Native ACPX closure paths must be safe, unique and sorted"); + if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(entry.sha256) || !Number.isSafeInteger(entry.size) || Number(entry.size) < 0 || Number(entry.size) > MAX_NATIVE_FILE_BYTES || typeof entry.executable !== "boolean") + throw new Error("Native ACPX closure file metadata is invalid"); + previous = entry.path; + total += Number(entry.size); + if (total > MAX_NATIVE_TREE_BYTES) + throw new Error("Native ACPX closure exceeds its byte bound"); + return { path: entry.path, sha256: entry.sha256, size: Number(entry.size), executable: entry.executable }; + }); + if (sha256(JSON.stringify(parsed)) !== expectedSha256) + throw new Error("Native ACPX closure manifest digest mismatch"); + return parsed; +} +export async function readNativeAcpxDistributionEntries(input) { + if (!validPath(input.executable) || (input.entrypoint !== undefined && !validPath(input.entrypoint)) || input.fixedArguments.length > 128 || input.fixedArguments.some(arg => typeof arg !== "string" || arg.length > 16_384 || arg.includes("\0")) || (input.isolatedCacheEnvironmentName !== undefined && input.isolatedCacheEnvironmentName !== "COPILOT_PKG_CACHE_HOME") + || (input.copilotDistributionDirectory !== undefined && (!validPath(input.copilotDistributionDirectory) || input.entrypoint !== undefined))) + throw new Error("Native ACPX launch declaration is invalid"); + const file = await open(input.manifestPath, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await file.stat({ bigint: true }); + if (!before.isFile() || before.size < 1n || before.size > BigInt(MAX_NATIVE_MANIFEST_BYTES)) + throw new Error("Native ACPX manifest must be a bounded regular file"); + const bytes = await file.readFile(); + if (!same(before, await file.stat({ bigint: true })) || bytes.length !== Number(before.size)) + throw new Error("Native ACPX manifest changed while read"); + const entries = parseNativeAcpxDistributionEntries(JSON.parse(bytes.toString("utf8")), input.expectedClosureSha256); + if (!entries.some(entry => entry.path === input.executable && entry.executable && entry.size > 0) || (input.entrypoint !== undefined && !entries.some(entry => entry.path === input.entrypoint && entry.size > 0))) + throw new Error("Native ACPX executable or entrypoint is absent from its closure"); + if (input.copilotDistributionDirectory !== undefined && !entries.some(entry => entry.path === `${input.copilotDistributionDirectory}/index.js` && entry.size > 0)) + throw new Error("Copilot owned entrypoint is absent from its closure"); + return entries; + } + finally { + await file.close(); + } +} +/** + * Freeze only manifest-admitted bytes. Native trees deliberately have a separate + * bound; the tighter JavaScript/npm snapshot limits remain unchanged. + */ +export async function createNativeAcpxDistributionSnapshot(input, entries) { + // Callers cannot substitute entries between manifest validation and copying. + entries = parseNativeAcpxDistributionEntries({ entries }, input.expectedClosureSha256); + if (process.platform !== "linux" && process.platform !== "darwin") + throw new Error("Native ACPX snapshots require Linux or macOS"); + const source = await realpath(input.distributionRoot); + const rootBefore = await lstat(source, { bigint: true }); + if (!rootBefore.isDirectory() || rootBefore.isSymbolicLink()) + throw new Error("Native ACPX distribution must be a directory"); + const heldRoot = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_DIRECTORY); + const privateRoot = await realpath(await mkdtemp(join(tmpdir(), "paperclip-acpx-native-"))); + const packageRoot = join(privateRoot, "distribution"); + const cacheRoot = join(privateRoot, "state"); + const directories = new Set([privateRoot, packageRoot]); + const digests = {}; + const close = async () => { + for (const path of directories) + await chmod(path, 0o700).catch(() => undefined); + await rm(privateRoot, { recursive: true, force: true }); + }; + let commandDirectory; + try { + if (!same(rootBefore, await heldRoot.stat({ bigint: true }))) + throw new Error("Native ACPX distribution root changed before snapshot"); + await mkdir(packageRoot, { mode: 0o700 }); + if (input.isolatedCacheEnvironmentName) + await mkdir(cacheRoot, { mode: 0o700 }); + // Build each private parent once. Level ordering prevents child creation + // from racing its parent; batches bound work and drain before any cleanup. + const parentLevels = new Map(); + for (const entry of entries) { + const parts = entry.path.split("/"); + for (let depth = 1; depth < parts.length; depth++) { + const path = join(packageRoot, ...parts.slice(0, depth)); + const level = parentLevels.get(depth) ?? new Set(); + level.add(path); + parentLevels.set(depth, level); + directories.add(path); + } + } + const directoryBatch = async (paths, operation) => { + for (let start = 0; start < paths.length; start += NATIVE_DIRECTORY_CONCURRENCY) { + const settled = await Promise.allSettled(paths.slice(start, start + NATIVE_DIRECTORY_CONCURRENCY).map(operation)); + const failed = settled.find(result => result.status === "rejected"); + if (failed?.status === "rejected") + throw failed.reason; + } + }; + for (const depth of [...parentLevels.keys()].sort((a, b) => a - b)) { + await directoryBatch([...parentLevels.get(depth)].sort(), path => mkdir(path, { mode: 0o700 })); + } + let failed = false; + let failure; + const recordFailure = (error) => { if (!failed) { + failed = true; + failure = error; + } }; + const copyEntry = async (entry) => { + const path = join(source, ...entry.path.split("/")); + if (await realpath(path) !== path) + throw new Error("Native ACPX closure contains a symbolic link"); + // Bind metadata to the descriptor we will read, without a redundant + // pathname stat. NONBLOCK prevents a raced-in FIFO from blocking open; + // only a bounded regular single-link file may reach the read below. + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let destination; + try { + const before = await file.stat({ bigint: true }); + if (!before.isFile() || before.nlink !== 1n || before.size !== BigInt(entry.size) || Boolean(before.mode & 73n) !== entry.executable) + throw new Error("Native ACPX closure file identity is invalid"); + const bytes = Buffer.alloc(Math.min(entry.size, NATIVE_COPY_CHUNK_BYTES)); + const target = join(packageRoot, ...entry.path.split("/")); + const streaming = entry.size > NATIVE_COPY_CHUNK_BYTES; + const contentHash = streaming ? createHash("sha256") : undefined; + // Partial large files remain inside the unpublished private tree. No + // digests, command lease or handoff exist until every copy is verified. + if (streaming) + destination = await open(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, entry.executable ? 0o500 : 0o400); + let offset = 0; + while (offset < entry.size) { + const bufferOffset = streaming ? 0 : offset; + const read = await file.read(bytes, bufferOffset, Math.min(bytes.length - bufferOffset, entry.size - offset), offset); + if (read.bytesRead === 0) + throw new Error("Native ACPX closure file ended during snapshot"); + if (destination) { + contentHash.update(bytes.subarray(0, read.bytesRead)); + let written = 0; + while (written < read.bytesRead) { + const result = await destination.write(bytes, written, read.bytesRead - written, offset + written); + if (result.bytesWritten === 0) + throw new Error("Native ACPX snapshot write made no progress"); + written += result.bytesWritten; + } + } + offset += read.bytesRead; + } + if (!same(before, await file.stat({ bigint: true })) || !same(before, await lstat(path, { bigint: true })) || await realpath(path) !== path) + throw new Error("Native ACPX closure file changed while read"); + if ((contentHash ? contentHash.digest("hex") : sha256(bytes)) !== entry.sha256) + throw new Error(`Native ACPX closure file digest mismatch: ${entry.path}`); + if (destination) { + const copied = await destination.stat({ bigint: true }); + if (!copied.isFile() || copied.nlink !== 1n || copied.size !== BigInt(entry.size)) + throw new Error("Native ACPX snapshot file identity is invalid"); + } + else + await writeFile(target, bytes, { mode: entry.executable ? 0o500 : 0o400, flag: "wx" }); + } + catch (error) { + recordFailure(error); + throw error; + } + finally { + // A failed close must not skip the other owned descriptor's retirement. + const closed = await Promise.allSettled([file, ...(destination ? [destination] : [])].map(async (handle) => { + try { + await handle.close(); + } + catch (error) { + recordFailure(error); + throw error; + } + })); + const failedClose = closed.find(result => result.status === "rejected"); + if (failedClose?.status === "rejected") + throw failedClose.reason; + } + }; + // Keep bounded capacity occupied when one file is slower than its peers. + // Every task catches its rejection before releasing capacity; once failed, + // no new copy is admitted and all owned descriptors drain before cleanup. + const active = new Set(); + let activeBytes = 0; + for (const entry of entries) { + const bufferBytes = Math.min(entry.size, NATIVE_COPY_CHUNK_BYTES); + while (!failed && (active.size >= NATIVE_COPY_CONCURRENCY + || activeBytes + bufferBytes > NATIVE_COPY_BUFFER_BYTES)) { + await Promise.race(active); + } + if (failed) + break; + activeBytes += bufferBytes; + const copying = copyEntry(entry).catch(recordFailure) + .finally(() => { activeBytes -= bufferBytes; active.delete(copying); }); + active.add(copying); + } + await Promise.all(active); + if (failed) + throw failure; + // Completion order must not change the module guard or manifest. + for (const entry of entries) + digests[join(packageRoot, ...entry.path.split("/"))] = entry.sha256; + if (!same(rootBefore, await heldRoot.stat({ bigint: true })) || !same(rootBefore, await lstat(source, { bigint: true }))) + throw new Error("Native ACPX distribution root changed during snapshot"); + const executable = join(packageRoot, ...input.executable.split("/")); + const args = [...(input.entrypoint === undefined ? [] : ["--require", join(packageRoot, GUARD), join(packageRoot, ...input.entrypoint.split("/"))]), ...input.fixedArguments]; + let copilotEntryDirectory; + if (input.copilotDistributionDirectory !== undefined) { + copilotEntryDirectory = join(packageRoot, COPILOT_ENTRY); + await mkdir(copilotEntryDirectory, { mode: 0o700 }); + directories.add(copilotEntryDirectory); + const shim = Buffer.from([ + `require(${JSON.stringify(join(packageRoot, GUARD))});`, + `import(require("node:url").pathToFileURL(${JSON.stringify(join(packageRoot, input.copilotDistributionDirectory, "index.js"))}).href).catch(error=>{console.error(error);process.exitCode=1;});`, + ].join("\n")); + const shimPath = join(copilotEntryDirectory, "index.js"); + await writeFile(shimPath, shim, { mode: 0o400, flag: "wx" }); + digests[shimPath] = sha256(shim); + } + if (input.entrypoint !== undefined || copilotEntryDirectory !== undefined) { + const guard = Buffer.from(nativeModuleGuard(digests)); + await writeFile(join(packageRoot, GUARD), guard, { mode: 0o400, flag: "wx" }); + digests[join(packageRoot, GUARD)] = sha256(guard); + } + const bootstrap = Buffer.from(nativeBootstrap(executable, digests[executable], args, input.isolatedCacheEnvironmentName ? { name: input.isolatedCacheEnvironmentName, value: cacheRoot } : undefined, copilotEntryDirectory)); + await writeFile(join(packageRoot, BOOTSTRAP), bootstrap, { mode: 0o400, flag: "wx" }); + digests[join(packageRoot, BOOTSTRAP)] = sha256(bootstrap); + // executable is null: commandLease's separately-qualified provider runtime + // slot is unused; this distribution is spawned by the trusted bootstrap. + const manifest = Buffer.from(JSON.stringify({ roots: [packageRoot], executable: null, digests })); + const manifestPath = join(privateRoot, "manifest.json"); + await writeFile(manifestPath, manifest, { mode: 0o400, flag: "wx" }); + await directoryBatch([...directories], path => chmod(path, 0o500)); + commandDirectory = await open(packageRoot, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_DIRECTORY); + return { commandDirectory, bootstrap, snapshot: { roots: [packageRoot], executable: null, digests, handoff: { path: manifestPath, digest: sha256(manifest) }, close } }; + } + catch (error) { + await commandDirectory?.close(); + await close(); + throw error; + } + finally { + await heldRoot.close(); + } +} +function nativeBootstrap(executable, executableDigest, args, cache, copilotEntryDirectory) { + return [ + 'const {spawn}=require("node:child_process");', + 'const fs=require("node:fs");', + // The commandLease's guarded bootstrap owns FDs 5 (guardian), 6 (exit), + // 7/8 (credential fences). Keep them open in the native process too. + 'const guarded=process.env.PAPERCLIP_ACPX_NATIVE_GUARDED==="1";', + 'const env={...process.env}; delete env.PAPERCLIP_ACPX_NATIVE_GUARDED; delete env.PAPERCLIP_ACPX_PRIVATE_SNAPSHOT; delete env.PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE;', + 'env.NODE_DISABLE_COMPILE_CACHE="1"; delete env.NODE_COMPILE_CACHE;', + ...(cache ? [`env[${JSON.stringify(cache.name)}]=${JSON.stringify(cache.value)};`] : []), + ...(copilotEntryDirectory ? [ + // Never permit the native loader to choose a different cached version, + // app, bootstrap mode or module path supplied by a workspace/caller. + 'for(const name of ["COPILOT_CLI_VERSION","COPILOT_CLI_DIST_DIR","COPILOT_CLI_RESOLVED_DIST_DIR","COPILOT_VOICE_SERVER_MODE","COPILOT_SHUTDOWN_FLUSH","NODE_OPTIONS","NODE_PATH"])delete env[name];', + `env.COPILOT_CLI_DIST_DIR=${JSON.stringify(copilotEntryDirectory)};`, + ] : []), + 'if(guarded)for(const fd of [5,6,7,8])fs.fstatSync(fd);', + `const executable=${JSON.stringify(executable)};const fd=fs.openSync(executable,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);`, + `if(require("node:crypto").createHash("sha256").update(fs.readFileSync(fd)).digest("hex")!==${JSON.stringify(executableDigest)})throw new Error("Native ACPX executable changed before spawn");`, + 'const stdio=guarded?[0,1,2,5,6,7,8]:[0,1,2]; const childExecutable=process.platform==="linux"?"/proc/self/fd/"+stdio.length:executable;stdio.push(fd);', + `const child=spawn(childExecutable,${JSON.stringify(args)},{env,cwd:process.cwd(),shell:false,detached:false,stdio});fs.closeSync(fd);`, + 'child.once("error",()=>process.exit(1)); child.once("exit",(code,signal)=>process.exit(signal?1:code??1));', + 'for(const signal of ["SIGTERM","SIGINT","SIGHUP"])process.on(signal,()=>child.kill(signal));', + ].join("\n"); +} +function nativeModuleGuard(digests) { + return [ + 'const {registerHooks,isBuiltin}=require("node:module");', + 'const {readFileSync,realpathSync}=require("node:fs");', + 'const {fileURLToPath}=require("node:url"); const {createHash}=require("node:crypto");', + `const digests=${JSON.stringify(digests)};`, + 'const verify=url=>{if(!url.startsWith("file:"))throw new Error("Native ACPX module must be a qualified file or builtin");const path=fileURLToPath(url);if(realpathSync(path)!==path||!Object.hasOwn(digests,path))throw new Error("Native ACPX module escaped its closed distribution");const bytes=readFileSync(path);if(createHash("sha256").update(bytes).digest("hex")!==digests[path])throw new Error("Native ACPX module digest mismatch");return bytes;};', + 'registerHooks({resolve(specifier,context,next){const result=next(specifier,context);if(!isBuiltin(specifier)&&!result.url.startsWith("node:"))verify(result.url);return result;},load(url,context,next){if(url.startsWith("node:"))return next(url,context);const bytes=verify(url);if(["module","commonjs","json"].includes(context.format))return {format:context.format,source:bytes,shortCircuit:true};return next(url,context);}});', + ].join("\n"); +} +function same(a, b) { + return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs; +} diff --git a/scripts/ci/pi-intel/snapshot-streaming-candidate.ts b/scripts/ci/pi-intel/snapshot-streaming-candidate.ts new file mode 100644 index 0000000000..d937e3ef22 --- /dev/null +++ b/scripts/ci/pi-intel/snapshot-streaming-candidate.ts @@ -0,0 +1,276 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { chmod, lstat, mkdir, mkdtemp, open, realpath, rm, writeFile, type FileHandle } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import type { AcpxPrivateSnapshot } from "./private-snapshot.js"; + +export interface NativeAcpxDistributionEntry { path: string; sha256: string; size: number; executable: boolean } +export interface NativeAcpxDistributionInput { + distributionRoot: string; + manifestPath: string; + /** Trusted profile pin, never taken from the installed manifest itself. */ + expectedClosureSha256: string; + executable: string; + /** When present, execute this JS entry with the distribution's bundled Node. */ + entrypoint?: string; + fixedArguments: readonly string[]; + isolatedCacheEnvironmentName?: "COPILOT_PKG_CACHE_HOME"; + /** Trusted Copilot profile only; resolved inside each immutable spawn lease. */ + copilotDistributionDirectory?: string; +} +export interface NativeAcpxDistributionSnapshot { + snapshot: AcpxPrivateSnapshot; + commandDirectory: FileHandle; + bootstrap: Buffer; +} + +const MAX_NATIVE_TREE_BYTES = 1024 * 1024 * 1024; +const MAX_NATIVE_FILE_BYTES = 384 * 1024 * 1024; +const MAX_NATIVE_MANIFEST_BYTES = 4 * 1024 * 1024; +// Bound active copy workers and their retained buffers. Large files stream +// through one chunk instead of excluding other copies or retaining whole files. +const NATIVE_COPY_CONCURRENCY = 32; +const NATIVE_DIRECTORY_CONCURRENCY = 32; +const NATIVE_COPY_BUFFER_BYTES = 32 * 1024 * 1024; +const NATIVE_COPY_CHUNK_BYTES = 1024 * 1024; +const BOOTSTRAP = ".paperclip-native-entry.cjs"; +const GUARD = ".paperclip-native-module-guard.cjs"; +const COPILOT_ENTRY = ".paperclip-copilot-entry"; +export const NATIVE_ACPX_BOOTSTRAP_NAME = BOOTSTRAP; +const sha256 = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); +const validPath = (value: unknown): value is string => typeof value === "string" && value.length > 0 && value.length <= 4_096 && !isAbsolute(value) && !/[\u0000-\u001f\u007f\\]/.test(value) && value.split("/").every(part => part.length > 0 && part !== "." && part !== ".."); + +/** Parse a closed, canonical execution inventory before opening any file in it. */ +export function parseNativeAcpxDistributionEntries(value: unknown, expectedSha256: string): NativeAcpxDistributionEntry[] { + if (!/^[a-f0-9]{64}$/.test(expectedSha256)) throw new Error("Native ACPX closure pin is invalid"); + const entries = value !== null && typeof value === "object" ? (value as { entries?: unknown }).entries : undefined; + if (!Array.isArray(entries) || entries.length === 0 || entries.length > 30_000) throw new Error("Native ACPX closure inventory is invalid"); + let total = 0; + let previous = ""; + const parsed = entries.map(raw => { + if (raw === null || typeof raw !== "object" || Array.isArray(raw)) throw new Error("Native ACPX closure entry is invalid"); + const entry = raw as Record; + if (Object.keys(entry).some(key => !["path", "sha256", "size", "executable"].includes(key)) || !validPath(entry.path) || entry.path <= previous || entry.path === BOOTSTRAP || entry.path === GUARD || entry.path === "manifest.json" || entry.path.startsWith(".paperclip-")) throw new Error("Native ACPX closure paths must be safe, unique and sorted"); + if (typeof entry.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(entry.sha256) || !Number.isSafeInteger(entry.size) || Number(entry.size) < 0 || Number(entry.size) > MAX_NATIVE_FILE_BYTES || typeof entry.executable !== "boolean") throw new Error("Native ACPX closure file metadata is invalid"); + previous = entry.path; + total += Number(entry.size); + if (total > MAX_NATIVE_TREE_BYTES) throw new Error("Native ACPX closure exceeds its byte bound"); + return { path: entry.path, sha256: entry.sha256, size: Number(entry.size), executable: entry.executable }; + }); + if (sha256(JSON.stringify(parsed)) !== expectedSha256) throw new Error("Native ACPX closure manifest digest mismatch"); + return parsed; +} + +export async function readNativeAcpxDistributionEntries(input: NativeAcpxDistributionInput): Promise { + if (!validPath(input.executable) || (input.entrypoint !== undefined && !validPath(input.entrypoint)) || input.fixedArguments.length > 128 || input.fixedArguments.some(arg => typeof arg !== "string" || arg.length > 16_384 || arg.includes("\0")) || (input.isolatedCacheEnvironmentName !== undefined && input.isolatedCacheEnvironmentName !== "COPILOT_PKG_CACHE_HOME") + || (input.copilotDistributionDirectory !== undefined && (!validPath(input.copilotDistributionDirectory) || input.entrypoint !== undefined))) throw new Error("Native ACPX launch declaration is invalid"); + const file = await open(input.manifestPath, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await file.stat({ bigint: true }); + if (!before.isFile() || before.size < 1n || before.size > BigInt(MAX_NATIVE_MANIFEST_BYTES)) throw new Error("Native ACPX manifest must be a bounded regular file"); + const bytes = await file.readFile(); + if (!same(before, await file.stat({ bigint: true })) || bytes.length !== Number(before.size)) throw new Error("Native ACPX manifest changed while read"); + const entries = parseNativeAcpxDistributionEntries(JSON.parse(bytes.toString("utf8")), input.expectedClosureSha256); + if (!entries.some(entry => entry.path === input.executable && entry.executable && entry.size > 0) || (input.entrypoint !== undefined && !entries.some(entry => entry.path === input.entrypoint && entry.size > 0))) throw new Error("Native ACPX executable or entrypoint is absent from its closure"); + if (input.copilotDistributionDirectory !== undefined && !entries.some(entry => entry.path === `${input.copilotDistributionDirectory}/index.js` && entry.size > 0)) throw new Error("Copilot owned entrypoint is absent from its closure"); + return entries; + } finally { await file.close(); } +} + +/** + * Freeze only manifest-admitted bytes. Native trees deliberately have a separate + * bound; the tighter JavaScript/npm snapshot limits remain unchanged. + */ +export async function createNativeAcpxDistributionSnapshot(input: NativeAcpxDistributionInput, entries: NativeAcpxDistributionEntry[]): Promise { + // Callers cannot substitute entries between manifest validation and copying. + entries = parseNativeAcpxDistributionEntries({ entries }, input.expectedClosureSha256); + if (process.platform !== "linux" && process.platform !== "darwin") throw new Error("Native ACPX snapshots require Linux or macOS"); + const source = await realpath(input.distributionRoot); + const rootBefore = await lstat(source, { bigint: true }); + if (!rootBefore.isDirectory() || rootBefore.isSymbolicLink()) throw new Error("Native ACPX distribution must be a directory"); + const heldRoot = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_DIRECTORY); + const privateRoot = await realpath(await mkdtemp(join(tmpdir(), "paperclip-acpx-native-"))); + const packageRoot = join(privateRoot, "distribution"); + const cacheRoot = join(privateRoot, "state"); + const directories = new Set([privateRoot, packageRoot]); + const digests: Record = {}; + const close = async () => { + for (const path of directories) await chmod(path, 0o700).catch(() => undefined); + await rm(privateRoot, { recursive: true, force: true }); + }; + let commandDirectory: FileHandle | undefined; + try { + if (!same(rootBefore, await heldRoot.stat({ bigint: true }))) throw new Error("Native ACPX distribution root changed before snapshot"); + await mkdir(packageRoot, { mode: 0o700 }); + if (input.isolatedCacheEnvironmentName) await mkdir(cacheRoot, { mode: 0o700 }); + // Build each private parent once. Level ordering prevents child creation + // from racing its parent; batches bound work and drain before any cleanup. + const parentLevels = new Map>(); + for (const entry of entries) { + const parts = entry.path.split("/"); + for (let depth = 1; depth < parts.length; depth++) { + const path = join(packageRoot, ...parts.slice(0, depth)); + const level = parentLevels.get(depth) ?? new Set(); + level.add(path); parentLevels.set(depth, level); directories.add(path); + } + } + const directoryBatch = async (paths: string[], operation: (path: string) => Promise): Promise => { + for (let start = 0; start < paths.length; start += NATIVE_DIRECTORY_CONCURRENCY) { + const settled = await Promise.allSettled(paths.slice(start, start + NATIVE_DIRECTORY_CONCURRENCY).map(operation)); + const failed = settled.find(result => result.status === "rejected"); + if (failed?.status === "rejected") throw failed.reason; + } + }; + for (const depth of [...parentLevels.keys()].sort((a, b) => a - b)) { + await directoryBatch([...parentLevels.get(depth)!].sort(), path => mkdir(path, { mode: 0o700 })); + } + let failed = false; + let failure: unknown; + const recordFailure = (error: unknown) => { if (!failed) { failed = true; failure = error; } }; + const copyEntry = async (entry: NativeAcpxDistributionEntry): Promise => { + const path = join(source, ...entry.path.split("/")); + if (await realpath(path) !== path) throw new Error("Native ACPX closure contains a symbolic link"); + // Bind metadata to the descriptor we will read, without a redundant + // pathname stat. NONBLOCK prevents a raced-in FIFO from blocking open; + // only a bounded regular single-link file may reach the read below. + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + let destination: FileHandle | undefined; + try { + const before = await file.stat({ bigint: true }); + if (!before.isFile() || before.nlink !== 1n || before.size !== BigInt(entry.size) || Boolean(before.mode & 0o111n) !== entry.executable) throw new Error("Native ACPX closure file identity is invalid"); + const bytes = Buffer.alloc(Math.min(entry.size, NATIVE_COPY_CHUNK_BYTES)); + const target = join(packageRoot, ...entry.path.split("/")); + const streaming = entry.size > NATIVE_COPY_CHUNK_BYTES; + const contentHash = streaming ? createHash("sha256") : undefined; + // Partial large files remain inside the unpublished private tree. No + // digests, command lease or handoff exist until every copy is verified. + if (streaming) destination = await open(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, entry.executable ? 0o500 : 0o400); + let offset = 0; + while (offset < entry.size) { + const bufferOffset = streaming ? 0 : offset; + const read = await file.read(bytes, bufferOffset, Math.min(bytes.length - bufferOffset, entry.size - offset), offset); + if (read.bytesRead === 0) throw new Error("Native ACPX closure file ended during snapshot"); + if (destination) { + contentHash!.update(bytes.subarray(0, read.bytesRead)); + let written = 0; + while (written < read.bytesRead) { + const result = await destination.write(bytes, written, read.bytesRead - written, offset + written); + if (result.bytesWritten === 0) throw new Error("Native ACPX snapshot write made no progress"); + written += result.bytesWritten; + } + } + offset += read.bytesRead; + } + if (!same(before, await file.stat({ bigint: true })) || !same(before, await lstat(path, { bigint: true })) || await realpath(path) !== path) throw new Error("Native ACPX closure file changed while read"); + if ((contentHash ? contentHash.digest("hex") : sha256(bytes)) !== entry.sha256) throw new Error(`Native ACPX closure file digest mismatch: ${entry.path}`); + if (destination) { + const copied = await destination.stat({ bigint: true }); + if (!copied.isFile() || copied.nlink !== 1n || copied.size !== BigInt(entry.size)) throw new Error("Native ACPX snapshot file identity is invalid"); + } else await writeFile(target, bytes, { mode: entry.executable ? 0o500 : 0o400, flag: "wx" }); + } catch (error) { recordFailure(error); throw error; } + finally { + // A failed close must not skip the other owned descriptor's retirement. + const closed = await Promise.allSettled([file, ...(destination ? [destination] : [])].map(async handle => { + try { await handle.close(); } catch (error) { recordFailure(error); throw error; } + })); + const failedClose = closed.find(result => result.status === "rejected"); + if (failedClose?.status === "rejected") throw failedClose.reason; + } + }; + // Keep bounded capacity occupied when one file is slower than its peers. + // Every task catches its rejection before releasing capacity; once failed, + // no new copy is admitted and all owned descriptors drain before cleanup. + const active = new Set>(); + let activeBytes = 0; + for (const entry of entries) { + const bufferBytes = Math.min(entry.size, NATIVE_COPY_CHUNK_BYTES); + while (!failed && (active.size >= NATIVE_COPY_CONCURRENCY + || activeBytes + bufferBytes > NATIVE_COPY_BUFFER_BYTES)) { + await Promise.race(active); + } + if (failed) break; + activeBytes += bufferBytes; + const copying = copyEntry(entry).catch(recordFailure) + .finally(() => { activeBytes -= bufferBytes; active.delete(copying); }); + active.add(copying); + } + await Promise.all(active); + if (failed) throw failure; + // Completion order must not change the module guard or manifest. + for (const entry of entries) digests[join(packageRoot, ...entry.path.split("/"))] = entry.sha256; + if (!same(rootBefore, await heldRoot.stat({ bigint: true })) || !same(rootBefore, await lstat(source, { bigint: true }))) throw new Error("Native ACPX distribution root changed during snapshot"); + const executable = join(packageRoot, ...input.executable.split("/")); + const args = [...(input.entrypoint === undefined ? [] : ["--require", join(packageRoot, GUARD), join(packageRoot, ...input.entrypoint.split("/"))]), ...input.fixedArguments]; + let copilotEntryDirectory: string | undefined; + if (input.copilotDistributionDirectory !== undefined) { + copilotEntryDirectory = join(packageRoot, COPILOT_ENTRY); + await mkdir(copilotEntryDirectory, { mode: 0o700 }); directories.add(copilotEntryDirectory); + const shim = Buffer.from([ + `require(${JSON.stringify(join(packageRoot, GUARD))});`, + `import(require("node:url").pathToFileURL(${JSON.stringify(join(packageRoot, input.copilotDistributionDirectory, "index.js"))}).href).catch(error=>{console.error(error);process.exitCode=1;});`, + ].join("\n")); + const shimPath = join(copilotEntryDirectory, "index.js"); + await writeFile(shimPath, shim, { mode: 0o400, flag: "wx" }); digests[shimPath] = sha256(shim); + } + if (input.entrypoint !== undefined || copilotEntryDirectory !== undefined) { + const guard = Buffer.from(nativeModuleGuard(digests)); + await writeFile(join(packageRoot, GUARD), guard, { mode: 0o400, flag: "wx" }); + digests[join(packageRoot, GUARD)] = sha256(guard); + } + const bootstrap = Buffer.from(nativeBootstrap(executable, digests[executable]!, args, input.isolatedCacheEnvironmentName ? { name: input.isolatedCacheEnvironmentName, value: cacheRoot } : undefined, copilotEntryDirectory)); + await writeFile(join(packageRoot, BOOTSTRAP), bootstrap, { mode: 0o400, flag: "wx" }); + digests[join(packageRoot, BOOTSTRAP)] = sha256(bootstrap); + // executable is null: commandLease's separately-qualified provider runtime + // slot is unused; this distribution is spawned by the trusted bootstrap. + const manifest = Buffer.from(JSON.stringify({ roots: [packageRoot], executable: null, digests })); + const manifestPath = join(privateRoot, "manifest.json"); + await writeFile(manifestPath, manifest, { mode: 0o400, flag: "wx" }); + await directoryBatch([...directories], path => chmod(path, 0o500)); + commandDirectory = await open(packageRoot, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_DIRECTORY); + return { commandDirectory, bootstrap, snapshot: { roots: [packageRoot], executable: null, digests, handoff: { path: manifestPath, digest: sha256(manifest) }, close } }; + } catch (error) { + await commandDirectory?.close(); await close(); throw error; + } finally { await heldRoot.close(); } +} + +function nativeBootstrap(executable: string, executableDigest: string, args: string[], cache?: { name: string; value: string }, copilotEntryDirectory?: string): string { + return [ + 'const {spawn}=require("node:child_process");', + 'const fs=require("node:fs");', + // The commandLease's guarded bootstrap owns FDs 5 (guardian), 6 (exit), + // 7/8 (credential fences). Keep them open in the native process too. + 'const guarded=process.env.PAPERCLIP_ACPX_NATIVE_GUARDED==="1";', + 'const env={...process.env}; delete env.PAPERCLIP_ACPX_NATIVE_GUARDED; delete env.PAPERCLIP_ACPX_PRIVATE_SNAPSHOT; delete env.PAPERCLIP_VERIFIED_RUNTIME_EXECUTABLE;', + 'env.NODE_DISABLE_COMPILE_CACHE="1"; delete env.NODE_COMPILE_CACHE;', + ...(cache ? [`env[${JSON.stringify(cache.name)}]=${JSON.stringify(cache.value)};`] : []), + ...(copilotEntryDirectory ? [ + // Never permit the native loader to choose a different cached version, + // app, bootstrap mode or module path supplied by a workspace/caller. + 'for(const name of ["COPILOT_CLI_VERSION","COPILOT_CLI_DIST_DIR","COPILOT_CLI_RESOLVED_DIST_DIR","COPILOT_VOICE_SERVER_MODE","COPILOT_SHUTDOWN_FLUSH","NODE_OPTIONS","NODE_PATH"])delete env[name];', + `env.COPILOT_CLI_DIST_DIR=${JSON.stringify(copilotEntryDirectory)};`, + ] : []), + 'if(guarded)for(const fd of [5,6,7,8])fs.fstatSync(fd);', + `const executable=${JSON.stringify(executable)};const fd=fs.openSync(executable,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);`, + `if(require("node:crypto").createHash("sha256").update(fs.readFileSync(fd)).digest("hex")!==${JSON.stringify(executableDigest)})throw new Error("Native ACPX executable changed before spawn");`, + 'const stdio=guarded?[0,1,2,5,6,7,8]:[0,1,2]; const childExecutable=process.platform==="linux"?"/proc/self/fd/"+stdio.length:executable;stdio.push(fd);', + `const child=spawn(childExecutable,${JSON.stringify(args)},{env,cwd:process.cwd(),shell:false,detached:false,stdio});fs.closeSync(fd);`, + 'child.once("error",()=>process.exit(1)); child.once("exit",(code,signal)=>process.exit(signal?1:code??1));', + 'for(const signal of ["SIGTERM","SIGINT","SIGHUP"])process.on(signal,()=>child.kill(signal));', + ].join("\n"); +} + +function nativeModuleGuard(digests: Record): string { + return [ + 'const {registerHooks,isBuiltin}=require("node:module");', + 'const {readFileSync,realpathSync}=require("node:fs");', + 'const {fileURLToPath}=require("node:url"); const {createHash}=require("node:crypto");', + `const digests=${JSON.stringify(digests)};`, + 'const verify=url=>{if(!url.startsWith("file:"))throw new Error("Native ACPX module must be a qualified file or builtin");const path=fileURLToPath(url);if(realpathSync(path)!==path||!Object.hasOwn(digests,path))throw new Error("Native ACPX module escaped its closed distribution");const bytes=readFileSync(path);if(createHash("sha256").update(bytes).digest("hex")!==digests[path])throw new Error("Native ACPX module digest mismatch");return bytes;};', + 'registerHooks({resolve(specifier,context,next){const result=next(specifier,context);if(!isBuiltin(specifier)&&!result.url.startsWith("node:"))verify(result.url);return result;},load(url,context,next){if(url.startsWith("node:"))return next(url,context);const bytes=verify(url);if(["module","commonjs","json"].includes(context.format))return {format:context.format,source:bytes,shortCircuit:true};return next(url,context);}});', + ].join("\n"); +} + +function same(a: { dev: bigint; ino: bigint; size: bigint; mtimeNs: bigint; ctimeNs: bigint }, b: typeof a): boolean { + return a.dev === b.dev && a.ino === b.ino && a.size === b.size && a.mtimeNs === b.mtimeNs && a.ctimeNs === b.ctimeNs; +} diff --git a/scripts/ci/pi-intel/snapshot-streaming-fixture.mjs b/scripts/ci/pi-intel/snapshot-streaming-fixture.mjs new file mode 100644 index 0000000000..8eb85fe8e6 --- /dev/null +++ b/scripts/ci/pi-intel/snapshot-streaming-fixture.mjs @@ -0,0 +1,56 @@ +// Diagnostic derivative only; original retained pack never changes. +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import {createHash} from 'node:crypto'; +import {join,resolve,dirname} from 'node:path'; +import {pathToFileURL,fileURLToPath} from 'node:url'; +import {validateEnvironment,verifySealedSnapshot} from './snapshot-pool-fixture.mjs'; +const sha=b=>createHash('sha256').update(b).digest('hex'); +export async function run(pack, pin, variant, report) { + validateEnvironment(process.env, '4'); + assert.ok(['baseline','candidate'].includes(variant)); + assert.equal(process.platform, 'darwin'); assert.equal(process.arch, 'x64'); + assert.equal(process.version, 'v24.21.0'); + const tmp = fs.realpathSync(process.env.TMPDIR); + assert.deepEqual(fs.readdirSync(tmp), []); + const tmpIdentity = fs.lstatSync(tmp); + assert.equal(tmpIdentity.mode & 0o777, 0o700); + const modulePath = variant==='baseline' ? join(pack,pin.modulePath) : join(dirname(fileURLToPath(import.meta.url)),'snapshot-streaming-candidate.mjs'); + const moduleSha256=variant==='baseline'?pin.moduleSha256:pin.candidateModuleSha256; + assert.equal(sha(fs.readFileSync(join(pack,pin.modulePath))),pin.moduleSha256); + assert.equal(sha(fs.readFileSync(modulePath)), moduleSha256); + assert.equal(sha(fs.readFileSync(join(pack, pin.closureManifestPath))), pin.closureManifestSha256); + const native = await import(pathToFileURL(modulePath).href); + const input = { distributionRoot: join(pack, pin.distributionPath), manifestPath: join(pack, pin.closureManifestPath), expectedClosureSha256: pin.closureDigest.replace(/^sha256:/, ''), executable: 'node/bin/node', entrypoint: 'pi-entry.cjs', fixedArguments: [] }; + const proof = { schema: 'paperclip.snapshot-streaming-copy/v1', variant, isQualification: false, poolRequestedAtProcessStart: 4, actualThreadCountMeasured: false, moduleSha256, admittedEnvironmentKeys: Object.keys(process.env).sort(), osAddedTextEncoding: process.env.__CF_USER_TEXT_ENCODING ?? null, providerCalls: 0, vendorProcessesStarted: 0, snapshotClosed: false, commandDirectoryClosed: false }; + let snapshot; + try { + const entries = await native.readNativeAcpxDistributionEntries(input); + assert.equal(entries.length, pin.entries); + assert.equal(entries.reduce((sum, e) => sum + e.size, 0), pin.distributionBytes); + const start = process.hrtime.bigint(); + snapshot = await native.createNativeAcpxDistributionSnapshot(input, entries); + proof.snapshotMilliseconds = Number(process.hrtime.bigint() - start) / 1e6; + // Deliberately outside the timed routine; this extra verification is identical in all four children. + proof.sealedOutput = verifySealedSnapshot(snapshot, entries, tmp); + proof.status = 'snapshot_verified_not_executed'; + } finally { + const errors = []; + if (snapshot) { + try { await snapshot.commandDirectory.close(); proof.commandDirectoryClosed = true; } catch (error) { errors.push(error); } + try { await snapshot.snapshot.close(); proof.snapshotClosed = true; } catch (error) { errors.push(error); } + } + try { + const current = fs.lstatSync(tmp); + assert.equal(current.dev, tmpIdentity.dev); assert.equal(current.ino, tmpIdentity.ino); + assert.deepEqual(fs.readdirSync(tmp), []); proof.temporaryRootEmpty = true; + } catch (error) { errors.push(error); } + proof.cleanupErrors = errors.map(error => String(error)); + fs.writeFileSync(report, JSON.stringify(proof, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); + if (errors.length) throw new AggregateError(errors, 'Snapshot cleanup incomplete'); + } +} +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const [pack, pinPath, variant, report] = process.argv.slice(2); + await run(pack, JSON.parse(fs.readFileSync(pinPath)), variant, report); +} diff --git a/scripts/ci/pi-intel/snapshot-streaming-fixture.test.mjs b/scripts/ci/pi-intel/snapshot-streaming-fixture.test.mjs new file mode 100644 index 0000000000..eb451c6f8f --- /dev/null +++ b/scripts/ci/pi-intel/snapshot-streaming-fixture.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import * as fs from 'node:fs'; +import { createHash } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { validateEnvironment, verifySealedSnapshot } from './snapshot-pool-fixture.mjs'; + +const modulePath = new URL('./snapshot-streaming-candidate.mjs',import.meta.url).pathname; +assert.ok(modulePath, 'Explicit retained built-module path is required for this calibration'); +assert.equal(createHash('sha256').update(fs.readFileSync(modulePath)).digest('hex'), '2ddacbff88558fb055a2037f904b3fc829140b138488f71bae9d23700c744a1d'); +const native = await import(pathToFileURL(modulePath).href); +const sha = bytes => createHash('sha256').update(bytes).digest('hex'); + +test('environment rejects ambient credentials, mismatched or unsupported pools', () => { + const good = { HOME: '/home', LANG: 'C', PATH: '/bin', TMPDIR: '/tmp', UV_THREADPOOL_SIZE: '4' }; + validateEnvironment(good, '4'); + assert.throws(() => validateEnvironment({ ...good, OPENROUTER_API_KEY: 'dummy' }, '4')); + assert.throws(() => validateEnvironment(good, '16')); + assert.throws(() => validateEnvironment({ ...good, UV_THREADPOOL_SIZE: '8' }, '8')); + validateEnvironment({ ...good, __CF_USER_TEXT_ENCODING: `0x${process.getuid().toString(16)}:0x0:0x0` }, '4'); + assert.throws(() => validateEnvironment({ ...good, __CF_USER_TEXT_ENCODING: `0x${(process.getuid() + 1).toString(16)}:0x0:0x0` }, '4')); + assert.throws(() => validateEnvironment({ ...good, __CF_USER_TEXT_ENCODING: 'unknown' }, '4')); +}); + +for (const mutation of ['none', 'changed bytes', 'wrong mode', 'extra file', 'extra directory', 'symbolic link', 'hardlink', 'handoff drift']) { + test(`actual compiled streaming snapshot calibration: ${mutation}`, async () => { + const source = fs.mkdtempSync(join(tmpdir(), 'pc-pool-calibration-')); + let value; + try { + const body=Buffer.alloc(2*1024*1024+7,42); + const entries = [{ path: 'entry.cjs', sha256: sha(body), size: body.length, executable: false }, { path: 'node', sha256: sha('node'), size: 4, executable: true }]; + fs.writeFileSync(join(source, 'entry.cjs'), body, { mode: 0o600 }); + fs.writeFileSync(join(source, 'node'), 'node', { mode: 0o700 }); + const manifestPath = join(source, 'closure.json'); + fs.writeFileSync(manifestPath, JSON.stringify({ entries })); + const input = { distributionRoot: source, manifestPath, expectedClosureSha256: sha(JSON.stringify(entries)), executable: 'node', entrypoint: 'entry.cjs', fixedArguments: [] }; + value = await native.createNativeAcpxDistributionSnapshot(input, await native.readNativeAcpxDistributionEntries(input)); + const root = value.snapshot.roots[0], target = join(root, 'entry.cjs'); + if (mutation !== 'none') fs.chmodSync(root, 0o700); + if (mutation === 'changed bytes') { fs.chmodSync(target, 0o600); fs.writeFileSync(target, 'wrong'); fs.chmodSync(target, 0o400); } + if (mutation === 'wrong mode') fs.chmodSync(target, 0o600); + if (mutation === 'extra file') fs.writeFileSync(join(root, 'extra'), 'x', { mode: 0o400 }); + if (mutation === 'extra directory') fs.mkdirSync(join(root, 'extra'), { mode: 0o500 }); + if (mutation === 'symbolic link') { fs.unlinkSync(target); fs.symlinkSync(join(source, 'entry.cjs'), target); } + if (mutation === 'hardlink') fs.linkSync(target, join(source, 'linked')); + if (mutation === 'handoff drift') value.snapshot.handoff.digest = '0'.repeat(64); + if (mutation !== 'none') fs.chmodSync(root, 0o500); + if (mutation === 'none') assert.equal(verifySealedSnapshot(value, entries, fs.realpathSync(tmpdir())).files, 4); + else assert.throws(() => verifySealedSnapshot(value, entries, fs.realpathSync(tmpdir()))); + } finally { + if (value) { + await value.commandDirectory.close(); + // The extra empty directory is an intentionally foreign fixture node; + // make it removable without changing the production close function. + const extra = join(value.snapshot.roots[0], 'extra'); + if (fs.existsSync(extra) && fs.lstatSync(extra).isDirectory()) fs.chmodSync(extra, 0o700); + await value.snapshot.close(); + } + fs.rmSync(source, { recursive: true, force: true }); + } + }); +} diff --git a/scripts/ci/pi-intel/snapshot-streaming-inputs.json b/scripts/ci/pi-intel/snapshot-streaming-inputs.json new file mode 100644 index 0000000000..08e344f84e --- /dev/null +++ b/scripts/ci/pi-intel/snapshot-streaming-inputs.json @@ -0,0 +1,86 @@ +{ + "schema": "paperclip.native-intel-snapshot-streaming-input/v1", + "sourceRevision": "b9e5d6ecdb05ab7244c90976e07c950f8d09b15b", + "resolvedLockSha256": "38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba", + "artifactRunId": "36959948724", + "artifactId": "11208570718", + "artifactWorkflowRevision": "9541eb18789bac35242eeea3e569b77fa90d1818", + "artifactZipBytes": 1076365959, + "artifactZipSha256": "b82cf6c843006781e2e6d5f0a06e0ff39497ffc457b94a511f8bdda66628aa43", + "selectedFiles": { + "receipt.json": { + "bytes": 32763, + "sha256": "47ef1f494c59c1249ac42f08308a2fb83edcb0759890960059cc367ebdf12bf0" + }, + "provider-pack.tar.gz": { + "bytes": 1029214229, + "sha256": "5a550004ca982c49a2d86b3196676f3ce1877b6655ec730d1971b4cec2ba547d" + }, + "provider-pack.json": { + "bytes": 3629, + "sha256": "db5b1d7d825a0533a6506755c6209ccc4d45776c3abbf6c87ff845d1d47e99b2" + }, + "pack-inventory.json": { + "bytes": 10633748, + "sha256": "172f0f4f2319f3716cca656cdeafd03b4f0db652def53101dfbf148850a20eaa" + }, + "paperclip-runnerd": { + "bytes": 32035568, + "sha256": "29d5b0fc2ae2a552b89a107e3777e2c8ee356ebdbb13a14ceab8865e1cea1d45" + }, + "source.tar": { + "bytes": 141742080, + "sha256": "b32d8e32639e6527804fe11bfa1aaefcefa87fca31e9155499e6b9a28f69dbd4" + }, + "source-input-inventory.json": { + "bytes": 1033702, + "sha256": "7c160bb1fc4d049378004d014e398d42dabf12e8a73ec9cdda7679113aaa6949" + }, + "resolved-pnpm-lock.yaml": { + "bytes": 607788, + "sha256": "38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba" + } + }, + "originalSidecarSha256": "7462825f4b82bce8e2a69425eafe4ca8b2c8812df345269c82fb75aa21f3043c", + "originalPackDigest": "sha256:9cf4f88d9fcf63746f7716f773cb8d062544c19e0a78e111e60f659a3596b13f", + "daemonSha256": "29d5b0fc2ae2a552b89a107e3777e2c8ee356ebdbb13a14ceab8865e1cea1d45", + "nodeSha256": "7abcf39bd37ab251015337ff75304d7555f0d8e88c6e0fbf04bce8ce34636f49", + "testPath": "packages/paperclip-runner/test/pi-closed-startup.test.mjs", + "testSha256": "8967cf9c8cd130b68bf9d64abef8cb8d352af00646e2288b341d8c6ae758b47a", + "observerSha256": "1fcf353beb90722f90401a7792a5853bf0807d964a1065e3f55b76a63bc4b4cc", + "profileDigest": "sha256:47306e6d2a9b59e8f9189f725ebb7a0a7f91826044d1739e1a35ab31f228ba1f", + "closureDigest": "sha256:03351f4a250a8db0e79411a9079b43a0ff05f72a2aff41fae17f1fc2de24bd41", + "model": "openrouter/deepseek/deepseek-v4-flash-0731", + "originalFailure": "Original 30s session.open timeout; settled35032ms; runner exit0 and owned cleanup verified;48SDK contracts passed; fresh Rust daemon and complete pack passed.", + "providerCalls": 0, + "timeoutChanges": false, + "archiveBytesMaximum": 268435456, + "retentionDays": 7, + "modulePath": "dist/drivers/acpx/native-distribution-integrity.js", + "moduleSha256": "2c321e2a6b356bdd92e19493ee1d848436bdc895e5aa98c4190a0d8658df15df", + "closureManifestPath": "provider-assets/pi/darwin-x64/native-closure.json", + "closureManifestSha256": "b54c812aa78e072bf9badd0e7e3bf491a948ef5ecef827c40de88be48e2e023b", + "distributionPath": "provider-assets/pi/darwin-x64/runtime", + "maximumCopies": 4, + "experimentDeadlineSeconds": 600, + "perCopyDeadlineSeconds": 90, + "cleanupReserveSeconds": 120, + "finalIntegrityReserveSeconds": 60, + "entries": 15671, + "distributionBytes": 538588650, + "variants": [ + "baseline", + "candidate", + "candidate", + "baseline" + ], + "candidateModuleSha256": "2ddacbff88558fb055a2037f904b3fc829140b138488f71bae9d23700c744a1d", + "candidateSourceSha256": "4fcd8a9b1f27d0dc11ca9ab582ddaf399e28ccb4fca3fefcd48e147d5ba2dc7f", + "baselineSourceSha256": "943ba0fb2d27d08b9b0d36778f08c7c0dbdb78893c7564babd4e67f67cf065fe", + "compilation": { + "tool": "TypeScript7.0.2", + "options": "targetES2023,moduleESNext,noResolve,noCheck,noSourceMap; emit only; separate narrow strict typecheck passed", + "baselineCalibration": "Exact retained baseline emitted bytes except original nonexecuting sourceMappingURL trailer", + "baselineEmittedSha256": "15a05f6e00e8ca58f07c54ee47d8daf4ab5cdeff99788ca8811ad815c6b44bbc" + } +} diff --git a/scripts/ci/pi-intel/snapshot_streaming.py b/scripts/ci/pi-intel/snapshot_streaming.py new file mode 100644 index 0000000000..cb085b0504 --- /dev/null +++ b/scripts/ci/pi-intel/snapshot_streaming.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Four baseline/candidate exact-snapshot ABBA measurements; no provider, daemon or qualification.""" +import argparse,datetime,hashlib,json,os,platform,shutil,signal,stat,subprocess,tarfile,tempfile,time +from pathlib import Path +from source_guard import require,sha +from native_daemon_reuse import extract_selected +from closed_inventory import closed_tree +from startup_diagnostic import restore_pack,finalize_diagnostic +from diagnostic_lifecycle import DiagnosticChild,DiagnosticInspection +from owned_processes import OwnedProcesses,atomic_json +HERE=Path(__file__).resolve().parent +PIN=json.loads((HERE/'snapshot-streaming-inputs.json').read_text()) + +def admit_copy(index,variant,remaining,cancelled,uncertain): + require(not cancelled and not uncertain,'Cancelled or uncertain experiment cannot launch another copy') + require(type(index) is int and 0<=index<4 and variant==["baseline","candidate","candidate","baseline"][index],'Only the exact four-copy ABBA sequence is admitted') + require(remaining>=90+120+60,'Full copy, cleanup and final-integrity windows no longer fit') + +def child_environment(home,tmp): + return {'PATH':'/usr/bin:/bin:/usr/sbin:/sbin','HOME':str(home),'LANG':'en_US.UTF-8','TMPDIR':str(tmp),'UV_THREADPOOL_SIZE':'4'} + +def verify_report(path,variant,pin=PIN): + st=path.lstat();require(stat.S_ISREG(st.st_mode) and st.st_nlink==1 and st.st_size<=16384,'Unsafe copy receipt') + r=json.loads(path.read_text()) + require(variant in ('baseline','candidate') and r['variant']==variant,'Wrong experiment variant') + require(r['status']=='snapshot_verified_not_executed' and r['isQualification'] is False and r['poolRequestedAtProcessStart']==4 and r['actualThreadCountMeasured'] is False,'Copy receipt identity/status mismatch') + expected=pin['moduleSha256'] if variant=='baseline' else pin['candidateModuleSha256'] + require(r['moduleSha256']==expected and r['providerCalls']==r['vendorProcessesStarted']==0,'Copy code/execution scope mismatch') + require(all(r[k] is True for k in ['snapshotClosed','commandDirectoryClosed','temporaryRootEmpty']) and r['cleanupErrors']==[],'Copy cleanup incomplete') + require(r['sealedOutput']=={'files':pin['entries']+2,'bytes':pin['distributionBytes'],'handoffVerified':True,'descriptorIdentityVerified':True},'Incomplete sealed snapshot verification') + require(type(r['snapshotMilliseconds']) in (int,float) and 0600: + proof['status']='comparison_failed_no_retry' + raise RuntimeError('Experiment exceeded its 600-second bound; cleanup completed') + if cancelled: + proof['status']='comparison_failed_no_retry' + raise InterruptedError(cancelled+'; owned cleanup completed') + try:assert_complete(proof) + except BaseException: + proof['status']='comparison_failed_no_retry';raise + finally:save() + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--archive',type=Path,required=True);p.add_argument('--output',type=Path,required=True) + raise SystemExit(execute(p.parse_args())) diff --git a/scripts/ci/pi-intel/test_snapshot_streaming.py b/scripts/ci/pi-intel/test_snapshot_streaming.py new file mode 100644 index 0000000000..e2f9672a27 --- /dev/null +++ b/scripts/ci/pi-intel/test_snapshot_streaming.py @@ -0,0 +1,79 @@ +import copy,json,tempfile,unittest +from pathlib import Path +from unittest.mock import patch +from snapshot_streaming import PIN,admit_copy,child_environment,verify_report,post_integrity,assert_complete,verify_baseline_source +from diagnostic_lifecycle import DiagnosticChild + +class SnapshotStreamingTests(unittest.TestCase): + def test_terminal_success_cannot_survive_failed_or_incomplete_finalization(self): + good={'status':'comparison_complete_not_qualification','cleanupUncertain':False,'scratchRemoved':True,'postRunIntegrity':{'complete':True,'passed':True,'checks':{'pack':True},'errors':{}}} + assert_complete(good) + for delta in [{'status':'diagnostic_failed'},{'cleanupUncertain':True},{'scratchRemoved':False},{'postRunIntegrity':{}},{'postRunIntegrity':{'complete':True,'passed':False,'checks':{'pack':False},'errors':{'pack':'changed'}}},{'postRunIntegrity':{'complete':True,'passed':True,'checks':{},'errors':{}}}]: + with self.subTest(delta=delta),self.assertRaises(RuntimeError):assert_complete(good|delta) + def test_exact_four_sequence_and_full_window(self): + for i,p in enumerate(['baseline','candidate','candidate','baseline']):admit_copy(i,p,270,None,False) + for args in [(4,'baseline',600,None,False),(0,'candidate',600,None,False),(0,'baseline',269.9,None,False),(0,'baseline',600,'cancel',False),(0,'baseline',600,None,True)]: + with self.assertRaises(RuntimeError):admit_copy(*args) + def test_only_explicit_noncredential_process_environment(self): + with patch.dict('os.environ',{'OPENROUTER_API_KEY':'must-not-propagate','UV_THREADPOOL_SIZE':'128'}): + e=child_environment('/private/home','/private/tmp') + self.assertEqual(e,{'HOME':'/private/home','TMPDIR':'/private/tmp','UV_THREADPOOL_SIZE':'4','LANG':'en_US.UTF-8','PATH':'/usr/bin:/bin:/usr/sbin:/sbin'}) + def receipt(self): + return {'variant':'baseline','status':'snapshot_verified_not_executed','isQualification':False,'poolRequestedAtProcessStart':4,'actualThreadCountMeasured':False,'moduleSha256':PIN['moduleSha256'],'providerCalls':0,'vendorProcessesStarted':0,'snapshotClosed':True,'commandDirectoryClosed':True,'temporaryRootEmpty':True,'cleanupErrors':[],'sealedOutput':{'files':PIN['entries']+2,'bytes':PIN['distributionBytes'],'handoffVerified':True,'descriptorIdentityVerified':True},'snapshotMilliseconds':125.0} + def test_exact_receipt(self): + with tempfile.TemporaryDirectory() as d: + path=Path(d)/'receipt.json';r=self.receipt();path.write_text(json.dumps(r));self.assertEqual(verify_report(path,'baseline'),r) + def test_false_or_incomplete_measurements_rejected(self): + deltas=[{'variant':'candidate'},{'poolRequestedAtProcessStart':16},{'moduleSha256':'0'*64},{'providerCalls':1},{'snapshotClosed':False},{'commandDirectoryClosed':False},{'temporaryRootEmpty':False},{'cleanupErrors':['error']},{'snapshotMilliseconds':float('nan')},{'snapshotMilliseconds':float('inf')},{'snapshotMilliseconds':0},{'sealedOutput':{}},{'status':'ok'}] + with tempfile.TemporaryDirectory() as d: + path=Path(d)/'receipt.json' + for delta in deltas: + with self.subTest(delta=delta): + path.write_text(json.dumps(self.receipt()|delta)) + with self.assertRaises(RuntimeError):verify_report(path,'baseline') + def test_post_checks_continue_after_pack_failure(self): + calls=[];pin={'candidateModuleSha256':'expected','candidateSourceSha256':'expected','selectedFiles':{'one':{'sha256':'1'},'two':{'sha256':'2'}}} + def digest(path):calls.append(path.name);return path.name=='one' and '1' or 'wrong' + with patch('snapshot_streaming.closed_tree',side_effect=RuntimeError('pack changed')),patch('snapshot_streaming.sha',side_effect=digest):r=post_integrity(Path('/pack'),[],Path('/original'),pin) + self.assertEqual(calls,['snapshot-streaming-candidate.mjs','snapshot-streaming-candidate.ts','one','two']);self.assertEqual(r['checks'],{'pack':False,'candidateModule':False,'candidateSource':False,'one':True,'two':False});self.assertFalse(r['passed']) + def test_before_launch_cancellation_never_calls_factory(self): + owner=type('Owner',(),{})();child=DiagnosticChild(owner);child.cancel('between copies') + with self.assertRaises(RuntimeError):child.execute(lambda:self.fail('cancelled spawn'),90) + self.assertFalse(child.attempted);self.assertFalse(child.uncertain) + def test_during_publication_cancellation_retires_owned_handle(self): + class Process: + pid=123;returncode=0 + def wait(self,timeout):return 0 + class Owner: + stops=0 + def observe(self,pid):child.cancel('during publication') + def stop(self,pid):self.stops+=1;return {'status':'stopped'} + def live(self):return [] + owner=Owner();child=DiagnosticChild(owner) + with self.assertRaises(RuntimeError):child.execute(Process,90) + self.assertTrue(child.retired);self.assertEqual(owner.stops,1) + with self.assertRaises(RuntimeError):child.execute(lambda:self.fail('second spawn'),90) + +class BaselineSourceTests(unittest.TestCase): + def test_safe_source_member_and_mismatched_hash_link_duplicate_rejected(self): + import tarfile,io,hashlib + name='packages/paperclip-runner/src/drivers/acpx/native-distribution-integrity.ts' + payload=b'baseline source';pin={'baselineSourceSha256':hashlib.sha256(payload).hexdigest()} + with tempfile.TemporaryDirectory() as d: + p=Path(d)/'source.tar' + for kind in ['valid','changed','link','duplicate','missing']: + with tarfile.open(p,'w') as t: + member=tarfile.TarInfo(name);member.size=len(payload) + if kind=='link':member.type=tarfile.SYMTYPE;member.linkname='/foreign';member.size=0 + if kind!='missing':t.addfile(member,io.BytesIO(b'changed source!' if kind=='changed' else payload)) + if kind=='duplicate':t.addfile(member,io.BytesIO(payload)) + if kind=='valid':self.assertEqual(verify_baseline_source(p,pin),payload) + else: + with self.assertRaises(RuntimeError):verify_baseline_source(p,pin) + def test_candidate_report_must_use_candidate_hash(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d)/'receipt.json';r=SnapshotStreamingTests().receipt()|{'variant':'candidate'};p.write_text(json.dumps(r)) + with self.assertRaises(RuntimeError):verify_report(p,'candidate') + r['moduleSha256']=PIN['candidateModuleSha256'];p.write_text(json.dumps(r));self.assertEqual(verify_report(p,'candidate'),r) + +if __name__=='__main__':unittest.main()