diff --git a/Dockerfile b/Dockerfile index c3c2917409b..3736f1f4e82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,9 +156,11 @@ RUN rm -rf packages/paperclip-runner/runner/target FROM build AS runner-provider-pack # Unstamped local builds remain usable, but cannot qualify a remote pack. # Never invent a source revision to make an unqualified pack look verified. -RUN mkdir -p /provider-pack \ +RUN mkdir -p /provider-pack /provider-pack-transport \ && if [ -n "${PAPERCLIP_BUILD_COMMIT}" ]; then \ - PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_BUILD_COMMIT}" node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack; \ + PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_BUILD_COMMIT}" node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \ + && node scripts/create-provider-pack-archive.mjs /provider-pack \ + && mv /provider-pack.archives /provider-pack.archive.json /provider-pack-transport/; \ else \ echo "Skipping remote provider pack: supply a full PAPERCLIP_BUILD_COMMIT to enable remote OpenCode/ACPX execution"; \ fi @@ -189,6 +191,7 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh COPY --chown=node:node --from=build /app /app COPY --from=runner-provider-pack /provider-pack /opt/paperclip-runner/provider-pack +COPY --from=runner-provider-pack /provider-pack-transport/ /opt/paperclip-runner/ # Managed deployments can remap node's UID at startup. This immutable pack # contains public code and integrity metadata, never credentials; it must remain # readable afterward. diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 12d8fadfbdb..c16823cbfed 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1925,6 +1925,24 @@ return a failure. Providers without this capability, explicit harness uploads, and npm installs keep the serial path. See the [native artifact controls](../server/src/services/native-runtime/native-artifact-preparation.test.ts). +Stamped app images also carry `provider-pack.archives/` and +`provider-pack.archive.json` beside their unpacked provider pack. The cloud build +compresses small files into one overlay and each file of at least 1 MiB into its +own zstd layer. The v2 receipt binds file and archive hashes, sizes, and modes to +the expected provider-pack digest. A sandbox can copy byte-identical large files +from an older image into the new private pack directory. It hashes the copies, +rejects symlink sources, and never shares writable hardlinks with the image. +Only missing or mismatched layers and the overlay are uploaded. The controller +checks those archives before upload; the sandbox checks all received hashes +before extraction. Normal full pack admission still runs afterward. + +Historical v1 `provider-pack.tar.zst` receipts remain readable. Missing sidecars, +remote zstd, compatible GNU tar/checksum tools, or the Node 18+ helper retain +directory staging; invalid sidecars fail +closed. Keep transport files outside the pack so they cannot include themselves. +Matching staged/preinstalled packs remain preferred and never transfer archives. +Daytona detects gzip/zstd headers and skips redundant transport compression. + ### Remote GitHub launcher staging Managed GitHub launchers stage their nine text files in one bounded control diff --git a/doc/observability.md b/doc/observability.md index 7666121fc75..4ce7e94d8af 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -408,6 +408,17 @@ above still apply. ## Native Runner Trace Spans +Provider-pack fallback staging separates `provider_pack.archive.verify` +(controller archive identity), `provider_pack.archive.probe` (remote helper support), +`provider_pack.archive.reuse` (copy and hash image files), +`provider_pack.archive.verify_missing` (verify only layers needing upload), +`provider_pack.archive.transfer` (compressed file +transfer), and `provider_pack.archive.extract` (received hash check and extraction). +They are children of the existing pack selection/upload path and also appear in +the local run timing log. Their fixed names add no paths, credentials, or content +attributes. The operator endpoint gate remains unchanged. Source installs without +prebuilt archives retain the existing directory upload span. + The native provider lifecycle emits one foldable OpenTelemetry trace, linked to the server attempt described above. This is the native-run trace schema version `3`. `task.provider_session` is the native diff --git a/doc/plans/2026-10-15-runner-startup-latency.md b/doc/plans/2026-10-15-runner-startup-latency.md new file mode 100644 index 00000000000..d39fd903fbe --- /dev/null +++ b/doc/plans/2026-10-15-runner-startup-latency.md @@ -0,0 +1,236 @@ +# Runner startup latency evidence + +## Master reconciliation — 2026-10-19 + +The user authorized updating PR #16493 to current `origin/master`, passing its +checks, and merging it. The performance changes now stand alone on master +`f3cd3f4ea8`; the unmerged Claude protocol, default, and platform PRs remain +separate. Master's concurrent native-artifact preparation and provider verification +remain intact. The stale stacked-branch workflow assertion is excluded. + +The original branch is preserved as +`archive/claude-startup-latency-pre-master-20261019` at +`ca3833fe8e4ca8f9fcc47e66041cb0f3cbab3b31`. The dated evidence below describes +its tested deployments, not a deployment of the rebased source. Fresh local verification passed 44 archive/provider tests and all token gates. +Review found that custom hosts with incompatible tar or no checksum utility +could enter archive staging and fail. Both archive formats now check the full +extraction tool set before staging. Six isolated-PATH regressions cover BSD tar, +older GNU tar, and missing sha256sum; unsupported hosts retain directory staging. +The first cloud policy job used the old stacked base from its push event and +incorrectly counted master's lockfile changes; a new push after retargeting uses +the current master base. Fresh source +verification is recorded in [PR #16493](https://github.com/paperclipai/paperclip/pull/16493). +Docker and Rust builds remain in cloud CI. No new paid provider runs, deployment, +or fleet image changes are part of this merge request. + +### Startup latency delivery — 2026-10-15 + +Owner: this chat, isolated `codex/claude-startup-latency` worktree based on +`6bb6159cdfe088bb2cc5ee5adad5de829f576c15` (runtime `15e37379e94535fd9692a4f0c4d3fadd2595b188`). +The user requested trace-led diagnosis and a fix on the dedicated test workspace. +This remains within the $50 qualification ceiling; Docker/Rust builds stay remote. +No merge, production rollout, or fleet-wide image change is authorized by this work. + +| Acceptance | Baseline | Required evidence | +| --- | --- | --- | +| Fresh sandbox startup | 133.446s native startup; 122.117s pack upload | Under 30s to prompt admission; timing/trace from exact serving source | +| Follow-up in a new sandbox | 136.285s resume; 125.798s pack upload | Under 30s, same provider session, successful artifact revision | +| Runtime admission | Older managed snapshot; verified pack fallback uploads | Exact pack/daemon identities, full integrity/architecture checks retained | +| Correctness | Two downloaded versions verified | Independently download new outputs and verify revision plus session continuity | +| Regressions | Shared native transport, staged/preinstalled/upload paths | Focused existing tests including corrupt/mismatched pack rejection; applicable CI | +| Deployment | Dedicated test organization only | Recorded startup spans and healthy target, usable preview, reviewable delivery | + +The first experiment makes no paid Claude prompt: separate archive creation, +transfer, and extraction, inspect the snapshot's preinstalled identity, and test +the verified preinstalled path. Existing run-log spans already locate the critical +path. Honeycomb had no matching exported trace for the first run; local native +spans provide before/after measurement without changing the operator endpoint. +Evidence directory: `/private/tmp/claude-startup-20261015/`. + +The owned no-prompt snapshot probe confirmed a preinstalled pack from source +`4a8178e9cf40bf77d1a22a2dc6ddc1a7d5189eae`, not the serving runtime. The pack +occupies 2.0 GiB. A bounded cloud compression experiment over 2,125,271,040 tar +bytes measured gzip 66.409s / 763,211,573 bytes and zstd 17.687s / 711,973,704 +bytes. These isolate compression on the sandbox CPU, not the exact controller's +historical phase breakdown. Both owned probe sandboxes were deleted; no provider +prompts were issued. The first benchmark approval was rejected under the mistaken +assumption that it uploaded the pack. Inspection established that only a 1,640-byte +script was uploaded and all pack/archive bytes stayed in the sandbox; re-review +approved that same command. + +The optional Honeycomb export setup would require refreshing an expired AWS +secret-store login. The user questioned that need; this work proceeds without it. +Existing native startup spans remain the before/after acceptance authority. + +The matching image build passed all 22 CLI/runtime identity checks in cloud run +[38803382803](https://github.com/paperclipai/paperclip-cloud/actions/runs/38803382803). +Candidate `fleet-sandbox-image-1791979001546`, snapshot +`fleet-sbx-040bf1ccbc2a-c19a1bc17412`, uses exact app image +`ghcr.io/paperclipai/paperclip@sha256:733f5cafb09c3d743b86fc2bdecf38077bde7db5cc55eb35ce017c1cd720e7ca`. +It remains unpromoted: current sandbox-image campaigns require fleet-wide default +promotion before targeting a stack. Avoid widening this task's rollout. Instead, +the runtime fix precompresses the app-owned pack at build time and uses its +hash-bound archive for the existing upload fallback. This improves the dedicated +workspace without a global image promotion. Actual deployed latency is pending. + +The actual fallback transport was then measured without model prompts using a +public qualified pack from `557c3064a30779e86e1ae312eab3d1d42c3a8b8d`. +Prebuilt gzip: 0.878s controller verification, 4.184s transfer, 16.340s extraction, +0.482s artifact verification. Prebuilt zstd through the actual Daytona +`performSyncIn` implementation: 0.788s verification, 0.076s capability probe, +6.009s transfer, 7.767s extraction, 0.412s artifact verification. All declared +artifact hashes matched; both owned sandboxes were deleted. Receipts are +`transfer-result.json` and `transfer-zstd-result.json` in the evidence directory. +These are transport probes, not a deployed application startup claim. + +The implementation uses build-time zstd with an explicit remote capability +probe. Missing archives or missing remote zstd retain directory staging; +malformed or corrupt archives fail closed. Source/custom installations and +verified staged/preinstalled pack selection retain their existing behavior. +The compressed sidecar adds approximately 0.98 GB to this measured image's +uncompressed layers; it trades one build/pull cost for every fresh sandbox's +repeated CPU work. Actual target image size will be recorded after the build. +Eleven archive contract tests pass. The focused native selection/transport suite +passes 56 selected tests. Local full server typecheck has fourteen pre-existing +missing/stale dependency errors; broad verification runs in a clean cloud job. + +Performance PR: [16493](https://github.com/paperclipai/paperclip/pull/16493), +stacked on the existing platform PR. Review identified redundant compression in +the shared Daytona file mapper and a missing zstd test-host gate. The mapper now +recognizes gzip/zstd content headers without relying on filenames. Thirty real +file-sync tests and eleven archive tests pass. Broad source verification and the +single-target deployment are running remotely; the first deployment watcher was +stopped before rollout so the reviewed candidate can replace it. No fleet default +or production setting was changed. + + +### First startup optimization deployed; follow-up threshold not met — 2026-10-15 + +Serving source `0215a521fd17ed18ff159f2f13da1e1d98647df2`, image +`ghcr.io/paperclipai/paperclip@sha256:e73177f4cb4158efb97678452041eae4506ad2a1e956eb6a1752fa9bbe5264f9`, +was deployed only to the dedicated QA tenant by +[38811327131](https://github.com/paperclipai/paperclip-cloud/actions/runs/38811327131). +Image build [38807675425](https://github.com/paperclipai/paperclip/actions/runs/38807675425) +ran in the cloud; the retained image artifact is 4,372,269,316 bytes. + +The strict 48-step user journey on [CLA-3](https://claude-runner-default-qa-oct-15.test.paperclip.app/CLA/issues/CLA-3) +produced two successful useful files and preserved the provider session. +Performance acceptance remains **failed**: the follow-up exceeded the unchanged +30-second native-startup target. No paid retries were started. + +| Turn | Run | Native startup | Archive transfer / extract | Task duration | API estimate | +| --- | --- | --- | --- | --- | --- | +| First | `d77d718b-810b-4244-bff5-3f7366199108` | 26.594s | 6.474s / 5.545s | 1m35s | $0.2166 | +| Follow-up | `50d643e2-82a1-44f9-93c2-b416ada77eeb` | 48.388s | 21.469s / 6.242s | 1m55s | $0.1475 | + +These startup spans exclude task preparation (9.155s / 17.504s) and model +response time. They must not be presented as time from click to first response. +The original 133.446s / 136.285s startup baseline remains retained. +Both new runs use provider session `67c358c5-cc9a-4155-a391-3fde8afabe9a`. +Independent browser-download checks confirm five original checklist items remain, +v2 adds Recovery, and downloading v1 again yields identical bytes. +V1 SHA-256: `7f7fe25076e182ee4a0994b8d4870a3bd621d3f3c367a08156dbbce88e8df408` (974 bytes); +v2: `fbdc337a1ff34c34936672f44d9cd29aae50bd4e9d062680c64572eb43e3ddb6` (1,421 bytes). +Evidence: `/private/tmp/claude-startup-user-20261015/`, including the ended journey, +HTML screenshot report, and independent-verification.json. The agent is paused +again. This pair adds $0.3641 estimated API usage; project lifetime usage is about +$0.7192 under its existing $2 cap, within the aggregate $50 qualification ceiling. + +Current main PR CI passes, Greptile is 5/5 with zero unresolved threads, and +Paperclip Review remains queued. The separate broad harness passed typecheck, +build, token gates, and runner typecheck, but its 15-minute full-test limit timed +out and three runner-eval support tests failed. A longer existing verification +run is in progress; no full-harness pass is claimed. Next experiment: quantify +hash-matched files already in the stale image to remove the large network transfer +without relaxing admission or changing fleet-wide sandbox defaults. + + +The second no-prompt experiment verified 2,012,177,122 bytes of identical large +files already present in the stale image (48 files; 3.548s hash inspection). +The v2 transport now builds a small-file overlay and separate large-file layers, +copies matching image files into a private directory, hashes the copies, and +transfers only missing layers. Required frames are concatenated without +recompression into one upload; all transferred hashes and final pack admission +remain enforced. Historical v1 transport remains readable. + +The actual Daytona mapper probe with the same public qualified pack measured: +0.031s receipt/overlay check, 0.065s capability check, 3.507s copy/hash reuse, +0.560s required-layer verification/batching, 1.293s transfer, 2.283s extraction, +and 0.469s declared-artifact verification. Upload fell from 978,057,504 bytes to +289,942,911 bytes. Receipt: `layer-probe-output.json` in the existing evidence +directory. The sandbox was deleted, and no provider prompt was sent. This is +a transport experiment, not acceptance evidence for the deployed app. +Twenty-four archive tests pass, including corruption, source/target symlinks, +private-copy isolation, historical readers, and missing-file transfer. The stale +workflow-security assertion now reflects the platform branch's trusted release +consumer checkout; all twelve focused security assertions pass. No workflow was +changed. The next cloud image will qualify the integrated source on the QA tenant. + +### Layered transport deployed; remaining resume latency — 2026-10-15 + +Serving source `81235a33604ccf9744077924bdef3db75b80e22d` is verified on the +dedicated QA workspace. The immutable image is +`ghcr.io/paperclipai/paperclip@sha256:85dbeb3ba1dc241a5772f416e1f0f9b9964d6f567157a90fbd1ddc97591c090a`. +[Cloud image build](https://github.com/paperclipai/paperclip/actions/runs/38814943155) +retained a 4,544,726,788-byte image artifact. +[Single-workspace deployment](https://github.com/paperclipai/paperclip-cloud/actions/runs/38817414740) +verified that source at 22:33:54 UTC, request +`d47ba355-9f90-4499-92cb-9bd5ce350c93`. No local Docker or Rust build was used. + +The 39-step ordinary-UI journey on +[CLA-4](https://claude-runner-default-qa-oct-15.test.paperclip.app/CLA/issues/CLA-4) +completed both tasks, independently downloaded both files, checked the original +again, and reloaded the saved result. Correctness passed. Performance acceptance +is still **failed** against the unchanged target of both startup spans below 30s. + +| Turn | Run | Native startup | Pack selection | Transfer / extract | Task duration | API estimate | +| --- | --- | --- | --- | --- | --- | --- | +| First | `40da6e01-e114-4244-886d-1dd159f6233f` | 20.017s | 12.461s | 2.066s / 2.762s | 1m12s | $0.2097 | +| Follow-up | `a4de4222-542d-468e-9016-69e883cfa15a` | 37.834s | 23.512s | 10.500s / 3.202s | 1m43s | $0.1582 | + +Relative to the original pair, native startup fell by 85.0% and 72.2%. These are +startup spans, not click-to-response times. Task preparation was 8.218s / 19.035s; +provider first-event latency after startup was 1.849s / 2.624s. The follow-up +created a replacement sandbox, restored its checkpoint, and retained session +`fc794198-95b6-4666-b228-a30cbd6cb5e6`. Both runs recorded `keep=false` cleanup. + +Independent checks: v1 is 801 bytes, SHA-256 +`1013d26f2fdb441d7c60ec500fa95c739ee94fe0f801eed59f96df67491ca973`; +v2 is 1,082 bytes, SHA-256 +`b64625202dbe68898a3d538fa640561643ae4bdde342477b29a02aa5d093001b`. +All five original checklist items remain verbatim, v2 adds two Recovery items, +and the second download of v1 is byte-identical. The agent is paused and timer +heartbeats remain disabled. This pair adds $0.3679 estimated API usage, for about +$1.0871 on the existing $2 project cap. No unknown-charge hold was released and +no automatic paid retry was started. The remaining balance is below the existing +$1 run reservation; another run must not be started without budget reconciliation. + +Evidence is retained at `/private/tmp/claude-startup-user-20261015-layers/` and +attached to the task: + +- [Screenshot report](https://claude-runner-default-qa-oct-15.test.paperclip.app/api/attachments/25f950a0-257e-4a66-97d0-f30bb6094983/content?download=1) +- [Journey JSON](https://claude-runner-default-qa-oct-15.test.paperclip.app/api/attachments/20e42fc1-0466-4e5c-9c6a-91bc5e3c3f4b/content?download=1) +- [Independent checks](https://claude-runner-default-qa-oct-15.test.paperclip.app/api/attachments/fa3a1f36-445a-4205-a3de-bb144d167ab9/content?download=1) + +The rendered report and gallery navigation were visually checked. It records +the failed performance target, including the earlier recorder binding error and +the actual UI friction; the ended journal has not been rewritten as a pass. + +The residual delay remains predominantly provider-pack preparation. A bounded +no-prompt content-reuse audit found another 138,023,392 matching bytes at different +paths, but scanning the old image took 8.985s and hashed 2.59 GB. That is not a +credible speed improvement, so no speculative content scan was added. Its owned +sandbox was deleted (`content-reuse-probe-output.json`). The matching image +already passed 22 runtime checks, but Cloud's supported sandbox-image campaign +requires it to be the fleet default first. Applying it only to this QA workspace +requires a scoped image-canary capability; neither a fleet-wide promotion nor a +managed-config bypass was performed. That image rollout is the next path to +removing transfer entirely, and it still needs fresh task/follow-up acceptance. + +[Main CI](https://github.com/paperclipai/paperclip/actions/runs/38814748461) +passed all 56 checks on the deployed source, including typecheck, sharded tests, +build, and token gates. Greptile reviewed that source at 5/5 with no unresolved +threads. Paperclip Review's separate task is blocked by its chat connection; +it is not counted as a passed review. The additional unsharded verification jobs +remain separately tracked; no full unsharded-harness pass is claimed. No AWS +login or Honeycomb configuration change was needed for this diagnosis. diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts index ff1c419f55e..81ed11847a5 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.test.ts @@ -1157,3 +1157,30 @@ describe.skipIf(!gnuTar)("automatic unsafe workspace export recovery", () => { expect(await fs.readdir(output)).toEqual([]); }); }); + + +describe("precompressed file transport", () => { + it.each(["gzip", "zstd"])("uploads %s bytes without trying to recompress them", async codec => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-compressed-transfer-")); + try { + const raw = crypto.randomBytes(ZSTD_MIN_SOURCE_BYTES_FOR_TEST + 1024); + const bytes = codec === "gzip" ? zlib.gzipSync(raw) : zlib.zstdCompressSync(raw); + expect(bytes.length).toBeGreaterThan(ZSTD_MIN_SOURCE_BYTES_FOR_TEST); + const source = path.join(root, "artifact-without-extension"); + await fs.writeFile(source, bytes); + const uploadedSources: string[] = []; + const uploadedDestinations: string[] = []; + const commands: RecordedCommand[] = []; + const sandbox = createRecordingSandbox({ probeReportsZstd: true, uploadedSources, uploadedDestinations, commands }); + const compressor = vi.spyOn(zlib, "createZstdCompress"); + await performSyncIn({ sandbox: sandbox as never, remoteDir: "/workspace", timeoutSeconds: 30, + operations: [{ operationId: "archive", files: [{ kind: "file", sourcePath: source, targetPath: "/workspace/archive", mode: 0o600 }] }] }); + expect(compressor).not.toHaveBeenCalled(); + expect(uploadedSources).toEqual([source]); + expect(await sha256OfFile(source)).toBe(crypto.createHash("sha256").update(bytes).digest("hex")); + } finally { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts index 89847f9ddbd..c155bf1b86d 100644 --- a/packages/plugins/sandbox-providers/daytona/src/file-sync.ts +++ b/packages/plugins/sandbox-providers/daytona/src/file-sync.ts @@ -479,6 +479,18 @@ function isZstdCompressionSupported(): boolean { return typeof zlib.createZstdCompress === "function"; } +async function hasCompressedHeader(sourcePath: string): Promise { + const file = await fs.open(sourcePath, "r"); + try { + const header = Buffer.alloc(4); + const { bytesRead } = await file.read(header, 0, header.length, 0); + return (bytesRead === 4 && header.readUInt32LE(0) === 0xfd2fb528) + || (bytesRead >= 3 && header[0] === 0x1f && header[1] === 0x8b && header[2] === 8); + } finally { + await file.close(); + } +} + /** * Stream-compress `sourcePath` to a new file with zstd at * {@link ZSTD_COMPRESSION_LEVEL}, never buffering the whole file in memory. @@ -781,6 +793,7 @@ async function syncInFileMappings(input: { for (const plan of compressCandidates) { let hostTempDir: string | null = null; try { + if (await hasCompressedHeader(plan.mapping.sourcePath)) continue; const compressed = await compressFileToHostTemp(plan.mapping.sourcePath); hostTempDir = compressed.dir; compressBytesIn += plan.sourceSize; diff --git a/scripts/create-provider-pack-archive.mjs b/scripts/create-provider-pack-archive.mjs new file mode 100644 index 00000000000..b599ea169ef --- /dev/null +++ b/scripts/create-provider-pack-archive.mjs @@ -0,0 +1,96 @@ +import { createHash } from 'node:crypto'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { lstat, mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { pipeline } from 'node:stream/promises'; +import { constants, createZstdCompress } from 'node:zlib'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export async function createProviderPackArchive(root) { + const manifest = JSON.parse(await readFile(resolve(root, 'provider-pack.json'), 'utf8')); + if (manifest.schema !== 'paperclip-runner/remote-provider-pack/v1' + || !/^sha256:[a-f0-9]{64}$/.test(manifest.digest ?? '')) throw Error('Invalid provider pack identity'); + const archive = `${root}.tar.zst`; + const tar = spawn('tar', ['--no-xattrs', '-cf', '-', '-C', root, '.'], { + env: { ...process.env, COPYFILE_DISABLE: '1' }, stdio: ['ignore', 'pipe', 'inherit'], + }); + const completed = new Promise((resolve, reject) => { + tar.on('error', reject); + tar.on('close', code => code === 0 ? resolve() : reject(Error('Provider pack archive creation failed'))); + }); + try { + await Promise.all([completed, pipeline(tar.stdout, + createZstdCompress({ params: { [constants.ZSTD_c_compressionLevel]: 3 } }), createWriteStream(archive))]); + } catch (error) { + tar.kill(); + throw error; + } + const hash = createHash('sha256'); + for await (const chunk of createReadStream(archive)) hash.update(chunk); + const receipt = { + schema: 'paperclip-runner/provider-pack-archive/v1', + encoding: 'tar+zstd', + providerPackDigest: manifest.digest, + sha256: hash.digest('hex'), + bytes: (await stat(archive)).size, + }; + await writeFile(`${root}.archive.json`, JSON.stringify(receipt) + '\n'); + return receipt; +} + +// Keep large immutable files separate so a fresh sandbox can reuse identical +// image bytes without trusting that image's older provider-pack identity. +export async function createProviderPackLayers(root) { + const manifest = JSON.parse(await readFile(resolve(root, 'provider-pack.json'), 'utf8')); + if (manifest.schema !== 'paperclip-runner/remote-provider-pack/v1' + || !/^sha256:[a-f0-9]{64}$/.test(manifest.digest ?? '')) throw Error('Invalid provider pack identity'); + const directory = `${root}.archives`; + await mkdir(directory, { recursive: true }); + const small = [], files = []; + const visit = async (relative = '') => { + for (const name of (await readdir(resolve(root, relative))).sort()) { + const path = relative ? `${relative}/${name}` : name; + const metadata = await lstat(resolve(root, path)); + if (metadata.isDirectory()) { small.push(path); await visit(path); } + else if (metadata.isFile() && metadata.size >= 1024 * 1024) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(resolve(root, path))) hash.update(chunk); + files.push({ path, bytes: metadata.size, mode: metadata.mode & 0o777, sha256: hash.digest('hex') }); + } else if (metadata.isFile() || metadata.isSymbolicLink()) small.push(path); + else throw Error('Unsupported provider pack entry'); + } + }; + await visit(); + const archive = async (name, paths) => { + const target = resolve(directory, name); + // Explicit NUL-separated names and no recursion avoid options, duplicate + // traversal, and hardlinks from an archive to files in a different layer. + const tar = spawn('tar', ['--no-xattrs', '--no-recursion', '-cf', '-', '-C', root, + '--null', '-T', '-'], { env: { ...process.env, COPYFILE_DISABLE: '1' }, stdio: ['pipe', 'pipe', 'inherit'] }); + const completed = new Promise((resolve, reject) => { + tar.on('error', reject); + tar.on('close', code => code === 0 ? resolve() : reject(Error('Provider pack layer creation failed'))); + }); + tar.stdin.on('error', () => {}); // The child/pipeline supplies the causal failure. + tar.stdin.end(paths.map(path => `./${path}\0`).join('')); + try { + await Promise.all([completed, pipeline(tar.stdout, + createZstdCompress({ params: { [constants.ZSTD_c_compressionLevel]: 3 } }), createWriteStream(target))]); + } catch (error) { tar.kill(); throw error; } + const hash = createHash('sha256'); + for await (const chunk of createReadStream(target)) hash.update(chunk); + return { sha256: hash.digest('hex'), bytes: (await stat(target)).size }; + }; + const overlay = await archive('overlay.tar.zst', small); + for (const [index, file] of files.entries()) file.archive = await archive(`${index}.tar.zst`, [file.path]); + const receipt = { schema: 'paperclip-runner/provider-pack-archive/v2', encoding: 'tar+zstd', + providerPackDigest: manifest.digest, overlay, files }; + await writeFile(`${root}.archive.json`, JSON.stringify(receipt) + '\n'); + return receipt; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + if (!process.argv[2] || process.argv[3]) throw Error('Usage: create-provider-pack-archive.mjs PACK_ROOT'); + await createProviderPackLayers(resolve(process.argv[2])); +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 0684682854e..254c8bc284c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -34,6 +34,7 @@ import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { hasAcknowledgedNativeReassignmentStopIntent, hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js"; import { stoppedCodexTurnIsTextOnly } from "./stopped-codex-turn.js"; import { prepareVerifiedRemoteProviderPack, runRemoteProviderVerification } from "./remote-provider-pack.js"; +import { readProviderPackArchive, stageProviderPackArchive } from "./provider-pack-archive.js"; import { selectRemotePiCompanion } from "./remote-pi-companion.js"; import { readNativeLocalProcessStop, PROCESS_START_REQUESTED } from "../native-local-process-stop.js"; import { remoteLeaseCleanupScope } from "../remote-execution-termination.js"; @@ -11783,6 +11784,7 @@ async function createRunnerdBackendWithinSessionClaim( } : sourceRuntimeContext; let remotePrepared = false; + let preinstalledProviderPackRoots: string[] = []; let remoteHarnessStatePrepared = false; let selectedRemoteMode: "dial_wss" | "listen_ws" | null = null; let remoteCaBundleMapping: { sourcePath: string; targetPath: string } | null = @@ -11921,6 +11923,7 @@ async function createRunnerdBackendWithinSessionClaim( ), usePreinstalled: async () => { let preinstalledProviderPack = await measureNativeRunnerSpan(input.trace, "provider_pack.discover", discoverPreinstalledProviderPack); + preinstalledProviderPackRoots = preinstalledProviderPack ? [preinstalledProviderPack] : []; if (preinstalledProviderPack) { try { await measureNativeRunnerSpan( @@ -11974,6 +11977,9 @@ async function createRunnerdBackendWithinSessionClaim( "runner_remote_provider_artifact_incompatible: this remote transport cannot stage a provider pack; preinstall the exact manifest-matched pack", ); } + const archive = await measureNativeRunnerSpan(input.trace, "provider_pack.archive.verify", () => + readProviderPackArchive(configuredProviderPackRoot, expectedProviderPackManifest.digest), + ); const escapedPackRoot = stagedRemoteProviderPackRoot.replaceAll( "'", "'\\''", @@ -11990,13 +11996,22 @@ async function createRunnerdBackendWithinSessionClaim( "runner_remote_provider_artifact_incompatible: stale provider pack could not be replaced", ); } - await measureNativeRunnerSpan(input.trace, "provider_pack.upload", () => stageRemoteRunnerDirectory({ - target: remoteTarget, - runner: remoteCommandRunner, - sourcePath: configuredProviderPackRoot, - targetPath: stagedRemoteProviderPackRoot, - mode: 0o700, - })); + await measureNativeRunnerSpan(input.trace, "provider_pack.upload", async () => { + if (archive && await stageProviderPackArchive({ + archive, + runner: remoteCommandRunner, + targetPath: stagedRemoteProviderPackRoot, + reuseRoots: preinstalledProviderPackRoots, + trace: input.trace, + })) return; + await stageRemoteRunnerDirectory({ + target: remoteTarget, + runner: remoteCommandRunner, + sourcePath: configuredProviderPackRoot, + targetPath: stagedRemoteProviderPackRoot, + mode: 0o700, + }); + }); await measureNativeRunnerSpan(input.trace, "provider_pack.verify_uploaded", () => verifyRemoteProviderPack(stagedRemoteProviderPackRoot), ); diff --git a/server/src/services/native-runtime/provider-pack-archive.test.ts b/server/src/services/native-runtime/provider-pack-archive.test.ts new file mode 100644 index 00000000000..8e150542c44 --- /dev/null +++ b/server/src/services/native-runtime/provider-pack-archive.test.ts @@ -0,0 +1,241 @@ +import { execFile, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it, vi } from "vitest"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { readProviderPackArchive, stageProviderPackArchive } from "./provider-pack-archive.js"; +// @ts-expect-error Build-time script has no TypeScript declaration. +import { createProviderPackArchive, createProviderPackLayers } from "../../../../scripts/create-provider-pack-archive.mjs"; + +const exec = promisify(execFile); +const hasZstd = spawnSync("zstd", ["--version"]).status === 0; +const roots: string[] = []; +const digest = `sha256:${"a".repeat(64)}`; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); }); + +async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "provider-archive-")); + roots.push(root); + const pack = path.join(root, "pack"); + await fs.mkdir(path.join(pack, "bin"), { recursive: true }); + await fs.writeFile(path.join(pack, "provider-pack.json"), JSON.stringify({ schema: "paperclip-runner/remote-provider-pack/v1", digest })); + await fs.writeFile(path.join(pack, "bin", "tool"), "#!/bin/sh\necho fixture\n", { mode: 0o755 }); + await fs.symlink("bin/tool", path.join(pack, "tool")); + await createProviderPackArchive(pack); + const archive = (await readProviderPackArchive(pack, digest))!; + if ("kind" in archive) throw Error("Expected historical single archive"); + return { root, pack, archive }; +} + +const tar = await Promise.all(["gtar", "tar"].map(async name => { + try { const { stdout } = await exec(name, ["--version"]); return stdout.includes("GNU tar") ? name : null; } + catch { return null; } +})).then(names => names.find(Boolean)); + +async function runner(root: string, corrupt = false, toolPath?: string) { + const bin = path.join(root, "commands"); + await fs.mkdir(bin); + const resolved = (await exec("sh", ["-c", 'command -v "$1"', "sh", tar!])).stdout.trim(); + await fs.symlink(resolved, path.join(bin, "tar")); + const syncIn = vi.fn>(async operations => { + for (const operation of operations) for (const file of operation.files) { + expect(file.kind).toBe("file"); + await fs.mkdir(path.dirname(file.targetPath), { recursive: true }); + await fs.copyFile(file.sourcePath, file.targetPath); + if (corrupt) await fs.appendFile(file.targetPath, "corrupted in transit"); + } + return { operations: await Promise.all(operations.map(async operation => ({ operationId: operation.operationId, filesTransferred: operation.files.length, + bytesTransferred: (await Promise.all(operation.files.map(async file => (await fs.stat(file.sourcePath)).size))).reduce((sum, size) => sum + size, 0) }))) }; + }); + const execute = vi.fn(async input => { + try { + const result = await exec(input.command, input.args ?? [], { env: { ...process.env, PATH: toolPath ?? `${bin}:${process.env.PATH}` } }); + return { exitCode: 0, signal: null, pid: null, startedAt: null, timedOut: false, stdout: result.stdout, stderr: result.stderr }; + } catch (error) { + return { exitCode: 1, signal: null, pid: null, startedAt: null, timedOut: false, stdout: "", stderr: String(error) }; + } + }); + return { syncIn, execute }; +} + +it("keeps historical/source packs without an archive on the existing path", async () => { + expect(await readProviderPackArchive("/does-not-exist/provider-pack", digest)).toBeNull(); +}); + +async function layeredFixture() { + const { root, pack } = await fixture(); + await fs.writeFile(path.join(pack, "bin", "large"), Buffer.alloc(1024 * 1024, 42), { mode: 0o755 }); + const receipt = await createProviderPackLayers(pack); + const archive = (await readProviderPackArchive(pack, digest))!; + if (!("kind" in archive)) throw Error("Expected layered archive"); + return { root, pack, receipt, archive }; +} + +it.skipIf(!tar || !hasZstd)("reuses only byte-matched image files, keeps modes, and never shares writable hardlinks", async () => { + const { root, pack, archive } = await layeredFixture(); + const remote = await runner(root); + const targetPath = path.join(root, "fresh"); + expect(await stageProviderPackArchive({ archive, runner: remote, targetPath, reuseRoots: [pack] })).toBe(true); + expect(remote.syncIn.mock.calls[0]![0][0]!.files).toHaveLength(1); + expect((await remote.syncIn.mock.results[0]!.value).operations[0]!.bytesTransferred).toBe(archive.overlay.bytes); + const original = path.join(pack, "bin/large"), copied = path.join(targetPath, "bin/large"); + expect(await fs.readFile(copied)).toEqual(await fs.readFile(original)); + expect((await fs.stat(copied)).mode & 0o777).toBe(0o755); + expect((await fs.stat(copied)).ino).not.toBe((await fs.stat(original)).ino); + await fs.writeFile(copied, "changed"); + expect((await fs.stat(original)).size).toBe(1024 * 1024); + expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool"); +}); + +it.skipIf(!tar || !hasZstd).each(["missing", "wrong-digest", "symlink", "ancestor-symlink"])("transfers a large layer when the image file is %s", async reason => { + const { root, pack, archive } = await layeredFixture(); + const remote = await runner(root); + const old = path.join(root, "old"); + await fs.cp(pack, old, { recursive: true }); + const large = path.join(old, "bin/large"); + if (reason === "missing") await fs.unlink(large); + if (reason === "wrong-digest") await fs.writeFile(large, Buffer.alloc(1024 * 1024, 43)); + if (reason === "symlink") { await fs.unlink(large); await fs.symlink(path.join(pack, "bin/large"), large); } + if (reason === "ancestor-symlink") { await fs.rm(path.join(old, "bin"), { recursive: true }); await fs.symlink(path.join(pack, "bin"), path.join(old, "bin")); } + const targetPath = path.join(root, "fresh"); + expect(await stageProviderPackArchive({ archive, runner: remote, targetPath, reuseRoots: [old] })).toBe(true); + expect((await remote.syncIn.mock.results[0]!.value).operations[0]!.bytesTransferred).toBe(archive.overlay.bytes + archive.files[0]!.archive.bytes); + expect(await fs.readFile(path.join(targetPath, "bin/large"))).toEqual(await fs.readFile(path.join(pack, "bin/large"))); +}); + +it.each(["traversal", "duplicate", "ancestor", "mode", "digest", "layer-symlink"])("rejects invalid layer metadata or bytes: %s", async reason => { + const { pack, receipt } = await layeredFixture(); + if (reason === "traversal") receipt.files[0].path = "../outside"; + if (reason === "duplicate") receipt.files.push(receipt.files[0]); + if (reason === "ancestor") receipt.files.push({ ...receipt.files[0], path: "bin" }); + if (reason === "mode") receipt.files[0].mode = 0o4755; + if (reason === "digest") receipt.overlay.sha256 = "0".repeat(64); + if (reason === "layer-symlink") { + const overlay = `${pack}.archives/overlay.tar.zst`; + await fs.rename(overlay, `${overlay}.original`); await fs.symlink(`${overlay}.original`, overlay); + } + await fs.writeFile(`${pack}.archive.json`, JSON.stringify(receipt)); + await expect(readProviderPackArchive(pack, digest)).rejects.toThrow(); +}); + +it.skipIf(!tar || !hasZstd)("rejects a corrupt missing layer before transfer and a corrupt upload before extraction", async () => { + const { root, pack, archive } = await layeredFixture(); + const remote = await runner(root, true); + const targetPath = path.join(root, "fresh"); + await expect(stageProviderPackArchive({ archive, runner: remote, targetPath })).rejects.toThrow("runner_remote_provider_archive_extract_failed"); + expect(await fs.readdir(targetPath)).toEqual([]); + expect((await fs.readdir(root)).some(name => name.includes(".upload-"))).toBe(false); + await fs.appendFile(`${pack}.archives/0.tar.zst`, "corrupt"); + remote.syncIn.mockClear(); + await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_invalid"); + expect(remote.syncIn).not.toHaveBeenCalled(); +}); + +it.skipIf(!tar || !hasZstd)("does not follow a layered target symlink or accept invalid reuse receipts", async () => { + const { root, archive } = await layeredFixture(); + const remote = await runner(root); + const targetPath = path.join(root, "target"), other = path.join(root, "other"); + await fs.mkdir(other); await fs.symlink(other, targetPath); + await expect(stageProviderPackArchive({ archive, runner: remote, targetPath })).rejects.toThrow("runner_remote_provider_archive_reuse_failed"); + expect(await fs.readdir(other)).toEqual([]); + remote.execute.mockResolvedValue({ exitCode: 0, timedOut: false, stdout: "[999999]", stderr: "", signal: null, pid: null, startedAt: null }); + await expect(stageProviderPackArchive({ archive, runner: remote, targetPath: `${targetPath}-2` })).rejects.toThrow("runner_remote_provider_archive_reuse_failed"); + expect(remote.syncIn).not.toHaveBeenCalled(); +}); + +it.each(["identity", "bytes", "sha256", "json", "symlink"])("rejects invalid archive %s", async failure => { + const { pack } = await fixture(); + const receiptPath = `${pack}.archive.json`; + const receipt = JSON.parse(await fs.readFile(receiptPath, "utf8")); + if (failure === "identity") receipt.providerPackDigest = `sha256:${"b".repeat(64)}`; + if (failure === "bytes") receipt.bytes++; + if (failure === "sha256") receipt.sha256 = "c".repeat(64); + await fs.writeFile(receiptPath, failure === "json" ? "{" : JSON.stringify(receipt)); + if (failure === "symlink") { + await fs.rename(`${pack}.tar.zst`, `${pack}.original`); + await fs.symlink(`${pack}.original`, `${pack}.tar.zst`); + } + await expect(readProviderPackArchive(pack, digest)).rejects.toThrow(); +}); + +it.skipIf(!tar || !hasZstd)("uploads build-time bytes to fresh targets, preserving files, modes and symlinks", async () => { + const { root, archive } = await fixture(); + const remote = await runner(root); + for (const name of ["first ' ; #", "follow-up"]) { + const targetPath = path.join(root, name); + await stageProviderPackArchive({ archive, runner: remote, targetPath }); + expect(await fs.readFile(path.join(targetPath, "bin/tool"), "utf8")).toBe("#!/bin/sh\necho fixture\n"); + expect((await fs.stat(path.join(targetPath, "bin/tool"))).mode & 0o777).toBe(0o755); + expect((await fs.stat(targetPath)).mode & 0o777).toBe(0o700); + expect(await fs.readlink(path.join(targetPath, "tool"))).toBe("bin/tool"); + } + expect((await fs.readdir(root)).some(name => name.includes(".upload-"))).toBe(false); + expect(remote.syncIn).toHaveBeenCalledTimes(2); +}); + +it.skipIf(!tar || !hasZstd)("rejects corrupt transferred bytes before extracting and cleans scratch", async () => { + const { root, archive } = await fixture(); + const targetPath = path.join(root, "target"); + await expect(stageProviderPackArchive({ archive, runner: await runner(root, true), targetPath })) + .rejects.toThrow("runner_remote_provider_archive_extract_failed"); + expect(await fs.readdir(root)).not.toContain("target"); + expect((await fs.readdir(root)).some(name => name.includes(".upload-"))).toBe(false); +}); + +it.skipIf(!tar || !hasZstd)("does not follow a remote target symlink", async () => { + const { root, archive } = await fixture(); + const targetPath = path.join(root, "target"); + const other = path.join(root, "other"); + await fs.mkdir(other); await fs.symlink(other, targetPath); + await expect(stageProviderPackArchive({ archive, runner: await runner(root), targetPath })) + .rejects.toThrow("runner_remote_provider_archive_extract_failed"); + expect(await fs.readdir(other)).toEqual([]); +}); + +it.skipIf(!tar || !hasZstd)("rejects malformed tar even with a matching transport digest", async () => { + const { root, archive } = await fixture(); + await fs.writeFile(archive.sourcePath, "invalid tar"); + archive.sha256 = createHash("sha256").update("invalid tar").digest("hex"); + await expect(stageProviderPackArchive({ archive, runner: await runner(root), targetPath: path.join(root, "target") })) + .rejects.toThrow("runner_remote_provider_archive_extract_failed"); + expect((await fs.readdir(root)).some(name => name.includes(".upload-"))).toBe(false); +}); + +it("keeps the directory fallback when the remote has no zstd", async () => { + const { archive } = await fixture(); + const remote = { syncIn: vi.fn(), execute: vi.fn().mockResolvedValue({ exitCode: 127, timedOut: false }) }; + expect(await stageProviderPackArchive({ archive, runner: remote, targetPath: "/unused" })).toBe(false); + expect(remote.syncIn).not.toHaveBeenCalled(); +}); + +it.skipIf(!tar || !hasZstd).each([ + ["v1", "bsd-tar"], ["v2", "bsd-tar"], + ["v1", "older-gnu-tar"], ["v2", "older-gnu-tar"], + ["v1", "missing-sha256sum"], ["v2", "missing-sha256sum"], +])("keeps %s on directory staging with %s before creating a target or uploading", async (version, unsupported) => { + const { root, archive } = version === "v1" ? await fixture() : await layeredFixture(); + const tools = path.join(root, "tools"); + await fs.mkdir(tools); + await fs.symlink(process.execPath, path.join(tools, "node")); + for (const name of ["sh", "zstd", "cut", ...(unsupported === "missing-sha256sum" ? [] : ["sha256sum"])]) { + const resolved = (await exec("sh", ["-c", 'command -v "$1"', "sh", name])).stdout.trim(); + await fs.symlink(resolved, path.join(tools, name)); + } + if (unsupported === "bsd-tar") { + await fs.writeFile(path.join(tools, "tar"), "#!/bin/sh\nprintf 'bsdtar 3.8.2\\n'\n", { mode: 0o755 }); + } else if (unsupported === "older-gnu-tar") { + await fs.writeFile(path.join(tools, "tar"), '#!/bin/sh\nif [ "$1" = "--version" ]; then printf "tar (GNU tar) 1.30\\n"; else exit 64; fi\n', { mode: 0o755 }); + } else { + const resolved = (await exec("sh", ["-c", 'command -v "$1"', "sh", tar!])).stdout.trim(); + await fs.symlink(resolved, path.join(tools, "tar")); + } + const remote = await runner(root, false, tools); + const targetPath = path.join(root, "fresh"); + expect(await stageProviderPackArchive({ archive, runner: remote, targetPath })).toBe(false); + expect(remote.execute).toHaveBeenCalledOnce(); + expect(remote.syncIn).not.toHaveBeenCalled(); + await expect(fs.lstat(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/server/src/services/native-runtime/provider-pack-archive.ts b/server/src/services/native-runtime/provider-pack-archive.ts new file mode 100644 index 00000000000..2dde86fc28a --- /dev/null +++ b/server/src/services/native-runtime/provider-pack-archive.ts @@ -0,0 +1,84 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { open, readFile } from "node:fs/promises"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeRunTrace } from "./native-run-trace.js"; +import { readProviderPackLayers, stageProviderPackLayers } from "./provider-pack-layers.js"; +import { providerPackExtractionProbe } from "./provider-pack-extraction.js"; + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +export async function readProviderPackArchive(root: string, expectedDigest: string) { + let receipt: { schema?: string; encoding?: string; providerPackDigest?: string; sha256?: string; bytes?: number }; + try { + receipt = JSON.parse(await readFile(`${root}.archive.json`, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw new Error("runner_remote_provider_archive_invalid", { cause: error }); + } + if (receipt.schema === "paperclip-runner/provider-pack-archive/v2") { + return readProviderPackLayers(root, expectedDigest, receipt); + } + if (receipt.schema !== "paperclip-runner/provider-pack-archive/v1" + || receipt.encoding !== "tar+zstd" + || receipt.providerPackDigest !== expectedDigest + || !/^[a-f0-9]{64}$/.test(receipt.sha256 ?? "") + || !Number.isSafeInteger(receipt.bytes) || receipt.bytes! <= 0) { + throw new Error("runner_remote_provider_archive_invalid"); + } + const sourcePath = `${root}.tar.zst`; + const file = await open(sourcePath, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await file.stat(); + if (!metadata.isFile() || metadata.size !== receipt.bytes) throw new Error("runner_remote_provider_archive_invalid"); + const hash = createHash("sha256"); + for await (const chunk of file.createReadStream({ autoClose: false })) hash.update(chunk); + if (hash.digest("hex") !== receipt.sha256) throw new Error("runner_remote_provider_archive_invalid"); + } finally { + await file.close(); + } + return { sourcePath, sha256: receipt.sha256!, bytes: receipt.bytes! }; +} + +export async function stageProviderPackArchive(input: { + archive: NonNullable>>; + runner: CommandManagedRuntimeRunner; + targetPath: string; + trace?: NativeRunTrace; + reuseRoots?: readonly string[]; +}): Promise { + if ("kind" in input.archive) return stageProviderPackLayers({ ...input, archive: input.archive, reuseRoots: input.reuseRoots ?? [] }); + const archive = input.archive; + if (!input.runner.syncIn) throw new Error("runner_remote_provider_archive_transport_unavailable"); + const archivePath = `${input.targetPath}.upload-${randomUUID()}.tar.zst`; + const measure = (name: string, fn: () => Promise) => input.trace ? input.trace.measure(name, fn) : fn(); + const capability = await measure("provider_pack.archive.probe", () => input.runner.execute({ + command: "sh", args: ["-c", providerPackExtractionProbe], + bypassSession: true, timeoutMs: 10_000, + })); + if (capability.exitCode !== 0 || capability.timedOut) return false; + try { + await measure("provider_pack.archive.transfer", () => input.runner.syncIn!([{ + operationId: `runner-provider-archive-${randomUUID()}`, + files: [{ kind: "file", sourcePath: archive.sourcePath, targetPath: archivePath, mode: 0o600 }], + }])); + const result = await measure("provider_pack.archive.extract", () => input.runner.execute({ + command: "sh", + args: ["-c", [ + "set -eu; umask 077;", + `test ! -L ${quote(archivePath)}; test -f ${quote(archivePath)};`, + `test "$(sha256sum -- ${quote(archivePath)} | cut -d ' ' -f 1)" = ${quote(archive.sha256)};`, + `test ! -L ${quote(input.targetPath)}; mkdir -p ${quote(input.targetPath)};`, + `tar --zstd -xf ${quote(archivePath)} --no-same-owner --same-permissions --delay-directory-restore -C ${quote(input.targetPath)};`, + `chmod 700 ${quote(input.targetPath)};`, + ].join(" ")], + bypassSession: true, + timeoutMs: 120_000, + })); + if (result.exitCode !== 0 || result.timedOut) throw new Error("runner_remote_provider_archive_extract_failed"); + return true; + } finally { + await input.runner.execute({ command: "sh", args: ["-c", `rm -f -- ${quote(archivePath)}`], + bypassSession: true, timeoutMs: 10_000 }).catch(() => undefined); + } +} diff --git a/server/src/services/native-runtime/provider-pack-extraction.ts b/server/src/services/native-runtime/provider-pack-extraction.ts new file mode 100644 index 00000000000..dedd94f09f9 --- /dev/null +++ b/server/src/services/native-runtime/provider-pack-extraction.ts @@ -0,0 +1,9 @@ +// Probe every tool and tar option used by either transport before staging bytes. +// A host with plain tar support can still use the existing directory fallback. +export const providerPackExtractionProbe = [ + "command -v zstd >/dev/null 2>&1", + "command -v sha256sum >/dev/null 2>&1", + "command -v cut >/dev/null 2>&1", + "tar_version=$(tar --zstd --ignore-zeros --no-same-owner --same-permissions --delay-directory-restore --version 2>/dev/null)", + 'case "$tar_version" in *"GNU tar"*) true ;; *) false ;; esac', +].join(" && "); diff --git a/server/src/services/native-runtime/provider-pack-layers.ts b/server/src/services/native-runtime/provider-pack-layers.ts new file mode 100644 index 00000000000..742b5e06f99 --- /dev/null +++ b/server/src/services/native-runtime/provider-pack-layers.ts @@ -0,0 +1,175 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants, createWriteStream } from "node:fs"; +import { lstat, mkdtemp, open, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, posix } from "node:path"; +import { pipeline } from "node:stream/promises"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { NativeRunTrace } from "./native-run-trace.js"; +import { providerPackExtractionProbe } from "./provider-pack-extraction.js"; + +type Archive = { sha256: string; bytes: number }; +type FileLayer = Archive & { path: string; mode: number; archive: Archive }; +export type ProviderPackLayers = { + kind: "layers"; + sourceRoot: string; + overlay: Archive; + files: FileLayer[]; +}; +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; +const validArchive = (value: unknown): value is Archive => { + const entry = value as Archive | null; + return !!entry && /^[a-f0-9]{64}$/.test(entry.sha256) + && Number.isSafeInteger(entry.bytes) && entry.bytes > 0; +}; + +async function verifyArchive(path: string, expected: Archive) { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await file.stat(); + if (!metadata.isFile() || metadata.size !== expected.bytes) throw Error("runner_remote_provider_archive_invalid"); + const hash = createHash("sha256"); + for await (const chunk of file.createReadStream({ autoClose: false })) hash.update(chunk); + if (hash.digest("hex") !== expected.sha256) throw Error("runner_remote_provider_archive_invalid"); + } finally { await file.close(); } +} + +export async function readProviderPackLayers(root: string, expectedDigest: string, value: unknown): Promise { + const receipt = value as { encoding: string; providerPackDigest: string; overlay: Archive; files: FileLayer[] }; + if (receipt.encoding !== "tar+zstd" || receipt.providerPackDigest !== expectedDigest + || !validArchive(receipt.overlay) || !Array.isArray(receipt.files) || receipt.files.length > 10_000) { + throw Error("runner_remote_provider_archive_invalid"); + } + const paths = new Set(); + for (const file of receipt.files) { + if (!validArchive(file) || !validArchive(file.archive) + || typeof file.path !== "string" || !file.path || file.path.length > 4096 + || file.path.startsWith("/") || file.path.includes("\\") || file.path.includes("\0") + || file.path.split("/").some(part => !part || part === "." || part === "..") + || !Number.isSafeInteger(file.mode) || file.mode < 0 || file.mode > 0o777 || paths.has(file.path)) { + throw Error("runner_remote_provider_archive_invalid"); + } + paths.add(file.path); + } + for (const path of paths) { + let parent = posix.dirname(path); + while (parent !== ".") { + if (paths.has(parent)) throw Error("runner_remote_provider_archive_invalid"); + parent = posix.dirname(parent); + } + } + const sourceRoot = `${root}.archives`; + if (!(await lstat(sourceRoot)).isDirectory()) throw Error("runner_remote_provider_archive_invalid"); + await verifyArchive(`${sourceRoot}/overlay.tar.zst`, receipt.overlay); + return { kind: "layers", sourceRoot, overlay: receipt.overlay, files: receipt.files }; +} + +// Copy first, then hash the private copy: no shared hardlinks, no mutation of +// the older image, and no time-of-check/time-of-use trust in its contents. +const reuseScript = ` +(async()=>{ + const fs=require('node:fs'),p=require('node:path'),c=require('node:crypto'); + const spec=JSON.parse(Buffer.from(process.argv[1],'base64').toString('utf8')); + const f=fs.promises;await f.mkdir(spec.target,{mode:448}); + const missing=[]; + const regularPath=async(root,relative)=>{ + let current=root;if(!(await f.lstat(current)).isDirectory())return false; + const parts=relative.split('/'); + for(let i=0;i{});}} + if(!reused)missing.push(index); + } + console.log(JSON.stringify(missing)); +})().catch(()=>process.exitCode=1); +`; + +export async function stageProviderPackLayers(input: { + archive: ProviderPackLayers; + runner: CommandManagedRuntimeRunner; + targetPath: string; + reuseRoots: readonly string[]; + trace?: NativeRunTrace; +}): Promise { + const { archive, runner } = input; + if (!runner.syncIn) throw Error("runner_remote_provider_archive_transport_unavailable"); + const measure = (name: string, fn: () => Promise) => input.trace ? input.trace.measure(name, fn) : fn(); + const spec = Buffer.from(JSON.stringify({ target: input.targetPath, roots: input.reuseRoots, files: archive.files })).toString("base64"); + // Respect the per-argument Linux limit; unusual packs retain directory staging. + if (Buffer.byteLength(spec) > 120_000) return false; + const capability = await measure("provider_pack.archive.probe", () => runner.execute({ + command: "sh", args: ["-c", `${providerPackExtractionProbe} && node -e 'process.exit(Number(process.versions.node.split(".")[0])>=18?0:1)'`], + bypassSession: true, timeoutMs: 10_000, + })); + if (capability.exitCode !== 0 || capability.timedOut) return false; + const reused = await measure("provider_pack.archive.reuse", () => runner.execute({ + command: "node", args: ["-e", reuseScript, spec], bypassSession: true, timeoutMs: 60_000, + })); + if (reused.exitCode !== 0 || reused.timedOut) throw Error("runner_remote_provider_archive_reuse_failed"); + let missing: number[]; + try { missing = JSON.parse(reused.stdout); } catch { throw Error("runner_remote_provider_archive_reuse_failed"); } + if (!Array.isArray(missing) || new Set(missing).size !== missing.length + || missing.some(index => !Number.isSafeInteger(index) || index < 0 || index >= archive.files.length)) { + throw Error("runner_remote_provider_archive_reuse_failed"); + } + const layers = [ + ...missing.map(index => ({ name: `${index}.tar.zst`, ...archive.files[index]!.archive })), + { name: "overlay.tar.zst", ...archive.overlay }, + ]; + const temporary = await mkdtemp(join(tmpdir(), "paperclip-provider-layers-")); + const bundlePath = join(temporary, "selected.tar.zst"); + const scratch = `${input.targetPath}.upload-${randomUUID()}`; + const remoteArchive = `${scratch}/selected.tar.zst`; + try { + // Concatenate independent zstd frames without recompression. One upload + // avoids a provider round trip for every small missing layer. + const hash = createHash("sha256"); + await measure("provider_pack.archive.verify_missing", () => pipeline((async function* () { + for (const layer of layers) { + const file = await open(`${archive.sourceRoot}/${layer.name}`, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await file.stat(), layerHash = createHash("sha256"); + if (!metadata.isFile() || metadata.size !== layer.bytes) throw Error("runner_remote_provider_archive_invalid"); + for await (const chunk of file.createReadStream({ autoClose: false })) { + layerHash.update(chunk); hash.update(chunk); yield chunk; + } + if (layerHash.digest("hex") !== layer.sha256) throw Error("runner_remote_provider_archive_invalid"); + } finally { await file.close(); } + } + })(), createWriteStream(bundlePath, { flags: "wx", mode: 0o600 }))); + const sha256 = hash.digest("hex"); + await measure("provider_pack.archive.transfer", () => runner.syncIn!([{ + operationId: `runner-provider-layers-${randomUUID()}`, + files: [{ kind: "file", sourcePath: bundlePath, targetPath: remoteArchive, mode: 0o600 }], + }])); + const result = await measure("provider_pack.archive.extract", () => runner.execute({ + command: "sh", args: ["-c", ["set -eu; umask 077;", + `test ! -L ${quote(input.targetPath)}; test -d ${quote(input.targetPath)};`, + `test ! -L ${quote(remoteArchive)}; test "$(sha256sum -- ${quote(remoteArchive)} | cut -d ' ' -f 1)" = ${quote(sha256)};`, + `tar --zstd --ignore-zeros -xf ${quote(remoteArchive)} --no-same-owner --same-permissions --delay-directory-restore -C ${quote(input.targetPath)};`, + `chmod 700 ${quote(input.targetPath)};`, + ].join(" ")], bypassSession: true, timeoutMs: 120_000, + })); + if (result.exitCode !== 0 || result.timedOut) throw Error("runner_remote_provider_archive_extract_failed"); + return true; + } finally { + await rm(temporary, { recursive: true, force: true }); + await runner.execute({ command: "sh", args: ["-c", `rm -rf -- ${quote(scratch)}`], + bypassSession: true, timeoutMs: 10_000 }).catch(() => undefined); + } +}