mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
fix: build qualification images from exact release artifacts
This commit is contained in:
1 parent
afaad3e9ca
commit
38cd5c95e5
6 files changed
+303
-6
No files matched your search
@@ -4,13 +4,21 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
target_branch:
|
target_branch:
|
||||||
description: "Branch in this repository to build; resolved to one immutable commit before checkout"
|
description: "Branch or full commit SHA in this repository to build"
|
||||||
type: string
|
type: string
|
||||||
required: false
|
required: false
|
||||||
publish_eval_image:
|
publish_eval_image:
|
||||||
description: "Publish an immutable Daytona qualification image on hosted Linux (no provider credentials)"
|
description: "Publish an immutable Daytona qualification image on hosted Linux (no provider credentials)"
|
||||||
type: boolean
|
type: boolean
|
||||||
default: false
|
default: false
|
||||||
|
release_run_id:
|
||||||
|
description: "Consume a successful exact-source Release Smoke pack, daemon and public-package artifact set"
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
|
release_base_image:
|
||||||
|
description: "Immutable Paperclip Daytona base image for the release-artifact consumer"
|
||||||
|
type: string
|
||||||
|
required: false
|
||||||
verify_source:
|
verify_source:
|
||||||
description: "Run broad source checks on hosted Linux"
|
description: "Run broad source checks on hosted Linux"
|
||||||
type: boolean
|
type: boolean
|
||||||
@@ -93,7 +101,11 @@ jobs:
|
|||||||
for id in "$ACTOR_ID" "$triggering_id"; do
|
for id in "$ACTOR_ID" "$triggering_id"; do
|
||||||
jq -e --argjson id "$id" 'index($id) != null' <<< "$ALLOWED_IDS" >/dev/null
|
jq -e --argjson id "$id" 'index($id) != null' <<< "$ALLOWED_IDS" >/dev/null
|
||||||
done
|
done
|
||||||
target_sha="$(gh api "repos/$REPOSITORY/git/ref/heads/$TARGET_BRANCH" --jq '.object.sha')"
|
if [[ "$TARGET_BRANCH" =~ ^[a-f0-9]{40}$ ]]; then
|
||||||
|
target_sha="$(gh api "repos/$REPOSITORY/commits/$TARGET_BRANCH" --jq .sha)"
|
||||||
|
else
|
||||||
|
target_sha="$(gh api "repos/$REPOSITORY/git/ref/heads/$TARGET_BRANCH" --jq '.object.sha')"
|
||||||
|
fi
|
||||||
[[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]
|
[[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]
|
||||||
echo "target_sha=$target_sha" >> "$GITHUB_OUTPUT"
|
echo "target_sha=$target_sha" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
@@ -105,6 +117,7 @@ jobs:
|
|||||||
timeout-minutes: 90
|
timeout-minutes: 90
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
actions: read
|
||||||
packages: write
|
packages: write
|
||||||
steps:
|
steps:
|
||||||
- name: Authorize an explicit maintainer image build
|
- name: Authorize an explicit maintainer image build
|
||||||
@@ -126,7 +139,7 @@ jobs:
|
|||||||
done
|
done
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||||
with:
|
with:
|
||||||
ref: ${{ needs.authorize_manual.outputs.target_sha }}
|
ref: ${{ inputs.release_run_id != '' && github.sha || needs.authorize_manual.outputs.target_sha }}
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
||||||
with:
|
with:
|
||||||
@@ -136,6 +149,7 @@ jobs:
|
|||||||
node-version: 24
|
node-version: 24
|
||||||
package-manager-cache: false
|
package-manager-cache: false
|
||||||
- name: Resolve source dependencies without lifecycle scripts
|
- name: Resolve source dependencies without lifecycle scripts
|
||||||
|
if: inputs.release_run_id == ''
|
||||||
run: |
|
run: |
|
||||||
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
|
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
|
||||||
mkdir -p remote-verification
|
mkdir -p remote-verification
|
||||||
@@ -149,7 +163,7 @@ jobs:
|
|||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
password: ${{ secrets.GITHUB_TOKEN }}
|
||||||
- name: Build checksum-verified Grok provider image
|
- name: Build checksum-verified Grok provider image
|
||||||
if: inputs.publish_eval_image
|
if: inputs.publish_eval_image && inputs.release_run_id == ''
|
||||||
env:
|
env:
|
||||||
SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
|
SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
|
||||||
run: |
|
run: |
|
||||||
@@ -170,6 +184,41 @@ jobs:
|
|||||||
docker buildx imagetools inspect "$immutable" >/dev/null
|
docker buildx imagetools inspect "$immutable" >/dev/null
|
||||||
echo "$immutable" > remote-verification/image.txt
|
echo "$immutable" > remote-verification/image.txt
|
||||||
echo "Image: $immutable" >> "$GITHUB_STEP_SUMMARY"
|
echo "Image: $immutable" >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
- name: Consume exact release artifacts into the qualification image
|
||||||
|
if: inputs.publish_eval_image && inputs.release_run_id != ''
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}
|
||||||
|
RELEASE_RUN_ID: ${{ inputs.release_run_id }}
|
||||||
|
RELEASE_BASE_IMAGE: ${{ inputs.release_base_image }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
test '${{ inputs.verify_source }}' != true
|
||||||
|
test '${{ inputs.verify_public_install }}' != true
|
||||||
|
test '${{ inputs.build_eval_viewer }}' != true
|
||||||
|
mkdir -p remote-verification
|
||||||
|
git rev-parse HEAD > remote-verification/tooling-source.txt
|
||||||
|
printf '%s\n' "$SOURCE_SHA" > remote-verification/source.txt
|
||||||
|
node scripts/prepare-runner-release-image.mjs remote-verification/release-context
|
||||||
|
image="ghcr.io/paperclipai/paperclip-daytona-runner:qualification-${SOURCE_SHA}-${GITHUB_RUN_ID}"
|
||||||
|
docker buildx build --platform linux/amd64 \
|
||||||
|
--build-arg "BASE_IMAGE=$RELEASE_BASE_IMAGE" \
|
||||||
|
--tag "$image" --metadata-file remote-verification/image.json --push \
|
||||||
|
remote-verification/release-context
|
||||||
|
digest="$(jq -r '."containerimage.digest"' remote-verification/image.json)"
|
||||||
|
[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]
|
||||||
|
immutable="${image%:*}@$digest"
|
||||||
|
docker run --rm --entrypoint cat "$immutable" /opt/paperclip-runner/provider-pack/provider-pack.json \
|
||||||
|
> remote-verification/image-provider-pack.json
|
||||||
|
cmp remote-verification/release-context/provider-pack.json remote-verification/image-provider-pack.json
|
||||||
|
docker run --rm --entrypoint sha256sum "$immutable" /usr/local/bin/paperclip-runnerd \
|
||||||
|
> remote-verification/image-daemon.sha256
|
||||||
|
expected="$(jq -r .daemonSha256 remote-verification/release-image-inputs.json)"
|
||||||
|
test "$(cut -d ' ' -f 1 remote-verification/image-daemon.sha256)" = "$expected"
|
||||||
|
printf '%s\n' "$immutable" > remote-verification/image.txt
|
||||||
|
rm -rf remote-verification/release-context
|
||||||
|
docker logout ghcr.io
|
||||||
|
echo "Image: $immutable" >> "$GITHUB_STEP_SUMMARY"
|
||||||
- name: Verify repository on hosted Linux
|
- name: Verify repository on hosted Linux
|
||||||
if: inputs.verify_source
|
if: inputs.verify_source
|
||||||
# Leave time for artifact retention within the job deadline.
|
# Leave time for artifact retention within the job deadline.
|
||||||
@@ -215,5 +264,7 @@ jobs:
|
|||||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||||
with:
|
with:
|
||||||
name: grok-remote-verification-${{ github.run_id }}
|
name: grok-remote-verification-${{ github.run_id }}
|
||||||
path: remote-verification/
|
path: |
|
||||||
|
remote-verification/
|
||||||
|
!remote-verification/release-context/**
|
||||||
retention-days: 7
|
retention-days: 7
|
||||||
@@ -94,7 +94,7 @@ jobs:
|
|||||||
run: node --test ./scripts/__tests__/e2e-shard.test.mjs
|
run: node --test ./scripts/__tests__/e2e-shard.test.mjs
|
||||||
|
|
||||||
- name: Test release verify workflow wiring
|
- name: Test release verify workflow wiring
|
||||||
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs
|
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs
|
||||||
|
|
||||||
- name: Test standalone package build concurrency
|
- name: Test standalone package build concurrency
|
||||||
run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
|
run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
|
||||||
|
|||||||
@@ -1427,3 +1427,22 @@ and pending accounting hold are retained. Cloud image alignment is the next gate
|
|||||||
Subscription follow-up remains unverified; remaining live acceptance uses API
|
Subscription follow-up remains unverified; remaining live acceptance uses API
|
||||||
keys per human steering. No local Docker or Rust build ran. Merge and production
|
keys per human steering. No local Docker or Rust build ran. Merge and production
|
||||||
deployment remain separate.
|
deployment remain separate.
|
||||||
|
|
||||||
|
### Release image artifact consumer — 2026-10-14
|
||||||
|
|
||||||
|
The first current-image x64 run failed before Claude startup. The independently
|
||||||
|
built image used a separately resolved lock and omitted the producer pack's Pi
|
||||||
|
assets, so it could not equal the pack manifest shipped in npm. The runtime
|
||||||
|
rejected it; its pending accounting hold stays preserved.
|
||||||
|
|
||||||
|
The manual cloud image workflow now optionally consumes one successful exact-
|
||||||
|
source Release Smoke artifact set. It validates repository/workflow/source,
|
||||||
|
artifact archive digests, canonical paths, package authority, daemon architecture
|
||||||
|
and daemon bytes before constructing the image. The release pack must equal the
|
||||||
|
manifest inside the public server tarball. This mode copies the producer's pack
|
||||||
|
and daemon over an immutable Paperclip base image without rebuilding either or
|
||||||
|
resolving dependencies. The resulting image is read back to compare pack and
|
||||||
|
daemon identities. Existing source-build mode remains available. Fifteen focused
|
||||||
|
negative/positive controls and 71 existing release workflow controls pass; full
|
||||||
|
current-head checks and fresh review run in hosted CI. The repaired image and
|
||||||
|
x64 task/revision are still pending. No local Docker or Rust build ran.
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
# syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e
|
||||||
|
ARG BASE_IMAGE
|
||||||
|
FROM ${BASE_IMAGE}
|
||||||
|
USER root
|
||||||
|
RUN rm -rf /opt/paperclip-runner/provider-pack
|
||||||
|
ADD provider-pack.tar.gz /opt/paperclip-runner/provider-pack/
|
||||||
|
COPY --chmod=0755 paperclip-runnerd /usr/local/bin/paperclip-runnerd
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { validateReleaseRun, selectReleaseArtifacts, validateReleaseImageBase, validateImageBindings } from '../prepare-runner-release-image.mjs';
|
||||||
|
|
||||||
|
const source = 'a'.repeat(40);
|
||||||
|
const run = { head_sha: source, conclusion: 'success', path: '.github/workflows/release-smoke.yml', repository: { full_name: 'paperclipai/paperclip' } };
|
||||||
|
const artifacts = ['runner-provider-packs-123-2-linux-x64', 'runner-release-123-2-linux-x64', 'claude-qualification-public-packages']
|
||||||
|
.map((name, index) => ({ id: index + 1, name, expired: false, digest: 'sha256:' + 'b'.repeat(64), size_in_bytes: 100 }));
|
||||||
|
|
||||||
|
test('release image consumes one successful exact-source producer set', () => {
|
||||||
|
assert.doesNotThrow(() => validateReleaseRun(run, source));
|
||||||
|
assert.deepEqual(selectReleaseArtifacts(artifacts, '123', 2), artifacts);
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const [name, patch] of [
|
||||||
|
['different source', { head_sha: 'c'.repeat(40) }],
|
||||||
|
['incomplete producer', { conclusion: null }],
|
||||||
|
['failed producer', { conclusion: 'failure' }],
|
||||||
|
['unrelated workflow', { path: '.github/workflows/ci.yml' }],
|
||||||
|
['fork artifact', { repository: { full_name: 'fork/paperclip' } }],
|
||||||
|
]) test('rejects ' + name, () => assert.throws(() => validateReleaseRun({ ...run, ...patch }, source)));
|
||||||
|
|
||||||
|
for (const [name, values] of [
|
||||||
|
['missing pack', artifacts.slice(1)],
|
||||||
|
['duplicate authority', [...artifacts, artifacts[0]]],
|
||||||
|
['expired artifact', artifacts.map((a, i) => i ? a : { ...a, expired: true })],
|
||||||
|
['unbound digest', artifacts.map((a, i) => i ? a : { ...a, digest: null })],
|
||||||
|
['oversized archive', artifacts.map((a, i) => i ? a : { ...a, size_in_bytes: 3 * 1024 ** 3 })],
|
||||||
|
['wrong attempt', artifacts.map((a, i) => i ? a : { ...a, name: 'runner-provider-packs-123-1-linux-x64' })],
|
||||||
|
]) test('rejects ' + name, () => assert.throws(() => selectReleaseArtifacts(values, '123', 2)));
|
||||||
|
|
||||||
|
test('base image must be immutable and repository-owned', () => {
|
||||||
|
assert.equal(validateReleaseImageBase('ghcr.io/paperclipai/paperclip-daytona-runner@sha256:' + source + 'b'.repeat(24)), 'ghcr.io/paperclipai/paperclip-daytona-runner@sha256:' + source + 'b'.repeat(24));
|
||||||
|
for (const base of ['ghcr.io/paperclipai/paperclip-daytona-runner:latest', 'ghcr.io/fork/paperclip-daytona-runner@sha256:' + 'a'.repeat(64), 'ubuntu:latest']) {
|
||||||
|
assert.throws(() => validateReleaseImageBase(base));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const daemonBytes = Buffer.alloc(64);
|
||||||
|
daemonBytes.write('7f454c46', 0, 'hex');
|
||||||
|
daemonBytes[4] = 2;
|
||||||
|
daemonBytes[5] = 1;
|
||||||
|
daemonBytes.writeUInt16LE(62, 18);
|
||||||
|
const daemonDigest = 'sha256:' + createHash('sha256').update(daemonBytes).digest('hex');
|
||||||
|
const binding = { source, pack: { digest: 'release', payload: { productionLock: 'original', candidateProviders: { pi: 'qualified' } } },
|
||||||
|
receipt: { schema: 'paperclip.runner.release-artifact.v1', sourceRevision: source, target: 'linux-x64', sha256: daemonDigest },
|
||||||
|
release: { sourceRevision: source, platforms: { 'linux-x64': { sha256: daemonDigest } } }, daemonBytes };
|
||||||
|
binding.publicPack = structuredClone(binding.pack);
|
||||||
|
|
||||||
|
test('image pack must equal the authority shipped in the public package', () => {
|
||||||
|
assert.equal(validateImageBindings(binding), daemonDigest.slice(7));
|
||||||
|
for (const changed of [{ productionLock: 'regenerated' }, { candidateProviders: {} }]) {
|
||||||
|
const different = structuredClone(binding.pack);
|
||||||
|
Object.assign(different.payload, changed);
|
||||||
|
assert.throws(() => validateImageBindings({ ...binding, publicPack: different }), /public package authority/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('image daemon must match source, receipt, public package and actual architecture', () => {
|
||||||
|
for (const changed of [{ sourceRevision: 'c'.repeat(40) }, { sha256: 'sha256:' + 'c'.repeat(64) }, { target: 'linux-arm64' }]) {
|
||||||
|
assert.throws(() => validateImageBindings({ ...binding, receipt: { ...binding.receipt, ...changed } }));
|
||||||
|
}
|
||||||
|
assert.throws(() => validateImageBindings({ ...binding, release: { ...binding.release, sourceRevision: 'c'.repeat(40) } }));
|
||||||
|
const wrongArch = Buffer.from(daemonBytes); wrongArch.writeUInt16LE(183, 18);
|
||||||
|
const digest = 'sha256:' + createHash('sha256').update(wrongArch).digest('hex');
|
||||||
|
assert.throws(() => validateImageBindings({ ...binding, daemonBytes: wrongArch,
|
||||||
|
receipt: { ...binding.receipt, sha256: digest }, release: { ...binding.release, platforms: { 'linux-x64': { sha256: digest } } } }));
|
||||||
|
});
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { createReadStream, createWriteStream } from 'node:fs';
|
||||||
|
import { chmod, copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { pipeline } from 'node:stream/promises';
|
||||||
|
import { Transform } from 'node:stream';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { isDeepStrictEqual } from 'node:util';
|
||||||
|
import { verifyReleaseProviderPack } from '../packages/paperclip-runner/scripts/release-provider-pack.mjs';
|
||||||
|
|
||||||
|
export function validateReleaseRun(run, source) {
|
||||||
|
if (!/^[a-f0-9]{40}$/.test(source) || run.head_sha !== source || run.conclusion !== 'success'
|
||||||
|
|| run.path !== '.github/workflows/release-smoke.yml' || run.repository?.full_name !== 'paperclipai/paperclip') {
|
||||||
|
throw Error('Image inputs require a successful exact-source Release Smoke run in paperclipai/paperclip');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function selectReleaseArtifacts(artifacts, runId, attempt) {
|
||||||
|
const names = [
|
||||||
|
`runner-provider-packs-${runId}-${attempt}-linux-x64`,
|
||||||
|
`runner-release-${runId}-${attempt}-linux-x64`,
|
||||||
|
];
|
||||||
|
const selected = names.map(name => artifacts.filter(a => a.name === name));
|
||||||
|
selected.push(artifacts.filter(a => /^[a-zA-Z0-9._-]+-public-packages$/.test(a.name)));
|
||||||
|
if (selected.some(matches => matches.length !== 1)) throw Error('Expected one pack, daemon and public-package artifact');
|
||||||
|
return selected.map(([a]) => {
|
||||||
|
if (a.expired || !Number.isSafeInteger(a.id) || a.id <= 0 || !/^sha256:[a-f0-9]{64}$/.test(a.digest)
|
||||||
|
|| !Number.isSafeInteger(a.size_in_bytes) || a.size_in_bytes < 1 || a.size_in_bytes > 2 * 1024 ** 3) {
|
||||||
|
throw Error('Invalid or expired release artifact');
|
||||||
|
}
|
||||||
|
return a;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateReleaseImageBase(base) {
|
||||||
|
if (!/^ghcr\.io\/paperclipai\/paperclip-daytona-runner@sha256:[a-f0-9]{64}$/.test(base)) {
|
||||||
|
throw Error('Release image base must be an immutable Paperclip Daytona image');
|
||||||
|
}
|
||||||
|
return base;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateImageBindings({ source, pack, publicPack, receipt, release, daemonBytes }) {
|
||||||
|
if (!isDeepStrictEqual(publicPack, pack)) throw Error('Image pack differs from public package authority');
|
||||||
|
const sha256 = createHash('sha256').update(daemonBytes).digest('hex');
|
||||||
|
if (receipt.schema !== 'paperclip.runner.release-artifact.v1' || receipt.sourceRevision !== source || receipt.target !== 'linux-x64'
|
||||||
|
|| receipt.sha256 !== `sha256:${sha256}` || release.sourceRevision !== source || release.platforms?.['linux-x64']?.sha256 !== `sha256:${sha256}`
|
||||||
|
|| daemonBytes.length < 64 || daemonBytes.subarray(0, 4).toString('hex') !== '7f454c46'
|
||||||
|
|| daemonBytes[4] !== 2 || daemonBytes[5] !== 1 || daemonBytes.readUInt16LE(18) !== 62) {
|
||||||
|
throw Error('Daemon source, digest or architecture mismatch');
|
||||||
|
}
|
||||||
|
return sha256;
|
||||||
|
}
|
||||||
|
|
||||||
|
const digestFile = async file => {
|
||||||
|
const hash = createHash('sha256');
|
||||||
|
for await (const chunk of createReadStream(file)) hash.update(chunk);
|
||||||
|
return hash.digest('hex');
|
||||||
|
};
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const source = process.env.SOURCE_SHA, runId = process.env.RELEASE_RUN_ID, token = process.env.GH_TOKEN;
|
||||||
|
const output = resolve(process.argv[2] ?? 'remote-verification/release-context');
|
||||||
|
validateReleaseImageBase(process.env.RELEASE_BASE_IMAGE);
|
||||||
|
if (!/^[1-9][0-9]*$/.test(runId ?? '') || !token) throw Error('Release run ID and GitHub token required');
|
||||||
|
const api = async endpoint => {
|
||||||
|
const response = await fetch(`https://api.github.com/repos/paperclipai/paperclip/${endpoint}`, {
|
||||||
|
headers: { authorization: `Bearer ${token}`, accept: 'application/vnd.github+json' }, redirect: 'error',
|
||||||
|
signal: AbortSignal.timeout(30_000),
|
||||||
|
});
|
||||||
|
if (!response.ok) throw Error(`GitHub release metadata request failed: ${response.status}`);
|
||||||
|
return response.json();
|
||||||
|
};
|
||||||
|
const run = await api(`actions/runs/${runId}`);
|
||||||
|
validateReleaseRun(run, source);
|
||||||
|
const artifacts = [];
|
||||||
|
for (let page = 1; ; page++) {
|
||||||
|
const result = await api(`actions/runs/${runId}/artifacts?per_page=100&page=${page}`);
|
||||||
|
artifacts.push(...result.artifacts);
|
||||||
|
if (result.artifacts.length < 100) break;
|
||||||
|
if (page === 10) throw Error('Unexpected artifact count');
|
||||||
|
}
|
||||||
|
const selected = selectReleaseArtifacts(artifacts, runId, run.run_attempt);
|
||||||
|
await mkdir(output, { recursive: false });
|
||||||
|
const dirs = [];
|
||||||
|
for (const [index, artifact] of selected.entries()) {
|
||||||
|
const response = await fetch(`https://api.github.com/repos/paperclipai/paperclip/actions/artifacts/${artifact.id}/zip`, {
|
||||||
|
headers: { authorization: `Bearer ${token}`, accept: 'application/vnd.github+json' }, redirect: 'manual',
|
||||||
|
signal: AbortSignal.timeout(30_000),
|
||||||
|
});
|
||||||
|
const url = response.headers.get('location');
|
||||||
|
if (response.status !== 302 || !url || new URL(url).protocol !== 'https:'
|
||||||
|
|| !new URL(url).hostname.endsWith('.blob.core.windows.net')) throw Error('Unexpected artifact download redirect');
|
||||||
|
const download = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(300_000) });
|
||||||
|
if (!download.ok || !download.body) throw Error('Release artifact download failed');
|
||||||
|
const archive = join(output, `artifact-${index}.zip`), hash = createHash('sha256');
|
||||||
|
let size = 0;
|
||||||
|
await pipeline(download.body, new Transform({ transform(chunk, encoding, callback) {
|
||||||
|
size += chunk.length;
|
||||||
|
if (size > artifact.size_in_bytes) return callback(Error('Artifact exceeds its declared size'));
|
||||||
|
hash.update(chunk); callback(null, chunk);
|
||||||
|
} }), createWriteStream(archive, { flags: 'wx', mode: 0o600 }));
|
||||||
|
if (size !== artifact.size_in_bytes || `sha256:${hash.digest('hex')}` !== artifact.digest) throw Error('Artifact digest or size mismatch');
|
||||||
|
const dir = join(output, `artifact-${index}`);
|
||||||
|
execFileSync('python3', ['-c', `import zipfile,sys,pathlib,stat
|
||||||
|
z=zipfile.ZipFile(sys.argv[1]); dest=pathlib.Path(sys.argv[2]); kind=int(sys.argv[3]); entries=z.infolist()
|
||||||
|
expected=[{'linux-x64/provider-pack.json','linux-x64/provider-pack.tar.gz'},{'linux-x64/paperclip-runnerd','linux-x64/receipt.json'}]
|
||||||
|
names=[e.filename for e in entries]
|
||||||
|
if len(names)!=len(set(names)): raise RuntimeError('Duplicate archive paths')
|
||||||
|
if kind<2:
|
||||||
|
if set(names)!=expected[kind]: raise RuntimeError('Unexpected release artifact entries')
|
||||||
|
else:
|
||||||
|
import re
|
||||||
|
if 'manifest.json' not in names or any(not re.fullmatch(r'(manifest.json|paperclipai-[a-z0-9.-]+\\.tgz)',n) for n in names): raise RuntimeError('Unexpected public package entries')
|
||||||
|
if sum(e.file_size for e in entries)>4*1024**3 or any(stat.S_ISLNK(e.external_attr>>16) for e in entries): raise RuntimeError('Unsafe artifact extraction')
|
||||||
|
z.extractall(dest)
|
||||||
|
`, archive, dir, String(index)], { stdio: 'pipe' });
|
||||||
|
await rm(archive);
|
||||||
|
dirs.push(dir);
|
||||||
|
}
|
||||||
|
const pack = JSON.parse(await readFile(join(dirs[0], 'linux-x64/provider-pack.json'), 'utf8'));
|
||||||
|
verifyReleaseProviderPack(pack, source, 'linux-x64');
|
||||||
|
const packages = JSON.parse(await readFile(join(dirs[2], 'manifest.json'), 'utf8'));
|
||||||
|
if (packages.schema !== 'paperclip.runner.public-packages/v1' || packages.sourceRevision !== source) throw Error('Wrong public-package source');
|
||||||
|
const server = packages.archives?.filter(a => a.name === '@paperclipai/server');
|
||||||
|
if (server?.length !== 1 || !/^paperclipai-server-[a-z0-9.-]+\.tgz$/.test(server[0].file)
|
||||||
|
|| !/^[a-f0-9]{64}$/.test(server[0].sha256)) throw Error('Invalid server archive');
|
||||||
|
const serverArchive = join(dirs[2], server[0].file);
|
||||||
|
if (await digestFile(serverArchive) !== server[0].sha256) throw Error('Server archive digest mismatch');
|
||||||
|
const fromServer = name => JSON.parse(execFileSync('tar', ['-xOf', serverArchive, `package/dist/vendor/paperclip-runner/${name}`], { encoding: 'utf8', maxBuffer: 1024 * 1024 }));
|
||||||
|
const publicPack = fromServer('remote-provider-packs/linux-x64/provider-pack.json');
|
||||||
|
const packArchive = join(dirs[0], 'linux-x64/provider-pack.tar.gz');
|
||||||
|
if (!isDeepStrictEqual(JSON.parse(execFileSync('tar', ['-xOf', packArchive, './provider-pack.json'], { encoding: 'utf8', maxBuffer: 1024 * 1024 })), pack)) throw Error('Pack archive manifest mismatch');
|
||||||
|
const receipt = JSON.parse(await readFile(join(dirs[1], 'linux-x64/receipt.json'), 'utf8'));
|
||||||
|
const daemon = join(dirs[1], 'linux-x64/paperclip-runnerd'), bytes = await readFile(daemon);
|
||||||
|
const release = fromServer('bin/release-manifest.json');
|
||||||
|
const sha256 = validateImageBindings({ source, pack, publicPack, receipt, release, daemonBytes: bytes });
|
||||||
|
await chmod(daemon, 0o700);
|
||||||
|
if (!isDeepStrictEqual(JSON.parse(execFileSync(daemon, ['--build-metadata'], { encoding: 'utf8', timeout: 15_000, maxBuffer: 65536 })), receipt.buildMetadata)) throw Error('Daemon metadata mismatch');
|
||||||
|
await copyFile(packArchive, join(output, 'provider-pack.tar.gz'));
|
||||||
|
await copyFile(daemon, join(output, 'paperclip-runnerd'));
|
||||||
|
await copyFile(new URL('../docker/daytona-runner/Dockerfile.release', import.meta.url), join(output, 'Dockerfile'));
|
||||||
|
await copyFile(join(dirs[0], 'linux-x64/provider-pack.json'), join(output, 'provider-pack.json'));
|
||||||
|
for (const dir of dirs) await rm(dir, { recursive: true });
|
||||||
|
const evidence = { sourceRevision: source, producerRunId: runId, producerAttempt: run.run_attempt, providerPackDigest: pack.digest,
|
||||||
|
daemonSha256: sha256, baseImage: process.env.RELEASE_BASE_IMAGE, artifacts: selected.map(a => ({ id: a.id, name: a.name, digest: a.digest })), providerCalls: 0 };
|
||||||
|
await writeFile(resolve(output, '../release-image-inputs.json'), JSON.stringify(evidence, null, 2) + '\n');
|
||||||
|
console.log(JSON.stringify(evidence));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) await main();
|
||||||
Reference in new issue
Block a user