diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index 2f09985a1af..47e88d5df5c 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -23,6 +23,10 @@ on: description: "Verify frozen Pi shipping source on native hosted ARM and Intel Macs; no provider calls" type: boolean default: false + verify_pi_intel_continuation: + description: "Continue native Intel admission from immutable retained package artifact 11517098137; no provider calls" + type: boolean + default: false build_eval_viewer: description: "Build the canonical eval report viewer on hosted Linux" type: boolean @@ -41,8 +45,8 @@ permissions: {} concurrency: # Publishing a newer image must not discard an earlier verification's evidence. - group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_pi_macos_install && 'mac-install' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} - cancel-in-progress: ${{ !inputs.verify_pi_macos_install }} + group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ (inputs.verify_pi_macos_install || inputs.verify_pi_intel_continuation) && 'mac-install' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} + cancel-in-progress: ${{ !inputs.verify_pi_macos_install && !inputs.verify_pi_intel_continuation }} jobs: runner: @@ -102,7 +106,7 @@ jobs: echo "target_sha=$target_sha" >> "$GITHUB_OUTPUT" manual_image: - if: ${{ !inputs.verify_pi_macos_install }} + if: ${{ !inputs.verify_pi_macos_install && !inputs.verify_pi_intel_continuation }} name: Build and verify on Linux needs: authorize_manual # Keep maintainer authorization below; use a native hosted Linux builder. @@ -299,3 +303,88 @@ jobs: ${{ runner.temp }}/pi-native-install-evidence/archives/*.tgz retention-days: 14 if-no-files-found: error + + pi_native_intel_continuation: + if: github.event_name == 'workflow_dispatch' && inputs.verify_pi_intel_continuation + name: Pi native Intel retained-package admission + needs: authorize_manual + runs-on: macos-15-intel + timeout-minutes: 35 + permissions: + contents: read + actions: read + steps: + - name: Authorize an explicit maintainer image build + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + REPOSITORY_ID: ${{ github.repository_id }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_IDS: ${{ vars.AWS_CI_TRUSTED_USER_IDS }} + run: | + set -euo pipefail + test "$REPOSITORY" = paperclipai/paperclip + test "$REPOSITORY_ID" = 1170821064 + jq -e 'type == "array" and length > 0 and all(.[]; type == "number")' <<< "$ALLOWED_IDS" >/dev/null + triggering_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" + for id in "$ACTOR_ID" "$triggering_id"; do + jq -e --argjson id "$id" 'index($id) != null' <<< "$ALLOWED_IDS" >/dev/null + done + - name: Checkout verification orchestration + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + path: orchestration + persist-credentials: false + - name: Checkout frozen shipping source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: 06a3d9739f402baa7f8be7aaa29dda6db7354bfb + path: source + fetch-depth: 0 + persist-credentials: false + - name: Verify immutable source ancestry + env: + TARGET_SHA: ${{ needs.authorize_manual.outputs.target_sha }} + ORCHESTRATION_SHA: ${{ github.sha }} + run: | + set -euo pipefail + test "$(git -C source rev-parse HEAD)" = 06a3d9739f402baa7f8be7aaa29dda6db7354bfb + test "$(git -C orchestration rev-parse HEAD)" = "$ORCHESTRATION_SHA" + git -C source merge-base --is-ancestor 06a3d9739f402baa7f8be7aaa29dda6db7354bfb "$TARGET_SHA" + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + with: + version: 9.15.4 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24.21.0 + package-manager-cache: false + - name: Download immutable retained native Intel packages + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api repos/paperclipai/paperclip/actions/artifacts/11517098137 > "$RUNNER_TEMP/pi-original-artifact.json" + jq -e '.name == "pi-native-install-x64-37694650556" and .expired == false and .digest == "sha256:b6923562c643ce7c970885692c2ce6d939c644ba66212870a7becfe9be454cac"' "$RUNNER_TEMP/pi-original-artifact.json" >/dev/null + gh run download 37694650556 --repo paperclipai/paperclip --name pi-native-install-x64-37694650556 --dir "$RUNNER_TEMP/pi-original-intel" + - name: Complete normal Intel package installation and Pi admission + timeout-minutes: 30 + env: + PI_QUALIFICATION_ORCHESTRATION_SHA: ${{ github.sha }} + run: | + set -euo pipefail + node orchestration/scripts/pi-qualification-intel-continuation.mjs "$PWD/source" "$RUNNER_TEMP/pi-original-intel" "$RUNNER_TEMP/pi-native-install-evidence" + - name: Retain native installation evidence and package archives + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pi-native-install-x64-continuation-${{ github.run_id }} + path: | + ${{ runner.temp }}/pi-native-install-evidence/*.json + ${{ runner.temp }}/pi-native-install-evidence/*.log + ${{ runner.temp }}/pi-native-install-evidence/archives/*.tgz + ${{ runner.temp }}/pi-native-install-evidence/original-build/ + ${{ runner.temp }}/pi-original-artifact.json + retention-days: 14 + if-no-files-found: error diff --git a/scripts/pi-qualification-intel-continuation.mjs b/scripts/pi-qualification-intel-continuation.mjs new file mode 100644 index 00000000000..9353b38e149 --- /dev/null +++ b/scripts/pi-qualification-intel-continuation.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import cp from 'node:child_process'; +import crypto from 'node:crypto'; +import path from 'node:path'; +import {pathToFileURL} from 'node:url'; +const [sourceDirectory, originalDirectory, evidenceDirectory] = process.argv.slice(2); +const w=fs.realpathSync(sourceDirectory), prior=fs.realpathSync(originalDirectory), r=path.resolve(evidenceDirectory); +assert.equal(process.platform,'darwin');assert.equal(process.arch,'x64'); +const source='06a3d9739f402baa7f8be7aaa29dda6db7354bfb'; +const receiptPin='60a7f44bf2c60ae8be54cd19ec4244e044ac0bee315d93fbf5ac8f86ea8377c0'; +const hash=p=>crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex'); +assert.equal(hash(path.join(prior,'receipt.json')),receiptPin); +const old=JSON.parse(fs.readFileSync(path.join(prior,'receipt.json'),'utf8')); +assert.equal(old.sourceRevision,source);assert.equal(old.orchestrationRevision,'f88624c7f749adb1255186734a2a0d7386e9ad0c'); +assert.equal(old.target,'darwin-x64');assert.equal(old.nativeHardware.rosetta,false); +assert.equal(old.nativeHardware.nodeArch,'x64');assert.equal(old.nativeHardware.uname,'x86_64'); +assert.equal(old.providerCalls,0);assert.equal(old.providerCredentials,false); +assert.equal(old.archives.length,18); +const completed=new Set(); +for(const s of old.steps.filter(x=>x.status==='passed')) { + assert.equal(s.exitCode,0);assert.equal(s.errorCode,null);assert.equal(hash(path.join(prior,s.label+'.log')),s.logSha256);completed.add(s.label); +} +for(const s of ['normal-workspace-build','normal-standalone-public-build','normal-public-npm-install','verify-normal-native-daemon'])assert.ok(completed.has(s)); +fs.mkdirSync(r,{recursive:true});const home=path.join(r,'home');fs.mkdirSync(home); +const env={HOME:home,PATH:process.env.PATH,LANG:'en_US.UTF-8',CI:'true',PAPERCLIP_TELEMETRY_ENABLED:'false',PAPERCLIP_UI_DEV_MIDDLEWARE:'false',PAPERCLIP_DISABLE_PLUGIN_AUTOBUILD:'1',npm_config_audit:'false',npm_config_fund:'false',npm_config_foreground_scripts:'true',npm_config_cache:path.join(r,'npm-cache')}; +const orchestration=process.env.PI_QUALIFICATION_ORCHESTRATION_SHA; +for(const key of Object.keys(process.env))delete process.env[key];Object.assign(process.env,env); +const query=(cmd,args,cwd=w)=>cp.execFileSync(cmd,args,{cwd,env,encoding:'utf8'}).trim(); +const receipt={schema:'paperclip.pi.native-macos-continuation.v1',status:'running',sourceRevision:source,orchestrationRevision:orchestration,target:'darwin-x64',providerCredentials:false,providerCalls:0,automaticRetries:0,steps:[],archives:[],startedAt:new Date().toISOString(),buildEvidence:{run:37694650556,artifactId:11517098137,receiptSha256:receiptPin,originalStatus:old.status,completedSteps:[...completed],reason:'Continue interrupted Pi setup from verified original packages; preserve cancelled attempt'},lockSha256:old.lockSha256,releaseVersion:old.releaseVersion}; +const save=()=>fs.writeFileSync(path.join(r,'receipt.json'),JSON.stringify(receipt,null,2)+'\n'); +const progress=x=>fs.writeSync(process.stdout.fd,JSON.stringify(x)+'\n'); +function run(label,command,args,cwd,timeout) { + const p=path.join(r,label+'.log'),fd=fs.openSync(p,'wx'),s={label,status:'running',startedAt:new Date().toISOString()};receipt.steps.push(s);save();progress(s);let x; + try{x=cp.spawnSync(command,args,{cwd,env,stdio:['ignore',fd,fd],timeout,killSignal:'SIGKILL'});}finally{fs.closeSync(fd);} + Object.assign(s,{status:x.status===0&&!x.error?'passed':'failed',exitCode:x.status,errorCode:x.error?.code??null,logSha256:hash(p),finishedAt:new Date().toISOString()});save();progress(s);assert.equal(s.status,'passed',label); +} +function graph(root,label) { + const entries=[],real=fs.realpathSync(root); + function visit(dir){for(const name of fs.readdirSync(dir).sort()){ + const p=path.join(dir,name),rel=path.relative(root,p),st=fs.lstatSync(p); + if(st.isSymbolicLink()){const resolved=fs.realpathSync(p);assert.ok(resolved===real||resolved.startsWith(real+path.sep));entries.push({path:rel,symlink:fs.readlinkSync(p)});} + else if(st.isDirectory())visit(p);else if(st.isFile())entries.push({path:rel,bytes:st.size,sha256:hash(p)});else assert.fail(rel); + }}visit(root);const p=path.join(r,label+'.json');fs.writeFileSync(p,JSON.stringify({root:real,entries},null,2)+'\n');return{entries:entries.length,sha256:hash(p)}; +} +try{ + assert.equal(query('git',['rev-parse','HEAD']),source);assert.equal(query('git',['status','--porcelain']),'');assert.equal(query('uname',['-m']),'x86_64'); + const translated=cp.spawnSync('sysctl',['-n','sysctl.proc_translated'],{encoding:'utf8'});assert.notEqual(translated.stdout.trim(),'1'); + receipt.nativeHardware={uname:'x86_64',node:process.version,nodeArch:process.arch,rosetta:false,osVersion:query('sw_vers',['-productVersion'])};assert.equal(process.version,'v24.21.0'); + fs.cpSync(prior,path.join(r,'original-build'),{recursive:true});fs.mkdirSync(path.join(r,'archives')); + for(const a of old.archives){const p=path.resolve(prior,a.path);assert.ok(p.startsWith(path.join(prior,'archives')+path.sep));assert.equal(hash(p),a.sha256);fs.copyFileSync(p,path.join(r,a.path));receipt.archives.push(a);}save(); + const consumer=path.join(r,'consumer');fs.mkdirSync(consumer);fs.writeFileSync(path.join(consumer,'package.json'),JSON.stringify({private:true,type:'module'})); + run('normal-public-npm-install','npm',['install','--omit=dev',...receipt.archives.map(a=>path.join(r,a.path))],consumer,1200000); + const lockPath=path.join(consumer,'package-lock.json'),lock=fs.readFileSync(lockPath);receipt.consumerLockSha256=hash(lockPath);fs.copyFileSync(lockPath,path.join(r,'consumer-package-lock.json')); + for(const a of receipt.archives)assert.equal(JSON.parse(fs.readFileSync(path.join(consumer,'node_modules',a.name,'package.json'))).version,receipt.releaseVersion); + const server=path.join(consumer,'node_modules/@paperclipai/server'),cli=path.join(consumer,'node_modules/paperclipai/dist/index.js'),daemon=path.join(server,'dist/vendor/paperclip-runner/bin/paperclip-runnerd');assert.equal(hash(daemon),old.daemonSha256); + run('verify-normal-native-daemon','codesign',['--verify','--strict',daemon],consumer,60000);receipt.binaryDescription=query('file',[daemon],consumer);assert.ok(receipt.binaryDescription.includes('Mach-O 64-bit executable x86_64'));receipt.daemonSha256=hash(daemon); + receipt.beforePiGraph=graph(path.join(consumer,'node_modules'),'installed-graph-before-pi');save(); + run('normal-pi-runtime-setup',process.execPath,[cli,'runtime','setup','pi'],consumer,300000); + const setup=JSON.parse(fs.readFileSync(path.join(r,'normal-pi-runtime-setup.log'),'utf8').trim());assert.equal(setup.status,'installed_verified');assert.equal(setup.target,'darwin-x64');assert.deepEqual(fs.readFileSync(lockPath),lock); + const {verifyPiInstallation}=await import(pathToFileURL(path.join(server,'dist/vendor/paperclip-runner/drivers/acpx/pi-installation.js'))); + const {QUALIFIED_ACPX_PROFILES}=await import(pathToFileURL(path.join(server,'dist/vendor/paperclip-runner/drivers/acpx/qualified-profiles.js'))); + assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion,18);assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest,'sha256:9d3e7d8269f1a0af94616552dfc69671932688b81dbbd5bab9ef356b3a9cbccb');receipt.installation=await verifyPiInstallation(QUALIFIED_ACPX_PROFILES.pi);save(); + receipt.afterPiGraph=graph(path.join(consumer,'node_modules'),'installed-graph-after-pi');save(); + run('public-pi-admission-probe',process.execPath,[path.join(w,'scripts/pi-public-install-probe.mjs'),server],consumer,100000); + receipt.probe=JSON.parse(fs.readFileSync(path.join(r,'public-pi-admission-probe.log'),'utf8'));assert.equal(receipt.probe.promptCalls,0);assert.equal(receipt.probe.cleanRunnerExit,true);assert.equal(receipt.probe.target,'darwin-x64');assert.equal(hash(daemon),old.daemonSha256); + receipt.status='passed';receipt.finishedAt=new Date().toISOString();save();progress({status:'passed',target:receipt.target,sourceRevision:source,daemonSha256:receipt.daemonSha256,providerCalls:0}); +}catch(e){receipt.status='failed';receipt.error=String(e);receipt.finishedAt=new Date().toISOString();save();progress({status:'failed',error:receipt.error,providerCalls:0});process.exitCode=1;}