diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index 6763d352bb6..97d53639b3b 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -31,6 +31,10 @@ on: description: "Verify frozen Pi 1.0 closed startup and SDK contracts on native Intel macOS (no provider prompts)" type: boolean default: false + diagnose_pi_startup: + description: "Instrument one retained S native Intel startup with unchanged deadlines (diagnostic only, no provider prompts)" + type: boolean + default: false diagnose_ajv_pack: description: "Diagnose only pinned AJV npm directory packing on Linux (no Runner build/tests)" type: boolean @@ -65,7 +69,7 @@ permissions: {} concurrency: # Publishing a newer image must not discard an earlier verification's evidence. - group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} + group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.diagnose_pi_startup && 'pi-startup-diagnostic' || inputs.verify_pi_intel && 'pi-native-intel' || inputs.diagnose_ajv_pack && 'ajv-diagnostic' || inputs.verify_runner && 'runner-verification' || inputs.source_e2e_support_only && 'source-e2e-support' || inputs.source_root_tests_only && 'source-root-tests' || inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} cancel-in-progress: true jobs: @@ -120,8 +124,9 @@ jobs: SOURCE_E2E_SUPPORT_ONLY: ${{ inputs.source_e2e_support_only }} IMAGE_MODE: ${{ inputs.publish_eval_image || inputs.verify_public_install || inputs.build_eval_viewer }} VERIFY_PI_INTEL: ${{ inputs.verify_pi_intel }} + DIAGNOSE_PI_STARTUP: ${{ inputs.diagnose_pi_startup }} DIAGNOSE_AJV: ${{ inputs.diagnose_ajv_pack }} - OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel || inputs.source_e2e_support_only }} + OTHER_MODE: ${{ inputs.publish_eval_image || inputs.verify_source || inputs.verify_public_install || inputs.build_eval_viewer || inputs.verify_pi_intel || inputs.diagnose_pi_startup || inputs.source_e2e_support_only }} run: | set -euo pipefail test "$REPOSITORY" = paperclipai/paperclip @@ -165,6 +170,7 @@ jobs: test "$EXPECTED_LOCK_SHA256" = 5ae57d1ddd475691dac1f1cfbd4836e54f7da1956b47e6e705b218ae3070ae2e fi if [ "$VERIFY_PI_INTEL" = true ]; then + test "$DIAGNOSE_PI_STARTUP" != true test "$VERIFY_RUNNER" != true test "$VERIFY_SOURCE" != true test "$SOURCE_ROOT_TESTS_ONLY" != true @@ -176,6 +182,18 @@ jobs: # Artifact storage and unrelated resource costs are not inferred here. test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public fi + if [ "$DIAGNOSE_PI_STARTUP" = true ]; then + test "$VERIFY_PI_INTEL" != true + test "$VERIFY_RUNNER" != true + test "$VERIFY_SOURCE" != true + test "$SOURCE_ROOT_TESTS_ONLY" != true + test "$SOURCE_E2E_SUPPORT_ONLY" != true + test "$DIAGNOSE_AJV" != true + test "$IMAGE_MODE" != true + test "$EXPECTED_SOURCE_SHA" = 586af4d7f3b07aa683338aedb4e5182603b70401 + test "$EXPECTED_LOCK_SHA256" = 38338a6867358440c5ab5993eaeb85fb501bc7df65d24acf52fbd63c880ee4ba + test "$(gh api "repos/$REPOSITORY" --jq '.visibility')" = public + fi if [ -n "$EXPECTED_SOURCE_SHA" ]; then test "$target_sha" = "$EXPECTED_SOURCE_SHA" fi @@ -248,6 +266,57 @@ jobs: retention-days: 7 if-no-files-found: error + manual_pi_startup_diagnostic: + name: Retained S native Intel startup timing diagnostic (not qualification) + if: github.event_name == 'workflow_dispatch' && inputs.diagnose_pi_startup + needs: authorize_manual + runs-on: macos-15-intel + timeout-minutes: 25 + permissions: + contents: read + actions: read + steps: + - name: Checkout trusted diagnostic helpers only + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.workflow_sha }} + path: trusted-ci + persist-credentials: false + - name: Download exact failed qualification artifacts + timeout-minutes: 5 + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api repos/paperclipai/paperclip/actions/artifacts/11200013599/zip \ + > "$RUNNER_TEMP/pi-startup-36940079040.zip" + printf '%s %s\n' 9e0209b69b2ee886c4cc8c9c74310a6c1a9010c81e41d618499b2a4d42f92830 \ + "$RUNNER_TEMP/pi-startup-36940079040.zip" | shasum -a 256 --check + - name: Run one instrumented startup with original assertions and deadlines + timeout-minutes: 18 + run: | + set -euo pipefail + python3 -B trusted-ci/scripts/ci/pi-intel/startup_diagnostic.py \ + --archive "$RUNNER_TEMP/pi-startup-36940079040.zip" \ + --output "$GITHUB_WORKSPACE/pi-startup-diagnostic-evidence" + - name: Admit complete diagnostic evidence within the storage bound + if: always() + id: diagnostic_evidence + timeout-minutes: 2 + run: | + python3 -B trusted-ci/scripts/ci/pi-intel/admit_diagnostic_evidence.py \ + --evidence "$GITHUB_WORKSPACE/pi-startup-diagnostic-evidence" \ + --github-output "$GITHUB_OUTPUT" + - name: Retain original and diagnostic identities, timings, assertions and cleanup + if: always() && steps.diagnostic_evidence.outputs.admitted == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: pi-startup-diagnostic-${{ github.run_id }} + include-hidden-files: true + path: pi-startup-diagnostic-evidence/ + retention-days: 7 + if-no-files-found: error + manual_runner_verify: name: Full offline Runner verification on GitHub-hosted Ubuntu if: github.event_name == 'workflow_dispatch' && inputs.verify_runner @@ -614,7 +683,7 @@ jobs: manual_image: name: Build and verify on EC2 - if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel && !inputs.source_e2e_support_only + if: github.event_name == 'workflow_dispatch' && !inputs.verify_runner && !inputs.verify_source && !inputs.diagnose_ajv_pack && !inputs.verify_pi_intel && !inputs.diagnose_pi_startup && !inputs.source_e2e_support_only needs: authorize_manual runs-on: runs-on/fleet=paperclip-public-pr-x64/env=public-ci timeout-minutes: 90 diff --git a/scripts/ci/pi-intel/admit_diagnostic_evidence.py b/scripts/ci/pi-intel/admit_diagnostic_evidence.py new file mode 100644 index 00000000000..87d46e403e9 --- /dev/null +++ b/scripts/ci/pi-intel/admit_diagnostic_evidence.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +"""Retain complete reconstructable diagnostic delta, capped at 256MiB for seven days.""" +import argparse,json +from pathlib import Path +from admit_evidence import admit_evidence +MAX_DIAGNOSTIC_BYTES=256*1024**2 +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--evidence',type=Path,required=True);p.add_argument('--github-output',type=Path,required=True);a=p.parse_args() + result=admit_evidence(a.evidence,MAX_DIAGNOSTIC_BYTES);accepted=result['status']=='admitted_complete_evidence' + with a.github_output.open('a') as f:f.write('admitted='+str(accepted).lower()+'\n') + print(json.dumps(result));raise SystemExit(0 if accepted else 1) diff --git a/scripts/ci/pi-intel/diagnostic_lifecycle.py b/scripts/ci/pi-intel/diagnostic_lifecycle.py new file mode 100644 index 00000000000..ddc9e43f137 --- /dev/null +++ b/scripts/ci/pi-intel/diagnostic_lifecycle.py @@ -0,0 +1,30 @@ +"""One diagnostic child, deferred cancellation through handle publication.""" +import time,subprocess +from launch_gate import LaunchGate +from source_guard import require +class DiagnosticChild: + def __init__(self,owner):self.owner=owner;self.child=None;self.attempted=False;self.uncertain=False;self.retired=False;self.cancelled=None;self.gate=LaunchGate(self.retire) + def cancel(self,reason):self.cancelled=reason;self.gate.cancel(reason) + def retire(self,child): + if self.retired:return + try: + result=self.owner.stop(child.pid) + require(result['status']=='stopped' and not self.owner.live(),'Diagnostic descendants remain') + child.wait(timeout=10);self.retired=True + except BaseException:self.uncertain=True;raise + def execute(self,factory,timeout,clock=time.monotonic,sleep=time.sleep): + require(not self.attempted and not self.uncertain,'Diagnostic may execute once only') + start=clock() + def before(): + if self.cancelled:self.gate.cancel(self.cancelled) + def launch():self.attempted=True;return factory() + def publish(child):self.child=child;self.owner.observe(child.pid) + try: + self.gate.spawn(launch,publish,before) + while self.child.poll() is None: + self.owner.observe(self.child.pid);require(clock()-start=previous,'Nonmonotonic timing data');previous=current + phases=[r['phase'] for r in rows] + require(phases[0]=='sidecar.entry' and phases[-1]=='sidecar.exit' and 'session.open.begin' in phases,'Required diagnostic entry/open/terminal marks missing; sink or startup incomplete') + require(phases.count('sidecar.entry')==phases.count('sidecar.exit')==1,'Duplicate sidecar execution') + return {'rows':rows,'elapsedFromEntryMs':[{'phase':r['phase'],'milliseconds':round((int(r['ns'])-int(rows[0]['ns']))/1e6,3)} for r in rows],'sha256':sha(path),'bytes':len(data),'completeSink':True} + +def reject_sink_failure(logs): + require(len(logs)<=8,'Unexpected diagnostic log count') + for path in logs: + st=path.lstat() + require(stat.S_ISREG(st.st_mode) and st.st_nlink==1 and st.st_size<=1024*1024,'Diagnostic stderr is missing, linked or oversized') + require(b'PC_STARTUP_DIAGNOSTIC_SINK_FAILED' not in path.read_bytes(),'Explicit diagnostic sink failure marker') + +def execute(args): + out=args.output.resolve();out.mkdir(mode=0o700,parents=True,exist_ok=False) + scratch=Path(tempfile.mkdtemp(prefix='pc-intel-diagnostic-',dir='/private/tmp'));scratch.chmod(0o700) + scratch_id=(scratch.stat().st_dev,scratch.stat().st_ino,scratch.stat().st_uid) + proof={'schema':'paperclip.native-intel-startup-diagnostic/v1','status':'preparing','sourceRevision':PIN['sourceRevision'],'runId':os.environ['GITHUB_RUN_ID'],'trustedWorkflowRevision':os.environ['GITHUB_WORKFLOW_SHA'],'originalQualificationRunId':PIN['artifactRunId'],'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'isQualification':False,'providerCalls':0,'credentialsRead':False,'runtimePromptsSubmitted':0,'nativeRecompiled':False,'dependencyInstallExecuted':False,'testSourceUnmodified':True,'deadlineChanged':False,'helpers':{p.name:sha(p) for p in HERE.iterdir() if p.is_file()},'commands':[],'cleanupUncertain':False,'scratchRemoved':False,'hardwareBareMetalClaim':False,'physicalColdDiskClaim':False,'cacheNote':'Artifact integrity verification necessarily reads payload before testing; no startup warmup is performed.'} + def save():atomic_json(out/'receipt.json',proof) + active=None + def cancelled(sig,_): + if active is not None:active.cancel('signal '+str(sig)) + else:raise InterruptedError('Diagnostic cancelled') + signal.signal(signal.SIGTERM,cancelled);signal.signal(signal.SIGINT,cancelled) + env={'PATH':'/usr/bin:/bin:/usr/sbin:/sbin','HOME':str(scratch/'home'),'LANG':'en_US.UTF-8','TMPDIR':str(scratch)};Path(env['HOME']).mkdir(mode=0o700) + def command(label,argv,timeout=30): + row={'label':label,'argv':list(map(str,argv)),'deadlineSeconds':timeout};proof['commands'].append(row);save() + with (out/(label+'.log')).open('xb') as log:r=subprocess.run(row['argv'],env=env,cwd=scratch,stdout=log,stderr=log,timeout=timeout) + row.update(exitCode=r.returncode,logSha256=sha(out/(label+'.log')));save();require(r.returncode==0,'Diagnostic preparation failed: '+label) + return (out/(label+'.log')).read_text() + save() + try: + require(platform.system()=='Darwin' and platform.machine()=='x86_64','Native Intel required') + for flag in ['sysctl.proc_translated','hw.optional.arm64']: + r=subprocess.run(['/usr/sbin/sysctl','-in',flag],capture_output=True,text=True,env=env,timeout=10) + require(r.stdout.strip() in ('','0'),'Rosetta/ARM rejected');proof[flag]={'value':r.stdout.strip(),'exitCode':r.returncode} + hardware=command('hardware',['/usr/sbin/sysctl','hw.machine','machdep.cpu.vendor','machdep.cpu.brand_string']) + require('GenuineIntel' in hardware and 'x86_64' in hardware,'Intel hardware evidence missing') + original=extract_selected(args.archive,scratch/'original-artifact',PIN) + references=out/'original-reference';references.mkdir(mode=0o700) + for name in ['receipt.json','provider-pack.json','pack-inventory.json','source-input-inventory.json','resolved-pnpm-lock.yaml']:shutil.copy2(original/name,references/name) + atomic_json(out/'original-input-pins.json',PIN) + receipt=json.loads((original/'receipt.json').read_text());require(receipt['sourceRevision']==PIN['sourceRevision'] and receipt['runId']==PIN['artifactRunId'] and receipt['trustedWorkflowRevision']==PIN['artifactWorkflowRevision'],'Original artifact provenance mismatch') + require(receipt['cleanupUncertain'] is False and receipt['status']=='failed_no_retry','Unexpected original qualification disposition') + proof['originalFailure']={'status':receipt['status'],'testFailures':receipt.get('testFailures'),'packArchiveSha256':sha(original/'provider-pack.tar.gz'),'daemonSha256':sha(original/'paperclip-runnerd')} + require(sha(original/'paperclip-runnerd')==PIN['daemonSha256'] and sha(original/'resolved-pnpm-lock.yaml')==PIN['resolvedLockSha256'],'Original daemon/lock mismatch') + inventory=json.loads((original/'pack-inventory.json').read_text());pack=restore_pack(original/'provider-pack.tar.gz',scratch/'unpacked',inventory) + node=pack/'node_modules/node/bin/node';daemon=scratch/'paperclip-runnerd';shutil.copyfile(original/'paperclip-runnerd',daemon);daemon.chmod(0o755) + require(sha(node)==PIN['nodeSha256'] and sha(daemon)==PIN['daemonSha256'],'Native executable changed') + require(json.loads(command('node-identity',[node,'-p','JSON.stringify([process.platform,process.arch,process.version])']))==['darwin','x64','v24.21.0'],'Node platform mismatch') + command('daemon-signature',['/usr/bin/codesign','--verify','--strict',daemon]);require('Mach-O 64-bit executable x86_64' in command('daemon-architecture',['/usr/bin/file',daemon]),'Daemon architecture mismatch') + proof['originalPackVerification']=json.loads(command('original-pack-verify',[node,HERE/'verify-pack.mjs',pack,PIN['sourceRevision'],'darwin','x64'],180)) + require(proof['originalPackVerification']['manifestDigest']==PIN['originalPackDigest'],'Original pack identity mismatch') + before_manifest=json.loads((pack/'provider-pack.json').read_text()) + sink=scratch/'startup-timings.jsonl';fd=os.open(sink,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600);st=os.fstat(fd);os.close(fd) + identity={k:str(getattr(st,'st_'+k)) for k in ['dev','ino','uid']};identity['path']=str(sink) + sidecar=pack/'dist/cli/acpx-runtime-sidecar.cjs';original_sidecar=sidecar.read_bytes();patched,patch=patch_sidecar(original_sidecar,identity) + # Preserve original file mode; this one private copy now has an explicit diagnostic identity. + sidecar.write_bytes(patched);atomic_json(out/'sidecar-patch.json',patch) + (out/'sidecar.diff').write_text(''.join(difflib.unified_diff(original_sidecar.decode().splitlines(True),patched.decode().splitlines(True),fromfile='original-S-sidecar',tofile='diagnostic-sidecar'))) + command('diagnostic-sidecar-syntax',[node,'--check',sidecar]) + proof['diagnosticPackVerification']=json.loads(command('diagnostic-pack-rebind',[node,HERE/'rebind-diagnostic-pack.mjs',pack,PIN['originalSidecarSha256'],PIN['sourceRevision']],180)) + after_manifest=json.loads((pack/'provider-pack.json').read_text());expected=copy.deepcopy(before_manifest) + expected['payload']['artifacts']['acpxSidecar']['sha256']=after_manifest['payload']['artifacts']['acpxSidecar']['sha256'] + for key in ['distDigest','bridgeDigest']:expected['payload'][key]=after_manifest['payload'][key] + expected['digest']=after_manifest['digest'];require(expected==after_manifest,'Diagnostic changed other manifest authority') + after=closed_tree(pack);proof['allowedPackDelta']=validate_pack_delta(inventory,after);atomic_json(out/'diagnostic-pack-inventory.json',after) + shutil.copy2(pack/'provider-pack.json',out/'diagnostic-provider-pack.json') + shutil.copy2(sidecar,out/'diagnostic-acpx-runtime-sidecar.cjs') + proof['diagnosticReconstruction']={'originalArtifactId':PIN['artifactId'],'originalZipSha256':PIN['artifactZipSha256'],'originalPackTarSha256':PIN['selectedFiles']['provider-pack.tar.gz']['sha256'],'replaceOnly':['dist/cli/acpx-runtime-sidecar.cjs','provider-pack.json'],'sidecarSha256':sha(out/'diagnostic-acpx-runtime-sidecar.cjs'),'manifestSha256':sha(out/'diagnostic-provider-pack.json'),'inventorySha256':sha(out/'diagnostic-pack-inventory.json'),'fullOriginalPackDuplicated':False,'diagnosticPackArchiveDuplicated':False} + test=out/'pi-closed-startup.test.mjs' + with tarfile.open(original/'source.tar','r:') as source: + member=source.getmember(PIN['testPath']);require(member.isfile(),'Original test is not regular');test.write_bytes(source.extractfile(member).read()) + require(sha(test)==PIN['testSha256'] and sha(HERE/'retain-test-state.mjs')==PIN['observerSha256'],'Test/observer changed');test.chmod(0o444);shutil.copy2(HERE/'retain-test-state.mjs',out/'retain-test-state.mjs') + retained=out/'retained-test-state';retained.mkdir(mode=0o700) + tenv={**env,'PAPERCLIP_TEST_PI_STARTUP_PACKAGE_ROOT':str(pack),'PAPERCLIP_TEST_PI_STARTUP_RUNNER_BINARY':str(daemon),'PI_INTEL_OWNED_TMP':str(scratch),'PI_INTEL_RETAINED_STATE':str(retained)} + owner=OwnedProcesses(out/'closed-startup-processes.json',scratch,pack,sidecar);active=DiagnosticChild(owner) + row={'label':'original-closed-startup-test-with-diagnostic-sidecar','argv':[str(node),'--import',str(HERE/'retain-test-state.mjs'),'--test',str(test)],'outerDeadlineSeconds':70,'originalTestTimeoutMs':60000,'originalAdmissionLimitMs':30000,'startedAt':datetime.datetime.now(datetime.timezone.utc).isoformat(),'startedMonotonicNs':str(time.monotonic_ns())};proof['commands'].append(row);proof['status']='diagnostic_running';save() + try: + with (out/'closed-startup.log').open('xb') as log: + result=active.execute(lambda:subprocess.Popen(row['argv'],env=tenv,cwd=pack,stdout=log,stderr=log,start_new_session=True),70) + finally: + row.update(finishedAt=datetime.datetime.now(datetime.timezone.utc).isoformat(),finishedMonotonicNs=str(time.monotonic_ns())) + row['supervisorElapsedMs']=round((int(row['finishedMonotonicNs'])-int(row['startedMonotonicNs']))/1e6,3) + proof['clockScope']='Supervisor uses Python monotonic_ns; sidecar uses Node hrtime.bigint. Compare elapsed spans; absolute cross-process clock equivalence is not asserted.' + save() + reports=list(retained.rglob('report.json')) + if len(reports)==1: + report=json.loads(reports[0].read_text());proof['originalTestReport']={k:report.get(k) for k in ['error','settledMs','durationMs']} + row.update(exitCode=result,logSha256=sha(out/'closed-startup.log'));proof['originalAssertionExitCode']=result;proof['shutdown']=owner.stop_result + reject_sink_failure([out/'closed-startup.log',*retained.rglob('runnerd.stderr.log')]) + proof['timing']=validate_timing_sink(sink,identity,set(patch['phases']));shutil.copy2(sink,out/'startup-timings.jsonl') + atomic_json(out/'timing-summary.json',proof['timing']) + require(closed_tree(pack)==after and sha(daemon)==PIN['daemonSha256'],'Diagnostic test mutated pack or daemon') + proof.update(status='diagnostic_complete_original_test_passed' if result==0 else 'diagnostic_complete_original_test_failed',isQualification=False,finishedAt=datetime.datetime.now(datetime.timezone.utc).isoformat());save() + return result + except BaseException as error: + proof.update(status='diagnostic_failed',errorType=type(error).__name__,error=str(error),finishedAt=datetime.datetime.now(datetime.timezone.utc).isoformat());raise + finally: + try: + if active is not None: + if active.child is not None:active.retire(active.child) + proof['cleanupUncertain']=active.uncertain + if (scratch/'startup-timings.jsonl').is_file() and not (out/'startup-timings.jsonl').exists(): + st=(scratch/'startup-timings.jsonl').lstat() + if stat.S_ISREG(st.st_mode) and st.st_nlink==1 and st.st_size<=32768:shutil.copyfile(scratch/'startup-timings.jsonl',out/'startup-timings.jsonl') + require(not proof['cleanupUncertain'],'Retain scratch after uncertain owned process cleanup') + st=scratch.lstat();require((st.st_dev,st.st_ino,st.st_uid)==scratch_id and stat.S_ISDIR(st.st_mode),'Scratch identity changed') + shutil.rmtree(scratch);proof['scratchRemoved']=True + except BaseException as cleanup_error: + proof.update(cleanupUncertain=True,cleanupError=type(cleanup_error).__name__+': '+str(cleanup_error));raise + finally:save() + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--archive',type=Path,required=True);p.add_argument('--output',type=Path,required=True) + raise SystemExit(execute(p.parse_args())) diff --git a/scripts/ci/pi-intel/startup_timing_patch.py b/scripts/ci/pi-intel/startup_timing_patch.py new file mode 100644 index 00000000000..f60ed9e4ce5 --- /dev/null +++ b/scripts/ci/pi-intel/startup_timing_patch.py @@ -0,0 +1,97 @@ +"""Sparse additive instrumentation of one pinned bundle; never edits vendor closure bytes.""" +import hashlib,json +from source_guard import require +ORIGINAL_SHA='97eedf930681729c6a1654e6c54bcc9657154e5dd2761f9922f4b01c7358f9f7' +PREFIX='__pcStartupDiagnostic' + +def patch_sidecar(original,sink): + require(hashlib.sha256(original).hexdigest()==ORIGINAL_SHA,'Diagnostic sidecar original digest mismatch') + require(set(sink)=={'path','dev','ino','uid'} and sink['path'].startswith('/private/tmp/pc-intel-diagnostic-') and sink['path'].endswith('/startup-timings.jsonl'),'Unexpected timing sink binding') + require(all(isinstance(sink[k],str) and sink[k].isdigit() for k in ['dev','ino','uid']),'Sink identity must use decimal strings') + text=original.decode();insertions=[];labels=[] + def mark(scope,anchor,label,after=False): + start=text.rindex(scope) if scope else 0 + end=text.find('\n// src/',start+len(scope)) if scope else len(text) + if end<0:end=len(text) + sub=text[start:end];require(sub.count(anchor)==1,'Missing/ambiguous timing anchor: '+label) + pos=start+sub.index(anchor)+(len(anchor) if after else 0) + insertions.append((pos,'\n'+PREFIX+'Mark('+json.dumps(label)+');\n'));labels.append(label) + mark('', '#!/usr/bin/env node\n','sidecar.entry',True) + mark('// src/cli/acpx-runtime-sidecar.ts','async function dispatch(request) {','sidecar.dispatch',True) + mark('// src/cli/acpx-runtime-sidecar.ts','if (request.command === "session.open") {','session.open.begin',True) + mark('// src/cli/acpx-runtime-sidecar.ts',' host = openedHost;','session.open.admitted') + mark('// src/cli/acpx-runtime-sidecar.ts',' const normalized = error51 instanceof Error ? error51 : new Error(String(error51));','sidecar.command.error') + mark('// src/cli/acpx-runtime-sidecar.ts','function requestShutdown(reason) {','sidecar.shutdown.request',True) + mark('// src/drivers/acpx/runtime-host.ts',' static async open(options, dependencies) {','host.open.begin',True) + mark('// src/drivers/acpx/runtime-host.ts',' const installation = await runAbortableAdmissionStage(','installation.begin') + mark('// src/drivers/acpx/runtime-host.ts',' if (installation.commandDigest !== profile.commandDigest) {','installation.end') + mark('// src/drivers/acpx/runtime-host.ts',' const sandbox = await runAbortableAdmissionStage(','sandbox.begin') + mark('// src/drivers/acpx/runtime-host.ts',' assertAcpxProfileEnvironment(options.agent, sandbox.launchEnvironment);','sandbox.lifetime.end') + mark('// src/drivers/acpx/runtime-host.ts',' command = await acquireAbortableAdmissionResource({','command.acquire.begin') + mark('// src/drivers/acpx/runtime-host.ts',' const commandOwner = createAcpxCommandLeaseOwner(','command.acquire.end') + mark('// src/drivers/acpx/runtime-host.ts',' runtime = await acquireAbortableAdmissionResource({','runtime.acquire.begin') + mark('// src/drivers/acpx/pi-installation.ts','async function verifyPiInstallation(profile) {','pi.install.begin',True) + mark('// src/drivers/acpx/pi-installation.ts',' const verified = await verifyPiRuntimeManifest(runtimeRoot, manifest);','pi.manifest.begin') + mark('// src/drivers/acpx/pi-installation.ts',' if (verified.manifestDigest !== metadata.manifestDigest)','pi.manifest.end') + mark('// src/drivers/acpx/pi-installation.ts',' return Object.freeze({','pi.install.end') + mark('// src/drivers/acpx/pi-verified-runtime.ts','async function inventoryPiRuntimeFiles(root) {','pi.discovery.begin',True) + mark('// src/drivers/acpx/pi-verified-runtime.ts',' await visit(physicalRoot);','pi.discovery.end',True) + mark('// src/drivers/acpx/pi-verified-runtime.ts',' for (let index = 0; index < regular.length;','pi.hash.begin') + mark('// src/drivers/acpx/pi-verified-runtime.ts',' return files;','pi.hash.end') + mark('// src/drivers/acpx/native-distribution-integrity.ts','async function createNativeAcpxDistributionSnapshot(input, entries) {','snapshot.begin',True) + mark('// src/drivers/acpx/native-distribution-integrity.ts',' const copyEntry = async (entry) => {','snapshot.directories.end') + mark('// src/drivers/acpx/native-distribution-integrity.ts',' for (let start = 0; start < entries.length; ) {','snapshot.copy.begin') + mark('// src/drivers/acpx/native-distribution-integrity.ts',' const executable = (0, import_node_path5.join)(packageRoot, ...input.executable.split("/"));','snapshot.copy.end') + mark('// src/drivers/acpx/native-distribution-integrity.ts',' await directoryBatch([...directories], (path3) => (0, import_promises6.chmod)(path3, 320));','snapshot.seal.begin') + mark('// src/drivers/acpx/native-distribution-integrity.ts',' return { commandDirectory, bootstrap, snapshot:','snapshot.seal.end') + mark('// src/drivers/acpx/installation-integrity.ts',' const lease = commandLease(','command.lease.begin') + mark('// src/drivers/acpx/installation-integrity.ts',' return {\n spawn(args = [], options = {}, lifetime) {','command.lease.end') + mark('// src/drivers/acpx/installation-integrity.ts',' if (consumed) throw new Error("Verified ACPX command lease is closed");','command.spawn.begin') + mark('// src/drivers/acpx/installation-integrity.ts',' const runtimeHandoff = verifiedRuntimeExecutableHandoff(runtimeTargetFd);','runtime.handoff.begin') + mark('// src/drivers/acpx/installation-integrity.ts',' const environment = sanitizedNodeEnvironment(options.env);','runtime.handoff.end') + mark('// src/drivers/acpx/installation-integrity.ts',' child = (0, import_node_child_process2.spawn)(','command.os_spawn.begin') + mark('// src/drivers/acpx/installation-integrity.ts',' if (guarded) {\n const guardianOwnerPipe','command.os_spawn.end') + mark('// src/drivers/acpx/codex-runtime-adapter.ts',' onAgentInitialize: (result) => {','wrapper.initialize.ack',True) + mark('// src/drivers/acpx/codex-runtime-adapter.ts',' commandLaunches.count += 1;','runtime.spawn.callback') + mark('// src/drivers/acpx/codex-runtime-adapter.ts',' const ensuredSession = Promise.resolve().then(','runtime.ensure.begin') + mark('// src/drivers/acpx/codex-runtime-adapter.ts',' cursorInstructions?.assertReady();','runtime.ensure.settled') + # Catch-all standalone function boundaries relevant to verified-runtime preparation. + for name in ['verifyAcpxProfileInstallation','verifyQualifiedRuntimeExecutable','openVerifiedRuntimeExecutable','openVerifiedCommandDirectory','createAcpxPrivateSnapshot','acquireAcpxProviderLifetimeLease']: + anchor='async function '+name+'(' + start=text.index(anchor);body=text.index(' {\n',start)+3 + label='entry.'+name + insertions.append((body,'\n'+PREFIX+'Mark('+json.dumps(label)+');\n'));labels.append(label) + require(len(labels)==len(set(labels)),'Duplicate diagnostic phase') + prelude=''' +// Explicit diagnostic derivative: bounded phase-only sink, no runtime policy changes. +const __pcStartupDiagnosticFs = require("node:fs"); +const __pcStartupDiagnosticBinding = SINK_BINDING; +const __pcStartupDiagnosticAllowed = new Set(ALLOWED_PHASES); +let __pcStartupDiagnosticFd = null, __pcStartupDiagnosticRows = 0, __pcStartupDiagnosticBytes = 0, __pcStartupDiagnosticFailed = false; +function __pcStartupDiagnosticMark(phase) { + if (__pcStartupDiagnosticFailed) return; + try { + if (!__pcStartupDiagnosticAllowed.has(phase) || ++__pcStartupDiagnosticRows > 128) throw new Error("diagnostic bound"); + if (__pcStartupDiagnosticFd === null) __pcStartupDiagnosticFd = __pcStartupDiagnosticFs.openSync(__pcStartupDiagnosticBinding.path, __pcStartupDiagnosticFs.constants.O_WRONLY | __pcStartupDiagnosticFs.constants.O_APPEND | __pcStartupDiagnosticFs.constants.O_NOFOLLOW); + const st = __pcStartupDiagnosticFs.fstatSync(__pcStartupDiagnosticFd, {bigint:true}); + if (!st.isFile() || st.nlink !== 1n || (st.mode & 511n) !== 384n || st.dev !== BigInt(__pcStartupDiagnosticBinding.dev) || st.ino !== BigInt(__pcStartupDiagnosticBinding.ino) || st.uid !== BigInt(__pcStartupDiagnosticBinding.uid)) throw new Error("diagnostic identity"); + const line = JSON.stringify({phase, ns:process.hrtime.bigint().toString()}) + "\\n"; + if ((__pcStartupDiagnosticBytes += Buffer.byteLength(line)) > 32768 || st.size + BigInt(Buffer.byteLength(line)) > 32768n) throw new Error("diagnostic bound"); + __pcStartupDiagnosticFs.writeSync(__pcStartupDiagnosticFd,line); + __pcStartupDiagnosticFs.writeSync(2,"PC_STARTUP_TIMING " + line); + } catch { + __pcStartupDiagnosticFailed = true; + try { __pcStartupDiagnosticFs.writeSync(2,"PC_STARTUP_DIAGNOSTIC_SINK_FAILED\\n"); } catch {} + } +} +process.once("exit", () => __pcStartupDiagnosticMark("sidecar.exit")); +'''.replace('SINK_BINDING',json.dumps(sink)).replace('ALLOWED_PHASES',json.dumps(labels+['sidecar.exit'])) + # Prelude must precede the first mark while preserving the original shebang. + insertions.append((text.index('\n')+1,prelude)) + # Stable ordering at the shared header offset puts prelude before entry mark. + ordered=sorted(enumerate(insertions),key=lambda x:(x[1][0],0 if x[1][1]==prelude else 1,x[0])) + result=[];at=0;patch=[] + for _,(pos,value) in ordered: + result.append(text[at:pos]);result.append(value);patch.append({'offset':pos,'inserted':value});at=pos + result.append(text[at:]);modified=''.join(result).encode() + return modified,{'schema':'paperclip.startup-timing-additive-patch/v1','originalSha256':ORIGINAL_SHA,'diagnosticSha256':hashlib.sha256(modified).hexdigest(),'sinkBinding':sink,'phases':labels+['sidecar.exit'],'insertions':patch,'onlyAdditions':True,'vendorClosureChanged':False} diff --git a/scripts/ci/pi-intel/test_startup_diagnostic.py b/scripts/ci/pi-intel/test_startup_diagnostic.py new file mode 100644 index 00000000000..009ceea1056 --- /dev/null +++ b/scripts/ci/pi-intel/test_startup_diagnostic.py @@ -0,0 +1,127 @@ +import copy,hashlib,json,os,subprocess,tempfile,unittest +from pathlib import Path +from startup_timing_patch import patch_sidecar,ORIGINAL_SHA +from startup_diagnostic import validate_pack_delta,validate_timing_sink,reject_sink_failure +from diagnostic_lifecycle import DiagnosticChild + +class FakeProcess: + pid=111;returncode=0 + def __init__(self,fail_wait=False):self.fail_wait=fail_wait + def poll(self):return 0 + def wait(self,timeout): + if self.fail_wait:raise RuntimeError('wait failed') + return 0 +class FakeOwner: + def __init__(self,fail_stop=False):self.fail_stop=fail_stop;self.stops=0 + def observe(self,pid):pass + def stop(self,pid): + self.stops+=1 + if self.fail_stop:raise RuntimeError('stop failed') + return {'status':'stopped'} + def live(self):return set() + +class DiagnosticTests(unittest.TestCase): + @classmethod + def setUpClass(cls):cls.original=Path(os.environ['PI_DIAGNOSTIC_ORIGINAL_SIDECAR']).read_bytes() + def binding(self,root): + p=root/'startup-timings.jsonl';p.touch(mode=0o600);s=p.stat() + return p,{'path':str(p),**{k:str(getattr(s,'st_'+k)) for k in ['dev','ino','uid']}} + def test_patch_is_exact_additions_only(self): + sink={'path':'/private/tmp/pc-intel-diagnostic-fixture/startup-timings.jsonl','dev':'1','ino':'2','uid':'501'} + patched,proof=patch_sidecar(self.original,sink);text=patched.decode();shift=0 + for change in proof['insertions']: + at=change['offset']+shift;self.assertEqual(text[at:at+len(change['inserted'])],change['inserted']);shift+=len(change['inserted']) + for change in reversed(proof['insertions']): + shift-=len(change['inserted']);at=change['offset']+shift;text=text[:at]+text[at+len(change['inserted']):] + self.assertEqual(text.encode(),self.original);self.assertEqual(hashlib.sha256(text.encode()).hexdigest(),ORIGINAL_SHA) + self.assertEqual(len(proof['phases']),46) + def test_wrong_original_rejected(self): + with self.assertRaises(RuntimeError):patch_sidecar(self.original+b' ',{'path':'unused','dev':'1','ino':'2','uid':'3'}) + def test_numeric_inode_rejected(self): + with self.assertRaises(RuntimeError):patch_sidecar(self.original,{'path':'/private/tmp/pc-intel-diagnostic-x/startup-timings.jsonl','dev':1,'ino':2,'uid':3}) + def run_prelude(self,extra='',identity_mutation=None): + with tempfile.TemporaryDirectory(prefix='pc-intel-diagnostic-',dir='/private/tmp') as tmp: + root=Path(tmp);sink,binding=self.binding(root) + if identity_mutation:identity_mutation(binding) + _,proof=patch_sidecar(self.original,binding);prelude=next(r['inserted'] for r in proof['insertions'] if '// Explicit diagnostic derivative' in r['inserted']) + script=root/'fixture.cjs';script.write_text(prelude+'\n__pcStartupDiagnosticMark("sidecar.entry");\n'+extra+'\n__pcStartupDiagnosticMark("session.open.begin");\n') + run=subprocess.run([os.environ['PI_DIAGNOSTIC_TEST_NODE'],script],env={'PATH':'/usr/bin:/bin'},capture_output=True,text=True,timeout=10) + self.assertEqual(run.returncode,0) + if identity_mutation or extra: + with self.assertRaises(RuntimeError):validate_timing_sink(sink,{k:str(getattr(sink.stat(),'st_'+k)) for k in ['dev','ino','uid']},set(proof['phases'])) + else: + rows=validate_timing_sink(sink,binding,set(proof['phases']))['rows'];self.assertEqual([r['phase'] for r in rows],['sidecar.entry','session.open.begin','sidecar.exit']) + return run.stderr,sink.stat().st_size + def test_sink_exact_identity_and_terminal(self):self.run_prelude() + def test_sink_wrong_inode_is_failure(self):self.assertIn('SINK_FAILED',self.run_prelude(identity_mutation=lambda b:b.update(ino='0'))[0]) + def test_sink_late_mode_change_prevents_complete_diagnosis(self):self.assertIn('SINK_FAILED',self.run_prelude('__pcStartupDiagnosticFs.chmodSync(__pcStartupDiagnosticBinding.path,0o400);')[0]) + def test_sink_existing_near_cap_cannot_overflow(self): + stderr,size=self.run_prelude('__pcStartupDiagnosticFs.appendFileSync(__pcStartupDiagnosticBinding.path,"x".repeat(32760-__pcStartupDiagnosticFs.statSync(__pcStartupDiagnosticBinding.path).size));') + self.assertIn('SINK_FAILED',stderr);self.assertLessEqual(size,32768) + def test_sink_row_cap_prevents_complete_diagnosis(self):self.assertIn('SINK_FAILED',self.run_prelude('for(let n=0;n<130;n++)__pcStartupDiagnosticMark("sidecar.dispatch");')[0]) + def test_sink_parser_rejects_bad_rows(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);p,b=self.binding(root) + for rows in [[{'phase':'sidecar.entry','ns':'2'},{'phase':'sidecar.exit','ns':'1'}],[{'phase':'unknown','ns':'1'}],[{'phase':'sidecar.entry','ns':'1','path':'secret'}]]: + p.write_text(''.join(json.dumps(r)+'\n' for r in rows)) + with self.assertRaises(RuntimeError):validate_timing_sink(p,b,{'sidecar.entry','sidecar.exit'}) + def test_explicit_failure_marker_rejects_even_with_valid_terminal(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);p,b=self.binding(root) + p.write_text(''.join(json.dumps({'phase':phase,'ns':str(i)})+'\n' for i,phase in enumerate(['sidecar.entry','session.open.begin','sidecar.exit']))) + self.assertTrue(validate_timing_sink(p,b,{'sidecar.entry','session.open.begin','sidecar.exit'})['completeSink']) + log=root/'runnerd.stderr.log';log.write_text('PC_STARTUP_DIAGNOSTIC_SINK_FAILED\n') + with self.assertRaises(RuntimeError):reject_sink_failure([log]) + def test_only_sidecar_and_manifest_delta(self): + before=[{'path':p,'kind':'file','mode':0o644,'sha256':'a','size':1} for p in ['dist/cli/acpx-runtime-sidecar.cjs','provider-pack.json','vendor']] + after=copy.deepcopy(before) + for row in after[:2]:row['sha256']='b' + self.assertEqual(len(validate_pack_delta(before,after)),2) + after[2]['sha256']='b' + with self.assertRaises(RuntimeError):validate_pack_delta(before,after) + def test_complete_small_evidence_admitted_with_diagnostic_cap(self): + from admit_evidence import admit_evidence + from admit_diagnostic_evidence import MAX_DIAGNOSTIC_BYTES + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);(root/'unique-sidecar').write_bytes(b'unique') + result=admit_evidence(root,MAX_DIAGNOSTIC_BYTES) + self.assertEqual(result['status'],'admitted_complete_evidence');self.assertEqual(result['maximumBytes'],268435456);self.assertEqual(result['retentionDays'],7) + self.assertEqual((root/'unique-sidecar').read_bytes(),b'unique') + def test_oversized_diagnostic_evidence_rejected_without_truncation(self): + from admit_evidence import admit_evidence + from admit_diagnostic_evidence import MAX_DIAGNOSTIC_BYTES + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);p=root/'oversized-unique-evidence' + with p.open('wb') as f:f.truncate(MAX_DIAGNOSTIC_BYTES+1) + result=admit_evidence(root,MAX_DIAGNOSTIC_BYTES) + self.assertEqual(result['status'],'rejected_storage_cap');self.assertFalse(result['truncated']);self.assertEqual(p.stat().st_size,MAX_DIAGNOSTIC_BYTES+1) + def test_cancel_before_spawn(self): + c=DiagnosticChild(FakeOwner());c.cancel('before');calls=[] + with self.assertRaises(RuntimeError):c.execute(lambda:calls.append(True),1) + self.assertEqual(calls,[]);self.assertFalse(c.uncertain) + def test_cancel_during_spawn_publishes_then_retires(self): + owner=FakeOwner();c=DiagnosticChild(owner);p=FakeProcess() + def factory():c.cancel('during');return p + with self.assertRaises(RuntimeError):c.execute(factory,1) + self.assertIs(c.child,p);self.assertTrue(c.retired);self.assertEqual(owner.stops,1) + def test_cancel_after_publication_retires_once(self): + owner=FakeOwner();c=DiagnosticChild(owner) + owner.observe=lambda pid:c.cancel('published') + with self.assertRaises(RuntimeError):c.execute(lambda:FakeProcess(),1) + self.assertTrue(c.retired);self.assertEqual(owner.stops,1) + def test_stop_exception_sticky_and_no_second_launch(self): + c=DiagnosticChild(FakeOwner(fail_stop=True)) + with self.assertRaises(RuntimeError):c.execute(lambda:FakeProcess(),1) + self.assertTrue(c.uncertain) + with self.assertRaises(RuntimeError):c.execute(lambda:self.fail('second launch'),1) + def test_wait_exception_sticky(self): + c=DiagnosticChild(FakeOwner()) + with self.assertRaises(RuntimeError):c.execute(lambda:FakeProcess(fail_wait=True),1) + self.assertTrue(c.uncertain) + def test_factory_exception_uncertain(self): + c=DiagnosticChild(FakeOwner()) + def factory():raise OSError('spawn uncertainty') + with self.assertRaises(OSError):c.execute(factory,1) + self.assertTrue(c.uncertain) + +if __name__=='__main__':unittest.main()