Merge verified tracing diagnostics and portable inventory coverage

Preserve hosted first-attempt admission and UI review repairs while incorporating the final reviewer recommendations.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -15,6 +15,7 @@ const { attemptDirectory, runControlledEval } = await import(new URL('./run.mjs'
 const { summarizeControlledCampaign } = await import(new URL('./report.mjs', runtime));
 const { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } = await import(new URL('./archive.mjs', runtime));
 const { readRegularFile, replaceEvidenceFile } = await import(new URL('./evidence-files.mjs', runtime));
+
 import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs';
 
 const childScript = options => `
@@ -22,11 +23,12 @@ import assert from 'node:assert/strict';
 import {createHash} from 'node:crypto';
 import {spawn} from 'node:child_process';
 import {once} from 'node:events';
-import {readFile,writeFile,rm,readdir,symlink,truncate} from 'node:fs/promises';
+import {readFile,writeFile,rm,readdir,symlink,truncate,mkdir} from 'node:fs/promises';
 import {dirname,join} from 'node:path';
 import {execFileSync} from 'node:child_process';
 const args=new Map();for(let i=2;i<process.argv.length;i+=2)args.set(process.argv[i],process.argv[i+1]);
 const request=JSON.parse(await readFile(args.get('--request'),'utf8'));
+if(${Boolean(options.failChild)})process.exit(3);
 if(${Boolean(options.resistantDescendant)}) {
  const orphan=${Boolean(options.orphanExit)};
  const descendant=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});"+(orphan?"process.send('ready');":"process.stdout.write('ready');")+"setInterval(()=>{},1000)"],{stdio:orphan?['ignore','inherit','inherit','ipc']:['ignore','pipe','ignore']});
@@ -51,6 +53,19 @@ if(args.has('--observability-config')) {
   await response.arrayBuffer();
  }
  nativeObservability={mode:'controller_otlp',configured:true,verification:'unverified',projectId:config.projectId,traceId:config.traceId,spanId:config.spanId,descriptorSha256:createHash('sha256').update(raw).digest('hex')};
+ if(${Boolean(options.realObservabilityValidator)}) {
+  const {createRequire}=await import('node:module');
+  const require=createRequire(${JSON.stringify(fileURLToPath(new URL('../../../../server/package.json', import.meta.url)))});
+  const {tsImport}=await import(require.resolve('tsx/esm/api'));
+  const {prepareEvalObservabilityRuntime}=await tsImport(${JSON.stringify(fileURLToPath(new URL('../../src/cli/eval-observability-runtime.ts', import.meta.url)))},import.meta.url);
+  await mkdir(process.env.HOME+'/codex',{recursive:true});
+  const prepared=await prepareEvalObservabilityRuntime(args.get('--observability-config'),request,
+   {evidence:{campaignId:descriptor.campaignId,executionId:descriptor.executionId},transportOptions:{environment:{CODEX_HOME:process.env.HOME+'/codex'}}},process.env);
+  assert.deepEqual(prepared.evidence,nativeObservability);
+  for(const signal of ['LOGS','TRACES'])assert(prepared.transportOptions.environment['OTEL_EXPORTER_OTLP_'+signal+'_HEADERS'].includes(encodeURIComponent('Bearer '+process.env.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN)));
+  nativeObservability=prepared.evidence;
+ }
+
 }
 if(${Boolean(options.waitForever)})await new Promise(()=>setInterval(()=>{},1000));
 if(!${Boolean(options.missingArtifact)})await writeFile(args.get('--output'),JSON.stringify({schema:'paperclip-runner/eval-session-artifact/v1',attemptId:request.attemptId,requestedModel:request.model,
@@ -831,3 +846,38 @@ test('failed final OTLP receipt write returns its saved incomplete report', asyn
   const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8'));
   await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs'));
 });
+
+
+for (const providerProfile of ['codex', 'claude']) test(`controller descriptor round trips through the real ${providerProfile} CLI validator`, async t => {
+  const f = await fixture(t, { providerProfile, observability: true, realObservabilityValidator: true, requests: 0 });
+  const result = await f.run();
+  assert.equal(result.code, 0, JSON.stringify(result.report) + '\n' + await readFile(join(attemptDirectory(f.manifest, 'attempt-0'), 'stderr.log'), 'utf8'));
+  assert.equal(result.report.observability.status, 'accepted');
+  const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' });
+  assert.equal((await summarizeControlledEvidence([bundle])).incompleteAttempts, 0);
+});
+
+test('retained failure codes distinguish native startup, child exit and invalid artifacts', async t => {
+  const startup = await fixture(t, { observability: true, requests: 0 });
+  const root = attemptDirectory(startup.manifest, 'attempt-0');
+  const probe = await open(join(startup.root, 'startup-probe'), 'wx');
+  const prototype = Object.getPrototypeOf(probe), original = prototype.sync; await probe.close();
+  let failed = false;
+  prototype.sync = async function (...args) {
+    const native = await stat(join(root, 'otlp-native.jsonl')).catch(() => null);
+    if (native && (await this.stat()).ino === native.ino) {
+      failed = true; throw Object.assign(new Error('synthetic native startup persistence failure'), { code: 'ENOSPC' });
+    }
+    return original.call(this, ...args);
+  };
+  t.after(() => { prototype.sync = original; });
+  const result = await startup.run();
+  prototype.sync = original;
+  assert.equal(failed, true); assert.equal(result.code, 2);
+  assert.equal(result.report.failure, 'observability_start_failed');
+  assert.equal((await summarizeControlledCampaign(startup.manifest)).committedUsd, '0.014000000');
+  const child = await fixture(t, { failChild: true, requests: 0 });
+  assert.equal((await child.run()).report.failure, 'child_failed');
+  const artifact = await fixture(t, { missingRoute: true, requests: 0 });
+  assert.equal((await artifact.run()).report.failure, 'artifact_invalid');
+});

packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs

@@ -1,6 +1,7 @@
 import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
 import { execFile } from 'node:child_process';
-import { mkdtemp, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises';
+import { mkdtemp, mkdir, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
@@ -13,6 +14,7 @@ const exec = promisify(execFile);
 const source = dirname(fileURLToPath(import.meta.url));
 const isolatedProbe = `
 import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
 import { registerHooks, isBuiltin, createRequire } from 'node:module';
 import { realpath } from 'node:fs/promises';
 import { pathToFileURL } from 'node:url';
@@ -101,3 +103,31 @@ test('copied controller runs without source dependencies, retains native evidenc
   await rm(extra);
   assert.equal((await verifyControllerRuntime(input)).receiptSha256, receipt.receiptSha256);
 });
+
+
+test('only the root runtime receipt is excluded from the verified file inventory', async t => {
+  const root = await mkdtemp(join(tmpdir(), 'runner-runtime-nested-receipt-'));
+  t.after(() => rm(root, { recursive: true, force: true }));
+  const sourceRevision = 'a'.repeat(40), files = {};
+  const digest = bytes => createHash('sha256').update(bytes).digest('hex');
+  let payloadBytes = 0;
+  for (const name of ['controlled-evals/run.mjs', 'controlled-evals/archive.mjs', 'controlled-evals/runtime.mjs', 'dependency/runtime.json']) {
+    const bytes = Buffer.from(name.endsWith('.json') ? '{"dependency":true}' : 'export {};');
+    await mkdir(dirname(join(root, name)), { recursive: true });
+    await writeFile(join(root, name), bytes);
+    files[name] = { sizeBytes: bytes.length, sha256: digest(bytes) }; payloadBytes += bytes.length;
+  }
+  const receipt = encodeRuntimeReceipt({ schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, payloadBytes, files });
+  await writeFile(join(root, 'runtime.json'), receipt);
+  const input = { directory: root, sourceRevision, receiptSha256: digest(receipt) };
+  assert.equal((await verifyControllerRuntime(input)).files, 5);
+  const nested = join(root, 'dependency/runtime.json'), original = await readFile(nested);
+  await writeFile(nested, Buffer.from('{"dependency":fake}'));
+  await assert.rejects(verifyControllerRuntime(input), /content mismatch|oversized/);
+  await writeFile(nested, original);
+  await mkdir(join(root, 'dependency/deeper'));
+  await writeFile(join(root, 'dependency/deeper/runtime.json'), '{}');
+  await assert.rejects(verifyControllerRuntime(input), /Unexpected runtime artifact file/);
+  await rm(join(root, 'dependency/deeper'), { recursive: true });
+  assert.equal((await verifyControllerRuntime(input)).files, 5);
+});

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -122,11 +122,13 @@ registerHooks({ resolve(specifier, context, nextResolve) {
   const ledgerPath = join(root, 'ledger.jsonl');
   const ledger = await costRuntime.openEvalCostLedger(ledgerPath, limits);
   let gateway, observability, observabilityEvidence, child, artifact, failure = null, cleanupVerified = false, secretExcluded = true;
-  let providerArtifactSha256;
+  let providerArtifactSha256, failureStage = 'controlled_attempt_incomplete';
   const secrets = [...new Set([key, observabilityKey].filter(Boolean))];
   const redact = raw => secrets.reduce((text, secret) => text.split(secret).join('[redacted]'), raw);
   try {
+    failureStage = 'request_admission_failed';
     await ledger.admitAttempt(attempt.executionId, attempt.attemptId);
+    failureStage = 'gateway_start_failed';
     gateway = await costRuntime.startEvalGateway({ ledger, identity: { executionId: attempt.executionId, attemptId: attempt.attemptId, phase: 'agent' },
       route: { ...model, mode: manifest.route, projectId: manifest.projectId, endpointName: provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY', apiKey: key },
       deadlineAt, transport, persist: receipt => atomicJson(join(root, 'receipts', `${receipt.requestId}.json`), receipt),
@@ -135,20 +137,25 @@ registerHooks({ resolve(specifier, context, nextResolve) {
     const descriptor = { schema: 'paperclip.runner-eval.gateway/v1', provider, model: attempt.model, baseUrl: gateway.baseUrl,
       campaignId: manifest.campaignId, executionId: attempt.executionId, attemptId: attempt.attemptId, upstreamRoute: manifest.route };
     const descriptorPath = join(root, 'gateway.json'), descriptorRaw = JSON.stringify(descriptor) + '\n';
+    failureStage = 'gateway_descriptor_failed';
     await writeFile(descriptorPath, descriptorRaw, { flag: 'wx', mode: 0o600 });
     const childEnvironment = Object.fromEntries(['PATH', 'LANG', 'LANGUAGE', 'LC_ALL', 'TZ'].flatMap(name => environment[name] === undefined ? [] : [[name, environment[name]]]));
     Object.assign(childEnvironment, { HOME: join(root, 'home'), TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: gateway.token });
     if (manifest.observability) {
+      failureStage = 'observability_start_failed';
       observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport, signal });
       secrets.push(...observability.secrets);
       childEnvironment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN = observability.token;
     }
     const artifactPath = join(root, 'provider-artifact.json');
+    failureStage = 'child_failed';
     signal?.throwIfAborted();
     if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn');
     child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, observabilityPath: observability?.descriptorPath,
       cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => Promise.all([gateway.close(), observability?.close()]) });
+    failureStage = 'output_retention_failed';
     for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 });
+    failureStage = child.code === 0 ? 'artifact_invalid' : 'child_failed';
     const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8');
     secretExcluded = secrets.every(secret => !raw.includes(secret));
     if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw));
@@ -158,6 +165,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
       executionId: attempt.executionId, attemptId: attempt.attemptId, descriptorSha256: sha256(descriptorRaw) };
     if (canonical(artifact.requestRouting) !== canonical(expected) || artifact.attemptId !== attempt.attemptId || artifact.requestedModel !== attempt.model) throw new Error('Provider did not retain the assigned route');
     if (observability && canonical(artifact.nativeObservability) !== canonical(observability.expectedEvidence)) throw new Error('Provider did not retain the assigned observability route');
+    failureStage = 'provider_cleanup_or_credential_unverified';
     const processInfo = artifact.snapshot?.process;
     const pids = [...new Set([child.pid, processInfo?.runnerPid, processInfo?.providerPid, processInfo?.sidecarPid, processInfo?.agentPid].filter(pid => Number.isSafeInteger(pid) && pid > 0))];
     cleanupVerified = !child.stopped && artifact.snapshot?.status === 'closed' && processInfo?.runnerExited === true
@@ -165,7 +173,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
       && Number.isSafeInteger(processInfo.providerPid) && processInfo.providerPid > 0 && pids.every(absent);
     if (!cleanupVerified || !secretExcluded) throw new Error('Controlled provider cleanup or credential exclusion is unverified');
   } catch {
-    failure = 'controlled_attempt_incomplete';
+    failure = failureStage;
   } finally {
     try { await gateway?.close(); } catch { failure = 'gateway_cleanup_unverified'; }
     try { await ledger.close(); } catch { failure = 'ledger_close_failed'; }