diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index e7f4dbb8ea5..6b374dfea81 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -5,6 +5,7 @@ import { mkdtemp, mkdir, open, readFile, readdir, rm, stat, symlink, truncate, w import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; import { attemptDirectory, runControlledEval } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; @@ -17,11 +18,12 @@ import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; import {spawn} from 'node:child_process'; import {once} from 'node:events'; -import {readFile,writeFile,rm,readdir,symlink,truncate} from 'node:fs/promises'; +import {readFile,writeFile,rm,readdir,symlink,truncate,mkdir} from 'node:fs/promises'; import {dirname,join} from 'node:path'; import {execFileSync} from 'node:child_process'; const args=new Map();for(let i=2;i{});"+(orphan?"process.send('ready');":"process.stdout.write('ready');")+"setInterval(()=>{},1000)"],{stdio:orphan?['ignore','inherit','inherit','ipc']:['ignore','pipe','ignore']}); @@ -46,6 +48,19 @@ if(args.has('--observability-config')) { await response.arrayBuffer(); } nativeObservability={mode:'controller_otlp',configured:true,verification:'unverified',projectId:config.projectId,traceId:config.traceId,spanId:config.spanId,descriptorSha256:createHash('sha256').update(raw).digest('hex')}; + if(${Boolean(options.realObservabilityValidator)}) { + const {createRequire}=await import('node:module'); + const require=createRequire(${JSON.stringify(fileURLToPath(new URL('../../../../server/package.json', import.meta.url)))}); + const {tsImport}=await import(require.resolve('tsx/esm/api')); + const {prepareEvalObservabilityRuntime}=await tsImport(${JSON.stringify(fileURLToPath(new URL('../../src/cli/eval-observability-runtime.ts', import.meta.url)))},import.meta.url); + await mkdir(process.env.HOME+'/codex',{recursive:true}); + const prepared=await prepareEvalObservabilityRuntime(args.get('--observability-config'),request, + {evidence:{campaignId:descriptor.campaignId,executionId:descriptor.executionId},transportOptions:{environment:{CODEX_HOME:process.env.HOME+'/codex'}}},process.env); + assert.deepEqual(prepared.evidence,nativeObservability); + for(const signal of ['LOGS','TRACES'])assert(prepared.transportOptions.environment['OTEL_EXPORTER_OTLP_'+signal+'_HEADERS'].includes(encodeURIComponent('Bearer '+process.env.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN))); + nativeObservability=prepared.evidence; + } + } if(${Boolean(options.waitForever)})await new Promise(()=>setInterval(()=>{},1000)); if(!${Boolean(options.missingArtifact)})await writeFile(args.get('--output'),JSON.stringify({schema:'paperclip-runner/eval-session-artifact/v1',attemptId:request.attemptId,requestedModel:request.model, @@ -765,3 +780,38 @@ test('failed final OTLP receipt write returns its saved incomplete report', asyn const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8')); await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs')); }); + + +for (const providerProfile of ['codex', 'claude']) test(`controller descriptor round trips through the real ${providerProfile} CLI validator`, async t => { + const f = await fixture(t, { providerProfile, observability: true, realObservabilityValidator: true, requests: 0 }); + const result = await f.run(); + assert.equal(result.code, 0, JSON.stringify(result.report) + '\n' + await readFile(join(attemptDirectory(f.manifest, 'attempt-0'), 'stderr.log'), 'utf8')); + assert.equal(result.report.observability.status, 'accepted'); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0' }); + assert.equal((await summarizeControlledEvidence([bundle])).incompleteAttempts, 0); +}); + +test('retained failure codes distinguish native startup, child exit and invalid artifacts', async t => { + const startup = await fixture(t, { observability: true, requests: 0 }); + const root = attemptDirectory(startup.manifest, 'attempt-0'); + const probe = await open(join(startup.root, 'startup-probe'), 'wx'); + const prototype = Object.getPrototypeOf(probe), original = prototype.sync; await probe.close(); + let failed = false; + prototype.sync = async function (...args) { + const native = await stat(join(root, 'otlp-native.jsonl')).catch(() => null); + if (native && (await this.stat()).ino === native.ino) { + failed = true; throw Object.assign(new Error('synthetic native startup persistence failure'), { code: 'ENOSPC' }); + } + return original.call(this, ...args); + }; + t.after(() => { prototype.sync = original; }); + const result = await startup.run(); + prototype.sync = original; + assert.equal(failed, true); assert.equal(result.code, 2); + assert.equal(result.report.failure, 'observability_start_failed'); + assert.equal((await summarizeControlledCampaign(startup.manifest)).committedUsd, '0.014000000'); + const child = await fixture(t, { failChild: true, requests: 0 }); + assert.equal((await child.run()).report.failure, 'child_failed'); + const artifact = await fixture(t, { missingRoute: true, requests: 0 }); + assert.equal((await artifact.run()).report.failure, 'artifact_invalid'); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 8b503c79c20..5389b2dba23 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -120,11 +120,13 @@ registerHooks({ resolve(specifier, context, nextResolve) { const ledgerPath = join(root, 'ledger.jsonl'); const ledger = await costRuntime.openEvalCostLedger(ledgerPath, limits); let gateway, observability, observabilityEvidence, child, artifact, failure = null, cleanupVerified = false, secretExcluded = true; - let providerArtifactSha256; + let providerArtifactSha256, failureStage = 'controlled_attempt_incomplete'; const secrets = [...new Set([key, observabilityKey].filter(Boolean))]; const redact = raw => secrets.reduce((text, secret) => text.split(secret).join('[redacted]'), raw); try { + failureStage = 'request_admission_failed'; await ledger.admitAttempt(attempt.executionId, attempt.attemptId); + failureStage = 'gateway_start_failed'; gateway = await costRuntime.startEvalGateway({ ledger, identity: { executionId: attempt.executionId, attemptId: attempt.attemptId, phase: 'agent' }, route: { ...model, mode: manifest.route, projectId: manifest.projectId, endpointName: provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY', apiKey: key }, deadlineAt, transport, persist: receipt => atomicJson(join(root, 'receipts', `${receipt.requestId}.json`), receipt), @@ -133,20 +135,25 @@ registerHooks({ resolve(specifier, context, nextResolve) { const descriptor = { schema: 'paperclip.runner-eval.gateway/v1', provider, model: attempt.model, baseUrl: gateway.baseUrl, campaignId: manifest.campaignId, executionId: attempt.executionId, attemptId: attempt.attemptId, upstreamRoute: manifest.route }; const descriptorPath = join(root, 'gateway.json'), descriptorRaw = JSON.stringify(descriptor) + '\n'; + failureStage = 'gateway_descriptor_failed'; await writeFile(descriptorPath, descriptorRaw, { flag: 'wx', mode: 0o600 }); const childEnvironment = Object.fromEntries(['PATH', 'LANG', 'LANGUAGE', 'LC_ALL', 'TZ'].flatMap(name => environment[name] === undefined ? [] : [[name, environment[name]]])); Object.assign(childEnvironment, { HOME: join(root, 'home'), TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: gateway.token }); if (manifest.observability) { + failureStage = 'observability_start_failed'; observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport, signal }); secrets.push(...observability.secrets); childEnvironment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN = observability.token; } const artifactPath = join(root, 'provider-artifact.json'); + failureStage = 'child_failed'; signal?.throwIfAborted(); if (Date.now() >= deadlineAt) throw new Error('Controlled attempt deadline reached before spawn'); child = await runChild({ cli: stagedCli, loader, requestPath: retainedRequestPath, artifactPath, descriptorPath, observabilityPath: observability?.descriptorPath, cwd: request.session.workingDirectory ?? root, environment: childEnvironment, deadlineAt, signal, closeGateway: () => Promise.all([gateway.close(), observability?.close()]) }); + failureStage = 'output_retention_failed'; for (const stream of ['stdout', 'stderr']) await writeFile(join(root, `${stream}.log`), redact(child[stream]), { flag: 'wx', mode: 0o600 }); + failureStage = child.code === 0 ? 'artifact_invalid' : 'child_failed'; const raw = (await readRegularFile(artifactPath, limitFor('provider-artifact.json'))).toString('utf8'); secretExcluded = secrets.every(secret => !raw.includes(secret)); if (!secretExcluded) await replaceEvidenceFile(artifactPath, redact(raw)); @@ -156,6 +163,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { executionId: attempt.executionId, attemptId: attempt.attemptId, descriptorSha256: sha256(descriptorRaw) }; if (canonical(artifact.requestRouting) !== canonical(expected) || artifact.attemptId !== attempt.attemptId || artifact.requestedModel !== attempt.model) throw new Error('Provider did not retain the assigned route'); if (observability && canonical(artifact.nativeObservability) !== canonical(observability.expectedEvidence)) throw new Error('Provider did not retain the assigned observability route'); + failureStage = 'provider_cleanup_or_credential_unverified'; const processInfo = artifact.snapshot?.process; const pids = [...new Set([child.pid, processInfo?.runnerPid, processInfo?.providerPid, processInfo?.sidecarPid, processInfo?.agentPid].filter(pid => Number.isSafeInteger(pid) && pid > 0))]; cleanupVerified = !child.stopped && artifact.snapshot?.status === 'closed' && processInfo?.runnerExited === true @@ -163,7 +171,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { && Number.isSafeInteger(processInfo.providerPid) && processInfo.providerPid > 0 && pids.every(absent); if (!cleanupVerified || !secretExcluded) throw new Error('Controlled provider cleanup or credential exclusion is unverified'); } catch { - failure = 'controlled_attempt_incomplete'; + failure = failureStage; } finally { try { await gateway?.close(); } catch { failure = 'gateway_cleanup_unverified'; } try { await ledger.close(); } catch { failure = 'ledger_close_failed'; }