diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 5c96e02108c..4a87b8a6aa2 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -372,7 +372,11 @@ records plus the controller root. It describes the retained requests only; `verification: "unverified"` remains until separate Braintrust readback. Neither field proves complete capture, semantic grading success, or invoice settlement. The root links revisions, the compiled request, execution, attempt, -provider, model, and cleanup status. Feedback persistence is explicitly +provider, model, and cleanup status. Native receipts are checked before the root +is built. Its `controller_status_before_root_export` includes known native export +failures; the final retained report remains authoritative for root-export failures +or later cancellation. Cancellation stays connected through root finalization. +Feedback persistence is explicitly unavailable in the Runner mock control plane; it does not report zero complaints or suggestions. Actual grading and provider failures remain in the immutable Runner artifact. diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index de56b6fbaa7..7162d41663e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -587,3 +587,35 @@ test('lost ledger finalization still closes both OTLP relays', async t => { const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8')); await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs')); }); + +for (const nativeFault of ['partial', 'quota']) test(`root includes ${nativeFault} native export failure before finalization`, async t => { + let rootPayload; + const f = await fixture(t, { observability: true, requests: 1, otelRequests: nativeFault === 'quota' ? 3 : 1, + otlpTransport: async (url, init) => { + if (new Headers(init.headers).get('x-bt-parent').startsWith('project_id:')) { + rootPayload = JSON.parse(Buffer.from(init.body).toString()); return Response.json({}); + } + return nativeFault === 'partial' ? Response.json({ partialSuccess: String(url).endsWith('/logs') ? { rejectedLogRecords: 1 } : { rejectedSpans: 1 } }) : Response.json({}); + } }); + const result = await f.run(); + assert.equal(result.code, 2); assert.equal(result.report.observability.traceFailure, 'native_export_incomplete'); + const attributes = rootPayload.resourceSpans[0].scopeSpans[0].spans[0].attributes; + assert(attributes.some(item => item.key === 'paperclip.eval.controller_status_before_root_export' && item.value.stringValue === 'incomplete')); +}); + +test('cancellation during root export stops forwarding and cannot close the attempt successfully', async t => { + const cancellation = new AbortController(); let rootAborted = false; + const f = await fixture(t, { observability: true, requests: 1, otlpTransport: async (_url, init) => { + if (!new Headers(init.headers).get('x-bt-parent').startsWith('project_id:')) return Response.json({}); + queueMicrotask(() => cancellation.abort()); + return new Promise((_resolve, reject) => init.signal.addEventListener('abort', () => { rootAborted = true; reject(new Error('synthetic root abort')); }, { once: true })); + } }); + const result = await f.run(0, () => success(), cancellation.signal); + assert.equal(rootAborted, true); assert.equal(result.code, 2); assert.equal(result.report.failure, 'controller_cancelled'); + assert.equal(result.report.status, 'incomplete'); assert.equal(result.report.cleanupVerified, true); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const rows = (await readFile(join(root, 'otlp-controller.jsonl'), 'utf8')).trim().split('\n').map(line => JSON.parse(line)); + assert.equal(rows.find(row => row.type === 'outcome').outcome, 'uncertain'); assert.equal(rows.at(-1).type, 'closed'); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.knownUsd, '0.000068000'); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/observability.mjs b/packages/paperclip-runner/scripts/controlled-evals/observability.mjs index 6ceeeb7dc5a..274a25eb438 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/observability.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/observability.mjs @@ -22,7 +22,7 @@ function rootTrace(manifest, attempt, identity, startedAt, finishedAt, cleanupVe 'paperclip.eval.source_revision': manifest.sourceRevision, 'paperclip.eval.definitions_revision': manifest.definitionsRevision, 'paperclip.eval.request_sha256': attempt.requestSha256, 'paperclip.eval.provider': attempt.provider, 'paperclip.eval.profile': attempt.acpxAgent ?? attempt.provider, 'paperclip.eval.model': attempt.model, - 'paperclip.eval.controller_status': failure ? 'incomplete' : 'closed', + 'paperclip.eval.controller_status_before_root_export': failure ? 'incomplete' : 'closed', 'paperclip.eval.cleanup_verified': String(cleanupVerified), 'paperclip.eval.feedback_persistence': 'unavailable_mock_control_plane', 'paperclip.eval.grading': 'retained_artifact_separate', }; @@ -34,18 +34,23 @@ function rootTrace(manifest, attempt, identity, startedAt, finishedAt, cleanupVe } const nativeEvidence = (identity, descriptorSha256) => ({ mode: 'controller_otlp', configured: true, verification: 'unverified', ...identity, descriptorSha256 }); -export async function startControlledObservability({ manifest, attempt, request, root, apiKey, deadlineAt, startedAt, transport }) { +export async function startControlledObservability({ manifest, attempt, request, root, apiKey, deadlineAt, startedAt, transport, signal }) { const identity = { projectId: manifest.observability.projectId, traceId: randomBytes(16).toString('hex'), spanId: randomBytes(8).toString('hex') }; let native, controller; + const onAbort = () => { void close().catch(() => {}); }; const close = async () => { + signal?.removeEventListener('abort', onAbort); const results = await Promise.allSettled([native?.close(), controller?.close()]); if (results.some(result => result.status === 'rejected')) throw new Error('OTLP closure unverified'); }; try { + signal?.throwIfAborted(); native = await costRuntime.startEvalOtlpRelay({ ...policyFor(manifest, attempt, identity, deadlineAt, true), deadlineAt, apiKey, receiptPath: join(root, 'otlp-native.jsonl'), transport }); controller = await costRuntime.startEvalOtlpRelay({ ...policyFor(manifest, attempt, identity, deadlineAt, false), deadlineAt, apiKey, receiptPath: join(root, 'otlp-controller.jsonl'), transport }); + signal?.addEventListener('abort', onAbort, { once: true }); + signal?.throwIfAborted(); const descriptor = { schema: 'paperclip.runner-eval.observability/v1', campaignId: manifest.campaignId, executionId: attempt.executionId, attemptId: attempt.attemptId, ...identity, baseUrl: native.baseUrl, launch: launchFor(manifest, request, identity, native.baseUrl) }; const raw = JSON.stringify(descriptor) + '\n'; @@ -56,17 +61,24 @@ export async function startControlledObservability({ manifest, attempt, request, expectedEvidence: nativeEvidence(identity, sha256(raw)), close, closeNative: native.close, finish: input => finished ??= (async () => { const finishedAt = new Date().toISOString(); + let traceFailure = input.failure; try { await native.close(); - const body = JSON.stringify(rootTrace(manifest, attempt, identity, startedAt, finishedAt, input.cleanupVerified, input.failure)); + signal?.throwIfAborted(); + try { + const receipts = costRuntime.inspectEvalOtlpReceipts(await readFile(join(root, 'otlp-native.jsonl')), + policyFor(manifest, attempt, identity, deadlineAt, true)); + if (receipts.status !== 'accepted' || !receipts.signals.logs.records || !receipts.signals.traces.records) traceFailure ??= 'native_export_incomplete'; + } catch { traceFailure ??= 'native_evidence_incomplete'; } + const body = JSON.stringify(rootTrace(manifest, attempt, identity, startedAt, finishedAt, input.cleanupVerified, traceFailure)); await writeFile(join(root, 'otlp-root.json'), body, { flag: 'wx', mode: 0o600 }); const response = await fetch(`${controller.baseUrl}/v1/traces`, { method: 'POST', redirect: 'error', - signal: AbortSignal.timeout(Math.max(1, Math.min(10_000, deadlineAt - Date.now()))), + signal: AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(Math.max(1, Math.min(10_000, deadlineAt - Date.now())))]), headers: { authorization: `Bearer ${controller.token}`, 'content-type': 'application/json', 'x-bt-parent': `project_id:${identity.projectId}` }, body }); await response.arrayBuffer(); } finally { await close(); } return inspectControlledObservability({ manifest, attempt, request, artifact: input.artifact, - report: { startedAt, traceFinishedAt: finishedAt, cleanupVerified: input.cleanupVerified, traceFailure: input.failure }, + report: { startedAt, traceFinishedAt: finishedAt, cleanupVerified: input.cleanupVerified, traceFailure }, deadlineAt, readEvidence: name => readFile(join(root, name)) }); })() }; } catch (error) { await close(); throw error; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 0c112cbc51e..005afa43922 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -136,7 +136,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { const childEnvironment = Object.fromEntries(['PATH', 'LANG', 'LANGUAGE', 'LC_ALL', 'TZ'].flatMap(name => environment[name] === undefined ? [] : [[name, environment[name]]])); Object.assign(childEnvironment, { HOME: join(root, 'home'), TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: gateway.token }); if (manifest.observability) { - observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport }); + observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport, signal }); secrets.push(...observability.secrets); childEnvironment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN = observability.token; } @@ -187,6 +187,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { if (observabilityEvidence.status !== 'accepted') failure ??= 'observability_export_incomplete'; } catch { failure ??= 'observability_evidence_incomplete'; } } + if (signal?.aborted) failure = 'controller_cancelled'; const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256, executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256, allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure,