From 1461bb48e16ae3f980b57c0cfa8e0f895d302160 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 14:41:15 -0700 Subject: [PATCH] fix(evals): retain cancellation and native failures through root export Close pending OTLP forwarding on cancellation through finalization. Inspect native receipts before building the root, and label its status as preceding the root export. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 6 +++- .../controlled-evals/controller.test.mjs | 32 +++++++++++++++++++ .../controlled-evals/observability.mjs | 22 ++++++++++--- .../scripts/controlled-evals/run.mjs | 3 +- 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 5c96e02108c..4a87b8a6aa2 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -372,7 +372,11 @@ records plus the controller root. It describes the retained requests only; `verification: "unverified"` remains until separate Braintrust readback. Neither field proves complete capture, semantic grading success, or invoice settlement. The root links revisions, the compiled request, execution, attempt, -provider, model, and cleanup status. Feedback persistence is explicitly +provider, model, and cleanup status. Native receipts are checked before the root +is built. Its `controller_status_before_root_export` includes known native export +failures; the final retained report remains authoritative for root-export failures +or later cancellation. Cancellation stays connected through root finalization. +Feedback persistence is explicitly unavailable in the Runner mock control plane; it does not report zero complaints or suggestions. Actual grading and provider failures remain in the immutable Runner artifact. diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index de56b6fbaa7..7162d41663e 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -587,3 +587,35 @@ test('lost ledger finalization still closes both OTLP relays', async t => { const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8')); await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs')); }); + +for (const nativeFault of ['partial', 'quota']) test(`root includes ${nativeFault} native export failure before finalization`, async t => { + let rootPayload; + const f = await fixture(t, { observability: true, requests: 1, otelRequests: nativeFault === 'quota' ? 3 : 1, + otlpTransport: async (url, init) => { + if (new Headers(init.headers).get('x-bt-parent').startsWith('project_id:')) { + rootPayload = JSON.parse(Buffer.from(init.body).toString()); return Response.json({}); + } + return nativeFault === 'partial' ? Response.json({ partialSuccess: String(url).endsWith('/logs') ? { rejectedLogRecords: 1 } : { rejectedSpans: 1 } }) : Response.json({}); + } }); + const result = await f.run(); + assert.equal(result.code, 2); assert.equal(result.report.observability.traceFailure, 'native_export_incomplete'); + const attributes = rootPayload.resourceSpans[0].scopeSpans[0].spans[0].attributes; + assert(attributes.some(item => item.key === 'paperclip.eval.controller_status_before_root_export' && item.value.stringValue === 'incomplete')); +}); + +test('cancellation during root export stops forwarding and cannot close the attempt successfully', async t => { + const cancellation = new AbortController(); let rootAborted = false; + const f = await fixture(t, { observability: true, requests: 1, otlpTransport: async (_url, init) => { + if (!new Headers(init.headers).get('x-bt-parent').startsWith('project_id:')) return Response.json({}); + queueMicrotask(() => cancellation.abort()); + return new Promise((_resolve, reject) => init.signal.addEventListener('abort', () => { rootAborted = true; reject(new Error('synthetic root abort')); }, { once: true })); + } }); + const result = await f.run(0, () => success(), cancellation.signal); + assert.equal(rootAborted, true); assert.equal(result.code, 2); assert.equal(result.report.failure, 'controller_cancelled'); + assert.equal(result.report.status, 'incomplete'); assert.equal(result.report.cleanupVerified, true); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const rows = (await readFile(join(root, 'otlp-controller.jsonl'), 'utf8')).trim().split('\n').map(line => JSON.parse(line)); + assert.equal(rows.find(row => row.type === 'outcome').outcome, 'uncertain'); assert.equal(rows.at(-1).type, 'closed'); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.knownUsd, '0.000068000'); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/observability.mjs b/packages/paperclip-runner/scripts/controlled-evals/observability.mjs index 6ceeeb7dc5a..274a25eb438 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/observability.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/observability.mjs @@ -22,7 +22,7 @@ function rootTrace(manifest, attempt, identity, startedAt, finishedAt, cleanupVe 'paperclip.eval.source_revision': manifest.sourceRevision, 'paperclip.eval.definitions_revision': manifest.definitionsRevision, 'paperclip.eval.request_sha256': attempt.requestSha256, 'paperclip.eval.provider': attempt.provider, 'paperclip.eval.profile': attempt.acpxAgent ?? attempt.provider, 'paperclip.eval.model': attempt.model, - 'paperclip.eval.controller_status': failure ? 'incomplete' : 'closed', + 'paperclip.eval.controller_status_before_root_export': failure ? 'incomplete' : 'closed', 'paperclip.eval.cleanup_verified': String(cleanupVerified), 'paperclip.eval.feedback_persistence': 'unavailable_mock_control_plane', 'paperclip.eval.grading': 'retained_artifact_separate', }; @@ -34,18 +34,23 @@ function rootTrace(manifest, attempt, identity, startedAt, finishedAt, cleanupVe } const nativeEvidence = (identity, descriptorSha256) => ({ mode: 'controller_otlp', configured: true, verification: 'unverified', ...identity, descriptorSha256 }); -export async function startControlledObservability({ manifest, attempt, request, root, apiKey, deadlineAt, startedAt, transport }) { +export async function startControlledObservability({ manifest, attempt, request, root, apiKey, deadlineAt, startedAt, transport, signal }) { const identity = { projectId: manifest.observability.projectId, traceId: randomBytes(16).toString('hex'), spanId: randomBytes(8).toString('hex') }; let native, controller; + const onAbort = () => { void close().catch(() => {}); }; const close = async () => { + signal?.removeEventListener('abort', onAbort); const results = await Promise.allSettled([native?.close(), controller?.close()]); if (results.some(result => result.status === 'rejected')) throw new Error('OTLP closure unverified'); }; try { + signal?.throwIfAborted(); native = await costRuntime.startEvalOtlpRelay({ ...policyFor(manifest, attempt, identity, deadlineAt, true), deadlineAt, apiKey, receiptPath: join(root, 'otlp-native.jsonl'), transport }); controller = await costRuntime.startEvalOtlpRelay({ ...policyFor(manifest, attempt, identity, deadlineAt, false), deadlineAt, apiKey, receiptPath: join(root, 'otlp-controller.jsonl'), transport }); + signal?.addEventListener('abort', onAbort, { once: true }); + signal?.throwIfAborted(); const descriptor = { schema: 'paperclip.runner-eval.observability/v1', campaignId: manifest.campaignId, executionId: attempt.executionId, attemptId: attempt.attemptId, ...identity, baseUrl: native.baseUrl, launch: launchFor(manifest, request, identity, native.baseUrl) }; const raw = JSON.stringify(descriptor) + '\n'; @@ -56,17 +61,24 @@ export async function startControlledObservability({ manifest, attempt, request, expectedEvidence: nativeEvidence(identity, sha256(raw)), close, closeNative: native.close, finish: input => finished ??= (async () => { const finishedAt = new Date().toISOString(); + let traceFailure = input.failure; try { await native.close(); - const body = JSON.stringify(rootTrace(manifest, attempt, identity, startedAt, finishedAt, input.cleanupVerified, input.failure)); + signal?.throwIfAborted(); + try { + const receipts = costRuntime.inspectEvalOtlpReceipts(await readFile(join(root, 'otlp-native.jsonl')), + policyFor(manifest, attempt, identity, deadlineAt, true)); + if (receipts.status !== 'accepted' || !receipts.signals.logs.records || !receipts.signals.traces.records) traceFailure ??= 'native_export_incomplete'; + } catch { traceFailure ??= 'native_evidence_incomplete'; } + const body = JSON.stringify(rootTrace(manifest, attempt, identity, startedAt, finishedAt, input.cleanupVerified, traceFailure)); await writeFile(join(root, 'otlp-root.json'), body, { flag: 'wx', mode: 0o600 }); const response = await fetch(`${controller.baseUrl}/v1/traces`, { method: 'POST', redirect: 'error', - signal: AbortSignal.timeout(Math.max(1, Math.min(10_000, deadlineAt - Date.now()))), + signal: AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(Math.max(1, Math.min(10_000, deadlineAt - Date.now())))]), headers: { authorization: `Bearer ${controller.token}`, 'content-type': 'application/json', 'x-bt-parent': `project_id:${identity.projectId}` }, body }); await response.arrayBuffer(); } finally { await close(); } return inspectControlledObservability({ manifest, attempt, request, artifact: input.artifact, - report: { startedAt, traceFinishedAt: finishedAt, cleanupVerified: input.cleanupVerified, traceFailure: input.failure }, + report: { startedAt, traceFinishedAt: finishedAt, cleanupVerified: input.cleanupVerified, traceFailure }, deadlineAt, readEvidence: name => readFile(join(root, name)) }); })() }; } catch (error) { await close(); throw error; } diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 0c112cbc51e..005afa43922 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -136,7 +136,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { const childEnvironment = Object.fromEntries(['PATH', 'LANG', 'LANGUAGE', 'LC_ALL', 'TZ'].flatMap(name => environment[name] === undefined ? [] : [[name, environment[name]]])); Object.assign(childEnvironment, { HOME: join(root, 'home'), TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: gateway.token }); if (manifest.observability) { - observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport }); + observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport, signal }); secrets.push(...observability.secrets); childEnvironment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN = observability.token; } @@ -187,6 +187,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { if (observabilityEvidence.status !== 'accepted') failure ??= 'observability_export_incomplete'; } catch { failure ??= 'observability_evidence_incomplete'; } } + if (signal?.aborted) failure = 'controller_cancelled'; const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256, executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256, allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure,