fix(evals): retain cancellation and native failures through root export

Close pending OTLP forwarding on cancellation through finalization. Inspect native receipts before building the root, and label its status as preceding the root export.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -372,7 +372,11 @@ records plus the controller root. It describes the retained requests only;
 `verification: "unverified"` remains until separate Braintrust readback.
 Neither field proves complete capture, semantic grading success, or invoice
 settlement. The root links revisions, the compiled request, execution, attempt,
-provider, model, and cleanup status. Feedback persistence is explicitly
+provider, model, and cleanup status. Native receipts are checked before the root
+is built. Its `controller_status_before_root_export` includes known native export
+failures; the final retained report remains authoritative for root-export failures
+or later cancellation. Cancellation stays connected through root finalization.
+Feedback persistence is explicitly
 unavailable in the Runner mock control plane; it does not report zero complaints
 or suggestions. Actual grading and provider failures remain in the immutable
 Runner artifact.

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -587,3 +587,35 @@ test('lost ledger finalization still closes both OTLP relays', async t => {
   const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8'));
   await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs'));
 });
+
+for (const nativeFault of ['partial', 'quota']) test(`root includes ${nativeFault} native export failure before finalization`, async t => {
+  let rootPayload;
+  const f = await fixture(t, { observability: true, requests: 1, otelRequests: nativeFault === 'quota' ? 3 : 1,
+    otlpTransport: async (url, init) => {
+      if (new Headers(init.headers).get('x-bt-parent').startsWith('project_id:')) {
+        rootPayload = JSON.parse(Buffer.from(init.body).toString()); return Response.json({});
+      }
+      return nativeFault === 'partial' ? Response.json({ partialSuccess: String(url).endsWith('/logs') ? { rejectedLogRecords: 1 } : { rejectedSpans: 1 } }) : Response.json({});
+    } });
+  const result = await f.run();
+  assert.equal(result.code, 2); assert.equal(result.report.observability.traceFailure, 'native_export_incomplete');
+  const attributes = rootPayload.resourceSpans[0].scopeSpans[0].spans[0].attributes;
+  assert(attributes.some(item => item.key === 'paperclip.eval.controller_status_before_root_export' && item.value.stringValue === 'incomplete'));
+});
+
+test('cancellation during root export stops forwarding and cannot close the attempt successfully', async t => {
+  const cancellation = new AbortController(); let rootAborted = false;
+  const f = await fixture(t, { observability: true, requests: 1, otlpTransport: async (_url, init) => {
+    if (!new Headers(init.headers).get('x-bt-parent').startsWith('project_id:')) return Response.json({});
+    queueMicrotask(() => cancellation.abort());
+    return new Promise((_resolve, reject) => init.signal.addEventListener('abort', () => { rootAborted = true; reject(new Error('synthetic root abort')); }, { once: true }));
+  } });
+  const result = await f.run(0, () => success(), cancellation.signal);
+  assert.equal(rootAborted, true); assert.equal(result.code, 2); assert.equal(result.report.failure, 'controller_cancelled');
+  assert.equal(result.report.status, 'incomplete'); assert.equal(result.report.cleanupVerified, true);
+  const root = attemptDirectory(f.manifest, 'attempt-0');
+  const rows = (await readFile(join(root, 'otlp-controller.jsonl'), 'utf8')).trim().split('\n').map(line => JSON.parse(line));
+  assert.equal(rows.find(row => row.type === 'outcome').outcome, 'uncertain'); assert.equal(rows.at(-1).type, 'closed');
+  const summary = await summarizeControlledCampaign(f.manifest);
+  assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.knownUsd, '0.000068000');
+});

packages/paperclip-runner/scripts/controlled-evals/observability.mjs

@@ -22,7 +22,7 @@ function rootTrace(manifest, attempt, identity, startedAt, finishedAt, cleanupVe
     'paperclip.eval.source_revision': manifest.sourceRevision, 'paperclip.eval.definitions_revision': manifest.definitionsRevision,
     'paperclip.eval.request_sha256': attempt.requestSha256, 'paperclip.eval.provider': attempt.provider,
     'paperclip.eval.profile': attempt.acpxAgent ?? attempt.provider, 'paperclip.eval.model': attempt.model,
-    'paperclip.eval.controller_status': failure ? 'incomplete' : 'closed',
+    'paperclip.eval.controller_status_before_root_export': failure ? 'incomplete' : 'closed',
     'paperclip.eval.cleanup_verified': String(cleanupVerified), 'paperclip.eval.feedback_persistence': 'unavailable_mock_control_plane',
     'paperclip.eval.grading': 'retained_artifact_separate',
   };
@@ -34,18 +34,23 @@ function rootTrace(manifest, attempt, identity, startedAt, finishedAt, cleanupVe
 }
 const nativeEvidence = (identity, descriptorSha256) => ({ mode: 'controller_otlp', configured: true, verification: 'unverified', ...identity, descriptorSha256 });
 
-export async function startControlledObservability({ manifest, attempt, request, root, apiKey, deadlineAt, startedAt, transport }) {
+export async function startControlledObservability({ manifest, attempt, request, root, apiKey, deadlineAt, startedAt, transport, signal }) {
   const identity = { projectId: manifest.observability.projectId, traceId: randomBytes(16).toString('hex'), spanId: randomBytes(8).toString('hex') };
   let native, controller;
+  const onAbort = () => { void close().catch(() => {}); };
   const close = async () => {
+    signal?.removeEventListener('abort', onAbort);
     const results = await Promise.allSettled([native?.close(), controller?.close()]);
     if (results.some(result => result.status === 'rejected')) throw new Error('OTLP closure unverified');
   };
   try {
+    signal?.throwIfAborted();
     native = await costRuntime.startEvalOtlpRelay({ ...policyFor(manifest, attempt, identity, deadlineAt, true), deadlineAt, apiKey,
       receiptPath: join(root, 'otlp-native.jsonl'), transport });
     controller = await costRuntime.startEvalOtlpRelay({ ...policyFor(manifest, attempt, identity, deadlineAt, false), deadlineAt, apiKey,
       receiptPath: join(root, 'otlp-controller.jsonl'), transport });
+    signal?.addEventListener('abort', onAbort, { once: true });
+    signal?.throwIfAborted();
     const descriptor = { schema: 'paperclip.runner-eval.observability/v1', campaignId: manifest.campaignId, executionId: attempt.executionId,
       attemptId: attempt.attemptId, ...identity, baseUrl: native.baseUrl, launch: launchFor(manifest, request, identity, native.baseUrl) };
     const raw = JSON.stringify(descriptor) + '\n';
@@ -56,17 +61,24 @@ export async function startControlledObservability({ manifest, attempt, request,
       expectedEvidence: nativeEvidence(identity, sha256(raw)), close, closeNative: native.close,
       finish: input => finished ??= (async () => {
         const finishedAt = new Date().toISOString();
+        let traceFailure = input.failure;
         try {
           await native.close();
-          const body = JSON.stringify(rootTrace(manifest, attempt, identity, startedAt, finishedAt, input.cleanupVerified, input.failure));
+          signal?.throwIfAborted();
+          try {
+            const receipts = costRuntime.inspectEvalOtlpReceipts(await readFile(join(root, 'otlp-native.jsonl')),
+              policyFor(manifest, attempt, identity, deadlineAt, true));
+            if (receipts.status !== 'accepted' || !receipts.signals.logs.records || !receipts.signals.traces.records) traceFailure ??= 'native_export_incomplete';
+          } catch { traceFailure ??= 'native_evidence_incomplete'; }
+          const body = JSON.stringify(rootTrace(manifest, attempt, identity, startedAt, finishedAt, input.cleanupVerified, traceFailure));
           await writeFile(join(root, 'otlp-root.json'), body, { flag: 'wx', mode: 0o600 });
           const response = await fetch(`${controller.baseUrl}/v1/traces`, { method: 'POST', redirect: 'error',
-            signal: AbortSignal.timeout(Math.max(1, Math.min(10_000, deadlineAt - Date.now()))),
+            signal: AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(Math.max(1, Math.min(10_000, deadlineAt - Date.now())))]),
             headers: { authorization: `Bearer ${controller.token}`, 'content-type': 'application/json', 'x-bt-parent': `project_id:${identity.projectId}` }, body });
           await response.arrayBuffer();
         } finally { await close(); }
         return inspectControlledObservability({ manifest, attempt, request, artifact: input.artifact,
-          report: { startedAt, traceFinishedAt: finishedAt, cleanupVerified: input.cleanupVerified, traceFailure: input.failure },
+          report: { startedAt, traceFinishedAt: finishedAt, cleanupVerified: input.cleanupVerified, traceFailure },
           deadlineAt, readEvidence: name => readFile(join(root, name)) });
       })() };
   } catch (error) { await close(); throw error; }

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -136,7 +136,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
     const childEnvironment = Object.fromEntries(['PATH', 'LANG', 'LANGUAGE', 'LC_ALL', 'TZ'].flatMap(name => environment[name] === undefined ? [] : [[name, environment[name]]]));
     Object.assign(childEnvironment, { HOME: join(root, 'home'), TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'), PAPERCLIP_RUNNER_EVAL_GATEWAY_TOKEN: gateway.token });
     if (manifest.observability) {
-      observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport });
+      observability = await startControlledObservability({ manifest, attempt, request, root, apiKey: observabilityKey, deadlineAt, startedAt, transport: otlpTransport, signal });
       secrets.push(...observability.secrets);
       childEnvironment.PAPERCLIP_RUNNER_EVAL_OTLP_TOKEN = observability.token;
     }
@@ -187,6 +187,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
         if (observabilityEvidence.status !== 'accepted') failure ??= 'observability_export_incomplete';
       } catch { failure ??= 'observability_evidence_incomplete'; }
     }
+    if (signal?.aborted) failure = 'controller_cancelled';
     const report = { schema: 'paperclip.runner-eval.control-result/v1', campaignId: manifest.campaignId, manifestSha256,
       executionId: attempt.executionId, attemptId: attempt.attemptId, requestSha256: attempt.requestSha256,
       allocationUsd: attempt.maxUsd, limits, startedAt, finishedAt: new Date().toISOString(), status: failure ? 'incomplete' : 'closed', failure,