From a74443ee6c83787b1d355e083ef08dc43f44c862 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 09:48:20 -0500 Subject: [PATCH] fix(evals): preserve incomplete results after OTLP closure errors Do not replace a saved failure report with a repeated rejected relay-close promise. A synthetic final receipt write fault proves the incomplete result, cost hold, and relay closure survive. Co-Authored-By: Paperclip --- .../controlled-evals/controller.test.mjs | 33 ++++++++++++++++++- .../scripts/controlled-evals/run.mjs | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index cf70b8d86dc..e7f4dbb8ea5 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { execFileSync } from 'node:child_process'; -import { mkdtemp, mkdir, readFile, readdir, rm, symlink, truncate, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, open, readFile, readdir, rm, stat, symlink, truncate, writeFile } from 'node:fs/promises'; import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -734,3 +734,34 @@ test('missing provider artifacts cannot bypass immutable result binding', async await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed)); await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/); }); + + +test('failed final OTLP receipt write returns its saved incomplete report', async t => { + const f = await fixture(t, { observability: true, requests: 1 }); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const probe = await open(join(f.root, 'prototype-probe'), 'wx'); + const prototype = Object.getPrototypeOf(probe), original = prototype.writeFile; + await probe.close(); + let failed = false; + prototype.writeFile = async function (data, ...options) { + if (String(data).includes('"type":"closed"')) { + const native = await stat(join(root, 'otlp-native.jsonl')).catch(() => null); + if (native && (await this.stat()).ino === native.ino) { + failed = true; + throw Object.assign(new Error('synthetic final receipt write failure'), { code: 'ENOSPC' }); + } + } + return original.call(this, data, ...options); + }; + t.after(() => { prototype.writeFile = original; }); + const result = await f.run(); + assert.equal(failed, true); assert.equal(result.code, 2); + assert.equal(result.report.status, 'incomplete'); + assert.deepEqual(JSON.parse(await readFile(join(root, 'result.json'), 'utf8')), result.report); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); assert.equal(summary.committedUsd, '0.014000000'); + const controller = (await readFile(join(root, 'otlp-controller.jsonl'), 'utf8')).trim().split('\n').map(line => JSON.parse(line)); + assert.equal(controller.at(-1).type, 'closed'); + const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8')); + await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs')); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 958c23296b5..8b503c79c20 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -201,7 +201,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 }); await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); return { code: failure ? 2 : child?.code ?? 2, report }; - } finally { await observability?.close(); } + } finally { try { await observability?.close(); } catch {} } }