diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 7e816c16af3..9f88a7ed867 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { execFileSync } from 'node:child_process'; -import { mkdtemp, mkdir, readFile, readdir, rm, symlink, truncate, writeFile } from 'node:fs/promises'; +import { mkdtemp, mkdir, open, readFile, readdir, rm, stat, symlink, truncate, writeFile } from 'node:fs/promises'; import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -737,3 +737,34 @@ test('missing provider artifacts cannot bypass immutable result binding', async await writeFile(join(directory, 'controller-evidence.json'), JSON.stringify(malformed)); await assert.rejects(readAttemptControlledEvidence(directory), /Malformed immutable controller result proof/); }); + + +test('failed final OTLP receipt write returns its saved incomplete report', async t => { + const f = await fixture(t, { observability: true, requests: 1 }); + const root = attemptDirectory(f.manifest, 'attempt-0'); + const probe = await open(join(f.root, 'prototype-probe'), 'wx'); + const prototype = Object.getPrototypeOf(probe), original = prototype.writeFile; + await probe.close(); + let failed = false; + prototype.writeFile = async function (data, ...options) { + if (String(data).includes('"type":"closed"')) { + const native = await stat(join(root, 'otlp-native.jsonl')).catch(() => null); + if (native && (await this.stat()).ino === native.ino) { + failed = true; + throw Object.assign(new Error('synthetic final receipt write failure'), { code: 'ENOSPC' }); + } + } + return original.call(this, data, ...options); + }; + t.after(() => { prototype.writeFile = original; }); + const result = await f.run(); + assert.equal(failed, true); assert.equal(result.code, 2); + assert.equal(result.report.status, 'incomplete'); + assert.deepEqual(JSON.parse(await readFile(join(root, 'result.json'), 'utf8')), result.report); + const summary = await summarizeControlledCampaign(f.manifest); + assert.equal(summary.knownUsd, '0.000068000'); assert.equal(summary.committedUsd, '0.014000000'); + const controller = (await readFile(join(root, 'otlp-controller.jsonl'), 'utf8')).trim().split('\n').map(line => JSON.parse(line)); + assert.equal(controller.at(-1).type, 'closed'); + const descriptor = JSON.parse(await readFile(join(root, 'observability.json'), 'utf8')); + await assert.rejects(fetch(descriptor.baseUrl + '/v1/logs')); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index d81a0a054bb..62a02c883e4 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -202,7 +202,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { await mkdir(dirname(resolve(outputPath)), { recursive: true, mode: 0o700 }); await writeFile(outputPath, JSON.stringify(resultArtifact, null, 2) + '\n', { flag: 'wx', mode: 0o600 }); return { code: failure ? 2 : child?.code ?? 2, report }; - } finally { await observability?.close(); } + } finally { try { await observability?.close(); } catch {} } }