From 4781487215116afd10a415d4db53b1ec8e004dd5 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 10:23:49 -0700 Subject: [PATCH] fix(evals): finish process group cancellation after CLI exit Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 4 +++ .../controlled-evals/controller.test.mjs | 27 +++++++++++++++++++ .../scripts/controlled-evals/run.mjs | 11 ++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 4319d7f76c6..b066a6de31f 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -510,3 +510,7 @@ Campaign reports verify the retained request binding and the digest of every request receipt before reducing a cell's allocation. Missing admission, request, or receipt evidence preserves known ledger charges and holds the rest of the full allocation. + +On cancellation, the controller keeps the scheduled process-group SIGKILL until +any remaining owned descendants receive it, even if the CLI exits first. A +forced stop still leaves cleanup unverified and retains the full allocation. diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 92ed44792c9..a4af19aa000 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -11,9 +11,17 @@ import { summarizeControlledCampaign } from './report.mjs'; const childScript = options => ` import assert from 'node:assert/strict'; import {createHash} from 'node:crypto'; +import {spawn} from 'node:child_process'; +import {once} from 'node:events'; import {readFile,writeFile} from 'node:fs/promises'; const args=new Map();for(let i=2;i{});process.stdout.write('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','ignore']}); + await once(descendant.stdout,'data'); + await writeFile(request.session.workingDirectory+'/descendant.pid',String(descendant.pid)); +} + const descriptorRaw=await readFile(args.get('--gateway-config'),'utf8'),descriptor=JSON.parse(descriptorRaw); for(const key of ['OPENAI_API_KEY','ANTHROPIC_API_KEY','BRAINTRUST_API_KEY','GH_TOKEN','NODE_OPTIONS','PYTHONPATH'])assert.equal(process.env[key],undefined); const statuses=[]; @@ -250,3 +258,22 @@ test('staged CLI relative imports survive a symlinked evidence directory', async assert.equal(result.code, 0); assert.equal((await summarizeControlledCampaign(f.manifest)).incompleteAttempts, 0); }); + +test('cancellation still kills an owned resistant descendant after the CLI exits', async t => { + const f = await fixture(t, { requests: 1, waitForever: true, resistantDescendant: true }); + f.manifest.attempts[0].timeoutMs = 15000; await f.save(); + const abort = new AbortController(); let ready; + const started = new Promise(resolve => { ready = resolve; }); + const running = f.run(0, async () => { ready(); return success(); }, abort.signal); + await started; + const pid = Number(await readFile(join(f.requests[0].request.session.workingDirectory, 'descendant.pid'), 'utf8')); + t.after(() => { try { process.kill(pid, 'SIGKILL'); } catch {} }); + abort.abort(); + const result = await running; + assert.equal(result.report.child.stopped, 'controller_cancelled'); + const absent = () => { try { process.kill(pid, 0); return false; } catch (error) { return error.code === 'ESRCH'; } }; + for (let retry = 0; retry < 100 && !absent(); retry++) await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(absent(), true); + assert.equal(result.report.cleanupVerified, false); + assert.equal((await summarizeControlledCampaign(f.manifest)).committedUsd, '0.014000000'); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 07740fe3a29..42fda084873 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -20,14 +20,16 @@ async function runChild(input) { const child = spawn(process.execPath, ['--import', input.loader, input.cli, '--request', input.requestPath, '--output', input.artifactPath, '--gateway-config', input.descriptorPath], { cwd: input.cwd, env: input.environment, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); - let stopped = null, force; + let stopped = null, force, escalation; const output = { stdout: [], stderr: [] }, sizes = { stdout: 0, stderr: 0 }; const stop = reason => { if (stopped) return; stopped = reason; void input.closeGateway().catch(() => {}); try { process.kill(-child.pid, 'SIGTERM'); } catch {} - force = setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }, 5000); + escalation = new Promise(resolve => { + force = setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} finally { resolve(); } }, 5000); + }); }; for (const stream of ['stdout', 'stderr']) child[stream].on('data', bytes => { const available = Math.max(0, 1_048_576 - sizes[stream]); @@ -40,6 +42,11 @@ async function runChild(input) { if (input.signal?.aborted) onAbort(); try { const [code, signal] = await once(child, 'exit'); + if (stopped) { + let groupPresent = true; + try { process.kill(-child.pid, 0); } catch (error) { if (error.code === 'ESRCH') groupPresent = false; } + if (groupPresent) await escalation; + } return { code, signal, pid: child.pid, stopped, stdout: Buffer.concat(output.stdout).toString(), stderr: Buffer.concat(output.stderr).toString() }; } finally { clearTimeout(timeout); clearTimeout(force); input.signal?.removeEventListener('abort', onAbort);