From dc58afb28f15edb57f2c9e1f4eeeff641899fce4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 28 Sep 2026 14:43:35 -0500 Subject: [PATCH] fix(runner): expose bounded completion reports in live eval sessions Co-Authored-By: Paperclip --- .../src/live/live-session.test.ts | 65 +++++++++++++++++++ .../paperclip-runner/src/live/live-session.ts | 56 +++++++++++++--- 2 files changed, 112 insertions(+), 9 deletions(-) diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts index 7f61d72a55f..f565840c151 100644 --- a/packages/paperclip-runner/src/live/live-session.test.ts +++ b/packages/paperclip-runner/src/live/live-session.test.ts @@ -892,6 +892,71 @@ describe("Capability live runnerd and Codex session", () => { await service.shutdown(session.id); }); + it("exposes native completion consistently on fresh and resumed sessions without granting semantic mutations", async () => { + const state = providerState(); + const store = new InMemoryCapabilityLiveSessionStore(); + const authority: Array[0]> = []; + const factory: CapabilityLiveTransportFactory = (options) => { + authority.push(options); + return fakeTransportFactory(state)(options); + }; + const firstService = new CapabilityLiveSessionService({ store, transportFactory: factory }); + const first = await firstService.create(); + const opened = state.transports[0]!.requests.find((request) => request.method === "thread/start")!; + const names = (opened.params.dynamicTools as Array<{ name: string }>).map((tool) => tool.name); + expect(names.filter((name) => name === "paperclip_finish")).toHaveLength(1); + expect(names.filter((name) => name === "paperclip_block")).toHaveLength(1); + expect(names).not.toContain("create_task"); + await first.suspend(); + const restoredService = new CapabilityLiveSessionService({ store, transportFactory: factory }); + const restored = await restoredService.restore(first.id); + // The facade appends native completion schemas to this admitted semantic list. + expect(authority[1]?.resumeDynamicTools?.map((tool) => tool.name)).toEqual( + names.filter((name) => name !== "paperclip_finish" && name !== "paperclip_block"), + ); + await restoredService.shutdown(restored.id); + await firstService.shutdown(first.id); + }); + + it("accepts only revision-bound advisory completion while mock mutations keep their own authority", async () => { + const state = providerState(); + const service = new CapabilityLiveSessionService({ transportFactory: fakeTransportFactory(state) }); + const session = await service.create(); + const before = session.mockState(); + const result = { + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Orientation complete", + completionClaim: { contractRevision: "paperclip-capability-live-v1", objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, + evidence: [], verification: [], + }; + const outcomes: Record[] = []; + state.onTurnStart = async () => { + const transport = state.transports.at(-1)!; + const turnId = [...state.turns.keys()].at(-1)!; + const call = (tool: string, arguments_: unknown, overrideTurn = turnId) => transport.invokeServerRequest({ + id: `call-${outcomes.length}`, method: "item/tool/call", + params: { threadId: state.threadId, turnId: overrideTurn, callId: `call-${outcomes.length}`, tool, arguments: arguments_ }, + }); + outcomes.push(await call("paperclip_finish", { ...result, completionClaim: { ...result.completionClaim, contractRevision: "stale" } })); + outcomes.push(await call("paperclip_finish", result, "wrong-turn")); + outcomes.push(await call("paperclip_block", result)); + outcomes.push(await call("paperclip_finish", result)); + outcomes.push(await call("paperclip_finish", { ...result, summary: "Conflicting report" })); + outcomes.push(await call("create_task", { title: "Unauthorized task", idempotencyKey: "unauthorized-task" })); + expect(session.snapshot().terminalTurns).toHaveLength(0); + }; + await session.sendMessage("Orient to the assigned task."); + expect(outcomes.map((outcome) => outcome.success)).toEqual([false, false, false, true, false, false]); + expect(session.snapshot().semanticResult).toMatchObject(result); + expect(session.mockState().tasks).toEqual(before.tasks); + expect(session.snapshot().terminalTurns).toHaveLength(1); + expect(session.snapshot().authorizationRecords).toContainEqual(expect.objectContaining({ operationId: "create_task", allowed: false })); + state.onTurnStart = undefined; + await session.sendMessage("Read the task again without reporting completion."); + expect(session.snapshot().semanticResult).toBeNull(); + await service.shutdown(session.id); + }); + it("passes caller-supplied native system instructions to the provider", async () => { const state = providerState(); const service = new CapabilityLiveSessionService({ diff --git a/packages/paperclip-runner/src/live/live-session.ts b/packages/paperclip-runner/src/live/live-session.ts index b7827b6bb5d..35339216162 100644 --- a/packages/paperclip-runner/src/live/live-session.ts +++ b/packages/paperclip-runner/src/live/live-session.ts @@ -7,7 +7,10 @@ import type { CodexRpcServerRequest, } from "../drivers/codex/app-server-transport.js"; import { redactCodexDiagnostic } from "../drivers/codex/app-server-transport.js"; -import { createSkilllessCodexThreadConfig } from "../drivers/codex/codex-app-server-driver.js"; +import { codexSemanticToolSpecs, createSkilllessCodexThreadConfig } from "../drivers/codex/codex-app-server-driver.js"; +import { createCodexTaskEnvelope } from "../contracts/codex.js"; +import { codexToolAcceptsResult, isCodexSemanticTool, isRetainableCodexPayload, rejectedCodexToolCall } from "../drivers/codex/codex-boundaries.js"; +import { validateCodexResultProposal } from "../mock-core/codex-runner.js"; import { resolveQualifiedAcpxProfile, type QualifiedAcpxAgent, @@ -58,6 +61,11 @@ const LIVE_COMPLETION_CONTRACT = Object.freeze({ revision: "paperclip-capability-live-v1", criterionIds: ["objective"], }); +const LIVE_COMPLETION_ENVELOPE = createCodexTaskEnvelope({ + objective: "Complete the current requested action without exceeding mock semantic authority.", + contractRevision: LIVE_COMPLETION_CONTRACT.revision, + criteria: LIVE_COMPLETION_CONTRACT.criterionIds.map((id) => ({ id, requirement: "Complete the current requested action." })), +}); const LIVE_BASE_INSTRUCTIONS = [ "You are operating one mock Paperclip issue through typed semantic tools.", "Use only the tools exposed in this thread; never call a Paperclip REST API.", @@ -1187,6 +1195,7 @@ export class CapabilityLiveSession { #providerThreadId = ""; #providerSessionId: string | null = null; #providerModel: { id: string; provider: string } | undefined; + #semanticResult: Record | null = null; #status: CapabilityLiveSessionStatus = "starting"; #activeTurnId: string | null = null; #providerRunBinding: { runId: string; turnId: string; itemId: string } | null; @@ -1251,6 +1260,11 @@ export class CapabilityLiveSession { this.#providerModel = options.snapshot?.providerModel === undefined ? undefined : structuredClone(options.snapshot.providerModel); + if (options.snapshot?.semanticResult) { + const validation = validateCodexResultProposal(options.snapshot.semanticResult, LIVE_COMPLETION_ENVELOPE); + if (validation.status !== "accepted") throw new Error("capability_live_invalid_semantic_result_checkpoint"); + this.#semanticResult = jsonValue(validation.result) as Record; + } this.#activeTurnId = options.snapshot?.activeTurnId ?? null; this.#providerRunBinding = options.snapshot?.providerRunBinding === undefined ? null @@ -1399,6 +1413,7 @@ export class CapabilityLiveSession { ...(this.#providerModel === undefined ? {} : { providerModel: structuredClone(this.#providerModel) }), + semanticResult: structuredClone(this.#semanticResult), status: this.#status, activeTurnId: this.#activeTurnId, ...(this.#providerRunBinding === null @@ -1517,6 +1532,7 @@ export class CapabilityLiveSession { settle: () => settleAdmission(), }; this.#pendingTurnAdmission = admission; + this.#semanticResult = null; this.#status = "running"; this.#clearIdleTimer(); this.#turnEventCount = 0; @@ -2330,6 +2346,14 @@ export class CapabilityLiveSession { itemId: `item_lab_${identityDigest}`, }; this.#providerRunBinding = providerRunBinding; + const authorizedTools = this.#dispatcher.listTools(this.#authority.runId); + const tools = (this.#config.toolExposure ?? "eager") === "lazy" + ? authorizedTools.filter((tool) => tool.annotations.exposure === "always") + : authorizedTools; + const semanticTools = [ + ...tools.map(dynamicToolSpec), + ...((this.#config.toolExposure ?? "eager") === "lazy" ? discoveryToolSpecs() : []), + ]; const transportBundle = this.#transportFactory({ ...this.#transportOptions, provider, @@ -2365,6 +2389,7 @@ export class CapabilityLiveSession { : {}), lifecyclePolicy: this.#config.lifecyclePolicy ?? { mode: "per_turn", idleTimeoutMs: null }, resumeActiveTurnId: resume ? this.#activeTurnId : null, + ...(resume ? { resumeDynamicTools: semanticTools } : {}), stateDirectory: resolve(this.#config.workingDirectory, ".paperclip-runner-prp", identityDigest), prpIdentity: { runnerInstanceId: `runner_lab_${identityDigest}`, @@ -2397,10 +2422,7 @@ export class CapabilityLiveSession { capabilities: { experimentalApi: true, requestAttestation: false }, }); this.#transport.notify("initialized"); - const authorizedTools = this.#dispatcher.listTools(this.#authority.runId); - const tools = (this.#config.toolExposure ?? "eager") === "lazy" - ? authorizedTools.filter((tool) => tool.annotations.exposure === "always") - : authorizedTools; + this.#appendEvidence("tool_exposure", null, { operationIds: tools.map((tool) => tool.name), scenarioId: this.#config.scenario.id, @@ -2471,10 +2493,7 @@ export class CapabilityLiveSession { baseInstructions: this.#transportOptions.baseInstructions ?? LIVE_BASE_INSTRUCTIONS, completionContract: LIVE_COMPLETION_CONTRACT, - dynamicTools: [ - ...tools.map(dynamicToolSpec), - ...((this.#config.toolExposure ?? "eager") === "lazy" ? discoveryToolSpecs() : []), - ], + dynamicTools: [...semanticTools, ...codexSemanticToolSpecs()], experimentalRawEvents: true, persistExtendedHistory: true, }); @@ -2565,6 +2584,25 @@ export class CapabilityLiveSession { contentItems: [{ type: "inputText", text: "Tool call was outside the active Capability thread and turn." }], }; } + if (isCodexSemanticTool(operationId)) { + // Native completion is an advisory run report. It cannot mutate the mock + // task, bypass semantic claims, or substitute for provider turn settlement. + if (terminalReplay || !isRetainableCodexPayload(request.params.arguments)) { + return rejectedCodexToolCall("Native completion requires a bounded active-turn report."); + } + const validation = validateCodexResultProposal(request.params.arguments, LIVE_COMPLETION_ENVELOPE); + if (validation.status !== "accepted" || !codexToolAcceptsResult(operationId, validation.result)) { + return rejectedCodexToolCall("Native completion must match the current completion contract and tool disposition."); + } + const result = jsonValue(validation.result) as Record; + if (this.#semanticResult !== null && JSON.stringify(this.#semanticResult) !== JSON.stringify(result)) { + return rejectedCodexToolCall("A different native completion report was already accepted for this turn."); + } + this.#semanticResult = result; + await this.#persist(); + return { success: true, contentItems: [{ type: "inputText", text: + "Native run report accepted. This does not change mock task state. End this provider turn after your final response; task mutations require their separately authorized semantic tools." }] }; + } if (operationId === DISCOVER_TOOL) { const args = record(request.params.arguments); try {