From 6391608e8f3bdee1becd09d7ca2ab24c00d9cb8b Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 8 Aug 2026 01:53:48 +0000 Subject: [PATCH] fix(runner): enforce phase 4 status authority Co-Authored-By: Paperclip --- .../paperclip-runner/src/cli/phase4-codex.ts | 4 +- .../src/contracts/harness-driver.ts | 20 ++ .../paperclip-runner/src/contracts/phase4.ts | 46 ++- .../codex/codex-app-server-driver.test.ts | 308 ++++++++++++++-- .../drivers/codex/codex-app-server-driver.ts | 196 +++++----- .../src/mock-core/phase4-codex-runner.ts | 340 +++++++++++++++--- 6 files changed, 763 insertions(+), 151 deletions(-) diff --git a/packages/paperclip-runner/src/cli/phase4-codex.ts b/packages/paperclip-runner/src/cli/phase4-codex.ts index 7dbc045adb8..fb679f0ac39 100644 --- a/packages/paperclip-runner/src/cli/phase4-codex.ts +++ b/packages/paperclip-runner/src/cli/phase4-codex.ts @@ -40,7 +40,9 @@ function formatSummary(trace: Awaited>): `provider session: ${trace.metadata.identity.providerSessionId}`, `model: ${trace.context.model} (${trace.context.modelProvider})`, `events: ${trace.events.length}`, - `result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`, + trace.result === null + ? `result: rejected — ${trace.resultDecision.issues.map((issue) => issue.message).join("; ")}` + : `result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`, checks, ].join("\n"); } diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index cafb8bfbc72..7cefd1a0351 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -11,9 +11,29 @@ export interface HarnessDriverDescriptor { export interface OpenHarnessSessionInput { runId: string; + normalizedSessionId: string; workingDirectory: string; } +export class HarnessCapabilityUnavailableError extends Error { + readonly operation: string; + + constructor(operation: string, detail: string) { + super(`${operation} is unavailable: ${detail}`); + this.name = "HarnessCapabilityUnavailableError"; + this.operation = operation; + } +} + +export class HarnessReconciliationError extends Error { + readonly recoverable = true; + + constructor(message: string) { + super(message); + this.name = "HarnessReconciliationError"; + } +} + export interface PersistedHarnessSession { driverKind: string; driverSessionId: string; diff --git a/packages/paperclip-runner/src/contracts/phase4.ts b/packages/paperclip-runner/src/contracts/phase4.ts index f879107b0f5..cdc13c63cd3 100644 --- a/packages/paperclip-runner/src/contracts/phase4.ts +++ b/packages/paperclip-runner/src/contracts/phase4.ts @@ -75,12 +75,15 @@ export interface Phase4RunTrace { metadata: Phase4RunMetadata; context: Phase4ModelContextSnapshot; events: PrpEvent[]; - result: PrpStructuredRunResult; + proposedResult: unknown | null; + result: PrpStructuredRunResult | null; + resultDecision: Phase4ResultDecision; liveSnapshot: SessionSnapshot; replaySnapshot: SessionSnapshot; diagnostics: string[]; assertions: { exactlyOneTerminalResult: boolean; + proposalAccepted: boolean; liveReplayParity: boolean; stableIdentity: boolean; sourceSequenceContinuous: boolean; @@ -91,6 +94,22 @@ export interface Phase4RunTrace { }; } +export interface Phase4ResultValidationIssue { + code: + | "schema_validation" + | "contract_revision_mismatch" + | "unknown_criterion" + | "missing_criterion" + | "duplicate_criterion" + | "invalid_disposition"; + path: string; + message: string; +} + +export type Phase4ResultDecision = + | { status: "accepted"; result: PrpStructuredRunResult; issues: [] } + | { status: "rejected"; result: null; issues: Phase4ResultValidationIssue[] }; + export function createPhase4TaskEnvelope(input: { objective: string; contractRevision?: string; @@ -273,3 +292,28 @@ export const PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA = { }, }, } as const; + +export function isSkilllessPhase4Context( + context: Phase4ModelContextSnapshot, + options: { dynamicTools: boolean } = { dynamicTools: true }, +): boolean { + const serialized = JSON.stringify(context).toLowerCase(); + const expectedTools = options.dynamicTools ? [...PHASE4_SEMANTIC_TOOL_NAMES] : []; + return ( + context.instructionSources.length === 0 && + context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS && + context.modelInputKinds.length === 1 && + context.modelInputKinds[0] === "text" && + context.dynamicToolNames.length === expectedTools.length && + context.dynamicToolNames.every((name) => + expectedTools.includes(name as (typeof expectedTools)[number]), + ) && + context.instructionPolicy.skillInstructions === false && + context.instructionPolicy.appInstructions === false && + context.instructionPolicy.collaborationInstructions === false && + !serialized.includes("paperclip_api_key") && + !serialized.includes("authorization: bearer") && + !serialized.includes("/api/issues/") && + !serialized.includes('"type":"skill"') + ); +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts index be3d8b95a95..248dfe49a35 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts @@ -4,7 +4,9 @@ import { PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA, PHASE4_RESULT_OUTPUT_SCHEMA, createPhase4TaskEnvelope, + isSkilllessPhase4Context, } from "../../contracts/phase4.js"; +import { HarnessCapabilityUnavailableError } from "../../contracts/harness-driver.js"; import { applyPrpEvent, createSessionSnapshotFromMetadata, @@ -17,9 +19,11 @@ import { } from "../../protocol/phase1-contract.js"; import { CodexAppServerDriver, - UnsupportedCodexOperationError, - isSkilllessContext, } from "./codex-app-server-driver.js"; +import { + runPhase4CodexTracer, + validatePhase4ResultProposal, +} from "../../mock-core/phase4-codex-runner.js"; import type { CodexAppServerTransport, CodexRpcNotification, @@ -61,6 +65,7 @@ class FakeCodexTransport implements CodexAppServerTransport { readonly queue = new TestQueue(); handler: CodexServerRequestHandler = async () => ({}); rejectMethods = new Map(); + readResponse: Record | null = null; constructor( readonly threadId = "thread-1", @@ -76,7 +81,13 @@ class FakeCodexTransport implements CodexAppServerTransport { } if (method === "thread/start" || method === "thread/resume") { return { - thread: { id: this.threadId, sessionId: this.providerSessionId, modelProvider: "openai", cwd: "/workspace", turns: [] }, + thread: { + id: this.threadId, + sessionId: this.providerSessionId, + modelProvider: "openai", + cwd: "/workspace", + turns: [], + }, model: "gpt-test", modelProvider: "openai", cwd: "/workspace", @@ -87,7 +98,7 @@ class FakeCodexTransport implements CodexAppServerTransport { } if (method === "turn/start") return { turn: { id: "turn-1", status: "inProgress", items: [] } }; if (method === "thread/read") { - return { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } }; + return this.readResponse ?? { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } }; } return {}; } @@ -160,17 +171,69 @@ async function collectUntilTerminal(events: AsyncIterable): Promise) + .reverse() + .map(([key, entry]) => [key, reverseObjectKeys(entry)]), + ); +} + +async function traceCompletedProposal( + proposal: PrpStructuredRunResult | null, + options: { + runId?: string; + normalizedSessionId?: string; + capabilities?: Record; + steer?: string; + interrupt?: boolean; + } = {}, +) { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport], { capabilities: options.capabilities }); + const traced = runPhase4CodexTracer({ + driver, + taskEnvelope: envelope, + workingDirectory: "/workspace", + runId: options.runId, + normalizedSessionId: options.normalizedSessionId, + steer: options.steer, + interrupt: options.interrupt, + }); + while (!transport.calls.some((call) => call.method === "turn/start")) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: "turn-1", status: "inProgress" }, + }); + if (proposal !== null) { + transport.push("item/completed", { + threadId: "thread-1", + turnId: "turn-1", + item: { id: "answer-1", type: "agentMessage", text: JSON.stringify(proposal) }, + }); + } + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + return { trace: await traced, transport }; +} + describe("Codex app-server Phase 4 driver", () => { - it("passes the common typed-event contract and produces exactly one terminal result", async () => { + it("passes the common typed-event contract and reports one provider turn terminal", async () => { const transport = new FakeCodexTransport(); const driver = makeDriver([transport]); const descriptor = await driver.descriptor(); - const session = await driver.openSession({ runId: "run-1", workingDirectory: "/workspace" }); + const session = await driver.openSession({ runId: "run-1", normalizedSessionId: "normalized-1", workingDirectory: "/workspace" }); const turn = await session.startTurn({ message: { role: "user", text: "Do the safe task." } }); transport.push("turn/started", { threadId: "thread-1", turn: { id: turn.turnId, status: "inProgress" } }); @@ -189,9 +252,10 @@ describe("Codex app-server Phase 4 driver", () => { const events = await collectUntilTerminal(session.events()); expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true); expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1); - expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(1); + expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1); + expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(0); expect(events.map((event) => event.eventType)).toEqual(expect.arrayContaining([ - "session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "run.terminal", + "session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "turn.completed", "runtime_request.created", "runtime_request.resolved", ])); @@ -208,7 +272,7 @@ describe("Codex app-server Phase 4 driver", () => { }; const metadata = { fixtureName: "phase4-conformance", - identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "session:run-1" }, + identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "normalized-1" }, capabilities, }; const live = events.reduce(applyPrpEvent, createSessionSnapshotFromMetadata(metadata)); @@ -220,10 +284,10 @@ describe("Codex app-server Phase 4 driver", () => { it("captures an exact skillless model/environment snapshot with credentials absent", async () => { const transport = new FakeCodexTransport(); - const session = await makeDriver([transport]).openSession({ runId: "run-context", workingDirectory: "/workspace" }); + const session = await makeDriver([transport]).openSession({ runId: "run-context", normalizedSessionId: "normalized-context", workingDirectory: "/workspace" }); const iterator = session.events()[Symbol.asyncIterator](); const first = await iterator.next(); - const context = first.value?.payload.context as Parameters[0]; + const context = first.value?.payload.context as Parameters[0]; expect(context).toMatchObject({ codexVersion: "codex-cli/0.132.0", model: "gpt-test", @@ -239,7 +303,7 @@ describe("Codex app-server Phase 4 driver", () => { environmentKeys: ["CODEX_HOME", "HOME", "LANG", "PATH"], envelope, }); - expect(isSkilllessContext(context)).toBe(true); + expect(isSkilllessPhase4Context(context)).toBe(true); expect(JSON.stringify(context)).not.toContain("must-not-pass"); expect(JSON.stringify(transport.calls)).not.toContain("RANDOM_SKILL_PATH"); expect(transport.calls.find((call) => call.method === "thread/start")?.params).toMatchObject({ @@ -263,7 +327,7 @@ describe("Codex app-server Phase 4 driver", () => { it("steers and interrupts an active turn without replacing the session", async () => { const transport = new FakeCodexTransport(); - const session = await makeDriver([transport]).openSession({ runId: "run-controls", workingDirectory: "/workspace" }); + const session = await makeDriver([transport]).openSession({ runId: "run-controls", normalizedSessionId: "normalized-controls", workingDirectory: "/workspace" }); const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } }); await session.steer?.({ turnId, message: { role: "user", text: "Use a shorter answer." } }); await session.interrupt?.({ turnId, reason: "operator requested" }); @@ -275,22 +339,28 @@ describe("Codex app-server Phase 4 driver", () => { it("makes duplicate semantic completion idempotent and rejects changed payloads", async () => { const transport = new FakeCodexTransport(); - const session = await makeDriver([transport]).openSession({ runId: "run-result", workingDirectory: "/workspace" }); + const session = await makeDriver([transport]).openSession({ runId: "run-result", normalizedSessionId: "normalized-result", workingDirectory: "/workspace" }); await session.startTurn({ message: { role: "user", text: "Complete." } }); const request = { id: 1, method: "item/tool/call", params: { threadId: "thread-1", turnId: "turn-1", callId: "call-1", tool: "paperclip_finish", arguments: result } }; expect(await transport.invoke(request)).toMatchObject({ success: true }); - expect(await transport.invoke({ ...request, id: 2 })).toMatchObject({ success: true }); + expect(await transport.invoke({ + ...request, + id: 2, + params: { ...request.params, arguments: reverseObjectKeys(result) }, + })).toMatchObject({ success: true }); const changed = structuredClone(result); changed.summary = "Changed after commit."; expect(await transport.invoke({ ...request, id: 3, params: { ...request.params, arguments: changed } })).toMatchObject({ success: false }); transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } }); const events = await collectUntilTerminal(session.events()); expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1); + expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1); }); it("normalizes finish and block tools into one canonical result contract", async () => { const transport = new FakeCodexTransport(); - const session = await makeDriver([transport]).openSession({ runId: "run-tools", workingDirectory: "/workspace" }); + const session = await makeDriver([transport]).openSession({ runId: "run-tools", normalizedSessionId: "normalized-tools", workingDirectory: "/workspace" }); await session.startTurn({ message: { role: "user", text: "Complete or block." } }); const blocked: PrpStructuredRunResult = { ...structuredClone(result), @@ -339,7 +409,7 @@ describe("Codex app-server Phase 4 driver", () => { const first = new FakeCodexTransport(); const second = new FakeCodexTransport(); const driver = makeDriver([first, second]); - const original = await driver.openSession({ runId: "run-recover", workingDirectory: "/workspace" }); + const original = await driver.openSession({ runId: "run-recover", normalizedSessionId: "normalized-recover", workingDirectory: "/workspace" }); await original.startTurn({ message: { role: "user", text: "Work." } }); const snapshot = await original.snapshot(); await original.close({ reason: "transport lost" }); @@ -351,12 +421,62 @@ describe("Codex app-server Phase 4 driver", () => { expect((await recovery?.session?.snapshot())?.activeTurnId).toBe("turn-1"); }); + it("reconciles a turn completed during transport loss without replacing either session identity", async () => { + const first = new FakeCodexTransport("driver-thread-loss", "provider-session-loss"); + const second = new FakeCodexTransport("driver-thread-loss", "provider-session-loss"); + second.readResponse = { + thread: { + id: "driver-thread-loss", + sessionId: "provider-session-loss", + tokenUsage: { total: { inputTokens: 21, outputTokens: 8 } }, + turns: [{ + id: "turn-1", + status: "completed", + items: [{ id: "answer-loss", type: "agentMessage", text: JSON.stringify(result) }], + }], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "runtime-run-loss", + normalizedSessionId: "controller-session-loss", + workingDirectory: "/workspace", + }); + await original.startTurn({ message: { role: "user", text: "Work through loss." } }); + const snapshot = await original.snapshot(); + await original.close({ reason: "transport lost before terminal delivery" }); + + const recovery = await driver.recoverSession?.(snapshot); + expect(recovery).toMatchObject({ recovered: true }); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + const collecting = collectUntilTerminal(recovered!.events()); + await recovered!.reconcile?.(); + const events = await collecting; + + expect(recovered!.ids()).toEqual({ + driverSessionId: "driver-thread-loss", + providerSessionId: "provider-session-loss", + displayId: "driver-thread-loss", + }); + expect(await recovered!.snapshot()).toMatchObject({ + runId: "runtime-run-loss", + normalizedSessionId: "controller-session-loss", + driverSessionId: "driver-thread-loss", + providerSessionId: "provider-session-loss", + }); + expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1); + expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1); + expect(events.every((event, index) => event.sourceSeq === snapshot.lastSourceSequence! + index + 1)).toBe(true); + expect(await recovered!.usage?.()).toEqual({ total: { inputTokens: 21, outputTokens: 8 } }); + }); + it("degrades unsupported operations with explicit redacted diagnostics", async () => { const transport = new FakeCodexTransport(); transport.rejectMethods.set("turn/steer", new Error("Bearer super-secret api_key=also-secret")); - const session = await makeDriver([transport]).openSession({ runId: "run-degrade", workingDirectory: "/workspace" }); + const session = await makeDriver([transport]).openSession({ runId: "run-degrade", normalizedSessionId: "normalized-degrade", workingDirectory: "/workspace" }); const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } }); - await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(UnsupportedCodexOperationError); + await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); const iterator = session.events()[Symbol.asyncIterator](); const events: PrpEvent[] = []; for (let index = 0; index < 5; index += 1) { @@ -368,4 +488,152 @@ describe("Codex app-server Phase 4 driver", () => { expect(JSON.stringify(diagnostic)).not.toContain("super-secret"); expect(JSON.stringify(diagnostic)).not.toContain("also-secret"); }); + + it("rejects proposals that do not match the exact task envelope", () => { + const wrongRevision = structuredClone(result); + wrongRevision.completionClaim.contractRevision = "phase4-demo-v0"; + expect(validatePhase4ResultProposal(wrongRevision, envelope)).toMatchObject({ + status: "rejected", + issues: [{ code: "contract_revision_mismatch" }], + }); + + const unknownCriterion = structuredClone(result); + unknownCriterion.completionClaim.criteria = [{ + criterionId: "not-in-envelope", + status: "satisfied", + evidenceRefs: [], + }]; + const criterionDecision = validatePhase4ResultProposal(unknownCriterion, envelope); + expect(criterionDecision).toMatchObject({ status: "rejected" }); + if (criterionDecision.status === "rejected") { + expect(criterionDecision.issues.map((issue) => issue.code)).toEqual( + expect.arrayContaining(["unknown_criterion", "missing_criterion"]), + ); + } + + const invalidDone = structuredClone(result); + invalidDone.completionClaim.objectiveSatisfied = false; + expect(validatePhase4ResultProposal(invalidDone, envelope)).toMatchObject({ + status: "rejected", + issues: [{ code: "invalid_disposition" }], + }); + }); + + it.each(["needs_review", "blocked"] as const)( + "keeps a completed runtime successful for an accepted %s advisory proposal", + async (disposition) => { + const proposal: PrpStructuredRunResult = disposition === "needs_review" + ? { + ...structuredClone(result), + reportedWorkDisposition: "needs_review", + attentionRequests: [{ kind: "review", summary: "Review the completed artifact." }], + } + : { + ...structuredClone(result), + reportedWorkDisposition: "blocked", + summary: "Waiting on fixture input.", + completionClaim: { + ...structuredClone(result.completionClaim), + objectiveSatisfied: false, + criteria: [{ criterionId: "file", status: "not_satisfied", evidenceRefs: [] }], + remainingWork: [{ description: "Provide fixture input.", blocksCompletion: true }], + }, + blocker: { + reasonCode: "fixture_input_missing", + owner: { kind: "external", name: "fixture owner" }, + unblockAction: "Provide fixture input.", + scope: "task_wide", + }, + artifacts: [], + }; + const { trace } = await traceCompletedProposal(proposal); + expect(trace.resultDecision.status).toBe("accepted"); + expect(trace.events.find((event) => event.eventType === "run.terminal")?.payload) + .toMatchObject({ + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: disposition, + }); + }, + ); + + it("records rejected and missing proposals as recovery evidence instead of review status", async () => { + const wrongRevision = structuredClone(result); + wrongRevision.completionClaim.contractRevision = "wrong-revision"; + const rejected = await traceCompletedProposal(wrongRevision); + expect(rejected.trace.result).toBeNull(); + expect(rejected.trace.proposedResult).toMatchObject({ reportedWorkDisposition: "done" }); + expect(rejected.trace.events.find((event) => event.eventType === "run.result.rejected")?.payload) + .toMatchObject({ recovery: { required: true, recoverable: true } }); + expect(rejected.trace.events.find((event) => event.eventType === "run.terminal")?.payload) + .toMatchObject({ runTerminalState: "succeeded", reportedWorkDisposition: "yielded" }); + + const missing = await traceCompletedProposal(null); + expect(missing.trace.resultDecision).toMatchObject({ status: "rejected" }); + expect(missing.trace.events.some((event) => + event.eventType === "run.result.proposed" && + event.payload.reportedWorkDisposition === "needs_review" + )).toBe(false); + }); + + it("preserves four independent stable identities under controller ownership", async () => { + const { trace } = await traceCompletedProposal(result, { + runId: "runtime-run-identity", + normalizedSessionId: "controller-session-identity", + }); + expect(trace.assertions.stableIdentity).toBe(true); + expect(new Set([ + trace.metadata.identity.runId, + trace.metadata.identity.normalizedSessionId, + trace.metadata.identity.driverSessionId, + trace.metadata.identity.providerSessionId, + ]).size).toBe(4); + expect(trace.events.every((event) => + event.runId === "runtime-run-identity" && + event.normalizedSessionId === "controller-session-identity" + )).toBe(true); + }); + + it("degrades declared unsupported capabilities without Codex-specific core branches", async () => { + const capabilities = { + resume: false, + read: false, + steering: false, + interruption: false, + usage: false, + reconciliation: false, + dynamicTools: false, + }; + const { trace, transport } = await traceCompletedProposal(result, { + capabilities, + steer: "Do not call this unsupported path.", + interrupt: true, + }); + expect(trace.resultDecision.status).toBe("accepted"); + expect(trace.assertions.contextIsSkillless).toBe(true); + expect(trace.diagnostics).toEqual(expect.arrayContaining([ + expect.stringContaining("steering is unavailable"), + expect.stringContaining("interruption is unavailable"), + ])); + expect(transport.calls.map((call) => call.method)).not.toEqual( + expect.arrayContaining(["turn/steer", "turn/interrupt", "thread/read"]), + ); + expect(transport.calls.find((call) => call.method === "thread/start")?.params.dynamicTools) + .toEqual([]); + + const directTransport = new FakeCodexTransport(); + const driver = makeDriver([directTransport], { capabilities }); + const session = await driver.openSession({ + runId: "run-no-capabilities", + normalizedSessionId: "normalized-no-capabilities", + workingDirectory: "/workspace", + }); + await expect(session.read?.()).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); + await expect(session.usage?.()).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); + const snapshot = await session.snapshot(); + expect(await driver.recoverSession?.(snapshot)).toMatchObject({ + recovered: false, + reason: "resume capability is unavailable", + }); + }); }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts index 1d8eba6b890..e2f2dfc6529 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts @@ -8,6 +8,10 @@ import type { OpenHarnessSessionInput, PersistedHarnessSession, } from "../../contracts/harness-driver.js"; +import { + HarnessCapabilityUnavailableError, + HarnessReconciliationError, +} from "../../contracts/harness-driver.js"; import { PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA, PHASE4_BLOCK_TOOL_NAME, @@ -68,16 +72,6 @@ class AsyncQueue implements AsyncIterable { } } -export class UnsupportedCodexOperationError extends Error { - readonly operation: string; - - constructor(operation: string, detail: string) { - super(`${operation} is unsupported: ${redactCodexDiagnostic(detail)}`); - this.name = "UnsupportedCodexOperationError"; - this.operation = operation; - } -} - export interface CodexAppServerDriverOptions { taskEnvelope: Phase4TaskEnvelope; transportFactory?: () => CodexAppServerTransport; @@ -102,7 +96,7 @@ type CodexCapabilities = Required< interface OpenedCodexThread { threadId: string; - providerSessionId: string; + providerSessionId: string | null; context: Phase4ModelContextSnapshot; } @@ -137,12 +131,6 @@ function terminalState(status: string): "completed" | "failed" | "interrupted" { return "completed"; } -function dispositionRunState( - result: PrpStructuredRunResult, -): "succeeded" | "failed" { - return result.reportedWorkDisposition === "done" ? "succeeded" : "failed"; -} - function tryParseResult(value: unknown): PrpStructuredRunResult | null { let candidate = value; if (typeof value === "string") { @@ -156,6 +144,18 @@ function tryParseResult(value: unknown): PrpStructuredRunResult | null { return validation.ok ? validation.result : null; } +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const object = record(value); + if (Object.keys(object).length > 0 || (typeof value === "object" && value !== null)) { + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + function finishToolSpec(): Record { return { name: PHASE4_COMPLETION_TOOL_NAME, @@ -242,7 +242,7 @@ export class CodexAppServerDriver implements HarnessDriver { return this.#session({ transport, runId: input.runId, - normalizedSessionId: `session:${input.runId}`, + normalizedSessionId: input.normalizedSessionId, opened, resumed: false, sourceSequence: 0, @@ -255,6 +255,9 @@ export class CodexAppServerDriver implements HarnessDriver { if (!this.#caps.resume) { return { recovered: false, reason: "resume capability is unavailable" }; } + if (!this.#caps.read) { + return { recovered: false, reason: "read capability is required for safe resume" }; + } if (!snapshot.runId || !snapshot.normalizedSessionId || !snapshot.driverSessionId) { return { recovered: false, reason: "persisted session identity is incomplete" }; } @@ -272,6 +275,13 @@ export class CodexAppServerDriver implements HarnessDriver { await transport.close(); return { recovered: false, reason: "provider resumed a different session" }; } + if ( + snapshot.providerSessionId && + opened.providerSessionId !== snapshot.providerSessionId + ) { + await transport.close(); + return { recovered: false, reason: "provider resumed a different provider session" }; + } return { recovered: true, session: this.#session({ @@ -319,7 +329,7 @@ export class CodexAppServerDriver implements HarnessDriver { if (threadId.length === 0) throw new Error("Codex thread response omitted thread.id"); return { threadId, - providerSessionId: text(thread.sessionId, threadId), + providerSessionId: text(thread.sessionId) || null, context: { protocolVersion: PHASE4_CODEX_PROTOCOL_VERSION, codexVersion: text(initialize.userAgent, "unknown"), @@ -385,6 +395,7 @@ class CodexHarnessSession implements HarnessSession { #result: PrpStructuredRunResult | null = null; #resultFingerprint: string | null = null; #terminal = false; + readonly #terminalTurns = new Map(); constructor(input: { transport: CodexAppServerTransport; @@ -502,13 +513,34 @@ class CodexHarnessSession implements HarnessSession { this.#requireCapability("reconciliation"); const snapshot = await this.read(); const thread = record(snapshot.thread); + if (text(thread.id) !== this.#opened.threadId) { + throw new HarnessReconciliationError("thread/read returned a different driver session"); + } + const providerSessionId = text(thread.sessionId); + if ( + this.#opened.providerSessionId !== null && + providerSessionId !== this.#opened.providerSessionId + ) { + throw new HarnessReconciliationError("thread/read returned a different provider session"); + } const turns = Array.isArray(thread.turns) ? thread.turns.map(record) : []; const active = turns.find((turn) => text(turn.status) === "inProgress"); - this.#activeTurnId = active === undefined ? null : text(active.id); + const reconciledUsage = record(thread.tokenUsage ?? snapshot.tokenUsage); + if (Object.keys(reconciledUsage).length > 0) this.#usage = reconciledUsage; + const terminal = [...turns] + .reverse() + .find((turn) => ["completed", "failed", "interrupted", "cancelled"].includes(text(turn.status))); + if (terminal !== undefined) { + this.#activeTurnId = null; + this.#mapTerminalTurn(terminal, text(terminal.id)); + } else { + this.#activeTurnId = active === undefined ? null : text(active.id); + } this.#emit("session.reconciled", { providerSessionId: this.#opened.providerSessionId, turnCount: turns.length, activeTurnId: this.#activeTurnId, + terminalTurnId: terminal === undefined ? null : text(terminal.id), }); return snapshot; } @@ -560,12 +592,7 @@ class CodexHarnessSession implements HarnessSession { return; } if (notification.method === "turn/completed") { - const status = text(turn.status, "completed"); - const eventType = status === "failed" ? "turn.failed" : status === "interrupted" ? "turn.interrupted" : "turn.completed"; - this.#captureResultFromTurn(turn); - this.#emit(eventType, { status, error: turn.error ?? null }, { turnId }); - this.#activeTurnId = null; - this.#finalize(status); + this.#mapTerminalTurn(turn, turnId); return; } if (notification.method === "item/started") { @@ -639,7 +666,7 @@ class CodexHarnessSession implements HarnessSession { }], }; } - const fingerprint = JSON.stringify(validation.result); + const fingerprint = canonicalJson(validation.result); if (this.#resultFingerprint !== null && this.#resultFingerprint !== fingerprint) { return { success: false, contentItems: [{ type: "inputText", text: "A different semantic result was already committed." }] }; } @@ -700,7 +727,7 @@ class CodexHarnessSession implements HarnessSession { #commitResult(result: PrpStructuredRunResult, itemId: string, turnId?: string): void { if (this.#result !== null) return; this.#result = structuredClone(result); - this.#resultFingerprint = JSON.stringify(result); + this.#resultFingerprint = canonicalJson(result); result.verification.forEach((verification, index) => { this.#emit("item.completed", { kind: "verification", @@ -720,35 +747,64 @@ class CodexHarnessSession implements HarnessSession { #finalize(turnStatus: string): void { if (this.#terminal) return; if (this.#result === null) { - this.#commitResult({ - schema: "paperclip.run_result.v1", - reportedWorkDisposition: "needs_review", - summary: "Codex ended without a valid semantic completion.", - completionClaim: { - contractRevision: this.#taskEnvelope.completionContract.revision, - objectiveSatisfied: false, - criteria: this.#taskEnvelope.completionContract.criteria.map((criterion) => ({ - criterionId: criterion.id, - status: "unknown", - evidenceRefs: [], - })), - remainingWork: [{ description: "Review the incomplete Codex turn.", blocksCompletion: true }], - }, - evidence: [], - verification: [], - attentionRequests: [], - artifacts: [], - }, "missing-semantic-result"); + this.#emit("harness.diagnostic", { + code: "semantic_result_missing", + message: `Turn ${turnStatus} without a schema-valid semantic proposal; recovery is required.`, + }); } - const result = this.#result; - if (result === null) throw new Error("semantic result finalization invariant failed"); this.#terminal = true; - this.#emit("run.terminal", { - schema: "paperclip.prp.terminal.v1", - turnTerminalState: terminalState(turnStatus), - runTerminalState: dispositionRunState(result), - reportedWorkDisposition: result.reportedWorkDisposition, + } + + #mapTerminalTurn(turn: Record, fallbackTurnId: string): void { + const turnId = text(turn.id, fallbackTurnId || this.#activeTurnId || ""); + const status = text(turn.status, "completed"); + const candidate = this.#resultFromTurn(turn) ?? this.#result; + const fingerprint = canonicalJson({ + terminalState: terminalState(status), + error: turn.error ?? null, + result: candidate, }); + const previous = this.#terminalTurns.get(turnId); + if (previous !== undefined) { + if (previous !== fingerprint) { + this.#emit("harness.diagnostic", { + code: "conflicting_turn_terminal", + message: "Provider sent materially different terminal facts for an already terminal turn.", + }, { turnId }); + } + return; + } + this.#terminalTurns.set(turnId, fingerprint); + if (candidate !== null && this.#result === null) { + this.#commitResult(candidate, text(this.#resultItemFromTurn(turn)?.id), turnId); + } + const eventType = + status === "failed" + ? "turn.failed" + : status === "interrupted" + ? "turn.interrupted" + : status === "cancelled" + ? "turn.cancelled" + : "turn.completed"; + this.#emit(eventType, { status, error: turn.error ?? null }, { turnId }); + this.#activeTurnId = null; + this.#finalize(status); + } + + #resultItemFromTurn(turn: Record): Record | null { + if (!Array.isArray(turn.items)) return null; + return ( + turn.items + .map(record) + .reverse() + .find((value) => text(value.type) === "agentMessage" && tryParseResult(value.text) !== null) ?? + null + ); + } + + #resultFromTurn(turn: Record): PrpStructuredRunResult | null { + const item = this.#resultItemFromTurn(turn); + return item === null ? null : tryParseResult(item.text); } #requireActiveTurn(turnId: string): void { @@ -761,8 +817,11 @@ class CodexHarnessSession implements HarnessSession { if (!this.#capabilities[operation]) throw this.#unsupported(operation, "capability not advertised"); } - #unsupported(operation: string, detail: unknown): UnsupportedCodexOperationError { - const error = new UnsupportedCodexOperationError(operation, String(detail)); + #unsupported(operation: string, detail: unknown): HarnessCapabilityUnavailableError { + const error = new HarnessCapabilityUnavailableError( + operation, + redactCodexDiagnostic(String(detail)), + ); this.#diagnoseUnsupported(operation, error.message); return error; } @@ -793,36 +852,13 @@ class CodexHarnessSession implements HarnessSession { ...(refs.itemId ? { itemId: refs.itemId } : {}), eventType, schemaVersion: 1, - priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1, + priority: eventType === "run.result.proposed" ? 0 : 1, emittedAt: this.#now().toISOString(), payload, }); } } -export function isSkilllessContext(context: Phase4ModelContextSnapshot): boolean { - const serialized = JSON.stringify(context).toLowerCase(); - return ( - context.instructionSources.length === 0 && - context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS && - context.modelInputKinds.length === 1 && - context.modelInputKinds[0] === "text" && - context.dynamicToolNames.length === PHASE4_SEMANTIC_TOOL_NAMES.length && - context.dynamicToolNames.every((name) => - PHASE4_SEMANTIC_TOOL_NAMES.includes( - name as (typeof PHASE4_SEMANTIC_TOOL_NAMES)[number], - ), - ) && - context.instructionPolicy.skillInstructions === false && - context.instructionPolicy.appInstructions === false && - context.instructionPolicy.collaborationInstructions === false && - !serialized.includes("paperclip_api_key") && - !serialized.includes("authorization: bearer") && - !serialized.includes("/api/issues/") && - !serialized.includes('"type":"skill"') - ); -} - function isSemanticTool(tool: string): boolean { return ( tool === PHASE4_COMPLETION_TOOL_NAME || diff --git a/packages/paperclip-runner/src/mock-core/phase4-codex-runner.ts b/packages/paperclip-runner/src/mock-core/phase4-codex-runner.ts index 08983dc23a3..62f26e14584 100644 --- a/packages/paperclip-runner/src/mock-core/phase4-codex-runner.ts +++ b/packages/paperclip-runner/src/mock-core/phase4-codex-runner.ts @@ -1,5 +1,8 @@ import type { HarnessDriver, HarnessSession } from "../contracts/harness-driver.js"; import { + isSkilllessPhase4Context, + type Phase4ResultDecision, + type Phase4ResultValidationIssue, type Phase4RunMetadata, type Phase4RunTrace, type Phase4TaskEnvelope, @@ -9,15 +12,13 @@ import { applyPrpEvent, createSessionSnapshotFromMetadata, } from "../reducer/session-reducer.js"; -import type { - PrpCapabilities, - PrpEvent, - PrpStructuredRunResult, -} from "../protocol/phase1-contract.js"; import { - UnsupportedCodexOperationError, - isSkilllessContext, -} from "../drivers/codex/codex-app-server-driver.js"; + validatePrpStructuredRunResult, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, + type PrpTerminalState, +} from "../protocol/phase1-contract.js"; export interface Phase4CodexTracerInput { driver: HarnessDriver; @@ -25,10 +26,12 @@ export interface Phase4CodexTracerInput { workingDirectory: string; message?: NativeUserMessage; runId?: string; + normalizedSessionId?: string; companyId?: string; issueId?: string; environmentLeaseId?: string; runnerInstanceId?: string; + controlPlaneInstanceId?: string; steer?: string; interrupt?: boolean; timeoutMs?: number; @@ -55,22 +58,128 @@ function prpCapabilities( function contextFrom(events: PrpEvent[]) { const event = events.find( - (candidate) => candidate.eventType === "session.started" || candidate.eventType === "session.resumed", + (candidate) => + candidate.eventType === "session.started" || candidate.eventType === "session.resumed", ); const context = event?.payload.context; if (typeof context !== "object" || context === null || Array.isArray(context)) { - throw new Error("Codex session did not emit its model-context snapshot"); + throw new Error("Harness session did not emit its model-context snapshot"); } return context as Phase4RunTrace["context"]; } -function resultFrom(events: PrpEvent[]): PrpStructuredRunResult { - const event = events.find((candidate) => candidate.eventType === "run.result.proposed"); - if (event === undefined) throw new Error("Codex session emitted no semantic result"); - return event.payload as PrpStructuredRunResult; +function dispositionIssues( + result: PrpStructuredRunResult, +): Phase4ResultValidationIssue[] { + const issues: Phase4ResultValidationIssue[] = []; + const claim = result.completionClaim; + if (result.reportedWorkDisposition === "done") { + if ( + claim.objectiveSatisfied !== true || + claim.criteria.some((criterion) => criterion.status !== "satisfied") || + claim.remainingWork.some((remaining) => remaining.blocksCompletion) || + result.blocker !== undefined + ) { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: "done requires a satisfied objective and criteria with no blocking work or blocker", + }); + } + } else if (result.reportedWorkDisposition === "needs_review") { + if (result.blocker !== undefined || result.attentionRequests.length === 0) { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: "needs_review requires an attention request and must not include a blocker", + }); + } + } else if (result.reportedWorkDisposition === "blocked") { + if ( + claim.objectiveSatisfied !== false || + result.blocker === undefined || + !claim.remainingWork.some((remaining) => remaining.blocksCompletion) + ) { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: "blocked requires an unsatisfied objective, blocker details, and blocking remaining work", + }); + } + } else { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: `${result.reportedWorkDisposition} is not a Phase 4 terminal proposal`, + }); + } + return issues; } -async function consumeToTerminal( +/** Validate an advisory provider proposal against the exact controller-owned task envelope. */ +export function validatePhase4ResultProposal( + proposal: unknown, + envelope: Phase4TaskEnvelope, +): Phase4ResultDecision { + const schema = validatePrpStructuredRunResult(proposal); + if (!schema.ok) { + return { + status: "rejected", + result: null, + issues: schema.issues.map((issue) => ({ + code: "schema_validation", + path: issue.path, + message: issue.message, + })), + }; + } + + const result = schema.result; + const issues: Phase4ResultValidationIssue[] = []; + if (result.completionClaim.contractRevision !== envelope.completionContract.revision) { + issues.push({ + code: "contract_revision_mismatch", + path: "/completionClaim/contractRevision", + message: `expected exact contract revision ${envelope.completionContract.revision}`, + }); + } + + const expectedIds = new Set(envelope.completionContract.criteria.map((criterion) => criterion.id)); + const observed = new Map(); + result.completionClaim.criteria.forEach((criterion, index) => { + observed.set(criterion.criterionId, (observed.get(criterion.criterionId) ?? 0) + 1); + if (!expectedIds.has(criterion.criterionId)) { + issues.push({ + code: "unknown_criterion", + path: `/completionClaim/criteria/${index}/criterionId`, + message: `criterion ${criterion.criterionId} is not in the task envelope`, + }); + } + }); + for (const criterionId of expectedIds) { + const count = observed.get(criterionId) ?? 0; + if (count === 0) { + issues.push({ + code: "missing_criterion", + path: "/completionClaim/criteria", + message: `criterion ${criterionId} is missing`, + }); + } else if (count > 1) { + issues.push({ + code: "duplicate_criterion", + path: "/completionClaim/criteria", + message: `criterion ${criterionId} appears more than once`, + }); + } + } + issues.push(...dispositionIssues(result)); + + return issues.length === 0 + ? { status: "accepted", result: structuredClone(result), issues: [] } + : { status: "rejected", result: null, issues }; +} + +async function consumeToTurnTerminal( session: HarnessSession, timeoutMs: number, onEvent?: (event: PrpEvent) => void, @@ -80,16 +189,19 @@ async function consumeToTerminal( for await (const event of session.events()) { events.push(event); onEvent?.(event); - if (event.eventType === "run.terminal") return events; + if (isTurnTerminal(event)) return events; } - throw new Error("Codex event stream closed before run.terminal"); + throw new Error("Harness event stream closed before a turn terminal fact"); }; let timer: ReturnType | undefined; try { return await Promise.race([ consume(), new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error(`Codex tracer timed out after ${timeoutMs}ms`)), timeoutMs); + timer = setTimeout( + () => reject(new Error(`Phase 4 tracer timed out after ${timeoutMs}ms`)), + timeoutMs, + ); }), ]); } finally { @@ -97,13 +209,43 @@ async function consumeToTerminal( } } +function isTurnTerminal(event: PrpEvent): boolean { + return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes( + event.eventType, + ); +} + +function runtimeTerminal(event: PrpEvent): Pick { + if (event.eventType === "turn.completed") { + return { turnTerminalState: "completed", runTerminalState: "succeeded" }; + } + if (event.eventType === "turn.failed") { + return { turnTerminalState: "failed", runTerminalState: "failed" }; + } + if (event.eventType === "turn.interrupted") { + return { turnTerminalState: "interrupted", runTerminalState: "cancelled" }; + } + return { turnTerminalState: "cancelled", runTerminalState: "cancelled" }; +} + +function sourceSequenceContinuous(events: PrpEvent[]): boolean { + const cursors = new Map(); + for (const event of events) { + const key = `${event.sourceKind}:${event.sourceInstanceId}`; + const previous = cursors.get(key); + if (previous !== undefined && event.sourceSeq !== previous + 1) return false; + cursors.set(key, event.sourceSeq); + } + return true; +} + /** - * Small in-memory Phase 4 mock core. It accepts only the public driver contract, - * reduces events live, and replays the same canonical stream after completion. + * Small in-memory Phase 4 mock core. The driver reports provider facts; this + * controller validates semantic proposals and alone emits the run terminal. */ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promise { const runId = input.runId ?? "phase4-safe-run"; - const normalizedSessionId = `session:${runId}`; + const normalizedSessionId = input.normalizedSessionId ?? "phase4-normalized-session"; const descriptor = await input.driver.descriptor(); const capabilities = prpCapabilities(descriptor); const metadata: Phase4RunMetadata = { @@ -120,12 +262,17 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi }, capabilities, }; - const session = await input.driver.openSession({ runId, workingDirectory: input.workingDirectory }); + const session = await input.driver.openSession({ + runId, + normalizedSessionId, + workingDirectory: input.workingDirectory, + }); + const capabilityDiagnostics: string[] = []; let signalTurnStarted: (() => void) | undefined; const turnStarted = new Promise((resolve) => { signalTurnStarted = resolve; }); - const consuming = consumeToTerminal(session, input.timeoutMs ?? 180_000, (event) => { + const consuming = consumeToTurnTerminal(session, input.timeoutMs ?? 180_000, (event) => { if (event.eventType === "turn.started") signalTurnStarted?.(); }); @@ -137,31 +284,37 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi await Promise.race([ turnStarted, consuming.then(() => { - throw new Error("Codex turn reached terminal state before accepting the control command"); + throw new Error("Harness turn reached terminal state before accepting the control command"); }), ]); } if (input.steer !== undefined) { - if (session.steer === undefined) throw new Error("driver did not expose steering"); - try { - await session.steer({ turnId: turn.turnId, message: { role: "user", text: input.steer } }); - } catch (error) { - // A short real-model turn can finish between turn/start and turn/steer. - // The driver already emitted the explicit capability diagnostic; keep - // consuming the same session so the mock core records its terminal. - if (!(error instanceof UnsupportedCodexOperationError)) throw error; + if (!descriptor.capabilities.steering || session.steer === undefined) { + capabilityDiagnostics.push("steering is unavailable by driver capability contract"); + } else { + try { + await session.steer({ + turnId: turn.turnId, + message: { role: "user", text: input.steer }, + }); + } catch (error) { + capabilityDiagnostics.push(`steering degraded: ${String(error)}`); + } } } if (input.interrupt) { - if (session.interrupt === undefined) throw new Error("driver did not expose interruption"); - try { - await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" }); - } catch (error) { - if (!(error instanceof UnsupportedCodexOperationError)) throw error; + if (!descriptor.capabilities.interruption || session.interrupt === undefined) { + capabilityDiagnostics.push("interruption is unavailable by driver capability contract"); + } else { + try { + await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" }); + } catch (error) { + capabilityDiagnostics.push(`interruption degraded: ${String(error)}`); + } } } - const events = await consuming; + const providerEvents = await consuming; const ids = session.ids(); const identity = { ...metadata.identity, @@ -169,6 +322,80 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi ...(ids.providerSessionId ? { providerSessionId: ids.providerSessionId } : {}), }; metadata.identity = identity; + + const proposals = providerEvents.filter( + (event) => event.eventType === "run.result.proposed", + ); + const proposedResult = proposals.length === 1 ? structuredClone(proposals[0]!.payload) : null; + const resultDecision: Phase4ResultDecision = + proposals.length === 1 + ? validatePhase4ResultProposal(proposedResult, input.taskEnvelope) + : { + status: "rejected", + result: null, + issues: [{ + code: "schema_validation", + path: "/run.result.proposed", + message: + proposals.length === 0 + ? "the completed turn emitted no semantic proposal" + : "the completed turn emitted multiple semantic proposals", + }], + }; + + let controlSequence = 0; + const controlPlaneInstanceId = input.controlPlaneInstanceId ?? "mock-core-phase4"; + const controlEvent = ( + eventType: PrpEvent["eventType"], + payload: Record, + refs: { turnId?: string; itemId?: string } = {}, + ): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `${controlPlaneInstanceId}:${runId}:${++controlSequence}`, + sourceSeq: controlSequence, + sourceInstanceId: controlPlaneInstanceId, + sourceKind: "control_plane", + runId, + normalizedSessionId, + ...(refs.turnId ? { turnId: refs.turnId } : {}), + ...(refs.itemId ? { itemId: refs.itemId } : {}), + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" || eventType.startsWith("run.result.") ? 0 : 1, + emittedAt: new Date().toISOString(), + payload, + }); + const controlEvents = capabilityDiagnostics.map((message) => + controlEvent("runner.diagnostic", { + code: "capability_unavailable", + message, + }), + ); + controlEvents.push( + resultDecision.status === "accepted" + ? controlEvent("run.result.accepted", { + contractRevision: input.taskEnvelope.completionContract.revision, + result: resultDecision.result, + }) + : controlEvent("run.result.rejected", { + reasonCode: "semantic_result_rejected", + issues: resultDecision.issues, + recovery: { required: true, recoverable: true }, + }), + ); + const providerTerminal = providerEvents.findLast(isTurnTerminal); + if (providerTerminal === undefined) throw new Error("turn terminal invariant failed"); + const runtime = runtimeTerminal(providerTerminal); + controlEvents.push(controlEvent("run.terminal", { + schema: "paperclip.prp.terminal.v1", + ...runtime, + reportedWorkDisposition: + resultDecision.status === "accepted" + ? resultDecision.result.reportedWorkDisposition + : "yielded", + }, providerTerminal.turnId ? { turnId: providerTerminal.turnId } : {})); + const events = [...providerEvents, ...controlEvents]; + const liveSnapshot = events.reduce( applyPrpEvent, createSessionSnapshotFromMetadata({ @@ -186,37 +413,52 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi }), ); const context = contextFrom(events); - const result = resultFrom(events); const terminalCount = events.filter((event) => event.eventType === "run.terminal").length; - const resultCount = events.filter((event) => event.eventType === "run.result.proposed").length; + const turnTerminalCount = events.filter(isTurnTerminal).length; + const decisionCount = events.filter( + (event) => + event.eventType === "run.result.accepted" || event.eventType === "run.result.rejected", + ).length; const serialized = JSON.stringify(context).toLowerCase(); const itemEvents = events.filter((event) => event.eventType.startsWith("item.")); + const providerSessionId = ids.providerSessionId ?? null; return { schema: "paperclip.runner.phase4.trace.v1", metadata, context, events, - result, + proposedResult, + result: resultDecision.result, + resultDecision, liveSnapshot, replaySnapshot, diagnostics: events - .filter((event) => event.eventType === "runner.diagnostic" || event.eventType === "harness.diagnostic") - .map((event) => String(event.payload.message ?? event.payload.code ?? "diagnostic")), + .filter( + (event) => + event.eventType === "runner.diagnostic" || + event.eventType === "harness.diagnostic" || + event.eventType === "run.result.rejected", + ) + .map((event) => String(event.payload.message ?? event.payload.reasonCode ?? event.payload.code ?? "diagnostic")), assertions: { - exactlyOneTerminalResult: terminalCount === 1 && resultCount === 1, + exactlyOneTerminalResult: + terminalCount === 1 && turnTerminalCount === 1 && decisionCount === 1, + proposalAccepted: resultDecision.status === "accepted", liveReplayParity: JSON.stringify(liveSnapshot) === JSON.stringify(replaySnapshot), stableIdentity: + providerSessionId !== null && + new Set([runId, normalizedSessionId, ids.driverSessionId, providerSessionId]).size === 4 && events.every( (event) => event.runId === runId && event.normalizedSessionId === normalizedSessionId, - ) && ids.driverSessionId.length > 0, - sourceSequenceContinuous: events.every( - (event, index) => event.sourceSeq === index + 1, - ), + ), + sourceSequenceContinuous: sourceSequenceContinuous(events), stableItemIdentity: itemEvents.every( (event) => typeof event.itemId === "string" && event.itemId.length > 0, ), - contextIsSkillless: isSkilllessContext(context), + contextIsSkillless: isSkilllessPhase4Context(context, { + dynamicTools: descriptor.capabilities.dynamicTools === true, + }), unrelatedSkillsAbsent: context.instructionSources.length === 0 && context.instructionPolicy.skillInstructions === false &&