import { CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, CODEX_INVALID_REQUEST, CODEX_METHOD_NOT_FOUND, CODEX_RESULT_OUTPUT_SCHEMA, CodexAppServerDriver, CodexRpcError, FakeCodexTransport, HarnessCapabilityUnavailableError, HarnessOperationAlreadyTerminalError, HarnessReconciliationError, HarnessStaleTurnError, WORKSPACE, applyPrpEvent, collectUntilTerminal, createCodexTaskEnvelope, createIsolatedCodexAppServerArgs, createSessionSnapshotFromMetadata, describe, envelope, expect, isSkilllessCodexContext, it, liveConsoleFixturePath, loadLiveConsoleConformanceFixture, makeDriver, replayPersistedCodexEvents, result, reverseObjectKeys, runCodexCodexTracer, traceCompletedProposal, validateCodexResultProposal, validatePrpEvent, vi, type CodexAppServerTransport, type CodexRpcNotification, type CodexRpcServerRequest, type CodexServerRequestHandler, type CodexTraceInterpretation, type HarnessRuntimeRequestResolution, type PrpCapabilities, type PrpEvent, type PrpStructuredRunResult, } from "./codex-app-server-driver.test-support.js"; describe("Codex app-server Codex driver", () => { it("captures an exact skillless model/environment snapshot with credentials absent", async () => { const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ runId: "run-context", normalizedSessionId: "normalized-context", workingDirectory: WORKSPACE, }); const iterator = session.events()[Symbol.asyncIterator](); const first = await iterator.next(); const context = first.value?.payload.context as Parameters< typeof isSkilllessCodexContext >[0]; expect(context).toMatchObject({ codexVersion: "codex-cli/0.132.0", model: "gpt-test", modelProvider: "openai", workingDirectory: WORKSPACE, approvalPolicy: "untrusted", instructionSources: [], instructionPolicy: { skillInstructions: false, appInstructions: false, collaborationInstructions: true, }, environmentKeys: ["LANG", "PATH"], envelope, }); expect(isSkilllessCodexContext(context)).toBe(true); expect(JSON.stringify(context)).not.toContain("must-not-pass"); expect(JSON.stringify(transport.calls)).not.toContain("RANDOM_SKILL_PATH"); expect( transport.calls.find((call) => call.method === "thread/start")?.params, ).toMatchObject({ approvalPolicy: "untrusted", config: { "skills.include_instructions": false, include_apps_instructions: false, include_collaboration_mode_instructions: true, "features.image_generation": false, }, permissions: "paperclip-runner-workspace-only", runtimeWorkspaceRoots: [WORKSPACE], dynamicTools: [{ name: "paperclip_finish" }, { name: "paperclip_block" }], }); const commandPolicy = transport.calls.find( (call) => call.method === "thread/start", )?.params.config as Record; expect(JSON.stringify(commandPolicy)).not.toContain("/isolated/home"); expect(JSON.stringify(commandPolicy)).not.toContain("/isolated/codex"); expect(JSON.stringify(commandPolicy)).not.toContain("CODEX_HOME"); const appServerArgs = createIsolatedCodexAppServerArgs({ PATH: "/bin", LANG: "C.UTF-8", HOME: "/isolated/home", CODEX_HOME: "/isolated/codex", }); expect(appServerArgs).toEqual( expect.arrayContaining([ expect.stringContaining( 'default_permissions="paperclip-runner-workspace-only"', ), expect.stringContaining( "permissions.paperclip-runner-workspace-only.filesystem=", ), "permissions.paperclip-runner-workspace-only.network.enabled=false", 'shell_environment_policy.inherit="none"', expect.stringContaining( 'shell_environment_policy.set={PATH="/bin",LANG="C.UTF-8"}', ), "--disable", "image_generation", ]), ); expect( appServerArgs.find((arg) => arg.startsWith("permissions.")), ).toContain('"/isolated/home"="none"'); expect( appServerArgs.find((arg) => arg.startsWith("permissions.")), ).toContain('"/isolated/codex"="none"'); expect(JSON.stringify(appServerArgs)).not.toContain("HOME="); expect(CODEX_RESULT_OUTPUT_SCHEMA.properties.schema).toEqual({ type: "string", const: "paperclip.run_result.v1", }); expect( CODEX_BLOCK_RESULT_OUTPUT_SCHEMA.properties.reportedWorkDisposition, ).toEqual({ type: "string", const: "blocked", }); }); it.each(["never", "on-request", "untrusted"] as const)( "pins approval policy %s for both thread start and resume", async (approvalPolicy) => { const first = new FakeCodexTransport(); const second = new FakeCodexTransport(); const driver = makeDriver([first, second], { approvalPolicy }); const original = await driver.openSession({ runId: `run-policy-${approvalPolicy}`, normalizedSessionId: `normalized-policy-${approvalPolicy}`, workingDirectory: WORKSPACE, }); const started = await original.events()[Symbol.asyncIterator]().next(); const snapshot = await original.snapshot(); await original.close({ reason: "policy recovery test" }); const recovered = await driver.recoverSession?.(snapshot); expect(recovered).toMatchObject({ recovered: true }); const resumed = await recovered!.session!.events()[Symbol.asyncIterator]().next(); expect(first.calls.find((call) => call.method === "thread/start")?.params) .toMatchObject({ approvalPolicy }); expect(second.calls.find((call) => call.method === "thread/resume")?.params) .toMatchObject({ approvalPolicy }); expect(started.value).toMatchObject({ eventType: "session.started", payload: { context: { approvalPolicy } } }); expect(resumed.value).toMatchObject({ eventType: "session.resumed", payload: { context: { approvalPolicy } } }); await recovered?.session?.close({ reason: "policy recovery complete" }); }, ); it("allows eval fixtures to opt out of Codex collaboration instructions", async () => { const transport = new FakeCodexTransport(); const session = await makeDriver([transport], { includeCollaborationModeInstructions: false, }).openSession({ runId: "run-no-collaboration-instructions", normalizedSessionId: "session-no-collaboration-instructions", workingDirectory: WORKSPACE, }); const first = await session.events()[Symbol.asyncIterator]().next(); expect(first.value?.payload.context).toMatchObject({ instructionPolicy: { collaborationInstructions: false }, }); expect( transport.calls.find((call) => call.method === "thread/start")?.params, ).toMatchObject({ config: { include_collaboration_mode_instructions: false }, }); }); it("negotiates genuine plan mode with collaboration instructions and read-only workspace access", async () => { const transport = new FakeCodexTransport(); const driver = makeDriver([transport], { requestedCollaborationMode: "plan", }); const session = await driver.openSession({ runId: "run-plan", normalizedSessionId: "session-plan", workingDirectory: WORKSPACE, }); expect( transport.calls.find((call) => call.method === "thread/start")?.params, ).toMatchObject({ permissions: "paperclip-runner-workspace-read-only", config: { include_collaboration_mode_instructions: true }, }); expect( transport.calls.find((call) => call.method === "thread/start")?.params, ).not.toHaveProperty("collaborationMode"); expect( transport.calls.some((call) => call.method === "collaborationMode/list"), ).toBe(true); await expect( session.startTurn({ message: { role: "user", text: "Author a plan." }, requestedCollaborationMode: "plan", }), ).resolves.toMatchObject({ effectiveCollaborationMode: "plan" }); expect( transport.calls.find((call) => call.method === "turn/start")?.params, ).toMatchObject({ collaborationMode: { mode: "plan", settings: { model: "gpt-test", reasoning_effort: "high", developer_instructions: null, }, }, permissions: "paperclip-runner-workspace-read-only", }); }); it("fails closed when the installed app-server does not confirm plan mode", async () => { const transport = new FakeCodexTransport(); transport.confirmCollaborationMode = false; await expect( makeDriver([transport], { requestedCollaborationMode: "plan", }).openSession({ runId: "run-plan-unsupported", normalizedSessionId: "session-plan-unsupported", workingDirectory: WORKSPACE, }), ).rejects.toThrow("planning_mode_unsupported"); }); it("refuses to turn host credential roots into model-writable workspaces", async () => { await expect( makeDriver([], { environment: { HOME: "/isolated/home", CODEX_HOME: WORKSPACE, }, }).openSession({ runId: "run-codex-home", normalizedSessionId: "normalized-codex-home", workingDirectory: WORKSPACE, }), ).rejects.toThrow("cannot overlap host CODEX_HOME"); await expect( makeDriver([], { environment: { HOME: `${WORKSPACE}/host-home`, CODEX_HOME: "/isolated/codex", }, }).openSession({ runId: "run-host-home", normalizedSessionId: "normalized-host-home", workingDirectory: WORKSPACE, }), ).rejects.toThrow("cannot contain the host HOME"); await expect( makeDriver([]).openSession({ runId: "run-root", normalizedSessionId: "normalized-root", workingDirectory: "/", }), ).rejects.toThrow("cannot be a filesystem root"); }); it("steers and interrupts an active turn without replacing the session", async () => { const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ runId: "run-controls", normalizedSessionId: "normalized-controls", workingDirectory: WORKSPACE, }); const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." }, }); await session.steer?.({ turnId, message: { role: "user", text: "Use a shorter answer." }, }); await session.interrupt?.({ turnId, reason: "operator requested" }); expect(transport.calls.map((call) => call.method)).toEqual([ "initialize", "thread/start", "thread/goal/get", "turn/start", "turn/steer", "turn/interrupt", ]); expect(session.ids()).toEqual({ driverSessionId: "thread-1", providerSessionId: "provider-session-1", displayId: "thread-1", }); }); it("loads the deterministic Live console wire fixture", async () => { const fixture = await loadLiveConsoleConformanceFixture( liveConsoleFixturePath, ); expect( fixture.runtimeRequests.map(({ requestKind }) => requestKind), ).toEqual([ "command_approval", "file_approval", "permission_approval", "user_input", "elicitation", ]); expect(fixture.goals.map(({ action }) => action)).toEqual([ "get", "set", "pause", "resume", "clear", ]); }); it("holds browser-resolved upstream requests and returns the exact fixture responses", async () => { const fixture = await loadLiveConsoleConformanceFixture( liveConsoleFixturePath, ); for (const scenario of fixture.runtimeRequests) { const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ runId: `run-${scenario.id}`, normalizedSessionId: `normalized-${scenario.id}`, workingDirectory: WORKSPACE, }); const { turnId } = await session.startTurn({ message: { role: "user", text: "Exercise request." }, }); const response = transport.invoke({ id: scenario.id, method: scenario.method, params: { threadId: "thread-1", turnId, itemId: `item-${scenario.id}`, reason: `fixture ${scenario.id}`, }, }); await new Promise((resolve) => setTimeout(resolve, 0)); expect(session.pendingRuntimeRequests?.()).toEqual([ expect.objectContaining({ requestId: scenario.id, requestKind: scenario.requestKind, method: scenario.method, turnId, }), ]); await expect( session.resolveRuntimeRequest?.({ requestId: scenario.id, turnId, resolution: { action: "forged", } as unknown as HarnessRuntimeRequestResolution, }), ).rejects.toThrow("unsupported action"); expect(session.pendingRuntimeRequests?.()).toHaveLength(1); await session.resolveRuntimeRequest?.({ requestId: scenario.id, turnId, resolution: scenario.resolution as HarnessRuntimeRequestResolution, }); expect(await response).toEqual(scenario.expectedResponse); expect(session.pendingRuntimeRequests?.()).toEqual([]); await expect( session.resolveRuntimeRequest?.({ requestId: scenario.id, turnId, resolution: { action: "cancel" }, }), ).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); await session.close({ reason: "fixture complete" }); } }); it("acknowledges same-turn steering and rejects stale or child steering", async () => { const fixture = await loadLiveConsoleConformanceFixture( liveConsoleFixturePath, ); const transport = new FakeCodexTransport("thread-root"); const session = await makeDriver([transport]).openSession({ runId: "run-live-console-controls", normalizedSessionId: "normalized-live-console-controls", workingDirectory: WORKSPACE, }); const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." }, }); await session.steer?.({ turnId, message: { role: "user", text: "Stay concise." }, correlationId: "queued-comment-1", }); transport.push("item/completed", { kind: "steering_acknowledgement", status: "acknowledged", commandId: "runnerd-steer-command", }); await new Promise((resolve) => setTimeout(resolve, 0)); await session.steer?.({ turnId, message: { role: "user", text: "Stay concise." }, correlationId: "queued-comment-1", }); expect( transport.calls.find((call) => call.method === "turn/steer")?.params, ).toMatchObject({ correlationId: "queued-comment-1" }); expect( transport.calls.filter((call) => call.method === "turn/steer"), ).toHaveLength(1); transport.rejectMethods.set( "turn/steer", new Error("provider rejected steering"), ); const rejectedSteering = await session .steer?.({ turnId, message: { role: "user", text: "Try this again later." }, correlationId: "queued-comment-2", }) .then( () => null, (error: unknown) => error, ); expect(rejectedSteering).toBeInstanceOf(Error); expect(rejectedSteering).not.toBeInstanceOf( HarnessCapabilityUnavailableError, ); expect(String(rejectedSteering)).toContain("provider rejected steering"); transport.rejectMethods.delete("turn/steer"); await expect( session.steer?.({ turnId: fixture.controls.staleTurnSteer.turnId!, message: { role: "user", text: "Stale." }, }), ).rejects.toBeInstanceOf(HarnessStaleTurnError); transport.push("thread/started", { thread: fixture.lineage.childThread }); await new Promise((resolve) => setTimeout(resolve, 0)); expect(session.lineage?.()).toEqual( expect.arrayContaining([ expect.objectContaining({ threadId: "thread-root", parentThreadId: null, depth: 0, }), expect.objectContaining({ threadId: "thread-child", parentThreadId: "thread-root", depth: 1, nickname: "Scout", role: "researcher", }), ]), ); await expect( session.steer?.({ turnId: "thread-child", message: { role: "user", text: "Do not emulate child steering." }, }), ).rejects.toBeInstanceOf(HarnessStaleTurnError); const events = session.events()[Symbol.asyncIterator](); const observed: PrpEvent[] = []; // PRP v2 adds a capability and authoritative goal snapshot immediately // after session start, so include those two events in this bounded read. for (let index = 0; index < 11; index += 1) { const next = await events.next(); if (next.done) break; observed.push(next.value); } expect(observed).toEqual( expect.arrayContaining([ expect.objectContaining({ eventType: "item.completed", itemId: `${turnId}:steer:queued-comment-1`, payload: expect.objectContaining({ kind: "steering_acknowledgement", status: "acknowledged", }), }), expect.objectContaining({ eventType: "harness.diagnostic", payload: expect.objectContaining({ code: "stale_turn_rejected" }), }), expect.objectContaining({ eventType: "item.started", payload: expect.objectContaining({ kind: "thread_lineage" }), }), ]), ); expect(observed.some((event) => event.eventType === "session.failed")) .toBe(false); await session.close({ reason: "fixture complete" }); }); it("queues an interrupt before turn identity and reports a terminal race precisely", async () => { const transport = new FakeCodexTransport(); let releaseStart!: (value: Record) => void; transport.turnStartResponse = new Promise((resolve) => { releaseStart = resolve; }); const session = await makeDriver([transport]).openSession({ runId: "run-interrupt-races", normalizedSessionId: "normalized-interrupt-races", workingDirectory: WORKSPACE, }); const starting = session.startTurn({ message: { role: "user", text: "Start slowly." }, }); await new Promise((resolve) => setTimeout(resolve, 0)); await session.interrupt?.({ reason: "operator" }); releaseStart({ turn: { id: "turn-1", status: "inProgress", items: [] } }); await expect(starting).resolves.toEqual({ turnId: "turn-1", effectiveCollaborationMode: "default", }); expect(transport.calls.map(({ method }) => method)).toEqual( expect.arrayContaining(["turn/start", "turn/interrupt"]), ); transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "interrupted", items: [] }, }); await new Promise((resolve) => setTimeout(resolve, 0)); await expect( session.interrupt?.({ turnId: "turn-1" }), ).rejects.toBeInstanceOf(HarnessOperationAlreadyTerminalError); await session.close({ reason: "fixture complete" }); }); it("maps all goal operations and advertises an exact unsupported state", async () => { const fixture = await loadLiveConsoleConformanceFixture( liveConsoleFixturePath, ); const transport = new FakeCodexTransport(); const session = await makeDriver([transport]).openSession({ runId: "run-goals", normalizedSessionId: "normalized-goals", workingDirectory: WORKSPACE, }); await session.goal?.({ action: "get" }); await session.goal?.({ action: "set", objective: "Ship the Live console tracer", tokenBudget: 4096, }); await session.goal?.({ action: "pause" }); await session.goal?.({ action: "resume" }); await session.goal?.({ action: "clear" }); const goalCalls = transport.calls.filter(({ method }) => method.startsWith("thread/goal/"), ); expect( goalCalls.slice(1).map(({ method, params }) => ({ method, params: Object.fromEntries( Object.entries(params).filter(([key]) => key !== "threadId"), ), })), ).toEqual(fixture.goals.map(({ method, params }) => ({ method, params }))); expect( (await makeDriver([new FakeCodexTransport()]).descriptor()).capabilities, ).toMatchObject({ goals: true }); // Both denials a real app-server sends: the method is absent, and the // build has the feature switched off. for (const denial of [ new CodexRpcError( '{"code":-32601,"message":"method not found"}', CODEX_METHOD_NOT_FOUND, ), new CodexRpcError( '{"code":-32600,"message":"goals feature is disabled"}', CODEX_INVALID_REQUEST, ), ]) { const unsupportedTransport = new FakeCodexTransport(); unsupportedTransport.rejectMethods.set("thread/goal/get", denial); const unsupportedDriver = makeDriver([unsupportedTransport]); const unsupported = await unsupportedDriver.openSession({ runId: "run-goals-unsupported", normalizedSessionId: "normalized-goals-unsupported", workingDirectory: WORKSPACE, }); expect((await unsupportedDriver.descriptor()).capabilities).toMatchObject( { goals: false }, ); await expect( unsupported.goal?.({ action: "get" }), ).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); for (const action of ["set", "pause", "resume", "clear"] as const) { await expect( unsupported.goal?.({ action, ...(action === "set" ? { objective: "Must not reach the provider" } : {}) }), ).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); } expect(unsupportedTransport.calls.filter(({ method }) => method === "thread/goal/set" || method === "thread/goal/clear", )).toEqual([]); await unsupported.close({ reason: "fixture complete" }); } await session.close({ reason: "fixture complete" }); }); it("keeps goal support advertised when the probe fails transiently", async () => { // A transport or protocol failure is evidence about this call, not about // what the provider implements, so it must not retire the capability. for (const failure of [ new Error("codex app-server transport closed"), new CodexRpcError('{"code":-32603,"message":"internal error"}', -32_603), ]) { const transport = new FakeCodexTransport(); transport.rejectMethods.set("thread/goal/get", failure); const driver = makeDriver([transport]); const session = await driver.openSession({ runId: "run-goals-transient", normalizedSessionId: "normalized-goals-transient", workingDirectory: WORKSPACE, }); expect((await driver.descriptor()).capabilities).toMatchObject({ goals: true, }); // The capability survives, so the operation is still offered and the // next call reaches the provider instead of failing closed locally. transport.rejectMethods.delete("thread/goal/get"); await expect(session.goal?.({ action: "get" })).resolves.toBeNull(); expect( transport.calls.filter(({ method }) => method === "thread/goal/get"), ).toHaveLength(2); await session.close({ reason: "fixture complete" }); } }); });