fix(runner): enforce phase 4 status authority

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-08 01:53:48 +00:00
1 parent 2161ddcc4a
commit 6391608e8f
6 files changed
+763 -151

No files matched your search

@@ -40,7 +40,9 @@ function formatSummary(trace: Awaited<ReturnType<typeof runPhase4CodexTracer>>):
`provider session: ${trace.metadata.identity.providerSessionId}`,
`model: ${trace.context.model} (${trace.context.modelProvider})`,
`events: ${trace.events.length}`,
`result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`,
trace.result === null
? `result: rejected — ${trace.resultDecision.issues.map((issue) => issue.message).join("; ")}`
: `result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`,
checks,
].join("\n");
}
@@ -11,9 +11,29 @@ export interface HarnessDriverDescriptor {
export interface OpenHarnessSessionInput {
runId: string;
normalizedSessionId: string;
workingDirectory: string;
}
export class HarnessCapabilityUnavailableError extends Error {
readonly operation: string;
constructor(operation: string, detail: string) {
super(`${operation} is unavailable: ${detail}`);
this.name = "HarnessCapabilityUnavailableError";
this.operation = operation;
}
}
export class HarnessReconciliationError extends Error {
readonly recoverable = true;
constructor(message: string) {
super(message);
this.name = "HarnessReconciliationError";
}
}
export interface PersistedHarnessSession {
driverKind: string;
driverSessionId: string;
@@ -75,12 +75,15 @@ export interface Phase4RunTrace {
metadata: Phase4RunMetadata;
context: Phase4ModelContextSnapshot;
events: PrpEvent[];
result: PrpStructuredRunResult;
proposedResult: unknown | null;
result: PrpStructuredRunResult | null;
resultDecision: Phase4ResultDecision;
liveSnapshot: SessionSnapshot;
replaySnapshot: SessionSnapshot;
diagnostics: string[];
assertions: {
exactlyOneTerminalResult: boolean;
proposalAccepted: boolean;
liveReplayParity: boolean;
stableIdentity: boolean;
sourceSequenceContinuous: boolean;
@@ -91,6 +94,22 @@ export interface Phase4RunTrace {
};
}
export interface Phase4ResultValidationIssue {
code:
| "schema_validation"
| "contract_revision_mismatch"
| "unknown_criterion"
| "missing_criterion"
| "duplicate_criterion"
| "invalid_disposition";
path: string;
message: string;
}
export type Phase4ResultDecision =
| { status: "accepted"; result: PrpStructuredRunResult; issues: [] }
| { status: "rejected"; result: null; issues: Phase4ResultValidationIssue[] };
export function createPhase4TaskEnvelope(input: {
objective: string;
contractRevision?: string;
@@ -273,3 +292,28 @@ export const PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA = {
},
},
} as const;
export function isSkilllessPhase4Context(
context: Phase4ModelContextSnapshot,
options: { dynamicTools: boolean } = { dynamicTools: true },
): boolean {
const serialized = JSON.stringify(context).toLowerCase();
const expectedTools = options.dynamicTools ? [...PHASE4_SEMANTIC_TOOL_NAMES] : [];
return (
context.instructionSources.length === 0 &&
context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS &&
context.modelInputKinds.length === 1 &&
context.modelInputKinds[0] === "text" &&
context.dynamicToolNames.length === expectedTools.length &&
context.dynamicToolNames.every((name) =>
expectedTools.includes(name as (typeof expectedTools)[number]),
) &&
context.instructionPolicy.skillInstructions === false &&
context.instructionPolicy.appInstructions === false &&
context.instructionPolicy.collaborationInstructions === false &&
!serialized.includes("paperclip_api_key") &&
!serialized.includes("authorization: bearer") &&
!serialized.includes("/api/issues/") &&
!serialized.includes('"type":"skill"')
);
}
@@ -4,7 +4,9 @@ import {
PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
PHASE4_RESULT_OUTPUT_SCHEMA,
createPhase4TaskEnvelope,
isSkilllessPhase4Context,
} from "../../contracts/phase4.js";
import { HarnessCapabilityUnavailableError } from "../../contracts/harness-driver.js";
import {
applyPrpEvent,
createSessionSnapshotFromMetadata,
@@ -17,9 +19,11 @@ import {
} from "../../protocol/phase1-contract.js";
import {
CodexAppServerDriver,
UnsupportedCodexOperationError,
isSkilllessContext,
} from "./codex-app-server-driver.js";
import {
runPhase4CodexTracer,
validatePhase4ResultProposal,
} from "../../mock-core/phase4-codex-runner.js";
import type {
CodexAppServerTransport,
CodexRpcNotification,
@@ -61,6 +65,7 @@ class FakeCodexTransport implements CodexAppServerTransport {
readonly queue = new TestQueue<CodexRpcNotification>();
handler: CodexServerRequestHandler = async () => ({});
rejectMethods = new Map<string, Error>();
readResponse: Record<string, unknown> | null = null;
constructor(
readonly threadId = "thread-1",
@@ -76,7 +81,13 @@ class FakeCodexTransport implements CodexAppServerTransport {
}
if (method === "thread/start" || method === "thread/resume") {
return {
thread: { id: this.threadId, sessionId: this.providerSessionId, modelProvider: "openai", cwd: "/workspace", turns: [] },
thread: {
id: this.threadId,
sessionId: this.providerSessionId,
modelProvider: "openai",
cwd: "/workspace",
turns: [],
},
model: "gpt-test",
modelProvider: "openai",
cwd: "/workspace",
@@ -87,7 +98,7 @@ class FakeCodexTransport implements CodexAppServerTransport {
}
if (method === "turn/start") return { turn: { id: "turn-1", status: "inProgress", items: [] } };
if (method === "thread/read") {
return { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } };
return this.readResponse ?? { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } };
}
return {};
}
@@ -160,17 +171,69 @@ async function collectUntilTerminal(events: AsyncIterable<PrpEvent>): Promise<Pr
const collected: PrpEvent[] = [];
for await (const event of events) {
collected.push(event);
if (event.eventType === "run.terminal") break;
if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) break;
}
return collected;
}
function reverseObjectKeys(value: unknown): unknown {
if (Array.isArray(value)) return value.map(reverseObjectKeys);
if (typeof value !== "object" || value === null) return value;
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.reverse()
.map(([key, entry]) => [key, reverseObjectKeys(entry)]),
);
}
async function traceCompletedProposal(
proposal: PrpStructuredRunResult | null,
options: {
runId?: string;
normalizedSessionId?: string;
capabilities?: Record<string, boolean>;
steer?: string;
interrupt?: boolean;
} = {},
) {
const transport = new FakeCodexTransport();
const driver = makeDriver([transport], { capabilities: options.capabilities });
const traced = runPhase4CodexTracer({
driver,
taskEnvelope: envelope,
workingDirectory: "/workspace",
runId: options.runId,
normalizedSessionId: options.normalizedSessionId,
steer: options.steer,
interrupt: options.interrupt,
});
while (!transport.calls.some((call) => call.method === "turn/start")) {
await new Promise((resolve) => setTimeout(resolve, 0));
}
transport.push("turn/started", {
threadId: "thread-1",
turn: { id: "turn-1", status: "inProgress" },
});
if (proposal !== null) {
transport.push("item/completed", {
threadId: "thread-1",
turnId: "turn-1",
item: { id: "answer-1", type: "agentMessage", text: JSON.stringify(proposal) },
});
}
transport.push("turn/completed", {
threadId: "thread-1",
turn: { id: "turn-1", status: "completed", items: [] },
});
return { trace: await traced, transport };
}
describe("Codex app-server Phase 4 driver", () => {
it("passes the common typed-event contract and produces exactly one terminal result", async () => {
it("passes the common typed-event contract and reports one provider turn terminal", async () => {
const transport = new FakeCodexTransport();
const driver = makeDriver([transport]);
const descriptor = await driver.descriptor();
const session = await driver.openSession({ runId: "run-1", workingDirectory: "/workspace" });
const session = await driver.openSession({ runId: "run-1", normalizedSessionId: "normalized-1", workingDirectory: "/workspace" });
const turn = await session.startTurn({ message: { role: "user", text: "Do the safe task." } });
transport.push("turn/started", { threadId: "thread-1", turn: { id: turn.turnId, status: "inProgress" } });
@@ -189,9 +252,10 @@ describe("Codex app-server Phase 4 driver", () => {
const events = await collectUntilTerminal(session.events());
expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true);
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(1);
expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1);
expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(0);
expect(events.map((event) => event.eventType)).toEqual(expect.arrayContaining([
"session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "run.terminal",
"session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "turn.completed",
"runtime_request.created", "runtime_request.resolved",
]));
@@ -208,7 +272,7 @@ describe("Codex app-server Phase 4 driver", () => {
};
const metadata = {
fixtureName: "phase4-conformance",
identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "session:run-1" },
identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "normalized-1" },
capabilities,
};
const live = events.reduce(applyPrpEvent, createSessionSnapshotFromMetadata(metadata));
@@ -220,10 +284,10 @@ describe("Codex app-server Phase 4 driver", () => {
it("captures an exact skillless model/environment snapshot with credentials absent", async () => {
const transport = new FakeCodexTransport();
const session = await makeDriver([transport]).openSession({ runId: "run-context", workingDirectory: "/workspace" });
const session = await makeDriver([transport]).openSession({ runId: "run-context", normalizedSessionId: "normalized-context", workingDirectory: "/workspace" });
const iterator = session.events()[Symbol.asyncIterator]();
const first = await iterator.next();
const context = first.value?.payload.context as Parameters<typeof isSkilllessContext>[0];
const context = first.value?.payload.context as Parameters<typeof isSkilllessPhase4Context>[0];
expect(context).toMatchObject({
codexVersion: "codex-cli/0.132.0",
model: "gpt-test",
@@ -239,7 +303,7 @@ describe("Codex app-server Phase 4 driver", () => {
environmentKeys: ["CODEX_HOME", "HOME", "LANG", "PATH"],
envelope,
});
expect(isSkilllessContext(context)).toBe(true);
expect(isSkilllessPhase4Context(context)).toBe(true);
expect(JSON.stringify(context)).not.toContain("must-not-pass");
expect(JSON.stringify(transport.calls)).not.toContain("RANDOM_SKILL_PATH");
expect(transport.calls.find((call) => call.method === "thread/start")?.params).toMatchObject({
@@ -263,7 +327,7 @@ describe("Codex app-server Phase 4 driver", () => {
it("steers and interrupts an active turn without replacing the session", async () => {
const transport = new FakeCodexTransport();
const session = await makeDriver([transport]).openSession({ runId: "run-controls", workingDirectory: "/workspace" });
const session = await makeDriver([transport]).openSession({ runId: "run-controls", normalizedSessionId: "normalized-controls", workingDirectory: "/workspace" });
const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } });
await session.steer?.({ turnId, message: { role: "user", text: "Use a shorter answer." } });
await session.interrupt?.({ turnId, reason: "operator requested" });
@@ -275,22 +339,28 @@ describe("Codex app-server Phase 4 driver", () => {
it("makes duplicate semantic completion idempotent and rejects changed payloads", async () => {
const transport = new FakeCodexTransport();
const session = await makeDriver([transport]).openSession({ runId: "run-result", workingDirectory: "/workspace" });
const session = await makeDriver([transport]).openSession({ runId: "run-result", normalizedSessionId: "normalized-result", workingDirectory: "/workspace" });
await session.startTurn({ message: { role: "user", text: "Complete." } });
const request = { id: 1, method: "item/tool/call", params: { threadId: "thread-1", turnId: "turn-1", callId: "call-1", tool: "paperclip_finish", arguments: result } };
expect(await transport.invoke(request)).toMatchObject({ success: true });
expect(await transport.invoke({ ...request, id: 2 })).toMatchObject({ success: true });
expect(await transport.invoke({
...request,
id: 2,
params: { ...request.params, arguments: reverseObjectKeys(result) },
})).toMatchObject({ success: true });
const changed = structuredClone(result);
changed.summary = "Changed after commit.";
expect(await transport.invoke({ ...request, id: 3, params: { ...request.params, arguments: changed } })).toMatchObject({ success: false });
transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } });
transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } });
const events = await collectUntilTerminal(session.events());
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1);
});
it("normalizes finish and block tools into one canonical result contract", async () => {
const transport = new FakeCodexTransport();
const session = await makeDriver([transport]).openSession({ runId: "run-tools", workingDirectory: "/workspace" });
const session = await makeDriver([transport]).openSession({ runId: "run-tools", normalizedSessionId: "normalized-tools", workingDirectory: "/workspace" });
await session.startTurn({ message: { role: "user", text: "Complete or block." } });
const blocked: PrpStructuredRunResult = {
...structuredClone(result),
@@ -339,7 +409,7 @@ describe("Codex app-server Phase 4 driver", () => {
const first = new FakeCodexTransport();
const second = new FakeCodexTransport();
const driver = makeDriver([first, second]);
const original = await driver.openSession({ runId: "run-recover", workingDirectory: "/workspace" });
const original = await driver.openSession({ runId: "run-recover", normalizedSessionId: "normalized-recover", workingDirectory: "/workspace" });
await original.startTurn({ message: { role: "user", text: "Work." } });
const snapshot = await original.snapshot();
await original.close({ reason: "transport lost" });
@@ -351,12 +421,62 @@ describe("Codex app-server Phase 4 driver", () => {
expect((await recovery?.session?.snapshot())?.activeTurnId).toBe("turn-1");
});
it("reconciles a turn completed during transport loss without replacing either session identity", async () => {
const first = new FakeCodexTransport("driver-thread-loss", "provider-session-loss");
const second = new FakeCodexTransport("driver-thread-loss", "provider-session-loss");
second.readResponse = {
thread: {
id: "driver-thread-loss",
sessionId: "provider-session-loss",
tokenUsage: { total: { inputTokens: 21, outputTokens: 8 } },
turns: [{
id: "turn-1",
status: "completed",
items: [{ id: "answer-loss", type: "agentMessage", text: JSON.stringify(result) }],
}],
},
};
const driver = makeDriver([first, second]);
const original = await driver.openSession({
runId: "runtime-run-loss",
normalizedSessionId: "controller-session-loss",
workingDirectory: "/workspace",
});
await original.startTurn({ message: { role: "user", text: "Work through loss." } });
const snapshot = await original.snapshot();
await original.close({ reason: "transport lost before terminal delivery" });
const recovery = await driver.recoverSession?.(snapshot);
expect(recovery).toMatchObject({ recovered: true });
const recovered = recovery?.session;
expect(recovered).toBeDefined();
const collecting = collectUntilTerminal(recovered!.events());
await recovered!.reconcile?.();
const events = await collecting;
expect(recovered!.ids()).toEqual({
driverSessionId: "driver-thread-loss",
providerSessionId: "provider-session-loss",
displayId: "driver-thread-loss",
});
expect(await recovered!.snapshot()).toMatchObject({
runId: "runtime-run-loss",
normalizedSessionId: "controller-session-loss",
driverSessionId: "driver-thread-loss",
providerSessionId: "provider-session-loss",
});
expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1);
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
expect(events.every((event, index) => event.sourceSeq === snapshot.lastSourceSequence! + index + 1)).toBe(true);
expect(await recovered!.usage?.()).toEqual({ total: { inputTokens: 21, outputTokens: 8 } });
});
it("degrades unsupported operations with explicit redacted diagnostics", async () => {
const transport = new FakeCodexTransport();
transport.rejectMethods.set("turn/steer", new Error("Bearer super-secret api_key=also-secret"));
const session = await makeDriver([transport]).openSession({ runId: "run-degrade", workingDirectory: "/workspace" });
const session = await makeDriver([transport]).openSession({ runId: "run-degrade", normalizedSessionId: "normalized-degrade", workingDirectory: "/workspace" });
const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } });
await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(UnsupportedCodexOperationError);
await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError);
const iterator = session.events()[Symbol.asyncIterator]();
const events: PrpEvent[] = [];
for (let index = 0; index < 5; index += 1) {
@@ -368,4 +488,152 @@ describe("Codex app-server Phase 4 driver", () => {
expect(JSON.stringify(diagnostic)).not.toContain("super-secret");
expect(JSON.stringify(diagnostic)).not.toContain("also-secret");
});
it("rejects proposals that do not match the exact task envelope", () => {
const wrongRevision = structuredClone(result);
wrongRevision.completionClaim.contractRevision = "phase4-demo-v0";
expect(validatePhase4ResultProposal(wrongRevision, envelope)).toMatchObject({
status: "rejected",
issues: [{ code: "contract_revision_mismatch" }],
});
const unknownCriterion = structuredClone(result);
unknownCriterion.completionClaim.criteria = [{
criterionId: "not-in-envelope",
status: "satisfied",
evidenceRefs: [],
}];
const criterionDecision = validatePhase4ResultProposal(unknownCriterion, envelope);
expect(criterionDecision).toMatchObject({ status: "rejected" });
if (criterionDecision.status === "rejected") {
expect(criterionDecision.issues.map((issue) => issue.code)).toEqual(
expect.arrayContaining(["unknown_criterion", "missing_criterion"]),
);
}
const invalidDone = structuredClone(result);
invalidDone.completionClaim.objectiveSatisfied = false;
expect(validatePhase4ResultProposal(invalidDone, envelope)).toMatchObject({
status: "rejected",
issues: [{ code: "invalid_disposition" }],
});
});
it.each(["needs_review", "blocked"] as const)(
"keeps a completed runtime successful for an accepted %s advisory proposal",
async (disposition) => {
const proposal: PrpStructuredRunResult = disposition === "needs_review"
? {
...structuredClone(result),
reportedWorkDisposition: "needs_review",
attentionRequests: [{ kind: "review", summary: "Review the completed artifact." }],
}
: {
...structuredClone(result),
reportedWorkDisposition: "blocked",
summary: "Waiting on fixture input.",
completionClaim: {
...structuredClone(result.completionClaim),
objectiveSatisfied: false,
criteria: [{ criterionId: "file", status: "not_satisfied", evidenceRefs: [] }],
remainingWork: [{ description: "Provide fixture input.", blocksCompletion: true }],
},
blocker: {
reasonCode: "fixture_input_missing",
owner: { kind: "external", name: "fixture owner" },
unblockAction: "Provide fixture input.",
scope: "task_wide",
},
artifacts: [],
};
const { trace } = await traceCompletedProposal(proposal);
expect(trace.resultDecision.status).toBe("accepted");
expect(trace.events.find((event) => event.eventType === "run.terminal")?.payload)
.toMatchObject({
turnTerminalState: "completed",
runTerminalState: "succeeded",
reportedWorkDisposition: disposition,
});
},
);
it("records rejected and missing proposals as recovery evidence instead of review status", async () => {
const wrongRevision = structuredClone(result);
wrongRevision.completionClaim.contractRevision = "wrong-revision";
const rejected = await traceCompletedProposal(wrongRevision);
expect(rejected.trace.result).toBeNull();
expect(rejected.trace.proposedResult).toMatchObject({ reportedWorkDisposition: "done" });
expect(rejected.trace.events.find((event) => event.eventType === "run.result.rejected")?.payload)
.toMatchObject({ recovery: { required: true, recoverable: true } });
expect(rejected.trace.events.find((event) => event.eventType === "run.terminal")?.payload)
.toMatchObject({ runTerminalState: "succeeded", reportedWorkDisposition: "yielded" });
const missing = await traceCompletedProposal(null);
expect(missing.trace.resultDecision).toMatchObject({ status: "rejected" });
expect(missing.trace.events.some((event) =>
event.eventType === "run.result.proposed" &&
event.payload.reportedWorkDisposition === "needs_review"
)).toBe(false);
});
it("preserves four independent stable identities under controller ownership", async () => {
const { trace } = await traceCompletedProposal(result, {
runId: "runtime-run-identity",
normalizedSessionId: "controller-session-identity",
});
expect(trace.assertions.stableIdentity).toBe(true);
expect(new Set([
trace.metadata.identity.runId,
trace.metadata.identity.normalizedSessionId,
trace.metadata.identity.driverSessionId,
trace.metadata.identity.providerSessionId,
]).size).toBe(4);
expect(trace.events.every((event) =>
event.runId === "runtime-run-identity" &&
event.normalizedSessionId === "controller-session-identity"
)).toBe(true);
});
it("degrades declared unsupported capabilities without Codex-specific core branches", async () => {
const capabilities = {
resume: false,
read: false,
steering: false,
interruption: false,
usage: false,
reconciliation: false,
dynamicTools: false,
};
const { trace, transport } = await traceCompletedProposal(result, {
capabilities,
steer: "Do not call this unsupported path.",
interrupt: true,
});
expect(trace.resultDecision.status).toBe("accepted");
expect(trace.assertions.contextIsSkillless).toBe(true);
expect(trace.diagnostics).toEqual(expect.arrayContaining([
expect.stringContaining("steering is unavailable"),
expect.stringContaining("interruption is unavailable"),
]));
expect(transport.calls.map((call) => call.method)).not.toEqual(
expect.arrayContaining(["turn/steer", "turn/interrupt", "thread/read"]),
);
expect(transport.calls.find((call) => call.method === "thread/start")?.params.dynamicTools)
.toEqual([]);
const directTransport = new FakeCodexTransport();
const driver = makeDriver([directTransport], { capabilities });
const session = await driver.openSession({
runId: "run-no-capabilities",
normalizedSessionId: "normalized-no-capabilities",
workingDirectory: "/workspace",
});
await expect(session.read?.()).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError);
await expect(session.usage?.()).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError);
const snapshot = await session.snapshot();
expect(await driver.recoverSession?.(snapshot)).toMatchObject({
recovered: false,
reason: "resume capability is unavailable",
});
});
});
@@ -8,6 +8,10 @@ import type {
OpenHarnessSessionInput,
PersistedHarnessSession,
} from "../../contracts/harness-driver.js";
import {
HarnessCapabilityUnavailableError,
HarnessReconciliationError,
} from "../../contracts/harness-driver.js";
import {
PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
PHASE4_BLOCK_TOOL_NAME,
@@ -68,16 +72,6 @@ class AsyncQueue<T> implements AsyncIterable<T> {
}
}
export class UnsupportedCodexOperationError extends Error {
readonly operation: string;
constructor(operation: string, detail: string) {
super(`${operation} is unsupported: ${redactCodexDiagnostic(detail)}`);
this.name = "UnsupportedCodexOperationError";
this.operation = operation;
}
}
export interface CodexAppServerDriverOptions {
taskEnvelope: Phase4TaskEnvelope;
transportFactory?: () => CodexAppServerTransport;
@@ -102,7 +96,7 @@ type CodexCapabilities = Required<
interface OpenedCodexThread {
threadId: string;
providerSessionId: string;
providerSessionId: string | null;
context: Phase4ModelContextSnapshot;
}
@@ -137,12 +131,6 @@ function terminalState(status: string): "completed" | "failed" | "interrupted" {
return "completed";
}
function dispositionRunState(
result: PrpStructuredRunResult,
): "succeeded" | "failed" {
return result.reportedWorkDisposition === "done" ? "succeeded" : "failed";
}
function tryParseResult(value: unknown): PrpStructuredRunResult | null {
let candidate = value;
if (typeof value === "string") {
@@ -156,6 +144,18 @@ function tryParseResult(value: unknown): PrpStructuredRunResult | null {
return validation.ok ? validation.result : null;
}
function canonicalJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
const object = record(value);
if (Object.keys(object).length > 0 || (typeof value === "object" && value !== null)) {
return `{${Object.keys(object)
.sort()
.map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`)
.join(",")}}`;
}
return JSON.stringify(value) ?? "undefined";
}
function finishToolSpec(): Record<string, unknown> {
return {
name: PHASE4_COMPLETION_TOOL_NAME,
@@ -242,7 +242,7 @@ export class CodexAppServerDriver implements HarnessDriver {
return this.#session({
transport,
runId: input.runId,
normalizedSessionId: `session:${input.runId}`,
normalizedSessionId: input.normalizedSessionId,
opened,
resumed: false,
sourceSequence: 0,
@@ -255,6 +255,9 @@ export class CodexAppServerDriver implements HarnessDriver {
if (!this.#caps.resume) {
return { recovered: false, reason: "resume capability is unavailable" };
}
if (!this.#caps.read) {
return { recovered: false, reason: "read capability is required for safe resume" };
}
if (!snapshot.runId || !snapshot.normalizedSessionId || !snapshot.driverSessionId) {
return { recovered: false, reason: "persisted session identity is incomplete" };
}
@@ -272,6 +275,13 @@ export class CodexAppServerDriver implements HarnessDriver {
await transport.close();
return { recovered: false, reason: "provider resumed a different session" };
}
if (
snapshot.providerSessionId &&
opened.providerSessionId !== snapshot.providerSessionId
) {
await transport.close();
return { recovered: false, reason: "provider resumed a different provider session" };
}
return {
recovered: true,
session: this.#session({
@@ -319,7 +329,7 @@ export class CodexAppServerDriver implements HarnessDriver {
if (threadId.length === 0) throw new Error("Codex thread response omitted thread.id");
return {
threadId,
providerSessionId: text(thread.sessionId, threadId),
providerSessionId: text(thread.sessionId) || null,
context: {
protocolVersion: PHASE4_CODEX_PROTOCOL_VERSION,
codexVersion: text(initialize.userAgent, "unknown"),
@@ -385,6 +395,7 @@ class CodexHarnessSession implements HarnessSession {
#result: PrpStructuredRunResult | null = null;
#resultFingerprint: string | null = null;
#terminal = false;
readonly #terminalTurns = new Map<string, string>();
constructor(input: {
transport: CodexAppServerTransport;
@@ -502,13 +513,34 @@ class CodexHarnessSession implements HarnessSession {
this.#requireCapability("reconciliation");
const snapshot = await this.read();
const thread = record(snapshot.thread);
if (text(thread.id) !== this.#opened.threadId) {
throw new HarnessReconciliationError("thread/read returned a different driver session");
}
const providerSessionId = text(thread.sessionId);
if (
this.#opened.providerSessionId !== null &&
providerSessionId !== this.#opened.providerSessionId
) {
throw new HarnessReconciliationError("thread/read returned a different provider session");
}
const turns = Array.isArray(thread.turns) ? thread.turns.map(record) : [];
const active = turns.find((turn) => text(turn.status) === "inProgress");
this.#activeTurnId = active === undefined ? null : text(active.id);
const reconciledUsage = record(thread.tokenUsage ?? snapshot.tokenUsage);
if (Object.keys(reconciledUsage).length > 0) this.#usage = reconciledUsage;
const terminal = [...turns]
.reverse()
.find((turn) => ["completed", "failed", "interrupted", "cancelled"].includes(text(turn.status)));
if (terminal !== undefined) {
this.#activeTurnId = null;
this.#mapTerminalTurn(terminal, text(terminal.id));
} else {
this.#activeTurnId = active === undefined ? null : text(active.id);
}
this.#emit("session.reconciled", {
providerSessionId: this.#opened.providerSessionId,
turnCount: turns.length,
activeTurnId: this.#activeTurnId,
terminalTurnId: terminal === undefined ? null : text(terminal.id),
});
return snapshot;
}
@@ -560,12 +592,7 @@ class CodexHarnessSession implements HarnessSession {
return;
}
if (notification.method === "turn/completed") {
const status = text(turn.status, "completed");
const eventType = status === "failed" ? "turn.failed" : status === "interrupted" ? "turn.interrupted" : "turn.completed";
this.#captureResultFromTurn(turn);
this.#emit(eventType, { status, error: turn.error ?? null }, { turnId });
this.#activeTurnId = null;
this.#finalize(status);
this.#mapTerminalTurn(turn, turnId);
return;
}
if (notification.method === "item/started") {
@@ -639,7 +666,7 @@ class CodexHarnessSession implements HarnessSession {
}],
};
}
const fingerprint = JSON.stringify(validation.result);
const fingerprint = canonicalJson(validation.result);
if (this.#resultFingerprint !== null && this.#resultFingerprint !== fingerprint) {
return { success: false, contentItems: [{ type: "inputText", text: "A different semantic result was already committed." }] };
}
@@ -700,7 +727,7 @@ class CodexHarnessSession implements HarnessSession {
#commitResult(result: PrpStructuredRunResult, itemId: string, turnId?: string): void {
if (this.#result !== null) return;
this.#result = structuredClone(result);
this.#resultFingerprint = JSON.stringify(result);
this.#resultFingerprint = canonicalJson(result);
result.verification.forEach((verification, index) => {
this.#emit("item.completed", {
kind: "verification",
@@ -720,35 +747,64 @@ class CodexHarnessSession implements HarnessSession {
#finalize(turnStatus: string): void {
if (this.#terminal) return;
if (this.#result === null) {
this.#commitResult({
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "needs_review",
summary: "Codex ended without a valid semantic completion.",
completionClaim: {
contractRevision: this.#taskEnvelope.completionContract.revision,
objectiveSatisfied: false,
criteria: this.#taskEnvelope.completionContract.criteria.map((criterion) => ({
criterionId: criterion.id,
status: "unknown",
evidenceRefs: [],
})),
remainingWork: [{ description: "Review the incomplete Codex turn.", blocksCompletion: true }],
},
evidence: [],
verification: [],
attentionRequests: [],
artifacts: [],
}, "missing-semantic-result");
this.#emit("harness.diagnostic", {
code: "semantic_result_missing",
message: `Turn ${turnStatus} without a schema-valid semantic proposal; recovery is required.`,
});
}
const result = this.#result;
if (result === null) throw new Error("semantic result finalization invariant failed");
this.#terminal = true;
this.#emit("run.terminal", {
schema: "paperclip.prp.terminal.v1",
turnTerminalState: terminalState(turnStatus),
runTerminalState: dispositionRunState(result),
reportedWorkDisposition: result.reportedWorkDisposition,
}
#mapTerminalTurn(turn: Record<string, unknown>, fallbackTurnId: string): void {
const turnId = text(turn.id, fallbackTurnId || this.#activeTurnId || "");
const status = text(turn.status, "completed");
const candidate = this.#resultFromTurn(turn) ?? this.#result;
const fingerprint = canonicalJson({
terminalState: terminalState(status),
error: turn.error ?? null,
result: candidate,
});
const previous = this.#terminalTurns.get(turnId);
if (previous !== undefined) {
if (previous !== fingerprint) {
this.#emit("harness.diagnostic", {
code: "conflicting_turn_terminal",
message: "Provider sent materially different terminal facts for an already terminal turn.",
}, { turnId });
}
return;
}
this.#terminalTurns.set(turnId, fingerprint);
if (candidate !== null && this.#result === null) {
this.#commitResult(candidate, text(this.#resultItemFromTurn(turn)?.id), turnId);
}
const eventType =
status === "failed"
? "turn.failed"
: status === "interrupted"
? "turn.interrupted"
: status === "cancelled"
? "turn.cancelled"
: "turn.completed";
this.#emit(eventType, { status, error: turn.error ?? null }, { turnId });
this.#activeTurnId = null;
this.#finalize(status);
}
#resultItemFromTurn(turn: Record<string, unknown>): Record<string, unknown> | null {
if (!Array.isArray(turn.items)) return null;
return (
turn.items
.map(record)
.reverse()
.find((value) => text(value.type) === "agentMessage" && tryParseResult(value.text) !== null) ??
null
);
}
#resultFromTurn(turn: Record<string, unknown>): PrpStructuredRunResult | null {
const item = this.#resultItemFromTurn(turn);
return item === null ? null : tryParseResult(item.text);
}
#requireActiveTurn(turnId: string): void {
@@ -761,8 +817,11 @@ class CodexHarnessSession implements HarnessSession {
if (!this.#capabilities[operation]) throw this.#unsupported(operation, "capability not advertised");
}
#unsupported(operation: string, detail: unknown): UnsupportedCodexOperationError {
const error = new UnsupportedCodexOperationError(operation, String(detail));
#unsupported(operation: string, detail: unknown): HarnessCapabilityUnavailableError {
const error = new HarnessCapabilityUnavailableError(
operation,
redactCodexDiagnostic(String(detail)),
);
this.#diagnoseUnsupported(operation, error.message);
return error;
}
@@ -793,36 +852,13 @@ class CodexHarnessSession implements HarnessSession {
...(refs.itemId ? { itemId: refs.itemId } : {}),
eventType,
schemaVersion: 1,
priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1,
priority: eventType === "run.result.proposed" ? 0 : 1,
emittedAt: this.#now().toISOString(),
payload,
});
}
}
export function isSkilllessContext(context: Phase4ModelContextSnapshot): boolean {
const serialized = JSON.stringify(context).toLowerCase();
return (
context.instructionSources.length === 0 &&
context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS &&
context.modelInputKinds.length === 1 &&
context.modelInputKinds[0] === "text" &&
context.dynamicToolNames.length === PHASE4_SEMANTIC_TOOL_NAMES.length &&
context.dynamicToolNames.every((name) =>
PHASE4_SEMANTIC_TOOL_NAMES.includes(
name as (typeof PHASE4_SEMANTIC_TOOL_NAMES)[number],
),
) &&
context.instructionPolicy.skillInstructions === false &&
context.instructionPolicy.appInstructions === false &&
context.instructionPolicy.collaborationInstructions === false &&
!serialized.includes("paperclip_api_key") &&
!serialized.includes("authorization: bearer") &&
!serialized.includes("/api/issues/") &&
!serialized.includes('"type":"skill"')
);
}
function isSemanticTool(tool: string): boolean {
return (
tool === PHASE4_COMPLETION_TOOL_NAME ||
@@ -1,5 +1,8 @@
import type { HarnessDriver, HarnessSession } from "../contracts/harness-driver.js";
import {
isSkilllessPhase4Context,
type Phase4ResultDecision,
type Phase4ResultValidationIssue,
type Phase4RunMetadata,
type Phase4RunTrace,
type Phase4TaskEnvelope,
@@ -9,15 +12,13 @@ import {
applyPrpEvent,
createSessionSnapshotFromMetadata,
} from "../reducer/session-reducer.js";
import type {
PrpCapabilities,
PrpEvent,
PrpStructuredRunResult,
} from "../protocol/phase1-contract.js";
import {
UnsupportedCodexOperationError,
isSkilllessContext,
} from "../drivers/codex/codex-app-server-driver.js";
validatePrpStructuredRunResult,
type PrpCapabilities,
type PrpEvent,
type PrpStructuredRunResult,
type PrpTerminalState,
} from "../protocol/phase1-contract.js";
export interface Phase4CodexTracerInput {
driver: HarnessDriver;
@@ -25,10 +26,12 @@ export interface Phase4CodexTracerInput {
workingDirectory: string;
message?: NativeUserMessage;
runId?: string;
normalizedSessionId?: string;
companyId?: string;
issueId?: string;
environmentLeaseId?: string;
runnerInstanceId?: string;
controlPlaneInstanceId?: string;
steer?: string;
interrupt?: boolean;
timeoutMs?: number;
@@ -55,22 +58,128 @@ function prpCapabilities(
function contextFrom(events: PrpEvent[]) {
const event = events.find(
(candidate) => candidate.eventType === "session.started" || candidate.eventType === "session.resumed",
(candidate) =>
candidate.eventType === "session.started" || candidate.eventType === "session.resumed",
);
const context = event?.payload.context;
if (typeof context !== "object" || context === null || Array.isArray(context)) {
throw new Error("Codex session did not emit its model-context snapshot");
throw new Error("Harness session did not emit its model-context snapshot");
}
return context as Phase4RunTrace["context"];
}
function resultFrom(events: PrpEvent[]): PrpStructuredRunResult {
const event = events.find((candidate) => candidate.eventType === "run.result.proposed");
if (event === undefined) throw new Error("Codex session emitted no semantic result");
return event.payload as PrpStructuredRunResult;
function dispositionIssues(
result: PrpStructuredRunResult,
): Phase4ResultValidationIssue[] {
const issues: Phase4ResultValidationIssue[] = [];
const claim = result.completionClaim;
if (result.reportedWorkDisposition === "done") {
if (
claim.objectiveSatisfied !== true ||
claim.criteria.some((criterion) => criterion.status !== "satisfied") ||
claim.remainingWork.some((remaining) => remaining.blocksCompletion) ||
result.blocker !== undefined
) {
issues.push({
code: "invalid_disposition",
path: "/reportedWorkDisposition",
message: "done requires a satisfied objective and criteria with no blocking work or blocker",
});
}
} else if (result.reportedWorkDisposition === "needs_review") {
if (result.blocker !== undefined || result.attentionRequests.length === 0) {
issues.push({
code: "invalid_disposition",
path: "/reportedWorkDisposition",
message: "needs_review requires an attention request and must not include a blocker",
});
}
} else if (result.reportedWorkDisposition === "blocked") {
if (
claim.objectiveSatisfied !== false ||
result.blocker === undefined ||
!claim.remainingWork.some((remaining) => remaining.blocksCompletion)
) {
issues.push({
code: "invalid_disposition",
path: "/reportedWorkDisposition",
message: "blocked requires an unsatisfied objective, blocker details, and blocking remaining work",
});
}
} else {
issues.push({
code: "invalid_disposition",
path: "/reportedWorkDisposition",
message: `${result.reportedWorkDisposition} is not a Phase 4 terminal proposal`,
});
}
return issues;
}
async function consumeToTerminal(
/** Validate an advisory provider proposal against the exact controller-owned task envelope. */
export function validatePhase4ResultProposal(
proposal: unknown,
envelope: Phase4TaskEnvelope,
): Phase4ResultDecision {
const schema = validatePrpStructuredRunResult(proposal);
if (!schema.ok) {
return {
status: "rejected",
result: null,
issues: schema.issues.map((issue) => ({
code: "schema_validation",
path: issue.path,
message: issue.message,
})),
};
}
const result = schema.result;
const issues: Phase4ResultValidationIssue[] = [];
if (result.completionClaim.contractRevision !== envelope.completionContract.revision) {
issues.push({
code: "contract_revision_mismatch",
path: "/completionClaim/contractRevision",
message: `expected exact contract revision ${envelope.completionContract.revision}`,
});
}
const expectedIds = new Set(envelope.completionContract.criteria.map((criterion) => criterion.id));
const observed = new Map<string, number>();
result.completionClaim.criteria.forEach((criterion, index) => {
observed.set(criterion.criterionId, (observed.get(criterion.criterionId) ?? 0) + 1);
if (!expectedIds.has(criterion.criterionId)) {
issues.push({
code: "unknown_criterion",
path: `/completionClaim/criteria/${index}/criterionId`,
message: `criterion ${criterion.criterionId} is not in the task envelope`,
});
}
});
for (const criterionId of expectedIds) {
const count = observed.get(criterionId) ?? 0;
if (count === 0) {
issues.push({
code: "missing_criterion",
path: "/completionClaim/criteria",
message: `criterion ${criterionId} is missing`,
});
} else if (count > 1) {
issues.push({
code: "duplicate_criterion",
path: "/completionClaim/criteria",
message: `criterion ${criterionId} appears more than once`,
});
}
}
issues.push(...dispositionIssues(result));
return issues.length === 0
? { status: "accepted", result: structuredClone(result), issues: [] }
: { status: "rejected", result: null, issues };
}
async function consumeToTurnTerminal(
session: HarnessSession,
timeoutMs: number,
onEvent?: (event: PrpEvent) => void,
@@ -80,16 +189,19 @@ async function consumeToTerminal(
for await (const event of session.events()) {
events.push(event);
onEvent?.(event);
if (event.eventType === "run.terminal") return events;
if (isTurnTerminal(event)) return events;
}
throw new Error("Codex event stream closed before run.terminal");
throw new Error("Harness event stream closed before a turn terminal fact");
};
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
consume(),
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(`Codex tracer timed out after ${timeoutMs}ms`)), timeoutMs);
timer = setTimeout(
() => reject(new Error(`Phase 4 tracer timed out after ${timeoutMs}ms`)),
timeoutMs,
);
}),
]);
} finally {
@@ -97,13 +209,43 @@ async function consumeToTerminal(
}
}
function isTurnTerminal(event: PrpEvent): boolean {
return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(
event.eventType,
);
}
function runtimeTerminal(event: PrpEvent): Pick<PrpTerminalState, "turnTerminalState" | "runTerminalState"> {
if (event.eventType === "turn.completed") {
return { turnTerminalState: "completed", runTerminalState: "succeeded" };
}
if (event.eventType === "turn.failed") {
return { turnTerminalState: "failed", runTerminalState: "failed" };
}
if (event.eventType === "turn.interrupted") {
return { turnTerminalState: "interrupted", runTerminalState: "cancelled" };
}
return { turnTerminalState: "cancelled", runTerminalState: "cancelled" };
}
function sourceSequenceContinuous(events: PrpEvent[]): boolean {
const cursors = new Map<string, number>();
for (const event of events) {
const key = `${event.sourceKind}:${event.sourceInstanceId}`;
const previous = cursors.get(key);
if (previous !== undefined && event.sourceSeq !== previous + 1) return false;
cursors.set(key, event.sourceSeq);
}
return true;
}
/**
* Small in-memory Phase 4 mock core. It accepts only the public driver contract,
* reduces events live, and replays the same canonical stream after completion.
* Small in-memory Phase 4 mock core. The driver reports provider facts; this
* controller validates semantic proposals and alone emits the run terminal.
*/
export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promise<Phase4RunTrace> {
const runId = input.runId ?? "phase4-safe-run";
const normalizedSessionId = `session:${runId}`;
const normalizedSessionId = input.normalizedSessionId ?? "phase4-normalized-session";
const descriptor = await input.driver.descriptor();
const capabilities = prpCapabilities(descriptor);
const metadata: Phase4RunMetadata = {
@@ -120,12 +262,17 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
},
capabilities,
};
const session = await input.driver.openSession({ runId, workingDirectory: input.workingDirectory });
const session = await input.driver.openSession({
runId,
normalizedSessionId,
workingDirectory: input.workingDirectory,
});
const capabilityDiagnostics: string[] = [];
let signalTurnStarted: (() => void) | undefined;
const turnStarted = new Promise<void>((resolve) => {
signalTurnStarted = resolve;
});
const consuming = consumeToTerminal(session, input.timeoutMs ?? 180_000, (event) => {
const consuming = consumeToTurnTerminal(session, input.timeoutMs ?? 180_000, (event) => {
if (event.eventType === "turn.started") signalTurnStarted?.();
});
@@ -137,31 +284,37 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
await Promise.race([
turnStarted,
consuming.then(() => {
throw new Error("Codex turn reached terminal state before accepting the control command");
throw new Error("Harness turn reached terminal state before accepting the control command");
}),
]);
}
if (input.steer !== undefined) {
if (session.steer === undefined) throw new Error("driver did not expose steering");
try {
await session.steer({ turnId: turn.turnId, message: { role: "user", text: input.steer } });
} catch (error) {
// A short real-model turn can finish between turn/start and turn/steer.
// The driver already emitted the explicit capability diagnostic; keep
// consuming the same session so the mock core records its terminal.
if (!(error instanceof UnsupportedCodexOperationError)) throw error;
if (!descriptor.capabilities.steering || session.steer === undefined) {
capabilityDiagnostics.push("steering is unavailable by driver capability contract");
} else {
try {
await session.steer({
turnId: turn.turnId,
message: { role: "user", text: input.steer },
});
} catch (error) {
capabilityDiagnostics.push(`steering degraded: ${String(error)}`);
}
}
}
if (input.interrupt) {
if (session.interrupt === undefined) throw new Error("driver did not expose interruption");
try {
await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" });
} catch (error) {
if (!(error instanceof UnsupportedCodexOperationError)) throw error;
if (!descriptor.capabilities.interruption || session.interrupt === undefined) {
capabilityDiagnostics.push("interruption is unavailable by driver capability contract");
} else {
try {
await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" });
} catch (error) {
capabilityDiagnostics.push(`interruption degraded: ${String(error)}`);
}
}
}
const events = await consuming;
const providerEvents = await consuming;
const ids = session.ids();
const identity = {
...metadata.identity,
@@ -169,6 +322,80 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
...(ids.providerSessionId ? { providerSessionId: ids.providerSessionId } : {}),
};
metadata.identity = identity;
const proposals = providerEvents.filter(
(event) => event.eventType === "run.result.proposed",
);
const proposedResult = proposals.length === 1 ? structuredClone(proposals[0]!.payload) : null;
const resultDecision: Phase4ResultDecision =
proposals.length === 1
? validatePhase4ResultProposal(proposedResult, input.taskEnvelope)
: {
status: "rejected",
result: null,
issues: [{
code: "schema_validation",
path: "/run.result.proposed",
message:
proposals.length === 0
? "the completed turn emitted no semantic proposal"
: "the completed turn emitted multiple semantic proposals",
}],
};
let controlSequence = 0;
const controlPlaneInstanceId = input.controlPlaneInstanceId ?? "mock-core-phase4";
const controlEvent = (
eventType: PrpEvent["eventType"],
payload: Record<string, unknown>,
refs: { turnId?: string; itemId?: string } = {},
): PrpEvent => ({
schema: "paperclip.prp.event.v1",
sourceEventId: `${controlPlaneInstanceId}:${runId}:${++controlSequence}`,
sourceSeq: controlSequence,
sourceInstanceId: controlPlaneInstanceId,
sourceKind: "control_plane",
runId,
normalizedSessionId,
...(refs.turnId ? { turnId: refs.turnId } : {}),
...(refs.itemId ? { itemId: refs.itemId } : {}),
eventType,
schemaVersion: 1,
priority: eventType === "run.terminal" || eventType.startsWith("run.result.") ? 0 : 1,
emittedAt: new Date().toISOString(),
payload,
});
const controlEvents = capabilityDiagnostics.map((message) =>
controlEvent("runner.diagnostic", {
code: "capability_unavailable",
message,
}),
);
controlEvents.push(
resultDecision.status === "accepted"
? controlEvent("run.result.accepted", {
contractRevision: input.taskEnvelope.completionContract.revision,
result: resultDecision.result,
})
: controlEvent("run.result.rejected", {
reasonCode: "semantic_result_rejected",
issues: resultDecision.issues,
recovery: { required: true, recoverable: true },
}),
);
const providerTerminal = providerEvents.findLast(isTurnTerminal);
if (providerTerminal === undefined) throw new Error("turn terminal invariant failed");
const runtime = runtimeTerminal(providerTerminal);
controlEvents.push(controlEvent("run.terminal", {
schema: "paperclip.prp.terminal.v1",
...runtime,
reportedWorkDisposition:
resultDecision.status === "accepted"
? resultDecision.result.reportedWorkDisposition
: "yielded",
}, providerTerminal.turnId ? { turnId: providerTerminal.turnId } : {}));
const events = [...providerEvents, ...controlEvents];
const liveSnapshot = events.reduce(
applyPrpEvent,
createSessionSnapshotFromMetadata({
@@ -186,37 +413,52 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
}),
);
const context = contextFrom(events);
const result = resultFrom(events);
const terminalCount = events.filter((event) => event.eventType === "run.terminal").length;
const resultCount = events.filter((event) => event.eventType === "run.result.proposed").length;
const turnTerminalCount = events.filter(isTurnTerminal).length;
const decisionCount = events.filter(
(event) =>
event.eventType === "run.result.accepted" || event.eventType === "run.result.rejected",
).length;
const serialized = JSON.stringify(context).toLowerCase();
const itemEvents = events.filter((event) => event.eventType.startsWith("item."));
const providerSessionId = ids.providerSessionId ?? null;
return {
schema: "paperclip.runner.phase4.trace.v1",
metadata,
context,
events,
result,
proposedResult,
result: resultDecision.result,
resultDecision,
liveSnapshot,
replaySnapshot,
diagnostics: events
.filter((event) => event.eventType === "runner.diagnostic" || event.eventType === "harness.diagnostic")
.map((event) => String(event.payload.message ?? event.payload.code ?? "diagnostic")),
.filter(
(event) =>
event.eventType === "runner.diagnostic" ||
event.eventType === "harness.diagnostic" ||
event.eventType === "run.result.rejected",
)
.map((event) => String(event.payload.message ?? event.payload.reasonCode ?? event.payload.code ?? "diagnostic")),
assertions: {
exactlyOneTerminalResult: terminalCount === 1 && resultCount === 1,
exactlyOneTerminalResult:
terminalCount === 1 && turnTerminalCount === 1 && decisionCount === 1,
proposalAccepted: resultDecision.status === "accepted",
liveReplayParity: JSON.stringify(liveSnapshot) === JSON.stringify(replaySnapshot),
stableIdentity:
providerSessionId !== null &&
new Set([runId, normalizedSessionId, ids.driverSessionId, providerSessionId]).size === 4 &&
events.every(
(event) =>
event.runId === runId && event.normalizedSessionId === normalizedSessionId,
) && ids.driverSessionId.length > 0,
sourceSequenceContinuous: events.every(
(event, index) => event.sourceSeq === index + 1,
),
),
sourceSequenceContinuous: sourceSequenceContinuous(events),
stableItemIdentity: itemEvents.every(
(event) => typeof event.itemId === "string" && event.itemId.length > 0,
),
contextIsSkillless: isSkilllessContext(context),
contextIsSkillless: isSkilllessPhase4Context(context, {
dynamicTools: descriptor.capabilities.dynamicTools === true,
}),
unrelatedSkillsAbsent:
context.instructionSources.length === 0 &&
context.instructionPolicy.skillInstructions === false &&