mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 18:31:34 +01:00
fix(runner): enforce phase 4 status authority
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
2161ddcc4a
commit
6391608e8f
6 files changed
+763
-151
No files matched your search
@@ -40,7 +40,9 @@ function formatSummary(trace: Awaited<ReturnType<typeof runPhase4CodexTracer>>):
|
||||
`provider session: ${trace.metadata.identity.providerSessionId}`,
|
||||
`model: ${trace.context.model} (${trace.context.modelProvider})`,
|
||||
`events: ${trace.events.length}`,
|
||||
`result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`,
|
||||
trace.result === null
|
||||
? `result: rejected — ${trace.resultDecision.issues.map((issue) => issue.message).join("; ")}`
|
||||
: `result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`,
|
||||
checks,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -11,9 +11,29 @@ export interface HarnessDriverDescriptor {
|
||||
|
||||
export interface OpenHarnessSessionInput {
|
||||
runId: string;
|
||||
normalizedSessionId: string;
|
||||
workingDirectory: string;
|
||||
}
|
||||
|
||||
export class HarnessCapabilityUnavailableError extends Error {
|
||||
readonly operation: string;
|
||||
|
||||
constructor(operation: string, detail: string) {
|
||||
super(`${operation} is unavailable: ${detail}`);
|
||||
this.name = "HarnessCapabilityUnavailableError";
|
||||
this.operation = operation;
|
||||
}
|
||||
}
|
||||
|
||||
export class HarnessReconciliationError extends Error {
|
||||
readonly recoverable = true;
|
||||
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "HarnessReconciliationError";
|
||||
}
|
||||
}
|
||||
|
||||
export interface PersistedHarnessSession {
|
||||
driverKind: string;
|
||||
driverSessionId: string;
|
||||
|
||||
@@ -75,12 +75,15 @@ export interface Phase4RunTrace {
|
||||
metadata: Phase4RunMetadata;
|
||||
context: Phase4ModelContextSnapshot;
|
||||
events: PrpEvent[];
|
||||
result: PrpStructuredRunResult;
|
||||
proposedResult: unknown | null;
|
||||
result: PrpStructuredRunResult | null;
|
||||
resultDecision: Phase4ResultDecision;
|
||||
liveSnapshot: SessionSnapshot;
|
||||
replaySnapshot: SessionSnapshot;
|
||||
diagnostics: string[];
|
||||
assertions: {
|
||||
exactlyOneTerminalResult: boolean;
|
||||
proposalAccepted: boolean;
|
||||
liveReplayParity: boolean;
|
||||
stableIdentity: boolean;
|
||||
sourceSequenceContinuous: boolean;
|
||||
@@ -91,6 +94,22 @@ export interface Phase4RunTrace {
|
||||
};
|
||||
}
|
||||
|
||||
export interface Phase4ResultValidationIssue {
|
||||
code:
|
||||
| "schema_validation"
|
||||
| "contract_revision_mismatch"
|
||||
| "unknown_criterion"
|
||||
| "missing_criterion"
|
||||
| "duplicate_criterion"
|
||||
| "invalid_disposition";
|
||||
path: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export type Phase4ResultDecision =
|
||||
| { status: "accepted"; result: PrpStructuredRunResult; issues: [] }
|
||||
| { status: "rejected"; result: null; issues: Phase4ResultValidationIssue[] };
|
||||
|
||||
export function createPhase4TaskEnvelope(input: {
|
||||
objective: string;
|
||||
contractRevision?: string;
|
||||
@@ -273,3 +292,28 @@ export const PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA = {
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
export function isSkilllessPhase4Context(
|
||||
context: Phase4ModelContextSnapshot,
|
||||
options: { dynamicTools: boolean } = { dynamicTools: true },
|
||||
): boolean {
|
||||
const serialized = JSON.stringify(context).toLowerCase();
|
||||
const expectedTools = options.dynamicTools ? [...PHASE4_SEMANTIC_TOOL_NAMES] : [];
|
||||
return (
|
||||
context.instructionSources.length === 0 &&
|
||||
context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS &&
|
||||
context.modelInputKinds.length === 1 &&
|
||||
context.modelInputKinds[0] === "text" &&
|
||||
context.dynamicToolNames.length === expectedTools.length &&
|
||||
context.dynamicToolNames.every((name) =>
|
||||
expectedTools.includes(name as (typeof expectedTools)[number]),
|
||||
) &&
|
||||
context.instructionPolicy.skillInstructions === false &&
|
||||
context.instructionPolicy.appInstructions === false &&
|
||||
context.instructionPolicy.collaborationInstructions === false &&
|
||||
!serialized.includes("paperclip_api_key") &&
|
||||
!serialized.includes("authorization: bearer") &&
|
||||
!serialized.includes("/api/issues/") &&
|
||||
!serialized.includes('"type":"skill"')
|
||||
);
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import {
|
||||
PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
|
||||
PHASE4_RESULT_OUTPUT_SCHEMA,
|
||||
createPhase4TaskEnvelope,
|
||||
isSkilllessPhase4Context,
|
||||
} from "../../contracts/phase4.js";
|
||||
import { HarnessCapabilityUnavailableError } from "../../contracts/harness-driver.js";
|
||||
import {
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata,
|
||||
@@ -17,9 +19,11 @@ import {
|
||||
} from "../../protocol/phase1-contract.js";
|
||||
import {
|
||||
CodexAppServerDriver,
|
||||
UnsupportedCodexOperationError,
|
||||
isSkilllessContext,
|
||||
} from "./codex-app-server-driver.js";
|
||||
import {
|
||||
runPhase4CodexTracer,
|
||||
validatePhase4ResultProposal,
|
||||
} from "../../mock-core/phase4-codex-runner.js";
|
||||
import type {
|
||||
CodexAppServerTransport,
|
||||
CodexRpcNotification,
|
||||
@@ -61,6 +65,7 @@ class FakeCodexTransport implements CodexAppServerTransport {
|
||||
readonly queue = new TestQueue<CodexRpcNotification>();
|
||||
handler: CodexServerRequestHandler = async () => ({});
|
||||
rejectMethods = new Map<string, Error>();
|
||||
readResponse: Record<string, unknown> | null = null;
|
||||
|
||||
constructor(
|
||||
readonly threadId = "thread-1",
|
||||
@@ -76,7 +81,13 @@ class FakeCodexTransport implements CodexAppServerTransport {
|
||||
}
|
||||
if (method === "thread/start" || method === "thread/resume") {
|
||||
return {
|
||||
thread: { id: this.threadId, sessionId: this.providerSessionId, modelProvider: "openai", cwd: "/workspace", turns: [] },
|
||||
thread: {
|
||||
id: this.threadId,
|
||||
sessionId: this.providerSessionId,
|
||||
modelProvider: "openai",
|
||||
cwd: "/workspace",
|
||||
turns: [],
|
||||
},
|
||||
model: "gpt-test",
|
||||
modelProvider: "openai",
|
||||
cwd: "/workspace",
|
||||
@@ -87,7 +98,7 @@ class FakeCodexTransport implements CodexAppServerTransport {
|
||||
}
|
||||
if (method === "turn/start") return { turn: { id: "turn-1", status: "inProgress", items: [] } };
|
||||
if (method === "thread/read") {
|
||||
return { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } };
|
||||
return this.readResponse ?? { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
@@ -160,17 +171,69 @@ async function collectUntilTerminal(events: AsyncIterable<PrpEvent>): Promise<Pr
|
||||
const collected: PrpEvent[] = [];
|
||||
for await (const event of events) {
|
||||
collected.push(event);
|
||||
if (event.eventType === "run.terminal") break;
|
||||
if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) break;
|
||||
}
|
||||
return collected;
|
||||
}
|
||||
|
||||
function reverseObjectKeys(value: unknown): unknown {
|
||||
if (Array.isArray(value)) return value.map(reverseObjectKeys);
|
||||
if (typeof value !== "object" || value === null) return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.reverse()
|
||||
.map(([key, entry]) => [key, reverseObjectKeys(entry)]),
|
||||
);
|
||||
}
|
||||
|
||||
async function traceCompletedProposal(
|
||||
proposal: PrpStructuredRunResult | null,
|
||||
options: {
|
||||
runId?: string;
|
||||
normalizedSessionId?: string;
|
||||
capabilities?: Record<string, boolean>;
|
||||
steer?: string;
|
||||
interrupt?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const transport = new FakeCodexTransport();
|
||||
const driver = makeDriver([transport], { capabilities: options.capabilities });
|
||||
const traced = runPhase4CodexTracer({
|
||||
driver,
|
||||
taskEnvelope: envelope,
|
||||
workingDirectory: "/workspace",
|
||||
runId: options.runId,
|
||||
normalizedSessionId: options.normalizedSessionId,
|
||||
steer: options.steer,
|
||||
interrupt: options.interrupt,
|
||||
});
|
||||
while (!transport.calls.some((call) => call.method === "turn/start")) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
transport.push("turn/started", {
|
||||
threadId: "thread-1",
|
||||
turn: { id: "turn-1", status: "inProgress" },
|
||||
});
|
||||
if (proposal !== null) {
|
||||
transport.push("item/completed", {
|
||||
threadId: "thread-1",
|
||||
turnId: "turn-1",
|
||||
item: { id: "answer-1", type: "agentMessage", text: JSON.stringify(proposal) },
|
||||
});
|
||||
}
|
||||
transport.push("turn/completed", {
|
||||
threadId: "thread-1",
|
||||
turn: { id: "turn-1", status: "completed", items: [] },
|
||||
});
|
||||
return { trace: await traced, transport };
|
||||
}
|
||||
|
||||
describe("Codex app-server Phase 4 driver", () => {
|
||||
it("passes the common typed-event contract and produces exactly one terminal result", async () => {
|
||||
it("passes the common typed-event contract and reports one provider turn terminal", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const driver = makeDriver([transport]);
|
||||
const descriptor = await driver.descriptor();
|
||||
const session = await driver.openSession({ runId: "run-1", workingDirectory: "/workspace" });
|
||||
const session = await driver.openSession({ runId: "run-1", normalizedSessionId: "normalized-1", workingDirectory: "/workspace" });
|
||||
const turn = await session.startTurn({ message: { role: "user", text: "Do the safe task." } });
|
||||
|
||||
transport.push("turn/started", { threadId: "thread-1", turn: { id: turn.turnId, status: "inProgress" } });
|
||||
@@ -189,9 +252,10 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
const events = await collectUntilTerminal(session.events());
|
||||
expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true);
|
||||
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
|
||||
expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(1);
|
||||
expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1);
|
||||
expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(0);
|
||||
expect(events.map((event) => event.eventType)).toEqual(expect.arrayContaining([
|
||||
"session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "run.terminal",
|
||||
"session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "turn.completed",
|
||||
"runtime_request.created", "runtime_request.resolved",
|
||||
]));
|
||||
|
||||
@@ -208,7 +272,7 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
};
|
||||
const metadata = {
|
||||
fixtureName: "phase4-conformance",
|
||||
identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "session:run-1" },
|
||||
identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "normalized-1" },
|
||||
capabilities,
|
||||
};
|
||||
const live = events.reduce(applyPrpEvent, createSessionSnapshotFromMetadata(metadata));
|
||||
@@ -220,10 +284,10 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
|
||||
it("captures an exact skillless model/environment snapshot with credentials absent", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-context", workingDirectory: "/workspace" });
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-context", normalizedSessionId: "normalized-context", workingDirectory: "/workspace" });
|
||||
const iterator = session.events()[Symbol.asyncIterator]();
|
||||
const first = await iterator.next();
|
||||
const context = first.value?.payload.context as Parameters<typeof isSkilllessContext>[0];
|
||||
const context = first.value?.payload.context as Parameters<typeof isSkilllessPhase4Context>[0];
|
||||
expect(context).toMatchObject({
|
||||
codexVersion: "codex-cli/0.132.0",
|
||||
model: "gpt-test",
|
||||
@@ -239,7 +303,7 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
environmentKeys: ["CODEX_HOME", "HOME", "LANG", "PATH"],
|
||||
envelope,
|
||||
});
|
||||
expect(isSkilllessContext(context)).toBe(true);
|
||||
expect(isSkilllessPhase4Context(context)).toBe(true);
|
||||
expect(JSON.stringify(context)).not.toContain("must-not-pass");
|
||||
expect(JSON.stringify(transport.calls)).not.toContain("RANDOM_SKILL_PATH");
|
||||
expect(transport.calls.find((call) => call.method === "thread/start")?.params).toMatchObject({
|
||||
@@ -263,7 +327,7 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
|
||||
it("steers and interrupts an active turn without replacing the session", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-controls", workingDirectory: "/workspace" });
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-controls", normalizedSessionId: "normalized-controls", workingDirectory: "/workspace" });
|
||||
const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } });
|
||||
await session.steer?.({ turnId, message: { role: "user", text: "Use a shorter answer." } });
|
||||
await session.interrupt?.({ turnId, reason: "operator requested" });
|
||||
@@ -275,22 +339,28 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
|
||||
it("makes duplicate semantic completion idempotent and rejects changed payloads", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-result", workingDirectory: "/workspace" });
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-result", normalizedSessionId: "normalized-result", workingDirectory: "/workspace" });
|
||||
await session.startTurn({ message: { role: "user", text: "Complete." } });
|
||||
const request = { id: 1, method: "item/tool/call", params: { threadId: "thread-1", turnId: "turn-1", callId: "call-1", tool: "paperclip_finish", arguments: result } };
|
||||
expect(await transport.invoke(request)).toMatchObject({ success: true });
|
||||
expect(await transport.invoke({ ...request, id: 2 })).toMatchObject({ success: true });
|
||||
expect(await transport.invoke({
|
||||
...request,
|
||||
id: 2,
|
||||
params: { ...request.params, arguments: reverseObjectKeys(result) },
|
||||
})).toMatchObject({ success: true });
|
||||
const changed = structuredClone(result);
|
||||
changed.summary = "Changed after commit.";
|
||||
expect(await transport.invoke({ ...request, id: 3, params: { ...request.params, arguments: changed } })).toMatchObject({ success: false });
|
||||
transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } });
|
||||
transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } });
|
||||
const events = await collectUntilTerminal(session.events());
|
||||
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
|
||||
expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("normalizes finish and block tools into one canonical result contract", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-tools", workingDirectory: "/workspace" });
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-tools", normalizedSessionId: "normalized-tools", workingDirectory: "/workspace" });
|
||||
await session.startTurn({ message: { role: "user", text: "Complete or block." } });
|
||||
const blocked: PrpStructuredRunResult = {
|
||||
...structuredClone(result),
|
||||
@@ -339,7 +409,7 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
const first = new FakeCodexTransport();
|
||||
const second = new FakeCodexTransport();
|
||||
const driver = makeDriver([first, second]);
|
||||
const original = await driver.openSession({ runId: "run-recover", workingDirectory: "/workspace" });
|
||||
const original = await driver.openSession({ runId: "run-recover", normalizedSessionId: "normalized-recover", workingDirectory: "/workspace" });
|
||||
await original.startTurn({ message: { role: "user", text: "Work." } });
|
||||
const snapshot = await original.snapshot();
|
||||
await original.close({ reason: "transport lost" });
|
||||
@@ -351,12 +421,62 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
expect((await recovery?.session?.snapshot())?.activeTurnId).toBe("turn-1");
|
||||
});
|
||||
|
||||
it("reconciles a turn completed during transport loss without replacing either session identity", async () => {
|
||||
const first = new FakeCodexTransport("driver-thread-loss", "provider-session-loss");
|
||||
const second = new FakeCodexTransport("driver-thread-loss", "provider-session-loss");
|
||||
second.readResponse = {
|
||||
thread: {
|
||||
id: "driver-thread-loss",
|
||||
sessionId: "provider-session-loss",
|
||||
tokenUsage: { total: { inputTokens: 21, outputTokens: 8 } },
|
||||
turns: [{
|
||||
id: "turn-1",
|
||||
status: "completed",
|
||||
items: [{ id: "answer-loss", type: "agentMessage", text: JSON.stringify(result) }],
|
||||
}],
|
||||
},
|
||||
};
|
||||
const driver = makeDriver([first, second]);
|
||||
const original = await driver.openSession({
|
||||
runId: "runtime-run-loss",
|
||||
normalizedSessionId: "controller-session-loss",
|
||||
workingDirectory: "/workspace",
|
||||
});
|
||||
await original.startTurn({ message: { role: "user", text: "Work through loss." } });
|
||||
const snapshot = await original.snapshot();
|
||||
await original.close({ reason: "transport lost before terminal delivery" });
|
||||
|
||||
const recovery = await driver.recoverSession?.(snapshot);
|
||||
expect(recovery).toMatchObject({ recovered: true });
|
||||
const recovered = recovery?.session;
|
||||
expect(recovered).toBeDefined();
|
||||
const collecting = collectUntilTerminal(recovered!.events());
|
||||
await recovered!.reconcile?.();
|
||||
const events = await collecting;
|
||||
|
||||
expect(recovered!.ids()).toEqual({
|
||||
driverSessionId: "driver-thread-loss",
|
||||
providerSessionId: "provider-session-loss",
|
||||
displayId: "driver-thread-loss",
|
||||
});
|
||||
expect(await recovered!.snapshot()).toMatchObject({
|
||||
runId: "runtime-run-loss",
|
||||
normalizedSessionId: "controller-session-loss",
|
||||
driverSessionId: "driver-thread-loss",
|
||||
providerSessionId: "provider-session-loss",
|
||||
});
|
||||
expect(events.filter((event) => event.eventType === "turn.completed")).toHaveLength(1);
|
||||
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
|
||||
expect(events.every((event, index) => event.sourceSeq === snapshot.lastSourceSequence! + index + 1)).toBe(true);
|
||||
expect(await recovered!.usage?.()).toEqual({ total: { inputTokens: 21, outputTokens: 8 } });
|
||||
});
|
||||
|
||||
it("degrades unsupported operations with explicit redacted diagnostics", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
transport.rejectMethods.set("turn/steer", new Error("Bearer super-secret api_key=also-secret"));
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-degrade", workingDirectory: "/workspace" });
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-degrade", normalizedSessionId: "normalized-degrade", workingDirectory: "/workspace" });
|
||||
const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } });
|
||||
await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(UnsupportedCodexOperationError);
|
||||
await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError);
|
||||
const iterator = session.events()[Symbol.asyncIterator]();
|
||||
const events: PrpEvent[] = [];
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
@@ -368,4 +488,152 @@ describe("Codex app-server Phase 4 driver", () => {
|
||||
expect(JSON.stringify(diagnostic)).not.toContain("super-secret");
|
||||
expect(JSON.stringify(diagnostic)).not.toContain("also-secret");
|
||||
});
|
||||
|
||||
it("rejects proposals that do not match the exact task envelope", () => {
|
||||
const wrongRevision = structuredClone(result);
|
||||
wrongRevision.completionClaim.contractRevision = "phase4-demo-v0";
|
||||
expect(validatePhase4ResultProposal(wrongRevision, envelope)).toMatchObject({
|
||||
status: "rejected",
|
||||
issues: [{ code: "contract_revision_mismatch" }],
|
||||
});
|
||||
|
||||
const unknownCriterion = structuredClone(result);
|
||||
unknownCriterion.completionClaim.criteria = [{
|
||||
criterionId: "not-in-envelope",
|
||||
status: "satisfied",
|
||||
evidenceRefs: [],
|
||||
}];
|
||||
const criterionDecision = validatePhase4ResultProposal(unknownCriterion, envelope);
|
||||
expect(criterionDecision).toMatchObject({ status: "rejected" });
|
||||
if (criterionDecision.status === "rejected") {
|
||||
expect(criterionDecision.issues.map((issue) => issue.code)).toEqual(
|
||||
expect.arrayContaining(["unknown_criterion", "missing_criterion"]),
|
||||
);
|
||||
}
|
||||
|
||||
const invalidDone = structuredClone(result);
|
||||
invalidDone.completionClaim.objectiveSatisfied = false;
|
||||
expect(validatePhase4ResultProposal(invalidDone, envelope)).toMatchObject({
|
||||
status: "rejected",
|
||||
issues: [{ code: "invalid_disposition" }],
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["needs_review", "blocked"] as const)(
|
||||
"keeps a completed runtime successful for an accepted %s advisory proposal",
|
||||
async (disposition) => {
|
||||
const proposal: PrpStructuredRunResult = disposition === "needs_review"
|
||||
? {
|
||||
...structuredClone(result),
|
||||
reportedWorkDisposition: "needs_review",
|
||||
attentionRequests: [{ kind: "review", summary: "Review the completed artifact." }],
|
||||
}
|
||||
: {
|
||||
...structuredClone(result),
|
||||
reportedWorkDisposition: "blocked",
|
||||
summary: "Waiting on fixture input.",
|
||||
completionClaim: {
|
||||
...structuredClone(result.completionClaim),
|
||||
objectiveSatisfied: false,
|
||||
criteria: [{ criterionId: "file", status: "not_satisfied", evidenceRefs: [] }],
|
||||
remainingWork: [{ description: "Provide fixture input.", blocksCompletion: true }],
|
||||
},
|
||||
blocker: {
|
||||
reasonCode: "fixture_input_missing",
|
||||
owner: { kind: "external", name: "fixture owner" },
|
||||
unblockAction: "Provide fixture input.",
|
||||
scope: "task_wide",
|
||||
},
|
||||
artifacts: [],
|
||||
};
|
||||
const { trace } = await traceCompletedProposal(proposal);
|
||||
expect(trace.resultDecision.status).toBe("accepted");
|
||||
expect(trace.events.find((event) => event.eventType === "run.terminal")?.payload)
|
||||
.toMatchObject({
|
||||
turnTerminalState: "completed",
|
||||
runTerminalState: "succeeded",
|
||||
reportedWorkDisposition: disposition,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("records rejected and missing proposals as recovery evidence instead of review status", async () => {
|
||||
const wrongRevision = structuredClone(result);
|
||||
wrongRevision.completionClaim.contractRevision = "wrong-revision";
|
||||
const rejected = await traceCompletedProposal(wrongRevision);
|
||||
expect(rejected.trace.result).toBeNull();
|
||||
expect(rejected.trace.proposedResult).toMatchObject({ reportedWorkDisposition: "done" });
|
||||
expect(rejected.trace.events.find((event) => event.eventType === "run.result.rejected")?.payload)
|
||||
.toMatchObject({ recovery: { required: true, recoverable: true } });
|
||||
expect(rejected.trace.events.find((event) => event.eventType === "run.terminal")?.payload)
|
||||
.toMatchObject({ runTerminalState: "succeeded", reportedWorkDisposition: "yielded" });
|
||||
|
||||
const missing = await traceCompletedProposal(null);
|
||||
expect(missing.trace.resultDecision).toMatchObject({ status: "rejected" });
|
||||
expect(missing.trace.events.some((event) =>
|
||||
event.eventType === "run.result.proposed" &&
|
||||
event.payload.reportedWorkDisposition === "needs_review"
|
||||
)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves four independent stable identities under controller ownership", async () => {
|
||||
const { trace } = await traceCompletedProposal(result, {
|
||||
runId: "runtime-run-identity",
|
||||
normalizedSessionId: "controller-session-identity",
|
||||
});
|
||||
expect(trace.assertions.stableIdentity).toBe(true);
|
||||
expect(new Set([
|
||||
trace.metadata.identity.runId,
|
||||
trace.metadata.identity.normalizedSessionId,
|
||||
trace.metadata.identity.driverSessionId,
|
||||
trace.metadata.identity.providerSessionId,
|
||||
]).size).toBe(4);
|
||||
expect(trace.events.every((event) =>
|
||||
event.runId === "runtime-run-identity" &&
|
||||
event.normalizedSessionId === "controller-session-identity"
|
||||
)).toBe(true);
|
||||
});
|
||||
|
||||
it("degrades declared unsupported capabilities without Codex-specific core branches", async () => {
|
||||
const capabilities = {
|
||||
resume: false,
|
||||
read: false,
|
||||
steering: false,
|
||||
interruption: false,
|
||||
usage: false,
|
||||
reconciliation: false,
|
||||
dynamicTools: false,
|
||||
};
|
||||
const { trace, transport } = await traceCompletedProposal(result, {
|
||||
capabilities,
|
||||
steer: "Do not call this unsupported path.",
|
||||
interrupt: true,
|
||||
});
|
||||
expect(trace.resultDecision.status).toBe("accepted");
|
||||
expect(trace.assertions.contextIsSkillless).toBe(true);
|
||||
expect(trace.diagnostics).toEqual(expect.arrayContaining([
|
||||
expect.stringContaining("steering is unavailable"),
|
||||
expect.stringContaining("interruption is unavailable"),
|
||||
]));
|
||||
expect(transport.calls.map((call) => call.method)).not.toEqual(
|
||||
expect.arrayContaining(["turn/steer", "turn/interrupt", "thread/read"]),
|
||||
);
|
||||
expect(transport.calls.find((call) => call.method === "thread/start")?.params.dynamicTools)
|
||||
.toEqual([]);
|
||||
|
||||
const directTransport = new FakeCodexTransport();
|
||||
const driver = makeDriver([directTransport], { capabilities });
|
||||
const session = await driver.openSession({
|
||||
runId: "run-no-capabilities",
|
||||
normalizedSessionId: "normalized-no-capabilities",
|
||||
workingDirectory: "/workspace",
|
||||
});
|
||||
await expect(session.read?.()).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError);
|
||||
await expect(session.usage?.()).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError);
|
||||
const snapshot = await session.snapshot();
|
||||
expect(await driver.recoverSession?.(snapshot)).toMatchObject({
|
||||
recovered: false,
|
||||
reason: "resume capability is unavailable",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,10 @@ import type {
|
||||
OpenHarnessSessionInput,
|
||||
PersistedHarnessSession,
|
||||
} from "../../contracts/harness-driver.js";
|
||||
import {
|
||||
HarnessCapabilityUnavailableError,
|
||||
HarnessReconciliationError,
|
||||
} from "../../contracts/harness-driver.js";
|
||||
import {
|
||||
PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
|
||||
PHASE4_BLOCK_TOOL_NAME,
|
||||
@@ -68,16 +72,6 @@ class AsyncQueue<T> implements AsyncIterable<T> {
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsupportedCodexOperationError extends Error {
|
||||
readonly operation: string;
|
||||
|
||||
constructor(operation: string, detail: string) {
|
||||
super(`${operation} is unsupported: ${redactCodexDiagnostic(detail)}`);
|
||||
this.name = "UnsupportedCodexOperationError";
|
||||
this.operation = operation;
|
||||
}
|
||||
}
|
||||
|
||||
export interface CodexAppServerDriverOptions {
|
||||
taskEnvelope: Phase4TaskEnvelope;
|
||||
transportFactory?: () => CodexAppServerTransport;
|
||||
@@ -102,7 +96,7 @@ type CodexCapabilities = Required<
|
||||
|
||||
interface OpenedCodexThread {
|
||||
threadId: string;
|
||||
providerSessionId: string;
|
||||
providerSessionId: string | null;
|
||||
context: Phase4ModelContextSnapshot;
|
||||
}
|
||||
|
||||
@@ -137,12 +131,6 @@ function terminalState(status: string): "completed" | "failed" | "interrupted" {
|
||||
return "completed";
|
||||
}
|
||||
|
||||
function dispositionRunState(
|
||||
result: PrpStructuredRunResult,
|
||||
): "succeeded" | "failed" {
|
||||
return result.reportedWorkDisposition === "done" ? "succeeded" : "failed";
|
||||
}
|
||||
|
||||
function tryParseResult(value: unknown): PrpStructuredRunResult | null {
|
||||
let candidate = value;
|
||||
if (typeof value === "string") {
|
||||
@@ -156,6 +144,18 @@ function tryParseResult(value: unknown): PrpStructuredRunResult | null {
|
||||
return validation.ok ? validation.result : null;
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
const object = record(value);
|
||||
if (Object.keys(object).length > 0 || (typeof value === "object" && value !== null)) {
|
||||
return `{${Object.keys(object)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "undefined";
|
||||
}
|
||||
|
||||
function finishToolSpec(): Record<string, unknown> {
|
||||
return {
|
||||
name: PHASE4_COMPLETION_TOOL_NAME,
|
||||
@@ -242,7 +242,7 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||
return this.#session({
|
||||
transport,
|
||||
runId: input.runId,
|
||||
normalizedSessionId: `session:${input.runId}`,
|
||||
normalizedSessionId: input.normalizedSessionId,
|
||||
opened,
|
||||
resumed: false,
|
||||
sourceSequence: 0,
|
||||
@@ -255,6 +255,9 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||
if (!this.#caps.resume) {
|
||||
return { recovered: false, reason: "resume capability is unavailable" };
|
||||
}
|
||||
if (!this.#caps.read) {
|
||||
return { recovered: false, reason: "read capability is required for safe resume" };
|
||||
}
|
||||
if (!snapshot.runId || !snapshot.normalizedSessionId || !snapshot.driverSessionId) {
|
||||
return { recovered: false, reason: "persisted session identity is incomplete" };
|
||||
}
|
||||
@@ -272,6 +275,13 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||
await transport.close();
|
||||
return { recovered: false, reason: "provider resumed a different session" };
|
||||
}
|
||||
if (
|
||||
snapshot.providerSessionId &&
|
||||
opened.providerSessionId !== snapshot.providerSessionId
|
||||
) {
|
||||
await transport.close();
|
||||
return { recovered: false, reason: "provider resumed a different provider session" };
|
||||
}
|
||||
return {
|
||||
recovered: true,
|
||||
session: this.#session({
|
||||
@@ -319,7 +329,7 @@ export class CodexAppServerDriver implements HarnessDriver {
|
||||
if (threadId.length === 0) throw new Error("Codex thread response omitted thread.id");
|
||||
return {
|
||||
threadId,
|
||||
providerSessionId: text(thread.sessionId, threadId),
|
||||
providerSessionId: text(thread.sessionId) || null,
|
||||
context: {
|
||||
protocolVersion: PHASE4_CODEX_PROTOCOL_VERSION,
|
||||
codexVersion: text(initialize.userAgent, "unknown"),
|
||||
@@ -385,6 +395,7 @@ class CodexHarnessSession implements HarnessSession {
|
||||
#result: PrpStructuredRunResult | null = null;
|
||||
#resultFingerprint: string | null = null;
|
||||
#terminal = false;
|
||||
readonly #terminalTurns = new Map<string, string>();
|
||||
|
||||
constructor(input: {
|
||||
transport: CodexAppServerTransport;
|
||||
@@ -502,13 +513,34 @@ class CodexHarnessSession implements HarnessSession {
|
||||
this.#requireCapability("reconciliation");
|
||||
const snapshot = await this.read();
|
||||
const thread = record(snapshot.thread);
|
||||
if (text(thread.id) !== this.#opened.threadId) {
|
||||
throw new HarnessReconciliationError("thread/read returned a different driver session");
|
||||
}
|
||||
const providerSessionId = text(thread.sessionId);
|
||||
if (
|
||||
this.#opened.providerSessionId !== null &&
|
||||
providerSessionId !== this.#opened.providerSessionId
|
||||
) {
|
||||
throw new HarnessReconciliationError("thread/read returned a different provider session");
|
||||
}
|
||||
const turns = Array.isArray(thread.turns) ? thread.turns.map(record) : [];
|
||||
const active = turns.find((turn) => text(turn.status) === "inProgress");
|
||||
this.#activeTurnId = active === undefined ? null : text(active.id);
|
||||
const reconciledUsage = record(thread.tokenUsage ?? snapshot.tokenUsage);
|
||||
if (Object.keys(reconciledUsage).length > 0) this.#usage = reconciledUsage;
|
||||
const terminal = [...turns]
|
||||
.reverse()
|
||||
.find((turn) => ["completed", "failed", "interrupted", "cancelled"].includes(text(turn.status)));
|
||||
if (terminal !== undefined) {
|
||||
this.#activeTurnId = null;
|
||||
this.#mapTerminalTurn(terminal, text(terminal.id));
|
||||
} else {
|
||||
this.#activeTurnId = active === undefined ? null : text(active.id);
|
||||
}
|
||||
this.#emit("session.reconciled", {
|
||||
providerSessionId: this.#opened.providerSessionId,
|
||||
turnCount: turns.length,
|
||||
activeTurnId: this.#activeTurnId,
|
||||
terminalTurnId: terminal === undefined ? null : text(terminal.id),
|
||||
});
|
||||
return snapshot;
|
||||
}
|
||||
@@ -560,12 +592,7 @@ class CodexHarnessSession implements HarnessSession {
|
||||
return;
|
||||
}
|
||||
if (notification.method === "turn/completed") {
|
||||
const status = text(turn.status, "completed");
|
||||
const eventType = status === "failed" ? "turn.failed" : status === "interrupted" ? "turn.interrupted" : "turn.completed";
|
||||
this.#captureResultFromTurn(turn);
|
||||
this.#emit(eventType, { status, error: turn.error ?? null }, { turnId });
|
||||
this.#activeTurnId = null;
|
||||
this.#finalize(status);
|
||||
this.#mapTerminalTurn(turn, turnId);
|
||||
return;
|
||||
}
|
||||
if (notification.method === "item/started") {
|
||||
@@ -639,7 +666,7 @@ class CodexHarnessSession implements HarnessSession {
|
||||
}],
|
||||
};
|
||||
}
|
||||
const fingerprint = JSON.stringify(validation.result);
|
||||
const fingerprint = canonicalJson(validation.result);
|
||||
if (this.#resultFingerprint !== null && this.#resultFingerprint !== fingerprint) {
|
||||
return { success: false, contentItems: [{ type: "inputText", text: "A different semantic result was already committed." }] };
|
||||
}
|
||||
@@ -700,7 +727,7 @@ class CodexHarnessSession implements HarnessSession {
|
||||
#commitResult(result: PrpStructuredRunResult, itemId: string, turnId?: string): void {
|
||||
if (this.#result !== null) return;
|
||||
this.#result = structuredClone(result);
|
||||
this.#resultFingerprint = JSON.stringify(result);
|
||||
this.#resultFingerprint = canonicalJson(result);
|
||||
result.verification.forEach((verification, index) => {
|
||||
this.#emit("item.completed", {
|
||||
kind: "verification",
|
||||
@@ -720,35 +747,64 @@ class CodexHarnessSession implements HarnessSession {
|
||||
#finalize(turnStatus: string): void {
|
||||
if (this.#terminal) return;
|
||||
if (this.#result === null) {
|
||||
this.#commitResult({
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "needs_review",
|
||||
summary: "Codex ended without a valid semantic completion.",
|
||||
completionClaim: {
|
||||
contractRevision: this.#taskEnvelope.completionContract.revision,
|
||||
objectiveSatisfied: false,
|
||||
criteria: this.#taskEnvelope.completionContract.criteria.map((criterion) => ({
|
||||
criterionId: criterion.id,
|
||||
status: "unknown",
|
||||
evidenceRefs: [],
|
||||
})),
|
||||
remainingWork: [{ description: "Review the incomplete Codex turn.", blocksCompletion: true }],
|
||||
},
|
||||
evidence: [],
|
||||
verification: [],
|
||||
attentionRequests: [],
|
||||
artifacts: [],
|
||||
}, "missing-semantic-result");
|
||||
this.#emit("harness.diagnostic", {
|
||||
code: "semantic_result_missing",
|
||||
message: `Turn ${turnStatus} without a schema-valid semantic proposal; recovery is required.`,
|
||||
});
|
||||
}
|
||||
const result = this.#result;
|
||||
if (result === null) throw new Error("semantic result finalization invariant failed");
|
||||
this.#terminal = true;
|
||||
this.#emit("run.terminal", {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
turnTerminalState: terminalState(turnStatus),
|
||||
runTerminalState: dispositionRunState(result),
|
||||
reportedWorkDisposition: result.reportedWorkDisposition,
|
||||
}
|
||||
|
||||
#mapTerminalTurn(turn: Record<string, unknown>, fallbackTurnId: string): void {
|
||||
const turnId = text(turn.id, fallbackTurnId || this.#activeTurnId || "");
|
||||
const status = text(turn.status, "completed");
|
||||
const candidate = this.#resultFromTurn(turn) ?? this.#result;
|
||||
const fingerprint = canonicalJson({
|
||||
terminalState: terminalState(status),
|
||||
error: turn.error ?? null,
|
||||
result: candidate,
|
||||
});
|
||||
const previous = this.#terminalTurns.get(turnId);
|
||||
if (previous !== undefined) {
|
||||
if (previous !== fingerprint) {
|
||||
this.#emit("harness.diagnostic", {
|
||||
code: "conflicting_turn_terminal",
|
||||
message: "Provider sent materially different terminal facts for an already terminal turn.",
|
||||
}, { turnId });
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.#terminalTurns.set(turnId, fingerprint);
|
||||
if (candidate !== null && this.#result === null) {
|
||||
this.#commitResult(candidate, text(this.#resultItemFromTurn(turn)?.id), turnId);
|
||||
}
|
||||
const eventType =
|
||||
status === "failed"
|
||||
? "turn.failed"
|
||||
: status === "interrupted"
|
||||
? "turn.interrupted"
|
||||
: status === "cancelled"
|
||||
? "turn.cancelled"
|
||||
: "turn.completed";
|
||||
this.#emit(eventType, { status, error: turn.error ?? null }, { turnId });
|
||||
this.#activeTurnId = null;
|
||||
this.#finalize(status);
|
||||
}
|
||||
|
||||
#resultItemFromTurn(turn: Record<string, unknown>): Record<string, unknown> | null {
|
||||
if (!Array.isArray(turn.items)) return null;
|
||||
return (
|
||||
turn.items
|
||||
.map(record)
|
||||
.reverse()
|
||||
.find((value) => text(value.type) === "agentMessage" && tryParseResult(value.text) !== null) ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
#resultFromTurn(turn: Record<string, unknown>): PrpStructuredRunResult | null {
|
||||
const item = this.#resultItemFromTurn(turn);
|
||||
return item === null ? null : tryParseResult(item.text);
|
||||
}
|
||||
|
||||
#requireActiveTurn(turnId: string): void {
|
||||
@@ -761,8 +817,11 @@ class CodexHarnessSession implements HarnessSession {
|
||||
if (!this.#capabilities[operation]) throw this.#unsupported(operation, "capability not advertised");
|
||||
}
|
||||
|
||||
#unsupported(operation: string, detail: unknown): UnsupportedCodexOperationError {
|
||||
const error = new UnsupportedCodexOperationError(operation, String(detail));
|
||||
#unsupported(operation: string, detail: unknown): HarnessCapabilityUnavailableError {
|
||||
const error = new HarnessCapabilityUnavailableError(
|
||||
operation,
|
||||
redactCodexDiagnostic(String(detail)),
|
||||
);
|
||||
this.#diagnoseUnsupported(operation, error.message);
|
||||
return error;
|
||||
}
|
||||
@@ -793,36 +852,13 @@ class CodexHarnessSession implements HarnessSession {
|
||||
...(refs.itemId ? { itemId: refs.itemId } : {}),
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1,
|
||||
priority: eventType === "run.result.proposed" ? 0 : 1,
|
||||
emittedAt: this.#now().toISOString(),
|
||||
payload,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function isSkilllessContext(context: Phase4ModelContextSnapshot): boolean {
|
||||
const serialized = JSON.stringify(context).toLowerCase();
|
||||
return (
|
||||
context.instructionSources.length === 0 &&
|
||||
context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS &&
|
||||
context.modelInputKinds.length === 1 &&
|
||||
context.modelInputKinds[0] === "text" &&
|
||||
context.dynamicToolNames.length === PHASE4_SEMANTIC_TOOL_NAMES.length &&
|
||||
context.dynamicToolNames.every((name) =>
|
||||
PHASE4_SEMANTIC_TOOL_NAMES.includes(
|
||||
name as (typeof PHASE4_SEMANTIC_TOOL_NAMES)[number],
|
||||
),
|
||||
) &&
|
||||
context.instructionPolicy.skillInstructions === false &&
|
||||
context.instructionPolicy.appInstructions === false &&
|
||||
context.instructionPolicy.collaborationInstructions === false &&
|
||||
!serialized.includes("paperclip_api_key") &&
|
||||
!serialized.includes("authorization: bearer") &&
|
||||
!serialized.includes("/api/issues/") &&
|
||||
!serialized.includes('"type":"skill"')
|
||||
);
|
||||
}
|
||||
|
||||
function isSemanticTool(tool: string): boolean {
|
||||
return (
|
||||
tool === PHASE4_COMPLETION_TOOL_NAME ||
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import type { HarnessDriver, HarnessSession } from "../contracts/harness-driver.js";
|
||||
import {
|
||||
isSkilllessPhase4Context,
|
||||
type Phase4ResultDecision,
|
||||
type Phase4ResultValidationIssue,
|
||||
type Phase4RunMetadata,
|
||||
type Phase4RunTrace,
|
||||
type Phase4TaskEnvelope,
|
||||
@@ -9,15 +12,13 @@ import {
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata,
|
||||
} from "../reducer/session-reducer.js";
|
||||
import type {
|
||||
PrpCapabilities,
|
||||
PrpEvent,
|
||||
PrpStructuredRunResult,
|
||||
} from "../protocol/phase1-contract.js";
|
||||
import {
|
||||
UnsupportedCodexOperationError,
|
||||
isSkilllessContext,
|
||||
} from "../drivers/codex/codex-app-server-driver.js";
|
||||
validatePrpStructuredRunResult,
|
||||
type PrpCapabilities,
|
||||
type PrpEvent,
|
||||
type PrpStructuredRunResult,
|
||||
type PrpTerminalState,
|
||||
} from "../protocol/phase1-contract.js";
|
||||
|
||||
export interface Phase4CodexTracerInput {
|
||||
driver: HarnessDriver;
|
||||
@@ -25,10 +26,12 @@ export interface Phase4CodexTracerInput {
|
||||
workingDirectory: string;
|
||||
message?: NativeUserMessage;
|
||||
runId?: string;
|
||||
normalizedSessionId?: string;
|
||||
companyId?: string;
|
||||
issueId?: string;
|
||||
environmentLeaseId?: string;
|
||||
runnerInstanceId?: string;
|
||||
controlPlaneInstanceId?: string;
|
||||
steer?: string;
|
||||
interrupt?: boolean;
|
||||
timeoutMs?: number;
|
||||
@@ -55,22 +58,128 @@ function prpCapabilities(
|
||||
|
||||
function contextFrom(events: PrpEvent[]) {
|
||||
const event = events.find(
|
||||
(candidate) => candidate.eventType === "session.started" || candidate.eventType === "session.resumed",
|
||||
(candidate) =>
|
||||
candidate.eventType === "session.started" || candidate.eventType === "session.resumed",
|
||||
);
|
||||
const context = event?.payload.context;
|
||||
if (typeof context !== "object" || context === null || Array.isArray(context)) {
|
||||
throw new Error("Codex session did not emit its model-context snapshot");
|
||||
throw new Error("Harness session did not emit its model-context snapshot");
|
||||
}
|
||||
return context as Phase4RunTrace["context"];
|
||||
}
|
||||
|
||||
function resultFrom(events: PrpEvent[]): PrpStructuredRunResult {
|
||||
const event = events.find((candidate) => candidate.eventType === "run.result.proposed");
|
||||
if (event === undefined) throw new Error("Codex session emitted no semantic result");
|
||||
return event.payload as PrpStructuredRunResult;
|
||||
function dispositionIssues(
|
||||
result: PrpStructuredRunResult,
|
||||
): Phase4ResultValidationIssue[] {
|
||||
const issues: Phase4ResultValidationIssue[] = [];
|
||||
const claim = result.completionClaim;
|
||||
if (result.reportedWorkDisposition === "done") {
|
||||
if (
|
||||
claim.objectiveSatisfied !== true ||
|
||||
claim.criteria.some((criterion) => criterion.status !== "satisfied") ||
|
||||
claim.remainingWork.some((remaining) => remaining.blocksCompletion) ||
|
||||
result.blocker !== undefined
|
||||
) {
|
||||
issues.push({
|
||||
code: "invalid_disposition",
|
||||
path: "/reportedWorkDisposition",
|
||||
message: "done requires a satisfied objective and criteria with no blocking work or blocker",
|
||||
});
|
||||
}
|
||||
} else if (result.reportedWorkDisposition === "needs_review") {
|
||||
if (result.blocker !== undefined || result.attentionRequests.length === 0) {
|
||||
issues.push({
|
||||
code: "invalid_disposition",
|
||||
path: "/reportedWorkDisposition",
|
||||
message: "needs_review requires an attention request and must not include a blocker",
|
||||
});
|
||||
}
|
||||
} else if (result.reportedWorkDisposition === "blocked") {
|
||||
if (
|
||||
claim.objectiveSatisfied !== false ||
|
||||
result.blocker === undefined ||
|
||||
!claim.remainingWork.some((remaining) => remaining.blocksCompletion)
|
||||
) {
|
||||
issues.push({
|
||||
code: "invalid_disposition",
|
||||
path: "/reportedWorkDisposition",
|
||||
message: "blocked requires an unsatisfied objective, blocker details, and blocking remaining work",
|
||||
});
|
||||
}
|
||||
} else {
|
||||
issues.push({
|
||||
code: "invalid_disposition",
|
||||
path: "/reportedWorkDisposition",
|
||||
message: `${result.reportedWorkDisposition} is not a Phase 4 terminal proposal`,
|
||||
});
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
async function consumeToTerminal(
|
||||
/** Validate an advisory provider proposal against the exact controller-owned task envelope. */
|
||||
export function validatePhase4ResultProposal(
|
||||
proposal: unknown,
|
||||
envelope: Phase4TaskEnvelope,
|
||||
): Phase4ResultDecision {
|
||||
const schema = validatePrpStructuredRunResult(proposal);
|
||||
if (!schema.ok) {
|
||||
return {
|
||||
status: "rejected",
|
||||
result: null,
|
||||
issues: schema.issues.map((issue) => ({
|
||||
code: "schema_validation",
|
||||
path: issue.path,
|
||||
message: issue.message,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const result = schema.result;
|
||||
const issues: Phase4ResultValidationIssue[] = [];
|
||||
if (result.completionClaim.contractRevision !== envelope.completionContract.revision) {
|
||||
issues.push({
|
||||
code: "contract_revision_mismatch",
|
||||
path: "/completionClaim/contractRevision",
|
||||
message: `expected exact contract revision ${envelope.completionContract.revision}`,
|
||||
});
|
||||
}
|
||||
|
||||
const expectedIds = new Set(envelope.completionContract.criteria.map((criterion) => criterion.id));
|
||||
const observed = new Map<string, number>();
|
||||
result.completionClaim.criteria.forEach((criterion, index) => {
|
||||
observed.set(criterion.criterionId, (observed.get(criterion.criterionId) ?? 0) + 1);
|
||||
if (!expectedIds.has(criterion.criterionId)) {
|
||||
issues.push({
|
||||
code: "unknown_criterion",
|
||||
path: `/completionClaim/criteria/${index}/criterionId`,
|
||||
message: `criterion ${criterion.criterionId} is not in the task envelope`,
|
||||
});
|
||||
}
|
||||
});
|
||||
for (const criterionId of expectedIds) {
|
||||
const count = observed.get(criterionId) ?? 0;
|
||||
if (count === 0) {
|
||||
issues.push({
|
||||
code: "missing_criterion",
|
||||
path: "/completionClaim/criteria",
|
||||
message: `criterion ${criterionId} is missing`,
|
||||
});
|
||||
} else if (count > 1) {
|
||||
issues.push({
|
||||
code: "duplicate_criterion",
|
||||
path: "/completionClaim/criteria",
|
||||
message: `criterion ${criterionId} appears more than once`,
|
||||
});
|
||||
}
|
||||
}
|
||||
issues.push(...dispositionIssues(result));
|
||||
|
||||
return issues.length === 0
|
||||
? { status: "accepted", result: structuredClone(result), issues: [] }
|
||||
: { status: "rejected", result: null, issues };
|
||||
}
|
||||
|
||||
async function consumeToTurnTerminal(
|
||||
session: HarnessSession,
|
||||
timeoutMs: number,
|
||||
onEvent?: (event: PrpEvent) => void,
|
||||
@@ -80,16 +189,19 @@ async function consumeToTerminal(
|
||||
for await (const event of session.events()) {
|
||||
events.push(event);
|
||||
onEvent?.(event);
|
||||
if (event.eventType === "run.terminal") return events;
|
||||
if (isTurnTerminal(event)) return events;
|
||||
}
|
||||
throw new Error("Codex event stream closed before run.terminal");
|
||||
throw new Error("Harness event stream closed before a turn terminal fact");
|
||||
};
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
consume(),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Codex tracer timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
timer = setTimeout(
|
||||
() => reject(new Error(`Phase 4 tracer timed out after ${timeoutMs}ms`)),
|
||||
timeoutMs,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
@@ -97,13 +209,43 @@ async function consumeToTerminal(
|
||||
}
|
||||
}
|
||||
|
||||
function isTurnTerminal(event: PrpEvent): boolean {
|
||||
return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(
|
||||
event.eventType,
|
||||
);
|
||||
}
|
||||
|
||||
function runtimeTerminal(event: PrpEvent): Pick<PrpTerminalState, "turnTerminalState" | "runTerminalState"> {
|
||||
if (event.eventType === "turn.completed") {
|
||||
return { turnTerminalState: "completed", runTerminalState: "succeeded" };
|
||||
}
|
||||
if (event.eventType === "turn.failed") {
|
||||
return { turnTerminalState: "failed", runTerminalState: "failed" };
|
||||
}
|
||||
if (event.eventType === "turn.interrupted") {
|
||||
return { turnTerminalState: "interrupted", runTerminalState: "cancelled" };
|
||||
}
|
||||
return { turnTerminalState: "cancelled", runTerminalState: "cancelled" };
|
||||
}
|
||||
|
||||
function sourceSequenceContinuous(events: PrpEvent[]): boolean {
|
||||
const cursors = new Map<string, number>();
|
||||
for (const event of events) {
|
||||
const key = `${event.sourceKind}:${event.sourceInstanceId}`;
|
||||
const previous = cursors.get(key);
|
||||
if (previous !== undefined && event.sourceSeq !== previous + 1) return false;
|
||||
cursors.set(key, event.sourceSeq);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Small in-memory Phase 4 mock core. It accepts only the public driver contract,
|
||||
* reduces events live, and replays the same canonical stream after completion.
|
||||
* Small in-memory Phase 4 mock core. The driver reports provider facts; this
|
||||
* controller validates semantic proposals and alone emits the run terminal.
|
||||
*/
|
||||
export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promise<Phase4RunTrace> {
|
||||
const runId = input.runId ?? "phase4-safe-run";
|
||||
const normalizedSessionId = `session:${runId}`;
|
||||
const normalizedSessionId = input.normalizedSessionId ?? "phase4-normalized-session";
|
||||
const descriptor = await input.driver.descriptor();
|
||||
const capabilities = prpCapabilities(descriptor);
|
||||
const metadata: Phase4RunMetadata = {
|
||||
@@ -120,12 +262,17 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
|
||||
},
|
||||
capabilities,
|
||||
};
|
||||
const session = await input.driver.openSession({ runId, workingDirectory: input.workingDirectory });
|
||||
const session = await input.driver.openSession({
|
||||
runId,
|
||||
normalizedSessionId,
|
||||
workingDirectory: input.workingDirectory,
|
||||
});
|
||||
const capabilityDiagnostics: string[] = [];
|
||||
let signalTurnStarted: (() => void) | undefined;
|
||||
const turnStarted = new Promise<void>((resolve) => {
|
||||
signalTurnStarted = resolve;
|
||||
});
|
||||
const consuming = consumeToTerminal(session, input.timeoutMs ?? 180_000, (event) => {
|
||||
const consuming = consumeToTurnTerminal(session, input.timeoutMs ?? 180_000, (event) => {
|
||||
if (event.eventType === "turn.started") signalTurnStarted?.();
|
||||
});
|
||||
|
||||
@@ -137,31 +284,37 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
|
||||
await Promise.race([
|
||||
turnStarted,
|
||||
consuming.then(() => {
|
||||
throw new Error("Codex turn reached terminal state before accepting the control command");
|
||||
throw new Error("Harness turn reached terminal state before accepting the control command");
|
||||
}),
|
||||
]);
|
||||
}
|
||||
if (input.steer !== undefined) {
|
||||
if (session.steer === undefined) throw new Error("driver did not expose steering");
|
||||
try {
|
||||
await session.steer({ turnId: turn.turnId, message: { role: "user", text: input.steer } });
|
||||
} catch (error) {
|
||||
// A short real-model turn can finish between turn/start and turn/steer.
|
||||
// The driver already emitted the explicit capability diagnostic; keep
|
||||
// consuming the same session so the mock core records its terminal.
|
||||
if (!(error instanceof UnsupportedCodexOperationError)) throw error;
|
||||
if (!descriptor.capabilities.steering || session.steer === undefined) {
|
||||
capabilityDiagnostics.push("steering is unavailable by driver capability contract");
|
||||
} else {
|
||||
try {
|
||||
await session.steer({
|
||||
turnId: turn.turnId,
|
||||
message: { role: "user", text: input.steer },
|
||||
});
|
||||
} catch (error) {
|
||||
capabilityDiagnostics.push(`steering degraded: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (input.interrupt) {
|
||||
if (session.interrupt === undefined) throw new Error("driver did not expose interruption");
|
||||
try {
|
||||
await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" });
|
||||
} catch (error) {
|
||||
if (!(error instanceof UnsupportedCodexOperationError)) throw error;
|
||||
if (!descriptor.capabilities.interruption || session.interrupt === undefined) {
|
||||
capabilityDiagnostics.push("interruption is unavailable by driver capability contract");
|
||||
} else {
|
||||
try {
|
||||
await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" });
|
||||
} catch (error) {
|
||||
capabilityDiagnostics.push(`interruption degraded: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const events = await consuming;
|
||||
const providerEvents = await consuming;
|
||||
const ids = session.ids();
|
||||
const identity = {
|
||||
...metadata.identity,
|
||||
@@ -169,6 +322,80 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
|
||||
...(ids.providerSessionId ? { providerSessionId: ids.providerSessionId } : {}),
|
||||
};
|
||||
metadata.identity = identity;
|
||||
|
||||
const proposals = providerEvents.filter(
|
||||
(event) => event.eventType === "run.result.proposed",
|
||||
);
|
||||
const proposedResult = proposals.length === 1 ? structuredClone(proposals[0]!.payload) : null;
|
||||
const resultDecision: Phase4ResultDecision =
|
||||
proposals.length === 1
|
||||
? validatePhase4ResultProposal(proposedResult, input.taskEnvelope)
|
||||
: {
|
||||
status: "rejected",
|
||||
result: null,
|
||||
issues: [{
|
||||
code: "schema_validation",
|
||||
path: "/run.result.proposed",
|
||||
message:
|
||||
proposals.length === 0
|
||||
? "the completed turn emitted no semantic proposal"
|
||||
: "the completed turn emitted multiple semantic proposals",
|
||||
}],
|
||||
};
|
||||
|
||||
let controlSequence = 0;
|
||||
const controlPlaneInstanceId = input.controlPlaneInstanceId ?? "mock-core-phase4";
|
||||
const controlEvent = (
|
||||
eventType: PrpEvent["eventType"],
|
||||
payload: Record<string, unknown>,
|
||||
refs: { turnId?: string; itemId?: string } = {},
|
||||
): PrpEvent => ({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `${controlPlaneInstanceId}:${runId}:${++controlSequence}`,
|
||||
sourceSeq: controlSequence,
|
||||
sourceInstanceId: controlPlaneInstanceId,
|
||||
sourceKind: "control_plane",
|
||||
runId,
|
||||
normalizedSessionId,
|
||||
...(refs.turnId ? { turnId: refs.turnId } : {}),
|
||||
...(refs.itemId ? { itemId: refs.itemId } : {}),
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: eventType === "run.terminal" || eventType.startsWith("run.result.") ? 0 : 1,
|
||||
emittedAt: new Date().toISOString(),
|
||||
payload,
|
||||
});
|
||||
const controlEvents = capabilityDiagnostics.map((message) =>
|
||||
controlEvent("runner.diagnostic", {
|
||||
code: "capability_unavailable",
|
||||
message,
|
||||
}),
|
||||
);
|
||||
controlEvents.push(
|
||||
resultDecision.status === "accepted"
|
||||
? controlEvent("run.result.accepted", {
|
||||
contractRevision: input.taskEnvelope.completionContract.revision,
|
||||
result: resultDecision.result,
|
||||
})
|
||||
: controlEvent("run.result.rejected", {
|
||||
reasonCode: "semantic_result_rejected",
|
||||
issues: resultDecision.issues,
|
||||
recovery: { required: true, recoverable: true },
|
||||
}),
|
||||
);
|
||||
const providerTerminal = providerEvents.findLast(isTurnTerminal);
|
||||
if (providerTerminal === undefined) throw new Error("turn terminal invariant failed");
|
||||
const runtime = runtimeTerminal(providerTerminal);
|
||||
controlEvents.push(controlEvent("run.terminal", {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
...runtime,
|
||||
reportedWorkDisposition:
|
||||
resultDecision.status === "accepted"
|
||||
? resultDecision.result.reportedWorkDisposition
|
||||
: "yielded",
|
||||
}, providerTerminal.turnId ? { turnId: providerTerminal.turnId } : {}));
|
||||
const events = [...providerEvents, ...controlEvents];
|
||||
|
||||
const liveSnapshot = events.reduce(
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata({
|
||||
@@ -186,37 +413,52 @@ export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promi
|
||||
}),
|
||||
);
|
||||
const context = contextFrom(events);
|
||||
const result = resultFrom(events);
|
||||
const terminalCount = events.filter((event) => event.eventType === "run.terminal").length;
|
||||
const resultCount = events.filter((event) => event.eventType === "run.result.proposed").length;
|
||||
const turnTerminalCount = events.filter(isTurnTerminal).length;
|
||||
const decisionCount = events.filter(
|
||||
(event) =>
|
||||
event.eventType === "run.result.accepted" || event.eventType === "run.result.rejected",
|
||||
).length;
|
||||
const serialized = JSON.stringify(context).toLowerCase();
|
||||
const itemEvents = events.filter((event) => event.eventType.startsWith("item."));
|
||||
const providerSessionId = ids.providerSessionId ?? null;
|
||||
return {
|
||||
schema: "paperclip.runner.phase4.trace.v1",
|
||||
metadata,
|
||||
context,
|
||||
events,
|
||||
result,
|
||||
proposedResult,
|
||||
result: resultDecision.result,
|
||||
resultDecision,
|
||||
liveSnapshot,
|
||||
replaySnapshot,
|
||||
diagnostics: events
|
||||
.filter((event) => event.eventType === "runner.diagnostic" || event.eventType === "harness.diagnostic")
|
||||
.map((event) => String(event.payload.message ?? event.payload.code ?? "diagnostic")),
|
||||
.filter(
|
||||
(event) =>
|
||||
event.eventType === "runner.diagnostic" ||
|
||||
event.eventType === "harness.diagnostic" ||
|
||||
event.eventType === "run.result.rejected",
|
||||
)
|
||||
.map((event) => String(event.payload.message ?? event.payload.reasonCode ?? event.payload.code ?? "diagnostic")),
|
||||
assertions: {
|
||||
exactlyOneTerminalResult: terminalCount === 1 && resultCount === 1,
|
||||
exactlyOneTerminalResult:
|
||||
terminalCount === 1 && turnTerminalCount === 1 && decisionCount === 1,
|
||||
proposalAccepted: resultDecision.status === "accepted",
|
||||
liveReplayParity: JSON.stringify(liveSnapshot) === JSON.stringify(replaySnapshot),
|
||||
stableIdentity:
|
||||
providerSessionId !== null &&
|
||||
new Set([runId, normalizedSessionId, ids.driverSessionId, providerSessionId]).size === 4 &&
|
||||
events.every(
|
||||
(event) =>
|
||||
event.runId === runId && event.normalizedSessionId === normalizedSessionId,
|
||||
) && ids.driverSessionId.length > 0,
|
||||
sourceSequenceContinuous: events.every(
|
||||
(event, index) => event.sourceSeq === index + 1,
|
||||
),
|
||||
),
|
||||
sourceSequenceContinuous: sourceSequenceContinuous(events),
|
||||
stableItemIdentity: itemEvents.every(
|
||||
(event) => typeof event.itemId === "string" && event.itemId.length > 0,
|
||||
),
|
||||
contextIsSkillless: isSkilllessContext(context),
|
||||
contextIsSkillless: isSkilllessPhase4Context(context, {
|
||||
dynamicTools: descriptor.capabilities.dynamicTools === true,
|
||||
}),
|
||||
unrelatedSkillsAbsent:
|
||||
context.instructionSources.length === 0 &&
|
||||
context.instructionPolicy.skillInstructions === false &&
|
||||
|
||||
Reference in new issue
Block a user