mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 14:02:36 +01:00
Give agents persistent Ed25519 identities encrypted with the existing instance master key. Create keys transactionally for new agents and lazily before supported managed runs, expose public identities in the API and agent UI, and protect private material during runtime delivery and output persistence. Preserve identities in recovery backups while giving imported and development-cloned agents fresh keys. Co-Authored-By: Paperclip <noreply@paperclip.ing>
997 lines
32 KiB
TypeScript
997 lines
32 KiB
TypeScript
import { readFileSync } from "node:fs";
|
|
import { describe, expect, it } from "vitest";
|
|
import type { HeartbeatRunEvent } from "@paperclipai/shared";
|
|
import { nativeRunEventsToTranscript } from "./native-run-events";
|
|
|
|
const RUN_ID = "10000000-0000-4000-8000-000000000001";
|
|
|
|
function event(
|
|
seq: number,
|
|
eventType: string,
|
|
payload: Record<string, unknown>,
|
|
overrides: Partial<HeartbeatRunEvent> = {},
|
|
): HeartbeatRunEvent {
|
|
return {
|
|
id: seq,
|
|
companyId: "10000000-0000-4000-8000-000000000002",
|
|
runId: RUN_ID,
|
|
agentId: "10000000-0000-4000-8000-000000000003",
|
|
seq,
|
|
eventType,
|
|
stream: "system",
|
|
level: "info",
|
|
color: null,
|
|
message: null,
|
|
payload: {
|
|
prpEvent: {
|
|
schema: "paperclip.prp.event.v1",
|
|
sourceEventId: `event-${seq}`,
|
|
sourceSeq: seq,
|
|
sourceInstanceId: "runner-1",
|
|
sourceKind: "runner",
|
|
runId: RUN_ID,
|
|
normalizedSessionId: "session-1",
|
|
eventType,
|
|
schemaVersion: 1,
|
|
priority: 1,
|
|
emittedAt: `2026-08-25T18:00:${String(seq).padStart(2, "0")}.000Z`,
|
|
payload,
|
|
},
|
|
},
|
|
createdAt: new Date("2026-08-25T18:00:00.000Z"),
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function itemEvent(
|
|
seq: number,
|
|
eventType: "item.started" | "item.delta" | "item.completed",
|
|
itemId: string,
|
|
payload: Record<string, unknown>,
|
|
): HeartbeatRunEvent {
|
|
const value = event(seq, eventType, payload);
|
|
(value.payload!.prpEvent as Record<string, unknown>).itemId = itemId;
|
|
return value;
|
|
}
|
|
|
|
function runResult(summary: string): Record<string, unknown> {
|
|
return {
|
|
schema: "paperclip.run_result.v1",
|
|
reportedWorkDisposition: "done",
|
|
summary,
|
|
completionClaim: {
|
|
contractRevision: "test-v1",
|
|
objectiveSatisfied: true,
|
|
criteria: [],
|
|
remainingWork: [],
|
|
},
|
|
evidence: [],
|
|
verification: [],
|
|
attentionRequests: [],
|
|
artifacts: [],
|
|
};
|
|
}
|
|
|
|
describe("provider notice presentation", () => {
|
|
it("omits unrelated information from saved chat without removing warnings or responses", () => {
|
|
const legacyNotice = {
|
|
schema: "paperclip.provider.notice.v1",
|
|
noticeId: "codex-warning",
|
|
severity: "warning",
|
|
category: "warning",
|
|
summary: "ignored unrelated provider information",
|
|
userActionable: true,
|
|
};
|
|
const entries = nativeRunEventsToTranscript([
|
|
event(1, "provider.notice.recorded", legacyNotice),
|
|
event(2, "harness.diagnostic", { code: "codex_unrelated_information" }),
|
|
event(3, "provider.notice.recorded", { ...legacyNotice, summary: "Repository is not trusted" }),
|
|
event(4, "provider.notice.recorded", { ...legacyNotice, severity: "error" }),
|
|
event(5, "provider.notice.recorded", { ...legacyNotice, category: "configWarning" }),
|
|
event(6, "item.completed", { kind: "agentMessage", channel: "final", text: "Here is the answer." }),
|
|
]);
|
|
expect(entries).toMatchObject([
|
|
{ kind: "provider_activity", summary: "Repository is not trusted" },
|
|
{ kind: "provider_activity", status: "failed", summary: legacyNotice.summary },
|
|
{ kind: "provider_activity", summary: legacyNotice.summary },
|
|
{ kind: "assistant", text: "Here is the answer." },
|
|
]);
|
|
});
|
|
|
|
it("preserves notice text as a notice rather than a synthetic tool call", () => {
|
|
const entries = nativeRunEventsToTranscript([
|
|
event(1, "provider.notice.recorded", {
|
|
schema: "paperclip.provider.notice.v1", noticeId: "warning-1",
|
|
severity: "warning", category: "configWarning", summary: "Repository is not trusted",
|
|
}),
|
|
event(2, "provider.notice.recorded", {
|
|
schema: "paperclip.provider.notice.v1", noticeId: "error-1",
|
|
severity: "error", message: "Provider connection failed",
|
|
}),
|
|
]);
|
|
expect(entries).toMatchObject([
|
|
{ kind: "provider_activity", family: "provider_notice", status: "informational", summary: "Repository is not trusted" },
|
|
{ kind: "provider_activity", family: "provider_notice", status: "failed", summary: "Provider connection failed" },
|
|
]);
|
|
expect(entries.some((entry) => entry.kind === "tool_call")).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe("nativeRunEventsToTranscript", () => {
|
|
describe("accepted response-wake authority", () => {
|
|
function fixture() {
|
|
const result = {
|
|
...runResult("The complete answer; the task remains open."),
|
|
reportedWorkDisposition: "yielded",
|
|
continuation: {
|
|
kind: "response_wake",
|
|
summary: "Wait for the next exact input.",
|
|
idempotencyKey: "next-input",
|
|
},
|
|
};
|
|
const accepted = event(2, "run.result.accepted", { result });
|
|
const terminal = event(3, "run.terminal", {
|
|
schema: "paperclip.prp.terminal.v1",
|
|
turnTerminalState: "completed",
|
|
runTerminalState: "succeeded",
|
|
reportedWorkDisposition: "yielded",
|
|
});
|
|
const acceptedEnvelope = accepted.payload!.prpEvent as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
const terminalEnvelope = terminal.payload!.prpEvent as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
for (const envelope of [acceptedEnvelope, terminalEnvelope]) {
|
|
envelope.sourceKind = "control_plane";
|
|
envelope.sourceInstanceId = "control-1";
|
|
envelope.turnId = "turn-1";
|
|
}
|
|
return { result, accepted, terminal, acceptedEnvelope, terminalEnvelope };
|
|
}
|
|
|
|
it("derives a closed marker only from a same-run accepted result and terminal", () => {
|
|
const { accepted, terminal } = fixture();
|
|
const entries = nativeRunEventsToTranscript([
|
|
accepted,
|
|
terminal,
|
|
structuredClone(accepted),
|
|
]);
|
|
expect(
|
|
entries.find((entry) => entry.kind === "run_result"),
|
|
).toMatchObject({
|
|
summary: "The complete answer; the task remains open.",
|
|
acceptedResponseWake: { runId: RUN_ID, sourceEventId: "event-2" },
|
|
});
|
|
expect(
|
|
entries.filter((entry) => entry.kind === "run_result"),
|
|
).toHaveLength(1);
|
|
});
|
|
|
|
it.each([
|
|
"provider_acceptance",
|
|
"provider_terminal",
|
|
"proposal",
|
|
"forged_marker",
|
|
"missing_attention",
|
|
"attention",
|
|
"other_continuation",
|
|
"blank_key",
|
|
"blank_summary",
|
|
"missing_source",
|
|
"wrong_run",
|
|
"wrong_event_type",
|
|
"wrong_schema",
|
|
"terminal_other_run",
|
|
"terminal_other_owner",
|
|
"terminal_other_session",
|
|
"terminal_other_turn",
|
|
"terminal_earlier",
|
|
"terminal_failed",
|
|
"terminal_interrupted",
|
|
"terminal_other_disposition",
|
|
"missing_terminal",
|
|
"conflicting_acceptance",
|
|
"pending_question",
|
|
"pending_approval",
|
|
])("does not derive response-wake authority from %s", (mode) => {
|
|
const { result, accepted, terminal, acceptedEnvelope, terminalEnvelope } =
|
|
fixture();
|
|
const mutableResult = result as Record<string, unknown>;
|
|
const terminalPayload = terminalEnvelope.payload as Record<
|
|
string,
|
|
unknown
|
|
>;
|
|
const events = [accepted, terminal];
|
|
if (mode === "provider_acceptance")
|
|
acceptedEnvelope.sourceKind = "runner";
|
|
if (mode === "provider_terminal") terminalEnvelope.sourceKind = "runner";
|
|
if (mode === "proposal") {
|
|
accepted.eventType = "run.result.proposed";
|
|
acceptedEnvelope.eventType = accepted.eventType;
|
|
acceptedEnvelope.payload = result;
|
|
}
|
|
if (mode === "forged_marker") {
|
|
result.continuation.kind = "monitor";
|
|
mutableResult.acceptedResponseWake = {
|
|
runId: RUN_ID,
|
|
sourceEventId: "event-2",
|
|
};
|
|
}
|
|
if (mode === "missing_attention") delete mutableResult.attentionRequests;
|
|
if (mode === "attention")
|
|
mutableResult.attentionRequests = [{ kind: "ask_user_questions" }];
|
|
if (mode === "other_continuation")
|
|
result.continuation.kind = "same_agent";
|
|
if (mode === "blank_key") result.continuation.idempotencyKey = " ";
|
|
if (mode === "blank_summary") mutableResult.summary = " ";
|
|
if (mode === "missing_source") acceptedEnvelope.sourceEventId = " ";
|
|
if (mode === "wrong_run") acceptedEnvelope.runId = "other-run";
|
|
if (mode === "wrong_event_type")
|
|
acceptedEnvelope.eventType = "item.completed";
|
|
if (mode === "wrong_schema") acceptedEnvelope.schema = "provider.result";
|
|
if (mode === "terminal_other_run")
|
|
terminal.runId = terminalEnvelope.runId = "other-run";
|
|
if (mode === "terminal_other_owner")
|
|
terminalEnvelope.sourceInstanceId = "other-owner";
|
|
if (mode === "terminal_other_session")
|
|
terminalEnvelope.normalizedSessionId = "other-session";
|
|
if (mode === "terminal_other_turn")
|
|
terminalEnvelope.turnId = "other-turn";
|
|
if (mode === "terminal_earlier") terminal.seq = 1;
|
|
if (mode === "terminal_failed")
|
|
terminalPayload.runTerminalState = "failed";
|
|
if (mode === "terminal_interrupted")
|
|
terminalPayload.turnTerminalState = "interrupted";
|
|
if (mode === "terminal_other_disposition")
|
|
terminalPayload.reportedWorkDisposition = "done";
|
|
if (mode === "missing_terminal") events.pop();
|
|
if (mode === "conflicting_acceptance") {
|
|
const other = structuredClone(accepted);
|
|
other.seq = 4;
|
|
(other.payload!.prpEvent as Record<string, unknown>).sourceEventId =
|
|
"other-accepted-result";
|
|
events.push(other);
|
|
}
|
|
if (mode === "pending_question" || mode === "pending_approval")
|
|
events.push(
|
|
event(4, "runtime_request.created", {
|
|
requestId: "pending-request",
|
|
requestKind:
|
|
mode === "pending_question" ? "user_input" : "command_approval",
|
|
status: "pending",
|
|
}),
|
|
);
|
|
expect(
|
|
nativeRunEventsToTranscript(events).find(
|
|
(entry) => entry.kind === "run_result",
|
|
)?.acceptedResponseWake,
|
|
).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
it("projects the cross-language duplicate-delivery fixture exactly once", () => {
|
|
const fixture = JSON.parse(readFileSync(
|
|
new URL(
|
|
"../../../../packages/paperclip-runner/protocol/fixtures/replay/duplicate-event.json",
|
|
import.meta.url,
|
|
),
|
|
"utf8",
|
|
)) as { events: Array<Record<string, unknown>> };
|
|
const persistedEvents = fixture.events.map((prpEvent, index) => ({
|
|
id: index + 1,
|
|
companyId: "company_replay",
|
|
runId: String(prpEvent.runId),
|
|
agentId: "agent_replay",
|
|
seq: index + 1,
|
|
eventType: String(prpEvent.eventType),
|
|
stream: "system" as const,
|
|
level: "info" as const,
|
|
color: null,
|
|
message: null,
|
|
payload: { prpEvent: structuredClone(prpEvent) },
|
|
createdAt: new Date(String(prpEvent.emittedAt)),
|
|
})) satisfies HeartbeatRunEvent[];
|
|
|
|
const transcript = nativeRunEventsToTranscript(persistedEvents);
|
|
expect(transcript.filter((entry) => entry.kind === "assistant")).toEqual([
|
|
expect.objectContaining({
|
|
text: "Exactly one projection.",
|
|
channel: "final",
|
|
itemId: "item_replay_duplicate",
|
|
}),
|
|
]);
|
|
expect(transcript.filter((entry) => entry.kind === "run_result")).toHaveLength(1);
|
|
expect(transcript.filter((entry) => entry.kind === "run_terminal")).toHaveLength(1);
|
|
});
|
|
|
|
it("preserves buffered output tails when a native turn ends", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "item.delta", { itemId: "reasoning-1", kind: "reasoning", text: "Thinking" }),
|
|
event(2, "turn.completed", { status: "completed", outputTails: [
|
|
{ itemId: "reasoning-1", payload: { kind: "reasoning", text: "-" } },
|
|
] }),
|
|
]);
|
|
expect(transcript.filter(entry => entry.kind === "thinking").map(entry => "text" in entry ? entry.text : "").join("")).toBe("Thinking-");
|
|
});
|
|
|
|
it("projects provider-neutral messages, tools, usage, and the final reply", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(6, "run.result.proposed", runResult("Done safely.")),
|
|
event(1, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "Done " }),
|
|
event(2, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "safely." }),
|
|
event(3, "item.completed", { itemId: "message-1", kind: "agentMessage", text: "Done safely." }),
|
|
event(4, "tool.execution.started", {
|
|
schema: "paperclip.tool.execution.v1",
|
|
executionId: "exec-1",
|
|
transport: "process",
|
|
operation: "execute",
|
|
name: "pnpm test",
|
|
status: "running",
|
|
output: null,
|
|
outputBytes: 0,
|
|
outputTruncated: false,
|
|
outputDigest: null,
|
|
}),
|
|
event(5, "tool.execution.completed", {
|
|
schema: "paperclip.tool.execution.v1",
|
|
executionId: "exec-1",
|
|
transport: "process",
|
|
operation: "execute",
|
|
name: "pnpm test",
|
|
status: "completed",
|
|
output: "all green",
|
|
outputBytes: 9,
|
|
outputTruncated: false,
|
|
outputDigest: null,
|
|
}),
|
|
event(7, "usage.reported", {
|
|
runDeltaAvailable: true,
|
|
runDelta: {
|
|
inputTokens: 12,
|
|
outputTokens: 3,
|
|
cacheReadTokens: 2,
|
|
providerCostUsd: 0.01,
|
|
},
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({ kind: "assistant", text: "Done safely." }),
|
|
expect.objectContaining({
|
|
kind: "tool_call",
|
|
name: "Bash",
|
|
toolUseId: "exec-1",
|
|
input: { command: "pnpm test" },
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "tool_result",
|
|
toolUseId: "exec-1",
|
|
content: "all green",
|
|
isError: false,
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "run_result",
|
|
summary: "Done safely.",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "result",
|
|
subtype: "paperclip_runner_usage",
|
|
inputTokens: 12,
|
|
outputTokens: 3,
|
|
cachedTokens: 2,
|
|
costUsd: 0.01,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("coalesces sparse Codex tool lifecycle events at the named write boundary", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "tool.execution.started", {
|
|
schema: "paperclip.tool.execution.v1",
|
|
executionId: "exec-write-plan",
|
|
transport: "dynamic",
|
|
operation: "unknown",
|
|
name: null,
|
|
status: "running",
|
|
output: null,
|
|
}),
|
|
itemEvent(2, "item.started", "exec-write-plan", {
|
|
kind: "dynamicToolCall",
|
|
item: { id: "exec-write-plan" },
|
|
}),
|
|
itemEvent(3, "item.started", "exec-write-plan", {
|
|
kind: "dynamicToolCall",
|
|
item: {
|
|
type: "tool_use",
|
|
id: "exec-write-plan",
|
|
name: "write_document",
|
|
input: {
|
|
key: "plan",
|
|
title: "Plan",
|
|
body: "Ship the durable runner.",
|
|
},
|
|
},
|
|
}),
|
|
itemEvent(4, "item.completed", "exec-write-plan", {
|
|
kind: "dynamicToolCall",
|
|
item: {
|
|
type: "tool_result",
|
|
id: "exec-write-plan",
|
|
tool_use_id: "exec-write-plan",
|
|
result: {
|
|
commandKind: "write_document",
|
|
revisionId: "revision-1",
|
|
},
|
|
},
|
|
}),
|
|
event(5, "tool.execution.completed", {
|
|
schema: "paperclip.tool.execution.v1",
|
|
executionId: "exec-write-plan",
|
|
transport: "dynamic",
|
|
operation: "unknown",
|
|
name: null,
|
|
status: "completed",
|
|
output: null,
|
|
}),
|
|
itemEvent(6, "item.completed", "exec-write-plan", {
|
|
kind: "dynamicToolCall",
|
|
item: { id: "exec-write-plan", status: "completed" },
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "tool_call",
|
|
name: "write_document",
|
|
toolUseId: "exec-write-plan",
|
|
input: {
|
|
key: "plan",
|
|
title: "Plan",
|
|
body: "Ship the durable runner.",
|
|
},
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "tool_result",
|
|
toolUseId: "exec-write-plan",
|
|
toolName: "write_document",
|
|
content: JSON.stringify({
|
|
commandKind: "write_document",
|
|
revisionId: "revision-1",
|
|
}),
|
|
isError: false,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("projects ACPX item-only tool lifecycles at the named write boundary", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
itemEvent(1, "item.started", "2", {
|
|
kind: "dynamicToolCall",
|
|
item: {
|
|
type: "tool_use",
|
|
id: "2",
|
|
name: "write_document",
|
|
input: {
|
|
key: "plan",
|
|
title: "Plan",
|
|
body: "Ship the durable runner.",
|
|
},
|
|
},
|
|
}),
|
|
itemEvent(2, "item.completed", "2", {
|
|
kind: "dynamicToolCall",
|
|
item: {
|
|
type: "tool_result",
|
|
id: "2",
|
|
tool_use_id: "2",
|
|
result: {
|
|
disposition: "applied",
|
|
document: {
|
|
key: "plan",
|
|
latestRevisionId: "revision-1",
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "tool_call",
|
|
name: "write_document",
|
|
toolUseId: "2",
|
|
input: {
|
|
key: "plan",
|
|
title: "Plan",
|
|
body: "Ship the durable runner.",
|
|
},
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "tool_result",
|
|
toolUseId: "2",
|
|
toolName: "write_document",
|
|
content: JSON.stringify({
|
|
disposition: "applied",
|
|
document: {
|
|
key: "plan",
|
|
latestRevisionId: "revision-1",
|
|
},
|
|
}),
|
|
isError: false,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("streams deltas until a loss-resistant completed item is available", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "Still " }),
|
|
event(2, "item.delta", { itemId: "message-1", kind: "agentMessage", text: "working" }),
|
|
])).toEqual([
|
|
expect.objectContaining({ kind: "assistant", text: "Still ", delta: true }),
|
|
expect.objectContaining({ kind: "assistant", text: "working", delta: true }),
|
|
]);
|
|
});
|
|
|
|
it("streams canonical kind-less deltas using item identity from item.started", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
itemEvent(1, "item.started", "message-1", {
|
|
kind: "assistant_message",
|
|
channel: "progress",
|
|
text: "",
|
|
}),
|
|
itemEvent(2, "item.delta", "message-1", { text: "Still " }),
|
|
itemEvent(3, "item.delta", "message-1", { text: "working" }),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "Still ",
|
|
delta: true,
|
|
channel: "progress",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "working",
|
|
delta: true,
|
|
channel: "progress",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("keeps a canonical assistant reply ahead of its semantic result card", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "item.completed", {
|
|
kind: "assistant_message",
|
|
text: "Canonical persisted reply.",
|
|
}),
|
|
event(2, "run.result.proposed", runResult(
|
|
"Structured fallback must not replace the reply.",
|
|
)),
|
|
]);
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "Canonical persisted reply.",
|
|
channel: "unknown",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "run_result",
|
|
summary: "Structured fallback must not replace the reply.",
|
|
}),
|
|
]);
|
|
expect(transcript.filter((entry) => entry.kind === "assistant")).toHaveLength(1);
|
|
});
|
|
|
|
it("sums run deltas without leaking session-cumulative usage", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "usage.reported", {
|
|
runDeltaAvailable: true,
|
|
runDelta: {
|
|
inputTokens: 12,
|
|
outputTokens: 3,
|
|
cacheReadTokens: 2,
|
|
providerCostUsd: 0.01,
|
|
},
|
|
cumulative: {
|
|
inputTokens: 112,
|
|
outputTokens: 53,
|
|
cacheReadTokens: 22,
|
|
providerCostUsd: 1.01,
|
|
},
|
|
}),
|
|
event(2, "usage.reported", {
|
|
runDeltaAvailable: true,
|
|
runDelta: {
|
|
inputTokens: 4,
|
|
outputTokens: 2,
|
|
cacheReadTokens: 1,
|
|
providerCostUsd: 0.005,
|
|
},
|
|
cumulative: {
|
|
inputTokens: 116,
|
|
outputTokens: 55,
|
|
cacheReadTokens: 23,
|
|
providerCostUsd: 1.015,
|
|
},
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "result",
|
|
subtype: "paperclip_runner_usage",
|
|
inputTokens: 16,
|
|
outputTokens: 5,
|
|
cachedTokens: 3,
|
|
costUsd: 0.015,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("sums delta-only usage reports into one run summary", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "usage.reported", {
|
|
runDeltaAvailable: true,
|
|
runDelta: { inputTokens: 2, outputTokens: 1, providerCostUsd: 0.01 },
|
|
}),
|
|
event(2, "usage.reported", {
|
|
runDeltaAvailable: true,
|
|
runDelta: { inputTokens: 3, outputTokens: 4, providerCostUsd: 0.02 },
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "result",
|
|
inputTokens: 5,
|
|
outputTokens: 5,
|
|
costUsd: 0.03,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("uses the latest explicitly session-scoped total when run deltas are unavailable", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "usage.reported", {
|
|
runDeltaAvailable: false,
|
|
runDelta: { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 },
|
|
cumulative: { inputTokens: 12, outputTokens: 3, providerCostUsd: 0.01 },
|
|
}),
|
|
event(2, "usage.reported", {
|
|
runDeltaAvailable: false,
|
|
runDelta: { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 },
|
|
cumulative: { inputTokens: 20, outputTokens: 5, providerCostUsd: 0.02 },
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "result",
|
|
subtype: "paperclip_runner_session_usage",
|
|
inputTokens: 20,
|
|
outputTokens: 5,
|
|
costUsd: 0.02,
|
|
text: expect.stringContaining("session-cumulative"),
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("fails closed for legacy usage reports without run-delta provenance", () => {
|
|
const transcript = nativeRunEventsToTranscript([
|
|
event(1, "usage.reported", {
|
|
runDelta: { inputTokens: 12, outputTokens: 3, providerCostUsd: 0.01 },
|
|
cumulative: { inputTokens: 112, outputTokens: 53, providerCostUsd: 1.01 },
|
|
}),
|
|
event(2, "usage.reported", {
|
|
runDelta: { inputTokens: 116, outputTokens: 55, providerCostUsd: 1.015 },
|
|
}),
|
|
]);
|
|
|
|
expect(transcript).toEqual([
|
|
expect.objectContaining({
|
|
kind: "result",
|
|
subtype: "paperclip_runner_session_usage",
|
|
inputTokens: 112,
|
|
outputTokens: 53,
|
|
costUsd: 1.01,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("uses the structured run summary when no agent message was emitted", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "run.result.proposed", runResult("Recovered final reply.")),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "run_result",
|
|
summary: "Recovered final reply.",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "Recovered final reply.",
|
|
channel: "final",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("prefers an explicit final item completed after the result proposal", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "run.result.proposed", runResult("Structured fallback.")),
|
|
itemEvent(2, "item.completed", "message-final", {
|
|
kind: "assistant_message",
|
|
channel: "final",
|
|
text: "The complete final reply.",
|
|
}),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "run_result",
|
|
summary: "Structured fallback.",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "The complete final reply.",
|
|
channel: "final",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("does not append a fallback after a channel-less final delta", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
itemEvent(1, "item.started", "message-final", {
|
|
kind: "assistant_message",
|
|
text: "",
|
|
}),
|
|
itemEvent(2, "item.delta", "message-final", {
|
|
text: "Streamed final reply.",
|
|
}),
|
|
event(3, "run.result.proposed", runResult("Structured fallback.")),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "Streamed final reply.",
|
|
delta: true,
|
|
channel: "unknown",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "run_result",
|
|
summary: "Structured fallback.",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("keeps progress separate from the final runner response", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "item.completed", {
|
|
itemId: "progress-1",
|
|
kind: "agentMessage",
|
|
channel: "progress",
|
|
text: "Checking the implementation.",
|
|
}),
|
|
event(2, "run.result.accepted", {
|
|
result: runResult("The implementation is ready."),
|
|
}),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "Checking the implementation.",
|
|
channel: "progress",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "run_result",
|
|
summary: "The implementation is ready.",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "assistant",
|
|
text: "The implementation is ready.",
|
|
channel: "final",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("projects provider-neutral activity without exposing provider envelopes", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "research.started", {
|
|
schema: "paperclip.research.v1",
|
|
researchId: "research-1",
|
|
query: "current behavior",
|
|
status: "running",
|
|
}),
|
|
event(2, "research.completed", {
|
|
schema: "paperclip.research.v1",
|
|
researchId: "research-1",
|
|
query: "current behavior",
|
|
status: "completed",
|
|
}),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "tool_call",
|
|
name: "research",
|
|
toolUseId: "research:research-1",
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "tool_result",
|
|
toolUseId: "research:research-1",
|
|
content: "current behavior",
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it.each(["running", "pending", "in_progress"])(
|
|
"keeps an explicitly %s activity open even when its event suffix looks terminal",
|
|
(status) => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "artifact.generated", {
|
|
schema: "paperclip.artifact.generated.v1",
|
|
artifactId: "artifact-1",
|
|
status,
|
|
reference: "artifacts/preview.png",
|
|
}),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "tool_call",
|
|
toolUseId: "artifact:artifact-1",
|
|
}),
|
|
]);
|
|
},
|
|
);
|
|
|
|
it("fails closed for unsupported versions, prefix lookalikes, and mismatched payload schemas", () => {
|
|
const unsupportedVersion = event(1, "model.verification.updated", {
|
|
schema: "paperclip.model.verification.v1",
|
|
verificationId: "verification-1",
|
|
status: "completed",
|
|
summary: "must not render",
|
|
});
|
|
(unsupportedVersion.payload!.prpEvent as Record<string, unknown>).schemaVersion = 2;
|
|
|
|
expect(nativeRunEventsToTranscript([
|
|
unsupportedVersion,
|
|
event(2, "model.provider_message.recorded", {
|
|
schema: "paperclip.model.provider_message.v1",
|
|
routeId: "route-1",
|
|
message: "provider envelope must not render",
|
|
}),
|
|
event(3, "model.verification.updated", {
|
|
schema: "paperclip.provider.native.v1",
|
|
verificationId: "verification-2",
|
|
status: "completed",
|
|
summary: "wrong payload schema must not render",
|
|
}),
|
|
])).toEqual([]);
|
|
});
|
|
|
|
it("fails closed for mismatched tool execution and run result schemas", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "tool.execution.started", {
|
|
schema: "paperclip.provider.native.v1",
|
|
executionId: "exec-1",
|
|
transport: "process",
|
|
operation: "execute",
|
|
status: "running",
|
|
}),
|
|
event(2, "run.result.proposed", {
|
|
schema: "paperclip.provider.native.v1",
|
|
summary: "malformed proposal must not render",
|
|
}),
|
|
event(3, "run.result.accepted", {
|
|
result: {
|
|
schema: "paperclip.provider.native.v1",
|
|
summary: "malformed accepted result must not render",
|
|
},
|
|
}),
|
|
event(4, "run.result.accepted", {
|
|
schema: "paperclip.run_result.v1",
|
|
summary: "accepted wrappers must not masquerade as results",
|
|
}),
|
|
])).toEqual([]);
|
|
});
|
|
|
|
it("fails closed for malformed, mismatched, and unknown event envelopes", () => {
|
|
const mismatched = event(1, "item.delta", {
|
|
itemId: "message-1",
|
|
kind: "agentMessage",
|
|
text: "must not render",
|
|
});
|
|
(mismatched.payload!.prpEvent as Record<string, unknown>).runId = "other-run";
|
|
const malformed = event(2, "item.delta", {});
|
|
malformed.payload = { providerNativeSecret: "must not render" };
|
|
|
|
expect(nativeRunEventsToTranscript([
|
|
mismatched,
|
|
malformed,
|
|
event(3, "extension.unknown", { explanation: "not a transcript row" }),
|
|
])).toEqual([]);
|
|
});
|
|
|
|
it("projects structured questions through resolution without losing identity", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "runtime_request.created", {
|
|
request: {
|
|
schema: "paperclip.runtime_request.v1",
|
|
requestId: "question-1",
|
|
requestKind: "elicitation",
|
|
type: "input",
|
|
status: "pending",
|
|
prompt: "Choose a rollout",
|
|
input: {
|
|
schema: "paperclip.question_set.v1",
|
|
questions: [{
|
|
id: "rollout",
|
|
prompt: "Which rollout?",
|
|
required: true,
|
|
answerMode: "single_select",
|
|
options: [{ id: "safe", label: "Safe" }],
|
|
}],
|
|
},
|
|
},
|
|
}),
|
|
event(2, "runtime_request.resolved", {
|
|
request: {
|
|
requestId: "question-1",
|
|
status: "resolved",
|
|
response: {
|
|
schema: "paperclip.question_response.v1",
|
|
answers: { rollout: { selectedOptionIds: ["safe"] } },
|
|
},
|
|
},
|
|
}),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "runtime_request",
|
|
requestId: "question-1",
|
|
status: "pending",
|
|
questionSet: expect.objectContaining({ schema: "paperclip.question_set.v1" }),
|
|
}),
|
|
expect.objectContaining({
|
|
kind: "runtime_request",
|
|
requestId: "question-1",
|
|
status: "resolved",
|
|
response: expect.objectContaining({ schema: "paperclip.question_response.v1" }),
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it.each([
|
|
["failed", "failed", "failed"],
|
|
["interrupted", "cancelled", "cancelled"],
|
|
["cancelled", "cancelled", "cancelled"],
|
|
] as const)(
|
|
"projects a visible %s terminal state",
|
|
(turnTerminalState, runTerminalState, expectedRunState) => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "run.terminal", {
|
|
schema: "paperclip.prp.terminal.v1",
|
|
turnTerminalState,
|
|
runTerminalState,
|
|
reportedWorkDisposition: "needs_review",
|
|
stopReason: {
|
|
schema: "paperclip.stop_reason.v1",
|
|
code: "provider_failed",
|
|
message: "Provider could not finish the turn.",
|
|
},
|
|
}),
|
|
])).toEqual([
|
|
expect.objectContaining({
|
|
kind: "run_terminal",
|
|
turnState: turnTerminalState,
|
|
runState: expectedRunState,
|
|
stopReason: "Provider could not finish the turn.",
|
|
}),
|
|
]);
|
|
},
|
|
);
|
|
|
|
it("fails closed instead of presenting malformed terminal state as success", () => {
|
|
expect(nativeRunEventsToTranscript([
|
|
event(1, "run.terminal", {
|
|
schema: "paperclip.prp.terminal.v1",
|
|
turnTerminalState: "mystery",
|
|
runTerminalState: "successful-ish",
|
|
reportedWorkDisposition: "done",
|
|
}),
|
|
])).toEqual([]);
|
|
});
|
|
});
|