mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
fix(runner): recover empty OpenCode replies without fabricating success
This commit is contained in:
1 parent
1e423103b3
commit
b0d0674fe7
4 files changed
+164
-14
No files matched your search
@@ -767,15 +767,26 @@ fn terminal_events(
|
||||
let Some(contract) = state.completion_contract.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
if rejected_completion {
|
||||
let empty_opencode_completion = state.config.provider == "opencode"
|
||||
&& event_type == "turn.completed"
|
||||
&& state.active_provider_result_fingerprint.is_none()
|
||||
&& state
|
||||
.last_agent_message
|
||||
.as_deref()
|
||||
.is_none_or(|text| text.trim().is_empty());
|
||||
if rejected_completion || empty_opencode_completion {
|
||||
return vec![
|
||||
NormalizedProviderEvent {
|
||||
event_type: "session.failed".to_owned(),
|
||||
priority: EventPriority::P0,
|
||||
payload: json!({
|
||||
"provider": state.config.provider,
|
||||
"code": "semantic_completion_rejected",
|
||||
"message": "The turn ended without correcting its rejected completion report.",
|
||||
"code": if rejected_completion { "semantic_completion_rejected" } else { "opencode_empty_response" },
|
||||
"message": if rejected_completion {
|
||||
"The turn ended without correcting its rejected completion report."
|
||||
} else {
|
||||
"OpenCode ended without a final answer or an accepted completion report."
|
||||
},
|
||||
"recoverable": false,
|
||||
}),
|
||||
},
|
||||
@@ -5733,6 +5744,7 @@ mod tests {
|
||||
#[test]
|
||||
fn an_undeclared_finish_rejection_does_not_change_terminal_authority() {
|
||||
let mut state = opencode_result_state();
|
||||
state.last_agent_message = Some("The requested answer is ready.".to_owned());
|
||||
state
|
||||
.admit_tool_call("undeclared-finish", "paperclip_finish", &json!({}))
|
||||
.unwrap_err();
|
||||
@@ -6023,14 +6035,11 @@ mod tests {
|
||||
}),
|
||||
ProviderToolBridge::default(),
|
||||
);
|
||||
state.last_agent_message = None;
|
||||
state.last_agent_message = Some("The actual final answer.".to_owned());
|
||||
|
||||
let events = terminal_events(&state, "turn.completed", None);
|
||||
|
||||
assert_eq!(
|
||||
events[0].payload["summary"],
|
||||
"OpenCode completed the requested work."
|
||||
);
|
||||
assert_eq!(events[0].payload["summary"], "The actual final answer.");
|
||||
assert_eq!(
|
||||
events[0].payload["evidence"][0]["ref"],
|
||||
"provider:opencode:agent-message"
|
||||
@@ -6039,6 +6048,21 @@ mod tests {
|
||||
assert!(!events[0].payload.to_string().contains("Codex"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opencode_empty_terminal_does_not_fabricate_a_successful_answer() {
|
||||
for message in [None, Some(" \n".to_owned())] {
|
||||
let mut state = opencode_result_state();
|
||||
state.last_agent_message = message;
|
||||
let events = terminal_events(&state, "turn.completed", None);
|
||||
assert!(!events
|
||||
.iter()
|
||||
.any(|event| event.event_type == "run.result.proposed"));
|
||||
assert_eq!(events[0].event_type, "session.failed");
|
||||
assert_eq!(events[0].payload["code"], "opencode_empty_response");
|
||||
assert_eq!(events[1].payload["runTerminalState"], "failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn goal_timestamps_normalize_seconds_milliseconds_and_iso() {
|
||||
for value in [
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { chmod, copyFile, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import { OpenCodeServerDriver } from "./opencode-server-driver.js";
|
||||
import type { PrpEvent } from "../../protocol/replay-contract.js";
|
||||
|
||||
it.each(["reply", "empty-again", "request-fails", "normal", "foreign-idle"] as const)("handles an empty OpenCode response: %s", async mode => {
|
||||
const root = await mkdtemp(join(tmpdir(), "opencode-empty-response-"));
|
||||
const command = join(root, "fake-opencode.mjs");
|
||||
await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command);
|
||||
await chmod(command, 0o755);
|
||||
const prompts: Record<string, any>[] = [];
|
||||
let stream: ReadableStreamDefaultController<Uint8Array> | undefined;
|
||||
const queued: Uint8Array[] = [];
|
||||
const emit = (value: unknown) => { const frame = new TextEncoder().encode(`data: ${JSON.stringify(value)}\n\n`); if (stream) stream.enqueue(frame); else queued.push(frame); };
|
||||
const driver = new OpenCodeServerDriver({
|
||||
model: "openrouter/controlled-model", runtimeDirectory: join(root, "runtime"), command,
|
||||
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "controlled-fixture-key" },
|
||||
fetch: async (input, init) => {
|
||||
const url = new URL(String(input));
|
||||
if (url.pathname === "/event") return new Response(new ReadableStream<Uint8Array>({ start(controller) { stream = controller; for (const frame of queued.splice(0)) controller.enqueue(frame); } }), { headers: { "Content-Type": "text/event-stream" } });
|
||||
if (url.pathname.endsWith("/prompt_async")) {
|
||||
const prompt = JSON.parse(String(init?.body)); prompts.push(prompt);
|
||||
if (prompts.length === 2 && mode === "request-fails") throw new Error("controlled correction request failed");
|
||||
const sessionID = decodeURIComponent(url.pathname.split("/")[2]!);
|
||||
const number = prompts.length, messageID = `assistant-${number}`;
|
||||
queueMicrotask(() => {
|
||||
if (mode === "foreign-idle") emit({ type: "session.idle", properties: { sessionID: "foreign-session" } });
|
||||
emit({ type: "message.updated", properties: { info: { id: messageID, sessionID, role: "assistant", finish: "stop", time: { completed: Date.now() } } } });
|
||||
const answer = mode === "normal" || mode === "foreign-idle" || number === 2 && mode === "reply";
|
||||
emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID,
|
||||
type: answer ? "text" : "reasoning", text: answer ? "Here is the actual final answer." : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } });
|
||||
emit({ type: "session.idle", properties: { sessionID } });
|
||||
emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } });
|
||||
});
|
||||
return new Response(null, { status: 204 });
|
||||
}
|
||||
return fetch(input, init);
|
||||
},
|
||||
});
|
||||
const session = await driver.openSession({ runId: "owned-run", normalizedSessionId: "owned-session", workingDirectory: root });
|
||||
try {
|
||||
const collected = (async () => { const rows: PrpEvent[] = []; for await (const event of session.events()) {
|
||||
rows.push(event); if (["turn.completed", "turn.failed"].includes(event.eventType)) break;
|
||||
} return rows; })();
|
||||
const { turnId } = await session.startTurn({ message: { role: "user", text: "Answer the current request." }, reasoningMode: "disabled" });
|
||||
const events = await collected;
|
||||
const succeeds = ["reply", "normal", "foreign-idle"].includes(mode);
|
||||
expect(events.at(-1)?.eventType).toBe(succeeds ? "turn.completed" : "turn.failed");
|
||||
expect(events.filter(event => event.eventType === "turn.started")).toHaveLength(1);
|
||||
expect(events.every(event => !event.turnId || event.turnId === turnId)).toBe(true);
|
||||
const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final");
|
||||
expect(final.map(event => event.payload.text)).toEqual(succeeds ? ["Here is the actual final answer."] : []);
|
||||
expect(prompts).toHaveLength(["normal", "foreign-idle"].includes(mode) ? 1 : 2);
|
||||
if (prompts.length === 2) {
|
||||
expect(prompts[1]).toMatchObject({ providerID: prompts[0]!.providerID, modelID: prompts[0]!.modelID, system: prompts[0]!.system, variant: "paperclip-no-reasoning" });
|
||||
expect(JSON.stringify(prompts[1]!.parts)).toContain("Do not repeat completed actions");
|
||||
}
|
||||
} finally {
|
||||
await session.close({ reason: "controlled test complete" });
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 30_000);
|
||||
@@ -2750,6 +2750,10 @@ describe("OpenCodeServerDriver", () => {
|
||||
id: "prt_oversized", messageID: "msg_oversized", sessionID, type: "tool", callID: "call_oversized", tool: "bash",
|
||||
state: { status: "completed", input: { command: "pnpm test" }, output, title: "pnpm test", metadata: { exit: 1, output }, time: { start: 1, end: 2 } },
|
||||
} } });
|
||||
send({ type: "message.part.updated", properties: { part: {
|
||||
id: "prt_final", messageID: "msg_oversized", sessionID, type: "text",
|
||||
text: "The suite completed: one test failed; 812 passed.", time: { start: 3, end: 4 },
|
||||
} } });
|
||||
send({ type: "session.idle", properties: { sessionID } });
|
||||
const observed = await events;
|
||||
expect(observed.at(-1)?.eventType).toBe("turn.completed");
|
||||
|
||||
@@ -540,6 +540,10 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
readonly #companyObservability: ReturnType<typeof createOpenCodeCompanyObservability>;
|
||||
#closed = false;
|
||||
#completionSettlement: Promise<void> | null = null;
|
||||
#turnStartSourceSeq = 0;
|
||||
#turnReasoningMode: OpenCodeReasoningMode = "default";
|
||||
#emptyResponseRecovery: { turnId: string; priorMessages: Set<string>; responseFinished: boolean } | null = null;
|
||||
#emptyResponseAckTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
#abort = new AbortController();
|
||||
|
||||
constructor(input: {
|
||||
@@ -686,14 +690,15 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
const turnId = `turn-${randomBytes(12).toString("hex")}`;
|
||||
this.#streamingParts.clear();
|
||||
this.#activeTurnId = turnId;
|
||||
this.#turnStartSourceSeq = this.#sourceSequence;
|
||||
this.#turnReasoningMode = reasoningMode;
|
||||
this.#emptyResponseRecovery = null;
|
||||
this.#emit("turn.submitted", {
|
||||
envelopeSchema: this.#taskEnvelope.schema,
|
||||
text: input.message.text,
|
||||
}, { turnId });
|
||||
this.#emit("turn.accepted", { turnId }, { turnId });
|
||||
this.#emit("turn.started", { status: "inProgress" }, { turnId });
|
||||
const [providerID, ...modelParts] = this.#model.split("/");
|
||||
const modelID = modelParts.join("/");
|
||||
// Keep the task envelope only on the initial task-mode wake. OpenCode
|
||||
// rebuilds system instructions from the latest user message, so system
|
||||
// instructions must still accompany every prompt (including recovery).
|
||||
@@ -703,12 +708,21 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
message: input.message.text,
|
||||
})
|
||||
: input.message.text;
|
||||
await this.#submitPrompt(prompt);
|
||||
this.#sendFullContext = false;
|
||||
return { turnId };
|
||||
}
|
||||
|
||||
async #submitPrompt(prompt: string): Promise<void> {
|
||||
const [providerID, ...modelParts] = this.#model.split("/");
|
||||
const modelID = modelParts.join("/");
|
||||
await api(
|
||||
this.#fetch,
|
||||
this.#runtime,
|
||||
`/session/${encodeURIComponent(this.#providerSessionId)}/prompt_async`,
|
||||
{
|
||||
method: "POST",
|
||||
signal: this.#abort.signal,
|
||||
body: JSON.stringify({
|
||||
// OpenCode 1.18's PromptPayload carries the provider and model at the
|
||||
// top level. Older examples used a nested `model` object; 1.18
|
||||
@@ -721,7 +735,7 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
// Always select the empty default variant on an ordinary turn so a
|
||||
// previous disabled turn cannot change this turn's provider defaults.
|
||||
...(providerID === "openrouter"
|
||||
? { variant: reasoningMode === "disabled" ? "paperclip-no-reasoning" : "paperclip-default" }
|
||||
? { variant: this.#turnReasoningMode === "disabled" ? "paperclip-no-reasoning" : "paperclip-default" }
|
||||
: {}),
|
||||
// Keep question enabled in the isolated config. OpenCode 1.18.32
|
||||
// turns a prompt's deprecated `tools` map into replacement session
|
||||
@@ -731,8 +745,39 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
}),
|
||||
},
|
||||
);
|
||||
this.#sendFullContext = false;
|
||||
return { turnId };
|
||||
}
|
||||
|
||||
#clearEmptyResponseAckTimer(): void {
|
||||
clearTimeout(this.#emptyResponseAckTimer);
|
||||
this.#emptyResponseAckTimer = undefined;
|
||||
}
|
||||
|
||||
#failEmptyResponse(turnId: string, message: string): void {
|
||||
if (this.#closed || this.#activeTurnId !== turnId || this.#result !== null) return;
|
||||
this.#emit("turn.failed", { status: "failed", error: { code: "opencode_empty_response", message, retryable: false } }, { turnId });
|
||||
this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" }));
|
||||
this.#activeTurnId = null;
|
||||
}
|
||||
|
||||
#recoverEmptyResponse(turnId: string): boolean {
|
||||
if (this.#result !== null || this.#pendingRuntimeRequests.size > 0) return false;
|
||||
const recovery = this.#emptyResponseRecovery;
|
||||
// Both idle event forms can remain queued from the empty response. Wait
|
||||
// for the correction's own completed assistant message before settling.
|
||||
if (recovery && !recovery.responseFinished) return true;
|
||||
const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq);
|
||||
if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary)) return false;
|
||||
if (recovery) {
|
||||
this.#failEmptyResponse(turnId, "OpenCode ended without a final answer or accepted completion report after one correction.");
|
||||
return true;
|
||||
}
|
||||
this.#emptyResponseRecovery = { turnId, priorMessages: new Set(this.#messageRoles.keys()), responseFinished: false };
|
||||
this.#emit("harness.diagnostic", { code: "opencode_empty_response_correction", message: "Requesting one final response after an empty OpenCode completion." }, { turnId });
|
||||
this.#emptyResponseAckTimer = setTimeout(() => this.#failEmptyResponse(turnId, "OpenCode did not start its requested final response."), 20_000);
|
||||
this.#emptyResponseAckTimer.unref();
|
||||
void this.#submitPrompt("Your response ended without a user-visible final answer or an accepted completion report. Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.")
|
||||
.catch(() => this.#failEmptyResponse(turnId, "OpenCode could not request a final response after an empty completion."));
|
||||
return true;
|
||||
}
|
||||
|
||||
async interrupt(input: { turnId?: string; reason?: string }): Promise<void> {
|
||||
@@ -992,6 +1037,7 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
await this.#completionSettlement;
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
this.#clearEmptyResponseAckTimer();
|
||||
this.#abort.abort();
|
||||
// Settle whatever is still pending, including a request whose turn
|
||||
// already went terminal without the driver observing it (see
|
||||
@@ -1583,7 +1629,8 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
// adjacent tokens. Content deduplication would silently lose the second.
|
||||
// Key id-less events by digest: a retained canonical copy of each recent
|
||||
// event would hold up to 10,000 large tool payloads in memory.
|
||||
const eventId = text(event.id) || (type === "message.part.delta"
|
||||
const idle = type === "session.idle" || type === "session.status" && text(record(properties.status).type) === "idle";
|
||||
const eventId = idle ? null : text(event.id) || (type === "message.part.delta"
|
||||
? null
|
||||
: `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`);
|
||||
if (eventId !== null && this.#seenProviderEvents.has(eventId)) return;
|
||||
@@ -1722,6 +1769,12 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
text(info.messageID, text(info.messageId)),
|
||||
);
|
||||
const role = text(info.role);
|
||||
const recovery = this.#emptyResponseRecovery;
|
||||
if (recovery?.turnId === turnId && role === "assistant" && messageId && !recovery.priorMessages.has(messageId)) {
|
||||
this.#clearEmptyResponseAckTimer();
|
||||
if ((Number.isFinite(record(info.time).completed) || text(info.finish)) && !["tool-calls", "tool_calls"].includes(text(info.finish)))
|
||||
recovery.responseFinished = true;
|
||||
}
|
||||
// Record the message's owning turn at the moment OpenCode first
|
||||
// reports it. A later turn that reuses the same native message id
|
||||
// legitimately reclaims ownership; a stale message never sees this
|
||||
@@ -1775,6 +1828,7 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
text(record(properties.status).type) === "idle")) &&
|
||||
turnId
|
||||
) {
|
||||
if (this.#recoverEmptyResponse(turnId)) return;
|
||||
const fingerprint = canonicalJson({
|
||||
status: "completed",
|
||||
semanticResult: this.#resultFingerprint,
|
||||
@@ -2147,7 +2201,11 @@ class OpenCodeHarnessSession implements HarnessSession {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (eventType === "run.result.proposed" || eventType === "runtime_request.created") {
|
||||
this.#clearEmptyResponseAckTimer();
|
||||
}
|
||||
if (["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(eventType)) {
|
||||
this.#clearEmptyResponseAckTimer();
|
||||
const performance = this.#runtime.performanceDiagnostics?.take(this.#performanceBoundaryMs);
|
||||
this.#performanceBoundaryMs = this.#now().getTime();
|
||||
if (performance) this.#emit("harness.diagnostic", performance, refs);
|
||||
|
||||
Reference in new issue
Block a user