diff --git a/server/src/services/native-runtime/native-completion-feedback.test.ts b/server/src/services/native-runtime/native-completion-feedback.test.ts index 841878e195b..1b0eb6fa773 100644 --- a/server/src/services/native-runtime/native-completion-feedback.test.ts +++ b/server/src/services/native-runtime/native-completion-feedback.test.ts @@ -410,6 +410,36 @@ describe("native final-response feedback", () => { await db.update(issues).set({ status: "in_progress", executionRunId: newerRunId }).where(eq(issues.id, value.issueId)); await expect(nativeCompletionFeedback(db, value.runId, report)).resolves.toContain("newer run owns"); }); + it("asks once to re-check Paperclip instead of resting blocked on its own stall", async () => { + // Chaos run 2b600576: a SIGSTOPped server left write_document/read_document + // unanswered; the agent reported the Paperclip document service as an + // external outage and the task rested blocked with no Retry. + const value = await fixture(); + const report: PrpStructuredRunResult = { ...done, reportedWorkDisposition: "blocked", + completionClaim: { ...done.completionClaim, objectiveSatisfied: false }, + blocker: { reasonCode: "external_service_unavailable", owner: { kind: "system", name: "Paperclip document service" }, + scope: "task_wide", unblockAction: "Restore document API responsiveness" } }; + await expect(nativeCompletionFeedback(db, value.runId, report)).rejects.toThrow(/Paperclip answered this report[\s\S]*same idempotencyKey/); + // Once per run: a Paperclip failure that persists after the re-check rests blocked. + await expect(nativeCompletionFeedback(db, value.runId, report)).resolves.toContain("Blocker report accepted"); + // Concatenated names such as "PaperclipAI document service" also name a Paperclip service. + const concatenated = await fixture(); + await expect(nativeCompletionFeedback(db, concatenated.runId, { ...report, blocker: { ...report.blocker!, owner: { kind: "system", name: "PaperclipAI document service" } } })) + .rejects.toThrow("Paperclip answered this report"); + const runnerNamed = await fixture(); + await expect(nativeCompletionFeedback(db, runnerNamed.runId, { ...report, blocker: { ...report.blocker!, owner: { kind: "system", name: "paperclip_runner" } } })) + .rejects.toThrow("Paperclip answered this report"); + // A genuinely external service, a non-system owner, or a finished run is not re-checked. + const other = await fixture(); + await expect(nativeCompletionFeedback(db, other.runId, { ...report, blocker: { ...report.blocker!, owner: { kind: "system", name: "GitHub API" } } })) + .resolves.toContain("Blocker report accepted"); + await expect(nativeCompletionFeedback(db, other.runId, { ...report, blocker: { ...report.blocker!, owner: { kind: "user", name: "Paperclip admin" } } })) + .resolves.toContain("Blocker report accepted"); + await expect(nativeCompletionFeedback(db, other.runId, { ...report, blocker: { ...report.blocker!, reasonCode: "missing_access" } })) + .resolves.toContain("Blocker report accepted"); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, other.runId)); + await expect(nativeCompletionFeedback(db, other.runId, report)).resolves.toContain("Blocker report accepted"); + }); it("asks a blocked provider to explain the cause and action instead of describing completed work", async () => { const value = await fixture(); const feedback = await nativeCompletionFeedback(db, value.runId, { ...done, reportedWorkDisposition: "blocked", diff --git a/server/src/services/native-runtime/native-completion-feedback.ts b/server/src/services/native-runtime/native-completion-feedback.ts index 9d3de432a54..18969541880 100644 --- a/server/src/services/native-runtime/native-completion-feedback.ts +++ b/server/src/services/native-runtime/native-completion-feedback.ts @@ -32,6 +32,31 @@ import { const repositoryYieldCorrection = 'This task has a saved repository preparation for its next admission. This is routine setup, not a blocker. Call paperclip_finish with reportedWorkDisposition="yielded" and omit continuation (or use null); Paperclip will select the saved workspace continuation. Preserve the completed work and explain what remains. Do not repeat setup, set a timer, or ask the user to restart.'; +// A Paperclip stall or restart leaves the agent's Paperclip tool calls +// unanswered until the server is back; they then return together. The agent can +// read that silence as an outage and report it as a blocker. This report reached +// the server and is being answered, which is the server's own proof that its +// control plane is reachable now. Ask once per run to re-check with fresh +// Paperclip calls instead of resting the task blocked on a past stall. +const paperclipServiceRecheckCorrection = "Paperclip answered this report, so its control plane is reachable now. Paperclip tool calls that did not return earlier were most likely delayed by a Paperclip restart or pause, and their results may have arrived since. Do not report this as a blocker. Re-read the current Paperclip state (for example read_document) before writing again; when a write is still needed, repeat it with the same idempotencyKey so it cannot apply twice. Then continue the task and report its real outcome. Report blocked only if a fresh Paperclip call fails now."; +const PAPERCLIP_SERVICE_RECHECK_LIMIT = 1_000; +const paperclipServiceRechecked = new Set(); + +function isPaperclipServiceOutageBlocker(result: PrpStructuredRunResult) { + const blocker = result.blocker; + return result.reportedWorkDisposition === "blocked" && blocker?.owner.kind === "system" + && blocker.reasonCode === "external_service_unavailable" && (blocker.owner.name ?? "").toLowerCase().includes("paperclip"); +} + +/** Once per run (per server process), so a real persistent failure still rests blocked. */ +function claimPaperclipServiceRecheck(runId: string) { + if (paperclipServiceRechecked.has(runId)) return false; + if (paperclipServiceRechecked.size >= PAPERCLIP_SERVICE_RECHECK_LIMIT) + paperclipServiceRechecked.delete(paperclipServiceRechecked.values().next().value!); + paperclipServiceRechecked.add(runId); + return true; +} + function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; @@ -323,6 +348,10 @@ export async function nativeCompletionFeedback( throw new Error(repositoryYieldCorrection); } } + if (isPaperclipServiceOutageBlocker(result) && issue.executionRunId === run.id && run.status === "running" + && claimPaperclipServiceRecheck(run.id)) { + throw new Error(paperclipServiceRecheckCorrection); + } if (result.reportedWorkDisposition === "yielded" && result.continuation?.kind === "monitor") { return `Monitor wait accepted. The task remains active and Paperclip will wake its assignee at or after ${issue.monitorNextCheckAt!.toISOString()} with issue_monitor_due. End this turn; do not poll or mark the task done.`; }