diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 57408cd3455..59fb391c5e6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -265,6 +265,7 @@ import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js"; import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { createNativeCredentialUpdateOwner, appendNativeCredentialUpdate } from "./native-credential-updates.js"; import { canonicalNativeJson, nativeSha256 } from "./canonical.js"; +import { RepeatedToolFailureGuard } from "./repeated-tool-failure-guard.js"; import { lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { dispatchToolCallback, recoverToolCallback, type SemanticInputAdmission } from "../../vendor/paperclip-runner/index.js"; @@ -2171,9 +2172,11 @@ export async function synchronizeCompletedProviderPlan(input: { }; } -class SessionAuthorityEpoch { +export class SessionAuthorityEpoch { readonly runId: string; #authority: PaperclipRunnerToolAuthority; + // Per run: a new run gets a new epoch and starts with no remembered failures. + readonly #repeatedFailures = new RepeatedToolFailureGuard(); #revoked = false; readonly #drainingRun?: { runId: string; capture?: (event: PrpEvent) => Promise }; @@ -2237,9 +2240,10 @@ class SessionAuthorityEpoch { async execute(call: Parameters[0]) { this.#assertCurrent(); + const execute = () => this.#repeatedFailures.run(call, () => this.#authority.execute(call)); return dispatchToolCallback(this.runId, call.callId, async () => this.toolTrace - ? await this.toolTrace.execute(call, () => this.#authority.execute(call)) - : await this.#authority.execute(call)); + ? await this.toolTrace.execute(call, execute) + : await execute()); } } diff --git a/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts b/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts new file mode 100644 index 00000000000..aef88f47394 --- /dev/null +++ b/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it, vi } from "vitest"; +import { HttpError, badRequest } from "../../errors.js"; +import { SessionAuthorityEpoch } from "./native-session-executor.js"; +import type { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import { executeRunnerApi } from "./runner-api-client.js"; + +const context = { companyId: "company-a", issueId: "issue-a", issueIdentifier: "API-1", runId: "run-a", workMode: "standard" }; + +function epochWith(execute: (call: { tool: string; callId: string; arguments: unknown }) => Promise) { + const authority = { execute: vi.fn(execute), close: () => {} }; + return { authority, epoch: new SessionAuthorityEpoch("run-a", authority as unknown as PaperclipRunnerToolAuthority) }; +} + +const failure = (epoch: SessionAuthorityEpoch, call: { tool: string; callId: string; arguments: unknown }) => + epoch.execute(call).then(() => null, (error: Error) => error); + +describe("repeated identical failing tool calls", () => { + // Recorded cohort run: a model host dropped `body`, and the model resent the + // same body-less PATCH twelve times, each refused before sending. + const dropped = { operationId: "PATCH /api/issues/{id}", pathParams: { id: "issue-a" }, query: {}, responseText: { limitBytes: 8192 } }; + + it("corrects the second identical failure and does not run the third", async () => { + const fetcher = vi.fn(); + const { authority, epoch } = epochWith(async call => executeRunnerApi(call.arguments as never, context, + { apiUrl: "http://127.0.0.1:3100", token: "t", fetch: fetcher, readFile: async () => { throw new Error("unused"); }, saveResponse: async () => { throw new Error("unused"); } })); + const call = (n: number) => ({ tool: "call_api", callId: `call-${n}`, arguments: dropped }); + + const first = await failure(epoch, call(1)); + expect(first?.message).toMatch(/^This request was not sent: PATCH \/api\/issues\/\{id\} requires body/); + expect(first?.message).not.toContain("failed 2 times"); + + const second = await failure(epoch, call(2)); + expect(second).toBeInstanceOf(HttpError); + expect((second as HttpError).status).toBe(400); + // The original correction stays visible, followed by what arrived and what to do. + expect(second?.message).toContain(first!.message); + expect(second?.message).toContain("failed 2 times with the same arguments"); + expect(second?.message).toContain("Arguments received each time: operationId, pathParams, query, responseText."); + expect(second?.message).toContain("Paperclip will not run this exact call again in this run"); + + for (const n of [3, 4, 5]) { + const blocked = await failure(epoch, call(n)); + expect(blocked?.message).toMatch(/^This call was not run: call_api already failed 2 times/); + expect(blocked?.message).toContain(first!.message); + } + expect(authority.execute).toHaveBeenCalledTimes(2); + expect(fetcher).not.toHaveBeenCalled(); + }); + + it("runs the call again once its arguments change", async () => { + const { authority, epoch } = epochWith(async call => { + if (!(call.arguments as { body?: unknown }).body) throw badRequest("body is required"); + return { ok: true }; + }); + await failure(epoch, { tool: "call_api", callId: "a", arguments: dropped }); + await failure(epoch, { tool: "call_api", callId: "b", arguments: dropped }); + await expect(epoch.execute({ tool: "call_api", callId: "c", arguments: { ...dropped, body: { title: "x" } } })).resolves.toEqual({ ok: true }); + expect(authority.execute).toHaveBeenCalledTimes(3); + }); + + it("passes failures that may not repeat through unchanged and keeps running them", async () => { + const outage = new HttpError(503, "Paperclip API unavailable"); + const { authority, epoch } = epochWith(async () => { throw outage; }); + for (const n of [1, 2, 3]) expect(await failure(epoch, { tool: "call_api", callId: `${n}`, arguments: dropped })).toBe(outage); + expect(authority.execute).toHaveBeenCalledTimes(3); + }); + + it("passes the first failure through unchanged and counts only matching errors", async () => { + let n = 0; + const { authority, epoch } = epochWith(async () => { throw badRequest(`refusal ${++n}`); }); + const errors = []; + for (const id of ["1", "2", "3"]) errors.push(await failure(epoch, { tool: "create_task", callId: id, arguments: { title: "t" } })); + expect(errors.map(error => error?.message)).toEqual(["refusal 1", "refusal 2", "refusal 3"]); + expect(authority.execute).toHaveBeenCalledTimes(3); + }); + it("counts identical calls sent in parallel and runs at most two of them", async () => { + const { authority, epoch } = epochWith(async () => { await new Promise(resolve => setTimeout(resolve, 5)); throw badRequest("body is required"); }); + const errors = await Promise.all([1, 2, 3, 4].map(n => failure(epoch, { tool: "call_api", callId: `p${n}`, arguments: dropped }))); + expect(authority.execute).toHaveBeenCalledTimes(2); + expect(errors[0]?.message).toBe("body is required"); + expect(errors[1]?.message).toContain("failed 2 times with the same arguments"); + expect(errors[2]?.message).toMatch(/^This call was not run/); + expect(errors[3]?.message).toMatch(/^This call was not run/); + }); + + it("does not hold back a different call while an identical one runs", async () => { + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const { epoch } = epochWith(async call => { if (call.callId === "slow") await gate; return call.callId; }); + const slow = epoch.execute({ tool: "list_agents", callId: "slow", arguments: {} }); + await expect(epoch.execute({ tool: "list_projects", callId: "other", arguments: {} })).resolves.toBe("other"); + release(); + await expect(slow).resolves.toBe("slow"); + }); +}); diff --git a/server/src/services/native-runtime/repeated-tool-failure-guard.ts b/server/src/services/native-runtime/repeated-tool-failure-guard.ts new file mode 100644 index 00000000000..b6b6c4127cd --- /dev/null +++ b/server/src/services/native-runtime/repeated-tool-failure-guard.ts @@ -0,0 +1,82 @@ +import { HttpError } from "../../errors.js"; +import { nativeSha256 } from "./canonical.js"; + +/** Identical failures after which the exact call is no longer run. */ +export const REPEATED_TOOL_FAILURE_LIMIT = 2; +const MAX_TRACKED_FAILURES = 64; + +type ToolCall = { tool: string; callId: string; arguments: unknown }; +type Failure = { count: number; status: number; message: string; details?: unknown }; + +/** + * Stops a model from resending the same rejected tool call. Some model hosts + * drop optional arguments, so the model can believe it fixed a call that + * arrives unchanged every time. Only failures that are certain to repeat for + * identical arguments count: request validation (400/422) and schema errors. + * Other failures, and any change to the arguments, pass through untouched. + * + * State lives for one run's tool authority, in memory, and is bounded. + */ +export class RepeatedToolFailureGuard { + readonly #failures = new Map(); + // Identical calls a model sends in parallel run one after another, so each + // sees the failures before it. Different calls never wait on each other. + readonly #inFlight = new Map>(); + + run(call: ToolCall, execute: () => Promise): Promise { + const key = callKey(call); + const previous = this.#inFlight.get(key); + const current = previous ? previous.then(() => this.#run(key, call, execute), () => this.#run(key, call, execute)) : this.#run(key, call, execute); + const settled = current.then(() => undefined, () => undefined); + this.#inFlight.set(key, settled); + void settled.then(() => { if (this.#inFlight.get(key) === settled) this.#inFlight.delete(key); }); + return current; + } + + async #run(key: string, call: ToolCall, execute: () => Promise): Promise { + const prior = this.#failures.get(key); + if (prior && prior.count >= REPEATED_TOOL_FAILURE_LIMIT) { + throw new HttpError(prior.status, `This call was not run: ${call.tool} already failed ${prior.count} times in this run with these exact arguments. ` + + `${receivedArguments(call.arguments)} The error each time was: ${prior.message} ` + + "Repeating it unchanged cannot succeed. Send it again only with the change that error asks for, or stop calling it and tell the user what blocked you.", prior.details); + } + try { + const result = await execute(); + this.#failures.delete(key); + return result; + } catch (error) { + const failure = deterministicFailure(error); + if (!failure) throw error; + const latest = this.#failures.get(key); + const count = latest && latest.message === failure.message && latest.status === failure.status ? latest.count + 1 : 1; + this.#remember(key, { ...failure, count }); + if (count < REPEATED_TOOL_FAILURE_LIMIT) throw error; + throw new HttpError(failure.status, `${failure.message} This exact call has now failed ${count} times with the same arguments. ` + + `${receivedArguments(call.arguments)} If you meant to send another argument, it did not arrive; send it as a top-level argument exactly as described above. ` + + "Paperclip will not run this exact call again in this run. Change the arguments, or stop and tell the user what blocked you.", failure.details); + } + } + + #remember(key: string, failure: Failure) { + this.#failures.delete(key); + this.#failures.set(key, failure); + if (this.#failures.size > MAX_TRACKED_FAILURES) this.#failures.delete(this.#failures.keys().next().value!); + } +} + +function callKey(call: ToolCall): string { + return nativeSha256({ tool: call.tool, arguments: call.arguments ?? null }); +} + +function deterministicFailure(error: unknown): Omit | null { + if (error instanceof HttpError && (error.status === 400 || error.status === 422)) { + return { status: error.status, message: error.message, details: error.details }; + } + if (error instanceof Error && error.name === "ZodError") return { status: 400, message: error.message }; + return null; +} + +function receivedArguments(value: unknown): string { + const names = value && typeof value === "object" && !Array.isArray(value) ? Object.keys(value).sort() : []; + return `Arguments received each time: ${names.length ? names.join(", ") : "none"}.`; +}