diff --git a/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts b/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts index aef88f47394..b51d5bb17e1 100644 --- a/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts +++ b/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts @@ -38,10 +38,16 @@ describe("repeated identical failing tool calls", () => { expect(second?.message).toContain("Arguments received each time: operationId, pathParams, query, responseText."); expect(second?.message).toContain("Paperclip will not run this exact call again in this run"); + // Both the correction and the refusal name a call the model can copy. + const example = '{"operationId":"PATCH /api/issues/{id}","pathParams":{"id":"issue-a"},"body":{"title":""}}'; + expect(first?.message).toContain(`for example: ${example}`); + expect(second?.message).toContain(example); + for (const n of [3, 4, 5]) { const blocked = await failure(epoch, call(n)); expect(blocked?.message).toMatch(/^This call was not run: call_api already failed 2 times/); expect(blocked?.message).toContain(first!.message); + expect(blocked?.message).toContain(example); } expect(authority.execute).toHaveBeenCalledTimes(2); expect(fetcher).not.toHaveBeenCalled(); @@ -92,4 +98,88 @@ describe("repeated identical failing tool calls", () => { release(); await expect(slow).resolves.toBe("slow"); }); + + // A route's own validation rejection comes back as data, not as a thrown error. + const rejected = (status: number | null, data: unknown = { error: "Validation error", details: [{ path: ["status"], message: "Invalid enum value" }] }) => + ({ ok: false, status, apiOperationId: "PATCH /api/issues/{id}", contentType: "application/json", data }); + const invalid = { operationId: "PATCH /api/issues/{id}", pathParams: { id: "issue-a" }, body: { status: "started" } }; + + it("counts a route's validation rejection returned as ok:false, corrects the second and does not run the third", async () => { + const { authority, epoch } = epochWith(async () => rejected(422)); + const call = (n: number) => ({ tool: "call_api", callId: `r${n}`, arguments: invalid }); + + expect(await epoch.execute(call(1))).toEqual(rejected(422)); + const second = await epoch.execute(call(2)) as Record<string, unknown>; + expect(second).toMatchObject({ ok: false, status: 422, data: rejected(422).data }); + expect(second.guidance).toContain("failed 2 times with the same arguments"); + expect(second.guidance).toContain("Arguments received each time: body, operationId, pathParams."); + expect(second.guidance).toContain("Paperclip will not run this exact call again in this run"); + + for (const n of [3, 4]) { + const blocked = await failure(epoch, call(n)); + expect(blocked).toBeInstanceOf(HttpError); + expect((blocked as HttpError).status).toBe(422); + expect(blocked?.message).toMatch(/^This call was not run: call_api already failed 2 times/); + expect(blocked?.message).toContain("The response each time was HTTP 422: "); + expect(blocked?.message).toContain("Invalid enum value"); + } + expect(authority.execute).toHaveBeenCalledTimes(2); + }); + + it("still runs a changed call after refusing a returned rejection", async () => { + const { authority, epoch } = epochWith(async call => (call.arguments as typeof invalid).body.status === "started" ? rejected(400) : { ok: true, status: 200 }); + for (const n of [1, 2, 3]) await epoch.execute({ tool: "call_api", callId: `r${n}`, arguments: invalid }).catch(() => null); + await expect(epoch.execute({ tool: "call_api", callId: "fixed", arguments: { ...invalid, body: { status: "in_progress" } } })) + .resolves.toEqual({ ok: true, status: 200 }); + expect(authority.execute).toHaveBeenCalledTimes(3); + }); + + it.each([ + ["server error", 500], ["unavailable", 503], ["rate limit", 429], ["timeout", 408], ["conflict", 409], ["not found", 404], ["no response", null], + ] as const)("keeps retrying a returned %s (%s), which may succeed unchanged", async (_label, status) => { + const { authority, epoch } = epochWith(async () => rejected(status, { error: "try again" })); + for (const n of [1, 2, 3, 4]) { + const result = await epoch.execute({ tool: "call_api", callId: `t${n}`, arguments: invalid }) as Record<string, unknown>; + expect(result).toEqual(rejected(status, { error: "try again" })); + } + expect(authority.execute).toHaveBeenCalledTimes(4); + }); + + it.each([ + ["rate limit", new HttpError(429, "Too many requests")], + ["conflict", new HttpError(409, "Issue was updated by someone else")], + ["timeout", Object.assign(new Error("The operation was aborted due to timeout"), { name: "TimeoutError" })], + ])("keeps retrying a thrown %s, which may succeed unchanged", async (_label, transient) => { + const { authority, epoch } = epochWith(async () => { throw transient; }); + for (const n of [1, 2, 3]) expect(await failure(epoch, { tool: "call_api", callId: `${n}`, arguments: invalid })).toBe(transient); + expect(authority.execute).toHaveBeenCalledTimes(3); + }); + + it("forgets a returned rejection once a different call succeeds, because the state it checked may have changed", async () => { + let unblocked = false; + const { authority, epoch } = epochWith(async call => call.tool === "call_api" + ? (unblocked ? { ok: true, status: 200 } : rejected(422, { error: "Resolve the blockers first" })) + : (unblocked = true, { ok: true })); + const patch = (n: number) => ({ tool: "call_api", callId: `s${n}`, arguments: invalid }); + await epoch.execute(patch(1)); + await epoch.execute(patch(2)); + await expect(epoch.execute({ tool: "update_blockers", callId: "unblock", arguments: { blockedByIssueIds: [] } })).resolves.toEqual({ ok: true }); + await expect(epoch.execute(patch(3))).resolves.toEqual({ ok: true, status: 200 }); + expect(authority.execute).toHaveBeenCalledTimes(4); + }); + + it("keeps counting a returned rejection when unrelated calls in between fail transiently", async () => { + const { authority, epoch } = epochWith(async call => call.tool === "call_api" + ? rejected(422) + : { ok: false, status: call.callId === "u1" ? 503 : call.callId === "u2" ? 429 : null, error: "try again" }); + const patch = (n: number) => ({ tool: "call_api", callId: `i${n}`, arguments: invalid }); + await epoch.execute(patch(1)); + await epoch.execute({ tool: "list_agents", callId: "u1", arguments: {} }); + const second = await epoch.execute(patch(2)) as Record<string, unknown>; + expect(second.guidance).toContain("failed 2 times with the same arguments"); + await epoch.execute({ tool: "list_agents", callId: "u2", arguments: {} }); + await epoch.execute({ tool: "list_agents", callId: "u3", arguments: {} }); + expect((await failure(epoch, patch(3)))?.message).toMatch(/^This call was not run/); + expect(authority.execute.mock.calls.filter(([call]) => call.tool === "call_api")).toHaveLength(2); + }); }); diff --git a/server/src/services/native-runtime/repeated-tool-failure-guard.ts b/server/src/services/native-runtime/repeated-tool-failure-guard.ts index b6b6c4127cd..25c4c08dac3 100644 --- a/server/src/services/native-runtime/repeated-tool-failure-guard.ts +++ b/server/src/services/native-runtime/repeated-tool-failure-guard.ts @@ -4,16 +4,30 @@ import { nativeSha256 } from "./canonical.js"; /** Identical failures after which the exact call is no longer run. */ export const REPEATED_TOOL_FAILURE_LIMIT = 2; const MAX_TRACKED_FAILURES = 64; +const MAX_RESPONSE_EXCERPT = 2_000; type ToolCall = { tool: string; callId: string; arguments: unknown }; -type Failure = { count: number; status: number; message: string; details?: unknown }; +type Failure = { + count: number; + status: number; + message: string; + details?: unknown; + /** Returned by the route as an `ok: false` response rather than refused before sending. */ + returned: boolean; +}; /** * Stops a model from resending the same rejected tool call. Some model hosts * drop optional arguments, so the model can believe it fixed a call that * arrives unchanged every time. Only failures that are certain to repeat for - * identical arguments count: request validation (400/422) and schema errors. - * Other failures, and any change to the arguments, pass through untouched. + * identical arguments count: request validation (400/422) thrown before the + * request is sent, schema errors, and a route's own 400/422 returned as an + * `ok: false` response. Timeouts, rate limits, conflicts, server errors and + * any change to the arguments pass through untouched. + * + * A route can reject a request because of the state it found, so a returned + * rejection is forgotten as soon as a different call succeeds; a refusal made + * before sending depends on the arguments alone and is remembered for the run. * * State lives for one run's tool authority, in memory, and is bounded. */ @@ -37,30 +51,47 @@ export class RepeatedToolFailureGuard { const prior = this.#failures.get(key); if (prior && prior.count >= REPEATED_TOOL_FAILURE_LIMIT) { throw new HttpError(prior.status, `This call was not run: ${call.tool} already failed ${prior.count} times in this run with these exact arguments. ` - + `${receivedArguments(call.arguments)} The error each time was: ${prior.message} ` + + `${receivedArguments(call.arguments)} ${prior.returned ? `The response each time was HTTP ${prior.status}: ` : "The error each time was: "}${prior.message} ` + "Repeating it unchanged cannot succeed. Send it again only with the change that error asks for, or stop calling it and tell the user what blocked you.", prior.details); } + let result: T; try { - const result = await execute(); - this.#failures.delete(key); - return result; + result = await execute(); } catch (error) { - const failure = deterministicFailure(error); + const failure = thrownFailure(error); if (!failure) throw error; - const latest = this.#failures.get(key); - const count = latest && latest.message === failure.message && latest.status === failure.status ? latest.count + 1 : 1; - this.#remember(key, { ...failure, count }); + const count = this.#record(key, failure); if (count < REPEATED_TOOL_FAILURE_LIMIT) throw error; - throw new HttpError(failure.status, `${failure.message} This exact call has now failed ${count} times with the same arguments. ` - + `${receivedArguments(call.arguments)} If you meant to send another argument, it did not arrive; send it as a top-level argument exactly as described above. ` - + "Paperclip will not run this exact call again in this run. Change the arguments, or stop and tell the user what blocked you.", failure.details); + throw new HttpError(failure.status, `${failure.message} ${repeatedNote(count, call)}`, failure.details); } + const failure = returnedFailure(result); + if (!failure) { + // Only a success can have changed the state a route checked. A transient + // failure (5xx, 429, timeout) leaves every remembered rejection in place. + if (!isFailedResponse(result)) { + this.#failures.delete(key); + this.#forgetReturnedFailures(); + } + return result; + } + const count = this.#record(key, failure); + if (count < REPEATED_TOOL_FAILURE_LIMIT) return result; + const response = result as Record<string, unknown>; + const guidance = typeof response.guidance === "string" ? `${response.guidance} ` : ""; + return { ...response, guidance: `${guidance}${repeatedNote(count, call)}` } as T; } - #remember(key: string, failure: Failure) { + #record(key: string, failure: Omit<Failure, "count">): number { + const latest = this.#failures.get(key); + const count = latest && latest.message === failure.message && latest.status === failure.status ? latest.count + 1 : 1; this.#failures.delete(key); - this.#failures.set(key, failure); + this.#failures.set(key, { ...failure, count }); if (this.#failures.size > MAX_TRACKED_FAILURES) this.#failures.delete(this.#failures.keys().next().value!); + return count; + } + + #forgetReturnedFailures() { + for (const [key, failure] of this.#failures) if (failure.returned) this.#failures.delete(key); } } @@ -68,14 +99,38 @@ function callKey(call: ToolCall): string { return nativeSha256({ tool: call.tool, arguments: call.arguments ?? null }); } -function deterministicFailure(error: unknown): Omit<Failure, "count"> | null { - if (error instanceof HttpError && (error.status === 400 || error.status === 422)) { - return { status: error.status, message: error.message, details: error.details }; +function isFailedResponse(result: unknown): boolean { + return Boolean(result && typeof result === "object" && (result as Record<string, unknown>).ok === false); +} + +function isValidationStatus(status: unknown): status is 400 | 422 { + return status === 400 || status === 422; +} + +function thrownFailure(error: unknown): Omit<Failure, "count"> | null { + if (error instanceof HttpError && isValidationStatus(error.status)) { + return { status: error.status, message: error.message, details: error.details, returned: false }; } - if (error instanceof Error && error.name === "ZodError") return { status: 400, message: error.message }; + if (error instanceof Error && error.name === "ZodError") return { status: 400, message: error.message, returned: false }; return null; } +/** A route's validation rejection, returned as data (call_api's `ok: false` with HTTP 400/422). */ +function returnedFailure(result: unknown): Omit<Failure, "count"> | null { + if (!result || typeof result !== "object" || Array.isArray(result)) return null; + const response = result as Record<string, unknown>; + if (response.ok !== false || !isValidationStatus(response.status)) return null; + const reason = response.data ?? response.message ?? response.error ?? null; + const text = typeof reason === "string" ? reason : JSON.stringify(reason); + return { status: response.status, message: text.length > MAX_RESPONSE_EXCERPT ? `${text.slice(0, MAX_RESPONSE_EXCERPT)}…` : text, returned: true }; +} + +function repeatedNote(count: number, call: ToolCall): string { + return `This exact call has now failed ${count} times with the same arguments. ${receivedArguments(call.arguments)} ` + + "If you meant to send another argument, it did not arrive; send it as a top-level argument exactly as described above. " + + "Paperclip will not run this exact call again in this run. Change the arguments, or stop and tell the user what blocked you."; +} + function receivedArguments(value: unknown): string { const names = value && typeof value === "object" && !Array.isArray(value) ? Object.keys(value).sort() : []; return `Arguments received each time: ${names.length ? names.join(", ") : "none"}.`; diff --git a/server/src/services/native-runtime/runner-api-client.ts b/server/src/services/native-runtime/runner-api-client.ts index 65c4b6e4009..869f8e0cec8 100644 --- a/server/src/services/native-runtime/runner-api-client.ts +++ b/server/src/services/native-runtime/runner-api-client.ts @@ -77,7 +77,8 @@ export function validateRunnerApiCall(value: unknown, context: RunnerApiContext) if (input.body !== undefined && !input.files?.length) { const requestSchema = operation.requestBody?.content?.[contentType]?.schema as { type?: string } | undefined; if (contentType.includes("json") && requestSchema?.type === "object" && (!input.body || typeof input.body !== "object" || Array.isArray(input.body))) { - throw badRequest("This operation requires body to be a JSON object. Pass the object directly, not a JSON-encoded string."); + throw badRequest("This operation requires body to be a JSON object. Pass the object directly, not a JSON-encoded string. " + + `For example: ${runnerApiCallExample(operation, input.pathParams)}`); } if (contentType.includes("json") && requestSchema?.type === "array" && !Array.isArray(input.body)) { throw badRequest("This operation requires body to be a JSON array. Pass the array directly, not a JSON-encoded string."); @@ -114,10 +115,70 @@ function assertBodyPresent(operation: RunnerApiOperation, input: RunnerApiCall, const received = [...receivedKeys].sort(); const misplaced = received.filter(key => ["query", "responseText", "saveResponse"].includes(key)); throw badRequest(`This request was not sent: ${operation.operationId} requires body, a JSON object with the request fields${requestFieldHint(operation)}. ` - + `Received arguments: ${received.join(", ")}. Resend with a top-level body argument: {"operationId": ${JSON.stringify(operation.operationId)}, "pathParams": {...}, "body": {...}}.` + + `Received arguments: ${received.join(", ")}. Resend with a top-level body argument holding the fields you mean to change, for example: ${runnerApiCallExample(operation, input.pathParams)}` + (misplaced.length ? ` ${misplaced.join(", ")} never carr${misplaced.length === 1 ? "ies" : "y"} request fields.` : "")); } +/** + * A call a model can copy: the path parameters it already sent and a body that + * satisfies the operation's request schema. Free-text values and ids it must + * look up appear as <field> placeholders. + */ +export function runnerApiCallExample(operation: RunnerApiOperation, pathParams?: Readonly<Record<string, string>>): string { + const params = Object.fromEntries([...operation.path.matchAll(/\{([^}]+)\}/g)].map(([, name]) => name) + .filter(name => name !== "companyId" || pathParams?.companyId).map(name => [name, pathParams?.[name] || `<${name}>`])); + const schema = operation.requestBody?.content?.["application/json"]?.schema as JsonSchema | undefined; + return JSON.stringify({ operationId: operation.operationId, ...(Object.keys(params).length ? { pathParams: params } : {}), body: exampleBody(schema) }); +} + +type JsonSchema = { type?: string; enum?: unknown[]; format?: string; pattern?: string; minLength?: number; maxLength?: number; + minimum?: number; exclusiveMinimum?: number | boolean; minItems?: number; items?: JsonSchema; properties?: Record<string, JsonSchema>; required?: string[]; + oneOf?: JsonSchema[]; anyOf?: JsonSchema[]; allOf?: JsonSchema[] }; + +function exampleBody(schema: JsonSchema | undefined): unknown { + if (!schema) return {}; + const variant = schema.oneOf?.[0] ?? schema.anyOf?.[0]; + if (variant && !schema.type && !schema.properties) return exampleBody(variant); + if (schema.type && schema.type !== "object") return exampleValue("body", schema, 0); + const properties = schema.properties ?? {}; + const required = (schema.required ?? []).filter(name => properties[name]); + // With no required field, show one field that takes free text: any value of + // its type is valid, and the model replaces it with what it meant to send. + const names = required.length ? required + : [Object.keys(properties).find(name => isFreeText(properties[name]!)) ?? Object.keys(properties)[0]].filter(Boolean) as string[]; + return Object.fromEntries(names.map(name => [name, exampleValue(name, properties[name]!, 0)])); +} + +function isFreeText(schema: JsonSchema): boolean { + return schema.type === "string" && !schema.enum && !schema.format && !schema.pattern && (schema.maxLength ?? Infinity) >= 3; +} + +function exampleValue(name: string, schema: JsonSchema, depth: number): unknown { + const variant = schema.oneOf?.[0] ?? schema.anyOf?.[0] ?? schema.allOf?.[0]; + if (variant && !schema.type) return exampleValue(name, variant, depth); + if (schema.enum?.length) return schema.enum[0]; + switch (schema.type) { + case "string": { + if (schema.format === "date-time") return "2026-01-01T00:00:00.000Z"; + if (schema.format === "date") return "2026-01-01"; + const text = `<${name}>`; + return schema.maxLength !== undefined && text.length > schema.maxLength ? "x".repeat(Math.max(schema.minLength ?? 1, 1)) : text; + } + case "integer": case "number": { + if (typeof schema.exclusiveMinimum === "number") return schema.exclusiveMinimum + 1; + return schema.exclusiveMinimum === true ? (schema.minimum ?? 0) + 1 : schema.minimum ?? 0; + } + case "boolean": return true; + case "array": return schema.minItems && schema.items && depth < 4 ? [exampleValue(name, schema.items, depth + 1)] : []; + case "object": { + if (depth >= 4) return {}; + const properties = schema.properties ?? {}; + return Object.fromEntries((schema.required ?? []).filter(key => properties[key]).map(key => [key, exampleValue(key, properties[key]!, depth + 1)])); + } + default: return `<${name}>`; + } +} + function requestFieldHint(operation: RunnerApiOperation): string { const schema = operation.requestBody?.content?.["application/json"]?.schema as { properties?: Record<string, unknown> } | undefined; const fields = Object.keys(schema?.properties ?? {}); diff --git a/server/src/services/native-runtime/runner-api.test.ts b/server/src/services/native-runtime/runner-api.test.ts index 6e83f4a2c66..7566836b1b1 100644 --- a/server/src/services/native-runtime/runner-api.test.ts +++ b/server/src/services/native-runtime/runner-api.test.ts @@ -4,9 +4,13 @@ import { runnerApiOperation, searchRunnerApi, } from "./runner-api-catalog.js"; +import Ajv from "ajv"; +import { updateIssueSchema } from "@paperclipai/shared"; import { + assertRunnerApiDispatchable, executeRunnerApi, readBoundedResponse, + runnerApiCallExample, RUNNER_API_CALL_HEADER, runnerApiUrl, validateRunnerApiCall, @@ -875,6 +879,34 @@ describe("runner API request boundary", () => { expect(error?.message).toContain(`Received arguments: ${Object.keys(call).sort().join(", ")}.`); expect(error?.message).toContain('Resend with a top-level body argument'); expect(error?.message).toMatch(/saveResponse never carr(y|ies) request fields/); + // The example in the correction is a call that is sent when copied as is. + const example = exampleIn(error!.message); + expect(example.pathParams).toEqual(call.pathParams); + const sent = vi.fn<typeof fetch>(async () => Response.json({ ok: true })); + await expect(executeRunnerApi(example as RunnerApiCall, context, io(sent))).resolves.toMatchObject({ ok: true }); + expect(sent).toHaveBeenCalledTimes(1); + }); + it("gives a copyable issue update that the issue route accepts", () => { + const example = JSON.parse(runnerApiCallExample(runnerApiOperation("PATCH /api/issues/{id}"), { id: "issue-a" })); + expect(example).toEqual({ operationId: "PATCH /api/issues/{id}", pathParams: { id: "issue-a" }, body: { title: "<title>" } }); + expect(updateIssueSchema.safeParse(example.body).success).toBe(true); + }); + it("gives every write that requires a body an example its request schema accepts", () => { + const AjvCtor = ((Ajv as unknown as { default?: typeof Ajv }).default ?? Ajv) as unknown as new (options: object) => { validate(schema: unknown, data: unknown): boolean; errorsText(): string }; + const ajv = new AjvCtor({ strict: false, validateFormats: false, allErrors: true }); + const rejected: string[] = []; + let checked = 0; + for (const operation of runnerApiCatalog()) { + const schema = operation.requestBody?.content?.["application/json"]?.schema; + if (!operation.requestBody?.required || !schema || operation.transport !== "rest") continue; + checked++; + const example = JSON.parse(runnerApiCallExample(operation, {})); + const { input } = validateRunnerApiCall(example, { ...context, workMode: operation.allowedModes[0] ?? "standard" }); + assertRunnerApiDispatchable(operation, input, example); + if (!ajv.validate(jsonSchemaFromOpenApi(schema), example.body)) rejected.push(`${operation.operationId}: ${ajv.errorsText()}`); + } + expect(checked).toBeGreaterThan(200); + expect(rejected).toEqual([]); }); it("still sends writes whose body is present or optional", async () => { const request = vi.fn<typeof fetch>(async () => Response.json({ ok: true })); @@ -922,3 +954,32 @@ describe("runner API request boundary", () => { ).rejects.toThrow("transfer limit"); }); }); + +/** The JSON call that follows "for example: " in a correction. */ +function exampleIn(message: string): Record<string, unknown> { + const start = message.indexOf("{", message.indexOf("for example: ")); + let depth = 0; + for (let index = start; index < message.length; index++) { + if (message[index] === "{") depth++; + if (message[index] === "}" && --depth === 0) return JSON.parse(message.slice(start, index + 1)) as Record<string, unknown>; + } + throw new Error("no example in message"); +} + +/** OpenAPI 3.0 schema keywords as JSON Schema, so Ajv can check them. */ +function jsonSchemaFromOpenApi(value: unknown): unknown { + if (Array.isArray(value)) return value.map(jsonSchemaFromOpenApi); + if (!value || typeof value !== "object") return value; + // Values the model must supply itself (ids, secrets, codes, evidence) appear as + // placeholders, so their text constraints are not checked; structure is. + // The schemas come from Zod, whose unions take the first match: anyOf, not oneOf. + const { nullable, exclusiveMinimum, exclusiveMaximum, minimum, maximum, pattern: _pattern, minLength: _minLength, oneOf, ...rest } = value as Record<string, unknown>; + const schema: Record<string, unknown> = Object.fromEntries(Object.entries(rest).map(([key, entry]) => [key, jsonSchemaFromOpenApi(entry)])); + if (oneOf) schema.anyOf = jsonSchemaFromOpenApi(oneOf); + if (exclusiveMinimum === true) schema.exclusiveMinimum = minimum; else if (minimum !== undefined) schema.minimum = minimum; + if (typeof exclusiveMinimum === "number") schema.exclusiveMinimum = exclusiveMinimum; + if (exclusiveMaximum === true) schema.exclusiveMaximum = maximum; else if (maximum !== undefined) schema.maximum = maximum; + if (typeof exclusiveMaximum === "number") schema.exclusiveMaximum = exclusiveMaximum; + if (nullable === true && typeof schema.type === "string") schema.type = [schema.type, "null"]; + return schema; +}