diff --git a/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.test.ts b/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.test.ts index 2c0344cd19b..c1f2f98a8e2 100644 --- a/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.test.ts +++ b/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.test.ts @@ -115,6 +115,27 @@ it("forwards closed tool delivery records after binding and drops any record tha message: JSON.stringify({ ...delivery, socketRequestIndex: 2, socketIdleMs: 5270 }) } }]); }); +it("forwards delivery records with argument names and drops names that could carry values", () => { + const frames: { params: { message: string } }[] = []; + const forward = createOpenCodeFinishDiagnosticForwarder(frame => frames.push(frame as never)); + forward.bind("session"); + const delivery = { schema: "paperclip.opencode.tool-delivery-diagnostic/v1", callIdSha256: "e".repeat(64), + socketRequestIndex: 1, socketIdleMs: null, handlerMs: 4, outcome: "result", written: true, + argumentNames: ["operationId", "pathParams", "query", "responseText"], argumentNamesTruncated: false, undeclaredArgumentCount: 0 }; + for (const bad of [ + { ...delivery, argumentNames: ["body value"] }, + { ...delivery, argumentNames: "operationId" }, + { ...delivery, argumentNames: Array.from({ length: 20 }, (_, n) => `${n}${"a".repeat(60)}`) }, + { ...delivery, argumentNamesTruncated: "no" }, + { ...delivery, undeclaredArgumentCount: -1 }, + { ...delivery, undeclaredArgumentCount: "1" }, + ]) forward.diagnostic(JSON.stringify(bad)); + expect(frames).toEqual([]); + forward.diagnostic(JSON.stringify(delivery)); + forward.diagnostic(JSON.stringify({ ...delivery, argumentNames: null })); + expect(frames.map(frame => JSON.parse(frame.params.message).argumentNames)).toEqual([delivery.argumentNames, null]); +}); + it("forwards closed input-coercion records and drops any that could carry values", () => { const frames: { params: { message: string } }[] = []; const forward = createOpenCodeFinishDiagnosticForwarder(frame => frames.push(frame as never)); diff --git a/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts b/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts index a718a856dd4..3e833bbafdb 100644 --- a/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts +++ b/packages/paperclip-runner/src/cli/opencode-proxy-finish-diagnostics.ts @@ -52,11 +52,15 @@ export function createOpenCodeFinishDiagnosticForwarder(send: (frame: Warning) = } const TOOL_DELIVERY_KEYS = "callIdSha256,handlerMs,outcome,schema,socketIdleMs,socketRequestIndex,written"; +const TOOL_DELIVERY_KEYS_WITH_ARGUMENTS = "argumentNames,argumentNamesTruncated,callIdSha256,handlerMs,outcome,schema,socketIdleMs,socketRequestIndex,undeclaredArgumentCount,written"; +const validArgumentNames = (names: unknown) => names === null || (Array.isArray(names) && JSON.stringify(names).length <= 1_024 + && names.every(name => typeof name === "string" && /^[A-Za-z0-9_.-]{1,64}$/.test(name))); // Delivery records carry only hashes, counters and timings. Reject anything // else so this channel cannot forward tool inputs or results. function validToolDelivery(value: Record): boolean { const count = (entry: unknown) => typeof entry === "number" && Number.isSafeInteger(entry) && entry >= 0; - return Object.keys(value).sort().join(",") === TOOL_DELIVERY_KEYS + const keys = Object.keys(value).sort().join(","); + return (keys === TOOL_DELIVERY_KEYS || (keys === TOOL_DELIVERY_KEYS_WITH_ARGUMENTS && validArgumentNames(value.argumentNames) && typeof value.argumentNamesTruncated === "boolean" && count(value.undeclaredArgumentCount))) && typeof value.callIdSha256 === "string" && /^[a-f0-9]{64}$/.test(value.callIdSha256) && count(value.socketRequestIndex) && count(value.handlerMs) && (value.socketIdleMs === null || count(value.socketIdleMs)) diff --git a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge-delivery.test.ts b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge-delivery.test.ts index d2543e2ce9e..3ca1f3a4ff3 100644 --- a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge-delivery.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge-delivery.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { Agent, request } from "node:http"; import { afterEach, expect, it } from "vitest"; import { startOpenCodeMcpBridge, type OpenCodeMcpBridge } from "./mcp-bridge.js"; +import { createOpenCodeFinishDiagnosticForwarder } from "../../cli/opencode-proxy-finish-diagnostics.js"; const bridges: OpenCodeMcpBridge[] = []; const agents: Agent[] = []; @@ -88,6 +89,9 @@ it("answers each tool call, including a parallel pair on a pooled keep-alive cli socketRequestIndex: expect.any(Number), socketIdleMs: delivery.socketRequestIndex === 1 ? null : expect.any(Number), handlerMs: expect.any(Number), + argumentNames: [], + argumentNamesTruncated: false, + undeclaredArgumentCount: 0, outcome: "result", written: true, }); @@ -140,3 +144,56 @@ it("records a rejected call as an answered error", async () => { callIdSha256: createHash("sha256").update("11").digest("hex"), outcome: "error", written: true, }); }); + +it("records the declared argument names a call delivered, never their values or undeclared names", async () => { + const messages: string[] = []; + const bridge = await startOpenCodeMcpBridge({ + tools: [{ name: "call_api", inputSchema: { type: "object", properties: { + operationId: { type: "string" }, pathParams: { type: "object" }, query: { type: "object" }, body: {}, responseText: { type: "object" }, + } } }], + handler: async () => ({ ok: false, status: 400 }), + onDiagnostic: message => { if (message.includes("tool-delivery-diagnostic")) messages.push(message); }, + }); + bridges.push(bridge); + await (await fetch(bridge.url, { + method: "POST", + headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 12, method: "tools/call", params: { name: "call_api", arguments: { + operationId: "PATCH /api/issues/{id}", pathParams: { id: "secret-issue" }, query: {}, responseText: { limitBytes: 8192 }, PRIVATE_FIELD: 1, + } } }), + })).json(); + await expect.poll(() => messages.length).toBe(1); + expect(JSON.parse(messages[0]!)).toMatchObject({ argumentNames: ["operationId", "pathParams", "query", "responseText"], + argumentNamesTruncated: false, undeclaredArgumentCount: 1, outcome: "result" }); + expect(messages[0]).not.toMatch(/PATCH|secret-issue|8192|PRIVATE_FIELD/); +}); + +it("bounds many long argument names so the forwarder keeps the whole delivery record", async () => { + const names = Array.from({ length: 32 }, (_, n) => `${String(n).padStart(2, "0")}${"a".repeat(62)}`); + const messages: string[] = []; + const bridge = await startOpenCodeMcpBridge({ + tools: [{ name: "wide_tool", inputSchema: { type: "object", properties: Object.fromEntries(names.map(name => [name, {}])) } }], + handler: async () => ({ ok: true }), + onDiagnostic: message => { if (message.includes("tool-delivery-diagnostic")) messages.push(message); }, + }); + bridges.push(bridge); + await (await fetch(bridge.url, { + method: "POST", + headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 13, method: "tools/call", params: { name: "wide_tool", + arguments: Object.fromEntries(names.map(name => [name, 1])) } }), + })).json(); + await expect.poll(() => messages.length).toBe(1); + const record = JSON.parse(messages[0]!); + expect(record.argumentNamesTruncated).toBe(true); + expect(record.argumentNames.length).toBeGreaterThan(0); + expect(record.argumentNames.length).toBeLessThan(names.length); + expect(record.argumentNames).toEqual(names.slice(0, record.argumentNames.length)); + expect(JSON.stringify(record.argumentNames).length).toBeLessThanOrEqual(1_024); + expect(messages[0]!.length).toBeLessThanOrEqual(2_048); + const frames: unknown[] = []; + const forward = createOpenCodeFinishDiagnosticForwarder(frame => frames.push(frame)); + forward.bind("session"); + forward.diagnostic(messages[0]!); + expect(frames).toHaveLength(1); +}); diff --git a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts index 5e947b226bc..1f7efdca6d4 100644 --- a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts +++ b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts @@ -65,6 +65,8 @@ const MAX_RESULT_BYTES = 1_048_576; const TOOL_DELIVERY_DIAGNOSTIC_SCHEMA = "paperclip.opencode.tool-delivery-diagnostic/v1"; const MAX_TOOL_DELIVERY_DIAGNOSTICS = 256; const MAX_TOOL_INPUT_COERCION_DIAGNOSTICS = 64; +const MAX_TOOL_DELIVERY_ARGUMENT_NAMES_CHARS = 1_024; +const TOOL_DELIVERY_ARGUMENT_NAME = /^[A-Za-z0-9_.-]{1,64}$/; type SocketUse = { requests: number; lastResponseAt: number | null }; type ToolDelivery = { @@ -73,6 +75,7 @@ type ToolDelivery = { receivedAt: number; }; type ToolDeliveryOutcome = "result" | "error" | "unanswered"; +type ToolDeliveryInput = { argumentNames: string[] | null; argumentNamesTruncated: boolean; undeclaredArgumentCount: number }; export function canonicalOpenCodeMcpToolName(name: string): string { const trimmed = name.trim(); @@ -131,11 +134,14 @@ export async function startOpenCodeMcpBridge( delivery: ToolDelivery, outcome: ToolDeliveryOutcome, written: boolean, + input?: ToolDeliveryInput, ) => { if (!options.onDiagnostic || deliveryDiagnostics++ >= MAX_TOOL_DELIVERY_DIAGNOSTICS) return; const diagnostic = { schema: TOOL_DELIVERY_DIAGNOSTIC_SCHEMA, callIdSha256: createHash("sha256").update(callId).digest("hex"), + ...(input ? { argumentNames: input.argumentNames, argumentNamesTruncated: input.argumentNamesTruncated, + undeclaredArgumentCount: input.undeclaredArgumentCount } : {}), socketRequestIndex: delivery.socketRequestIndex, socketIdleMs: delivery.socketIdleMs, handlerMs: Date.now() - delivery.receivedAt, @@ -200,6 +206,18 @@ export async function startOpenCodeMcpBridge( }; } +function boundedArgumentNames(names: string[] | null): Pick { + if (!names) return { argumentNames: null, argumentNamesTruncated: false }; + const kept: string[] = []; + let size = 2; + for (const name of [...names].sort()) { + size += JSON.stringify(name).length + (kept.length ? 1 : 0); + if (size > MAX_TOOL_DELIVERY_ARGUMENT_NAMES_CHARS) return { argumentNames: kept, argumentNamesTruncated: true }; + kept.push(name); + } + return { argumentNames: kept, argumentNamesTruncated: false }; +} + function inputShape(input: unknown): "omitted" | "null" | "array" | "object" | "string" | "number" | "boolean" | "other" { if (input === undefined) return "omitted"; if (input === null) return "null"; @@ -250,7 +268,7 @@ async function handleRequest( response: ServerResponse, context: { delivery: ToolDelivery; - diagnoseToolDelivery: (callId: string, delivery: ToolDelivery, outcome: ToolDeliveryOutcome, written: boolean) => void; + diagnoseToolDelivery: (callId: string, delivery: ToolDelivery, outcome: ToolDeliveryOutcome, written: boolean, input?: ToolDeliveryInput) => void; secret: string; tools: OpenCodeMcpToolDefinition[]; admittedTools: OpenCodeMcpToolDefinition[]; @@ -345,8 +363,9 @@ async function handleRequest( // client that never consumes a written answer can be told apart from a // bridge that never answered it. let outcome: ToolDeliveryOutcome = "unanswered"; + let deliveredInput: ToolDeliveryInput | undefined; response.once("close", () => { - context.diagnoseToolDelivery(callId, context.delivery, outcome, response.writableFinished); + context.diagnoseToolDelivery(callId, context.delivery, outcome, response.writableFinished, deliveredInput); }); const rawName = typeof params.name === "string" ? params.name : ""; // A declared operation may itself start with paperclip_. Preserve that @@ -361,6 +380,14 @@ async function handleRequest( return; } const received = params.arguments ?? {}; + const declaredSchema = context.admittedTools.find(entry => entry.name === tool)?.inputSchema.properties; + const declaredArguments = isRecord(declaredSchema) ? declaredSchema : {}; + const receivedNames = isRecord(received) ? Object.keys(received) : null; + const declaredNames = receivedNames?.filter(name => Object.hasOwn(declaredArguments, name) && TOOL_DELIVERY_ARGUMENT_NAME.test(name)) ?? null; + deliveredInput = { + ...boundedArgumentNames(declaredNames), + undeclaredArgumentCount: (receivedNames?.length ?? 0) - (declaredNames?.length ?? 0), + }; const validator = context.validators.get(tool); const admitted = admitToolInput(validator, received); const { args, schemaValid } = admitted; diff --git a/server/src/__tests__/native-status-arbiter-corpus.test.ts b/server/src/__tests__/native-status-arbiter-corpus.test.ts index a3309bed6a5..f0701e88be0 100644 --- a/server/src/__tests__/native-status-arbiter-corpus.test.ts +++ b/server/src/__tests__/native-status-arbiter-corpus.test.ts @@ -3874,11 +3874,14 @@ describe("P6-31 Section 18.13 executable status-authority corpus", () => { ]); }); - it("binds a tools-refresh continuation to its existing durable wake without a duplicate", async () => { + it.each([ + ["tools-refresh", (runId: string) => `connection-intent:tools:${runId}:fixture-digest`], + ["tool-call-loop", (runId: string) => `agent_tool_call_loop:${runId}`], + ] as const)("binds a %s continuation to its existing durable wake without a duplicate", async (name, keyFor) => { const template = corpus.fixtures.find((candidate) => candidate.mode === "native")!; - const fixture = { ...template, id: "in-feed-tools-refresh", given: { ...template.given, priorIssueStatus: "in_progress" } }; + const fixture = { ...template, id: `in-feed-${name}`, given: { ...template.given, priorIssueStatus: "in_progress" } }; const seeded = await seedFixture(fixture); - const key = `connection-intent:tools:${seeded.runId}:fixture-digest`; + const key = keyFor(seeded.runId); const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "assignment", status: "queued", idempotencyKey: key, payload: { issueId: seeded.issueId, mutation: "connection_tools_refreshed" } }).returning(); const decision: NativeStatusDecision = { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, statusAction: "in_progress", toStatus: "in_progress", diff --git a/server/src/__tests__/native-tool-call-loop-continuation.test.ts b/server/src/__tests__/native-tool-call-loop-continuation.test.ts new file mode 100644 index 00000000000..34d7a706343 --- /dev/null +++ b/server/src/__tests__/native-tool-call-loop-continuation.test.ts @@ -0,0 +1,126 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + agents, + agentWakeupRequests, + authUsers, + companies, + companyMemberships, + createDb, + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + heartbeatRuns, + issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; +import { heartbeatService } from "../services/heartbeat.ts"; +import { type AdapterExecutionContext, type AdapterExecutionResult } from "../adapters/index.ts"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { ToolCallLoopError, toolCallLoopContinuationKey } from "../services/native-runtime/repeated-tool-failure-guard.js"; + +const mockAdapterExecute = vi.hoisted(() => + vi.fn(async (_context: AdapterExecutionContext): Promise => ({ + exitCode: 0, signal: null, timedOut: false, errorMessage: null, + summary: "Tool-call loop continuation test run.", provider: "test", model: "test-model", + })), +); + +vi.mock("../adapters/index.ts", async () => { + const actual = await vi.importActual("../adapters/index.ts"); + return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute: mockAdapterExecute })) }; +}); + +const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +if (!embeddedPostgresSupport.supported) { + console.warn( + `Skipping tool-call loop continuation tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + ); +} + +describeEmbeddedPostgres("the continuation of a turn stopped by a tool-call loop", () => { + let db!: ReturnType; + let heartbeat!: ReturnType; + let tempDb: Awaited> | null = null; + + beforeAll(async () => { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-tool-call-loop-continuation-"); + db = createDb(tempDb.connectionString); + heartbeat = heartbeatService(db, { runtimeEnv: { ...process.env, PAPERCLIP_IN_WORKTREE: "false" } }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterAll(async () => { + await drainHeartbeatRunsToQuiescence(db, heartbeat); + await db?.$client.end(); + await tempDb?.cleanup(); + }); + + it("keeps its own keyed wake when a comment wake is already deferred behind the looping run", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + await db.insert(companies).values({ + id: companyId, name: "Loop Co", issuePrefix: `L${companyId.replace(/-/g, "").slice(0, 5).toUpperCase()}`, + defaultResponsibleUserId: "responsible-user", requireBoardApprovalForNewAgents: false, + }); + await db.insert(authUsers).values({ + id: "responsible-user", name: "Fixture user", email: "responsible-user@example.test", + createdAt: new Date(), updatedAt: new Date(), + }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ + companyId, principalType: "user", principalId: "responsible-user", status: "active", membershipRole: "member", + }); + await db.insert(agents).values({ + id: agentId, companyId, name: "Looper", role: "engineer", status: "idle", adapterType: "codex_local", + adapterConfig: {}, runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } }, permissions: {}, + }); + await db.insert(issues).values({ + id: issueId, companyId, title: "Describe the subtasks", status: "todo", priority: "medium", + assigneeAgentId: agentId, responsibleUserId: "responsible-user", + }); + + let finishLoopingRun!: () => void; + const loopingRunFinished = new Promise((resolve) => { finishLoopingRun = resolve; }); + mockAdapterExecute.mockImplementationOnce(async () => { + await loopingRunFinished; + return { exitCode: 0, signal: null, timedOut: false, errorMessage: null, + summary: "Stopped after a tool-call loop.", provider: "test", model: "test-model" }; + }); + + const loopingRun = await heartbeat.wakeup(agentId, { + source: "assignment", triggerDetail: "system", reason: "issue_assigned", + requestedByActorType: "system", payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" }, + }); + expect(loopingRun).not.toBeNull(); + await vi.waitFor(() => expect(mockAdapterExecute).toHaveBeenCalledTimes(1), { timeout: 30_000 }); + + const [commentWake] = await db.insert(agentWakeupRequests).values({ + companyId, agentId, source: "automation", triggerDetail: "system", reason: "issue_commented", + status: "deferred_issue_execution", requestedByActorType: "user", requestedByActorId: "responsible-user", + payload: { issueId, _paperclipWakeContext: { issueId, wakeReason: "issue_commented" } }, + }).returning(); + + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId: loopingRun!.id, workMode: "standard" }); + await authority.scheduleToolCallLoopContinuation(new ToolCallLoopError("call_api", 6, 6, ["operationId", "pathParams"], 400, "refused")); + await heartbeat.dispatchPendingNativeStatusWakeups({ companyId }); + await heartbeat.dispatchPendingNativeStatusWakeups({ companyId }); + + const key = toolCallLoopContinuationKey(loopingRun!.id); + const keyed = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.idempotencyKey, key), + )); + expect(keyed.some((wake) => wake.status === "deferred_issue_execution" && wake.id !== commentWake!.id)).toBe(true); + expect(keyed.every((wake) => !["skipped", "failed", "cancelled"].includes(wake.status))).toBe(true); + const [comment] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, commentWake!.id)); + expect(comment).toMatchObject({ status: "deferred_issue_execution", coalescedCount: 0, idempotencyKey: null }); + + finishLoopingRun(); + await vi.waitFor(async () => { + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + expect(runs.some((run) => run.id !== loopingRun!.id && run.contextSnapshot?.livenessContinuationReason === "agent_tool_call_loop" + && run.contextSnapshot?.forceFreshSession === true)).toBe(true); + }, { timeout: 30_000 }); + }, 90_000); +}); diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 70882ed0a8a..d9c07e2ab65 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -36,6 +36,7 @@ import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAIN import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js"; import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-review-participant.js"; +import { toolCallLoopContinuationKey } from "./repeated-tool-failure-guard.js"; import { conversationNativeDecision, isConversation, settleConversationTurnForRun } from "../agent-conversations.js"; import { issueTreeControlService } from "../issue-tree-control.js"; import { COMPLETION_REVIEW_STAGE_REASON, completionReviewStageEntry } from "../issue-execution-policy.js"; @@ -1765,6 +1766,8 @@ export async function finalizeNativeRun(input: { // into a blocker; the committer rechecks that under the issue lock. failedTurnContinuationUsed: terminalState === "failed" && ownsProviderFailureDecision && !reviewContext && await isFailedTurnContinuationRun(input.db, run, authoritativeIssue.id), + toolCallLoopContinuation: assessment.reportedDisposition === "yielded" + && assessment.continuation?.idempotencyKey === toolCallLoopContinuationKey(run.id), ...(reviewContext ? { nativeReviewOutcome: nativeReview ? nativeReview.interaction.status === "pending" ? "pending" as const : "resolved" as const : "stale" as const } : {}), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c28c1295484..8e463045596 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -21,6 +21,7 @@ import { trace } from "@opentelemetry/api"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { configuredEnvironment, configuredEnvironmentProjection } from "../../vendor/paperclip-runner/index.js"; import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from "vitest"; +import { TOOL_CALL_LOOP_REFUSAL_LIMIT, ToolCallLoopError, toolCallLoopContinuationKey } from "./repeated-tool-failure-guard.js"; import { enableReattachPhase, isReattachPhaseActive, joinReattachPhase, onReattachPhaseEnd, resetReattachPhaseForTests, type ReattachPhaseSummary } from "../heartbeat/run-admission.js"; import { createSystemRecovery, failSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure } from "../system-recovery-state.js"; import { registerRunStartHandover } from "../run-start-handover.js"; @@ -81,7 +82,9 @@ import { type NativeExecutionInputV3, type NativeExecutionInput, type PrpEvent, + type PrpStructuredRunResult, } from "@paperclipai/paperclip-runner"; +import { HttpError } from "../../errors.js"; import { existsSync } from "node:fs"; import { createHash, generateKeyPairSync, randomUUID } from "node:crypto"; import { createServer as createNetServer } from "node:net"; @@ -468,6 +471,8 @@ import { nativeGovernedWaitResult, nativeConversationReplyResult, nativeToolsRefreshWaitResult, + nativeToolCallLoopWaitResult, + SessionAuthorityEpoch, parseRemoteExecutableCandidate, buildRemoteCodexLauncherCommand, mayUsePreinstalledRunnerArtifact, @@ -7912,6 +7917,66 @@ describe("native conversation replies", () => { }); describe("native governed waits", () => { + it("a turn whose model repeats a refused call N times ends with agent_tool_call_loop and schedules a continuation", async () => { + const wakes: Array<{ id: string; key: string; loop: ToolCallLoopError }> = []; + const authority = { + execute: vi.fn(async () => { throw new HttpError(400, "body is required"); }), + scheduleToolCallLoopContinuation: vi.fn(async (loop: ToolCallLoopError) => { + if (!wakes.length) wakes.push({ id: "wake-loop", key: toolCallLoopContinuationKey("run-1"), loop }); + return true; + }), + close: () => {}, + }; + const epoch = new SessionAuthorityEpoch("run-1", authority as never); + const contract = { revision: "4", objective: "Describe the subtasks", criteria: [{ id: "describe", requirement: "Describe them" }] }; + const observation = createGovernedWaitEventObservation(async () => wakes[0] + ? nativeToolCallLoopWaitResult({ wakeId: wakes[0].id, key: wakes[0].key, completionContract: contract }) : null); + const args = { operationId: "PATCH /api/issues/{id}", pathParams: { id: "issue-a" }, query: {}, responseText: { limitBytes: 8192 } }; + const toolEvent = (n: number): PrpEvent => ({ + schema: "paperclip.prp.event.v1", sourceInstanceId: "runner-1", sourceEventId: `runner-1:item:${n}`, sourceSeq: n, + sourceKind: "runner", runId: "run-1", normalizedSessionId: "session-1", turnId: "turn-1", itemId: `call-${n}`, + eventType: "item.completed", schemaVersion: 1, priority: 0, emittedAt: "2026-11-15T05:00:00.000Z", + payload: { kind: "dynamicToolCall", item: { type: "tool_result", is_error: true } }, + }); + + let parked: PrpStructuredRunResult | null = null; + let calls = 0; + let lastError: unknown; + while (!parked && calls < 2 + TOOL_CALL_LOOP_REFUSAL_LIMIT + 5) { + calls += 1; + lastError = await epoch.execute({ tool: "call_api", callId: `call-${calls}`, arguments: args }).then(() => null, (error: unknown) => error); + await observation.observe(toolEvent(calls), true); + parked = observation.consume(toolEvent(calls)); + } + + expect(calls).toBe(2 + TOOL_CALL_LOOP_REFUSAL_LIMIT); + expect(authority.execute).toHaveBeenCalledTimes(2); + expect(lastError).toBeInstanceOf(ToolCallLoopError); + expect(lastError).toMatchObject({ code: "agent_tool_call_loop", refusals: TOOL_CALL_LOOP_REFUSAL_LIMIT }); + expect(authority.scheduleToolCallLoopContinuation).toHaveBeenCalledTimes(1); + expect(parked).toMatchObject({ + reportedWorkDisposition: "yielded", + completionClaim: { objectiveSatisfied: false }, + evidence: [{ ref: "wakeup:wake-loop" }], + continuation: { kind: "same_agent", idempotencyKey: "agent_tool_call_loop:run-1", summary: "Continue the task in a fresh session." }, + }); + }); + + it("keeps the refused call's error when the loop continuation cannot be queued", async () => { + const authority = { + execute: vi.fn(async () => { throw new HttpError(400, "body is required"); }), + scheduleToolCallLoopContinuation: vi.fn(async () => { throw new Error("queue unavailable"); }), + close: () => {}, + }; + const epoch = new SessionAuthorityEpoch("run-1", authority as never); + let error: unknown; + for (let n = 0; n < 2 + TOOL_CALL_LOOP_REFUSAL_LIMIT; n++) { + error = await epoch.execute({ tool: "call_api", callId: `c-${n}`, arguments: {} }).then(() => null, (failure: unknown) => failure); + } + expect(error).toBeInstanceOf(ToolCallLoopError); + expect(authority.scheduleToolCallLoopContinuation).toHaveBeenCalledTimes(1); + }); + it("yields to an existing tools-refresh wake without claiming completion or a human interaction", () => { const result = nativeToolsRefreshWaitResult({ wakeId: "wake-1", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index adb11edef8b..cffe9cd56c0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -271,7 +271,7 @@ import { redactRunnerApiPayloads } from "./runner-api-log-redaction.js"; import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; import { createNativeCredentialUpdateOwner, appendNativeCredentialUpdate } from "./native-credential-updates.js"; import { canonicalNativeJson, nativeSha256 } from "./canonical.js"; -import { RepeatedToolFailureGuard } from "./repeated-tool-failure-guard.js"; +import { RepeatedToolFailureGuard, ToolCallLoopError, toolCallLoopContinuationKey } from "./repeated-tool-failure-guard.js"; import { lockSourceRecoveryCoordinators } from "./native-source-recovery-ownership.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { dispatchToolCallback, recoverToolCallback, type SemanticInputAdmission } from "../../vendor/paperclip-runner/index.js"; @@ -1933,6 +1933,29 @@ export function nativeToolsRefreshWaitResult(input: { }; } +export function nativeToolCallLoopWaitResult(input: { + wakeId: string; + key: string; + completionContract: NativeExecutionInput["completionContract"]["contract"]; +}): PrpStructuredRunResult { + const ref = `wakeup:${input.wakeId}`; + return { + ...nativeToolsRefreshWaitResult(input), + summary: "Paperclip stopped a tool call that kept failing the same way. The task continues in a fresh session.", + completionClaim: { + contractRevision: input.completionContract.revision, + objectiveSatisfied: false, + criteria: input.completionContract.criteria.map((criterion) => ({ + criterionId: criterion.id, + status: "unknown", + evidenceRefs: [ref], + })), + remainingWork: [{ description: "Continue the task in a fresh session.", blocksCompletion: true }], + }, + continuation: { kind: "same_agent", summary: "Continue the task in a fresh session.", idempotencyKey: input.key }, + }; +} + /** * Partial item-verdict responses deliberately leave their original durable * interaction pending. They are already authority-checked before entering the @@ -2293,7 +2316,14 @@ export class SessionAuthorityEpoch { async execute(call: Parameters[0]) { this.#assertCurrent(); - const execute = () => this.#repeatedFailures.run(call, () => this.#authority.execute(call)); + const execute = () => this.#repeatedFailures.run(call, () => this.#authority.execute(call)).catch(async (error: unknown) => { + if (error instanceof ToolCallLoopError) { + await this.#authority.scheduleToolCallLoopContinuation?.(error).catch((scheduleError: unknown) => { + logger.warn({ err: scheduleError, runId: this.runId, tool: error.tool }, "tool-call loop continuation was not queued"); + }); + } + throw error; + }); return dispatchToolCallback(this.runId, call.callId, async () => this.toolTrace ? await this.toolTrace.execute(call, execute) : await execute()); @@ -12254,10 +12284,28 @@ async function executePaperclipNativeSessionAttempt( ), ) .limit(1); - return refresh?.key - ? nativeToolsRefreshWaitResult({ - wakeId: refresh.id, - key: refresh.key, + if (refresh?.key) + return nativeToolsRefreshWaitResult({ + wakeId: refresh.id, + key: refresh.key, + completionContract: input.execution.completionContract.contract, + }); + const loopKey = toolCallLoopContinuationKey(input.execution.binding.runId); + const [loop] = await input.db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, input.execution.binding.companyId), + eq(agentWakeupRequests.idempotencyKey, loopKey), + notInArray(agentWakeupRequests.status, ["skipped", "failed", "cancelled"]), + ), + ) + .limit(1); + return loop + ? nativeToolCallLoopWaitResult({ + wakeId: loop.id, + key: loopKey, completionContract: input.execution.completionContract.contract, }) : null; diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts index d8fc0b129f7..c1f69eb2620 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts @@ -8,6 +8,7 @@ import { LOW_TRUST_REVIEW_PRESET, upsertIssueDocumentSchema } from "@paperclipai import { and, eq, inArray } from "drizzle-orm"; import { activityLog, + agentWakeupRequests, agents, authUsers, approvals, @@ -30,6 +31,7 @@ import { initializeRunIdentity, reserveSteeredIdentity, reconcileSteeredIdentity import { documentService } from "../documents.js"; import { issueService } from "../issues.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; +import { TOOL_CALL_LOOP_HINT, ToolCallLoopError } from "./repeated-tool-failure-guard.js"; import { PaperclipRunnerSemanticAuthority } from "./runner-semantic-authority.js"; import { createAssignedMcpTools } from "./assigned-mcp-tools.js"; import type { ToolGatewayService } from "../tool-gateway.js"; @@ -1610,6 +1612,32 @@ describe("PaperclipRunnerToolAuthority", () => { .rejects.toThrow(/conversation cannot be a parent/i); }); + it.each([ + ["the assignee", null], + ["a completion reviewer", { nativeReviewInteractionId: "7f0b4f3e-2c1d-4a5b-8c6d-0e1f2a3b4c5d", nativeReviewDecisionId: "8a1c5e4f-3d2e-4b6c-9d7e-1f2a3b4c5d6e" }], + ] as const)("queues one fresh-session continuation outbox for %s stopped by a tool-call loop", async (_who, nativeReview) => { + const authority = new PaperclipRunnerToolAuthority(db, { + companyId, agentId, issueId, runId, workMode: "standard", ...(nativeReview ? { nativeReview } : {}), + }); + const key = `agent_tool_call_loop:${runId}`; + const loop = new ToolCallLoopError("call_api", 6, 6, ["operationId", "pathParams"], 400, "refused"); + await expect(authority.scheduleToolCallLoopContinuation(loop)).resolves.toBe(true); + await expect(authority.scheduleToolCallLoopContinuation(loop)).resolves.toBe(true); + const rows = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + agentId, status: "queued", runId: null, requestedByActorType: "system", requestedByActorId: "native-status-committer", + reason: nativeReview ? "native_completion_review" : "issue_continuation_needed", + payload: { + issueId, sourceRunId: runId, continuationCause: "agent_tool_call_loop", tool: "call_api", argumentNames: ["operationId", "pathParams"], + ...(nativeReview ?? {}), + _paperclipWakeContext: { issueId, forceFreshSession: true, livenessContinuationSourceRunId: runId, + livenessContinuationReason: "agent_tool_call_loop", livenessContinuationInstruction: TOOL_CALL_LOOP_HINT, ...(nativeReview ?? {}) }, + }, + }); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key)); + }); + it("creates ordinary children, preserves blockers, and deduplicates across runs", async () => { const wakes: Array<{ agentId: string; options: Record }> = []; diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 8b61d8fcb8a..77406c5b089 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -79,6 +79,8 @@ import { assertRunnerApiDispatchable, executeRunnerApi, validateRunnerApiCall, R import { finishPendingRunnerApiCaptureDeletes, markRunnerApiCaptureEvictions, RUNNER_API_CAPTURE_EVICTION_PASSES, runnerApiCaptureCleanupStalled, runnerApiCaptureUsageBytes } from "./runner-api-capture-retention.js"; import { acquireRunnerApiResponseSlot, RUNNER_API_RESPONSE_DEADLINE_MS, RunnerApiCaptureReservationReclaimedError, runnerApiCompanyCaptureMaxBytes, RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES, RunnerApiResponseLimitError } from "./runner-api-response-limits.js"; import { and, desc, eq, inArray, isNull, notInArray, sql } from "drizzle-orm"; +import { AGENT_TOOL_CALL_LOOP, TOOL_CALL_LOOP_HINT, toolCallLoopContinuationKey, type ToolCallLoopError } from "./repeated-tool-failure-guard.js"; +import { enqueueWake } from "./status-decision-committer.js"; import type { Db } from "@paperclipai/db"; import { authUsers, @@ -350,6 +352,25 @@ export class PaperclipRunnerToolAuthority { #taskAttachmentCache = new TaskAttachmentReadCache(); #skillLibraryCache = new SkillLibraryReadCache(); close() { this.#taskAttachmentCache.close(); this.#skillLibraryCache.close(); this.binding.assignedMcpTools?.close?.(); } + + async scheduleToolCallLoopContinuation(loop: ToolCallLoopError): Promise { + const review = this.binding.nativeReview; + const loopContext = { sourceRunId: this.binding.runId, continuationCause: AGENT_TOOL_CALL_LOOP, + tool: loop.tool, refusals: loop.refusals, runRefusals: loop.runRefusals, argumentNames: loop.argumentNames }; + await this.db.transaction(tx => enqueueWake({ + tx: tx as unknown as Db, + originRunId: this.binding.runId, + companyId: this.binding.companyId, + issueId: this.binding.issueId, + agentId: this.binding.agentId, + reason: review ? "native_completion_review" : "issue_continuation_needed", + idempotencyKey: toolCallLoopContinuationKey(this.binding.runId), + payload: { ...(review ?? {}), ...loopContext }, + contextSnapshot: { ...(review ?? {}), forceFreshSession: true, livenessContinuationSourceRunId: this.binding.runId, + livenessContinuationReason: AGENT_TOOL_CALL_LOOP, livenessContinuationInstruction: TOOL_CALL_LOOP_HINT }, + })); + return true; + } constructor(readonly db: Db, readonly binding: Binding) {} definitions(): Array> { diff --git a/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts b/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts index b51d5bb17e1..a522c014918 100644 --- a/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts +++ b/server/src/services/native-runtime/repeated-tool-failure-guard.test.ts @@ -3,6 +3,7 @@ import { HttpError, badRequest } from "../../errors.js"; import { SessionAuthorityEpoch } from "./native-session-executor.js"; import type { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { executeRunnerApi } from "./runner-api-client.js"; +import { TOOL_CALL_LOOP_REFUSAL_LIMIT, TOOL_CALL_LOOP_RUN_REFUSAL_LIMIT, ToolCallLoopError } from "./repeated-tool-failure-guard.js"; const context = { companyId: "company-a", issueId: "issue-a", issueIdentifier: "API-1", runId: "run-a", workMode: "standard" }; @@ -53,6 +54,39 @@ describe("repeated identical failing tool calls", () => { expect(fetcher).not.toHaveBeenCalled(); }); + it("after N refusals the guard raises a turn-stopping loop error", async () => { + const { authority, epoch } = epochWith(async () => { throw badRequest("body is required"); }); + const call = (n: number) => ({ tool: "call_api", callId: `loop-${n}`, arguments: dropped }); + await failure(epoch, call(1)); + await failure(epoch, call(2)); + for (let n = 1; n < TOOL_CALL_LOOP_REFUSAL_LIMIT; n++) { + const refused = await failure(epoch, call(2 + n)); + expect(refused).not.toBeInstanceOf(ToolCallLoopError); + expect(refused?.message).toMatch(/^This call was not run/); + } + const stopped = await failure(epoch, call(2 + TOOL_CALL_LOOP_REFUSAL_LIMIT)); + expect(stopped).toBeInstanceOf(ToolCallLoopError); + expect(stopped).toMatchObject({ code: "agent_tool_call_loop", tool: "call_api", refusals: TOOL_CALL_LOOP_REFUSAL_LIMIT, + argumentNames: ["operationId", "pathParams", "query", "responseText"] }); + expect(authority.execute).toHaveBeenCalledTimes(2); + }); + + it("stops the turn once refusals across different calls reach the run limit", async () => { + const { epoch } = epochWith(async () => { throw badRequest("body is required"); }); + let stopped: Error | null = null; + let refusals = 0; + for (let variant = 0; !stopped && variant < TOOL_CALL_LOOP_RUN_REFUSAL_LIMIT; variant++) { + const args = { ...dropped, pathParams: { id: `issue-${variant}` } }; + for (let n = 0; n < 2 + TOOL_CALL_LOOP_REFUSAL_LIMIT - 1 && !stopped; n++) { + const error = await failure(epoch, { tool: "call_api", callId: `v${variant}-${n}`, arguments: args }); + if (error instanceof ToolCallLoopError) stopped = error; + else if (error?.message.startsWith("This call was not run")) refusals++; + } + } + expect(stopped).toBeInstanceOf(ToolCallLoopError); + expect(refusals + 1).toBe(TOOL_CALL_LOOP_RUN_REFUSAL_LIMIT); + }); + it("runs the call again once its arguments change", async () => { const { authority, epoch } = epochWith(async call => { if (!(call.arguments as { body?: unknown }).body) throw badRequest("body is required"); diff --git a/server/src/services/native-runtime/repeated-tool-failure-guard.ts b/server/src/services/native-runtime/repeated-tool-failure-guard.ts index 25c4c08dac3..f1739fcab9c 100644 --- a/server/src/services/native-runtime/repeated-tool-failure-guard.ts +++ b/server/src/services/native-runtime/repeated-tool-failure-guard.ts @@ -3,6 +3,16 @@ import { nativeSha256 } from "./canonical.js"; /** Identical failures after which the exact call is no longer run. */ export const REPEATED_TOOL_FAILURE_LIMIT = 2; +export const TOOL_CALL_LOOP_REFUSAL_LIMIT = 6; +export const TOOL_CALL_LOOP_RUN_REFUSAL_LIMIT = 20; +export const AGENT_TOOL_CALL_LOOP = "agent_tool_call_loop"; +export function toolCallLoopContinuationKey(runId: string): string { + return `${AGENT_TOOL_CALL_LOOP}:${runId}`; +} + +export const TOOL_CALL_LOOP_HINT = "The previous turn kept sending one tool call that Paperclip had already refused. " + + "Do not send it again unchanged. Prefer a dedicated Paperclip tool, such as set_task_title, report_progress or write_document, over call_api. " + + "If call_api is the only way, send every required argument, including body, as a top-level argument."; const MAX_TRACKED_FAILURES = 64; const MAX_RESPONSE_EXCERPT = 2_000; @@ -14,8 +24,18 @@ type Failure = { details?: unknown; /** Returned by the route as an `ok: false` response rather than refused before sending. */ returned: boolean; + refusals?: number; }; +export class ToolCallLoopError extends HttpError { + readonly code = AGENT_TOOL_CALL_LOOP; + constructor(readonly tool: string, readonly refusals: number, readonly runRefusals: number, + readonly argumentNames: string[], status: number, message: string, details?: unknown) { + super(status, message, details); + this.name = "ToolCallLoopError"; + } +} + /** * Stops a model from resending the same rejected tool call. Some model hosts * drop optional arguments, so the model can believe it fixed a call that @@ -36,6 +56,7 @@ export class RepeatedToolFailureGuard { // Identical calls a model sends in parallel run one after another, so each // sees the failures before it. Different calls never wait on each other. readonly #inFlight = new Map>(); + #runRefusals = 0; run(call: ToolCall, execute: () => Promise): Promise { const key = callKey(call); @@ -50,6 +71,14 @@ export class RepeatedToolFailureGuard { async #run(key: string, call: ToolCall, execute: () => Promise): Promise { const prior = this.#failures.get(key); if (prior && prior.count >= REPEATED_TOOL_FAILURE_LIMIT) { + prior.refusals = (prior.refusals ?? 0) + 1; + this.#runRefusals += 1; + if (prior.refusals >= TOOL_CALL_LOOP_REFUSAL_LIMIT || this.#runRefusals >= TOOL_CALL_LOOP_RUN_REFUSAL_LIMIT) { + throw new ToolCallLoopError(call.tool, prior.refusals, this.#runRefusals, argumentNames(call.arguments), prior.status, + `This call was not run, and Paperclip stopped this turn: ${call.tool} was refused ${prior.refusals} times in this run with these exact arguments. ` + + `${receivedArguments(call.arguments)} The task continues in a fresh session. ${TOOL_CALL_LOOP_HINT}`, + prior.details); + } throw new HttpError(prior.status, `This call was not run: ${call.tool} already failed ${prior.count} times in this run with these exact arguments. ` + `${receivedArguments(call.arguments)} ${prior.returned ? `The response each time was HTTP ${prior.status}: ` : "The error each time was: "}${prior.message} ` + "Repeating it unchanged cannot succeed. Send it again only with the change that error asks for, or stop calling it and tell the user what blocked you.", prior.details); @@ -131,7 +160,11 @@ function repeatedNote(count: number, call: ToolCall): string { + "Paperclip will not run this exact call again in this run. Change the arguments, or stop and tell the user what blocked you."; } +function argumentNames(value: unknown): string[] { + return value && typeof value === "object" && !Array.isArray(value) ? Object.keys(value).sort() : []; +} + function receivedArguments(value: unknown): string { - const names = value && typeof value === "object" && !Array.isArray(value) ? Object.keys(value).sort() : []; + const names = argumentNames(value); return `Arguments received each time: ${names.length ? names.join(", ") : "none"}.`; } diff --git a/server/src/services/native-runtime/status-arbiter.test.ts b/server/src/services/native-runtime/status-arbiter.test.ts index 0e57f8f1b7b..579ddd39eaf 100644 --- a/server/src/services/native-runtime/status-arbiter.test.ts +++ b/server/src/services/native-runtime/status-arbiter.test.ts @@ -151,6 +151,12 @@ describe("native status authority", () => { expect(decision.effects.some((effect) => ["enqueue_continuation", "schedule_retry"].includes(effect.kind))).toBe(false); }); + it("lets a reviewer stopped by a tool-call loop continue through its queued review wake instead of a resume recovery", () => { + const decision = arbitrate({ priorIssueStatus: "in_review", nativeReviewOutcome: "pending", toolCallLoopContinuation: true }); + expect(decision).toMatchObject({ statusAction: "preserve", toStatus: "in_review", reasonCode: "native_review_unresolved" }); + expect(decision.effects).toEqual([{ kind: "release_checkout" }]); + }); + it("treats only the authorized Board response_wake as passive and preserves governance", () => { const passive = assessment({ reportedDisposition: "yielded", diff --git a/server/src/services/native-runtime/status-arbiter.ts b/server/src/services/native-runtime/status-arbiter.ts index 7fd22e7506a..26e3643e5e2 100644 --- a/server/src/services/native-runtime/status-arbiter.ts +++ b/server/src/services/native-runtime/status-arbiter.ts @@ -240,6 +240,7 @@ export function arbitrateNativeStatus(input: { reviewOwnerUserId?: string | null; /** Review decisions own task state; a reviewer's finish report cannot override them. */ nativeReviewOutcome?: "resolved" | "pending" | "stale"; + toolCallLoopContinuation?: boolean; /** * This run, as the configured participant, recorded the approval that * completed the task's final execution-policy stage. That durable decision is @@ -375,7 +376,7 @@ export function arbitrateNativeStatus(input: { unblockDescriptor: null, effects: [ { kind: "release_checkout" }, - ...(input.nativeReviewOutcome === "pending" ? [{ + ...(input.nativeReviewOutcome === "pending" && !input.toolCallLoopContinuation ? [{ kind: "record_recovery" as const, cause: "native_review_unresolved", nextAction: "The named reviewer must resolve the pending completion review or explain the concrete blocker. Do not restart the worker's assignment or create another review request.", agentId: input.agentId, diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 37877b261d7..80da7787b0e 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1,4 +1,5 @@ import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js"; +import { toolCallLoopContinuationKey } from "./repeated-tool-failure-guard.js"; import { assertAgentInteraction } from "../agent-interaction-policy.js"; import { forbidden, HttpError } from "../../errors.js"; import { agentInteractionDeliveryDenial, agentWakeInteractionTexts, INTERACTION_DELIVERY_DENIED, withInteractionDeliveryOrigin } from "../agent-interaction-delivery.js"; @@ -795,7 +796,7 @@ async function materializeDecisionEffect(input: { if ( effect.idempotencyKey.startsWith( `connection-intent:tools:${input.runId}:`, - ) + ) || effect.idempotencyKey === toolCallLoopContinuationKey(input.runId) ) { const [refresh] = await input.tx .select({ id: agentWakeupRequests.id })