diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index 6da5c8d45f3..e9242bee000 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -963,6 +963,7 @@ impl AcpxProviderSession { let mut next_bridge = self.tool_bridge.clone(); let mut next_reserved_bridge = self.reserved_tool_bridge.clone(); let mut reconciled_events = Vec::with_capacity(events.len()); + let mut retryable_rejections = Vec::new(); for event in events { let mut expose_event = true; match &event { @@ -998,9 +999,19 @@ impl AcpxProviderSession { if let Err(error) = bridge.begin_call(call_id.clone(), operation_id.clone(), input.clone()) { - return Err(self.fail_closed(LocalRunnerError::invalid(format!( - "ACPX provider tool authorization failed: {error}" - )))); + if !error.is_retryable() { + return Err(self.fail_closed(LocalRunnerError::invalid(format!( + "ACPX provider tool authorization failed: {error}" + )))); + } + if let Err(error) = next_state.complete_tool(call_id, operation_id) { + return Err(self.fail_closed(error)); + } + retryable_rejections.push(( + call_id.clone(), + error.safe_provider_message().unwrap_or_default().to_owned(), + )); + expose_event = false; } } AcpxProviderStateEvent::SemanticResult(result) => { @@ -1125,6 +1136,22 @@ impl AcpxProviderSession { self.state = next_state; self.tool_bridge = next_bridge; self.reserved_tool_bridge = next_reserved_bridge; + if let Some(turn_id) = (!retryable_rejections.is_empty()) + .then(|| self.state.active_turn_id().map(str::to_owned)) + .flatten() + { + for (call_id, message) in retryable_rejections { + let response = self + .transport + .request_tool_resolve(json!({ + "callId": call_id, + "turnId": turn_id, + "error": {"message": message}, + })) + .map_err(|(error, _)| self.fail_closed(error))?; + self.verify_resolution(&response, "tool")?; + } + } if let Some(turn_id) = terminal_turn { self.offer_provider_turn(&turn_id, terminal_completed, terminal_live); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index 8bd51438e54..370a840ec86 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -104,7 +104,10 @@ fn run() -> Result<(), Box> { .try_into() .map_err(|_| "lifetime fence requires three ports")?; // Only the receiver-admission fixture writes this task-owned command journal. - let mut admission_journal = if matches!(mode, "admission-tool" | "admission-tool-oversized") { + let mut admission_journal = if matches!( + mode, + "admission-tool" | "admission-tool-oversized" | "admission-tool-burst" + ) { let path = args .windows(2) .find(|pair| pair[0] == "--journal") @@ -641,7 +644,10 @@ fn run() -> Result<(), Box> { } if let Some(journal) = admission_journal.as_mut() { write_json(journal, &json!({"request":request}))?; - let response = if command == "tool.resolve" { + let response = if command == "tool.resolve" && mode == "admission-tool-burst" { + json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION, + "id":id,"ok":true,"result":{"resolved":true}}) + } else if command == "tool.resolve" { json!({"protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION, "id":id,"ok":true,"result":{"resolved": request["params"]["callId"] == "call-admission" @@ -660,7 +666,20 @@ fn run() -> Result<(), Box> { ) }; write_json(&mut stdout, &response)?; - if command == "turn.start" { + if command == "turn.start" && mode == "admission-tool-burst" { + for index in 0..11 { + let frame = json!({ + "protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION, + "sequence":next_sequence,"eventType":"runtime.tool_called", + "runId":"run-1","turnId":request["params"]["turnId"], + "payload":{"callId":format!("call-burst-{index}"),"operationId":"issues.read", + "input":{"id":format!("issue-{index}")}}, + }); + write_json(journal, &json!({"emitted":frame}))?; + write_json(&mut stdout, &frame)?; + next_sequence += 1; + } + } else if command == "turn.start" { let frame = json!({ "protocolVersion":GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION, "sequence":next_sequence,"eventType":"runtime.tool_called", diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 7e811ce6c7d..02907217626 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -72,7 +72,7 @@ fn invalid_tool_call_result( "message": error .safe_provider_message() .unwrap_or(GENERIC_INVALID_TOOL_CALL_MESSAGE), - "retryable": false, + "retryable": error.is_retryable(), }, }), is_error: true, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs index b1ab2939aad..bf9d33ebd24 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs @@ -39,6 +39,9 @@ const MAX_SETTLED_CALL_IDS: usize = 65_536; const REPLAY_FILTER_WORDS: usize = 32_768; const ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE: &str = "durable provider tool receipt limit reached for the active turn"; +const TOOL_CALLS_IN_FLIGHT_MESSAGE: &str = + "too many provider tool calls are in flight for the retained result reserve"; +const TOOL_CALLS_IN_FLIGHT_HINT: &str = "Too many tool calls are in flight. This call was not executed. Wait for the pending tool results, then retry it."; const COMPLETION_INPUT_SCHEMA_HINT: &str = "Invalid paperclip_finish arguments. Required fields: reportedWorkDisposition, summary, completionClaim, evidence, and verification. When reportedWorkDisposition is yielded, continuation must include kind=response_wake or monitor, summary, and idempotencyKey."; const HUMAN_INPUT_SCHEMA_HINT: &str = "Invalid request_human_input arguments. Put version and supersedeOnUserComment at payload.version and payload.supersedeOnUserComment, never at the root. For questions, use payload.version=1 and a complete payload.questionSet. Preserve question and option IDs. Correct the arguments and retry."; const APPROVAL_INPUT_SCHEMA_HINT: &str = "Invalid request_human_input arguments. Put version and supersedeOnUserComment at payload.version and payload.supersedeOnUserComment, never at the root. Keep approval requests in their confirmation or checkbox interactionKind; do not replace them with questions. Correct the arguments and retry."; @@ -310,6 +313,17 @@ impl ProviderBridgeError { self.message == ACTIVE_TURN_RECEIPT_LIMIT_MESSAGE } + fn tool_calls_in_flight() -> Self { + Self { + message: TOOL_CALLS_IN_FLIGHT_MESSAGE.to_owned(), + safe_provider_message: Some(TOOL_CALLS_IN_FLIGHT_HINT.to_owned()), + } + } + + pub fn is_retryable(&self) -> bool { + self.message == TOOL_CALLS_IN_FLIGHT_MESSAGE + } + pub fn safe_provider_message(&self) -> Option<&str> { self.safe_provider_message.as_deref() } @@ -779,7 +793,7 @@ impl ProviderToolBridge { .and_then(|total| total.checked_add(result_reserve)) .is_none_or(|total| total > MAX_RETAINED_TOOL_VALUE_BYTES) { - return Err(ProviderBridgeError::active_turn_receipt_limit()); + return Err(ProviderBridgeError::tool_calls_in_flight()); } // Reserve the complete encoded receipt, including this exact input and // a maximum-sized result, before accepting work. An admitted call can @@ -791,7 +805,7 @@ impl ProviderToolBridge { return Err(ProviderBridgeError::active_turn_receipt_limit()); } if self.ensure_settled_result_capacity(Some(&call)).is_err() { - return Err(ProviderBridgeError::active_turn_receipt_limit()); + return Err(ProviderBridgeError::tool_calls_in_flight()); } self.pending.insert(call_id, call.clone()); Ok(call) @@ -1656,6 +1670,108 @@ mod tests { bridge } + fn get_task_bridge() -> ProviderToolBridge { + let operation = AuthorizedTool { + operation_id: "get_task".to_owned(), + version: 1, + description: "Read a task.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object"}), + }; + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(std::slice::from_ref(&operation)) + .unwrap(), + operations: vec![operation], + }) + .unwrap(); + bridge + } + + fn settle_get_task(bridge: &mut ProviderToolBridge, call_id: &str, result_bytes: usize) { + bridge + .begin_call(call_id.to_owned(), "get_task".to_owned(), json!({})) + .unwrap(); + bridge + .apply_result(ToolResult { + call_id: call_id.to_owned(), + operation_id: "get_task".to_owned(), + result: json!({"body": "x".repeat(result_bytes)}), + is_error: false, + }) + .unwrap(); + } + + #[test] + fn parallel_calls_over_the_result_reserve_reject_only_the_overflow_call() { + let mut bridge = get_task_bridge(); + settle_get_task(&mut bridge, "settled", 600 * 1024); + for index in 0..9 { + bridge + .begin_call( + format!("parallel-{index}"), + "get_task".to_owned(), + json!({}), + ) + .expect("calls within the result reserve are admitted"); + } + let error = bridge + .begin_call("parallel-9".to_owned(), "get_task".to_owned(), json!({})) + .expect_err("the call that overflows the reserve is rejected"); + assert!(!error.is_active_turn_receipt_limit()); + assert!(error.is_retryable()); + assert_eq!( + error.safe_provider_message(), + Some(TOOL_CALLS_IN_FLIGHT_HINT) + ); + assert!(!bridge.durable_run_receipt_limit_reached()); + assert!(!bridge.has_call_receipt("parallel-9")); + assert_eq!(bridge.pending_calls().count(), 9); + + bridge + .apply_result(ToolResult { + call_id: "parallel-0".to_owned(), + operation_id: "get_task".to_owned(), + result: json!({"ok": true}), + is_error: false, + }) + .unwrap(); + bridge + .begin_call("parallel-9".to_owned(), "get_task".to_owned(), json!({})) + .expect("the rejected call is admitted once a pending result settles"); + assert!(!bridge.durable_run_receipt_limit_reached()); + } + + #[test] + fn held_result_bytes_over_the_limit_still_stop_the_turn() { + let mut bridge = get_task_bridge(); + let mut settled = 0; + let error = loop { + match bridge.begin_call(format!("held-{settled}"), "get_task".to_owned(), json!({})) { + Ok(_) => { + bridge + .apply_result(ToolResult { + call_id: format!("held-{settled}"), + operation_id: "get_task".to_owned(), + result: json!({"body": "x".repeat(760 * 1024)}), + is_error: false, + }) + .unwrap(); + settled += 1; + } + Err(error) => break error, + } + }; + assert!(settled >= 9); + assert_eq!(bridge.pending_calls().count(), 0); + assert!(error.is_active_turn_receipt_limit()); + assert!(!error.is_retryable()); + assert!(bridge.durable_run_receipt_limit_reached()); + } + #[test] fn completion_summary_enforces_the_canonical_unicode_character_limit() { let mut within_limit = completion_bridge(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs index 9478a755975..3d383bc310d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs @@ -385,6 +385,68 @@ fn receiver_admits_normal_tool_event_and_resolves_only_correlated_call_once() { check_tool_receiver_admission(false); } +#[test] +fn receiver_rejects_only_the_tool_call_that_overflows_the_result_reserve() { + let root = std::env::temp_dir().join(format!("acpx-burst-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let journal = root.join("commands.jsonl"); + let mut fixture = config("admission-tool-burst"); + fixture + .transport + .args + .extend(["--journal".to_owned(), journal.to_str().unwrap().to_owned()]); + let mut session = AcpxProviderSession::start(&fixture).unwrap(); + session + .start_turn("turn-admission", "Read the issues", &std::env::temp_dir()) + .unwrap(); + let mut exposed = 0; + while (0..10).any(|index| { + session + .state() + .pending_tool(&format!("call-burst-{index}")) + .is_none() + }) || exposed < 10 + { + exposed += session + .poll_event(Duration::from_secs(1)) + .unwrap() + .map_or(0, |events| events.len()); + } + for _ in 0..3 { + let _ = session.poll_event(Duration::from_millis(100)).unwrap(); + } + assert_eq!(exposed, 10); + assert!(session.state().pending_tool("call-burst-10").is_none()); + session + .deliver_tool_result(&paperclip_runner_core::provider_bridge::ToolResult { + call_id: "call-burst-0".to_owned(), + operation_id: "issues.read".to_owned(), + result: json!({"id":"issue-0"}), + is_error: false, + }) + .unwrap(); + session.shutdown("burst test complete").unwrap(); + let rows: Vec = std::fs::read_to_string(&journal) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + let resolutions: Vec<_> = rows + .iter() + .filter_map(|row| row.get("request")) + .filter(|request| request["command"] == "tool.resolve") + .map(|request| request["params"].clone()) + .collect(); + assert_eq!(resolutions.len(), 2); + assert_eq!(resolutions[0]["callId"], "call-burst-10"); + assert!(resolutions[0]["error"]["message"] + .as_str() + .unwrap() + .contains("Too many tool calls are in flight")); + assert_eq!(resolutions[1]["callId"], "call-burst-0"); + std::fs::remove_dir_all(root).unwrap(); +} + #[test] fn pi_thinking_effective_identity_is_admitted_before_any_turn() { use paperclip_runner_core::acpx_provider_session::{ diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index a472054ee17..79952d5869b 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -5,29 +5,74 @@ import { and, eq } from "drizzle-orm"; import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; import { accountingForScheduledRetry, executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_MESSAGE } from "./native-provider-failure.js"; +import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS, NATIVE_TOOL_RECEIPT_LIMIT_HINT, + NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS, NATIVE_TOOL_RECEIPT_LIMIT_MESSAGE, nativeRunHasStopRequest } from "./native-tool-receipt-limit.js"; +import { NativeStatusRaceError } from "./status-decision-committer.js"; -/** Called under the finalizer's issue/coordinator locks, in the status-effect transaction. */ -export async function materializeNativeProviderCapacityRetry(input: { +type NativeRetryTarget = { tx: Db; companyId: string; issueId: string; runId: string; agentId: string; -}) { +}; + +type NativeRetryLane = { + reason: string; + maxRetries: number; + delayMs: (attempt: number) => number; + message: string; + idempotencyPrefix: string; + exhaustedError: string; + context?: (run: typeof heartbeatRuns.$inferSelect) => Record; + refuseStopped?: boolean; +}; + +/** Called under the finalizer's issue/coordinator locks, in the status-effect transaction. */ +export async function materializeNativeProviderCapacityRetry(input: NativeRetryTarget) { + return materializeNativeFreshSessionRetry(input, { + reason: NATIVE_PROVIDER_OVERLOADED_CODE, + maxRetries: NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, + delayMs: attempt => 60_000 * 2 ** (attempt - 1), + message: NATIVE_PROVIDER_OVERLOADED_MESSAGE, + idempotencyPrefix: "native-capacity", + exhaustedError: "native_capacity_retry_exhausted", + }); +} + +export async function materializeNativeToolReceiptLimitContinuation(input: NativeRetryTarget) { + return materializeNativeFreshSessionRetry(input, { + reason: NATIVE_TOOL_RECEIPT_LIMIT_CODE, + maxRetries: NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS, + delayMs: () => NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS, + message: NATIVE_TOOL_RECEIPT_LIMIT_MESSAGE, + idempotencyPrefix: "native-tool-receipt-limit", + exhaustedError: "native_tool_receipt_limit_continuation_exhausted", + refuseStopped: true, + context: run => ({ + livenessContinuationSourceRunId: run.id, + livenessContinuationReason: NATIVE_TOOL_RECEIPT_LIMIT_CODE, + livenessContinuationInstruction: NATIVE_TOOL_RECEIPT_LIMIT_HINT, + }), + }); +} + +async function materializeNativeFreshSessionRetry(input: NativeRetryTarget, lane: NativeRetryLane) { const [run] = await input.tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.agentId, input.agentId), eq(heartbeatRuns.nativeIssueId, input.issueId), )).for("update"); if (!run || run.runtimeMode !== "native") throw new Error("native_capacity_retry_run_binding_mismatch"); + if (lane.refuseStopped && nativeRunHasStopRequest(run.resultJson)) throw new NativeStatusRaceError(); // Share the predecessor claim with every other failure-retry lane. const [existing] = await input.tx.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.retryOfRunId, run.id), )).limit(1); if (existing) return existing.id; const attempt = executionFailureRetryCount(run) + 1; - if (attempt > NATIVE_PROVIDER_CAPACITY_MAX_RETRIES) throw new Error("native_capacity_retry_exhausted"); + if (attempt > lane.maxRetries) throw new Error(lane.exhaustedError); const now = new Date(); - const dueAt = new Date(now.getTime() + 60_000 * 2 ** (attempt - 1)); + const dueAt = new Date(now.getTime() + lane.delayMs(attempt)); const retryId = randomUUID(); const predecessorContext = { ...run.contextSnapshot }; // Resume receipts and delivered wake input belong to the failed run. The @@ -45,18 +90,19 @@ export async function materializeNativeProviderCapacityRetry(input: { taskId: input.issueId, forceFreshSession: true, retryOfRunId: run.id, - retryReason: NATIVE_PROVIDER_OVERLOADED_CODE, - wakeReason: NATIVE_PROVIDER_OVERLOADED_CODE, + retryReason: lane.reason, + wakeReason: lane.reason, scheduledRetryAttempt: attempt, scheduledRetryAt: dueAt.toISOString(), - executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), + executionRetryAccounting: accountingForScheduledRetry(run, lane.reason, attempt), + ...lane.context?.(run), }; await notifyIssueAttentionRetentionWork(input.tx, { companyId: input.companyId, issueId: input.issueId, runId: run.id }); await notifyScheduledRetryDispatch(input.tx, { companyId: run.companyId, runId: run.id }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", - reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, - idempotencyKey: `native-capacity:${run.companyId}:${run.id}`, + reason: lane.reason, status: "queued", runId: retryId, + idempotencyKey: `${lane.idempotencyPrefix}:${run.companyId}:${run.id}`, requestedByActorType: "system", requestedByActorId: "native_finalizer", payload: contextSnapshot, }).returning({ id: agentWakeupRequests.id }); @@ -64,9 +110,9 @@ export async function materializeNativeProviderCapacityRetry(input: { id: retryId, companyId: run.companyId, agentId: run.agentId, invocationSource: "automation", triggerDetail: "system", status: "scheduled_retry", responsibleUserId: run.responsibleUserId, wakeupRequestId: wake.id, - retryOfRunId: run.id, scheduledRetryReason: NATIVE_PROVIDER_OVERLOADED_CODE, + retryOfRunId: run.id, scheduledRetryReason: lane.reason, scheduledRetryAttempt: attempt, scheduledRetryAt: dueAt, - error: NATIVE_PROVIDER_OVERLOADED_MESSAGE, errorCode: NATIVE_PROVIDER_OVERLOADED_CODE, + error: lane.message, errorCode: lane.reason, contextSnapshot, }); return retryId; diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index df6232eb978..eb5dd3977eb 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -32,6 +32,7 @@ import { settleExternalConversation } from "../slack-conversation-lifecycle.js"; import { executionFailureRetryCount } from "../execution-recovery-attempt.js"; import { isFailedTurnContinuationRun } from "./native-failed-turn-continuation.js"; import { readPersistedNativeFailureEvidence } from "./native-provider-failure-evidence.js"; +import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, cancelStoppedRunContinuation, nativeRunHasStopRequest, readPersistedNativeToolReceiptLimitStop } from "./native-tool-receipt-limit.js"; import { explainFailedRun, explainNativeTurnFailure, NATIVE_RUN_FAILED_UNEXPLAINED } from "./native-run-failure-explanation.js"; import { dismissAutomaticCompletionReviews } from "./automatic-completion-reviews.js"; import { findSupersededExternalReviewRequests } from "./superseded-external-review-requests.js"; @@ -1607,6 +1608,10 @@ export async function finalizeNativeRun(input: { const allowIncompleteContinuation = record(sourceWake?.payload).continuationIdempotencyKey !== "native-completion-incomplete"; let supersedesAssessmentId: string | null = coordinator.assessmentId; for (let attempt = 0; attempt < 3; attempt += 1) { + const savedStop = attempt === 0 ? run.resultJson : await input.db.select({ resultJson: heartbeatRuns.resultJson }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId))) + .then(rows => rows[0]?.resultJson ?? run.resultJson); + const stopRequested = nativeRunHasStopRequest(savedStop); const authoritativeIssue = await input.db .select() .from(issues) @@ -1747,6 +1752,7 @@ export async function finalizeNativeRun(input: { // issue route decides `status: "done"`. Review runs keep their own path. const reviewStage = isConversation(authoritativeIssue) || reviewContext ? { kind: "none" as const } : completionReviewStageEntry(authoritativeIssue, run.agentId); + await cancelStoppedRunContinuation(input.db, { companyId: run.companyId, resultJson: savedStop }, toolCallLoopContinuationKey(run.id)); const statusInput: Parameters[0] = { requiredQualityReviewSatisfied: qualityReview.satisfied, executionStageApprovedByRun: await executionStageApprovedByRun(input.db, run, authoritativeIssue), @@ -1761,13 +1767,16 @@ export async function finalizeNativeRun(input: { providerModelRejected: providerFailure?.errorCode === "native_provider_model_rejected" && ownsProviderFailureDecision, providerOverloaded: providerFailure?.errorCode === "native_provider_overloaded" && ownsProviderFailureDecision, providerFailureSuperseded: providerFailure?.errorCode === "native_provider_overloaded" && !ownsProviderFailureDecision, + toolReceiptLimitStopped: terminalState !== "succeeded" && ownsProviderFailureDecision && !reviewContext + && !stopRequested && await readPersistedNativeToolReceiptLimitStop(input.db, run, resultRow.turnId), failureRetryCount: executionFailureRetryCount(run), // Only the task's current owner may turn its own failed continuation // into a blocker; the committer rechecks that under the issue lock. failedTurnContinuationUsed: terminalState === "failed" && ownsProviderFailureDecision && !reviewContext && await isFailedTurnContinuationRun(input.db, run, authoritativeIssue.id), toolCallLoopContinuation: assessment.reportedDisposition === "yielded" - && assessment.continuation?.idempotencyKey === toolCallLoopContinuationKey(run.id), + && assessment.continuation?.idempotencyKey === toolCallLoopContinuationKey(run.id) + && !stopRequested, ...(reviewContext ? { nativeReviewOutcome: nativeReview ? nativeReview.interaction.status === "pending" ? "pending" as const : "resolved" as const : "stale" as const } : {}), @@ -1922,6 +1931,7 @@ export async function finalizeNativeRun(input: { : undefined, requireProviderFailureOwner: decision.reasonCode?.startsWith("native_provider_overloaded") || decision.reasonCode === "native_run_failed_continuation_exhausted" + || decision.reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE ? { agentId: run.agentId, reviewContext } : undefined, requireBoardResponseWaitSource: decision.reasonCode === "board_response_waiting" || repairBoardResponseWait diff --git a/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts b/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts new file mode 100644 index 00000000000..594a1ee0262 --- /dev/null +++ b/server/src/services/native-runtime/native-tool-receipt-limit.integration.test.ts @@ -0,0 +1,196 @@ +import { randomUUID } from "node:crypto"; +import { and, eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, companies, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, statusDecisionEffects, statusDecisions } from "@paperclipai/db"; +import type { ControlPlanePort, NativeExecutionInputV1, PrpEvent, PrpTerminalState } from "@paperclipai/paperclip-runner"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT } from "../../vendor/paperclip-runner/testing.js"; + +const provider = vi.hoisted(() => ({ execute: vi.fn() })); +const receiptLimitRead = vi.hoisted(() => ({ after: null as null | (() => Promise) })); +vi.mock("./native-tool-receipt-limit.js", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, readPersistedNativeToolReceiptLimitStop: async (...args: Parameters) => { + const stopped = await original.readPersistedNativeToolReceiptLimitStop(...args); + const after = receiptLimitRead.after; + receiptLimitRead.after = null; + await after?.(); + return stopped; + } }; +}); +vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ + ...(await importOriginal()), executeNativeSession: provider.execute, +})); +import { executePaperclipNativeSession } from "./native-session-executor.js"; +import { buildNativeCompletionContract } from "./completion-contracts.js"; +import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; +import { finalizeNativeRun } from "./native-run-finalizer.js"; +import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS, NATIVE_TOOL_RECEIPT_LIMIT_HINT, + readPersistedNativeToolReceiptLimitStop } from "./native-tool-receipt-limit.js"; +import { heartbeatService } from "../heartbeat.js"; +import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; +import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; +import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; +import { materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; +import { NativeStatusRaceError } from "./status-decision-committer.js"; + +describe("native turn stopped at its tool receipt limit", () => { + let temporary: Awaited>; + let db: ReturnType; + const companyId = randomUUID(), agentId = randomUUID(); + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("native-tool-receipt-limit-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Receipt limit regression", issuePrefix: "RCL" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Receipt worker", status: "active", adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "test-model" } }); + }); + afterAll(async () => { await temporary?.cleanup(); }); + + let finalTurnId: string | null = null; + async function stopAtReceiptLimit(runId: string, issueId: string, diagnosticCode: string, beforeFinalize?: () => Promise, + diagnosticTurnId?: string) { + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + const [run] = await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativeIssueId: issueId, error: null, errorCode: null }).where(eq(heartbeatRuns.id, runId)).returning(); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); + const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); + const contractInput = buildNativeCompletionContract(issue, { revision: contract.revision }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const execution: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", binding: { companyId, runId, issueId, agentId, executionWorkspaceId: runId }, + provider: { kind: "codex", model: "test-model" }, + task: { identifier: issueId, title: issue.title, description: null, prompt: "Continue the task using its history.", workMode: "standard" }, + workspace: { cwd: "/tmp", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: native.normalizedSessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + completionContract: { id: contract.id, sha256: contract.canonicalSha256, schemaVersion: "paperclip.completion-contract.v1", contract: contractInput }, + interactionResponses: [], credentialBindings: [], + }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const event: PrpEvent = { + schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId, turnId: diagnosticTurnId ?? native.turnId, + sourceInstanceId: native.runnerInstanceId, sourceEventId: "receipt-limit", sourceSeq: 1, sourceKind: "runner", eventType: "harness.diagnostic", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), + payload: { provider: "opencode", code: diagnosticCode, operationId: "get_task", callId: "call-80", + message: "The active provider turn reached its durable semantic-tool receipt limit and was interrupted", paperclipExecuted: false }, + }; + const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "interrupted", runTerminalState: "cancelled", reportedWorkDisposition: "needs_review" }; + const result = { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "needs_review" as const, + summary: "The OpenCode run stopped before it completed.", + completionClaim: { ...CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim, contractRevision: contractInput.revision }, + attentionRequests: [{ kind: "review" as const, summary: "Review the stopped OpenCode run before continuing.", ownerClass: "human" as const }] }; + provider.execute.mockReset().mockImplementation(async (options: { controlPlane: ControlPlanePort }) => { + await options.controlPlane.openRun({ identity: { companyId, issueId, runId, agentId, sessionId: native.normalizedSessionId }, backendKind: "mock", sourceInstanceId: native.runnerInstanceId }); + await options.controlPlane.appendEvent(event); + await options.controlPlane.completeRun({ result, terminal, turnId: native.turnId }); + return { result, terminal, turnId: native.turnId, normalizedSessionId: native.normalizedSessionId, providerSessionId: "fixture-session", driverKind: "codex_app_server", driverVersion: "fixture", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + const unexpected = async () => { throw new Error("Unexpected provider call"); }; + await executePaperclipNativeSession({ db, execution, runnerInstanceId: native.runnerInstanceId, backend: { descriptor: unexpected, openSession: unexpected } }); + await beforeFinalize?.(); + for (let replay = 0; replay < 2; replay++) await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + expect(provider.execute).toHaveBeenCalledTimes(1); + finalTurnId = native.turnId; + return (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]!; + } + + function release() { + const unexpected = vi.fn(async () => { throw new Error("Unexpected recovery"); }); + return createReleaseIssueExecution({ + issueLock: createPostgresWakeQueueAdapter(db, { resolveResponsibleUserId: unexpected, getRoutineEnv: unexpected, resolveSessionBeforeForWakeup: unexpected }), + recovery: { escalateStrandedAssignedIssue: unexpected, escalateStrandedRecoveryIssueInPlace: unexpected }, + }); + } + + it("continues the task at once in a fresh session and shows no Retry", async () => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Parallel reads task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "receipt-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "receipt-owner", contextSnapshot: { issueId, wakeCommentId: "original-message" } }); + const stopped = await stopAtReceiptLimit(runId, issueId, NATIVE_TOOL_RECEIPT_LIMIT_CODE); + expect(stopped.status).toBe("cancelled"); + expect(await readPersistedNativeToolReceiptLimitStop(db, stopped, finalTurnId)).toBe(true); + expect(await readPersistedNativeToolReceiptLimitStop(db, stopped, "another-turn")).toBe(false); + + const [retry, ...others] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, runId))); + expect(others).toHaveLength(0); + expect(retry).toMatchObject({ status: "scheduled_retry", scheduledRetryReason: NATIVE_TOOL_RECEIPT_LIMIT_CODE, scheduledRetryAttempt: 1, + contextSnapshot: { issueId, forceFreshSession: true, livenessContinuationSourceRunId: runId, + livenessContinuationReason: NATIVE_TOOL_RECEIPT_LIMIT_CODE, livenessContinuationInstruction: NATIVE_TOOL_RECEIPT_LIMIT_HINT } }); + expect(retry!.contextSnapshot).not.toHaveProperty("wakeCommentId"); + expect(retry!.scheduledRetryAt!.getTime() - Date.now()).toBeLessThanOrEqual(NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + expect(decision).toMatchObject({ reasonCode: NATIVE_TOOL_RECEIPT_LIMIT_CODE }); + expect(await db.select().from(statusDecisionEffects).where(eq(statusDecisionEffects.decisionId, decision!.id))) + .toEqual(expect.arrayContaining([expect.objectContaining({ effectKind: "schedule_retry", targetType: "heartbeat_run", targetId: retry!.id })])); + + expect((await release()({ companyId, runId, now: new Date() })).outcome.kind).toBe("released"); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue).toMatchObject({ status: "in_progress", assigneeAgentId: agentId }); + expect(await taskInterruptionPresentation(db, issue!)).toMatchObject({ state: "retry_scheduled", canRetry: false, userAction: null }); + const scheduler = heartbeatService(db, { runtimeEnv: {} }); + expect(await scheduler.promoteDueScheduledRetries(new Date())).toMatchObject({ promoted: 0 }); + expect(await scheduler.promoteDueScheduledRetries(retry!.scheduledRetryAt!)).toMatchObject({ promoted: 1, runIds: [retry!.id] }); + expect(await taskInterruptionPresentation(db, issue!)).toMatchObject({ state: "working", canRetry: false, userAction: null }); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.runId, retry!.id))).toHaveLength(1); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, retry!.id)); + }); + + it("does not continue a turn a person stopped while it settled", async () => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Stopped task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "receipt-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "receipt-owner", contextSnapshot: { issueId } }); + await stopAtReceiptLimit(runId, issueId, NATIVE_TOOL_RECEIPT_LIMIT_CODE, () => db.update(heartbeatRuns) + .set({ resultJson: { cancelledByActorType: "user", cancelledByUserId: "receipt-owner" } }).where(eq(heartbeatRuns.id, runId))); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + expect(decision?.reasonCode).not.toBe(NATIVE_TOOL_RECEIPT_LIMIT_CODE); + }); + + it("settles the cancellation when a person stops the run after finalization first read it", async () => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Late stop task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "receipt-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "receipt-owner", contextSnapshot: { issueId } }); + await stopAtReceiptLimit(runId, issueId, NATIVE_TOOL_RECEIPT_LIMIT_CODE, async () => { + receiptLimitRead.after = () => db.update(heartbeatRuns) + .set({ resultJson: { cancelledByActorType: "user", cancelledByUserId: "receipt-owner" } }).where(eq(heartbeatRuns.id, runId)); + }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)); + expect(coordinator).toMatchObject({ phase: "committed", failureCode: null }); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + expect(decision).toMatchObject({ reasonCode: "cancellation_run_only" }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); + }); + + it("does not continue when only an earlier turn hit the receipt limit", async () => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Earlier turn task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "receipt-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "receipt-owner", contextSnapshot: { issueId } }); + const stopped = await stopAtReceiptLimit(runId, issueId, NATIVE_TOOL_RECEIPT_LIMIT_CODE, undefined, "earlier-provider-turn"); + expect(await readPersistedNativeToolReceiptLimitStop(db, stopped, finalTurnId)).toBe(false); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + expect(decision).toMatchObject({ reasonCode: "cancellation_run_only" }); + }); + + it("refuses to create the continuation under the run lock once a person's Stop is saved", async () => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Locked stop task", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "cancelled", runtimeMode: "native", nativeIssueId: issueId, + contextSnapshot: { issueId }, resultJson: { cancelledByActorType: "user", cancelledByUserId: "receipt-owner" } }); + await expect(db.transaction(tx => materializeNativeToolReceiptLimitContinuation({ tx: tx as never, companyId, issueId, runId, agentId }))) + .rejects.toBeInstanceOf(NativeStatusRaceError); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, `native-tool-receipt-limit:${companyId}:${runId}`))).toHaveLength(0); + }); + + it("leaves another cancelled turn to its existing recovery", async () => { + const issueId = randomUUID(), runId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, title: "Other diagnostic task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: "receipt-owner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, responsibleUserId: "receipt-owner", contextSnapshot: { issueId } }); + const stopped = await stopAtReceiptLimit(runId, issueId, "semantic_tool_denied"); + expect(await readPersistedNativeToolReceiptLimitStop(db, stopped, finalTurnId)).toBe(false); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, runId))).toHaveLength(0); + const [decision] = await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId)); + expect(decision).toMatchObject({ reasonCode: "cancellation_run_only" }); + }); +}); diff --git a/server/src/services/native-runtime/native-tool-receipt-limit.ts b/server/src/services/native-runtime/native-tool-receipt-limit.ts new file mode 100644 index 00000000000..a4228c169c2 --- /dev/null +++ b/server/src/services/native-runtime/native-tool-receipt-limit.ts @@ -0,0 +1,60 @@ +import { and, eq, inArray, isNull, sql } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRunEvents, type heartbeatRuns, type Db } from "@paperclipai/db"; +import { validatePrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { nativeSha256 } from "./canonical.js"; +import { readRunCancellation } from "../run-cancellation.js"; + +export const NATIVE_TOOL_RECEIPT_LIMIT_CODE = "semantic_tool_turn_receipt_limit"; +export const NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS = 2; +export const NATIVE_TOOL_RECEIPT_LIMIT_CONTINUATION_DELAY_MS = 5_000; +export const NATIVE_TOOL_RECEIPT_LIMIT_MESSAGE = "The agent's turn held too many tool results at once. Paperclip continues the task in a fresh session."; +export const NATIVE_TOOL_RECEIPT_LIMIT_HINT = "The previous turn stopped because it held too many tool results at once. " + + "Continue the task from its saved work. Make fewer tool calls in parallel, and do not re-read data you already have."; + +export function nativeRunHasStopRequest(resultJson: Record | null | undefined): boolean { + if (!resultJson) return false; + const cancellation = readRunCancellation(resultJson); + return Boolean(resultJson.nativeCancellation || resultJson.executionCancellation || resultJson.cancelledByActorType + || resultJson.cancelledByUserId || resultJson.interruptedByActorType || resultJson.queuedCommentInterruptQueueId + || resultJson.issueMutationStopId || resultJson.operatorInterrupted === true + || (cancellation && cancellation.initiator.type !== "system" && cancellation.initiator.type !== "provider")); +} + +export async function cancelStoppedRunContinuation( + db: Pick, + run: Pick, + idempotencyKey: string, +): Promise { + if (!nativeRunHasStopRequest(run.resultJson)) return 0; + const now = new Date(); + const cancelled = await db.update(agentWakeupRequests).set({ + status: "cancelled", finishedAt: now, updatedAt: now, error: "The run that asked for this continuation was stopped", + }).where(and( + eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.idempotencyKey, idempotencyKey), + inArray(agentWakeupRequests.status, ["queued", "claimed", "deferred_issue_execution"]), isNull(agentWakeupRequests.runId), + )).returning({ id: agentWakeupRequests.id }); + return cancelled.length; +} + +export async function readPersistedNativeToolReceiptLimitStop( + db: Pick, + run: Pick, + turnId: string | null, +): Promise { + if (run.runtimeMode !== "native" || !run.runnerInstanceId || !run.nativeSessionId || !turnId) return false; + const rows = await db.select().from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + eq(heartbeatRunEvents.sourceInstanceId, run.runnerInstanceId), eq(heartbeatRunEvents.eventType, "harness.diagnostic"), + sql`${heartbeatRunEvents.payload}->'prpEvent'->'payload'->>'code' = ${NATIVE_TOOL_RECEIPT_LIMIT_CODE}`, + sql`${heartbeatRunEvents.payload}->'prpEvent'->>'turnId' = ${turnId}`, + )).limit(4); + return rows.some(row => { + const prpEvent = (row.payload as Record | null)?.prpEvent; + const parsed = validatePrpEvent(prpEvent); + return parsed.ok && parsed.event.runId === run.id && parsed.event.sourceKind === "runner" + && parsed.event.turnId === turnId && parsed.event.normalizedSessionId === run.nativeSessionId + && parsed.event.sourceInstanceId === run.runnerInstanceId && parsed.event.eventType === row.eventType + && parsed.event.sourceEventId === row.sourceEventId && parsed.event.sourceSeq === row.sourceSeq + && row.sourcePayloadSha256 === nativeSha256(parsed.event); + }); +} diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts index c1f69eb2620..4fbc48db02d 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts @@ -32,6 +32,7 @@ import { documentService } from "../documents.js"; import { issueService } from "../issues.js"; import { PaperclipRunnerToolAuthority } from "./paperclip-runner-tool-authority.js"; import { TOOL_CALL_LOOP_HINT, ToolCallLoopError } from "./repeated-tool-failure-guard.js"; +import { cancelStoppedRunContinuation } from "./native-tool-receipt-limit.js"; import { PaperclipRunnerSemanticAuthority } from "./runner-semantic-authority.js"; import { createAssignedMcpTools } from "./assigned-mcp-tools.js"; import type { ToolGatewayService } from "../tool-gateway.js"; @@ -1638,6 +1639,53 @@ describe("PaperclipRunnerToolAuthority", () => { await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key)); }); + it("queues no tool-call loop continuation for a run a person stopped, and cancels one queued before the Stop", async () => { + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId, workMode: "standard" }); + const key = `agent_tool_call_loop:${runId}`; + const loop = new ToolCallLoopError("call_api", 6, 6, ["operationId"], 400, "refused"); + await expect(authority.scheduleToolCallLoopContinuation(loop)).resolves.toBe(true); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + const stopped = { ...before!, resultJson: { ...before!.resultJson, cancelledByActorType: "user", cancelledByUserId: "board-user" } }; + await db.update(heartbeatRuns).set({ resultJson: stopped.resultJson }).where(eq(heartbeatRuns.id, runId)); + try { + await expect(cancelStoppedRunContinuation(db, stopped, key)).resolves.toBe(1); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key))) + .map(row => row.status)).toEqual(["cancelled"]); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key)); + await expect(authority.scheduleToolCallLoopContinuation(loop)).resolves.toBe(false); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key))).toHaveLength(0); + } finally { + await db.update(heartbeatRuns).set({ resultJson: before!.resultJson }).where(eq(heartbeatRuns.id, runId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key)); + } + }); + + it("orders the loop continuation's locks task first, like Stop, so a concurrent Stop wins without a deadlock", async () => { + const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId, workMode: "standard" }); + const key = `agent_tool_call_loop:${runId}`; + const loop = new ToolCallLoopError("call_api", 6, 6, ["operationId"], 400, "refused"); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + let holdingTask!: () => void; + const taskLocked = new Promise(resolve => { holdingTask = resolve; }); + try { + const stop = db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, issueId)).for("update"); + holdingTask(); + await new Promise(resolve => setTimeout(resolve, 300)); + await tx.update(heartbeatRuns).set({ resultJson: { ...before!.resultJson, cancelledByActorType: "user", cancelledByUserId: "board-user" } }) + .where(eq(heartbeatRuns.id, runId)); + }); + await taskLocked; + const scheduled = authority.scheduleToolCallLoopContinuation(loop); + await expect(stop).resolves.toBeUndefined(); + await expect(scheduled).resolves.toBe(false); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key))).toHaveLength(0); + } finally { + await db.update(heartbeatRuns).set({ resultJson: before!.resultJson }).where(eq(heartbeatRuns.id, runId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.idempotencyKey, key)); + } + }); + it("creates ordinary children, preserves blockers, and deduplicates across runs", async () => { const wakes: Array<{ agentId: string; options: Record }> = []; diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 77406c5b089..4df41025bf0 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -80,6 +80,7 @@ import { finishPendingRunnerApiCaptureDeletes, markRunnerApiCaptureEvictions, RU import { acquireRunnerApiResponseSlot, RUNNER_API_RESPONSE_DEADLINE_MS, RunnerApiCaptureReservationReclaimedError, runnerApiCompanyCaptureMaxBytes, RUNNER_API_RESPONSE_MAX_BYTES, RUNNER_API_RESPONSE_RUN_MAX_BYTES, RunnerApiResponseLimitError } from "./runner-api-response-limits.js"; import { and, desc, eq, inArray, isNull, notInArray, sql } from "drizzle-orm"; import { AGENT_TOOL_CALL_LOOP, TOOL_CALL_LOOP_HINT, toolCallLoopContinuationKey, type ToolCallLoopError } from "./repeated-tool-failure-guard.js"; +import { nativeRunHasStopRequest } from "./native-tool-receipt-limit.js"; import { enqueueWake } from "./status-decision-committer.js"; import type { Db } from "@paperclipai/db"; import { @@ -357,7 +358,13 @@ export class PaperclipRunnerToolAuthority { const review = this.binding.nativeReview; const loopContext = { sourceRunId: this.binding.runId, continuationCause: AGENT_TOOL_CALL_LOOP, tool: loop.tool, refusals: loop.refusals, runRefusals: loop.runRefusals, argumentNames: loop.argumentNames }; - await this.db.transaction(tx => enqueueWake({ + return this.db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.id, this.binding.issueId), eq(issues.companyId, this.binding.companyId))).for("update"); + const [run] = await tx.select({ resultJson: heartbeatRuns.resultJson }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, this.binding.runId), eq(heartbeatRuns.companyId, this.binding.companyId))).for("share"); + if (!run || nativeRunHasStopRequest(run.resultJson)) return false; + await enqueueWake({ tx: tx as unknown as Db, originRunId: this.binding.runId, companyId: this.binding.companyId, @@ -368,8 +375,9 @@ export class PaperclipRunnerToolAuthority { payload: { ...(review ?? {}), ...loopContext }, contextSnapshot: { ...(review ?? {}), forceFreshSession: true, livenessContinuationSourceRunId: this.binding.runId, livenessContinuationReason: AGENT_TOOL_CALL_LOOP, livenessContinuationInstruction: TOOL_CALL_LOOP_HINT }, - })); - return true; + }); + return true; + }); } constructor(readonly db: Db, readonly binding: Binding) {} diff --git a/server/src/services/native-runtime/status-arbiter.test.ts b/server/src/services/native-runtime/status-arbiter.test.ts index 579ddd39eaf..2a90adef678 100644 --- a/server/src/services/native-runtime/status-arbiter.test.ts +++ b/server/src/services/native-runtime/status-arbiter.test.ts @@ -157,6 +157,20 @@ describe("native status authority", () => { expect(decision.effects).toEqual([{ kind: "release_checkout" }]); }); + it("continues a turn stopped at its tool receipt limit in a fresh session, within its bound", () => { + const stopped = arbitrate({ terminalState: "cancelled", toolReceiptLimitStopped: true }); + expect(stopped).toMatchObject({ statusAction: "preserve", toStatus: "in_progress", reasonCode: "semantic_tool_turn_receipt_limit" }); + expect(stopped.effects).toEqual([ + { kind: "schedule_retry", cause: "semantic_tool_turn_receipt_limit", summary: "Continue the task in a fresh session.", agentId: expect.any(String) }, + { kind: "release_run_resources" }, + ]); + expect(arbitrate({ terminalState: "failed", toolReceiptLimitStopped: true }).effects) + .toEqual([expect.objectContaining({ kind: "schedule_retry", cause: "semantic_tool_turn_receipt_limit" })]); + expect(arbitrate({ terminalState: "cancelled", toolReceiptLimitStopped: true, failureRetryCount: 2 }).reasonCode).toBe("cancellation_run_only"); + expect(arbitrate({ terminalState: "cancelled", toolReceiptLimitStopped: true, priorIssueStatus: "blocked" }).reasonCode).toBe("cancellation_run_only"); + expect(arbitrate({ terminalState: "cancelled" }).reasonCode).toBe("cancellation_run_only"); + }); + it("treats only the authorized Board response_wake as passive and preserves governance", () => { const passive = assessment({ reportedDisposition: "yielded", diff --git a/server/src/services/native-runtime/status-arbiter.ts b/server/src/services/native-runtime/status-arbiter.ts index 26e3643e5e2..594f3011fef 100644 --- a/server/src/services/native-runtime/status-arbiter.ts +++ b/server/src/services/native-runtime/status-arbiter.ts @@ -1,6 +1,7 @@ import { COMPLETION_REVIEW_STAGE_REASON, COMPLETION_REVIEW_STAGE_REJECTED_REASON, type CompletionReviewStageEntry } from "../issue-execution-policy.js"; import type { NativeEvidenceAssessment } from "./evidence-classifier.js"; import { NATIVE_MODEL_REJECTION_MESSAGE, NATIVE_PROVIDER_CAPACITY_MAX_RETRIES, NATIVE_PROVIDER_OVERLOADED_CODE, NATIVE_PROVIDER_OVERLOADED_MESSAGE } from "./native-provider-failure.js"; +import { NATIVE_TOOL_RECEIPT_LIMIT_CODE, NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS } from "./native-tool-receipt-limit.js"; export const NATIVE_STATUS_ARBITER_POLICY_VERSION = "phase6-v14"; @@ -256,6 +257,7 @@ export function arbitrateNativeStatus(input: { providerModelRejected?: boolean; providerOverloaded?: boolean; providerFailureSuperseded?: boolean; + toolReceiptLimitStopped?: boolean; failureRetryCount?: number; /** * This run was itself the automatic continuation of an earlier failed turn. @@ -368,6 +370,20 @@ export function arbitrateNativeStatus(input: { ], }; } + if (input.toolReceiptLimitStopped && input.terminalState !== "succeeded" && !input.nativeReviewOutcome && + !input.governanceGate && !input.hasUnresolvedIssueBlockers && !input.hasActivePauseHold && + input.priorIssueStatus !== "blocked" && + (input.failureRetryCount ?? 0) < NATIVE_TOOL_RECEIPT_LIMIT_MAX_CONTINUATIONS) { + return { + policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, + statusAction: "preserve", toStatus: input.priorIssueStatus, + reasonCode: NATIVE_TOOL_RECEIPT_LIMIT_CODE, unblockDescriptor: null, + effects: [ + { kind: "schedule_retry", cause: NATIVE_TOOL_RECEIPT_LIMIT_CODE, summary: "Continue the task in a fresh session.", agentId: input.agentId }, + ...(input.terminalState === "cancelled" ? [{ kind: "release_run_resources" as const }] : []), + ], + }; + } if (input.nativeReviewOutcome) { return { policyVersion: NATIVE_STATUS_ARBITER_POLICY_VERSION, diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index 80da7787b0e..d64421cecc5 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -87,8 +87,9 @@ import { type ActivityPublication, } from "../activity-log.js"; import { emitAgentTaskRun } from "../agent-task-run-telemetry.js"; -import { materializeNativeProviderCapacityRetry } from "./native-provider-capacity-retry.js"; +import { materializeNativeProviderCapacityRetry, materializeNativeToolReceiptLimitContinuation } from "./native-provider-capacity-retry.js"; import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-provider-failure.js"; +import { NATIVE_TOOL_RECEIPT_LIMIT_CODE } from "./native-tool-receipt-limit.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; export class NativeStatusRaceError extends Error { @@ -824,6 +825,19 @@ async function materializeDecisionEffect(input: { summary: effect.summary, }, }; + const [stopped] = effect.idempotencyKey === toolCallLoopContinuationKey(input.runId) + ? await input.tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, effect.agentId), + eq(agentWakeupRequests.idempotencyKey, effect.idempotencyKey), eq(agentWakeupRequests.status, "cancelled"), + )).limit(1) + : []; + if (stopped) + return { + effectKind: effect.kind, + targetType: "agent_wakeup_request", + targetId: stopped.id, + payload: { continuationKind: effect.continuationKind, summary: effect.summary, cancelled: true }, + }; } if (effect.continuationKind === "monitor") throw new Error("Monitor delivery belongs to the issue monitor scheduler"); const wakeId = await enqueueWake({ @@ -895,8 +909,10 @@ async function materializeDecisionEffect(input: { } if (effect.kind === "schedule_retry") { failAt("continuation_materialization", input.failpoint); - if (effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE) { - const retryId = await materializeNativeProviderCapacityRetry({ + if (effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE || effect.cause === NATIVE_TOOL_RECEIPT_LIMIT_CODE) { + const materialize = effect.cause === NATIVE_PROVIDER_OVERLOADED_CODE + ? materializeNativeProviderCapacityRetry : materializeNativeToolReceiptLimitContinuation; + const retryId = await materialize({ tx: input.tx as unknown as Db, companyId: input.companyId, issueId: input.issue.id, runId: input.runId, agentId: effect.agentId, }); @@ -1699,8 +1715,8 @@ export async function commitNativeStatusDecision(input: { if (reasonCode === "native_provider_model_rejected" && !input.requireModelRejectionOwner) { throw new Error("native_model_rejection_owner_required"); } - if ((reasonCode.startsWith("native_provider_overloaded") || reasonCode === "native_run_failed_continuation_exhausted") - && !input.requireProviderFailureOwner) { + if ((reasonCode.startsWith("native_provider_overloaded") || reasonCode === "native_run_failed_continuation_exhausted" + || reasonCode === NATIVE_TOOL_RECEIPT_LIMIT_CODE) && !input.requireProviderFailureOwner) { throw new Error("native_provider_failure_owner_required"); } const publications: ActivityPublication[] = [];