diff --git a/doc/run-log-events.md b/doc/run-log-events.md index 26b84266d8f..2da5478d001 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -359,6 +359,23 @@ attention request. A true conflict includes the call and operation IDs and both result digests in its error; full arguments belong to the canonical semantic input record, not the error message. +When the tool bridge rejects a recorded result for a call it still holds, for +example a result envelope that names another operation or a value over the +retained limit, the runner still settles that call. It returns a bounded, +non-retryable error to the provider and records +`semantic_tool_result_undeliverable` with warning severity, call ID, operation +ID, the rejection reason, and the rejected result's digest. The provider is not +left waiting for its own client timeout, and the call does not block later +turns or runs. + +`run.attach` settles calls that a closed provider turn left pending. The +provider stopped waiting when that turn ended, and the new run's controller +never delivers the old run's results. Each such call is recorded as a +non-retryable failure with an unknown outcome, and the attachment records one +`semantic_tool_result_expired` diagnostic with the call and operation IDs. A +call that a live provider turn still waits on keeps blocking attachment as +`durable_pending_tool_calls`. + Incomplete close emits the bounded `native_session_settlement_incomplete` runner diagnostic. Its evidence includes runner suspension, provider drain, final provider state, pending call/operation/source-event IDs and input digests, diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 7ea574589c3..befe1482b09 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -700,6 +700,15 @@ result replay as `semantic_tool_result_duplicate` in a diagnostic event, with no task failure. Conflict errors identify the call, operation, and both result digests without including the result bodies. +A recorded result that the bridge rejects for a call it still holds settles +that call with a non-retryable runner error, recorded as +`semantic_tool_result_undeliverable`; the provider receives that error +immediately. `run.attach` settles calls left pending by a closed provider turn +as `semantic_tool_result_expired`, an error result with an unknown outcome: the +provider stopped waiting when its turn ended, and the next run's authority +never delivers the previous run's results. Neither settlement executes the +operation again. + The controller checkpoints a reusable session only after every admitted tool has a matching completed result-delivery command and the provider has drained. A callback rejected before business dispatch can report diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index d364d83b045..7e811ce6c7d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -27,8 +27,8 @@ use crate::durable::{ }; use crate::provider_bridge::{ authorized_tool_catalog_digest, semantic_value_digest, AuthorizedToolSet, DurableReplayFilter, - PendingToolCall, ProviderBridgeError, ProviderToolBridge, ToolResult, MAX_PENDING_CALLS, - TOOL_SET_SCHEMA, + PendingToolCall, ProviderBridgeError, ProviderToolBridge, ToolResult, EXPIRED_RESULT_CODE, + MAX_PENDING_CALLS, TOOL_SET_SCHEMA, UNDELIVERABLE_RESULT_CODE, }; use crate::provider_events::{ codex_context_usage, normalize_codex_notification, normalize_codex_notification_in, @@ -3443,7 +3443,9 @@ impl CodexCommandExecutor { }) } - fn attach_run(&mut self, payload: &Value) -> Result<(), DurableRunnerError> { + /// Returns the calls a closed turn left pending that this attachment + /// settled as expired. + fn attach_run(&mut self, payload: &Value) -> Result, DurableRunnerError> { self.retain_completed_accounting_tail()?; let mut next_state = self .state @@ -3473,6 +3475,23 @@ impl CodexCommandExecutor { if next_state.ambiguous_turn_start_pending { blockers.push("durable_ambiguous_turn_start"); } + // A call left pending by a closed provider turn can never be answered: + // the provider stopped waiting when its turn ended, and the new run's + // controller never delivers the old run's results. Record it as an + // expired failure with an unknown outcome so the session continues. + // A call the live provider still waits on keeps blocking attachment. + let expired_tool_calls = if blockers.is_empty() { + next_state + .tool_bridge + .expire_closed_turn_calls() + .map_err(|error| { + DurableRunnerError::invalid(format!( + "run.attach could not settle tool calls left by a closed turn: {error}" + )) + })? + } else { + Vec::new() + }; if next_state.tool_bridge.pending_calls().next().is_some() { blockers.push("durable_pending_tool_calls"); } @@ -3669,7 +3688,7 @@ impl CodexCommandExecutor { }; self.persist_state(&next_state)?; self.state = Some(next_state); - Ok(()) + Ok(expired_tool_calls) } fn open_session(&mut self) -> Result { @@ -5002,15 +5021,54 @@ impl CodexCommandExecutor { .pending_calls() .find(|call| call.call_id == result.call_id) .map(|call| (call.operation_id.clone(), call.input.clone())); - next_state - .tool_bridge - .apply_result(result.clone()) - .map_err(|error| { - DurableRunnerError::invalid(format!( + // A recorded result that this bridge rejects must still settle its + // pending call. Otherwise the provider waits until its own client + // times out, and the call blocks every later turn and run. + let mut undeliverable = None; + let result = match next_state.tool_bridge.apply_result(result.clone()) { + Ok(_) => result, + Err(error) => { + let rejected = DurableRunnerError::invalid(format!( "semantic tool result was rejected for call {} operation {}: {error}", result.call_id, result.operation_id - )) - })?; + )); + if was_completed { + // An exact replay of a delivery whose result was already + // rejected and settled (for example after a crash before + // the command finished) is a duplicate, not a conflict. + if next_state + .tool_bridge + .is_settled_undeliverable_replay(&result) + { + return Ok(CommandExecution { + result: json!({ "status": "duplicate", "callId": result.call_id }), + events: vec![( + "harness.diagnostic".to_owned(), + EventPriority::P0, + json!({ + "code": "semantic_tool_result_duplicate", "severity": "warning", + "callId": result.call_id, "operationId": result.operation_id, + "resultDigest": semantic_value_digest(&result.result), + }), + )], + }); + } + return Err(rejected); + } + match next_state.tool_bridge.settle_undeliverable_result(&result) { + Ok(Some(settlement)) => { + undeliverable = Some(json!({ + "code": UNDELIVERABLE_RESULT_CODE, "severity": "warning", + "callId": result.call_id, "operationId": settlement.operation_id, + "reason": error.to_string(), + "resultDigest": semantic_value_digest(&result.result), + })); + settlement + } + _ => return Err(rejected), + } + } + }; if was_completed { return Ok(CommandExecution { result: json!({ "status": "duplicate", "callId": result.call_id }), @@ -5041,11 +5099,24 @@ impl CodexCommandExecutor { next_state.push_event(semantic_result_event(&identity, &result))?; self.persist_state(&next_state)?; self.state = Some(next_state); + let diagnostics = undeliverable + .map(|diagnostic| { + ( + "harness.diagnostic".to_owned(), + EventPriority::P0, + diagnostic, + ) + }) + .into_iter() + .collect::>(); if delivery_detached { - return Ok(CommandExecution::result(json!({ - "status": "settled_after_turn", - "callId": result.call_id, - }))); + return Ok(CommandExecution { + result: json!({ + "status": "settled_after_turn", + "callId": result.call_id, + }), + events: diagnostics, + }); } let provider = self.ensure_provider()?; if terminal_tool_authoritative { @@ -5062,10 +5133,13 @@ impl CodexCommandExecutor { "failed to return semantic tool result to Codex: {error}" )) })?; - Ok(CommandExecution::result(json!({ - "status": "delivered", - "callId": result.call_id, - }))) + Ok(CommandExecution { + result: json!({ + "status": "delivered", + "callId": result.call_id, + }), + events: diagnostics, + }) } fn stop_owned_provider(&mut self) -> Result<(), DurableRunnerError> { @@ -5915,12 +5989,26 @@ impl CommandExecutor for CodexCommandExecutor { match command.command_type.as_str() { "run.prepare" => self.prepare(&command.payload), "run.attach" => { - if self.state.is_none() && command.payload.get("provider").is_some() { - self.prepare(&command.payload)?; - } else { - self.attach_run(&command.payload)?; - } + let expired = + if self.state.is_none() && command.payload.get("provider").is_some() { + self.prepare(&command.payload)?; + Vec::new() + } else { + self.attach_run(&command.payload)? + }; let mut execution = self.open_session()?; + if !expired.is_empty() { + execution.events.push(( + "harness.diagnostic".to_owned(), + EventPriority::P0, + json!({ + "code": EXPIRED_RESULT_CODE, "severity": "warning", + "calls": expired.iter().map(|call| json!({ + "callId": call.call_id, "operationId": call.operation_id, + })).collect::>(), + }), + )); + } let provider = self .state .as_ref() diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs index e9e99c8c220..b1ab2939aad 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_bridge.rs @@ -897,6 +897,72 @@ impl ProviderToolBridge { self.pending.values() } + /// Settle a pending call whose recorded result this bridge rejected, such + /// as a malformed envelope or an oversized value. The provider then gets a + /// bounded error now instead of waiting on a call nothing can answer, and + /// the call stops blocking later turns. Returns None when the call is not + /// pending, so a conflicting or unknown result still fails as before. + pub fn settle_undeliverable_result( + &mut self, + rejected: &ToolResult, + ) -> Result, ProviderBridgeError> { + let Some(pending) = self.pending.get(&rejected.call_id) else { + return Ok(None); + }; + let mut settlement = runner_tool_failure( + pending, + UNDELIVERABLE_RESULT_CODE, + "Paperclip ran this tool, but its result could not be returned to you. The action may already have completed; do not repeat it. Check the current state with a read tool instead.", + ); + // Bind the settlement to the exact rejected result, so a replay of the + // same delivery after a crash is recognized as a duplicate. + settlement.result["error"]["rejectedResultDigest"] = + Value::String(rejected_result_digest(rejected)); + self.apply_result(settlement.clone())?; + Ok(Some(settlement)) + } + + /// True when this exact result was already rejected and its call settled + /// by `settle_undeliverable_result`. + pub fn is_settled_undeliverable_replay(&self, result: &ToolResult) -> bool { + self.completed + .get(&result.call_id) + .or_else(|| self.settled_results.get(&result.call_id)) + .is_some_and(|entry| { + let error = &entry.result.result["error"]; + entry.result.is_error + && entry.result.operation_id == result.operation_id + && error["code"] == UNDELIVERABLE_RESULT_CODE + && error["rejectedResultDigest"] + == Value::String(rejected_result_digest(result)) + }) + } + + /// Settle the calls a closed provider turn left pending. Their provider + /// turn ended without them, and a new run's authority can never deliver + /// the old run's result, so they would block every later run. Each is + /// recorded as a timed-out failure with an unknown outcome. + pub fn expire_closed_turn_calls(&mut self) -> Result, ProviderBridgeError> { + if !self.turn_closed { + return Ok(Vec::new()); + } + let expired = self + .pending + .values() + .map(|pending| { + runner_tool_failure( + pending, + EXPIRED_RESULT_CODE, + "This Paperclip tool call timed out before its result reached you. Its outcome is unknown: check the current state before repeating any change.", + ) + }) + .collect::>(); + for settlement in &expired { + self.apply_result(settlement.clone())?; + } + Ok(expired) + } + /// Close provider admission/delivery, without inventing an operation outcome. /// This marker is durable, including when interruption beats result delivery. pub fn cancel_pending_calls( @@ -1307,6 +1373,26 @@ where deserializer.deserialize_map(RetainedResultsVisitor) } +pub const UNDELIVERABLE_RESULT_CODE: &str = "semantic_tool_result_undeliverable"; +pub const EXPIRED_RESULT_CODE: &str = "semantic_tool_result_expired"; + +fn rejected_result_digest(result: &ToolResult) -> String { + semantic_value_digest(&serde_json::json!({ + "operationId": result.operation_id, "isError": result.is_error, "result": result.result, + })) +} + +fn runner_tool_failure(pending: &PendingToolCall, code: &str, message: &str) -> ToolResult { + ToolResult { + call_id: pending.call_id.clone(), + operation_id: pending.operation_id.clone(), + result: serde_json::json!({ + "error": { "code": code, "retryable": false, "message": message }, + }), + is_error: true, + } +} + fn conflicting_tool_result(existing: &ToolResult, incoming: &ToolResult) -> ProviderBridgeError { // Correlate both receipts without leaking arguments or result content into // a user-visible error or diagnostic stream. @@ -2262,6 +2348,121 @@ mod tests { .expect("a new durable run resets the full legacy ledger"); } + fn parallel_read_bridge() -> ProviderToolBridge { + let operations = ["list_documents", "list_workflows"] + .map(|operation_id| AuthorizedTool { + operation_id: operation_id.to_owned(), + version: 1, + description: "Read company records.".to_owned(), + input_schema: json!({"type": "object"}), + response_schema: json!({"type": "object", "additionalProperties": true}), + }) + .to_vec(); + let mut bridge = ProviderToolBridge::default(); + bridge + .prepare(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }) + .unwrap(); + for (call_id, operation_id) in [("4", "list_documents"), ("5", "list_workflows")] { + bridge + .begin_call(call_id.to_owned(), operation_id.to_owned(), json!({})) + .unwrap(); + } + bridge + } + + #[test] + fn a_rejected_result_in_a_parallel_pair_still_settles_its_own_call() { + // Chat-eval cohort 13 issued list_documents and list_workflows together. + // The workflow result named its REST route as `operationId`, which the + // bridge reads as a mismatched semantic envelope. + let mut bridge = parallel_read_bridge(); + bridge + .apply_result(ToolResult { + call_id: "4".to_owned(), + operation_id: "list_documents".to_owned(), + result: json!({"documents": []}), + is_error: false, + }) + .unwrap(); + let rejected = ToolResult { + call_id: "5".to_owned(), + operation_id: "list_workflows".to_owned(), + result: json!({"ok": true, "status": 200, + "operationId": "GET /api/companies/{companyId}/workflows", "data": []}), + is_error: false, + }; + let error = bridge.apply_result(rejected.clone()).unwrap_err(); + assert!(error.to_string().contains("envelope does not match")); + assert_eq!(bridge.pending_calls().count(), 1); + + let settlement = bridge + .settle_undeliverable_result(&rejected) + .unwrap() + .expect("the pending call settles"); + assert!(settlement.is_error); + assert_eq!(settlement.operation_id, "list_workflows"); + assert_eq!( + settlement.result["error"]["code"], + UNDELIVERABLE_RESULT_CODE + ); + assert_eq!(bridge.pending_calls().count(), 0); + // Its sibling keeps its own result, and a settled call is not settled twice. + assert_eq!( + bridge + .replay_result("4", "list_documents", &json!({})) + .unwrap() + .unwrap() + .result, + json!({"documents": []}) + ); + assert!(bridge + .settle_undeliverable_result(&rejected) + .unwrap() + .is_none()); + assert!(bridge.is_settled_undeliverable_replay(&rejected)); + let mut different = rejected.clone(); + different.result["data"] = json!(["other"]); + assert!(!bridge.is_settled_undeliverable_replay(&different)); + bridge.settle_turn("provider_turn_terminated").unwrap(); + assert!(bridge.is_settled_undeliverable_replay(&rejected)); + bridge.prepare_turn().expect("the next turn starts"); + } + + #[test] + fn only_a_closed_turn_expires_its_pending_calls() { + let mut bridge = parallel_read_bridge(); + assert!(bridge.expire_closed_turn_calls().unwrap().is_empty()); + assert_eq!(bridge.pending_calls().count(), 2); + + bridge.settle_turn("provider_turn_terminated").unwrap(); + let expired = bridge.expire_closed_turn_calls().unwrap(); + assert_eq!( + expired + .iter() + .map(|call| call.call_id.as_str()) + .collect::>(), + ["4", "5"] + ); + assert!(expired + .iter() + .all(|call| call.is_error && call.result["error"]["code"] == EXPIRED_RESULT_CODE)); + assert_eq!(bridge.pending_calls().count(), 0); + let operations = bridge.authorized_tools().cloned().collect::>(); + bridge + .attach_run(AuthorizedToolSet { + schema: TOOL_SET_SCHEMA.to_owned(), + schema_version: 1, + catalog_digest: authorized_tool_catalog_digest(&operations).unwrap(), + operations, + }) + .expect("the next run attaches"); + } + #[test] fn active_receipts_that_block_the_next_turn_are_named() { let mut bridge = completion_bridge(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 629f117c7d1..225108ceba3 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -5853,6 +5853,176 @@ fn durable_backend_settles_tools_before_a_natural_terminal_event() { fs::remove_dir_all(directory).unwrap(); } +fn start_turn_with_pending_tool(executor: &mut CodexCommandExecutor, config: CodexProviderConfig) { + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider": config, "authorizedTools": task_context_tool_set()}), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text": "Call the tool."}), + )) + .unwrap(); +} + +#[test] +fn a_rejected_recorded_result_still_answers_the_provider_and_frees_the_session() { + // Chat-eval cohort 13: the controller recorded a REST-shaped result whose + // `{ ok, operationId }` the bridge reads as a mismatched semantic envelope. + // The rejected delivery left OpenCode waiting ~290 s and the call pending, + // so the next run's run.attach failed with durable_pending_tool_calls. + let directory = temporary_directory("rejected-recorded-result"); + let config = provider_config(&directory, &["--require-dynamic-tool", "--emit-tool-call"]); + let mut executor = + CodexCommandExecutor::with_runner_config(&directory, &durable_config(&directory)); + start_turn_with_pending_tool(&mut executor, config); + let deadline = event_deadline(); + let mut input_seen = false; + while !input_seen && std::time::Instant::now() < deadline { + input_seen = poll_and_ack(&mut executor) + .unwrap() + .iter() + .any(|event| event.event_type == "semantic_tool.input"); + } + assert!(input_seen, "the provider requests its tool"); + + let rejected = json!({ + "callId": "semantic-call-1", "operationId": "get_task_context", "isError": false, + "result": {"ok": true, "status": 200, "operationId": "GET /api/companies/{companyId}/workflows", + "contentType": "application/json; charset=utf-8", "data": []}, + }); + let delivered = executor + .execute(&command( + "result", + 4, + "semantic_tool.result", + rejected.clone(), + )) + .expect("a recorded result must settle its call even when the bridge rejects it"); + assert_eq!(delivered.result["status"], "delivered"); + let mut observed = Vec::new(); + let deadline = event_deadline(); + while !observed.iter().any(|event: &PolledEvent| { + event.event_type == "turn.failed" || event.event_type == "turn.completed" + }) && std::time::Instant::now() < deadline + { + observed.extend(poll_and_ack(&mut executor).unwrap()); + } + let diagnostic = delivered + .events + .iter() + .find(|(_, _, payload)| payload["code"] == "semantic_tool_result_undeliverable") + .map(|(_, _, payload)| payload) + .expect("the rejected delivery is named"); + assert_eq!(diagnostic["callId"], "semantic-call-1"); + assert!(diagnostic["reason"] + .as_str() + .unwrap() + .contains("envelope does not match")); + let settled = observed + .iter() + .find(|event| event.event_type == "semantic_tool.result") + .expect("the call has a recorded settlement"); + assert_eq!(settled.payload["semantic_tool"]["outcome"], "failed"); + let calls = fs::read_to_string(directory.join("calls.log")).unwrap(); + assert!( + calls.contains("tool-response:failure"), + "the provider must receive an answer, not wait for its client timeout: {calls}" + ); + + // A crash before the command finished replays the same delivery. + let replay = executor + .execute(&command( + "result-replay", + 5, + "semantic_tool.result", + rejected, + )) + .expect("an exact replay of the rejected delivery is a duplicate"); + assert_eq!(replay.result["status"], "duplicate"); + + executor + .execute(&command( + "attach", + 6, + "run.attach", + json!({"authorizedTools": task_context_tool_set()}), + )) + .expect("the next run attaches to the same session"); + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn run_attach_expires_a_call_left_pending_by_a_closed_turn() { + // The provider stopped waiting when its turn ended, and the next run's + // controller never delivers the old run's result. Attachment records the + // call as expired instead of refusing every later run. + let directory = temporary_directory("closed-turn-pending-attach"); + let config = provider_config( + &directory, + &[ + "--require-dynamic-tool", + "--emit-tool-call", + "--finish-turn-with-pending-tool", + ], + ); + let runner_config = durable_config(&directory); + let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + start_turn_with_pending_tool(&mut first, config); + let mut observed = Vec::new(); + let deadline = event_deadline(); + while !observed.iter().any(|event| event == "turn.completed") + && std::time::Instant::now() < deadline + { + observed.extend( + poll_and_ack(&mut first) + .unwrap() + .into_iter() + .map(|event| event.event_type), + ); + } + assert!(observed.iter().any(|event| event == "semantic_tool.input")); + assert!(!observed.iter().any(|event| event == "semantic_tool.result")); + // A deploy lands here: the next Runner restores this saved state. + first.shutdown().unwrap(); + drop(first); + + let mut resumed = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let attached = resumed + .execute(&command( + "attach", + 4, + "run.attach", + json!({"authorizedTools": task_context_tool_set()}), + )) + .expect("a call left by a closed turn must not block the next run"); + let expired = attached + .events + .iter() + .find(|(event_type, _, payload)| { + event_type == "harness.diagnostic" && payload["code"] == "semantic_tool_result_expired" + }) + .map(|(_, _, payload)| payload.clone()) + .expect("the expired call is named"); + assert_eq!( + expired["calls"], + json!([{"callId": "semantic-call-1", "operationId": "get_task_context"}]) + ); + resumed.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + #[test] fn durable_terminal_delivery_reconciliation_does_not_launch_a_cold_provider() { let directory = temporary_directory("cold-terminal-delivery"); diff --git a/server/src/services/native-runtime/native-case-tools.integration.test.ts b/server/src/services/native-runtime/native-case-tools.integration.test.ts index 16e85ab18a5..ed18e6a9930 100644 --- a/server/src/services/native-runtime/native-case-tools.integration.test.ts +++ b/server/src/services/native-runtime/native-case-tools.integration.test.ts @@ -154,7 +154,16 @@ describe("native case tools through authenticated production routes", () => { expect(created).toMatchObject({ ok: true, status: 201, data: { name: input.name } }); expect(created.data).not.toHaveProperty("prompt"); expect(await f.call("create_workflow", input)).toEqual(created); - expect(await f.call("list_workflows")).toMatchObject({ ok: true, data: [{ id: created.data.id, name: input.name }] }); + const listed = await f.call("list_workflows"); + expect(listed).toMatchObject({ ok: true, data: [{ id: created.data.id, name: input.name }] }); + // The runner reads `{ ok, operationId }` as a semantic envelope for this + // exact tool and call. A REST route there makes the runner reject the + // result, so the agent never receives it (chat-eval cohort 13). + for (const [result, route] of [[created, "POST /api/companies/{companyId}/workflows"], + [listed, "GET /api/companies/{companyId}/workflows"]] as const) { + expect(result).not.toHaveProperty("operationId"); + expect(result).toMatchObject({ apiOperationId: route }); + } let offsetBytes = 0, text = "", snapshot: string | undefined; for (let i = 0; ; i++) { expect(i).toBeLessThan(20); @@ -162,6 +171,8 @@ describe("native case tools through authenticated production routes", () => { ok: boolean; data: string; snapshot: string; responseText: { nextOffsetBytes: number | null }; }; expect(page.ok).toBe(true); + expect(page).toMatchObject({ apiOperationId: "GET /api/workflows/{id}" }); + expect(page).not.toHaveProperty("operationId"); expect(page.snapshot).toMatch(/^[a-f0-9]{64}$/); snapshot ??= page.snapshot; expect(page.snapshot).toBe(snapshot); @@ -175,7 +186,9 @@ describe("native case tools through authenticated production routes", () => { workflowId: created.data.id, kickoff: { toAgentId: f.agentId, phase: "custom phase", instructions: "Read workflow" } }) as { taskId: string; caseId: string }; expect((await documentService(server.db).getIssueDocumentByKey(receipt.taskId, "workflow"))?.body).toBe(prompt); await expect(f.call("read_workflow", { workflowId: created.data.id, responseText: { offsetBytes: 8192 } })).rejects.toThrow("Invalid tool input for read_workflow: A nonzero responseText.offsetBytes continues an earlier read: also send the snapshot it returned"); - expect(await f.call("update_workflow", { workflowId: created.data.id, idempotencyKey: "workflow-edit", prompt: prompt.replaceAll("Reusable", "Adjusted") })).toMatchObject({ ok: true }); + const updated = await f.call("update_workflow", { workflowId: created.data.id, idempotencyKey: "workflow-edit", prompt: prompt.replaceAll("Reusable", "Adjusted") }); + expect(updated).toMatchObject({ ok: true, apiOperationId: "PATCH /api/workflows/{id}" }); + expect(updated).not.toHaveProperty("operationId"); expect(await f.call("read_workflow", { workflowId: created.data.id, snapshot, responseText: { offsetBytes: 8192 } })).toMatchObject({ ok: false, status: 409 }); const restarted = await f.call("read_workflow", { workflowId: created.data.id, responseText: { limitBytes: 4 } }) as { ok: boolean; data: string; snapshot: string }; expect(restarted.ok).toBe(true); diff --git a/server/src/services/native-runtime/native-workflow-tools.ts b/server/src/services/native-runtime/native-workflow-tools.ts index 54cc3fbcffa..59209a1d0d8 100644 --- a/server/src/services/native-runtime/native-workflow-tools.ts +++ b/server/src/services/native-runtime/native-workflow-tools.ts @@ -38,9 +38,16 @@ export async function executeNativeWorkflowTool(input: { ...(input.tool === "read_workflow" ? { responseText: responseText ?? { offsetBytes: 0 } } : {}), }; const key = idempotencyKey ?? createHash("sha256").update(JSON.stringify(request)).digest("hex"); - const result = await input.callApi(`workflow-tool:${input.tool}:${key}`, request); - if (input.tool !== "read_workflow" || !result || typeof result !== "object") return result; - const etag = "etag" in result && typeof result.etag === "string" ? result.etag : ""; + const raw = await input.callApi(`workflow-tool:${input.tool}:${key}`, request); + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw; + // Name the REST route apiOperationId, as every other API-backed tool does. + // The runner reads `{ ok, operationId }` as a semantic result envelope that + // must name this tool and call. A REST route there is a mismatch, so the + // result is rejected and never reaches the agent. + const { operationId, ...response } = raw as Record; + const result: Record = { ...response, apiOperationId: operationId }; + if (input.tool !== "read_workflow") return result; + const etag = typeof result.etag === "string" ? result.etag : ""; const snapshot = /^(?:W\/)?"([a-f0-9]{64})"$/.exec(etag)?.[1]; return { ...result, ...(snapshot ? { snapshot } : {}), guidance: "Concatenate data JSON text pages using read_workflow with the same workflowId, returned snapshot, and responseText.offsetBytes=responseText.nextOffsetBytes until null. A 409 conflict requires restarting at offset zero without snapshot. Adoption captures the latest prompt at handoff acceptance." }; }