diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6efb240dc2b..dceb36391cd 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1539,6 +1539,18 @@ does not prove that the provider stopped or that a retry is safe; it remains non-recoverable until the existing ownership and recovery checks establish the next permitted action. +Provider rejections additionally use the versioned `ProviderFailure` cause. +The provider adapter extracts and redacts the original explanation before the +process boundary. Startup command receipts and failed turns retain the same +contract through Runner, persistence, and restart finalization. The task's current +interruption and the historical run detail show this explanation, including when +its category is unknown. Later cleanup errors do not erase the saved cause. +Task read projections reapply registered run-secret and current-user redaction +before returning provider text to detail or list views. +The category and retryability describe provider evidence; they do not grant +retry authority, prove stopped execution, or change the existing recovery budget. +An HTTP 429 alone does not establish a quota reset time or a safe retry. + Local Codex crash replacement can use a complete interrupted-turn inventory, authenticated process-stop evidence, and unchanged retained-state fingerprints. Only text and an exactly receipted task-completion call qualify for this path; diff --git a/doc/run-log-events.md b/doc/run-log-events.md index e2bf5455176..ed4dbc3a3f4 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -92,6 +92,17 @@ They also become native Observability spans only with an operator OTLP endpoint. ## Native PRP Run-Log Events +Provider startup command failures and failed provider turns can carry a +`providerFailure` cause with schema `paperclip.provider-failure.v1`. The cause +retains a normalized category, a bounded and redacted provider explanation, the +startup/turn phase, and available status, code, request ID, or retry time. Provider +adapters extract these fields; arbitrary exception objects, request bodies, +headers, environment variables, and stacks are excluded. Unknown categories retain +their explanation. The cause stays in company-scoped run records and the run log; +it is not added to first-party Telemetry, Observability attributes, or Sentry +failure reports. The reportable run error keeps its existing generic text. Startup +receipts retain it across controller restart before any provider turn exists. + Completed OpenCode builtin `bash` parts preserve the numeric process exit from `state.metadata.exit` in `tool.execution.completed.payload.exitCode`. The tool status stays `completed` even when the command exits with a nonzero code. This diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index 9c9f142b7ff..2b8bff89cca 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -255,6 +255,14 @@ Capability ID: `runs-adapters/runs`. Review: `baseline`. +Run reads and task interruption projections can include a sanitized +`paperclip.provider-failure.v1` cause. Existing company and task/run read policy +applies. The cause exposes no credential values or new secret access and grants +no retry authority. Startup receipts and failed-turn events use the same cause; +unknown categories retain their explanation. Transport/controller restart and +PostgreSQL finalization tests cover retention. These tests do not independently +qualify actor access through the routes. + Scoped source-only timing evidence: workspace preparation spans carry only the existing hashed run ID. Controller startup phase spans carry no custom attributes. Native controller setup and settlement phases use that same run hash, including diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index ce07cbb01bd..3a4056e0085 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -344,6 +344,14 @@ a separate acceptance requirement. Automated: [live run routes](../../server/src/__tests__/agent-live-run-routes.test.ts) and [run event sequencing](../../server/src/__tests__/heartbeat-run-event-sequencing.test.ts) cover run data contracts. +For `runs-adapters/runs`, verify a provider rejection during startup and during a +turn. The task interruption and run detail must show the same sanitized provider +explanation after reload. Unknown provider categories keep their message. A newer +attempt or a completed task hides the stale task notice, while historical run +details retain it. Check that existing retry and access gates remain effective. +The transport restart regression and PostgreSQL finalization integration cover +the persisted cause; component tests cover plain-text rendering and read-only use. + Manual: Start a disposable task, open its running transcript, and follow it to a terminal outcome. Reload and compare task/run IDs, status, output, and cost. Verify a failed run remains distinguishable from a completed run with useful output. For Legacy OpenCode on a disposable Linux sandbox, run a Bash tool that reads diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index ba573e65c6c..ae8b5a3ad0d 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -99,8 +99,8 @@ "record:sdk:codex": "pnpm run build:typescript && playwright test --config examples/playwright.real.config.ts", "generate:protocol-manifest": "node scripts/generate-protocol-manifest.mjs", "check:protocol-manifest": "node scripts/generate-protocol-manifest.mjs --check", - "generate:protocol-types": "node scripts/generate-protocol-schema-module.mjs && node scripts/generate-acpx-sidecar-contract.mjs && node scripts/generate-runtime-context-contract.mjs", - "check:protocol-types": "node scripts/generate-protocol-schema-module.mjs --check && node scripts/generate-acpx-sidecar-contract.mjs --check && node scripts/generate-runtime-context-contract.mjs --check", + "generate:protocol-types": "node scripts/generate-protocol-schema-module.mjs && node scripts/generate-acpx-sidecar-contract.mjs && node scripts/generate-runtime-context-contract.mjs && node scripts/generate-provider-failure-contract.mjs", + "check:protocol-types": "node scripts/generate-protocol-schema-module.mjs --check && node scripts/generate-acpx-sidecar-contract.mjs --check && node scripts/generate-runtime-context-contract.mjs --check && node scripts/generate-provider-failure-contract.mjs --check", "generate:capability-contract": "node scripts/generate-capability-contract.mjs", "check:capability-contract": "node scripts/generate-capability-contract.mjs --check", "generate:semantic-contracts": "pnpm run build:typescript && node scripts/generate-semantic-contracts.mjs", diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json index c91793e19e3..c060b075bea 100644 --- a/packages/paperclip-runner/protocol/manifest.json +++ b/packages/paperclip-runner/protocol/manifest.json @@ -92,6 +92,11 @@ "id": "https://paperclip.dev/schemas/prp/v1/provider-event.schema.json", "sha256": "14143d53f28969fa3f940def7331a45ee0eed560dd343506ac41b6a3dfe59985" }, + { + "path": "schemas/provider-failure.schema.json", + "id": "https://paperclip.dev/schemas/prp/v1/provider-failure.schema.json", + "sha256": "d8bc6a9a165d48c125aa0b2193f6dc212450156086f17ef852f93a3a4c260dd0" + }, { "path": "schemas/question-adapter-fixture.schema.json", "id": "https://paperclip.dev/schemas/prp/v1/question-adapter-fixture.schema.json", diff --git a/packages/paperclip-runner/protocol/provider-schemas/acpx-sidecar.schema.json b/packages/paperclip-runner/protocol/provider-schemas/acpx-sidecar.schema.json index 46ad48aab61..bd1717d102f 100644 --- a/packages/paperclip-runner/protocol/provider-schemas/acpx-sidecar.schema.json +++ b/packages/paperclip-runner/protocol/provider-schemas/acpx-sidecar.schema.json @@ -197,7 +197,8 @@ "properties": { "code": { "type": "string", "minLength": 1, "maxLength": 160 }, "message": { "type": "string", "maxLength": 8192 }, - "retryable": { "type": "boolean" } + "retryable": { "type": "boolean" }, + "providerFailure": { "$ref": "https://paperclip.dev/schemas/prp/v1/provider-failure.schema.json" } }, "additionalProperties": false } diff --git a/packages/paperclip-runner/protocol/schemas/provider-failure.schema.json b/packages/paperclip-runner/protocol/schemas/provider-failure.schema.json new file mode 100644 index 00000000000..8a3ed526027 --- /dev/null +++ b/packages/paperclip-runner/protocol/schemas/provider-failure.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://paperclip.dev/schemas/prp/v1/provider-failure.schema.json", + "title": "Provider failure", + "type": "object", + "required": ["schema", "provider", "phase", "category", "message", "retryability"], + "properties": { + "schema": { "const": "paperclip.provider-failure.v1" }, + "provider": { "type": "string", "minLength": 1, "maxLength": 80, "pattern": "^[a-zA-Z0-9_.:/-]+$" }, + "phase": { "enum": ["startup", "turn"] }, + "category": { "enum": ["rate_limit", "quota_exhausted", "authentication", "permission", "model_unavailable", "invalid_request", "overloaded", "timeout", "unknown"] }, + "message": { "type": "string", "minLength": 1, "maxLength": 4000 }, + "providerCode": { "type": "string", "minLength": 1, "maxLength": 160, "pattern": "^[a-zA-Z0-9_.:/-]+$" }, + "httpStatus": { "type": "integer", "minimum": 400, "maximum": 599 }, + "requestId": { "type": "string", "minLength": 1, "maxLength": 160, "pattern": "^[a-zA-Z0-9_.:/-]+$" }, + "retryAt": { "type": "string", "format": "date-time" }, + "retryability": { "enum": ["transient", "requires_action", "unknown"] } + }, + "additionalProperties": false +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index e614aea8880..1065f6fa223 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -1111,6 +1111,7 @@ impl AcpxCommandExecutor { )?; let mut session = AcpxProviderSession::start(&config).map_err(|error| { DurableRunnerError::invalid(format!("failed to start ACPX provider: {error}")) + .with_provider_failure(error.provider_failure()) })?; if session.identity().profile_digest != state.descriptor.command_digest { let _ = session.shutdown("qualified ACPX profile digest mismatch"); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 3fe3a59c2ee..b22e7c9126b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -517,6 +517,7 @@ impl AcpxSidecarTransport { )); } let error = response.error.expect("failed response has validated error"); + let provider_failure = error.provider_failure.clone(); return Ok(CommandOutcome::Rejected(LocalRunnerError::invalid( format!( "ACPX sidecar command {} was rejected (retryable={}, classification={})", @@ -524,7 +525,7 @@ impl AcpxSidecarTransport { error.retryable, response_error_classification(&error), ), - ))); + ).with_provider_failure(provider_failure))); } } } @@ -724,11 +725,13 @@ struct ResponseFrame { } #[derive(Deserialize)] -#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] struct ResponseError { code: String, message: String, retryable: bool, + #[serde(default)] + provider_failure: Option, } #[derive(Deserialize)] @@ -1176,6 +1179,7 @@ mod tests { "private-token-canary COPILOT_AUTH_REQUIRED https://user:secret@example.invalid" .to_owned(), retryable: false, + provider_failure: None, }; let classification = response_error_classification(&error); assert_eq!(classification, expected); @@ -1190,6 +1194,7 @@ mod tests { code: code.to_owned(), message: message.to_owned(), retryable: false, + provider_failure: None, }; for (code, expected) in [ ("ACPX_ADMISSION_SANDBOX", "admission_sandbox"), diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 7670e86b943..abdcd69411b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -3270,9 +3270,21 @@ impl CodexProvider { format!("Codex {method} returned an invalid error response"), ))); } - return Err(ProviderRequestError::Rejected(LocalRunnerError::invalid( - format!("Codex {method} failed: {}", redact_text(&error.to_string())), - ))); + return Err(ProviderRequestError::Rejected( + LocalRunnerError::invalid(format!( + "Codex {method} failed: {}", + redact_text(&error.to_string()) + )) + .with_provider_failure( + if self.config.provider == "codex" + && matches!(method, "initialize" | "thread/start" | "thread/resume") + { + crate::provider_failure::codex_startup_failure(error) + } else { + None + }, + ), + )); } return Ok(message.get("result").cloned().unwrap_or(Value::Null)); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs index 7c72e5e372c..a894798b303 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/mod.rs @@ -38,11 +38,20 @@ pub fn redact_diagnostic_text(input: &str) -> String { } #[derive(Clone, Debug, PartialEq, Eq)] -pub struct DurableRunnerError(String); +pub struct DurableRunnerError(String, Option); impl DurableRunnerError { pub fn invalid(message: impl Into) -> Self { - Self(message.into()) + Self(message.into(), None) + } + + pub fn with_provider_failure(mut self, value: Option) -> Self { + self.1 = crate::provider_failure::validated_provider_failure(value); + self + } + + pub fn provider_failure(&self) -> Option { + self.1.clone() } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs index f09940c7769..ffb7b00a5f1 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs @@ -1919,13 +1919,14 @@ fn process_command( "{} command failed: {message}", command.command_type )); - let result = state.fail_command( - command, - json!({ - "code": "command_execution_failed", - "message": message, - }), - )?; + let mut failure = json!({ + "code": "command_execution_failed", + "message": message, + }); + if let Some(provider_failure) = error.provider_failure() { + failure["providerFailure"] = provider_failure; + } + let result = state.fail_command(command, failure)?; store.save(state)?; return Ok((result, CommandLifecycle::for_terminal(command))); } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs index 8b063e95c93..50875c5d446 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs @@ -31,6 +31,7 @@ pub mod process_supervisor; pub mod provider_backend; pub mod provider_bridge; pub mod provider_events; +pub mod provider_failure; pub mod qualified_launch; pub mod question_response; pub mod replay; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/local_runner.rs b/packages/paperclip-runner/runner/crates/runner-core/src/local_runner.rs index d98ccb04665..839322c0097 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/local_runner.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/local_runner.rs @@ -22,11 +22,20 @@ const HARNESS_MESSAGE_SCHEMA: &str = "paperclip.fake_harness.message.v1"; const CONTROLLER_COMMAND_QUEUE_CAPACITY: usize = 256; #[derive(Clone, Debug, PartialEq, Eq)] -pub struct LocalRunnerError(String); +pub struct LocalRunnerError(String, Option); impl LocalRunnerError { pub fn invalid(message: impl Into) -> Self { - Self(message.into()) + Self(message.into(), None) + } + + pub fn with_provider_failure(mut self, value: Option) -> Self { + self.1 = crate::provider_failure::validated_provider_failure(value); + self + } + + pub fn provider_failure(&self) -> Option { + self.1.clone() } } diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 6a7c12a8d3b..2adaafd0420 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -1907,6 +1907,7 @@ impl CodexCommandExecutor { "failed to start {} provider: {error}", state.config.provider )) + .with_provider_failure(error.provider_failure()) }) } @@ -2051,6 +2052,7 @@ impl CodexCommandExecutor { DurableRunnerError::invalid(format!( "failed to resume {provider_name} provider: {error}" )) + .with_provider_failure(error.provider_failure()) })?; let admission = (|| -> Result<(), DurableRunnerError> { provider.enable_durable_tool_call_replays(); @@ -2535,6 +2537,7 @@ impl CodexCommandExecutor { .start_observed_provider(ProviderStartupTrigger::Ensure, process_generation) .map_err(|error| { DurableRunnerError::invalid(format!("failed to start Codex provider: {error}")) + .with_provider_failure(error.provider_failure()) })?; let admission = (|| -> Result<(), DurableRunnerError> { provider.enable_durable_tool_call_replays(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_failure.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_failure.rs new file mode 100644 index 00000000000..540293946b1 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_failure.rs @@ -0,0 +1,160 @@ +use serde_json::{json, Value}; +use std::sync::OnceLock; + +pub fn validated_provider_failure(value: Option) -> Option { + static VALIDATOR: OnceLock = OnceLock::new(); + let value = value?; + let validator = VALIDATOR.get_or_init(|| { + let schema: Value = serde_json::from_str(include_str!( + "../../../../protocol/schemas/provider-failure.schema.json" + )) + .expect("provider failure schema is valid JSON"); + jsonschema::validator_for(&schema).expect("provider failure schema compiles") + }); + validator.is_valid(&value).then_some(value) +} + +pub(crate) fn codex_startup_failure(error: &Value) -> Option { + let original = error.get("message")?.as_str()?; + let response = (original.len() <= 16_000) + .then(|| serde_json::from_str::(original).ok()) + .flatten() + .unwrap_or(Value::Null); + let detail = response.get("error").unwrap_or(error); + let mut secrets: Vec = std::env::vars() + .filter(|(key, value)| { + let key = key.to_ascii_lowercase(); + !value.is_empty() + && [ + "token", + "secret", + "password", + "api_key", + "apikey", + "authorization", + "credential", + ] + .iter() + .any(|word| key.contains(word)) + }) + .flat_map(|(_, value)| { + let escaped = serde_json::to_string(&value).unwrap_or_default(); + let encoded: String = value + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || b"-_.!~*'()".contains(&byte) { + char::from(byte).to_string() + } else { + format!("%{byte:02X}") + } + }) + .collect(); + vec![value, escaped[1..escaped.len() - 1].to_owned(), encoded] + }) + .collect(); + secrets.sort_unstable_by_key(|value| std::cmp::Reverse(value.len())); + let sanitize = |text: &str| -> String { + let mut text = text.to_owned(); + for secret in &secrets { + if !secret.is_empty() { + text = text.replace(secret, "[REDACTED]"); + } + } + crate::durable::redact_sensitive_text_values(&text) + .chars() + .filter(|character| { + (!character.is_control() || matches!(character, '\n' | '\t')) + && !matches!(character, '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}') + }) + .take(4000) + .collect::() + .trim() + .to_owned() + }; + let token = |value: Option<&Value>| { + value + .and_then(Value::as_str) + .filter(|text| { + !text.is_empty() + && text.len() <= 160 + && text + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.:/-".contains(&byte)) + && sanitize(text) == *text + }) + .map(str::to_owned) + }; + let code = token(detail.get("code").or_else(|| detail.get("type"))); + let status = response + .get("status") + .or_else(|| error.pointer("/data/httpStatusCode")) + .and_then(Value::as_u64) + .filter(|status| (400..=599).contains(status)); + let category = match code.as_deref() { + Some( + "usageLimitExceeded" + | "insufficient_quota" + | "credit_balance_exhausted" + | "organization_spend_limit_exceeded" + | "project_spend_limit_exceeded" + | "organization_usage_limit_exceeded", + ) => "quota_exhausted", + Some("rate_limit_exceeded" | "rate_limit_error" | "slow_down") => "rate_limit", + Some("serverOverloaded" | "server_is_overloaded") => "overloaded", + Some("unauthorized" | "invalid_api_key") => "authentication", + Some("model_not_found") => "model_unavailable", + _ => match status { + Some(429) => "rate_limit", + Some(401) => "authentication", + Some(403) => "permission", + Some(503 | 529) => "overloaded", + Some(408 | 504) => "timeout", + _ => "unknown", + }, + }; + let mut failure = json!({ "schema": "paperclip.provider-failure.v1", "provider": "openai", "phase": "startup", + "category": category, "message": sanitize(detail.get("message").and_then(Value::as_str).unwrap_or(original)), + "retryability": match category { "quota_exhausted" | "authentication" | "permission" | "model_unavailable" | "invalid_request" => "requires_action", + "overloaded" | "timeout" => "transient", _ => "unknown" } }); + if let Some(code) = code { + failure["providerCode"] = json!(code); + } + if let Some(status) = status { + failure["httpStatus"] = json!(status); + } + if let Some(request_id) = token(response.get("request_id")) { + failure["requestId"] = json!(request_id); + } + validated_provider_failure(Some(failure)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codex_startup_keeps_provider_fields_without_response_bodies_or_credentials() { + let error = json!({ "code": -32603, "message": json!({ "status": 429, + "error": { "code": "insufficient_quota", "message": "Allowance exhausted. Bearer fixture-credential" }, + "request_id": "req_fixture", "responseBody": "private body marker" }).to_string() }); + let failure = codex_startup_failure(&error).unwrap(); + assert_eq!(failure["category"], "quota_exhausted"); + assert_eq!(failure["httpStatus"], 429); + assert_eq!(failure["requestId"], "req_fixture"); + assert!(failure["message"] + .as_str() + .unwrap() + .contains("Allowance exhausted.")); + assert!(!failure.to_string().contains("fixture-credential")); + assert!(!failure.to_string().contains("private body marker")); + } + + #[test] + fn unknown_codex_startup_errors_keep_bounded_unicode_text() { + let failure = + codex_startup_failure(&json!({ "code": -32603, "message": "šŸš€".repeat(6000) })) + .unwrap(); + assert_eq!(failure["category"], "unknown"); + assert_eq!(failure["message"].as_str().unwrap().chars().count(), 4000); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 581e9de1ac5..c74854a127d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -234,6 +234,7 @@ fn failed_provider_startup_is_persistently_fenced_before_another_process_can_res .execute(&command("open", 2, "session.open", json!({}))) .unwrap_err(); assert!(failure.to_string().contains("no rollout found")); + assert_eq!(failure.provider_failure().unwrap()["provider"], "openai"); let persisted: Value = serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap()) .unwrap(); diff --git a/packages/paperclip-runner/scripts/generate-provider-failure-contract.mjs b/packages/paperclip-runner/scripts/generate-provider-failure-contract.mjs new file mode 100644 index 00000000000..670ab36903f --- /dev/null +++ b/packages/paperclip-runner/scripts/generate-provider-failure-contract.mjs @@ -0,0 +1,23 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +const root = new URL("../", import.meta.url); +const schema = JSON.parse(await readFile(new URL("protocol/schemas/provider-failure.schema.json", root), "utf8")); +const properties = schema.properties; +const declarations = Object.entries(properties).map(([key, field]) => { + const type = "const" in field ? JSON.stringify(field.const) + : field.enum ? field.enum.map(value => JSON.stringify(value)).join(" | ") + : field.type === "integer" ? "number" : "string"; + return ` ${key}${schema.required.includes(key) ? "" : "?"}: ${type};`; +}).join("\n"); +const generated = `export interface ProviderFailure {\n${declarations}\n}\n\nconst fields = ${JSON.stringify(properties, null, 2)} as const;\n\nexport function readProviderFailure(value: unknown): ProviderFailure | null {\n try {\n if (!value || typeof value !== "object" || Array.isArray(value)) return null;\n const input = value as Record;\n const output: Record = {};\n for (const [key, field] of Object.entries(fields)) {\n const item = input[key];\n if (item === undefined && !${JSON.stringify(schema.required)}.includes(key)) continue;\n if ("const" in field) {\n if (item !== field.const) return null;\n } else if ("enum" in field) {\n if (typeof item !== "string" || !(field.enum as readonly string[]).includes(item)) return null;\n } else if (field.type === "integer") {\n if (typeof item !== "number" || !Number.isInteger(item) || item < field.minimum || item > field.maximum) return null;\n } else {\n if (typeof item !== "string") return null;\n if ("maxLength" in field && ([...item].length < field.minLength || [...item].length > field.maxLength)) return null;\n if ("pattern" in field && !new RegExp(field.pattern).test(item)) return null;\n if ("format" in field && !/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z$/.test(item)) return null;\n if ("format" in field && !Number.isFinite(Date.parse(item))) return null;\n }\n output[key] = item;\n }\n return output as unknown as ProviderFailure;\n } catch { return null; }\n}\n`; +const paths = [new URL("src/contracts/provider-failure.generated.ts", root), + new URL("../shared/src/provider-failure.generated.ts", root)]; +for (const path of paths) { + if (process.argv.includes("--check")) { + if (await readFile(path, "utf8").catch(() => "") !== generated) { + process.stderr.write(`Provider failure contract is stale: ${fileURLToPath(path)}\n`); + process.exitCode = 1; + } + } else await writeFile(path, generated); +} diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts index 5543589604d..5a96c9aebea 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -143,7 +143,7 @@ describe("qualified ACPX runtime sidecar", () => { it.each(["projection", "validation", "emission"])("preserves standard usage when optional Cursor notice %s fails", async failure => { const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8"); const start = source.indexOf(" try {\n const usageAfter = await readSidecarHostStatusWithin(activeHost);"); - const end = source.indexOf(" terminal = boundedSidecarValue(result);", start); + const end = source.indexOf(" terminal = openParams ? acpxTerminalResult(", start); expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); const emitted: unknown[] = [], diagnostics: unknown[] = []; const after = { lastRequestId: "request-1", requestTokenUsage: { "prompt-1": { input_tokens: 12, output_tokens: 3 } } }; diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index da96cf45300..155f963fd99 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -2,6 +2,8 @@ import { resolveGrokDisableWebSearch } from "../drivers/acpx/grok-web-search.js"; import { redactObservabilityDiagnostic } from "../drivers/agent-observability.js"; import { resolveClaudeEffort } from "../drivers/acpx/claude-effort.js"; +import { acpxProviderFailure, acpxTerminalResult } from "../drivers/acpx/provider-failure.js"; +import { providerFailureFromError } from "../drivers/provider-failure.js"; import { resolveGrokEffort } from "../drivers/acpx/grok-effort.js"; import { parseProviderMode } from "../contracts/provider-mode.js"; import { assertAcpxExecutionAdmission, parseAcpxCandidateStartupAdmission, type AcpxCandidateStartupAdmission } from "../drivers/acpx/candidate-admission.js"; @@ -251,6 +253,8 @@ async function receiveLine(line: string): Promise { ), message: safeMessage(normalized), retryable: normalizedRecord.retryable === true, + ...(request.command === "session.open" && initializedAgent + ? { providerFailure: providerFailureFromError(error) ?? acpxProviderFailure(error, initializedAgent, "startup", process.env) ?? undefined } : {}), }); } } @@ -712,7 +716,7 @@ async function pumpTurn( // not replace the provider's authoritative completed/cancelled result. diagnostic("acpx_terminal_usage_unavailable", safeMessage(error)); } - terminal = boundedSidecarValue(result); + terminal = openParams ? acpxTerminalResult(result, openParams.agent, process.env) : boundedSidecarValue(result); } catch (error) { terminal = { status: "failed", @@ -720,6 +724,7 @@ async function pumpTurn( ...(error instanceof AcpxApprovalRequiredError || error instanceof AcpxProviderProcessLostError ? { code: error.code } : {}), message: safeMessage(error), retryable: false, + ...(openParams ? { providerFailure: acpxProviderFailure(error, openParams.agent, "turn", process.env) ?? undefined } : {}), }, }; } finally { diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 8457c47a396..337698660d5 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -1,3 +1,4 @@ +import type { ProviderFailure } from "./provider-failure.generated.js"; import type { NativeRunRuntimeGrant } from "./runtime-context.js"; import type { NativeRunIdentity, @@ -279,8 +280,11 @@ export interface NativeSessionBackend { /** A provider failed terminal is not a missing completion proposal. */ export class NativeProviderTerminalFailure extends Error { readonly code = "native_provider_terminal_failed"; - constructor(readonly providerCode: string, readonly recoverable: boolean, message = "Provider session ended with a failed terminal") { + declare readonly providerFailure?: ProviderFailure; + constructor(readonly providerCode: string, readonly recoverable: boolean, message = "Provider session ended with a failed terminal", + providerFailure?: ProviderFailure | null) { super(message); this.name = "NativeProviderTerminalFailure"; + if (providerFailure) Object.defineProperty(this, "providerFailure", { value: providerFailure, enumerable: false }); } } diff --git a/packages/paperclip-runner/src/contracts/provider-failure.generated.ts b/packages/paperclip-runner/src/contracts/provider-failure.generated.ts new file mode 100644 index 00000000000..e7f9071a254 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/provider-failure.generated.ts @@ -0,0 +1,103 @@ +export interface ProviderFailure { + schema: "paperclip.provider-failure.v1"; + provider: string; + phase: "startup" | "turn"; + category: "rate_limit" | "quota_exhausted" | "authentication" | "permission" | "model_unavailable" | "invalid_request" | "overloaded" | "timeout" | "unknown"; + message: string; + providerCode?: string; + httpStatus?: number; + requestId?: string; + retryAt?: string; + retryability: "transient" | "requires_action" | "unknown"; +} + +const fields = { + "schema": { + "const": "paperclip.provider-failure.v1" + }, + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 80, + "pattern": "^[a-zA-Z0-9_.:/-]+$" + }, + "phase": { + "enum": [ + "startup", + "turn" + ] + }, + "category": { + "enum": [ + "rate_limit", + "quota_exhausted", + "authentication", + "permission", + "model_unavailable", + "invalid_request", + "overloaded", + "timeout", + "unknown" + ] + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "providerCode": { + "type": "string", + "minLength": 1, + "maxLength": 160, + "pattern": "^[a-zA-Z0-9_.:/-]+$" + }, + "httpStatus": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "requestId": { + "type": "string", + "minLength": 1, + "maxLength": 160, + "pattern": "^[a-zA-Z0-9_.:/-]+$" + }, + "retryAt": { + "type": "string", + "format": "date-time" + }, + "retryability": { + "enum": [ + "transient", + "requires_action", + "unknown" + ] + } +} as const; + +export function readProviderFailure(value: unknown): ProviderFailure | null { + try { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const input = value as Record; + const output: Record = {}; + for (const [key, field] of Object.entries(fields)) { + const item = input[key]; + if (item === undefined && !["schema","provider","phase","category","message","retryability"].includes(key)) continue; + if ("const" in field) { + if (item !== field.const) return null; + } else if ("enum" in field) { + if (typeof item !== "string" || !(field.enum as readonly string[]).includes(item)) return null; + } else if (field.type === "integer") { + if (typeof item !== "number" || !Number.isInteger(item) || item < field.minimum || item > field.maximum) return null; + } else { + if (typeof item !== "string") return null; + if ("maxLength" in field && ([...item].length < field.minLength || [...item].length > field.maxLength)) return null; + if ("pattern" in field && !new RegExp(field.pattern).test(item)) return null; + if ("format" in field && !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(item)) return null; + if ("format" in field && !Number.isFinite(Date.parse(item))) return null; + } + output[key] = item; + } + return output as unknown as ProviderFailure; + } catch { return null; } +} diff --git a/packages/paperclip-runner/src/control-plane/acpx-controller-recovery.test.ts b/packages/paperclip-runner/src/control-plane/acpx-controller-recovery.test.ts index b147497b092..e44b6c7caf5 100644 --- a/packages/paperclip-runner/src/control-plane/acpx-controller-recovery.test.ts +++ b/packages/paperclip-runner/src/control-plane/acpx-controller-recovery.test.ts @@ -49,6 +49,7 @@ function sidecarSource(root: string, selectedProfile = profile, agent = "claude" " const reply = (r,result) => send({protocolVersion,id:r.id,ok:true,result});", " const lines=rl.createInterface({input:process.stdin}); lines.on('close',()=>process.exit(0)); lines.on('line', line => {", " const r=JSON.parse(line), p=r.params || {};", + " if(r.command==='session.open'&&fs.existsSync(root+'/startup-failure.json')){send({protocolVersion,id:r.id,ok:false,error:{code:'ACPX_SESSION_ENSURE_FAILED',message:'Internal error',retryable:false,providerFailure:JSON.parse(fs.readFileSync(root+'/startup-failure.json','utf8'))}});return;}", " switch(r.command) {", " case 'initialize': reply(r,{protocolVersion,sidecarPid:process.pid,profile:{agent:" + JSON.stringify(agent) + "},capabilities:{persistentSessions:true,exactModelVerification:true,permissions:'runner_policy',semanticTools:'runner_bridge',structuredInput:'paperclip.question_set.v1'}});break;", " case 'session.open': identity={kind:'acpx',normalizedSessionId:p.normalizedSessionId,acpxRecordId:'record-1',backendSessionId:'backend-1',agentSessionId:'agent-1',profileDigest,workspaceDigest:'sha256:'+'2'.repeat(64),requestedModel:p.model,effectiveModel:p.model,permissionMode:p.permissionMode,mode:p.mode,piThinkingLevel:p.piThinkingLevel,claudeEffort:p.claudeEffort,grokEffort:p.grokEffort,grokDisableWebSearch:p.grokDisableWebSearch,providerLifetimeFenceCandidates:ports};reply(r,{sidecarPid:process.pid,identity,status:{},turnControls:{steering:false,queuedFollowUp:false}});break;", @@ -67,6 +68,23 @@ function sidecarSource(root: string, selectedProfile = profile, agent = "claude" ].join("\n"); } +it("preserves a startup provider failure through the real sidecar transport and restored controller journal", async () => { + const run = await fixture(); + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", message: "This request would exceed your account's rate limit.", httpStatus: 429, retryability: "unknown" }; + try { + writeFileSync(resolve(run.root, "startup-failure.json"), JSON.stringify(providerFailure)); + expect(await run.command("run.prepare", { provider: run.descriptor })).toMatchObject({ status: "completed" }); + const failed = await run.command("session.open"); + expect(failed).toMatchObject({ status: "failed", result: { result: { code: "command_execution_failed", providerFailure } } }); + await run.stopController(); + await run.replaceController(); + expect(run.controller().getCommand("command-2")).toMatchObject({ status: "failed", + result: { result: { providerFailure } } }); + expect(run.launches()).toHaveLength(1); + } finally { await run.cleanup(); } +}, 20_000); + async function fixture(maxLifetimeMs?: number, candidateAdmission?: AcpxCandidateStartupAdmission, beforeLaunchDelayMs = 0) { const root = mkdtempSync(resolve(tmpdir(), "acpx-controller-recovery-")); const { executable: nodeCommand } = await copyOwnedNodeRuntime(resolve(root, "node")); @@ -156,7 +174,7 @@ async function fixture(maxLifetimeMs?: number, candidateAdmission?: AcpxCandidat rmSync(stateDirectory,{recursive:true,force:true}); decodeDurablePrpCheckpoint(checkpoint.records).restoreSessionDirectory(root); }; - return {root,child,exit,command,ready,startedAt,deadlineAt,processGroupId: bounded?.processGroupId ?? child.pid, + return {root,child,exit,command,ready,descriptor,startedAt,deadlineAt,processGroupId: bounded?.processGroupId ?? child.pid, controller:()=>controller, stopController:stopped, restoreCheckpoint, diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts index 76f0d7c914e..ddbe54291b5 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts @@ -34,6 +34,21 @@ const HANDLE: AcpRuntimeHandle = { }; describe("Codex ACPX runtime adapter", () => { + it("retains the provider cause before assigning the generic session failure code", async () => { + const runtime = fakeRuntime(); + const rejection = Object.assign(new Error("Internal error"), { code: -32603, + data: { details: "API error: 429 This request would exceed your account's rate limit." } }); + vi.mocked(runtime.ensureSession).mockRejectedValue(rejection); + const options = openOptions(fakeCommand()); + options.profile = { ...options.profile, agent: "claude" }; + await expect(openCodexAcpxRuntime(options, { createRegistry: () => registry(), createStore: () => store(), + createRuntime: () => runtime })).rejects.toMatchObject({ + code: "ACPX_SESSION_ENSURE_FAILED", + providerFailure: { provider: "anthropic", category: "rate_limit", httpStatus: 429, + message: rejection.data.details, phase: "startup" }, + }); + expect(runtime.startTurn).not.toHaveBeenCalled(); + }); it.each([false, true])("acknowledges configured Claude effort before admission (restoring=%s)", async restoring => { const runtime = fakeRuntime(); const child = fakeChild(); diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts index 742050327f8..30cb7421fc0 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts @@ -4,6 +4,7 @@ import { kimiReadOnlyPermissionAllowed } from "./kimi-config.js"; import { resolveGrokDisableWebSearch } from "./grok-web-search.js"; import { resolveGrokEffort, createGrokEffortAdmission } from "./grok-effort.js"; import { resolveClaudeEffort, createClaudeEffortAdmission } from "./claude-effort.js"; +import { acpxProviderFailure } from "./provider-failure.js"; import { createPiThinkingAdmission, resolvePiThinkingLevel } from "./pi-thinking.js"; import { usesPreconfiguredHome } from "../preconfigured-home.js"; import type { ChildProcess } from "node:child_process"; @@ -599,6 +600,10 @@ export async function openQualifiedAcpxRuntime( return ensuredHandle; }) : ensuredSession) .catch((error: unknown) => { + const providerFailure = acpxProviderFailure(error, options.profile.agent, "startup", options.launchEnvironment); + if (providerFailure && error instanceof Error) { + Object.defineProperty(error, "providerFailure", { value: providerFailure, enumerable: false }); + } throw classifySessionEnsureFailure(error); }); let lateCleanup: Promise | null = null; diff --git a/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts b/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts new file mode 100644 index 00000000000..e18e978fd25 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts @@ -0,0 +1,52 @@ +import type { ProviderFailure } from "../../contracts/provider-failure.generated.js"; +import { createProviderFailure } from "../provider-failure.js"; +import { boundedSidecarValue } from "./sidecar-protocol.js"; + +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +function acpError(value: unknown, depth = 0): Record | null { + if (depth > 5) return null; + const entry = record(value); + if (typeof entry.code === "number" && typeof entry.message === "string") return entry; + for (const key of ["acp", "cause", "error"]) { + const found = acpError(entry[key], depth + 1); + if (found) return found; + } + return null; +} + +export function acpxProviderFailure(error: unknown, agent: string, phase: ProviderFailure["phase"], + environment: NodeJS.ProcessEnv = {}, terminal = false): ProviderFailure | null { + const acp = acpError(error); + const data = record(acp?.data); + const detail = record(data.error); + const fallback = terminal ? record(error) : {}; + const message = typeof detail.message === "string" ? detail.message + : typeof data.details === "string" ? data.details + : typeof acp?.message === "string" ? acp.message : fallback.message; + if (typeof message !== "string") return null; + const status = data.status ?? data.statusCode ?? detail.status + ?? (agent === "claude" ? Number(/^API error: (\d{3})\b/.exec(message)?.[1]) : undefined); + const code = detail.type ?? detail.code ?? data.errorType; + const categories: Record = { + authentication_error: "authentication", permission_error: "permission", overloaded_error: "overloaded", + rate_limit_error: "rate_limit", billing_error: "quota_exhausted", invalid_request_error: "invalid_request", + insufficient_quota: "quota_exhausted", model_not_found: "model_unavailable", + }; + return createProviderFailure({ provider: agent === "claude" ? "anthropic" : agent === "codex" ? "openai" : agent, + phase, message, httpStatus: status, providerCode: code, requestId: data.request_id, + category: typeof code === "string" ? categories[code] : undefined, environment }); +} + +export function acpxTerminalResult(result: unknown, agent: string, environment: NodeJS.ProcessEnv = {}): Record { + const raw = record(result); + const providerFailure = raw.status === "failed" ? acpxProviderFailure(raw.error, agent, "turn", environment, true) : null; + const terminal = boundedSidecarValue(result); + if (providerFailure) { + terminal.status = "failed"; + terminal.error = { ...record(terminal.error), providerFailure }; + } + return terminal; +} diff --git a/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts b/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts index 26f9367d34b..fd36c5aa8b3 100644 --- a/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts +++ b/packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts @@ -1,4 +1,5 @@ import type { AcpxCandidateStartupAdmission } from "./candidate-admission.js"; +import type { ProviderFailure } from "../../contracts/provider-failure.generated.js"; import type { QualifiedAcpxAgent } from "./qualified-profiles.js"; import type { NativeRuntimeContextSnapshot } from "../../contracts/runtime-context.js"; import type { NativeAcpxPermissionMode } from "../../contracts/native-execution.js"; @@ -27,7 +28,7 @@ export interface AcpxSidecarResponse { id: number; ok: boolean; result?: Record; - error?: { code: string; message: string; retryable: boolean }; + error?: { code: string; message: string; retryable: boolean; providerFailure?: ProviderFailure }; } export interface AcpxSidecarEvent { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 18b5cff263c..995424acc63 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -1,4 +1,5 @@ import { usesPreconfiguredHome } from "../preconfigured-home.js"; +import { codexProviderFailure } from "./provider-failure.js"; import { writeManagedCodexObservability, redactObservabilityDiagnostic } from "../agent-observability.js"; import { trustCodexStartupRoot } from "./codex-startup-trust.js"; import { readCodexThreadState, readCodexTurnMetadata } from "./codex-history.js"; @@ -358,6 +359,10 @@ export class CodexAppServerDriver implements HarnessDriver { await cancellation.close().catch(() => {}); if (error instanceof NativeSessionProtocolIntegrityError) throw error; if (input.signal?.aborted) input.signal.throwIfAborted(); + if (error instanceof CodexRpcError && (this.#options.driverIdentity?.kind ?? DRIVER_KIND) === DRIVER_KIND) { + const providerFailure = codexProviderFailure(error, "startup", this.#options.environment); + if (providerFailure) Object.defineProperty(error, "providerFailure", { value: providerFailure, enumerable: false }); + } throw error; } finally { cancellation.detach(); @@ -989,6 +994,7 @@ export class CodexAppServerDriver implements HarnessDriver { dynamicTools: this.#providerDynamicTools(), skillInputs: this.#options.skillInputs, reasoningEffort: this.#options.reasoningEffort, + diagnosticEnvironment: this.#options.environment, dynamicToolHandler: this.#options.dynamicToolHandler, completionFeedback: this.#options.completionFeedback, }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts index 7810e82bb4a..4cb21eaa922 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts @@ -115,6 +115,17 @@ class WarmAttachTransport extends FakeCodexTransport { } describe("Codex app-server Codex driver", () => { + it.each(["codex_app_server", "acpx_runtime"])("keeps startup provider evidence for the selected %s driver", async kind => { + const transport = new FakeCodexTransport(); + transport.rejectMethods.set("thread/start", new CodexRpcError(JSON.stringify({ error: { + code: "insufficient_quota", message: "The provider allowance is exhausted." }, status: 429 }), -32603)); + const driver = makeDriver([transport], { driverIdentity: { kind, displayName: "Fixture", version: "1" } }); + const error = await driver.openSession({ runId: "startup-cause", normalizedSessionId: "startup-cause", workingDirectory: WORKSPACE }).catch(error => error); + if (kind === "codex_app_server") expect(error.providerFailure).toMatchObject({ provider: "openai", phase: "startup", + category: "quota_exhausted", message: "The provider allowance is exhausted.", httpStatus: 429 }); + else expect(error.providerFailure).toBeUndefined(); + }); + it("accepts runner-proven warm attachment when the host active-turn reducer is stale", async () => { const transport = new WarmAttachTransport(); const driver = makeDriver([transport]); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts index 53f9da0b699..382e43c0c33 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -103,6 +103,7 @@ export class CodexSessionState { readonly goalReason: string | null; readonly skillInputs: NonNullable; readonly reasoningEffort: string | undefined; + readonly diagnosticEnvironment: NodeJS.ProcessEnv; readonly dynamicTools: readonly Readonly>[]; readonly completionFeedback: CodexAppServerDriverOptions["completionFeedback"]; readonly dynamicToolHandler: CodexAppServerDriverOptions["dynamicToolHandler"]; @@ -180,6 +181,7 @@ export class CodexSessionState { goalReason: string | null; skillInputs?: CodexAppServerDriverOptions["skillInputs"]; reasoningEffort?: string; + diagnosticEnvironment?: NodeJS.ProcessEnv; dynamicTools: readonly Readonly>[]; completionFeedback?: CodexAppServerDriverOptions["completionFeedback"]; dynamicToolHandler?: CodexAppServerDriverOptions["dynamicToolHandler"]; @@ -205,6 +207,7 @@ export class CodexSessionState { this.goalReason = input.goalReason; this.skillInputs = structuredClone(input.skillInputs ?? []); this.reasoningEffort = input.reasoningEffort; + this.diagnosticEnvironment = input.diagnosticEnvironment ?? {}; this.dynamicTools = input.dynamicTools; this.dynamicToolHandler = input.dynamicToolHandler; this.completionFeedback = input.completionFeedback; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts index 582730399aa..777a061a9ac 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts @@ -1,4 +1,6 @@ import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; +import { readProviderFailure } from "../../contracts/provider-failure.generated.js"; +import { codexProviderFailure } from "./provider-failure.js"; import { isNativeRestartInterruption } from "../../contracts/native-session-backend.js"; import type { PrpStructuredRunResult } from "../../protocol/replay-contract.js"; import { boundedCodexPayload as boundedPayload, isRetainableCodexPayload } from "./codex-boundaries.js"; @@ -163,14 +165,14 @@ export function mapTerminalTurn( state.cancelPendingRequests("turn_terminal"); state.activeTurnId = null; state.turnStarted = false; - state.emit( - eventType, - boundedPayload({ - status, - error: turn.error ?? null, - }), - { turnId }, - ); + const providerFailure = status === "failed" ? readProviderFailure(record(turn.error).providerFailure) + ?? (state.driverKind === "codex_app_server" ? codexProviderFailure(turn.error, "turn", state.diagnosticEnvironment) : null) : null; + const payload = boundedPayload({ status, error: turn.error ?? null }); + if (providerFailure) { + payload.status = status; + payload.error = { ...record(payload.error), providerFailure }; + } + state.emit(eventType, payload, { turnId }); finalize(state, status); if (status === "failed" && isNativeRestartInterruption(turn.error) && state.result === null && !state.dispositionOnlyRecoveryConsumed) { diff --git a/packages/paperclip-runner/src/drivers/codex/diagnostic-redaction.ts b/packages/paperclip-runner/src/drivers/codex/diagnostic-redaction.ts index 856b9365e41..656c08bebb5 100644 --- a/packages/paperclip-runner/src/drivers/codex/diagnostic-redaction.ts +++ b/packages/paperclip-runner/src/drivers/codex/diagnostic-redaction.ts @@ -1,33 +1 @@ -/** Redact diagnostic text without importing the provider process or its credentials. */ -export function redactCodexDiagnostic(message: string): string { - return message - .replaceAll(/\u001b\[[0-?]*[ -/]*[@-~]/g, "") - .replace(/Bearer\s+[A-Za-z0-9._~+\/-]+/gi, "Bearer [REDACTED]") - .replace(/Basic\s+([A-Za-z0-9+/=]+)/gi, (match, encoded: string) => { - try { - // Only redact an actual RFC 7617 credential. Ordinary prose such as - // ā€œBasic APIā€ must remain readable. - const decoded = Buffer.from(encoded, "base64").toString("utf8"); - return decoded.includes(":") ? "Basic [REDACTED]" : match; - } catch { - return match; - } - }) - .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED]@") - .replace( - /([?&](?:api[_-]?key|token|secret|password)=)[^&#\s]+/gi, - "$1[REDACTED]", - ) - .replace( - /(["'](?:api[_-]?key|token|secret|password|authorization)["']\s*:\s*["'])[^"']+/gi, - "$1[REDACTED]", - ) - .replace( - /(api[_-]?key|token|secret|password)\s*[=:]\s*[^\s,;]+/gi, - "$1=[REDACTED]", - ) - .replace( - /((?:[A-Z][A-Z0-9]*_)+API_KEY)=[^\s]+/g, - "$1=[REDACTED]", - ); -} +export { redactProviderDiagnostic as redactCodexDiagnostic } from "../diagnostic-redaction.js"; diff --git a/packages/paperclip-runner/src/drivers/codex/provider-failure.ts b/packages/paperclip-runner/src/drivers/codex/provider-failure.ts new file mode 100644 index 00000000000..183f9da400a --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/provider-failure.ts @@ -0,0 +1,32 @@ +import type { ProviderFailure } from "../../contracts/provider-failure.generated.js"; +import { createProviderFailure } from "../provider-failure.js"; + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +export function codexProviderFailure(value: unknown, phase: ProviderFailure["phase"], environment: NodeJS.ProcessEnv = {}): ProviderFailure | null { + const error = record(value); + if (typeof error.message !== "string") return null; + let response: Record = {}; + if (error.message.length <= 16_000) { + try { response = record(JSON.parse(error.message)); } catch {} + } + const detail = record(response.error); + const info = error.codexErrorInfo; + const kind = typeof info === "string" ? info : Object.keys(record(info))[0]; + const code = detail.code ?? detail.type ?? kind; + const categories: Record = { + usageLimitExceeded: "quota_exhausted", insufficient_quota: "quota_exhausted", + credit_balance_exhausted: "quota_exhausted", organization_spend_limit_exceeded: "quota_exhausted", + project_spend_limit_exceeded: "quota_exhausted", organization_usage_limit_exceeded: "quota_exhausted", + rate_limit_exceeded: "rate_limit", rate_limit_error: "rate_limit", slow_down: "rate_limit", + serverOverloaded: "overloaded", server_is_overloaded: "overloaded", unauthorized: "authentication", + invalid_api_key: "authentication", model_not_found: "model_unavailable", + }; + return createProviderFailure({ provider: "openai", phase, + message: typeof detail.message === "string" ? detail.message : error.message, + category: typeof code === "string" ? categories[code] : undefined, + providerCode: code, httpStatus: response.status ?? record(record(info)[kind ?? ""]).httpStatusCode, + requestId: response.request_id, environment }); +} diff --git a/packages/paperclip-runner/src/drivers/diagnostic-redaction.ts b/packages/paperclip-runner/src/drivers/diagnostic-redaction.ts new file mode 100644 index 00000000000..d17113d69d8 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/diagnostic-redaction.ts @@ -0,0 +1,33 @@ +/** Redact diagnostic text without importing the provider process or its credentials. */ +export function redactProviderDiagnostic(message: string): string { + return message + .replaceAll(/\u001b\[[0-?]*[ -/]*[@-~]/g, "") + .replace(/Bearer\s+[A-Za-z0-9._~+\/-]+/gi, "Bearer [REDACTED]") + .replace(/Basic\s+([A-Za-z0-9+/=]+)/gi, (match, encoded: string) => { + try { + // Only redact an actual RFC 7617 credential. Ordinary prose such as + // ā€œBasic APIā€ must remain readable. + const decoded = Buffer.from(encoded, "base64").toString("utf8"); + return decoded.includes(":") ? "Basic [REDACTED]" : match; + } catch { + return match; + } + }) + .replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED]@") + .replace( + /([?&](?:api[_-]?key|token|secret|password)=)[^&#\s]+/gi, + "$1[REDACTED]", + ) + .replace( + /(["'](?:api[_-]?key|token|secret|password|authorization)["']\s*:\s*["'])[^"']+/gi, + "$1[REDACTED]", + ) + .replace( + /(api[_-]?key|token|secret|password)\s*[=:]\s*[^\s,;]+/gi, + "$1=[REDACTED]", + ) + .replace( + /((?:[A-Z][A-Z0-9]*_)+API_KEY)=[^\s]+/g, + "$1=[REDACTED]", + ); +} diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 12880cf0453..b8837776a18 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -2789,6 +2789,37 @@ describe("OpenCodeServerDriver", () => { } }); + it("keeps the provider explanation when an API error response body exceeds the payload limit", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-large-error-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-large-error-workspace-")); + roots.push(root, workspace); + let stream!: ReadableStreamDefaultController; + let streamOpened!: () => void; + const opened = new Promise(resolve => { streamOpened = resolve; }); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + fetch: async (input, init) => String(input).endsWith("/event") + ? new Response(new ReadableStream({ start(controller) { stream = controller; streamOpened(); } }), { headers: { "Content-Type": "text/event-stream" } }) + : fetch(input, init), + }); + const session = await driver.openSession({ runId: "large-error", normalizedSessionId: "large-error", workingDirectory: workspace }); + try { + const events = collectTurnEvents(session.events()); + await session.startTurn({ message: { role: "user", text: "diagnostic check" } }); + await opened; + stream.enqueue(new TextEncoder().encode(`data: ${JSON.stringify({ type: "session.error", properties: { + sessionID: session.ids().providerSessionId, error: { name: "APIError", data: { message: "Provider is busy.", statusCode: 503, + responseBody: "private-body".repeat(7000) } }, + } })}\n\n`)); + const observed = await events; + const failure = observed.find(event => event.eventType === "turn.failed"); + expect(failure?.payload.error).toMatchObject({ providerFailure: { provider: "openrouter", category: "overloaded", message: "Provider is busy." } }); + expect(JSON.stringify(failure)).not.toContain("private-body"); + } finally { await session.close({ reason: "test complete" }); } + }); + it("fails the running turn with the exit cause when OpenCode stops mid-turn", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-server-lost-")); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index a5c701ecc79..b3684dee58a 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -24,6 +24,7 @@ import { pipeline } from "node:stream/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { parseOpenCodeReasoningMode, type OpenCodeReasoningMode } from "./reasoning-mode.js"; +import { openCodeProviderFailure } from "./provider-failure.js"; import { createOpenCodeCompanyObservability } from "./company-observability.js"; import { startOpenCodePerformanceDiagnostics } from "./performance-diagnostics.js"; import { PROVIDER_PERFORMANCE_ENV } from "../../contracts/provider-performance.js"; @@ -1951,7 +1952,9 @@ class OpenCodeHarnessSession implements HarnessSession { this.#settlePendingRuntimeRequestsForTurn(turnId); this.#emit( "turn.failed", - { status: "failed", error: bounded(properties.error ?? properties) }, + { status: "failed", error: { ...bounded(providerError), + providerFailure: openCodeProviderFailure(providerError, this.#model.split("/")[0]!, + Object.fromEntries(this.#runtime.sensitiveValues.map((value, index) => [`TOKEN_${index}`, value]))) ?? undefined } }, { turnId }, ); this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" })); diff --git a/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts b/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts new file mode 100644 index 00000000000..f6532b999aa --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts @@ -0,0 +1,11 @@ +import { createProviderFailure } from "../provider-failure.js"; + +export function openCodeProviderFailure(error: Record, provider: string, + environment: NodeJS.ProcessEnv = {}) { + const data = error.data && typeof error.data === "object" ? error.data as Record : {}; + const message = typeof data.message === "string" ? data.message : error.message; + if (typeof message !== "string") return null; + return createProviderFailure({ provider, phase: "turn", message, httpStatus: data.statusCode, + providerCode: data.code ?? error.name, requestId: data.requestID, + category: error.name === "ProviderAuthError" ? "authentication" : undefined, environment }); +} diff --git a/packages/paperclip-runner/src/drivers/provider-failure.test.ts b/packages/paperclip-runner/src/drivers/provider-failure.test.ts new file mode 100644 index 00000000000..34ee587df5c --- /dev/null +++ b/packages/paperclip-runner/src/drivers/provider-failure.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { readProviderFailure } from "../contracts/provider-failure.generated.js"; +import { acpxProviderFailure, acpxTerminalResult } from "./acpx/provider-failure.js"; +import { codexProviderFailure } from "./codex/provider-failure.js"; +import { openCodeProviderFailure } from "./opencode/provider-failure.js"; +import { createProviderFailure, providerFailureFromError } from "./provider-failure.js"; + +describe("provider failure boundary", () => { + const message = "API error: 429 This request would exceed your account's rate limit. Please try again later Ā· model not changed"; + const acp = { code: -32603, message: "Internal error", data: { details: message, prompt: "private request body" } }; + + it.each(["startup", "turn"] as const)("preserves Claude's nested ACP explanation during %s", phase => { + const error = Object.assign(new Error("ACPX session ensure failed"), { cause: { acp } }); + expect(acpxProviderFailure(error, "claude", phase)).toEqual({ schema: "paperclip.provider-failure.v1", + provider: "anthropic", phase, category: "rate_limit", message, httpStatus: 429, retryability: "unknown" }); + expect(JSON.stringify(acpxProviderFailure(error, "claude", phase))).not.toContain("private request body"); + }); + + it("preserves unknown provider explanations without English-message classification", () => { + const failure = acpxProviderFailure({ ...acp, data: { details: "Unrecognized upstream failure šŸš€" } }, "kimi", "turn"); + expect(failure).toMatchObject({ provider: "kimi", category: "unknown", message: "Unrecognized upstream failure šŸš€", retryability: "unknown" }); + expect(acpxProviderFailure(new Error("internal runtime failure"), "claude", "startup")).toBeNull(); + }); + + it("keeps the ACPX terminal cause when the raw result exceeds its payload limit", () => { + const terminal = acpxTerminalResult({ status: "failed", error: { ...acp, + data: { ...acp.data, responseBody: "private-body".repeat(7000) } } }, "claude"); + expect(terminal).toMatchObject({ status: "failed", omitted: true, error: { providerFailure: { message, httpStatus: 429 } } }); + expect(JSON.stringify(terminal)).not.toContain("private-body"); + }); + + it("distinguishes structured OpenAI quota failures from rate limits", () => { + const error = { message: JSON.stringify({ type: "error", status: 429, + error: { code: "insufficient_quota", message: "Your current allowance is exhausted." }, request_id: "req_fixture" }) }; + expect(codexProviderFailure(error, "turn")).toMatchObject({ provider: "openai", category: "quota_exhausted", + httpStatus: 429, message: "Your current allowance is exhausted.", requestId: "req_fixture", retryability: "requires_action" }); + expect(codexProviderFailure({ message: "Rate limited.", codexErrorInfo: { httpConnectionFailed: { httpStatusCode: 429 } } }, "turn")) + .toMatchObject({ category: "rate_limit", retryability: "unknown" }); + }); + + it("extracts OpenCode API error fields without copying response bodies or headers", () => { + const failure = openCodeProviderFailure({ name: "APIError", data: { message: "Service is busy.", statusCode: 503, + responseBody: "private request", responseHeaders: { authorization: "secret" } } }, "openrouter"); + expect(failure).toMatchObject({ provider: "openrouter", httpStatus: 503, message: "Service is busy.", category: "overloaded" }); + expect(JSON.stringify(failure)).not.toMatch(/private request|secret|responseHeaders/); + }); + + it("redacts credentials before truncation and preserves ordinary words and Unicode", () => { + const secret = "fixture-unusual-secret"; + const failure = createProviderFailure({ provider: "anthropic", phase: "startup", + message: `Basic API failure šŸš€ ${secret} ${encodeURIComponent(secret)} Bearer abcdef https://user:pass@example.invalid?token=hidden sk-ant-privatecredential123`, + environment: { CUSTOM_API_KEY: secret } }); + expect(failure?.message).toContain("Basic API failure šŸš€"); + expect(failure?.message).not.toMatch(/fixture-unusual-secret|abcdef|user:pass|hidden|privatecredential/); + const long = createProviderFailure({ provider: "new-provider", phase: "turn", message: "šŸš€".repeat(6000) }); + expect([...long!.message]).toHaveLength(4000); + expect(readProviderFailure(JSON.parse(JSON.stringify(long)))).toEqual(long); + }); + + it("redacts overlapping environment secrets without leaving a credential suffix", () => { + expect(createProviderFailure({ provider: "openai", phase: "startup", message: "fixture-long-private-value", + environment: { FIRST_TOKEN: "fixture", SECOND_TOKEN: "fixture-long-private-value" } })?.message).toBe("[REDACTED]"); + }); + + it("carries only the selected fields and rejects malformed persisted causes", () => { + const failure = createProviderFailure({ provider: "openai", phase: "turn", message: "Upstream failure." })!; + expect(readProviderFailure({ ...failure, headers: { authorization: "secret" } })).toEqual(failure); + expect(readProviderFailure({ ...failure, message: "" })).toBeNull(); + expect(readProviderFailure({ ...failure, httpStatus: 999 })).toBeNull(); + expect(providerFailureFromError({ runnerCommandResult: { result: { providerFailure: failure } } })).toEqual(failure); + expect(providerFailureFromError({ runnerCommandResult: { result: { message: "arbitrary private text" } } })).toBeNull(); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/provider-failure.ts b/packages/paperclip-runner/src/drivers/provider-failure.ts new file mode 100644 index 00000000000..b1697fe2d48 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/provider-failure.ts @@ -0,0 +1,58 @@ +import { readProviderFailure, type ProviderFailure } from "../contracts/provider-failure.generated.js"; +import { redactProviderDiagnostic } from "./diagnostic-redaction.js"; + +export function providerFailureMessage(message: string, environment: NodeJS.ProcessEnv = {}): string { + const secrets = Object.entries(environment).flatMap(([key, value]) => + value && /token|secret|password|api[_-]?key|authorization|credential|otlp.*headers/i.test(key) + ? [value, encodeURIComponent(value), JSON.stringify(value).slice(1, -1)] : []).sort((left, right) => right.length - left.length); + for (const secret of secrets) { + if (secret) message = message.replaceAll(secret, "[REDACTED]"); + } + return [...redactProviderDiagnostic(message) + .replace(/\b(?:sk-(?:ant-|proj-)?[A-Za-z0-9_-]{8,}|gh[pousr]_[A-Za-z0-9_]+)\b/g, "[REDACTED]") + .replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/g, "")] + .slice(0, 4000).join("").trim(); +} + +export function createProviderFailure(input: { + provider: string; + phase: ProviderFailure["phase"]; + message: string; + category?: ProviderFailure["category"]; + providerCode?: unknown; + httpStatus?: unknown; + requestId?: unknown; + retryAt?: unknown; + environment?: NodeJS.ProcessEnv; +}): ProviderFailure | null { + const httpStatus = typeof input.httpStatus === "number" && Number.isInteger(input.httpStatus) + && input.httpStatus >= 400 && input.httpStatus <= 599 ? input.httpStatus : undefined; + const category = input.category ?? (httpStatus === 429 ? "rate_limit" : httpStatus === 401 ? "authentication" + : httpStatus === 403 ? "permission" : httpStatus === 503 || httpStatus === 529 ? "overloaded" + : httpStatus === 408 || httpStatus === 504 ? "timeout" : "unknown"); + const retryAt = typeof input.retryAt === "string" && Number.isFinite(Date.parse(input.retryAt)) + ? new Date(input.retryAt).toISOString() : undefined; + const token = (value: unknown) => typeof value === "string" && /^[a-zA-Z0-9_.:/-]{1,160}$/.test(value) + && providerFailureMessage(value, input.environment) === value ? value : undefined; + return readProviderFailure({ + schema: "paperclip.provider-failure.v1", provider: token(input.provider)?.slice(0, 80) ?? "unknown", phase: input.phase, + category, message: providerFailureMessage(input.message, input.environment), + ...(token(input.providerCode) ? { providerCode: token(input.providerCode) } : {}), + ...(token(input.requestId) ? { requestId: token(input.requestId) } : {}), + ...(httpStatus ? { httpStatus } : {}), ...(retryAt ? { retryAt } : {}), + retryability: ["authentication", "permission", "quota_exhausted", "model_unavailable", "invalid_request"].includes(category) + ? "requires_action" : ["overloaded", "timeout"].includes(category) || (category === "rate_limit" && retryAt) + ? "transient" : "unknown", + }); +} + +export function providerFailureFromError(error: unknown, depth = 0): ProviderFailure | null { + if (!error || typeof error !== "object" || depth > 5) return null; + const value = error as Record; + const result = value.runnerCommandResult as Record | undefined; + const detail = result?.result as Record | undefined; + const direct = readProviderFailure(value.providerFailure) ?? readProviderFailure(detail?.providerFailure); + if (direct) return direct; + return providerFailureFromError(value.cause, depth + 1) + ?? (Array.isArray(value.errors) ? value.errors.slice(0, 3).map(item => providerFailureFromError(item, depth + 1)).find(Boolean) ?? null : null); +} diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 665c5a4a0af..3f5ecb3dc77 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -119,3 +119,6 @@ export { RUNNER_GITHUB_LAUNCHER_VERSION, bindRunnerLaunchResources, createRunner export type { RunnerProviderLaunchRequirements, RunnerProviderLaunchEnvironment, PreparedRunnerProviderLaunch, RunnerLaunchPreparer } from "./live/runner-provider-launch.js"; export { resolveGrokDisableWebSearch } from "./drivers/acpx/grok-web-search.js"; + +export { readProviderFailure, type ProviderFailure } from "./contracts/provider-failure.generated.js"; +export { providerFailureFromError } from "./drivers/provider-failure.js"; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts index e776729200f..9c97cd48540 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts @@ -732,6 +732,8 @@ it.each([ command.result?.result?.message?.includes("no rollout found"), ), ).toBe(true); + expect(failed.commands.find((command: { type: string }) => command.type === "turn.stop")?.result?.result?.providerFailure) + .toMatchObject({ provider: "openai", phase: "startup", category: "unknown", message: "no rollout found for thread id" }); expect(await readFile(fakeState)).toEqual(originalProviderHome); expect( await Promise.all(files.map((file) => readFile(join(original, file)))), diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 3eff4e3b3e2..fc4952e1f19 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -9885,6 +9885,22 @@ it("retains the authenticated command failure without implicitly logging its str expect(structured).toHaveProperty("result.message", "closed runtime failure"); }); +it.each(["RPC failed: Private provider explanation.", "failed to start ACPX provider: ACPX sidecar command session.open was rejected (retryable=false, classification=session_ensure_failed)"])("keeps a provider explanation private and retains command diagnosis: %s", async message => { + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "unknown", message: "Private provider explanation.", retryability: "unknown" }; + const result = { result: { code: "command_execution_failed", message, providerFailure } }; + let failure: unknown; + try { + await waitForRunnerCommand({ type: "session.open", deadline: Date.now() + 1000, abortOnClose: true, + isClosed: () => false, throwIfFailed: () => {}, command: () => ({ status: "failed", result }), + runnerHasExited: async () => false, failureCode: () => "startup_failed" }); + } catch (error) { failure = error; } + expect(failure).toHaveProperty("runnerCommandResult.result.providerFailure", providerFailure); + expect(failure).toHaveProperty("runnerCommandResult.result.message", message); + expect((failure as Error).message).not.toContain(providerFailure.message); + expect(JSON.stringify(failure)).not.toContain(providerFailure.message); +}); + it("shares Pi's absolute cold deadline across recovery barriers and rejects a late ACK", async () => { vi.useFakeTimers(); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index ccdf72eaa11..28e1f6bb694 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,3 +1,4 @@ +import { readProviderFailure } from "../contracts/provider-failure.generated.js"; import { confirmAntigravityProbeNativeRetirement } from "../drivers/acpx/antigravity-probe-retirement.js"; import { resolveGrokDisableWebSearch, admittedGrokWebSearchPolicy } from "../drivers/acpx/grok-web-search.js"; import { PRP_COMPLETION_TOOL_NAME, PRP_COMPLETION_RESULT_BRIDGE_INPUT_SCHEMA } from "../contracts/completion-result.js"; @@ -8091,7 +8092,11 @@ export async function waitForRunnerCommand(input: { const command = input.command(); if (command?.status === "completed") return; if (command !== undefined && command.status !== "pending") { - const error = new Error(`PRP command ${input.type} ${command.status}: ${JSON.stringify(command.result)}`) as RunnerCommandFailure; + const detail = record(record(command.result).result); + const reportedResult = readProviderFailure(detail.providerFailure) + ? { ...record(command.result), result: { code: detail.code, message: "Provider request failed" } } : command.result; + const error = new Error(`PRP command ${input.type} ${command.status}: ${JSON.stringify(reportedResult, + (key, value) => key === "providerFailure" ? undefined : value)}`) as RunnerCommandFailure; Object.defineProperty(error, "runnerCommandResult", { value: structuredClone(command.result), enumerable: false }); throw error; } diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 445d3e5f615..2b8475c317e 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -9,6 +9,7 @@ import type { NativeExecutionInput, NativeSessionExecutionResult, } from "./contracts/native-execution.js"; +import { readProviderFailure } from "./contracts/provider-failure.generated.js"; import { buildNativeModelEnvelope, parseNativeExecutionInput, @@ -1111,6 +1112,7 @@ async function consumeTurn( typeof failure.code === "string" ? failure.code : "provider_session_failed", failure.recoverable === true || payload.recoverable === true, typeof failure.message === "string" ? failure.message : undefined, + readProviderFailure(failure.providerFailure), ); } const request = @@ -2732,6 +2734,7 @@ export async function executeNativeSession( modelRejected ? `native_provider_model_rejected: ${message}` : message, + readProviderFailure(failure.providerFailure), ); } if (settledCompletion === null && options.resolveMissingResult) { diff --git a/packages/paperclip-runner/test/acpx-sidecar-contract.test.mjs b/packages/paperclip-runner/test/acpx-sidecar-contract.test.mjs index bb5c3f3f6c2..2b262d59641 100644 --- a/packages/paperclip-runner/test/acpx-sidecar-contract.test.mjs +++ b/packages/paperclip-runner/test/acpx-sidecar-contract.test.mjs @@ -15,7 +15,13 @@ const schema = JSON.parse( "utf8", ), ); -const validate = new Ajv2020({ allErrors: true, strict: true }).compile(schema); +const providerFailureSchema = JSON.parse(await readFile(new URL("../protocol/schemas/provider-failure.schema.json", import.meta.url), "utf8")); +function compileSidecarSchema(value) { + return new Ajv2020({ allErrors: true, strict: true, + formats: { "date-time": /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/ }, + }).addSchema(providerFailureSchema).compile(value); +} +const validate = compileSidecarSchema(schema); const protocolVersion = readAcpxSidecarProtocolVersion(schema); const messages = [ @@ -47,11 +53,24 @@ test("the ACPX sidecar schema accepts each versioned message family", () => { } }); +test("failed responses accept the bounded provider cause and reject extra diagnostic fields", () => { + const failure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", message: "Account rate limit reached.", httpStatus: 429, retryability: "unknown" }; + const response = { protocolVersion, id: 1, ok: false, + error: { code: "ACPX_SESSION_ENSURE_FAILED", message: "Internal error", retryable: false } }; + assert.equal(validate(response), true); + const withCause = value => ({ ...response, error: { ...response.error, providerFailure: value } }); + assert.equal(validate(withCause(failure)), true, JSON.stringify(validate.errors)); + for (const change of [{ httpStatus: 999 }, { message: "x".repeat(4001) }, { headers: { authorization: "private" } }, { category: "invented" }]) { + assert.equal(validate(withCause({ ...failure, ...change })), false); + } +}); + test("candidate startup and sidecar admission share one strict identity-bound JSON record", () => { const candidateAdmission = { schema: "paperclip.acpx-candidate-admission.v1", companyId: "company-1", agent: "antigravity", model: "gemini-3.8-flash-low", profileDigest: `sha256:${"a".repeat(64)}`, nativeClosureDigest: `sha256:${"b".repeat(64)}`, platform: "darwin", architecture: "arm64", expiresAt: 1800000060000, runId: "run-1", normalizedSessionId: "session-1" }; - const startup = new Ajv2020({ allErrors: true, strict: true }).compile({ ...schema, oneOf: undefined, $ref: "#/$defs/candidateAdmission" }); + const startup = compileSidecarSchema({ ...schema, oneOf: undefined, $ref: "#/$defs/candidateAdmission" }); assert.equal(startup(candidateAdmission), true, JSON.stringify(startup.errors)); for (const command of ["initialize", "session.open"]) { const request = { protocolVersion, id: 1, command, params: { agent: "antigravity", model: candidateAdmission.model, candidateAdmission } }; diff --git a/packages/paperclip-runner/test/pi-closed-startup.test.mjs b/packages/paperclip-runner/test/pi-closed-startup.test.mjs index 378e166acf3..cb57a2a6b9c 100644 --- a/packages/paperclip-runner/test/pi-closed-startup.test.mjs +++ b/packages/paperclip-runner/test/pi-closed-startup.test.mjs @@ -19,7 +19,7 @@ if (process.argv[2] === "--pi-no-key-probe") { environment: { PATH: "/usr/bin:/bin", LANG: "en_US.UTF-8" }, onEvidence: value => evidence.push(value), }); - let error = null; let closeError = null; let settledMs; + let error = null; let commandFailure = null; let closeError = null; let settledMs; try { await bundle.transport.request("initialize", { clientInfo: { name: "pi-closed-startup-regression", version: "1" } }); await bundle.transport.request("thread/start", { @@ -27,12 +27,12 @@ if (process.argv[2] === "--pi-no-key-probe") { baseInstructions: "Credential-free admission probe. No prompt is submitted.", permissions: "paperclip-runner-workspace-read-only", dynamicTools: [], }); - } catch (failure) { error = String(failure); } + } catch (failure) { error = String(failure); commandFailure = failure.runnerCommandResult; } finally { settledMs = Math.round(performance.now() - started); try { await bundle.transport.close(); } catch (failure) { closeError = String(failure); } } - await writeFile(join(root, "report.json"), JSON.stringify({ error, closeError, settledMs, durationMs: Math.round(performance.now() - started), evidence }), { mode: 0o600 }); + await writeFile(join(root, "report.json"), JSON.stringify({ error, commandFailure, closeError, settledMs, durationMs: Math.round(performance.now() - started), evidence }), { mode: 0o600 }); } else { const packageRoot = process.env.PAPERCLIP_TEST_PI_STARTUP_PACKAGE_ROOT; const daemon = process.env.PAPERCLIP_TEST_PI_STARTUP_RUNNER_BINARY; @@ -55,7 +55,10 @@ if (process.argv[2] === "--pi-no-key-probe") { const report = JSON.parse(await readFile(join(root, "report.json"), "utf8")); assert.equal(report.closeError, null, report.closeError); assert.match(report.error, /session\.open failed/); - assert.match(report.error, /retryable=false, classification=session_ensure_failed/); + assert.match(report.commandFailure?.result?.message ?? "", /retryable=false, classification=session_ensure_failed/); + assert.equal(report.commandFailure?.result?.providerFailure?.schema, "paperclip.provider-failure.v1"); + assert.equal(report.commandFailure.result.providerFailure.provider, "pi"); + assert.equal(report.commandFailure.result.providerFailure.phase, "startup"); assert.doesNotMatch(report.error, /timed out/); assert.ok(report.settledMs < 60_000, `session.open took ${report.settledMs} ms`); assert.ok(report.evidence.some(item => item.runnerExited === true && item.runnerExitCode === 0)); diff --git a/packages/paperclip-runner/test/protocol-contract.test.mjs b/packages/paperclip-runner/test/protocol-contract.test.mjs index a3f457e19f2..992242c4802 100644 --- a/packages/paperclip-runner/test/protocol-contract.test.mjs +++ b/packages/paperclip-runner/test/protocol-contract.test.mjs @@ -27,7 +27,7 @@ async function fixture(relativePath) { test("all schema IDs are unique and all external references resolve", async () => { const schemas = await loadSchemaCatalog(resolve(protocolRoot, "schemas")); - assert.equal(schemas.length, 29); + assert.equal(schemas.length, 30); assert.doesNotThrow(() => compileProtocolValidators(schemas)); }); diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 81150730214..366ed7a870e 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -2883,3 +2883,5 @@ export { type RemoteWorkspaceAuthority, type RemoteWorkspaceRunReference, } from "./remote-workspace.js"; + +export { readProviderFailure, type ProviderFailure } from "./provider-failure.generated.js"; diff --git a/packages/shared/src/provider-failure.generated.ts b/packages/shared/src/provider-failure.generated.ts new file mode 100644 index 00000000000..e7f9071a254 --- /dev/null +++ b/packages/shared/src/provider-failure.generated.ts @@ -0,0 +1,103 @@ +export interface ProviderFailure { + schema: "paperclip.provider-failure.v1"; + provider: string; + phase: "startup" | "turn"; + category: "rate_limit" | "quota_exhausted" | "authentication" | "permission" | "model_unavailable" | "invalid_request" | "overloaded" | "timeout" | "unknown"; + message: string; + providerCode?: string; + httpStatus?: number; + requestId?: string; + retryAt?: string; + retryability: "transient" | "requires_action" | "unknown"; +} + +const fields = { + "schema": { + "const": "paperclip.provider-failure.v1" + }, + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 80, + "pattern": "^[a-zA-Z0-9_.:/-]+$" + }, + "phase": { + "enum": [ + "startup", + "turn" + ] + }, + "category": { + "enum": [ + "rate_limit", + "quota_exhausted", + "authentication", + "permission", + "model_unavailable", + "invalid_request", + "overloaded", + "timeout", + "unknown" + ] + }, + "message": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "providerCode": { + "type": "string", + "minLength": 1, + "maxLength": 160, + "pattern": "^[a-zA-Z0-9_.:/-]+$" + }, + "httpStatus": { + "type": "integer", + "minimum": 400, + "maximum": 599 + }, + "requestId": { + "type": "string", + "minLength": 1, + "maxLength": 160, + "pattern": "^[a-zA-Z0-9_.:/-]+$" + }, + "retryAt": { + "type": "string", + "format": "date-time" + }, + "retryability": { + "enum": [ + "transient", + "requires_action", + "unknown" + ] + } +} as const; + +export function readProviderFailure(value: unknown): ProviderFailure | null { + try { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const input = value as Record; + const output: Record = {}; + for (const [key, field] of Object.entries(fields)) { + const item = input[key]; + if (item === undefined && !["schema","provider","phase","category","message","retryability"].includes(key)) continue; + if ("const" in field) { + if (item !== field.const) return null; + } else if ("enum" in field) { + if (typeof item !== "string" || !(field.enum as readonly string[]).includes(item)) return null; + } else if (field.type === "integer") { + if (typeof item !== "number" || !Number.isInteger(item) || item < field.minimum || item > field.maximum) return null; + } else { + if (typeof item !== "string") return null; + if ("maxLength" in field && ([...item].length < field.minLength || [...item].length > field.maxLength)) return null; + if ("pattern" in field && !new RegExp(field.pattern).test(item)) return null; + if ("format" in field && !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(item)) return null; + if ("format" in field && !Number.isFinite(Date.parse(item))) return null; + } + output[key] = item; + } + return output as unknown as ProviderFailure; + } catch { return null; } +} diff --git a/packages/shared/src/types/execution-projection.ts b/packages/shared/src/types/execution-projection.ts index ff28fd01dc1..f3b9ef7d5c4 100644 --- a/packages/shared/src/types/execution-projection.ts +++ b/packages/shared/src/types/execution-projection.ts @@ -99,6 +99,7 @@ export interface ExecutionReconciliation { /** A task-level view of one interruption. Diagnostics keep the underlying runs * and recovery records; ordinary product surfaces render this projection. */ export interface TaskInterruptionPresentation { + providerFailure?: import("../provider-failure.generated.js").ProviderFailure; intentId: string | null; responsibility: "system" | "user"; state: "working" | "retry_scheduled" | "failed" | "waiting_for_user" | "resolved"; diff --git a/scripts/pi-public-install-probe.mjs b/scripts/pi-public-install-probe.mjs index e6e020eeb4e..a0091e50bc2 100644 --- a/scripts/pi-public-install-probe.mjs +++ b/scripts/pi-public-install-probe.mjs @@ -65,6 +65,7 @@ const started = performance.now(); const watchdog = setTimeout(() => { console.error('Pi public-install probe exceeded its cleanup deadline'); process.exit(1); }, 85_000); let bundle; let failure; +let commandFailure; let settledMs; try { const workspace = join(root, 'workspace'); await mkdir(workspace); @@ -81,11 +82,14 @@ try { baseInstructions: 'Credential-free admission probe. No prompt is submitted.', permissions: 'paperclip-runner-workspace-read-only', dynamicTools: [], }); - } catch (error) { failure = String(error); } + } catch (error) { failure = String(error); commandFailure = error.runnerCommandResult; } settledMs = Math.round(performance.now() - started); await bundle.transport.close(); assert.match(failure ?? '', /session\.open failed/); - assert.match(failure, /retryable=false, classification=session_ensure_failed/); + assert.match(commandFailure?.result?.message ?? '', /retryable=false, classification=session_ensure_failed/); + assert.equal(commandFailure?.result?.providerFailure?.schema, 'paperclip.provider-failure.v1'); + assert.equal(commandFailure.result.providerFailure.provider, 'pi'); + assert.equal(commandFailure.result.providerFailure.phase, 'startup'); assert.doesNotMatch(failure, /timed out/); assert.ok(settledMs < 60_000, `Pi admission took ${settledMs} ms`); assert.ok(evidence.some(item => item.runnerExited === true && item.runnerExitCode === 0)); diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index dd357024bfd..8e44c6f60c2 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -2,7 +2,8 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issues, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; -import { taskInterruptionPresentations } from "../services/task-interruption-presentation.js"; +import { taskInterruptionPresentation, taskInterruptionPresentations } from "../services/task-interruption-presentation.js"; +import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { captureInterruptionQuery, seedInterruptionFixture } from "./helpers/task-interruption-fixture.js"; @@ -15,6 +16,48 @@ describe("task interruption latest run query", () => { beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("interruption-query-"); db = createDb(database.connectionString); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + it("redacts registered run secrets from both task detail and list provider explanations", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + try { + const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const secret = `private-runtime-value-${randomUUID()}`; + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "unknown", message: `Provider rejected runtime value ${secret}.`, retryability: "unknown" }; + await db.update(heartbeatRuns).set({ resultJson: { providerFailure } }).where(eq(heartbeatRuns.id, run!.id)); + await createRunSecretRedactionRegistry(db).register(f.companyId, run!.id, secret); + const list = await taskInterruptionPresentations(db, f.companyId, f.input); + const detail = await taskInterruptionPresentation(db, f.input[0]!); + for (const presentation of [list.get(f.input[0]!.id), detail]) { + expect(presentation?.providerFailure?.message).toContain("Provider rejected runtime value"); + expect(JSON.stringify(presentation)).not.toContain(secret); + } + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id)))[0]?.resultJson).toEqual({ providerFailure }); + } finally { vi.unstubAllEnvs(); } + }); + + it.each(["decryption failure", "expanded message"])("keeps task reads available after provider display %s", async scenario => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + try { + const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const secret = "pw-short"; + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "unknown", message: secret + "šŸš€".repeat(4000 - secret.length), retryability: "unknown" }; + await db.update(heartbeatRuns).set({ resultJson: { providerFailure } }).where(eq(heartbeatRuns.id, run!.id)); + await createRunSecretRedactionRegistry(db).register(f.companyId, run!.id, secret); + if (scenario === "decryption failure") vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "b".repeat(64)); + const presentation = await taskInterruptionPresentation(db, f.input[0]!); + expect(presentation).toMatchObject({ state: "failed", canRetry: true }); + if (scenario === "decryption failure") expect(presentation).not.toHaveProperty("providerFailure"); + else { + expect(presentation?.providerFailure?.message).toContain("REDACTED"); + expect([...presentation!.providerFailure!.message]).toHaveLength(4000); + } + expect(JSON.stringify(presentation)).not.toContain(secret); + } finally { vi.unstubAllEnvs(); } + }); + it("does not inspect unrelated company history for a small issue page", async () => { const f = await seedInterruptionFixture(db, { selected: 2, history: 3, unrelated: 1500, payloadBytes: 4096 }); const query = await captureInterruptionQuery(db, taskInterruptionPresentations, f.companyId, f.input); diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 9b8a9810b2a..feafc254d80 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -379,6 +379,20 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( expect(f.deps.finalizeAgentStatus).toHaveBeenCalledOnce(); }); + it.each(["execution", "setup"] as const)("retains a provider startup cause at the outer %s settlement boundary", async phase => { + const f = await fixture(); + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", message: "This request would exceed your account's rate limit.", httpStatus: 429, retryability: "unknown" }; + const error = Object.assign(new Error("PRP command session.open failed"), { + runnerCommandResult: { result: { code: "command_execution_failed", providerFailure } }, + }); + f.deps.runLogStore.finalize.mockRejectedValueOnce(new Error("secondary log cleanup failure")); + if (phase === "setup") await f.completion.failRunSetup(f.setup(error)); + else await f.completion.failRun(f.failure(error)); + expect(await read(f.run.id)).toMatchObject({ status: "failed", error: "PRP command session.open failed", + resultJson: { providerFailure } }); + }); + it("still records failure and releases issue ownership when log finalization and progress flushing fail", async () => { const f = await fixture(); f.deps.runLogStore.finalize.mockRejectedValueOnce(new Error("log unavailable")); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 9a0696a1d11..a5a31d2ccf8 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1,4 +1,5 @@ import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; +import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; import { nativeTaskSessionPublicationHandled } from "../native-runtime/native-task-session-publication.js"; import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { measureRunPhase } from "../execution-preparation-timing.js"; @@ -1071,9 +1072,11 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC // A process adapter may throw while its owned Stop is joining the // child. Let the cancellation write settle before attempting failure. await processRunCancellationSettlements.get(run.id)?.settled; + const providerFailure = providerFailureFromError(err); + const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const message = redactCurrentUserText( identityRedactor.redact(err instanceof Error ? err.message : "Unknown adapter failure"), - await getCurrentUserRedactionOptions(), + currentUserRedactionOptions, ); const workspaceValidationFailure = isWorkspaceValidationFailure(err) ? err @@ -1155,6 +1158,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC errorMessage: message, resultJson: { ...parseObject(stopSnapshot?.resultJson), + ...(providerFailure ? { providerFailure: { ...providerFailure, + message: redactCurrentUserText(identityRedactor.redact(providerFailure.message), currentUserRedactionOptions) } } : {}), ...requiredWorkspaceRestoreEvidence, ...(workspaceValidationFailure?.resultJson ?? configurationIncompleteFailure?.resultJson ?? @@ -1308,11 +1313,13 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC signal.reason instanceof LegacyControllerLeaseLostError; // Setup code before adapter.execute threw (e.g. ensureRuntimeState, resolveWorkspaceForRun). // The inner catch did not fire, so we must record the failure here. + const providerFailure = providerFailureFromError(outerErr); + const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); const message = redactCurrentUserText( identityRedactor.redact(outerErr instanceof Error ? outerErr.message : "Unknown setup failure"), - await getCurrentUserRedactionOptions(), + currentUserRedactionOptions, ); // A missing secret/env binding is a known pre-dispatch configuration gap, // not an opaque setup crash. Surface it with its own errorCode so the @@ -1390,6 +1397,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const connectionFailure = readGitConnectionFailure(outerErr); const setupFailureResultJson = { ...setupFailureDetails, + ...(providerFailure ? { providerFailure: { ...providerFailure, + message: redactCurrentUserText(identityRedactor.redact(providerFailure.message), currentUserRedactionOptions) } } : {}), ...(connectionFailure ? { connectionFailure } : {}), // A lease lost during setup surfaces as the thrown abort reason, or as // the signal's reason when a setup step threw a different error after it. diff --git a/server/src/services/native-runtime/native-provider-failure-evidence.ts b/server/src/services/native-runtime/native-provider-failure-evidence.ts index cc89f4130b4..02f21282af2 100644 --- a/server/src/services/native-runtime/native-provider-failure-evidence.ts +++ b/server/src/services/native-runtime/native-provider-failure-evidence.ts @@ -19,8 +19,7 @@ async function observePersistedNativeTurnFailure( terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null; let execution; try { execution = parseNativeExecutionInput(record(run.runnerProfileJson).nativeExecutionInput); } catch { return null; } - if ((execution.provider.kind !== "codex" && execution.provider.kind !== "opencode") || - execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId || + if (execution.binding.runId !== run.id || execution.binding.companyId !== run.companyId || execution.binding.agentId !== run.agentId || execution.binding.issueId !== run.nativeIssueId || execution.session.normalizedSessionId !== run.nativeSessionId || execution.completionContract.id !== run.completionContractId || @@ -63,6 +62,7 @@ export async function readPersistedNativeFailureEvidence( return { providerFailure: observed?.forTerminal(turnId, terminal) ?? null, turnFailure: observed?.unclassifiedTurnFailure(turnId, terminal) ?? null, + providerExplanation: observed?.providerFailureForTerminal(turnId, terminal) ?? null, }; } diff --git a/server/src/services/native-runtime/native-provider-failure.test.ts b/server/src/services/native-runtime/native-provider-failure.test.ts index f8ce42f76f5..9406db76a27 100644 --- a/server/src/services/native-runtime/native-provider-failure.test.ts +++ b/server/src/services/native-runtime/native-provider-failure.test.ts @@ -19,6 +19,19 @@ const terminal: PrpTerminalState = { const observation = () => createNativeProviderFailureObservation({ kind: "codex", model }); describe("native provider failure observation", () => { + it.each(["acpx", "codex", "opencode"])("retains a typed explanation for %s without granting recovery authority", kind => { + const observed = createNativeProviderFailureObservation({ kind }); + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "new-provider", phase: "turn", + category: "unknown", message: "An unfamiliar provider rejection.", retryability: "unknown" }; + const failed = { ...event, payload: { status: "failed", error: { providerFailure } } }; + observed.observe({ ...failed, sourceKind: "control_plane" }); + expect(observed.providerFailureForTerminal("turn", terminal)).toBeNull(); + observed.observe(failed); + expect(observed.providerFailureForTerminal("turn", terminal)).toEqual(providerFailure); + expect(observed.forTerminal("turn", terminal)).toBeNull(); + expect(observed.providerFailureForTerminal("other-turn", terminal)).toBeNull(); + expect(observed.providerFailureForTerminal("turn", { ...terminal, runTerminalState: "succeeded" })).toBeNull(); + }); it("surfaces a structured capacity failure without copying arbitrary provider text", () => { const observed = createNativeProviderFailureObservation({ kind: "codex" }); observed.observe({ ...event, payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded", message: "Private arbitrary response" } } }); diff --git a/server/src/services/native-runtime/native-provider-failure.ts b/server/src/services/native-runtime/native-provider-failure.ts index db5d0e28814..ef858593b97 100644 --- a/server/src/services/native-runtime/native-provider-failure.ts +++ b/server/src/services/native-runtime/native-provider-failure.ts @@ -1,4 +1,5 @@ import type { PrpEvent, PrpTerminalState } from "../../vendor/paperclip-runner/index.js"; +import { readProviderFailure, type ProviderFailure } from "@paperclipai/shared"; export const NATIVE_MODEL_REJECTION_MESSAGE = "The selected model is not supported by the current ChatGPT connection. Choose a supported model or a compatible AI connection."; @@ -184,8 +185,14 @@ export function createNativeProviderFailureObservation(provider: { kind: string; let rejectedTurnId: string | null = null; let overloadedTurnId: string | null = null; const failedTurns = new Map(); + const providerFailures = new Map(); return { observe(event: PrpEvent) { + if (event.sourceKind === "runner" && event.eventType === "turn.failed" && event.turnId + && event.payload.status === "failed") { + const failure = readProviderFailure(record(event.payload.error).providerFailure); + if (failure && !providerFailures.has(event.turnId)) providerFailures.set(event.turnId, failure); + } if ((provider.kind !== "codex" && provider.kind !== "opencode") || event.sourceKind !== "runner" || event.eventType !== "turn.failed" || !event.turnId) return; const payload = record(event.payload); @@ -219,6 +226,10 @@ export function createNativeProviderFailureObservation(provider: { kind: string; error.message !== `The '${provider.model}' model is not supported when using Codex with a ChatGPT account.`) return; rejectedTurnId = event.turnId; }, + providerFailureForTerminal(turnId: string | null | undefined, terminal: PrpTerminalState) { + if (!turnId || terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null; + return providerFailures.get(turnId) ?? null; + }, forTerminal(turnId: string | null | undefined, terminal: PrpTerminalState) { if (!turnId || terminal.turnTerminalState !== "failed" || terminal.runTerminalState !== "failed") return null; if (turnId === overloadedTurnId) return { diff --git a/server/src/services/native-runtime/native-provider-overload.integration.test.ts b/server/src/services/native-runtime/native-provider-overload.integration.test.ts index 82516e895d2..08c328fc9ea 100644 --- a/server/src/services/native-runtime/native-provider-overload.integration.test.ts +++ b/server/src/services/native-runtime/native-provider-overload.integration.test.ts @@ -25,11 +25,13 @@ import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapter import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; import { buildExecutionContinuation } from "../execution-continuation.js"; import { waitForPendingRunFailureReports } from "../run-failure-report.js"; +import { projectTaskInterruption } from "../task-interruption-presentation.js"; describe("durable native provider capacity retry", () => { let temporary: Awaited>; let db: ReturnType; const companyId = randomUUID(), agentId = randomUUID(), otherAgentId = randomUUID(); + const providerDisplayMessage = "Distinctive upstream explanation for the task display only."; beforeAll(async () => { vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "unexpected"); temporary = await startEmbeddedPostgresTestDatabase("native-capacity-retry-"); @@ -60,10 +62,12 @@ describe("durable native provider capacity retry", () => { interactionResponses: [], credentialBindings: [], }; await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeExecutionInput: execution } }).where(eq(heartbeatRuns.id, runId)); + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "openai", phase: "turn", + category: "overloaded", message: providerDisplayMessage, retryability: "transient" }; const event: PrpEvent = { schema: "paperclip.prp.event.v1", runId, normalizedSessionId: native.normalizedSessionId, turnId: native.turnId, sourceInstanceId: native.runnerInstanceId, sourceEventId: "overload", sourceSeq: 1, sourceKind: "runner", eventType: "turn.failed", schemaVersion: 1, priority: 0, emittedAt: new Date().toISOString(), - payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded", message: NATIVE_PROVIDER_OVERLOADED_MESSAGE } }, + payload: { status: "failed", error: { codexErrorInfo: "serverOverloaded", message: NATIVE_PROVIDER_OVERLOADED_MESSAGE, providerFailure } }, }; const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "needs_review" }; const result = { ...CONTROL_PLANE_CONFORMANCE_RESULT, reportedWorkDisposition: "needs_review" as const, @@ -86,6 +90,10 @@ describe("durable native provider capacity retry", () => { for (let replay = 0; replay < 2; replay++) await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); expect(provider.execute).toHaveBeenCalledTimes(1); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]).toMatchObject({ status: "failed", error: NATIVE_PROVIDER_OVERLOADED_MESSAGE, errorCode: "native_provider_overloaded" }); + const [recoveredRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(recoveredRun.resultJson).toMatchObject({ providerFailure }); + expect(projectTaskInterruption(issue, { run: recoveredRun, paused: false, approval: false })) + .toMatchObject({ providerFailure }); expect(await db.select().from(statusDecisions).where(eq(statusDecisions.runId, runId))).toHaveLength(1); } @@ -110,6 +118,7 @@ describe("durable native provider capacity retry", () => { // Exhaustion ends automatic recovery, so Sentry receives the failure. expect(sentry.captureRunFailure).toHaveBeenCalledOnce(); expect(sentry.captureRunFailure.mock.calls[0]![0]).toMatchObject({ runId, runStatus: "failed" }); + expect(JSON.stringify(sentry.captureRunFailure.mock.calls)).not.toContain(providerDisplayMessage); expect((await db.select().from(issues).where(eq(issues.id, issueId)))[0]).toMatchObject({ status: "blocked", unblockDescriptor: { action: expect.stringContaining("Automatic retries exhausted") } }); break; } diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 50b626d9adb..1cbba5703cd 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -1391,7 +1391,7 @@ export async function finalizeNativeRun(input: { (!planWait || nativeSha256(planWait.result) !== nativeSha256(result))) { throw new Error("native_plan_wait_authority_lost"); } - const { providerFailure, turnFailure } = await readPersistedNativeFailureEvidence( + const { providerFailure, turnFailure, providerExplanation } = await readPersistedNativeFailureEvidence( input.db, run, resultRow.turnId, envelope.terminal as PrpTerminalState, ); const ownsProviderFailureDecision = @@ -1582,6 +1582,7 @@ export async function finalizeNativeRun(input: { ); const clearExecutionFailure = input.projectRunStatus && finalizationPhase === "committed" && terminalState === "succeeded"; const finalizationMetadata = { + ...(providerExplanation ? { providerFailure: providerExplanation } : {}), ...(providerFailure ? { nativeProviderFailure: providerFailure.diagnostic } : {}), ...(turnFailure ? { nativeTurnFailure: turnFailure } : {}), finalizationPhase, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 715e7c599a7..f6df58888bd 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js"; import { readClaimedNativeStopRecovery } from "./native-cancellation-request.js"; import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; @@ -10220,6 +10221,7 @@ async function executePaperclipNativeSessionWithinScope( const { exhausted } = recoveryProjection; const integrityFailure = sourceFailureCode === "native_event_replay_conflict"; + const providerFailure = providerFailureFromError(error); const message = redactIdentityText( error instanceof Error ? error.message : String(error), ).slice(0, 2_000); @@ -10259,6 +10261,7 @@ async function executePaperclipNativeSessionWithinScope( failureCode, failureDetail: identityRedactor.redact({ message, + ...(providerFailure ? { providerFailure } : {}), ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}), originalFailureCode: error instanceof NativeProviderTerminalFailure @@ -10390,7 +10393,9 @@ async function executePaperclipNativeSessionWithinScope( nativePhase: phase, nativePhaseUpdatedAt: now, error: message, - resultJson: sql`(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) - 'remoteRunnerRecovery') || ${JSON.stringify(remoteRunnerRecovery ? { remoteRunnerRecovery } : {})}::jsonb`, + resultJson: sql`(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) - 'remoteRunnerRecovery') + || ${JSON.stringify(remoteRunnerRecovery ? { remoteRunnerRecovery } : {})}::jsonb + || ${providerFailure ? sql`jsonb_build_object('providerFailure', coalesce(${heartbeatRuns.resultJson}->'providerFailure', ${JSON.stringify(providerFailure)}::jsonb))` : sql`'{}'::jsonb`}`, errorCode: ownershipUnverified ? NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE : sourceFailureCode, @@ -10671,6 +10676,7 @@ async function executePaperclipNativeSessionWithinScope( if (collectInstructions && !collectedByOwner) await timeController("instruction_collection", () => instructionCopy!.collectStopped()); } const providerFailure = providerFailureObservation.forTerminal(native.turnId, native.terminal); + const providerExplanation = providerFailureObservation.providerFailureForTerminal(native.turnId, native.terminal); // Keep the generic failure behavior, but record why the turn failed. const unclassifiedTurnFailure = providerFailureObservation.unclassifiedTurnFailure(native.turnId, native.terminal); const adapterResult: AdapterExecutionResult = { @@ -10688,6 +10694,7 @@ async function executePaperclipNativeSessionWithinScope( ...((input.execution.schema === "paperclip.native-execution-input.v6" || input.execution.schema === "paperclip.native-execution-input.v8") ? { providerAccounting: { usage: null, cost: null, externallyBilled: true }, externalStopConfirmed: false } : {}), ...(providerFailure ? { nativeProviderFailure: providerFailure.diagnostic } : {}), ...(unclassifiedTurnFailure ? { nativeTurnFailure: unclassifiedTurnFailure } : {}), + ...(providerExplanation ? { providerFailure: providerExplanation } : {}), ...(native.goalRolloverRequired ? { goalRolloverRequired: true } : {}), planSynchronizations, }, diff --git a/server/src/services/task-interruption-presentation.test.ts b/server/src/services/task-interruption-presentation.test.ts index 9c730d1c5fc..7c2b98bc963 100644 --- a/server/src/services/task-interruption-presentation.test.ts +++ b/server/src/services/task-interruption-presentation.test.ts @@ -7,6 +7,17 @@ const run = { id: "run", agentId: "agent", status: "failed", scheduledRetryAt: n resultJson: { projectAccessDenied: { projectId: "22222222-2222-4222-8222-222222222222" } } }; const agent = { status: "idle", pauseReason: null }; +it("preserves a provider explanation on a failed task without changing retry authority", () => { + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", message: "API error: 429 This request would exceed your account's rate limit.", + httpStatus: 429, retryability: "unknown" }; + const failedRun = { ...run, errorCode: "native_session_interrupted", resultJson: { providerFailure } }; + const presentation = projectTaskInterruption(issue, { run: failedRun, agent, paused: false, approval: false }); + expect(presentation).toMatchObject({ state: "failed", canRetry: true, providerFailure }); + expect(projectTaskInterruption({ ...issue, status: "done" }, { run: failedRun, agent, paused: false, approval: false })) + .not.toHaveProperty("providerFailure"); +}); + describe("task project-access interruption", () => { it("explains a project denial without offering a futile retry", () => { expect(projectTaskInterruption(issue, { run, agent, paused: false, approval: false })).toMatchObject({ diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index 8f2820fee54..fc0b5574abc 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -2,13 +2,17 @@ import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; import { agents, agentWakeupRequests, approvals, companies, heartbeatRuns, issueApprovals, issueRecoveryActions, issues, issueTreeHoldMembers, issueTreeHolds, projects, type Db } from "@paperclipai/db"; import { unionAll } from "drizzle-orm/pg-core"; -import { HEARTBEAT_RUN_STATUSES, type TaskInterruptionPresentation } from "@paperclipai/shared"; +import { HEARTBEAT_RUN_STATUSES, readProviderFailure, type TaskInterruptionPresentation } from "@paperclipai/shared"; import { captureTaskInterruptionFailure } from "../sentry.js"; import { isAiAuthenticationBlocked } from "./ai-auth-failure.js"; import { readSystemRecovery } from "./system-recovery-state.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { authorizationService } from "./authorization.js"; import { taskWorkspaceProjectRequirements } from "./task-workspace-access.js"; +import { createRunSecretRedactionRegistry } from "./run-secret-redaction.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { redactCurrentUserText } from "../log-redaction.js"; +import { logger } from "../middleware/logger.js"; type ProjectionIssue = Pick; type ProjectionRun = Pick issue.companyId === companyId); const result = new Map(); @@ -110,7 +116,8 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i and ${heartbeatRuns.error} = 'Project is outside this low-trust boundary.' then ${heartbeatRuns.error} else null end`, errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, resultJson: sql>`jsonb_build_object('configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete', - 'projectAccessDenied', ${heartbeatRuns.resultJson}->'projectAccessDenied')`, + 'projectAccessDenied', ${heartbeatRuns.resultJson}->'projectAccessDenied', + 'providerFailure', ${heartbeatRuns.resultJson}->'providerFailure')`, }).from(requested).innerJoinLateral(latestRuns, sql`true`).innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, latestRuns.id))) .leftJoin(agents, and(eq(agents.companyId, heartbeatRuns.companyId), eq(agents.id, heartbeatRuns.agentId))) .orderBy(requestedIssueId), @@ -187,6 +194,27 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i agentAdapter: run.agentAdapter ?? "unknown" }); } } + const disclosures = [...result.values()].flatMap(presentation => presentation?.providerFailure && presentation.observedRunId + ? [{ id: presentation.observedRunId, providerFailure: presentation.providerFailure, presentation }] : []); + if (disclosures.length) { + try { + const [safe, settings] = await Promise.all([ + createRunSecretRedactionRegistry(db).redactForRuns(companyId, disclosures.map(({ id, providerFailure }) => ({ id, providerFailure }))), + instanceSettingsService(db).getGeneral(), + ]); + disclosures.forEach(({ presentation }, index) => { + const value = safe[index]?.providerFailure; + const message = typeof value?.message === "string" + ? [...redactCurrentUserText(value.message, { enabled: settings.censorUsernameInLogs })].slice(0, 4000).join("") : ""; + const failure = readProviderFailure({ ...value, message }); + if (failure) presentation.providerFailure = failure; + else delete presentation.providerFailure; + }); + } catch { + for (const { presentation } of disclosures) delete presentation.providerFailure; + logger.warn({ companyId, runCount: disclosures.length }, "Could not redact provider failure display; explanations omitted"); + } + } return result; } diff --git a/server/src/vendor/paperclip-runner/index.ts b/server/src/vendor/paperclip-runner/index.ts index 2abc5ca4213..3df1e702c6e 100644 --- a/server/src/vendor/paperclip-runner/index.ts +++ b/server/src/vendor/paperclip-runner/index.ts @@ -88,6 +88,7 @@ const sourceUrl = new URL( const runner = (await import(sourceUrl.href)) as RunnerModule; export const DurablePrpControlPlane = runner.DurablePrpControlPlane; +export const providerFailureFromError = runner.providerFailureFromError; export const decodeDurablePrpCheckpoint = runner.decodeDurablePrpCheckpoint; export const inspectDurablePrpCheckpointHeader = runner.inspectDurablePrpCheckpointHeader; export const RUNNER_PRP_PROTOCOL_MIN = runner.RUNNER_PRP_PROTOCOL_MIN; diff --git a/ui/src/components/ExecutionBlockerNotice.test.tsx b/ui/src/components/ExecutionBlockerNotice.test.tsx index e79d926ebca..bfe07d25284 100644 --- a/ui/src/components/ExecutionBlockerNotice.test.tsx +++ b/ui/src/components/ExecutionBlockerNotice.test.tsx @@ -58,4 +58,16 @@ describe("ordinary task interruption", () => { expect(container.querySelector('[role="alert"]')?.textContent).toContain("Your task access changed."); expect(container.querySelectorAll("button")).toHaveLength(1); }); + it("shows an unknown provider's original explanation as text and preserves read-only access", async () => { + const providerFailure = { schema: "paperclip.provider-failure.v1" as const, provider: "new-provider", phase: "startup" as const, + category: "unknown" as const, retryability: "unknown" as const, httpStatus: 429, + message: "Upstream failure " }; + await render({ ...failed, providerFailure }, { canRetry: false }); + expect(container.textContent).toContain(providerFailure.message); + expect(container.textContent).toContain("Failed before work started"); + expect(container.textContent).toContain("HTTP 429"); + expect(container.querySelector("img, button")).toBeNull(); + await render({ ...failed, providerFailure, state: "resolved" }); + expect(container.textContent).toBe(""); + }); }); diff --git a/ui/src/components/ExecutionBlockerNotice.tsx b/ui/src/components/ExecutionBlockerNotice.tsx index 0a343207d22..721dfda3a0f 100644 --- a/ui/src/components/ExecutionBlockerNotice.tsx +++ b/ui/src/components/ExecutionBlockerNotice.tsx @@ -4,6 +4,7 @@ import { Button } from "./ui/button"; import { cn } from "../lib/utils"; import { useSecondTick } from "../hooks/useSecondTick"; import { TaskProjectAccessNotice } from "./TaskProjectAccessNotice"; +import { ProviderFailureNotice } from "./ProviderFailureNotice"; /** One ordinary interruption surface. Technical records remain in diagnostics. */ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimistic, takingLonger, error, className, issueId, agentName, projectName }: { @@ -24,6 +25,12 @@ export function ExecutionBlockerNotice({ presentation, canRetry, onRetry, optimi canRetry={canRetry && presentation.canRetry} onRetry={onRetry} error={error} className={className} />; if (error && !optimistic && !takingLonger) return
{error.message}{canRetry && }
; if (takingLonger) return
Taking longer than usualā€¦{canRetry && }
; + if (!optimistic && presentation?.providerFailure && ["failed", "retry_scheduled", "waiting_for_user"].includes(presentation.state)) { + return Retry : undefined}> + {presentation.state === "retry_scheduled" &&

Paperclip will retry automatically.

} + ; + } if (optimistic || !presentation || presentation.responsibility !== "system" || !["failed", "retry_scheduled"].includes(presentation.state)) return null; const seconds = Math.max(0, Math.ceil((Date.parse(presentation.nextRetryAt ?? "") - Date.now()) / 1000)); diff --git a/ui/src/components/ProviderFailureNotice.tsx b/ui/src/components/ProviderFailureNotice.tsx new file mode 100644 index 00000000000..9ad8fa86d08 --- /dev/null +++ b/ui/src/components/ProviderFailureNotice.tsx @@ -0,0 +1,30 @@ +import type { ReactNode } from "react"; +import type { ProviderFailure } from "@paperclipai/shared"; +import { InlineBanner } from "./InlineBanner"; + +const labels: Record = { + rate_limit: "Rate limit reached", quota_exhausted: "Usage allowance exhausted", + authentication: "Authentication failed", permission: "Access denied", + model_unavailable: "Model unavailable", invalid_request: "Request rejected", + overloaded: "Provider at capacity", timeout: "Provider timed out", unknown: "Provider error", +}; + +export function ProviderFailureNotice({ failure, actions, children, className }: { + failure: ProviderFailure; actions?: ReactNode; children?: ReactNode; className?: string; +}) { + const provider = ({ anthropic: "Claude", openai: "OpenAI" } as Record)[failure.provider] ?? failure.provider; + return ( + +
+

{failure.message}

+

+ {failure.phase === "startup" ? "Failed before work started" : "Failed during work"} + {failure.httpStatus ? ` Ā· HTTP ${failure.httpStatus}` : ""} + {failure.providerCode ? ` Ā· ${failure.providerCode}` : ""} +

+ {failure.requestId &&

Request: {failure.requestId}

} + {children} +
+ + ); +} diff --git a/ui/src/pages/AgentDetail.tsx b/ui/src/pages/AgentDetail.tsx index f8093cbc2cb..575d92b67e5 100644 --- a/ui/src/pages/AgentDetail.tsx +++ b/ui/src/pages/AgentDetail.tsx @@ -3,6 +3,8 @@ import { grokBotApi } from "@/api/grokBot"; import { GrokBotWorkSummary } from "@/components/GrokBotWorkSummary"; import { PrimaryAgentCrownButton } from "@/components/primary-agent/PrimaryAgentPresentation"; import { AiConnectionPoolRunDetails } from "@/components/ai-connections/AiConnectionPoolRunDetails"; +import { ProviderFailureNotice } from "@/components/ProviderFailureNotice"; +import { readProviderFailure } from "@paperclipai/shared"; import { AgentConnectionInstructions } from "@/features/connections/ConnectionInstructions"; import type { AgentInstructionCandidate, AgentInstructionsBundle } from "@paperclipai/shared"; import { InstructionHistory } from "../components/InstructionHistory"; @@ -3268,7 +3270,9 @@ function RunDetail({ run: initialRun, agentRouteId, adapterType, adapterConfig } )} - {run.error && ( + {readProviderFailure(run.resultJson?.providerFailure) ? ( + + ) : run.error && (
{run.error} {run.errorCode && ({run.errorCode})}