Files
PaperClipAI/tests/runner-e2e/failure-classifier.ts
DottaandPaperclip 4c6adcadcb test(runner-e2e): verify pending Pi input after provider loss
Add one explicit Daytona Product cell that admits the exact Pi child before faulting it, then requires production expiry of the original native question, failed-run Blocked disposition, stale-answer rejection, distinct unanswered fallback, and no replay through owned retirement. Retain the existing local scope and pending qualification.

Calibrate public lifecycle evidence, generated observer one-shot dispatch, and candidate failure classification. Runtime, provider profile, dependency closure, deadlines, and lockfiles are unchanged.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-10-01 19:26:25 -05:00

62 lines
3.4 KiB
TypeScript

import { ObservedStateTimeout, RemoteAdmissionReadError } from "./api.js";
import type { FailureClass } from "./types.js";
const TRANSIENT =
/(?:\b429\b|\b5\d\d\b|rate.?limit|ECONN(?:RESET|REFUSED)|socket hang up|network (?:error|interruption|timeout)|service unavailable|(?:provider|server|bootstrap|browser|webserver|health|daytona|sandbox|ingress|preview|connection|harness).*(?:temporar|timed? out|timeout|closed|failed|unavailable|interrupt|reset|refused|create|start|connect)|(?:timed? out|timeout).*(?:provider|server|bootstrap|browser|webserver|health|daytona|sandbox|ingress|preview|connection|harness))/i;
const PERMANENT =
/(?:missing (?:credential|fixture secret)|invalid.*(?:credential|api key)|unauthorized|forbidden|qualification|model.*(?:unsupported|incompatible)|artifact.*incompatible|runner_remote_.*(?:incompatible|unavailable)|immutable image digest)/i;
// These controller checkpoint failures are reproducible protocol defects, even
// when their outer error is described as a provider transport failure.
const NON_RETRYABLE_SESSION_CLOSE =
/(?:native_session_close_unrecoverable|runner did not durably suspend before checkpoint)/i;
// Recovery cannot repair a provider session-open rejection that explicitly
// declares itself non-retryable. Keep generic transport/start failures transient.
const NON_RETRYABLE_ACPX_SESSION_OPEN =
/native_session_recovery_failed[\s\S]*ACPX sidecar command session\.open was rejected\s*\([^)]*\bretryable\s*=\s*false\b/i;
const CANDIDATE =
/(?:matcher|expected.*observed|marker|issue status|run status|runtime mode|wrong output|missing output)/i;
// File-transfer deadlines are sandbox transport failures. Do not generalize
// this to all RPC timeouts: runner protocol defects must remain visible.
const SANDBOX_TRANSFER_TIMEOUT =
/RPC call "environmentSync(?:In|Out)" timed out after \d+ms/i;
export function classifyFailure(error: unknown): FailureClass {
if (error instanceof ObservedStateTimeout || error instanceof RemoteAdmissionReadError) return error.failureClass;
const message =
error instanceof Error ? `${error.name}: ${error.message}` : String(error);
if (/browser bootstrap failed before task creation/i.test(message))
return "transient_infrastructure";
if (/secret.*(?:leak|plaintext|redaction)/i.test(message))
return "secret_leak";
if (/cleanup|teardown|lease.*release/i.test(message)) {
return TRANSIENT.test(message)
? "transient_infrastructure"
: "cleanup_failure";
}
if (PERMANENT.test(message)) return "permanent_infrastructure";
// A deliberately killed Pi child is the fixture stimulus. Missing lifecycle
// proof must not become a retryable transport failure because of its wording.
if (/pi_provider_death_proof:/.test(message)) return "candidate_failure";
if (
/chat_idle_state_invariant/.test(message) ||
NON_RETRYABLE_SESSION_CLOSE.test(message) ||
NON_RETRYABLE_ACPX_SESSION_OPEN.test(message)
)
return "candidate_failure";
if (TRANSIENT.test(message) || SANDBOX_TRANSFER_TIMEOUT.test(message))
return "transient_infrastructure";
if (CANDIDATE.test(message)) return "candidate_failure";
return "candidate_failure";
}
export function shouldRetryFailure(
failureClass: FailureClass,
maxAutomaticRetries = 1,
) {
return (
maxAutomaticRetries > 0 &&
(failureClass === "transient_infrastructure" ||
failureClass === "provider_variance")
);
}