diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 2b435c75464..60dd9a190ff 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -45,6 +45,7 @@ import { type DurableRecoveryIdentity, type DurableWarmRunTransition, } from "./prp-transport-types.js"; +import { crashPoint } from "../crash-point.js"; const protocol = "paperclip.runner"; export const RUNNER_PRP_PROTOCOL_MIN = 1; @@ -4030,9 +4031,12 @@ export class DurablePrpControlPlane { this.#ackWarmTransition(connection, receipt); return; } + const turnStart = command.type === "turn.start"; + if (turnStart) crashPoint("runner.turn_start.after_runner_accept_before_controller_checkpoint"); command.status = status; command.result = structuredClone(result); this.#store.save(); + if (turnStart) crashPoint("runner.turn_start.after_controller_journal_before_db_checkpoint"); this.#ackTerminalCommandResult(connection, command); this.#releaseServingLifecycle(connection, command); if (!this.#isTerminalLifecycleCommand(command) || connection.terminalLifecycleCommandId === null) { diff --git a/packages/paperclip-runner/src/crash-point.ts b/packages/paperclip-runner/src/crash-point.ts new file mode 100644 index 00000000000..bfa45677208 --- /dev/null +++ b/packages/paperclip-runner/src/crash-point.ts @@ -0,0 +1,50 @@ +/** + * Runner-side crash points: the same `PAPERCLIP_CRASH_POINT` contract as + * `@paperclipai/shared/crash-points`, for the controller code in this package + * (it runs inside the Paperclip server process). This package does not depend + * on `@paperclipai/shared`, so it keeps this small copy; the names are checked + * against the shared registry by server/src/__tests__/crash-point-call-sites.test.ts. + * + * Disarmed (always under NODE_ENV=production) a call is one comparison. + * runnerd never sees the variable: its environment is an allowlist. + */ +import { appendFileSync, existsSync } from "node:fs"; + +export type RunnerCrashPointName = + | "runner.process.after_spawn_before_identity" + | "runner.turn_start.after_runner_accept_before_controller_checkpoint" + | "runner.turn_start.after_controller_journal_before_db_checkpoint"; + +type Armed = { name: string; nth: number; hits: number; receiptPath: string | null; gatePath: string | null }; + +function armFromEnvironment(): Armed | null { + const env = typeof process === "undefined" ? undefined : process.env; + const raw = env?.PAPERCLIP_CRASH_POINT?.trim(); + if (!env || !raw || env.NODE_ENV === "production") return null; + const separator = raw.lastIndexOf(":"); + const name = separator === -1 ? raw : raw.slice(0, separator); + // The shared helper validates the name; this copy only arms its own names. + if (!name.startsWith("runner.")) return null; + const nth = separator === -1 ? 1 : Number(raw.slice(separator + 1)); + if (!Number.isInteger(nth) || nth < 1) return null; + return { name, nth, hits: 0, receiptPath: env.PAPERCLIP_CRASH_POINT_RECEIPT?.trim() || null, + gatePath: env.PAPERCLIP_CRASH_POINT_GATE?.trim() || null }; +} + +const armed: Armed | null = armFromEnvironment(); + +export function crashPoint(name: RunnerCrashPointName): void { + if (armed === null || armed.name !== name) return; + if (armed.gatePath !== null) { + if (!existsSync(armed.gatePath)) return; + armed.gatePath = null; + } + armed.hits += 1; + if (armed.hits !== armed.nth) return; + const receipt = { point: name, hit: armed.hits, pid: process.pid, at: new Date().toISOString() }; + if (armed.receiptPath) { + try { appendFileSync(armed.receiptPath, `${JSON.stringify(receipt)}\n`); } catch { /* the log line still records it */ } + } + try { process.stderr.write(`[crash-point] SIGKILL ${JSON.stringify(receipt)}\n`); } catch { /* ignore */ } + process.kill(process.pid, "SIGKILL"); +} diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 07f74b734af..7ce9f59ce14 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -104,6 +104,7 @@ import { } from "../drivers/runtime-context-materializer.js"; import { resolvePackagedRunnerBinary } from "./runner-binary.js"; import { RUNNERD_CANONICAL_ITEM, RUNNERD_RUNTIME_ACTIVITY } from "../drivers/codex/codex-driver-values.js"; +import { crashPoint } from "../crash-point.js"; // URL directory conversion preserves a trailing separator while path-derived // build artifacts do not. Normalize once so a source build cannot be @@ -5990,6 +5991,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { }); this.#handle = handle; this.#watchRunner(handle); + // A remote launcher returns before runnerd exists; only a local spawn reaches this gap. + if (!this.options.runnerProcessLauncher) crashPoint("runner.process.after_spawn_before_identity"); await this.#publishSpawnedProcess(handle); this.#assertStartupOpen(); await registration?.activate?.(); @@ -6827,6 +6830,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { if (handle) { this.#handle = handle; this.#watchRunner(handle); + // A remote launcher returns before runnerd exists; only a local spawn reaches this gap. + if (!this.options.runnerProcessLauncher) crashPoint("runner.process.after_spawn_before_identity"); await this.#publishSpawnedProcess(handle); } this.#assertStartupOpen(); diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts index b796ac92cfc..a8b4a023551 100644 --- a/packages/shared/src/crash-point-registry.ts +++ b/packages/shared/src/crash-point-registry.ts @@ -45,8 +45,8 @@ export const CRASH_POINTS = { invariants: ["I2", "I5", "I7"], }, "runner.process.after_spawn_before_identity": { - description: "runnerd is spawned (own process group, survives the server); its pid and process identity are not yet recorded on the run. Recovery must find or stop the orphan by identity.", - environments: ["local", "daytona"], + description: "runnerd is spawned (own process group, survives the server); its pid and process identity are not yet recorded on the run. Recovery must find or stop the orphan by identity. Local spawns only: a remote launcher returns before runnerd exists.", + environments: ["local"], kinds: ["native_executor_claim", "detached_process_marker"], invariants: ["I1", "I2", "I5"], }, diff --git a/server/src/__tests__/crash-point-call-sites.test.ts b/server/src/__tests__/crash-point-call-sites.test.ts new file mode 100644 index 00000000000..ea26342286b --- /dev/null +++ b/server/src/__tests__/crash-point-call-sites.test.ts @@ -0,0 +1,52 @@ +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { CRASH_POINTS } from "@paperclipai/shared/crash-points"; +import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../../.."); +const SOURCE_ROOTS = ["server/src", "packages/paperclip-runner/src"]; +const CALL = /\bcrashPoint\(\s*"([^"]+)"\s*\)/g; + +function sourceFiles(directory: string): string[] { + return readdirSync(directory, { withFileTypes: true }).flatMap(entry => { + const full = path.join(directory, entry.name); + if (entry.isDirectory()) return entry.name === "node_modules" || entry.name === "__tests__" ? [] : sourceFiles(full); + return /\.tsx?$/.test(entry.name) && !/\.test\.tsx?$/.test(entry.name) ? [full] : []; + }); +} + +function callSites(): Map { + const sites = new Map(); + for (const root of SOURCE_ROOTS) { + for (const file of sourceFiles(path.join(repositoryRoot, root))) { + for (const match of readFileSync(file, "utf8").matchAll(CALL)) { + const list = sites.get(match[1]!) ?? []; + list.push(path.relative(repositoryRoot, file)); + sites.set(match[1]!, list); + } + } + } + return sites; +} + +describe("crash-point call sites", () => { + const sites = callSites(); + + it("instruments every registered crash point", () => { + const missing = Object.keys(CRASH_POINTS).filter(name => !sites.has(name)); + expect(missing).toEqual([]); + }); + + it("calls only registered crash points", () => { + const unknown = [...sites.keys()].filter(name => !(name in CRASH_POINTS)); + expect(unknown).toEqual([]); + }); + + it("names only ownable kinds of the Phase 1 registry", () => { + const known = new Set(OWNABLE_KINDS.map(kind => kind.id)); + const unknown = Object.entries(CRASH_POINTS).flatMap(([name, definition]) => + definition.kinds.filter(kind => !known.has(kind)).map(kind => `${name}: ${kind}`)); + expect(unknown).toEqual([]); + }); +}); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index bc9f3b4df0f..3e50b257a80 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -110,6 +110,7 @@ import { type SandboxOrphanCleanupSpool, } from "./sandbox-orphan-cleanup-spool.js"; import { logger } from "../middleware/logger.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; // The constant error kind for the durable orphan-cleanup-write-failed log. The // log never reads the caught exception, because the exception can carry a @@ -2286,6 +2287,7 @@ function createSandboxEnvironmentDriver( } catch { cleanupStatus = "failed"; } + if (cleanupStatus === "success") crashPoint("environment.lease_release.after_provider_before_lease"); const releaseStatus = input.lease.leasePolicy === "retain_on_failure" && input.status === "failed" ? "retained" as const : input.status; @@ -3025,6 +3027,8 @@ function createSandboxEnvironmentDriver( } else { cleanupStatus = "failed"; } + // Daytona and other plugin providers release here, not in the built-in path. + if (cleanupStatus === "success") crashPoint("environment.lease_release.after_provider_before_lease"); // A failed release verification leaves the provider resource active. The // cleanup reaper retries only `pending_cleanup` leases, so route a failed @@ -3112,6 +3116,8 @@ function createSandboxEnvironmentDriver( } catch { cleanupStatus = "failed"; } + // Only a destroy the provider confirmed reaches this gap. + if (cleanupStatus === "success") crashPoint("environment.teardown.after_destroy_before_lease"); return await environmentsSvc.releaseLease( input.lease.id, @@ -3736,6 +3742,7 @@ export function environmentRuntimeService( )`, )).returning(), { companyId: input.leaseRow.companyId, runId: input.leaseRow.heartbeatRunId ?? undefined }); if (!claimed) return null; + crashPoint("environment.teardown.after_claim_before_destroy"); const lease = toEnvironmentLeaseSnapshot(claimed); let renewalInFlight = false; const renewal = setInterval(() => { @@ -3972,6 +3979,7 @@ export function environmentRuntimeService( sql`${environmentLeases.metadata}->${intentKey} = ${JSON.stringify(leaseSnapshot.metadata?.[intentKey])}::jsonb`, )).returning(), { companyId: leaseRow.companyId, runId: leaseRow.heartbeatRunId ?? undefined }); if (!claimed) continue; + crashPoint("environment.stop_retain.after_claim_before_stop"); const snapshot = toEnvironmentLeaseSnapshot(claimed); try { const driver = requireDriverKey(getLeaseDriverKey(snapshot, environment)); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index ea26ccf6390..52f70ed5ac2 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -52,6 +52,7 @@ import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-wo import { nativeRunHasNoWorkspaceAccess } from "./workspace-restore-recovery-state.js"; import { prepareAndSelectHeartbeatRuntime } from "./heartbeat/runtime-selection.js"; import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; export { MAX_TURN_CONTINUATION_WAKE_REASON, parseSandboxProviderPluginNotReadyFailureMessage, @@ -2947,6 +2948,7 @@ export function heartbeatService( reserveRunBudget(db, run.companyId, run.id, readNonEmptyString(runLedgerScope.projectId), runLedgerScope, runOptions.nativeLeaseOwner)) .catch(async (error: unknown) => { throw await relinquished() ? new NativeControllerDetachedForRestartError() : error; }); + crashPoint("budget.reservation.after_hold_before_dispatch"); let entered = false, handedOff = false; try { return await dispatchResolvedInteractionContinuationAfterAdmission((markDispatchStarted) => { @@ -3934,6 +3936,7 @@ export function heartbeatService( providerResourceDisposition: providerResourceDispositionForRun, nativeLifecycleTelemetry: nativeLifecycleTelemetryForRun, })); + crashPoint("environment.lease.after_release_before_runtime_services"); await releaseRuntimeServicesForRun(run.id).catch(() => undefined); caseHandoffCleanupSettled = true; } diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 8acb6924cf7..3fd4eef82a8 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -191,6 +191,7 @@ import { isBoardOwnedBudgetBlock } from "../budget-block-ownership.js"; import type { instanceSettingsService } from "../instance-settings.js"; import type { createRunDispatch } from "../../modules/run-dispatch/index.js"; import { hasRetainedExternalChatRetry, type PostCommitEffect as WakeQueuePostCommitEffect, type createWakeQueue } from "../../modules/wake-queue/index.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; type HeartbeatRunState = ReturnType; @@ -1550,6 +1551,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende publishRunLifecyclePluginEvent(claimed); if (!nativeReviewContext) { + crashPoint("wake.claim.after_run_running_before_wakeup_claimed"); await setWakeupStatus(claimed.wakeupRequestId, "claimed", { claimedAt }); } diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index d61039733f6..e584d1fb407 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -107,6 +107,7 @@ import { import { isNativeRunnerOwnershipHeld, } from "./native-runner-ownership.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) @@ -1697,6 +1698,7 @@ export async function finalizeNativeRun(input: { : undefined, failpoint: input.failpoint, }); + crashPoint("native.finalize.after_decision_before_projection"); const now = new Date(); const finalizationFailed = decision.effects.some( (effect) => effect.kind === "record_finalization_error", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 51704b21a96..0e6dca5c55b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -24,6 +24,7 @@ import { rehydratePinnedSkillAssets } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeRecoveryUsageDelivery } from "./native-recovery-usage-delivery.js"; import { createNativeTurnAccounting, NATIVE_LOST_TURN_USAGE_EVIDENCE, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js"; import { measureRunnerOperation, recordRunnerTimingReceipt } from "./runner-operation-timing.js"; import { formatNativeRunLogEvent } from "./native-run-log-export.js"; @@ -10930,6 +10931,7 @@ async function executePaperclipNativeSessionWithinScope( // Binds a later identity-less stop proof to the host that launched. payload: { hostname: hostname() }, }); + crashPoint("native.process_start.after_requested_before_spawn"); }, input: runnerExecution, startup: executionContext && (!usesPluginTask || executionContext.pluginSupported) ? executionContext : undefined, diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index 649eeedcb97..39af5d919de 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -35,6 +35,7 @@ import { createNativeControllerRecovery, NATIVE_CONTROLLER_RECOVERY_MAX_BYTES, t import { eligibleNativeTaskSessionPublication, publishNativeTaskSessionInTransaction, NATIVE_TASK_SESSION_PUBLICATION_KEY, type NativeTaskSessionPublicationContext, type NativeTaskSessionPublicationIssue } from "./native-task-session-publication.js"; import { lockNativeRunnerCheckpointOwner, nativeRunnerCheckpointOwnerLock, validateNativeRunnerCheckpointOwner, inspectNativeRunnerUpgradeHeader, type NativeRunnerCheckpointOwner } from "./native-runner-checkpoint.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; export interface PaperclipControlPlaneBinding { companyId: string; @@ -632,6 +633,7 @@ export class PaperclipControlPlanePort implements ControlPlanePort { this.#assertActive(options); return true; }); + if (insertedResult) crashPoint("native.result.after_accept_before_finalize"); if (insertedResult) { try { await this.#onResultCommitted?.(); } catch {} } } } diff --git a/server/src/services/run-cost-accounting.ts b/server/src/services/run-cost-accounting.ts index 2ed480afa39..3240d034f82 100644 --- a/server/src/services/run-cost-accounting.ts +++ b/server/src/services/run-cost-accounting.ts @@ -21,6 +21,7 @@ import { priceAnthropicReceipt } from "./anthropic-pricing.js"; import { priceCodexReceipt } from "./codex-pricing.js"; import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; import { replayUsageReceipts, indexPendingUsageReceipts, readPendingRunUsageReceipts, persistUsageReceiptInTransaction, reconcileLateRunUsageReceiptInTransaction, type UsageReceiptIndex } from "./usage-receipts.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; const terminalStatuses = ["succeeded", "failed", "timed_out", "cancelled", "interrupted"]; const object = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; @@ -422,6 +423,8 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService timeoutMinutes: ACCOUNTING_CLOSURE_TIMEOUT_MS / 60_000 } }, publications); return true; }), { "budget.accounting.owner": "run" }); + // Only a closed account reaches this gap; an open one returned false above. + if (accounted) crashPoint("accounting.closure.after_commit_before_spool_unlink"); for (const file of recoveredFiles) { try { await fs.rm(file, { force: true }); } catch (error) { logger.warn({ err: error, runId }, "Accounting committed; recovered receipt cleanup will retry"); } diff --git a/server/src/services/sandbox-creation-operations.ts b/server/src/services/sandbox-creation-operations.ts index 0b5b7bee8c7..8e11ce48c37 100644 --- a/server/src/services/sandbox-creation-operations.ts +++ b/server/src/services/sandbox-creation-operations.ts @@ -8,6 +8,7 @@ import { assertEnvironmentCompanyBinding, environmentService } from "./environme import { ENVIRONMENT_CLEANUP_RETRY_MS, recoverableSandboxCreationCondition } from "./environment-cleanup-state.js"; import { logger } from "../middleware/logger.js"; import { notifyEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSelect; @@ -122,7 +123,9 @@ export async function createOwnedSandbox( metadata: { ...existing.metadata, creationOperationId: operation.id } }; } try { + crashPoint("sandbox.create.after_intent_before_provider"); const lease = await create({ id, runId }); + crashPoint("sandbox.create.after_provider_before_lease"); return { ...lease, metadata: { ...lease.metadata, creationOperationId: id } }; } catch (error) { if (recoverableTaskCreation(operation)) { diff --git a/server/src/services/usage-receipts.ts b/server/src/services/usage-receipts.ts index fa216c42a1b..e8c45042aea 100644 --- a/server/src/services/usage-receipts.ts +++ b/server/src/services/usage-receipts.ts @@ -22,6 +22,7 @@ import { applyCostCorrectionInTransaction } from "./accounting-corrections.js"; import { logActivity, type ActivityPublication } from "./activity-log.js"; import { createCostEventInTransaction } from "./costs.js"; import { queueBudgetEvaluation } from "./budget-evaluation-work.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; const token = z.number().int().nonnegative().max(2_147_483_647); const price = z.number().finite().nonnegative().nullable().optional(); @@ -694,7 +695,12 @@ export async function createRunUsageRecorder(db: Db, input: { companyId: string; } const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), ...input, sourceId, sequence: ++sequence, receivedAt: new Date().toISOString(), receipt, liveCostCents }; const file = await spoolUsageReceipt(envelope, directory); - try { await persistUsageReceipt(db, envelope); await fs.rm(file, { force: true }); } + crashPoint("accounting.usage_receipt.after_spool_before_db"); + try { + await persistUsageReceipt(db, envelope); + crashPoint("accounting.usage_receipt.after_db_before_unlink"); + await fs.rm(file, { force: true }); + } catch (error) { // The fsynced receipt is durable even when the DB transaction failed. // Wake the owned spool recovery path; never certify that failed write as free. diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index ab2aa012b7f..06c6358d6b9 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -13,6 +13,7 @@ import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction. import { instanceSettingsService } from "./instance-settings.js"; import { currentProcessSpaceHash, processExists, sameProcessSpace } from "./controller-process-identity.js"; import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect; @@ -585,7 +586,10 @@ export function workspaceOperationService(db: Db) { throw insertError; } createdIds.push(id); - if (recordInput.phase === "workspace_finalize") hasFinalization = true; + if (recordInput.phase === "workspace_finalize") { + hasFinalization = true; + crashPoint("workspace.finalize.after_start_before_copyback"); + } let heartbeatTimer: NodeJS.Timeout | null = null; let timeoutTimer: NodeJS.Timeout | null = null; @@ -667,6 +671,7 @@ export function workspaceOperationService(db: Db) { try { const result = await settle(); + if (recordInput.phase === "workspace_finalize") crashPoint("workspace.finalize.after_copyback_before_done"); await append("system", result.system ?? null); await append("stdout", result.stdout ?? null); await append("stderr", result.stderr ?? null);