diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts index 6e9694699cb..3cbf2d290bc 100644 --- a/packages/shared/src/crash-point-registry.ts +++ b/packages/shared/src/crash-point-registry.ts @@ -47,6 +47,13 @@ export const CRASH_POINTS = { kinds: ["sandbox_creation_intent", "environment_run_lease_active"], invariants: ["I2", "I7"], }, + "native.provider_artifacts.before_capture": { + description: "A remote OpenCode session opened and the controller ledger holds no turn command; the launch-artifact capture gate has not saved its receipt. Fail mode throws the typed transfer timeout, so the task must continue on a fresh sandbox without a Retry.", + environments: ["daytona"], + kinds: ["native_executor_claim"], + invariants: ["I2", "I7"], + prerequisite: "an OpenCode agent on a sandbox environment, in a run that starts a new provider session (not a continuation that carries saved launch artifacts)", + }, "native.process_start.after_requested_before_spawn": { description: "native.process_start_requested is in the run log; runnerd has not been spawned. No provider work started, so a resumed attempt must not be charged.", environments: ["local", "daytona"], diff --git a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts index 7ffd99c0dd4..5c715bd3a43 100644 --- a/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts +++ b/server/src/__tests__/heartbeat-native-predispatch-retry.test.ts @@ -1,15 +1,19 @@ import { randomUUID } from "node:crypto"; -import { and, eq } from "drizzle-orm"; +import { existsSync, mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { and, eq, inArray } from "drizzle-orm"; import { activityLog, agents, authUsers, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues } from "@paperclipai/db"; import { NativePredecessorRetirementUnprovenError } from "../services/native-runtime/predecessor-retirement.js"; +import { NativeProviderArtifactTransferTimeoutError } from "../services/native-runtime/native-provider-artifacts.js"; +import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; import type { NativeRunIdentity, NativeSession, NativeSessionBackend, PrpEvent } from "@paperclipai/paperclip-runner"; import type { ExecutionContinuationEnvelope } from "@paperclipai/shared"; import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils"; import { describe, expect, it, vi } from "vitest"; import { heartbeatService, type HeartbeatServiceOptions } from "../services/heartbeat.js"; import { instanceSettingsService } from "../services/instance-settings.js"; -import { closeIdleWarmNativeSessionsForRestart } from "../services/native-runtime/native-session-executor.js"; +import { closeIdleWarmNativeSessionsForRestart, scopedRunnerdStateRoot } from "../services/native-runtime/native-session-executor.js"; import { nativeRestartFixture, waitForCondition } from "./helpers/native-restart-fixture.js"; import { getRecoveryPostgresSupport } from "./helpers/recovery-postgres-fixture.js"; @@ -35,8 +39,13 @@ vi.mock("../services/budget-reservations.js", async importOriginal => { const support = await getRecoveryPostgresSupport({}); +type AdmissionFailure = { remaining: number; turnNeverIssued: boolean; environmentId: string; provider: string; + providerLeaseId: string; leasePolicy: "reuse_by_environment" | "reuse_by_execution_workspace"; failedRuns: string[]; + sessionRoots: string[] }; + function nativeBackend(db: ReturnType, options: { failDescriptorOnce?: { failed: boolean }; - unretiredPredecessorOnce?: { failed: boolean; environmentId: string }; turnMessages?: Map } = {}): + unretiredPredecessorOnce?: { failed: boolean; environmentId: string }; turnMessages?: Map; + admissionFailure?: AdmissionFailure } = {}): NonNullable { return execution => { let identity: NativeRunIdentity; @@ -100,7 +109,27 @@ function nativeBackend(db: ReturnType, options: { failDescripto openSession: async input => { identity = input.identity; return session; }, recoverSession: async snapshot => { identity = snapshot.identity; return { recovered: true, session }; }, }; - return backend; + const failure = options.admissionFailure; + if (!failure) return backend; + return Object.assign(backend, { + providerTurnNeverIssued: () => failure.turnNeverIssued, + providerAdmissionPersisted: async () => { + if (failure.remaining <= 0) return; + failure.remaining -= 1; + const runId = execution.binding.runId; + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await db.insert(environmentLeases).values({ companyId: run!.companyId, environmentId: failure.environmentId, + issueId: execution.binding.issueId, heartbeatRunId: runId, provider: failure.provider, + providerLeaseId: `${failure.providerLeaseId}-${failure.failedRuns.length}`, leasePolicy: failure.leasePolicy, + status: "active", metadata: { remoteCwd: "/workspace" } }); + failure.failedRuns.push(runId); + const root = scopedRunnerdStateRoot(execution); + mkdirSync(root, { recursive: true, mode: 0o700 }); + writeFileSync(join(root, "failed-attempt"), runId); + failure.sessionRoots.push(root); + throw new NativeProviderArtifactTransferTimeoutError("upload", 120_000, 512 * 1024 * 1024); + }, + }); }; } @@ -252,6 +281,139 @@ describe.skipIf(!support.supported)("heartbeat native pre-dispatch failure retry } }, 90_000); + it.each([ + { target: "remote sandbox", provider: "e2b", leasePolicy: "reuse_by_execution_workspace" as const }, + { target: "plugin-task sandbox", provider: "daytona", leasePolicy: "reuse_by_environment" as const }, + ])("continues on a fresh $target when the provider artifacts fail to save before any turn", async ({ provider, leasePolicy }) => { + const { companyId } = fixture.current; + const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); + const [environment] = await db.insert(environments).values({ name: `capture-${randomUUID()}`, driver: "sandbox" }).returning(); + const turnMessages = new Map(); + const failure: AdmissionFailure = { remaining: 1, turnNeverIssued: true, environmentId: environment!.id, provider, + providerLeaseId: `sandbox-${provider}`, leasePolicy, failedRuns: [], sessionRoots: [] }; + const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: nativeBackend(db, { admissionFailure: failure, turnMessages }) }); + try { + const { issueId, runId } = await startTask(db, heartbeat); + expect(failure.failedRuns).toEqual([runId]); + expect(turnMessages.has(runId)).toBe(false); + expect(existsSync(failure.sessionRoots[0]!)).toBe(false); + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(failed).toMatchObject({ status: "failed", runtimeMode: "native", + resultJson: expect.objectContaining({ failureCause: "native_failed_before_provider_turn", + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + predecessorRetirementFallback: expect.objectContaining({ version: 1, providerWorkStarted: false, + predecessorRunId: null, replacedProviderLeaseIds: [`sandbox-${provider}-0`] }) }) }); + expect(failed!.error).toBe("Setting up the computer took too long, so the task is continuing on a fresh one."); + expect(failed!.error).not.toMatch(/[a-z0-9]+_[a-z0-9_]+|\d|\(|\)/i); + expect(failed!.runnerProfileJson).toMatchObject({ failedBeforeProviderTurn: { + cause: "native_failed_before_provider_turn", technicalCause: "native_provider_artifacts_transfer_timeout" } }); + const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.heartbeatRunId, runId), + eq(environmentLeases.providerLeaseId, `sandbox-${provider}-0`))); + expect(lease?.metadata).toMatchObject({ replaceBeforeReuse: { schema: "paperclip.replace-before-reuse.v1", runId } }); + const successors = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId))); + expect(successors).toEqual([expect.objectContaining({ scheduledRetryReason: "interrupted_preparation", scheduledRetryAttempt: 1 })]); + expect(successors[0]!.contextSnapshot?.executionRetryAccounting).toMatchObject({ failureRetries: 0 }); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue!.status).not.toBe("blocked"); + expect(await taskInterruptionPresentation(db, issue!)).toMatchObject({ state: "working", canRetry: false }); + expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId))).toHaveLength(0); + + expect(await heartbeat.promoteDueScheduledRetries()).toMatchObject({ promoted: 1 }); + await heartbeat.resumeQueuedRuns(); + await waitForCondition("the continuation to succeed", async () => + (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, successors[0]!.id)))[0]?.status === "succeeded"); + await waitForCondition("the continuation to settle", () => heartbeat.getTaskDrainStatus().quiescent); + expect(turnMessages.has(successors[0]!.id)).toBe(true); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId)))).toHaveLength(1); + const [settled] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(await taskInterruptionPresentation(db, settled!)).toMatchObject({ state: "resolved", canRetry: false }); + } finally { + await heartbeat.drainActiveRunExecutions(); + await closeIdleWarmNativeSessionsForRestart(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]))); + await db.$client.end(); + } + }, 90_000); + + it("keeps today's ambiguous hold when the Runner ledger cannot prove no turn was sent", async () => { + const { companyId } = fixture.current; + const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); + const [environment] = await db.insert(environments).values({ name: `capture-${randomUUID()}`, driver: "sandbox" }).returning(); + const failure: AdmissionFailure = { remaining: 1, turnNeverIssued: false, environmentId: environment!.id, provider: "daytona", + providerLeaseId: "sandbox-turn-maybe-sent", leasePolicy: "reuse_by_environment", failedRuns: [], sessionRoots: [] }; + const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: nativeBackend(db, { admissionFailure: failure }) }); + try { + const { runId } = await startTask(db, heartbeat); + const [failed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(failed?.status).toBe("failed"); + expect(failed?.resultJson ?? {}).not.toHaveProperty("predecessorRetirementFallback"); + expect(failed?.resultJson?.failureCause).not.toBe("native_failed_before_provider_turn"); + const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.heartbeatRunId, runId), + eq(environmentLeases.providerLeaseId, "sandbox-turn-maybe-sent-0"))); + expect(lease?.metadata ?? {}).not.toHaveProperty("replaceBeforeReuse"); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.retryOfRunId, runId), eq(heartbeatRuns.scheduledRetryReason, "interrupted_preparation")))).toHaveLength(0); + } finally { + await heartbeat.drainActiveRunExecutions(); + await closeIdleWarmNativeSessionsForRestart(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]))); + await db.$client.end(); + } + }, 90_000); + + it("backs off a lasting pre-turn failure and stops at the shared lane cap without hammering", async () => { + const { companyId } = fixture.current; + const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); + const [environment] = await db.insert(environments).values({ name: `capture-${randomUUID()}`, driver: "sandbox" }).returning(); + const failure: AdmissionFailure = { remaining: Number.POSITIVE_INFINITY, turnNeverIssued: true, environmentId: environment!.id, + provider: "daytona", providerLeaseId: "sandbox-lasting", leasePolicy: "reuse_by_environment", failedRuns: [], sessionRoots: [] }; + const heartbeat = heartbeatService(db, { nativeSessionBackendFactory: nativeBackend(db, { admissionFailure: failure }) }); + const successorOf = async (runId: string) => (await db.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.retryOfRunId, runId)))); + const runToEnd = async (runId: string) => { + await heartbeat.resumeQueuedRuns(); + await waitForCondition("the attempt to end", async () => + (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0]?.status === "failed"); + await waitForCondition("the attempt to settle", () => heartbeat.getTaskDrainStatus().quiescent); + }; + try { + const { issueId, runId } = await startTask(db, heartbeat); + const [first] = await successorOf(runId); + expect(first).toMatchObject({ scheduledRetryReason: "interrupted_preparation", scheduledRetryAttempt: 1 }); + expect((await heartbeat.promoteDueScheduledRetries()).runIds).toEqual([first!.id]); + const secondStartedAt = Date.now(); + await runToEnd(first!.id); + const [second] = await successorOf(first!.id); + expect(second).toMatchObject({ scheduledRetryReason: "interrupted_preparation", scheduledRetryAttempt: 2 }); + expect(second!.scheduledRetryAt!.getTime() - secondStartedAt).toBeGreaterThanOrEqual(15_000); + expect((await heartbeat.promoteDueScheduledRetries()).runIds).not.toContain(second!.id); + expect(failure.failedRuns).toHaveLength(2); + const [waiting] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(await taskInterruptionPresentation(db, waiting!)).toMatchObject({ state: "working", canRetry: false }); + + await db.update(heartbeatRuns).set({ scheduledRetryAttempt: 12, scheduledRetryAt: new Date(Date.now() - 1_000), + contextSnapshot: { ...second!.contextSnapshot, executionRetryAccounting: { + ...(second!.contextSnapshot?.executionRetryAccounting as Record), interruptedPreparations: 12 } } }) + .where(eq(heartbeatRuns.id, second!.id)); + expect((await heartbeat.promoteDueScheduledRetries()).runIds).toEqual([second!.id]); + await runToEnd(second!.id); + expect(await successorOf(second!.id)).toHaveLength(0); + expect(failure.failedRuns).toHaveLength(3); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(await taskInterruptionPresentation(db, issue!)).toMatchObject({ state: "failed", canRetry: true }); + } finally { + await heartbeat.drainActiveRunExecutions(); + await closeIdleWarmNativeSessionsForRestart(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.status, ["queued", "scheduled_retry"]))); + await db.$client.end(); + } + }, 120_000); + it("keeps the hold when the failure follows entry into the native executor", async () => { const { companyId } = fixture.current; const db = createDb(fixture.current.temporary.connectionString, { maxConnections: 4 }); diff --git a/server/src/services/heartbeat/predecessor-retirement-fallback.ts b/server/src/services/heartbeat/predecessor-retirement-fallback.ts index b9b91955888..648ce5aee87 100644 --- a/server/src/services/heartbeat/predecessor-retirement-fallback.ts +++ b/server/src/services/heartbeat/predecessor-retirement-fallback.ts @@ -1,4 +1,4 @@ -import { and, eq, isNotNull, sql } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, sql } from "drizzle-orm"; import { environmentLeases, executionWorkspaces, heartbeatRuns, nativeRunResults, type Db } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; import { persistActivity } from "../activity-log.js"; @@ -7,6 +7,9 @@ import { findNativePredecessorRetirementUnproven, REPLACE_BEFORE_REUSE_KEY, REPLACE_BEFORE_REUSE_SCHEMA, type PredecessorRetirementFallback, type ReplaceBeforeReuseMarker, } from "../native-runtime/predecessor-retirement.js"; +import { FAILED_BEFORE_PROVIDER_TURN_KEY, FAILED_BEFORE_PROVIDER_TURN_SCHEMA, findNativeFailedBeforeProviderTurn, + readFailedBeforeProviderTurnRecord } from "../native-runtime/failed-before-provider-turn.js"; +import { abandonUnverifiedRemoteWorkspaceEpoch } from "../remote-workspace-epoch-abandonment.js"; import { releaseUnchangedRemoteWorkspaceWriter } from "../remote-workspaces.js"; type Run = Pick; @@ -26,9 +29,10 @@ type Run = Pick; */ export async function preparePredecessorRetirementFallback(db: Db, run: Run, error: unknown) : Promise { - const failure = findNativePredecessorRetirementUnproven(error); + const beforeProviderTurn = findNativeFailedBeforeProviderTurn(error); + const failure = findNativePredecessorRetirementUnproven(error) ?? beforeProviderTurn; if (!failure) return null; - if (await hasNativeExecutionStartEvidence(db, { companyId: run.companyId, id: run.id })) return null; + if (!beforeProviderTurn && await hasNativeExecutionStartEvidence(db, { companyId: run.companyId, id: run.id })) return null; const [result] = await db.select({ id: nativeRunResults.id }).from(nativeRunResults) .where(and(eq(nativeRunResults.companyId, run.companyId), eq(nativeRunResults.runId, run.id))).limit(1); if (result) return null; @@ -39,9 +43,14 @@ export async function preparePredecessorRetirementFallback(db: Db, run: Run, err if (!current || !["running", "failed"].includes(current.status) || current.runnerProfileJson?.nativeWorkspaceSync != null) return null; const leases = (await tx.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), - eq(environmentLeases.leasePolicy, "reuse_by_environment"), isNotNull(environmentLeases.providerLeaseId), + beforeProviderTurn ? inArray(environmentLeases.leasePolicy, ["reuse_by_environment", "reuse_by_execution_workspace"]) + : eq(environmentLeases.leasePolicy, "reuse_by_environment"), isNotNull(environmentLeases.providerLeaseId), )).for("update")).filter(lease => !failure.providerLeaseId || lease.providerLeaseId === failure.providerLeaseId); - if (leases.length === 0) return null; + if (leases.length === 0 && !beforeProviderTurn) return null; + if (beforeProviderTurn) await tx.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ + [FAILED_BEFORE_PROVIDER_TURN_KEY]: { schema: FAILED_BEFORE_PROVIDER_TURN_SCHEMA, cause: beforeProviderTurn.code, proof: beforeProviderTurn.proof, + technicalCause: beforeProviderTurn.technicalCause } })}::jsonb` }) + .where(and(eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id))); const marker: ReplaceBeforeReuseMarker = { schema: REPLACE_BEFORE_REUSE_SCHEMA, runId: run.id, predecessorRunId: failure.predecessorRunId, recordedAt: new Date().toISOString() }; for (const lease of leases) { @@ -85,6 +94,13 @@ export async function releaseUnchangedFallbackWorkspace(db: Db, run: Run): Promi // the task. Do not queue a second attempt beside it. if (authority.writerRunId !== run.id) return false; // The heartbeat that owns this writer is the caller, and it started no runner. - return Boolean(await releaseUnchangedRemoteWorkspaceWriter(tx, { workspace, authority, callerOwnsWriter: true })); + if (await releaseUnchangedRemoteWorkspaceWriter(tx, { workspace, authority, callerOwnsWriter: true })) return true; + if (!readFailedBeforeProviderTurnRecord(current.profile) || !authority.location || authority.epoch !== reference.data.epoch) return false; + const [writer] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id))).for("update", { skipLocked: true }); + if (!writer) return false; + await tx.transaction(savepoint => abandonUnverifiedRemoteWorkspaceEpoch(savepoint, { companyId: run.companyId, + workspaceId: workspace.id, runId: run.id, epoch: authority.epoch, snapshotId: authority.snapshotId, location: authority.location! })); + return true; }); } diff --git a/server/src/services/native-runtime/failed-before-provider-turn.ts b/server/src/services/native-runtime/failed-before-provider-turn.ts new file mode 100644 index 00000000000..ca8ea9701c5 --- /dev/null +++ b/server/src/services/native-runtime/failed-before-provider-turn.ts @@ -0,0 +1,55 @@ +export const NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE = "native_failed_before_provider_turn"; + +export const FAILED_BEFORE_PROVIDER_TURN_KEY = "failedBeforeProviderTurn"; +export const FAILED_BEFORE_PROVIDER_TURN_SCHEMA = "paperclip.failed-before-provider-turn.v1"; + +export type FailedBeforeProviderTurnProof = { source: "runner_ledger"; noProviderTurn: true; provenAt: string }; + +export type FailedBeforeProviderTurnRecord = { + schema: typeof FAILED_BEFORE_PROVIDER_TURN_SCHEMA; cause: typeof NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE; + proof: FailedBeforeProviderTurnProof; technicalCause?: string; +}; + +const TRANSFER_TIMEOUT_CODE = "native_provider_artifacts_transfer_timeout"; + +function technicalCauseCode(cause: unknown): string { + const code = cause && typeof cause === "object" ? (cause as { code?: unknown }).code : undefined; + return typeof code === "string" && /^[a-z][a-z0-9_]{0,79}$/.test(code) ? code : "unknown"; +} + +export function readFailedBeforeProviderTurnRecord(profile: unknown): FailedBeforeProviderTurnRecord | null { + if (!profile || typeof profile !== "object" || Array.isArray(profile)) return null; + const value = (profile as Record)[FAILED_BEFORE_PROVIDER_TURN_KEY]; + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const record = value as Record; + const proof = record.proof && typeof record.proof === "object" ? record.proof as Record : {}; + return record.schema === FAILED_BEFORE_PROVIDER_TURN_SCHEMA && record.cause === NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE + && proof.source === "runner_ledger" && proof.noProviderTurn === true && typeof proof.provenAt === "string" + ? record as FailedBeforeProviderTurnRecord : null; +} + +export class NativeFailedBeforeProviderTurnError extends Error { + readonly code = NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE; + readonly predecessorRunId = null; + readonly proof: FailedBeforeProviderTurnProof = { source: "runner_ledger", noProviderTurn: true, provenAt: new Date().toISOString() }; + readonly technicalCause: string; + constructor(readonly providerLeaseId: string | null, cause: unknown) { + const technicalCause = technicalCauseCode(cause); + super(technicalCause === TRANSFER_TIMEOUT_CODE + ? "Setting up the computer took too long, so the task is continuing on a fresh one." + : "The computer could not be set up before the agent started, so the task is continuing on a fresh one.", { cause }); + this.name = "NativeFailedBeforeProviderTurnError"; + this.technicalCause = technicalCause; + } +} + +export function findNativeFailedBeforeProviderTurn(error: unknown): NativeFailedBeforeProviderTurnError | null { + const pending: unknown[] = [error]; + for (let seen = 0; pending.length > 0 && seen < 16; seen++) { + const current = pending.shift(); + if (current instanceof NativeFailedBeforeProviderTurnError) return current; + if (current instanceof AggregateError) pending.push(...current.errors); + if (current instanceof Error && current.cause !== undefined) pending.push(current.cause); + } + return null; +} diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index b49c7a5cb57..122c1c5f4a1 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -23,7 +23,8 @@ import { getStartupRecoveryState } from "../../startup-recovery-state.js"; import { assertNativeHarnessBackupOwner, durableNativeHarnessStorage, NATIVE_HARNESS_DOWNLOAD_IDLE_TIMEOUT_MS, nativeHarnessBackupOwner, persistNativeHarnessBackup, restoreDurableNativeHarnessBackup } from "./native-harness-backup-storage.js"; import { restoreNativeProviderArtifacts, retainNativeProviderArtifacts, retainExistingNativeProviderArtifacts, - retainReadyNativeProviderArtifacts, type NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; + retainReadyNativeProviderArtifacts, NativeProviderArtifactTransferTimeoutError, nativeProviderArtifactTransferBounds, + type NativeProviderArtifactReceipt } from "./native-provider-artifacts.js"; import { prepareRemoteOpenCodeArtifacts, type OpenCodeLaunchArtifacts } from "./remote-opencode-artifacts.js"; import { prepareRuntimeContextSource, rehydratePinnedInstructionAsset } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; @@ -59,6 +60,7 @@ import { assertAttachedPortableTask, readyPortableTaskSource, unlaunchedPortable import { environmentTaskOperationSchema, environmentTaskResultSchema } from "@paperclipai/plugin-sdk"; import { preparePluginRunnerCredentials } from "./plugin-runner-credentials.js"; import { createPluginRunnerTaskOwner, verifyPluginRunnerTaskReattachment } from "./plugin-runner-task.js"; +import { findNativeFailedBeforeProviderTurn, NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE, NativeFailedBeforeProviderTurnError } from "./failed-before-provider-turn.js"; import { findNativePredecessorRetirementUnproven, NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE, NativePredecessorRetirementDirectoryBusyError, NativePredecessorRetirementUnprovenError } from "./predecessor-retirement.js"; import { retireSandboxRunnerPortHolders } from "./sandbox-runner-port-holders.js"; import { AgentDirectoryOperationBusyError } from "../agent-directory-working-copies.js"; @@ -2524,6 +2526,11 @@ function scrubRunnerdQuarantineLaunchState(root: string): void { } } +function retireFailedBeforeTurnSessionRoot(execution: NativeExecutionInput): void { + const root = scopedRunnerdStateRoot(execution); + if (existsSync(root)) quarantineRunnerdStateRoot(root, "identity_indeterminate"); +} + function quarantineRunnerdStateRoot( root: string, reason: "identity_indeterminate" | "identity_mismatch", @@ -9006,8 +9013,10 @@ export function nativeSessionFailureSourceCode( | "runner_ingress_startup_failed" | "local_runtime_api_listener_unavailable" | typeof NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE + | typeof NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE | "native_session_interrupted" { if (findNativePredecessorRetirementUnproven(error)) return NATIVE_PREDECESSOR_RETIREMENT_UNPROVEN_CODE; + if (findNativeFailedBeforeProviderTurn(error)) return NATIVE_FAILED_BEFORE_PROVIDER_TURN_CODE; if (error instanceof NativePermissionDeclinedError) return "native_permission_declined"; if (error instanceof NativeGovernedWaitSettlementPendingError) return "native_governed_wait_settlement_pending"; if (error instanceof NativeProviderStateIncompatibleError) return NATIVE_PROVIDER_STATE_INCOMPATIBLE; @@ -12684,7 +12693,21 @@ async function executePaperclipNativeSessionAttempt( ?? (error instanceof Error && /^native_[a-z_]+$/.test(error.message) ? error.message : "unknown"); try { await input.onLog?.("stderr", `[paperclip-runner] native provider-pack admission receipt could not be saved (cause=${cause})\n`); } catch {} } - await runnerdBackend?.providerAdmissionPersisted?.(active.admissionAbort!.signal); + const admissionBackend: Pick | null = + runnerdBackend ?? (input.backend as ManagedRunnerdBackend | undefined) ?? null; + try { + await admissionBackend?.providerAdmissionPersisted?.(active.admissionAbort!.signal); + } catch (admissionError) { + if (active.cancelRequested || active.admissionAbort!.signal.aborted + || admissionBackend?.providerTurnNeverIssued?.() !== true) throw admissionError; + const beforeTurn = new NativeFailedBeforeProviderTurnError(sandboxProviderLeaseId(input.runnerExecutionTarget), admissionError); + const detail = admissionError instanceof NativeProviderArtifactTransferTimeoutError ? `: ${admissionError.message}` : ""; + logger.warn({ runId: input.execution.binding.runId, technicalCause: beforeTurn.technicalCause, + err: admissionError instanceof Error ? admissionError.message : String(admissionError) }, + "native session failed before its first provider turn; continuing on a fresh sandbox"); + try { await input.onLog?.("stderr", `[paperclip-runner] the session failed before its first provider turn (cause=${beforeTurn.technicalCause}${detail})\n`); } catch {} + throw beforeTurn; + } if (session.resolveRuntimeRequest) await liveQuestions.attach(); if (active.cancelRequested) { await active.cancellationSettled; @@ -12903,8 +12926,10 @@ async function executePaperclipNativeSessionAttempt( // An unretired predecessor failed this run before it launched a Runner. // It is not a provider failure: the heartbeat boundary sends the next // attempt to a fresh sandbox, or keeps today's preservation. - findNativePredecessorRetirementUnproven(error) + findNativePredecessorRetirementUnproven(error) || + findNativeFailedBeforeProviderTurn(error) ) { + if (findNativeFailedBeforeProviderTurn(error)) retireFailedBeforeTurnSessionRoot(input.execution); // This is not a provider failure and must not overwrite the durable // result/coordinator state. The heartbeat boundary will either hand an // already-materialized result to the finalizer or retain the durable @@ -16101,6 +16126,7 @@ type ManagedRunnerdBackend = NativeSessionBackend & { sessionCheckpointPersisted?(snapshot: PersistedNativeSession): void; providerAdmissionPersisted?(signal?: AbortSignal): Promise; providerAdmissionRequired?(): boolean; + providerTurnNeverIssued?(): boolean; }; async function createRunnerdBackendWithinSessionClaim( @@ -18588,6 +18614,9 @@ async function createRunnerdBackendWithinSessionClaim( return { bindManagedSession: (session, options) => replacementBackend ? replacementBackend.bindManagedSession(session, options) : wrapManagedSession(session), + providerTurnNeverIssued: () => replacementBackend?.providerTurnNeverIssued + ? replacementBackend.providerTurnNeverIssued() + : runnerControlPlane !== undefined && runnerdStateHasNoProviderTurn(runnerControlPlane.store.state as unknown as Record), providerAdmissionRequired: () => replacementBackend?.providerAdmissionRequired ? replacementBackend.providerAdmissionRequired() : Boolean(artifactStorage && (providerArtifacts || !legacyProviderContinuity && !liveProviderRecovery)), @@ -18606,6 +18635,8 @@ async function createRunnerdBackendWithinSessionClaim( const signal = providerArtifactsSignal(stop); if (!remoteTarget) throw new Error("native_provider_artifacts_admission_missing"); const launch = openCodeLaunchArtifacts, manifest = expectedProviderPackManifest; + crashPoint("native.provider_artifacts.before_capture", { fail: () => + new NativeProviderArtifactTransferTimeoutError("upload", nativeProviderArtifactTransferBounds.minimumStepMs, 0) }); const reuse = () => retainReadyNativeProviderArtifacts({ db: input.db, owner: backupOwner, manifest, launchRoot: launch.root, accountUid: launch.accountUid }); providerArtifacts = await reuse() ?? await shareNativeProviderArtifactCapture({ diff --git a/server/src/services/remote-workspace-unchanged-proof.ts b/server/src/services/remote-workspace-unchanged-proof.ts index 67477468de5..4fe08424857 100644 --- a/server/src/services/remote-workspace-unchanged-proof.ts +++ b/server/src/services/remote-workspace-unchanged-proof.ts @@ -1,4 +1,4 @@ -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { executionWorkspaceRepositories, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, nativeRunnerCheckpoints, nativeRunResults, workspaceOperations, type Db, @@ -7,6 +7,10 @@ import { remoteWorkspaceRunReferenceSchema, type RemoteWorkspaceAuthority } from import { executionStartedEventPredicate } from "./pre-dispatch-execution-evidence.js"; import { runOwnerFenceHeldSql } from "./remote-workspace-writer-hold.js"; import { carriesInheritedRunnerIdentity } from "./native-runtime/native-inherited-runner-retry.js"; +import { readFailedBeforeProviderTurnRecord } from "./native-runtime/failed-before-provider-turn.js"; + +const PROVIDER_TURN_EVENT_TYPES = ["turn.started", "turn.completed", "turn.failed", "provider.event", "provider.rpc_result", + "tool.execution.started", "item.started", "item.completed"]; /** * A writer's prepared files equal its saved version only while nothing ran in @@ -48,7 +52,7 @@ const ENDED_WITHOUT_RESULT = ["failed", "cancelled", "interrupted", "timed_out"] export type UnchangedProofRefusal = | "baseline_missing" | "authority_mismatch" | "writer_not_ended" | "writer_owner_live" | "process_recorded" | "reference_mismatch" | "provider_session_started" | "execution_recorded" | "result_recorded" - | "runner_commanded" | "workspace_operation_recorded" | "repository_unsaved"; + | "runner_commanded" | "workspace_operation_recorded" | "repository_unsaved" | "predecessor_save_unsettled"; /** * Whether the ended writer of `authority` provably left its files exactly as @@ -78,8 +82,9 @@ export async function proveRemoteWorkspaceUnchanged(tx: Pick, inpu .where(and(eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, runId))); if (!run || !ENDED_WITHOUT_RESULT.includes(run.status)) return { proven: false, reason: "writer_not_ended" }; if (!input.callerOwnsWriter && run.ownerFenceHeld) return { proven: false, reason: "writer_owner_live" }; + const beforeProviderTurn = readFailedBeforeProviderTurnRecord(run.runnerProfileJson) !== null; if (run.sessionIdAfter !== null) return { proven: false, reason: "process_recorded" }; - if ((run.processPid !== null || run.processGroupId !== null || run.processStartedAt !== null) + if (!beforeProviderTurn && (run.processPid !== null || run.processGroupId !== null || run.processStartedAt !== null) && !await carriesInheritedRunnerIdentity(tx, run)) return { proven: false, reason: "process_recorded" }; const profile = object(run.runnerProfileJson); const reference = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace); @@ -97,7 +102,7 @@ export async function proveRemoteWorkspaceUnchanged(tx: Pick, inpu const [event] = await tx.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, workspace.companyId), eq(heartbeatRunEvents.runId, runId), - executionStartedEventPredicate())).limit(1); + beforeProviderTurn ? inArray(heartbeatRunEvents.eventType, PROVIDER_TURN_EVENT_TYPES) : executionStartedEventPredicate())).limit(1); if (event) return { proven: false, reason: "execution_recorded" }; const [result] = await tx.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and( eq(nativeRunResults.companyId, workspace.companyId), eq(nativeRunResults.runId, runId))).limit(1); @@ -105,13 +110,19 @@ export async function proveRemoteWorkspaceUnchanged(tx: Pick, inpu // The controller writes this checkpoint before every runner command. const [commanded] = await tx.select({ runId: nativeRunnerCheckpoints.runId }).from(nativeRunnerCheckpoints).where(and( eq(nativeRunnerCheckpoints.companyId, workspace.companyId), eq(nativeRunnerCheckpoints.runId, runId))).limit(1); - if (commanded) return { proven: false, reason: "runner_commanded" }; + if (commanded && !beforeProviderTurn) return { proven: false, reason: "runner_commanded" }; // Provisioning and saves record their operation before they touch files. const [operation] = await tx.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.heartbeatRunId, runId), sql`(${workspaceOperations.executionWorkspaceId} is null or ${workspaceOperations.executionWorkspaceId} = ${workspace.id})`, sql`${workspaceOperations.phase} not in (${sql.join(FILE_NEUTRAL_OPERATION_PHASES.map(phase => sql`${phase}`), sql`, `)})`)).limit(1); if (operation) return { proven: false, reason: "workspace_operation_recorded" }; + if (beforeProviderTurn) { + const [unsettled] = await tx.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id), + eq(workspaceOperations.status, "running"), sql`${workspaceOperations.heartbeatRunId} is distinct from ${runId}`)).limit(1); + if (unsettled) return { proven: false, reason: "predecessor_save_unsettled" }; + } // A requested repository is cloned before its save is recorded. Only // repositories already in the saved version are proven untouched. const repositories = await tx.select().from(executionWorkspaceRepositories).where(and( diff --git a/server/src/services/remote-workspace-unchanged-release.test.ts b/server/src/services/remote-workspace-unchanged-release.test.ts index 736138c0c18..1846ed527cd 100644 --- a/server/src/services/remote-workspace-unchanged-release.test.ts +++ b/server/src/services/remote-workspace-unchanged-release.test.ts @@ -1,14 +1,14 @@ import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { cp, mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { access, cp, mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, closeRegisteredClients, companies, createDb, environmentLeases, environments, executionWorkspaces, - heartbeatRunEvents, heartbeatRuns, issues, workspaceOperations, + heartbeatRunEvents, heartbeatRuns, issues, nativeRunnerCheckpoints, workspaceOperations, } from "@paperclipai/db"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import type { EnvironmentLease } from "@paperclipai/shared"; @@ -20,6 +20,11 @@ import { } from "./remote-workspaces.js"; import { UNCHANGED_BASELINE_KEY } from "./remote-workspace-unchanged-proof.js"; import { releaseUnchangedFallbackWorkspace } from "./heartbeat/predecessor-retirement-fallback.js"; +import { FAILED_BEFORE_PROVIDER_TURN_KEY, FAILED_BEFORE_PROVIDER_TURN_SCHEMA } from "./native-runtime/failed-before-provider-turn.js"; +import { readReplaceBeforeReuseMarker, REPLACE_BEFORE_REUSE_KEY, REPLACE_BEFORE_REUSE_SCHEMA } from "./native-runtime/predecessor-retirement.js"; +import { retireReplacedReusableSandboxLease, reusableSandboxReplacementPlan } from "./reusable-sandbox-resume-replacement.js"; +import { ABANDONED_WORKSPACE_FILES_PARKED_REASON, abandonedWorkspaceFilesHeldOnLease } from "./remote-workspace-abandoned-files-hold.js"; +import { heartbeatService } from "./heartbeat.js"; const exec = promisify(execFile); const support = await getEmbeddedPostgresTestSupport(); @@ -120,8 +125,8 @@ describe.skipIf(!support.supported)("remote workspace writer that ended without }; const event = (eventType: string) => db.insert(heartbeatRunEvents).values({ companyId, runId: runB, agentId, seq: 1, stream: "system", level: "info", eventType }); - return { companyId, runA, runB, savedA, workspace, end, saveB, snapshotOperations, reserveNext, parkWaiter, event, - captures: () => captures }; + return { companyId, runA, runB, issueId, savedA, workspace, end, saveB, snapshotOperations, reserveNext, parkWaiter, event, + remoteRoot: b.remoteRoot, leaseB: b.lease, captures: () => captures }; } async function expectFullSave(f: Awaited>, changed = false) { @@ -189,7 +194,7 @@ describe.skipIf(!support.supported)("remote workspace writer that ended without const f = await preparedFollowUp(provision ? { runtimeProvisionCommand: provision } : {}); // The heartbeat still holds the run's controller lease when it settles the failure. await f.end({ status: "failed", errorCode: "adapter_failed", controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }); - expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId })).toBe(released); + expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId }).then(Boolean)).toBe(released); expect(readRemoteWorkspaceAuthority((await f.workspace()).metadata)).toMatchObject(released ? { dirty: false, writerRunId: null, snapshotId: f.savedA.snapshotId } : { dirty: true, writerRunId: f.runB }); }); @@ -207,7 +212,7 @@ describe.skipIf(!support.supported)("remote workspace writer that ended without processGroupId: own ? 41_002 : 41_001, processStartedAt }).where(eq(heartbeatRuns.id, f.runB)); if (event) await f.event("native.process_identity_recorded"); await f.end({ status: "failed", errorCode: "native_session_interrupted", controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }); - expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId })).toBe(released); + expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId }).then(Boolean)).toBe(released); expect(readRemoteWorkspaceAuthority((await f.workspace()).metadata)).toMatchObject(released ? { dirty: false, writerRunId: null, snapshotId: f.savedA.snapshotId } : { dirty: true, writerRunId: f.runB }); }); @@ -267,4 +272,119 @@ describe.skipIf(!support.supported)("remote workspace writer that ended without await f.end(); await expectFullSave(f); }); + + async function failBeforeProviderTurn(f: Awaited>, options: { proof: boolean }) { + const agentId = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runB)))[0]!.agentId; + await db.insert(heartbeatRunEvents).values([ + { companyId: f.companyId, runId: f.runB, agentId, seq: 10, stream: "system", level: "info", eventType: "native.process_start_requested" }, + { companyId: f.companyId, runId: f.runB, agentId, seq: 11, stream: "system", level: "info", eventType: "session.started", + sourceInstanceId: "runner-b", sourceEventId: "runner-b:1", sourceSeq: 1 }, + ]); + await db.update(heartbeatRuns).set({ nativeIssueId: f.issueId }).where(eq(heartbeatRuns.id, f.runB)); + await db.insert(nativeRunnerCheckpoints).values({ companyId: f.companyId, runId: f.runB, issueId: f.issueId, + part: "command:session.open", material: { type: "session.open", status: "completed" } }); + if (options.proof) await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ + [FAILED_BEFORE_PROVIDER_TURN_KEY]: { schema: FAILED_BEFORE_PROVIDER_TURN_SCHEMA, cause: "native_failed_before_provider_turn", + proof: { source: "runner_ledger", noProviderTurn: true, provenAt: new Date().toISOString() } } })}::jsonb` }) + .where(eq(heartbeatRuns.id, f.runB)); + await f.end({ status: "failed", errorCode: "adapter_failed", processPid: 41_003, processStartedAt: new Date(), + controllerLeaseExpiresAt: new Date(Date.now() + 60_000) }); + } + + async function markForReplacement(f: Awaited>) { + await db.update(environmentLeases).set({ status: "released", executionWorkspaceId: (await f.workspace()).id, + leasePolicy: "reuse_by_environment", metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ + [REPLACE_BEFORE_REUSE_KEY]: { schema: REPLACE_BEFORE_REUSE_SCHEMA, runId: f.runB, predecessorRunId: null, + recordedAt: new Date().toISOString() } })}::jsonb` }).where(eq(environmentLeases.id, f.leaseB.id)); + } + + async function nextAcquisitionReplaces(f: Awaited>, runC: string) { + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseB.id)); + const replace = readReplaceBeforeReuseMarker(lease!.metadata) !== null && await reusableSandboxReplacementPlan({ db, + lease: lease as EnvironmentLease, companyId: f.companyId, heartbeatRunId: runC, + executionWorkspaceId: lease!.executionWorkspaceId, adapterType: null, + cause: new Error("native_predecessor_retirement_unproven") }) === "retire"; + if (replace) await retireReplacedReusableSandboxLease({ db, lease: lease as EnvironmentLease, companyId: f.companyId, heartbeatRunId: runC }); + return (await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseB.id)))[0]!; + } + + async function sweepCleanup() { + const calls: string[] = []; + const runtime = new Proxy({}, { get: (_target, name) => typeof name === "string" && name !== "then" + ? async () => { calls.push(name); return name === "isPendingCleanupWorkerReady" ? true : { destroyed: true }; } : undefined }); + await heartbeatService(db, { environmentRuntime: runtime as never }).sweepPendingCleanupLeases({ backoffMs: 0 }); + return calls; + } + + it("hands on the workspace unchanged when the Runner ledger proves the failed run sent no provider turn", async () => { + const f = await preparedFollowUp(); + await failBeforeProviderTurn(f, { proof: true }); + expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId })).toBeTruthy(); + const workspace = await f.workspace(); + expect(readRemoteWorkspaceAuthority(workspace.metadata)).toMatchObject({ dirty: false, writerRunId: null, + snapshotId: f.savedA.snapshotId, epoch: 2 }); + expect(workspace.metadata?.[UNCHANGED_BASELINE_KEY]).toMatchObject({ runId: f.runB, releasedAt: expect.any(String) }); + expect(workspace.metadata?.remoteWorkspaceAbandonedEpochs).toBeUndefined(); + }); + + it("replaces the marked sandbox when the failed run provably left its files unchanged", async () => { + const f = await preparedFollowUp(); + await failBeforeProviderTurn(f, { proof: true }); + await markForReplacement(f); + const released = await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId }); + const { runC } = await f.reserveNext(); + expect(await nextAcquisitionReplaces(f, runC)).toMatchObject({ status: "pending_cleanup" }); + expect(released).toBe(true); + expect(await sweepCleanup()).toEqual(expect.arrayContaining(["destroyRunLease"])); + }); + + it("keeps today's preservation for the same evidence without the Runner ledger proof", async () => { + const f = await preparedFollowUp(); + await failBeforeProviderTurn(f, { proof: false }); + expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId })).toBe(false); + expect(readRemoteWorkspaceAuthority((await f.workspace()).metadata)).toMatchObject({ dirty: true, writerRunId: f.runB }); + }); + + it.each([ + { label: "a provider turn was recorded", setup: async (f: Awaited>) => { await f.event("turn.started"); } }, + { label: "a predecessor's save on this sandbox is unsettled", setup: async (f: Awaited>) => { + await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: (await f.workspace()).id, + heartbeatRunId: f.runA, phase: "remote_workspace_snapshot", status: "running" }); + } }, + { label: "setup wrote into the workspace", provision: "echo provisioned > provisioned.txt", + setup: async () => undefined }, + ])("never claims unchanged when $label: continues from the last committed save and keeps the files unapplied", async ({ setup, provision }) => { + const f = await preparedFollowUp(provision ? { runtimeProvisionCommand: provision } : {}); + await writeFile(path.join(f.remoteRoot, "unsaved.md"), "kept\n"); + await setup(f); + await failBeforeProviderTurn(f, { proof: true }); + expect(await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId })).toBeTruthy(); + const workspace = await f.workspace(); + expect(readRemoteWorkspaceAuthority(workspace.metadata)).toMatchObject({ dirty: false, writerRunId: null, location: null, + snapshotId: f.savedA.snapshotId, epoch: 3 }); + expect(workspace.metadata?.[UNCHANGED_BASELINE_KEY]).not.toHaveProperty("releasedAt"); + expect(workspace.metadata?.remoteWorkspaceAbandonedEpochs).toEqual([expect.objectContaining({ epoch: 2, runId: f.runB, + snapshotId: f.savedA.snapshotId, location: expect.objectContaining({ remoteCwd: f.remoteRoot }) })]); + await expect(access(path.join(f.remoteRoot, "unsaved.md"))).resolves.toBeUndefined(); + const { runC } = await f.reserveNext(); + expect(readRemoteWorkspaceAuthority((await f.workspace()).metadata)).toMatchObject({ writerRunId: runC, location: null }); + }); + + it("continues on a fresh sandbox and parks the one that holds abandoned files, which cleanup leaves alone", async () => { + const f = await preparedFollowUp({ runtimeProvisionCommand: "echo provisioned > provisioned.txt" }); + await writeFile(path.join(f.remoteRoot, "unsaved.md"), "kept\n"); + await failBeforeProviderTurn(f, { proof: true }); + await markForReplacement(f); + const released = await releaseUnchangedFallbackWorkspace(db, { id: f.runB, companyId: f.companyId }); + const { runC } = await f.reserveNext(); + expect(released).toBe(true); + const lease = await nextAcquisitionReplaces(f, runC); + expect(lease).toMatchObject({ status: "retained", failureReason: ABANDONED_WORKSPACE_FILES_PARKED_REASON }); + expect(lease.metadata).toHaveProperty(REPLACE_BEFORE_REUSE_KEY); + expect(await abandonedWorkspaceFilesHeldOnLease(db, lease)).toBe(true); + expect(await sweepCleanup()).not.toEqual(expect.arrayContaining(["destroyRunLease"])); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseB.id))) + .toEqual([expect.objectContaining({ status: "retained" })]); + await expect(access(path.join(f.remoteRoot, "unsaved.md"))).resolves.toBeUndefined(); + }); }); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index d0b9c1dd35f..17e8cd7664a 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -1,5 +1,5 @@ import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; -import { AGENT_DIRECTORY_WAIT_RETRY_REASON } from "./execution-recovery-attempt.js"; +import { AGENT_DIRECTORY_WAIT_RETRY_REASON, INTERRUPTED_PREPARATION_RETRY_REASON } from "./execution-recovery-attempt.js"; import { boundSetupCommandFailure, readSetupCommandFailure } from "./workspace-setup-command-failure.js"; import { and, desc, eq, inArray, isNull, not, notInArray, sql, type SQL } from "drizzle-orm"; import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; @@ -314,6 +314,9 @@ function projectTaskInterruptionState(issue: ProjectionIssue, input: ProjectionS } if (run.status === "scheduled_retry" && resourceWaitRetryReasons.has(run.scheduledRetryReason ?? "") && !system) return { ...base, state: "working" }; + if (run.status === "scheduled_retry" && run.scheduledRetryReason === INTERRUPTED_PREPARATION_RETRY_REASON && !system + && (!run.scheduledRetryAt || now - run.scheduledRetryAt.getTime() < NATIVE_RESUME_DISPATCH_GRACE_MS)) + return { ...base, state: "working" }; // The native coordinator resumes this same run by itself. A manual Retry // would race it, so show the resume until it is due and briefly while its // dispatch runs. A resume that never ran falls through to one Retry. diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 0ec057daec8..1c319e547b2 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -266,6 +266,11 @@ async function main(): Promise { supervisor.useCrashPointControl({ controlPath: path.join(supervisor.paths.temporaryRoot, "crash-point.control"), receiptPath: path.join(waveDir, "crash-point-receipts.jsonl") }); } + const externalCrashControl = process.env.RESTART_CHAOS_EXTERNAL_CRASH_CONTROL; + if (!crashPoint && !plan.options.failRecoveryStep && externalCrashControl) { + supervisor.useCrashPointControl({ controlPath: externalCrashControl, + receiptPath: process.env.RESTART_CHAOS_EXTERNAL_CRASH_RECEIPT ?? path.join(waveDir, "crash-point-receipts.jsonl") }); + } let sweeper: DaytonaWaveSweeper | null = null; let sandboxSweep: DaytonaSweepResult | null = null; let runnerLogTarget: { companyId: string; environmentId: string } | null = null; diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index b977fbb58bd..247e5601cb2 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -230,7 +230,7 @@ "packages/shared/src/connector-identity.test.ts": "d58357707faefa7c98bb906cdaeb4db39d31081a", "packages/shared/src/connector-identity.ts": "5ede83c9c5225dbe8af9cf3a49c74e477ef5b955", "packages/shared/src/constants.ts": "3149ee1cc5ffa34b50ecd493276b3a70fc46adc2", - "packages/shared/src/crash-point-registry.ts": "6e9694699cb8de9c934f92170b9cfcab2eed4a1a", + "packages/shared/src/crash-point-registry.ts": "3cbf2d290bc111061376b43020848623e493737c", "packages/shared/src/crash-points.test.ts": "3564a1f62d22f54a3c2fb0a72b3a6cde17c957d2", "packages/shared/src/crash-points.ts": "c15259f39610e352eac5423a491548ec1bae94fe", "packages/shared/src/customer-success.ts": "a2d02aaa8223164c0eae4267b2a6da82242eaa6f",