diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index f8cbe5453a7..a5e84a855ec 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -8,13 +8,14 @@ import { z } from "zod"; import { agentWakeupRequests, chatConversations, chatEndpoints, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, toolConnections, workspaceOperations, type Db } from "@paperclipai/db"; import { canContinueCancelledRun, readRunCancellation } from "./run-cancellation.js"; import { canRetryStoppedRun } from "./cancelled-native-startup.js"; +import { heldRetryOffersRetry, heldTaskRetryRefusals } from "./task-retry-eligibility.js"; import { queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; import { EXECUTION_RECONCILIATION_CAUSES, readRunFailureCause, withoutSqlStatementText, type ExecutionBlocker } from "@paperclipai/shared"; import { hasRequiredWorkspaceRecovery, LEGACY_WORKSPACE_RECOVERY_SCHEMA, nativeRunHasNoWorkspaceAccess } from "./workspace-restore-recovery-state.js"; import { unreconciledNativeExportPredicate } from "./native-runtime/native-workspace-reconstruction.js"; import { hasRecoveredCleanupStopReceipt, hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; -import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, recoveredWorkMessage, workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; +import { completedWorkspaceRecoveryCondition, hasVerifiedWorkspaceRepair, recoveredWorkMessage, savedWorkOffersRetry, workspaceRecoveryDisplay } from "./workspace-recovery-receipt.js"; import { nativeManualAdmissionHoldsExecution } from "./native-runtime/native-runner-ownership.js"; import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; @@ -312,6 +313,14 @@ function ownNativeResumePredicate(runId: string) { and ${issueRecoveryActions.evidence}->>'runId' = ${runId}), false)`; } +export function restoreHoldPredicate() { + return or( + unreconciledNativeExportPredicate(), + sql`${issueRecoveryActions.evidence}->>'workspaceRestoreFailure' = 'restore_unsafe_archive'`, + sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`, + ); +} + export async function getExecutionBlocker(db: Db, companyId: string, issueId: string, options?: { conversationResetCommentId?: string | null; /** The native run being dispatched; its own scheduled resume does not block it. */ @@ -330,11 +339,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st eq(issueRecoveryActions.sourceIssueId, issueId), executionBlockerPredicate(), not(restoreHoldReleasedForCurrentOwnerPredicate()), - or( - unreconciledNativeExportPredicate(), - sql`${issueRecoveryActions.evidence}->>'workspaceRestoreFailure' = 'restore_unsafe_archive'`, - sql`${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema' = ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`, - ), + restoreHoldPredicate(), )).orderBy(desc(issueRecoveryActions.updatedAt)).limit(1); // A persisted user /new is an ordered context command, not a retry of uncertain work. // The normal issue execution lock still serializes it behind any active turn. @@ -411,8 +416,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st // hold to protect its files. Offer the same Retry that admission accepts, so // an infrastructure failure (such as a restart) is not left to a new message. // An intentional Stop keeps "send a new message" as its only continuation. - const recoveredFailure = Boolean(restoreHold && (run?.status === "failed" || run?.status === "timed_out") && !workspaceRepairRequired && - workspaceRecoveryState?.status === "recovered"); + const recoveredFailure = Boolean(restoreHold && savedWorkOffersRetry(workspaceRecoveryState, run?.status, workspaceRepairRequired, action.evidence)); const canRetry = Boolean((!restoreHold || recoveredFailure) && run && conversation?.assigneeAgentId === run.agentId && !["done", "cancelled"].includes(conversation.status) && await canRetryStoppedRun(db, run)); const runError = run && canRetry && isPreDispatchReviewWait(run) @@ -435,14 +439,16 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st )).where(and( eq(chatConversations.companyId, companyId), eq(chatConversations.issueId, issueId), )).limit(1) : []; - const retryOffered = canRetry && (!chatBinding || recoveredFailure); + const heldRefusal = canRetry ? (await heldTaskRetryRefusals(db, companyId, [issueId])).get(issueId) : undefined; + const retryOffered = canRetry && (!chatBinding || recoveredFailure) && heldRetryOffersRetry(heldRefusal); const workspaceRecovery = workspaceRecoveryState?.status === "recovered" - ? { ...workspaceRecoveryState, message: recoveredWorkMessage(run?.status, retryOffered) } : workspaceRecoveryState; + ? { ...workspaceRecoveryState, message: recoveredWorkMessage(run?.status, retryOffered) } + : workspaceRecoveryState && { ...workspaceRecoveryState, canRetry: workspaceRecoveryState.canRetry && retryOffered }; let nextAction = action.status === "active" && action.cause === EXECUTION_FINALIZATION_DEADLINE_EXCEEDED ? EXECUTION_FINALIZATION_DEADLINE_MESSAGE : action.nextAction; // Retry is a person's control on this task. Say so, so an agent reading this // record points the person to it instead of re-running the work itself. - if (canRetry && !eligibleContinuation) nextAction += ` ${EXECUTION_BLOCKER_PERSON_RETRY_ACTION}`; + if (canRetry && heldRetryOffersRetry(heldRefusal) && !eligibleContinuation) nextAction += ` ${EXECUTION_BLOCKER_PERSON_RETRY_ACTION}`; if (chatBinding && !restoreHold) { if (chatBinding.state === "endpoint_removed" || !chatBinding.endpointStatus || chatBinding.endpointStatus === "archived") { nextAction = "This chat connection was removed. Inspect the stopped run and create a new task to continue the work."; diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 37c509ded51..ef2894a8cd0 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -77,6 +77,7 @@ const continuationRunFields = { executionStage: heartbeatRuns.executionStage, }; const terminal = ["failed", "interrupted", "timed_out", "cancelled"]; +const PREVIOUS_EXECUTION_UNVERIFIED = "Paperclip can't confirm that the previous attempt stopped before it started work, so Retry can't continue from it. Your message is saved."; export function isUserCommentContinuationReason(reason: string | null | undefined): boolean { return reason === "issue_commented" || reason === "issue_reopened_via_comment" || reason === "issue_assigned"; @@ -193,6 +194,8 @@ export async function admitExplicitNativeContinuation(input: { }): Promise<{ previousRunId: string; commentId: string | null; failedRunId?: string; taskRetryRequestId?: string; queuedCommentInterruptId?: string; readiness?: ContinuationReadiness } | null> { const { db, companyId, issueId, agentId, actorId, commentId } = input; const blocked = (reason: string, message: string) => { input.onBlocked?.(reason, message); return null; }; + let blocker: Awaited> = null; + const refused = (reason: string, message: string) => blocker ? blocked(reason, message) : null; if (input.actorType !== "user" || !actorId) return null; const retry = input.reason === "retry_failed_run" && z.string().guid().safeParse(input.failedRunId).success; @@ -280,7 +283,7 @@ export async function admitExplicitNativeContinuation(input: { const authorizedAt = nativeQueuedSend ? interruptRequestedAt! : response?.comment.createdAt ?? comment?.createdAt ?? retryWake?.requestedAt ?? new Date(); if (!input.dryRun) await preserveTaskRemoteWorkspaceWriter(db, companyId, issueId); - const blocker = await getExecutionBlocker(db, companyId, issueId); + blocker = await getExecutionBlocker(db, companyId, issueId); const [boundary] = await db.select({ createdAt: issueComments.createdAt }).from(issues).innerJoin(issueComments, and( eq(issueComments.companyId, issues.companyId), eq(issueComments.issueId, issues.id), eq(issueComments.id, issues.conversationBoundaryCommentId), @@ -349,7 +352,7 @@ export async function admitExplicitNativeContinuation(input: { )).for("update"); if (!source || (source.nativeIssueId ?? source.issueId ?? source.contextSnapshot?.issueId) !== issueId || adapterExecutionControls.has(source.id) || - !(await isFailedNativePreparation(db, source, coordinator) || await isPreDispatchExecutionWaitVerified(db, source))) return null; + !(await isFailedNativePreparation(db, source, coordinator) || await isPreDispatchExecutionWaitVerified(db, source))) return refused("previous_execution_unverified", PREVIOUS_EXECUTION_UNVERIFIED); sourceChecks.push({ cause: "legacy_execution_requires_reconciliation", evidence: { runId: source.id } }); } if (!sourceChecks.length && !nativeQueuedSend) return null; @@ -385,7 +388,7 @@ export async function admitExplicitNativeContinuation(input: { eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, run.id), )).for("update"); if (!lockedRun || lockedRun.status !== run.status || lockedRun.agentId !== run.agentId || - lockedRun.finishedAt?.getTime() !== run.finishedAt.getTime()) return null; + lockedRun.finishedAt?.getTime() !== run.finishedAt.getTime()) return refused("execution_settling", "Waiting for the previous run to stop. Your message will start automatically."); run = lockedRun; // A user's message written after their Stop request is new direction even // when the stopped native run settled later. @@ -421,12 +424,16 @@ export async function admitExplicitNativeContinuation(input: { // a conversation. Those adapters retain their reconciliation contract. if (historicalAdapter && !isConversationAdapter(historicalAdapter)) { if (!(taskRetry && run.id !== taskRetry.sourceRunId && freshIntent && agentIsNative && - historicalAdapter === "paperclip_runner" && run.status === "failed") || - run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date() || - await hasPendingRunCleanup(db, companyId, run.id) || - !await isFailedNativePreparationRetryAncestor(db, run, coordinator)) return null; + historicalAdapter === "paperclip_runner" && run.status === "failed")) + return refused("previous_execution_unverified", PREVIOUS_EXECUTION_UNVERIFIED); + if (run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt > new Date()) + return refused("execution_settling", "Waiting for the previous run to stop. Your message will start automatically."); + if (await hasPendingRunCleanup(db, companyId, run.id)) + return refused("cleanup_pending", "Waiting for the previous run to finish cleanup. Your message is saved."); + if (!await isFailedNativePreparationRetryAncestor(db, run, coordinator)) + return refused("previous_execution_unverified", PREVIOUS_EXECUTION_UNVERIFIED); } - if (!historicalAdapter && agent.adapterType === "paperclip_runner") return null; + if (!historicalAdapter && agent.adapterType === "paperclip_runner") return refused("previous_execution_unverified", PREVIOUS_EXECUTION_UNVERIFIED); } const interruptedByThisClick = queuedInterrupt && interruptIntent?.targetRunId === run.id && run.resultJson?.operatorInterrupted === true && @@ -511,7 +518,7 @@ export async function admitExplicitNativeContinuation(input: { if (retry && run.status === "cancelled" && !canContinueCancelledRun(run) && !cancelledStartup) return blocked("cancelled_by_operator", "Inspect the stopped run and send a new message to continue."); if (cancelledStartup) cancelledStartupIds.add(run.id); - if (run.runtimeMode !== "native" && !unusedAdmission && !legacyUserTurn && !cancelledStartup) return null; + if (run.runtimeMode !== "native" && !unusedAdmission && !legacyUserTurn && !cancelledStartup) return refused("previous_execution_unverified", PREVIOUS_EXECUTION_UNVERIFIED); // A provider failure can finish the normal result/assessment commit path. // Its accepted failed result is immutable history, not a live controller. // Only a new user turn may pass this gate; the process/lease stop proofs @@ -559,7 +566,7 @@ export async function admitExplicitNativeContinuation(input: { if (!leases.every(lease => hasRemoteTerminationReceipt(lease) || hasRecoveredCleanupStopReceipt(lease))) return blocked("remote_cleanup", "Waiting for the previous environment to stop. Your message will start automatically."); if (run.runtimeMode === "native" && !input.dryRun && !leases.every(lease => completeTerminatedRemoteNativeSessionCleanup({ companyId, runId: run.id, remoteCleanupScope: remoteLeaseCleanupScope(lease)!, - }))) return null; + }))) return refused("remote_cleanup", "Waiting for the previous environment to stop. Your message will start automatically."); } else { if (leases.some(lease => !lease.releasedAt || lease.cleanupStatus === "failed")) return blocked("local_cleanup", "Waiting for the previous environment to finish cleanup. Your message will start automatically."); if (!unusedAdmission && !cancelledStartup && !failedPreparation) { @@ -652,7 +659,7 @@ export async function admitExplicitNativeContinuation(input: { const nativeSources = sources.filter(run => run.runtimeMode === "native"); const executedSources = sources.filter(run => run.runtimeMode === "native" || run.errorCode !== "execution_reconciliation_required" || run.startedAt); if (!executedSources.length || (retry && !sources.some(run => run.id === input.failedRunId)) || - (taskRetry && !sources.some(run => run.id === taskRetry.sourceRunId))) return null; + (taskRetry && !sources.some(run => run.id === taskRetry.sourceRunId))) return refused("previous_execution_unverified", PREVIOUS_EXECUTION_UNVERIFIED); const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( eq(heartbeatRuns.companyId, companyId), or(eq(heartbeatRuns.nativeIssueId, issueId), sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`), diff --git a/server/src/services/remote-workspace-snapshot-reuse-receipt.ts b/server/src/services/remote-workspace-snapshot-reuse-receipt.ts index bc01c4df918..d2389ca199e 100644 --- a/server/src/services/remote-workspace-snapshot-reuse-receipt.ts +++ b/server/src/services/remote-workspace-snapshot-reuse-receipt.ts @@ -1,5 +1,5 @@ -import { and, eq, isNull, sql } from "drizzle-orm"; -import { executionWorkspaceSnapshots, workspaceOperations, type Db } from "@paperclipai/db"; +import { and, eq, inArray, isNull, sql } from "drizzle-orm"; +import { executionWorkspaceSnapshots, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; import type { RemoteWorkspaceRunReference } from "@paperclipai/shared"; type Snapshot = typeof executionWorkspaceSnapshots.$inferSelect; @@ -48,3 +48,30 @@ export async function verifiedWorkspaceSnapshotReuse(db: Pick, inp || typeof settled.quiescedAt !== "string" || !Number.isFinite(Date.parse(settled.quiescedAt))) return null; return receipt.id; } + +export async function verifiedUnchangedRestoredSnapshot(db: Pick, input: { + companyId: string; issueId: string; runId: string; reference: RemoteWorkspaceRunReference; snapshot: Snapshot; +}): Promise { + const { reference, snapshot } = input; + if (reference.state !== "saved" || reference.snapshotId !== snapshot.id || snapshot.companyId !== input.companyId + || snapshot.executionWorkspaceId !== reference.workspaceId || !["committed", "superseded"].includes(snapshot.status) + || !snapshot.sha256) return null; + const [run] = await db.select({ resultJson: heartbeatRuns.resultJson, executionStage: heartbeatRuns.executionStage }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + const recovery = run?.resultJson?.executionRecovery as Record | undefined; + if (run?.executionStage !== "preparing" || recovery?.kind !== "bootstrap" || recovery.providerWorkStarted !== false) return null; + const operations = await db.select({ id: workspaceOperations.id, phase: workspaceOperations.phase, + status: workspaceOperations.status, metadata: workspaceOperations.metadata, issueId: workspaceOperations.issueId, + workspaceId: workspaceOperations.executionWorkspaceId }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.heartbeatRunId, input.runId), + inArray(workspaceOperations.phase, ["remote_workspace_restore", "remote_workspace_snapshot"]))); + if (operations.some(operation => operation.phase === "remote_workspace_snapshot")) return null; + const restores = operations.filter(operation => { + const cleanup = operation.metadata?.exportCleanup as Record | undefined; + return operation.status === "succeeded" && operation.issueId === input.issueId + && operation.workspaceId === reference.workspaceId && operation.metadata?.epoch === reference.epoch + && operation.metadata?.snapshotId === snapshot.id && cleanup?.leaseId === reference.leaseId + && cleanup?.providerLeaseId === reference.providerLeaseId && cleanup?.remoteCwd === reference.remoteCwd; + }); + return restores.length === 1 ? restores[0]!.id : null; +} diff --git a/server/src/services/saved-work-retry-eligibility.test.ts b/server/src/services/saved-work-retry-eligibility.test.ts new file mode 100644 index 00000000000..fc2602a5868 --- /dev/null +++ b/server/src/services/saved-work-retry-eligibility.test.ts @@ -0,0 +1,329 @@ +import { createHash, randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, companies, createDb, environmentLeases, executionWorkspaces, + executionWorkspaceSnapshots, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, issues, + workspaceOperations } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { requestTaskRetry } from "./task-retry.js"; +import { heartbeatService } from "./heartbeat.js"; +import { getExecutionBlocker } from "./execution-blocker.js"; +import { taskInterruptionPresentation } from "./task-interruption-presentation.js"; +import { REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA } from "./remote-workspace-epoch-abandonment.js"; + +const logs = vi.hoisted(() => ({ content: new Map() })); +vi.mock("./run-log-store.js", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, getRunLogStore: () => ({ ...actual.getRunLogStore(), + read: async (handle: { logRef: string }) => { + const content = logs.content.get(handle.logRef); + if (content === undefined) throw new Error("Object log not found"); + return { content }; + } }) }; +}); + +describe("Retry of a task whose saved work recovery settled", () => { + let postgres: Awaited>; + let db: ReturnType; + beforeAll(async () => { + postgres = await startEmbeddedPostgresTestDatabase("saved-work-retry-"); + db = createDb(postgres.connectionString); + }, 30_000); + afterAll(async () => { await db?.$client.end(); await postgres?.cleanup(); }); + beforeEach(() => { logs.content.clear(); }); + + const actorId = "board"; + const terminated = (companyId: string, runId: string, leaseId: string, providerLeaseId: string, at: Date) => ({ + pendingCleanupInFlight: false, remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", + companyId, runId, leaseId, provider: "daytona", providerLeaseId, state: "destroyed", confirmedAt: at.toISOString() } }); + + async function base(prefix: string) { + const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), workspaceId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: prefix, issuePrefix: `${prefix}${companyId.slice(0, 5)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", adapterType: "paperclip_runner", status: "idle", + runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Continue the task", status: "blocked", assigneeAgentId: agentId }); + return { companyId, issueId, agentId, workspaceId }; + } + + async function sourceUnavailable() { + const f = await base("SU"); + const { companyId, issueId, agentId, workspaceId } = f; + const firstId = randomUUID(), preparedId = randomUUID(), heldId = randomUUID(), actionId = randomUUID(); + const leaseId = randomUUID(), providerLeaseId = randomUUID(), snapshotId = randomUUID(), now = Date.now(); + const remoteCwd = "/home/daytona/paperclip-workspace"; + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, + mode: "isolated_workspace", strategyType: "git_worktree", name: "Task workspace", providerType: "cloud_sandbox", + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 9, dirty: false, source: { kind: "empty" }, + location: null, snapshotId, writerRunId: null }, + remoteWorkspaceAbandonedEpochs: [{ schema: REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA, runId: preparedId, epoch: 8, + location: { leaseId, providerLeaseId, remoteCwd, environmentId: randomUUID() }, snapshotId, + abandonedAt: new Date(now - 500_000).toISOString(), providerRef: remoteCwd }] } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + await db.insert(heartbeatRuns).values({ id: firstId, companyId, agentId, issueId, status: "failed", runtimeMode: "legacy", + errorCode: "process_lost", executionStage: "preparing", invocationSource: "automation", + createdAt: new Date(now - 900_000), startedAt: new Date(now - 880_000), finishedAt: new Date(now - 800_000), + contextSnapshot: { issueId }, runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } }, + resultJson: { stopReason: "process_lost", executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + const scheduledRetryAt = new Date(now - 780_000), outputAt = new Date(now - 700_000); + const lines = ["[paperclip] Execution workspace reuse freshness action \"replace\" because execution workspace configuration changed: repo/base ref, environment.\n", + `[paperclip] Skipping saved session resume for task "${issueId}" because effective run configuration changed: environment.\n`]; + const attemptId = randomUUID(); + const content = lines.map((chunk, index) => JSON.stringify({ ts: (index ? outputAt : new Date(outputAt.getTime() - 5)).toISOString(), + attemptId, stream: "stdout", chunk, seq: index + 1 }) + "\n").join(""); + const logRef = `${companyId}/${agentId}/${preparedId}`; + logs.content.set(logRef, content); + const error = "input.provider.profile does not match the qualified ACPX v1 profile"; + await db.insert(heartbeatRuns).values({ id: preparedId, companyId, agentId, issueId, status: "failed", runtimeMode: "legacy", + errorCode: "adapter_failed", error, executionStage: "preparing", invocationSource: "automation", retryOfRunId: firstId, + createdAt: new Date(now - 790_000), startedAt: new Date(now - 760_000), finishedAt: new Date(now - 690_000), + scheduledRetryAt, scheduledRetryAttempt: 1, scheduledRetryReason: "transient_failure", + controllerBootId: randomUUID(), logStore: "s3_segments", logRef, logBytes: Buffer.byteLength(content), + logSha256: createHash("sha256").update(content).digest("hex"), logCompressed: false, stdoutExcerpt: lines.join(""), + stderrExcerpt: "", lastOutputAt: outputAt, lastOutputSeq: 2, lastOutputStream: "stdout", lastOutputBytes: Buffer.byteLength(content), + usageJson: { ledgerScope: { issueId, projectId: null, billingCode: null } }, + contextSnapshot: { issueId, retryOfRunId: firstId, wakeReason: "transient_failure_retry", retryReason: "transient_failure", + scheduledRetryAt: scheduledRetryAt.toISOString(), scheduledRetryAttempt: 1 }, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" }, + remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId, providerLeaseId, remoteCwd, + epoch: 8, state: "prepared", snapshotId }, + chatControlRecoveryAdmission: { phase: "required", runId: preparedId, agentId, issueId, version: 1, companyId } }, + resultJson: { stopReason: "adapter_failed", failureCause: "native_execution_input_invalid", timeoutFired: false, + timeoutSource: "config", executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, timeoutConfigured: false, + effectiveTimeoutSec: 0, systemRecoveryAdoption: { actionId, policyVersion: 1 }, workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] } } }); + const event = { companyId, agentId, runId: preparedId, stream: "system" }; + await db.insert(heartbeatRunEvents).values([ + { ...event, seq: 1, eventType: "lifecycle", level: "info", message: "Scheduled retry became due and was promoted to the queued run pool", + payload: { scheduledRetryAt: scheduledRetryAt.toISOString(), scheduledRetryReason: "transient_failure", scheduledRetryAttempt: 1 } }, + { ...event, seq: 2, eventType: "lifecycle", level: "info", message: "run started" }, + { ...event, seq: 3, eventType: "error", level: "error", message: error }, + { ...event, seq: 4, eventType: "lifecycle", level: "info", message: "task.run.measured", + payload: { phase: "task.run.measured", outcome: "failed", durationMs: 318_581, durableSettlement: true } }, + ]); + const releasedAt = new Date(now - 650_000); + await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId, heartbeatRunId: preparedId, + executionWorkspaceId: workspaceId, provider: "daytona", providerLeaseId, status: "expired", + leasePolicy: "reuse_by_environment", releasedAt, cleanupStatus: "success", + metadata: terminated(companyId, preparedId, leaseId, providerLeaseId, releasedAt) }); + await db.insert(issueRecoveryActions).values({ id: actionId, companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", fingerprint: preparedId, + status: "resolved", outcome: "blocked", + nextAction: "Automatic recovery of saved work failed (last failure: source_unavailable). Retry to try again; the agent will not repeat its work.", + evidence: { runId: preparedId, + systemRecovery: { schema: "paperclip.system-recovery.v1", policyVersion: 1, capturePolicyVersion: 2, phase: "verify_execution", + state: "exhausted", failure: { code: "source_unavailable", retryClass: "use_alternative_strategy", evidenceFingerprint: "a".repeat(64) }, + attempts: 1, intentId: null, revision: 1, attemptId: randomUUID(), deadlineAt: now + 600_000, retriesUsed: 0, + sourceRunId: preparedId, generationId: randomUUID(), nextAttemptAt: null, firstFailureAt: now - 600_000, + freshAttemptRunId: null, providerCooldownAt: null, preservationOperationIds: [] }, + automaticRecovery: { runId: preparedId, policy: "preserve_without_replay_v1", replay: "blocked", + recordedAt: new Date(now - 680_000).toISOString(), actionOutcome: "unknown" }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: preparedId, stage: "verifying_stop", status: "blocked", + attempt: 1, attemptLimit: 6, retryable: true, requestIds: [], + message: "Automatic recovery of saved work failed (last failure: source_unavailable). Retry to try again; the agent will not repeat its work." }, + originalFailureCode: "adapter_failed", continuationDelivery: "invalidated", + workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] }, + historicalWorkspaceSource: { schema: "paperclip.historical-workspace-source.v1", runId: preparedId, leaseIds: [leaseId] }, + previousWorkspaceRecovery: null } }); + await db.insert(heartbeatRuns).values({ id: heldId, companyId, agentId, issueId, status: "cancelled", runtimeMode: "legacy", + errorCode: "execution_reconciliation_required", error: "Recovering saved work…", invocationSource: "automation", + triggerDetail: "system", retryOfRunId: preparedId, createdAt: new Date(now - 680_000), finishedAt: new Date(now - 600_000), + contextSnapshot: { issueId, taskId: issueId, source: "issue.continuation_recovery", wakeReason: "issue_continuation_needed", + retryOfRunId: preparedId }, + resultJson: { stopReason: "execution_reconciliation_required", timeoutFired: false, + executionWait: { issueId, recoveryActionId: actionId }, timeoutSource: "stale_queued_run_gate", + timeoutConfigured: false, effectiveTimeoutSec: 0 } }); + return { ...f, preparedId, heldId, actionId }; + } + + async function restoredSnapshot() { + const f = await base("RS"); + const { companyId, issueId, agentId, workspaceId } = f; + const earlierId = randomUUID(), sourceId = randomUUID(), actionId = randomUUID(), leaseId = randomUUID(); + const providerLeaseId = randomUUID(), snapshotId = randomUUID(), now = Date.now(), sha256 = "c".repeat(64); + const remoteCwd = `/home/daytona/paperclip-workspace/.paperclip-workspaces/${workspaceId}-1-${leaseId}`; + const location = { leaseId, providerLeaseId, remoteCwd, environmentId: randomUUID() }; + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, + mode: "isolated_workspace", strategyType: "git_worktree", name: "Task workspace", providerType: "cloud_sandbox", + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 3, dirty: false, source: { kind: "empty" }, + location, snapshotId, writerRunId: null } } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issueId)); + await db.insert(heartbeatRuns).values({ id: earlierId, companyId, agentId, issueId, status: "succeeded", runtimeMode: "native", + createdAt: new Date(now - 7_200_000), startedAt: new Date(now - 7_100_000), finishedAt: new Date(now - 7_000_000), + contextSnapshot: { issueId } }); + const [earlierSave] = await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId: workspaceId, + heartbeatRunId: earlierId, issueId, phase: "remote_workspace_snapshot", status: "succeeded", + metadata: { boundary: "final", epoch: 3, snapshotId } }).returning(); + await db.insert(executionWorkspaceSnapshots).values({ id: snapshotId, companyId, executionWorkspaceId: workspaceId, + sourceOperationId: earlierSave.id, sourceRunId: earlierId, generation: 3, epoch: 3, status: "committed", + objectKey: `${companyId}/workspace-snapshots/${workspaceId}/${snapshotId}/archive`, sha256, byteSize: 48_507_938, + key: { kind: "fixture" }, manifestDigest: "d".repeat(64) }); + const error = "Request failed with status code 502"; + await db.insert(heartbeatRuns).values({ id: sourceId, companyId, agentId, issueId, status: "failed", runtimeMode: "legacy", + errorCode: "setup_failed", error, executionStage: "preparing", createdAt: new Date(now - 300_000), + startedAt: new Date(now - 280_000), finishedAt: new Date(now - 220_000), controllerBootId: randomUUID(), + usageJson: { ledgerScope: { issueId, projectId: null, billingCode: null } }, + contextSnapshot: { issueId, wakeReason: "task_retry" }, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" }, + remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId, providerLeaseId, remoteCwd, + epoch: 3, state: "saved", snapshotId } }, + resultJson: { stopReason: "adapter_failed", failureCause: "JsonRpcCallError", timeoutFired: false, timeoutSource: "config", + executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, timeoutConfigured: false, effectiveTimeoutSec: 0, + systemRecoveryAdoption: { actionId, policyVersion: 1 }, workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] } } }); + const event = { companyId, agentId, runId: sourceId, stream: "system" }; + await db.insert(heartbeatRunEvents).values([ + { ...event, seq: 1, eventType: "error", level: "error", message: error }, + { ...event, seq: 2, eventType: "lifecycle", level: "info", message: "task.run.measured", + payload: { phase: "task.run.measured", outcome: "failed", durationMs: 61_906, durableSettlement: true } }, + ]); + const releasedAt = new Date(now - 200_000); + await db.insert(environmentLeases).values({ id: leaseId, companyId, issueId, heartbeatRunId: sourceId, + executionWorkspaceId: workspaceId, provider: "daytona", providerLeaseId, status: "expired", + leasePolicy: "reuse_by_environment", releasedAt, cleanupStatus: "success", + metadata: terminated(companyId, sourceId, leaseId, providerLeaseId, releasedAt) }); + await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId: workspaceId, heartbeatRunId: sourceId, issueId, + phase: "remote_workspace_restore", status: "succeeded", metadata: { epoch: 3, snapshotId, + exportCleanup: { state: "cleaned", leaseId, providerLeaseId, remoteCwd, lastError: null, + stagingRoot: "/home/daytona/paperclip-workspace/.paperclip-workspaces", environmentId: location.environmentId } } }); + const [recovered] = await db.insert(workspaceOperations).values({ companyId, executionWorkspaceId: workspaceId, + heartbeatRunId: sourceId, issueId, phase: "workspace_recovery", status: "succeeded", + metadata: { epoch: 3, sha256, leaseId, storage: "remote_workspace", generation: 3, snapshotId, providerLeaseId, + recoveryActionId: actionId } }).returning(); + await db.insert(workspaceOperations).values({ companyId, heartbeatRunId: sourceId, issueId, phase: "workspace_finalize", + status: "succeeded", metadata: { owningService: "workspace_recovery", preservationOperationIds: [recovered.id] } }); + await db.insert(issueRecoveryActions).values({ id: actionId, companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", fingerprint: sourceId, + status: "resolved", outcome: "blocked", nextAction: "Saved work recovered.", + evidence: { runId: sourceId, + systemRecovery: { schema: "paperclip.system-recovery.v1", policyVersion: 1, capturePolicyVersion: 2, phase: "reconcile_effects", + state: "complete", failure: null, attempts: 1, intentId: null, revision: 3, attemptId: randomUUID(), deadlineAt: null, + retriesUsed: 0, sourceRunId: sourceId, generationId: randomUUID(), nextAttemptAt: null, firstFailureAt: null, + freshAttemptRunId: null, providerCooldownAt: null, preservationOperationIds: [recovered.id] }, + automaticRecovery: { runId: sourceId, policy: "preserve_without_replay_v1", replay: "blocked", + recordedAt: new Date(now - 210_000).toISOString(), actionOutcome: "unknown" }, + workspaceRecovery: { schema: "paperclip.workspace-recovery.v1", runId: sourceId, stage: "saving_receipts", status: "recovered", + attempt: 1, attemptLimit: 6, retryable: true, requestIds: [] }, + originalFailureCode: "setup_failed", continuationDelivery: "invalidated", workspaceRestoreFailure: "restore_failed", + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [leaseId] }, + historicalWorkspaceSource: { schema: "paperclip.historical-workspace-source.v1", runId: sourceId, leaseIds: [leaseId] }, + previousWorkspaceRecovery: null } }); + return { ...f, sourceId, actionId, snapshotId }; + } + + async function retry(f: { companyId: string; issueId: string }, observedRunId: string) { + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId, + observedRunId, idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + expect(receipt).toMatchObject({ accepted: true }); + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + return wake; + } + + async function issue(issueId: string) { + const [row] = await db.select().from(issues).where(eq(issues.id, issueId)); + return row; + } + + it("offers the same Retry in the task view and the blocker when saved work is no longer available", async () => { + const f = await sourceUnavailable(); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + const interruption = await taskInterruptionPresentation(db, await issue(f.issueId)); + expect(blocker?.workspaceRecovery).toMatchObject({ status: "blocked", canRetry: true }); + expect(blocker?.canRetry).toBe(true); + expect(interruption?.canRetry).toBe(true); + }); + + it("offers no Retry in either view while saved-work recovery waits for an unconfirmed file copy to stop", async () => { + const f = await sourceUnavailable(); + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + workspaceOwnerRecovery: { state: "operator_required", owner: { pid: 3, token: randomUUID(), hostname: "previous-host" } } })}::jsonb` }) + .where(eq(issueRecoveryActions.id, f.actionId)); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + const interruption = await taskInterruptionPresentation(db, await issue(f.issueId)); + expect(blocker).toMatchObject({ canRetry: false, workspaceRecovery: { status: "blocked", canRetry: false } }); + expect(interruption).toMatchObject({ canRetry: false, retryWait: { reason: "workspace_repair_required" } }); + }); + + it("offers no Retry for a held Retry with the generic wait over a hold waiting for an unconfirmed file copy to stop", async () => { + const f = await sourceUnavailable(); + await db.update(issueRecoveryActions).set({ evidence: sql`${issueRecoveryActions.evidence} || ${JSON.stringify({ + workspaceOwnerRecovery: { state: "operator_required", owner: { pid: 3, token: randomUUID(), hostname: "previous-host" } } })}::jsonb` }) + .where(eq(issueRecoveryActions.id, f.actionId)); + const [agent] = await db.select({ id: agents.id }).from(agents).where(eq(agents.companyId, f.companyId)); + const requestId = randomUUID(); + await db.insert(agentWakeupRequests).values({ id: requestId, companyId: f.companyId, agentId: agent.id, source: "on_demand", + triggerDetail: "manual", reason: "task_retry", status: "deferred_issue_execution", requestedByActorType: "user", + requestedByActorId: actorId, idempotencyKey: `task-retry:${randomUUID()}`, + payload: { issueId: f.issueId, executionWait: { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }, + taskRetry: { schema: "paperclip.task-retry.v1", generationId: requestId, sourceRunId: f.heldId, actorId, + actorSource: "local_implicit", strategy: "evaluate" } } }); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + const interruption = await taskInterruptionPresentation(db, await issue(f.issueId)); + expect(blocker?.canRetry).toBe(false); + expect(interruption).toMatchObject({ canRetry: false, retryWait: { reason: "execution_recovery" } }); + }); + + it("continues in a fresh session from the recorded work when saved work is no longer available", async () => { + const f = await sourceUnavailable(); + const wake = await retry(f, f.heldId); + expect(wake.payload?.executionWait).toBeUndefined(); + expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, wake.runId!)); + expect(["queued", "running"]).toContain(successor.status); + expect(successor.contextSnapshot).toMatchObject({ taskRetryRequestId: wake.id }); + }); + + it("verifies the repair of a restored snapshot that the failed preparation never changed", async () => { + const f = await restoredSnapshot(); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + const interruption = await taskInterruptionPresentation(db, await issue(f.issueId)); + expect(blocker).toMatchObject({ workspaceRepairRequired: false, canRetry: true, + workspaceRecovery: { status: "recovered" } }); + expect(interruption?.canRetry).toBe(true); + const wake = await retry(f, f.sourceId); + expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + const [successor] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, wake.runId!)); + expect(["queued", "running"]).toContain(successor.status); + }); + + it("keeps the repair unverified when the failed preparation may have changed the restored snapshot", async () => { + const f = await restoredSnapshot(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.sourceId)); + await db.update(heartbeatRuns).set({ resultJson: { ...run.resultJson, + executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }).where(eq(heartbeatRuns.id, f.sourceId)); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + expect(blocker).toMatchObject({ workspaceRepairRequired: true, canRetry: false }); + expect(await taskInterruptionPresentation(db, await issue(f.issueId))).toMatchObject({ canRetry: false }); + }); + + it("names a refusal that another click cannot change and offers no Retry for it in either view", async () => { + const f = await restoredSnapshot(); + await db.update(workspaceOperations).set({ status: "failed" }).where(and(eq(workspaceOperations.heartbeatRunId, f.sourceId), + eq(workspaceOperations.phase, "remote_workspace_restore"))); + await db.update(issueRecoveryActions).set({ evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, + '{workspaceRecovery,status}', '"blocked"')` }).where(eq(issueRecoveryActions.id, f.actionId)); + const wake = await retry(f, f.sourceId); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null, + payload: { executionWait: { reason: "previous_execution_unverified" } } }); + const interruption = await taskInterruptionPresentation(db, await issue(f.issueId)); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + expect(interruption).toMatchObject({ canRetry: false, retryWait: { reason: "previous_execution_unverified" } }); + expect(blocker?.canRetry).toBe(false); + }); + + it("names the admission refusal of a held Retry and offers Retry only when admission would take it", async () => { + const f = await restoredSnapshot(); + await db.update(heartbeatRuns).set({ controllerLeaseExpiresAt: new Date(Date.now() + 3_600_000) }) + .where(eq(heartbeatRuns.id, f.sourceId)); + const wake = await retry(f, f.sourceId); + expect(wake).toMatchObject({ status: "deferred_issue_execution", runId: null }); + const reason = (wake.payload?.executionWait as { reason?: string } | undefined)?.reason; + expect(reason).toBeTruthy(); + expect(reason).not.toBe("execution_recovery"); + const interruption = await taskInterruptionPresentation(db, await issue(f.issueId)); + expect(interruption?.retryWait?.reason).toBe(reason); + expect(interruption?.retryWait?.message).not.toMatch(/execution checks/); + }); +}); diff --git a/server/src/services/task-interruption-presentation.ts b/server/src/services/task-interruption-presentation.ts index f941f789b7c..8ee3505183c 100644 --- a/server/src/services/task-interruption-presentation.ts +++ b/server/src/services/task-interruption-presentation.ts @@ -1,7 +1,7 @@ import { STORAGE_UNAVAILABLE_WAIT_RETRY_REASON } from "../storage/storage-unavailable.js"; import { AGENT_DIRECTORY_WAIT_RETRY_REASON } from "./execution-recovery-attempt.js"; import { boundSetupCommandFailure, readSetupCommandFailure } from "./workspace-setup-command-failure.js"; -import { and, desc, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, inArray, isNull, not, sql, type SQL } from "drizzle-orm"; import { unidentifiedLocalLaunchRefusals } from "./native-local-process-stop.js"; import { nativeSessionCleanupBlock } from "../vendor/paperclip-runner/index.js"; import { agents, agentWakeupRequests, approvals, chatActions, companies, environmentLeases, heartbeatRuns, issueApprovals, issueRecoveryActions, @@ -23,6 +23,7 @@ import { canonicalExecutionUuid } from "./execution-identity.js"; import { chatActionDeadline } from "./chat-action-work.js"; import { NATIVE_CLEANUP_SETTLING_MESSAGE, NATIVE_SESSION_CLEANUP_QUARANTINED } from "./native-runtime/completed-turn-cleanup.js"; import { isQueuedDependencyWait } from "./run-cancellation.js"; +import { heldRetryOffersRetry } from "./task-retry-eligibility.js"; import { exhaustedNativeFinalizationCondition, savedResultRetryPendingCondition } from "./native-runtime/exhausted-finalization-retry.js"; import { AI_CONNECTION_BUSY_RETRY_REASON, AI_CONNECTION_POOL_WAIT_RETRY_REASON, RUNNER_REVISION_WAIT_RETRY_REASON, WORKSPACE_BUSY_RETRY_REASON } from "../modules/run-dispatch/index.js"; @@ -161,6 +162,7 @@ const retryWaitMessages: Record = { // A new click replaces a Retry declined for its requester's access with a // fresh attempt under the clicker's access, so Retry stays available. retry_authority_denied: "Retry could not start with the requesting user's access. Press Retry to start it with yours.", + previous_execution_unverified: "Retry can't continue because Paperclip can't confirm that the previous attempt stopped before it started work. Your message is saved.", // An edited saved message can never be replayed; only new input continues. retry_source_changed: "Retry cannot continue because its saved message changed. Send a new message to continue from saved work.", }; @@ -339,8 +341,7 @@ function projectTaskInterruptionState(issue: ProjectionIssue, input: ProjectionS const reason = Object.hasOwn(retryWaitMessages, retryWait.executionWaitReason) ? retryWait.executionWaitReason : "execution_recovery"; // Keep the latest input receipt so message submission can settle independently. return { ...base, state: "failed", - canRetry: !["cancelled_by_operator", "retry_source_changed", "workspace_owner_stop_required"].includes(reason) - && (reason !== "cleanup_quarantined" || !input.cleanupSettling), retryWait: { reason, + canRetry: heldRetryOffersRetry(reason, input.cleanupSettling), retryWait: { reason, message: retryWaitMessages[reason] ?? "Retry is waiting for execution checks. Your message is saved." } }; } const queuedDuringRun = Boolean(wait && run.status === "succeeded" && ["todo", "in_progress", "in_review"].includes(issue.status) && run.finishedAt && wait.requestedAt.getTime() <= run.finishedAt.getTime() @@ -744,6 +745,22 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i agentAdapter: run.agentAdapter ?? "unknown" }); } } + const offered = [...result].flatMap(([issueId, presentation]) => presentation?.canRetry ? [issueId] : []); + if (offered.length) { + const { executionBlockerPredicate, getExecutionBlocker, restoreHoldPredicate, restoreHoldReleasedForCurrentOwnerPredicate } = + await import("./execution-blocker.js"); + const held = await db.selectDistinct({ issueId: issueRecoveryActions.sourceIssueId }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.sourceIssueId, offered), + executionBlockerPredicate(), not(restoreHoldReleasedForCurrentOwnerPredicate()), restoreHoldPredicate())); + for (const { issueId } of held) { + const blocker = await getExecutionBlocker(db, companyId, issueId); + const presentation = result.get(issueId); + if (!presentation || !blocker || blocker.canRetry) continue; + presentation.canRetry = false; + if (!presentation.retryWait && blocker.workspaceRecovery && blocker.workspaceRecovery.status !== "recovered") + presentation.retryWait = { reason: "workspace_repair_required", message: retryWaitMessages.workspace_repair_required! }; + } + } // Redact each explanation with the registry of the run that recorded it: the // workspace failure is the observed run's, while a scheduled retry's provider // failure may be its predecessor's. diff --git a/server/src/services/task-retry-eligibility.ts b/server/src/services/task-retry-eligibility.ts new file mode 100644 index 00000000000..e4e3ef1719b --- /dev/null +++ b/server/src/services/task-retry-eligibility.ts @@ -0,0 +1,30 @@ +import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; +import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; + +const REFUSALS_A_CLICK_CANNOT_CHANGE: ReadonlySet = new Set([ + "cancelled_by_operator", "retry_source_changed", "workspace_owner_stop_required", "previous_execution_unverified", +]); + +export function heldRetryOffersRetry(reason: string | null | undefined, cleanupSettling = false): boolean { + if (!reason) return true; + return !REFUSALS_A_CLICK_CANNOT_CHANGE.has(reason) && (reason !== "cleanup_quarantined" || !cleanupSettling); +} + +export async function heldTaskRetryRefusals(db: Pick, companyId: string, issueIds: string[]): Promise> { + if (!issueIds.length) return new Map(); + const issueId = sql`${agentWakeupRequests.payload}->>'issueId'`; + const rows = await db.selectDistinctOn([issueId], { issueId, sourceRunId: sql`${agentWakeupRequests.payload}#>>'{taskRetry,sourceRunId}'`, + reason: sql`${agentWakeupRequests.payload}#>>'{executionWait,reason}'`, + latestRunId: sql`(select latest.id::text from ${heartbeatRuns} latest + where latest.company_id = ${companyId} and (latest.issue_id::text = ${agentWakeupRequests.payload}->>'issueId' + or latest.native_issue_id::text = ${agentWakeupRequests.payload}->>'issueId' + or latest.context_snapshot->>'issueId' = ${agentWakeupRequests.payload}->>'issueId') + order by latest.created_at desc, latest.id desc limit 1)` }) + .from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), + isNull(agentWakeupRequests.runId), inArray(issueId, issueIds), + sql`${agentWakeupRequests.payload}#>>'{executionWait,reason}' is not null`)) + .orderBy(issueId, desc(agentWakeupRequests.requestedAt)); + return new Map(rows.flatMap(row => row.reason && row.sourceRunId && row.sourceRunId === row.latestRunId + ? [[row.issueId, row.reason] as const] : [])); +} diff --git a/server/src/services/verified-preparation-log.test.ts b/server/src/services/verified-preparation-log.test.ts index 0ab4e7096e9..391bf3fdc75 100644 --- a/server/src/services/verified-preparation-log.test.ts +++ b/server/src/services/verified-preparation-log.test.ts @@ -65,6 +65,50 @@ describe("verified host preparation output", () => { expect(await hasVerifiedPreparationLog(f.run)).toBe(false); }); + it("verifies a log made only of the host's workspace notices written before the adapter starts", async () => { + const f = source(); + const issueId = randomUUID(), ts = f.run.lastOutputAt!; + const chunks = [f.record.chunk, + "[paperclip] Execution workspace reuse freshness action \"replace\" because execution workspace configuration changed: repo/base ref, environment.\n", + `[paperclip] Skipping saved session resume for task "${issueId}" because effective run configuration changed: environment.\n`]; + const content = chunks.map((chunk, index) => JSON.stringify({ ...f.record, chunk, seq: index + 1, ts: ts.toISOString() }) + "\n").join(""); + Object.assign(f.run, { lastOutputSeq: 3, stdoutExcerpt: chunks.join(""), logBytes: Buffer.byteLength(content), + lastOutputBytes: Buffer.byteLength(content), logSha256: createHash("sha256").update(content).digest("hex") }); + fixture.read.mockResolvedValue({ content }); + expect(await hasVerifiedPreparationLog(f.run)).toBe(true); + const notices = chunks.slice(1); + const withoutSkills = notices.map((chunk, index) => JSON.stringify({ ...f.record, chunk, seq: index + 1, ts: ts.toISOString() }) + "\n").join(""); + Object.assign(f.run, { contextSnapshot: null, lastOutputSeq: 2, stdoutExcerpt: notices.join(""), logBytes: Buffer.byteLength(withoutSkills), + lastOutputBytes: Buffer.byteLength(withoutSkills), logSha256: createHash("sha256").update(withoutSkills).digest("hex") }); + fixture.read.mockResolvedValue({ content: withoutSkills }); + expect(await hasVerifiedPreparationLog(f.run)).toBe(true); + }); + + it("accepts two notices of the same attempt", async () => { + const f = source(); + const notice = "[paperclip] Skipping saved session resume because effective run configuration changed: environment.\n"; + const content = [1, 2].map(seq => JSON.stringify({ ...f.record, chunk: notice, seq }) + "\n").join(""); + Object.assign(f.run, { lastOutputSeq: 2, stdoutExcerpt: notice + notice, logBytes: Buffer.byteLength(content), + lastOutputBytes: Buffer.byteLength(content), logSha256: createHash("sha256").update(content).digest("hex") }); + fixture.read.mockResolvedValue({ content }); + expect(await hasVerifiedPreparationLog(f.run)).toBe(true); + }); + + it.each(["provider_line", "other_attempt", "skills_after_notice"] as const)("keeps a log with other output held: %s", async state => { + const f = source(); + const notice = "[paperclip] Skipping saved session resume because effective run configuration changed: environment.\n"; + const records = [{ ...f.record, chunk: notice, seq: 1 }, { ...f.record, seq: 2 }]; + if (state !== "skills_after_notice") records[1]!.chunk = notice; + if (state === "provider_line") records[1]!.chunk = "[paperclip] Provider wrote a file\n"; + if (state === "other_attempt") records[1]!.attemptId = randomUUID(); + const content = records.map(record => JSON.stringify(record) + "\n").join(""); + Object.assign(f.run, { lastOutputSeq: 2, stdoutExcerpt: records.map(record => record.chunk).join(""), + logBytes: Buffer.byteLength(content), lastOutputBytes: Buffer.byteLength(content), + logSha256: createHash("sha256").update(content).digest("hex") }); + fixture.read.mockResolvedValue({ content }); + expect(await hasVerifiedPreparationLog(f.run)).toBe(false); + }); + it("cancels a slow read, fails closed, and leaves an unrelated recovery responsive", async () => { const slow = source(), fast = source(); let signal: AbortSignal | undefined; diff --git a/server/src/services/verified-preparation-log.ts b/server/src/services/verified-preparation-log.ts index 161d18441ea..715f4b117f3 100644 --- a/server/src/services/verified-preparation-log.ts +++ b/server/src/services/verified-preparation-log.ts @@ -3,7 +3,13 @@ import { isUuidLike } from "@paperclipai/shared"; import { getRunLogStore } from "./run-log-store.js"; const MAX_BYTES = 2048; +const MAX_RECORDS = 4; const READ_TIMEOUT_MS = 2000; +const SKILLS_PREFIX = "[paperclip] Enabled run-scoped skills from issue mentions: "; +const HOST_WORKSPACE_NOTICES = [ + /^\[paperclip\] Execution workspace reuse freshness action "[a-z_]+" because [^\r\n]+\.\n$/, + /^\[paperclip\] Skipping saved session resume (?:for task "[^"\r\n]+" )?because [^\r\n]+\.\n$/, +]; type PreparationLog = { companyId: string; agentId: string; id: string; contextSnapshot: Record | null; @@ -12,14 +18,26 @@ type PreparationLog = { lastOutputAt: Date | null; lastOutputSeq: number; lastOutputStream: string | null; lastOutputBytes: number | null; }; -export async function hasVerifiedPreparationLog(run: PreparationLog): Promise { - const versions = run.contextSnapshot?.skillVersions; - if (run.contextSnapshot?.skillSelectionSnapshot !== true || !versions || typeof versions !== "object" +function isMentionedSkillsNotice(chunk: string, context: Record | null) { + const versions = context?.skillVersions; + if (context?.skillSelectionSnapshot !== true || !versions || typeof versions !== "object" || Array.isArray(versions) || Object.keys(versions).length === 0 - || run.logStore !== "s3_segments" || run.logRef !== `${run.companyId}/${run.agentId}/${run.id}` + || !chunk.startsWith(SKILLS_PREFIX) || !chunk.endsWith("\n")) return false; + const keys = chunk.slice(SKILLS_PREFIX.length, -1).split(", "); + return keys.length > 0 && new Set(keys).size === keys.length + && keys.every(key => { + const version = (versions as Record)[key]; + return key.length > 0 && Object.hasOwn(versions, key) && !/[\r\n]/.test(key) + && typeof version === "string" && isUuidLike(version); + }); +} + +export async function hasVerifiedPreparationLog(run: PreparationLog): Promise { + if (run.logStore !== "s3_segments" || run.logRef !== `${run.companyId}/${run.agentId}/${run.id}` || !Number.isSafeInteger(run.logBytes) || run.logBytes! <= 0 || run.logBytes! > MAX_BYTES || !run.logSha256 || !/^[a-f0-9]{64}$/.test(run.logSha256) || run.logCompressed - || run.lastOutputSeq !== 1 || run.lastOutputStream !== "stdout" || !run.lastOutputAt + || !Number.isSafeInteger(run.lastOutputSeq) || run.lastOutputSeq < 1 || run.lastOutputSeq > MAX_RECORDS + || run.lastOutputStream !== "stdout" || !run.lastOutputAt || run.lastOutputBytes !== run.logBytes || run.stderrExcerpt !== "") return false; const controller = new AbortController(); let timer: ReturnType | undefined; @@ -31,22 +49,25 @@ export async function hasVerifiedPreparationLog(run: PreparationLog): Promise | null; - if (!record || Array.isArray(record) || typeof record !== "object" - || Object.keys(record).some(key => !["ts", "attemptId", "stream", "chunk", "seq"].includes(key)) - || record.ts !== run.lastOutputAt.toISOString() || typeof record.attemptId !== "string" || !isUuidLike(record.attemptId) - || record.seq !== 1 || record.stream !== "stdout" || typeof record.chunk !== "string" - || record.chunk !== run.stdoutExcerpt) return false; - const prefix = "[paperclip] Enabled run-scoped skills from issue mentions: "; - if (!record.chunk.startsWith(prefix) || !record.chunk.endsWith("\n")) return false; - const keys = record.chunk.slice(prefix.length, -1).split(", "); - return keys.length > 0 && new Set(keys).size === keys.length - && keys.every(key => { - const version = (versions as Record)[key]; - return key.length > 0 && Object.hasOwn(versions, key) && !/[\r\n]/.test(key) - && typeof version === "string" && isUuidLike(version); - }); + || createHash("sha256").update(result.content).digest("hex") !== run.logSha256 + || !result.content.endsWith("\n")) return false; + const records = result.content.slice(0, -1).split("\n").map(line => JSON.parse(line) as Record | null); + if (records.length !== run.lastOutputSeq) return false; + const attemptId = records[0]?.attemptId; + let excerpt = ""; + for (const [index, record] of records.entries()) { + if (!record || Array.isArray(record) || typeof record !== "object" + || Object.keys(record).some(key => !["ts", "attemptId", "stream", "chunk", "seq"].includes(key)) + || typeof record.ts !== "string" || !Number.isFinite(Date.parse(record.ts)) + || typeof record.attemptId !== "string" || !isUuidLike(record.attemptId) || record.attemptId !== attemptId + || record.seq !== index + 1 || record.stream !== "stdout" || typeof record.chunk !== "string") return false; + const chunk = record.chunk; + const notice = index === 0 && isMentionedSkillsNotice(chunk, run.contextSnapshot) + || HOST_WORKSPACE_NOTICES.some(pattern => pattern.test(chunk)); + if (!notice) return false; + excerpt += chunk; + } + return records.at(-1)!.ts === run.lastOutputAt.toISOString() && excerpt === run.stdoutExcerpt; } catch { return false; } finally { if (timer) clearTimeout(timer); controller.abort(); } } diff --git a/server/src/services/workspace-recovery-admission.test.ts b/server/src/services/workspace-recovery-admission.test.ts index 7236d811fa0..b5bccc7cb05 100644 --- a/server/src/services/workspace-recovery-admission.test.ts +++ b/server/src/services/workspace-recovery-admission.test.ts @@ -969,8 +969,9 @@ describe("automatic retained workspace recovery", () => { const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); expect(wake.payload?.executionWait).toMatchObject({ reason: "remote_cleanup" }); const [task] = await db.select().from(issues).where(eq(issues.id, f.issueId)); - expect(await taskInterruptionPresentation(db, task)).toMatchObject({ state: "failed", canRetry: true, + expect(await taskInterruptionPresentation(db, task)).toMatchObject({ state: "failed", canRetry: false, retryWait: { reason: "remote_cleanup", message: "Retry is waiting for the previous environment to stop. Your message is saved." } }); + expect((await getExecutionBlocker(db, f.companyId, f.issueId))?.canRetry).toBe(false); const commentId = randomUUID(), messageWakeId = randomUUID(); await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "board", body: "Keep my saved message while the previous environment stops." }); diff --git a/server/src/services/workspace-recovery-receipt.ts b/server/src/services/workspace-recovery-receipt.ts index 11910813161..33e402fece0 100644 --- a/server/src/services/workspace-recovery-receipt.ts +++ b/server/src/services/workspace-recovery-receipt.ts @@ -5,8 +5,8 @@ import { z } from "zod"; import { readNativeWorkspaceSyncReference, type NativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js"; import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js"; import { readSnapshot } from "./workspace-recovery-snapshot.js"; -import { exhaustedSystemRecoveryAction, readSystemRecovery } from "./system-recovery-state.js"; -import { verifiedWorkspaceSnapshotReuse } from "./remote-workspace-snapshot-reuse-receipt.js"; +import { exhaustedSystemRecoveryAction, isCapturePolicyRearmable, readSystemRecovery } from "./system-recovery-state.js"; +import { verifiedUnchangedRestoredSnapshot, verifiedWorkspaceSnapshotReuse } from "./remote-workspace-snapshot-reuse-receipt.js"; import { isAbandonedPreparedRemoteWorkspace } from "./remote-workspace-abandoned-preparation.js"; export const workspaceRecoverySchema = z.object({ @@ -62,6 +62,17 @@ export function recoveredWorkMessage(runStatus: string | null | undefined, retry : "Saved work recovered. Send a message to continue."; } +export function savedWorkOffersRetry(state: { status: string; canRetry: boolean } | undefined, + runStatus: string | null | undefined, workspaceRepairRequired: boolean, evidence: Record) { + if (!state || (runStatus !== "failed" && runStatus !== "timed_out")) return false; + if (state.status === "recovered") return !workspaceRepairRequired; + const ownerRecovery = evidence.workspaceOwnerRecovery as { state?: unknown } | undefined; + const recovery = readSystemRecovery(evidence); + return state.canRetry && !(ownerRecovery?.state === "operator_required" && evidence.workspaceOwnerStop == null) + && recovery?.state === "exhausted" && !recovery.freshAttemptRunId && recovery.failure?.retryClass !== "recheck_evidence" + && !isCapturePolicyRearmable(recovery); +} + export function workspaceRecoveryDisplay(evidence: Record, runId: string | null, run?: { status: string | null; retryOffered: boolean }): ExecutionBlocker["workspaceRecovery"] { const state = readWorkspaceRecovery(evidence); @@ -175,7 +186,8 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick