diff --git a/server/src/__tests__/interrupted-preparation-hold-settle.test.ts b/server/src/__tests__/interrupted-preparation-hold-settle.test.ts index 253e3a89aee..b79ca3c3ff9 100644 --- a/server/src/__tests__/interrupted-preparation-hold-settle.test.ts +++ b/server/src/__tests__/interrupted-preparation-hold-settle.test.ts @@ -153,6 +153,88 @@ suite("held interrupted preparations on the shared ownership path", () => { expect(audit?.details).toMatchObject({ reason: "native_bootstrap_stopped_before_provider", ownerProof: "pid_dead" }); }); + // Staging release-swap shape: a Runner preparation that was + // itself an automatic retry had prepared its remote workspace when the + // deploy's SIGTERM interrupted it. Workspace settlement then handed the + // workspace on at its saved version (the run never changed it), which turns + // the run's reference to `saved`. No restore is owed, but the preparation + // proof still demanded a workspace-repair receipt that only exists after a + // copy-back, so the hold never settled, every automatic continuation was + // refused ("could not start yet"), and the person's Retry waited forever on + // previous_execution_unverified. + async function shutdownPreparationWithUnchangedWorkspace(options: { released?: boolean; + receipt?: (runId: string, snapshotId: string) => Record } = {}) { + const released = options.released !== false; + const f = await interruptedPreparation({ run: { scheduledRetryAttempt: 1, scheduledRetryReason: "transient_failure" } }); + const workspaceId = randomUUID(), leaseId = randomUUID(), snapshotId = randomUUID(); + const source = { kind: "git", repoUrl: "https://github.com/paperclipai/paperclip.git", ref: "a".repeat(40), + branchMode: "create", branchName: "IPH-1-change" }; + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: f.issueId, mode: "isolated_workspace", + strategyType: "git_worktree", name: "Remote workspace", status: "active", providerType: "git_worktree", + metadata: { + remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 4, source, location: null, snapshotId, + dirty: !released, writerRunId: released ? null : f.runId }, + remoteWorkspaceUnchangedBaseline: { schema: "paperclip.remote-workspace-unchanged-baseline.v1", epoch: 4, + runId: f.runId, snapshotId, ...(released ? { releasedAt: new Date().toISOString() } : {}), + ...options.receipt?.(f.runId, snapshotId) }, + } }); + await db.update(heartbeatRuns).set({ + contextSnapshot: { issueId: f.issueId, retryReason: "transient_failure", + executionRetryAccounting: { version: 1, failureRetries: 1, maxTurnContinuations: 0 } }, + runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" }, + remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", state: released ? "saved" : "prepared", + workspaceId, epoch: 4, leaseId, providerLeaseId: `sandbox-${leaseId}`, + remoteCwd: `/home/daytona/paperclip-workspace/.paperclip-workspaces/${workspaceId}-4-${leaseId}`, snapshotId } } }) + .where(eq(heartbeatRuns.id, f.runId)); + return { ...f, workspaceId }; + } + + it("replays a shutdown-interrupted preparation whose remote workspace was handed on unchanged", async () => { + heartbeatService(db); + const f = await shutdownPreparationWithUnchangedWorkspace(); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toMatchObject({ cause: "legacy_execution_requires_reconciliation" }); + expect(await pass()).toMatchObject({ inProgress: 1, settled: 1 }); + expect(await hold(f.holdId)).toMatchObject({ status: "resolved", outcome: "false_positive" }); + expect(await getExecutionBlocker(db, companyId, f.issueId)).toBeNull(); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]).toMatchObject({ status: "in_progress" }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, f.runId))) + .toEqual([expect.objectContaining({ scheduledRetryReason: "interrupted_preparation" })]); + }); + + it("keeps the hold while the interrupted preparation still holds its remote workspace unsaved", async () => { + heartbeatService(db); + const f = await shutdownPreparationWithUnchangedWorkspace({ released: false }); + const before = await hold(f.holdId); + expect((await pass())?.settled ?? 0).toBe(0); + expect(await hold(f.holdId)).toEqual(before); + }); + + it.each([ + ["the receipt names another run", () => ({ runId: randomUUID() })], + ["the receipt is for another epoch", () => ({ epoch: 3 })], + ["the receipt is for another saved version", () => ({ snapshotId: randomUUID() })], + ["the receipt has no release time", () => ({ releasedAt: undefined })], + ] as const)("keeps the hold of a handed-on preparation when %s", async (_label, receipt) => { + heartbeatService(db); + const f = await shutdownPreparationWithUnchangedWorkspace({ receipt }); + const before = await hold(f.holdId); + expect((await pass())?.settled ?? 0).toBe(0); + expect(await hold(f.holdId)).toEqual(before); + }); + + it("starts a person's Retry of a shutdown-interrupted preparation whose remote workspace was handed on", async () => { + const heartbeat = heartbeatService(db); + const f = await shutdownPreparationWithUnchangedWorkspace(); + const receipt = await requestTaskRetry({ db, companyId, issueId: f.issueId, actorId: "local-board", + actorSource: "local_implicit", idempotencyKey: randomUUID(), observedRunId: f.runId }); + expect(receipt).toMatchObject({ accepted: true }); + await heartbeat.resumeTaskRetry(companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake?.status).not.toBe("deferred_issue_execution"); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.issueId, f.issueId), + ne(heartbeatRuns.id, f.runId)))).not.toHaveLength(0); + }, 60_000); + it.each([ ["provider work may have started", async () => { const f = await interruptedPreparation(); diff --git a/server/src/services/legacy-preparation-retry-hold.test.ts b/server/src/services/legacy-preparation-retry-hold.test.ts index 4b7092950b6..3f3fe9dab5d 100644 --- a/server/src/services/legacy-preparation-retry-hold.test.ts +++ b/server/src/services/legacy-preparation-retry-hold.test.ts @@ -95,6 +95,34 @@ describe("Retry of a legacy preparation that automatic recovery stopped", () => expect(successor.contextSnapshot).toMatchObject({ previousRunId: f.source.id, taskRetryRequestId: receipt.requestId }); }); + // Staging shape: a setup failure during preparation. The failed preparation had restored the task's saved + // version into its sandbox, and workspace settlement handed the workspace on + // unchanged (reference `saved`, unchanged-baseline receipt for this run). + // No copy-back is owed, so the person's Retry must start. + it("starts the person's Retry when the stopped preparation handed its remote workspace on unchanged", async () => { + const f = await seed(); + const workspaceId = randomUUID(), snapshotId = randomUUID(); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.leaseId)); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId, sourceIssueId: f.issueId, + mode: "isolated_workspace", strategyType: "git_worktree", name: "Remote workspace", providerType: "git_worktree", + metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 4, location: null, dirty: false, + writerRunId: null, snapshotId, source: { kind: "git", repoUrl: "https://github.com/paperclipai/paperclip.git", + ref: "a".repeat(40), branchMode: "create", branchName: "L-1-change" } }, + remoteWorkspaceUnchangedBaseline: { schema: "paperclip.remote-workspace-unchanged-baseline.v1", epoch: 4, + runId: f.source.id, snapshotId, releasedAt: new Date().toISOString() } } }); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...f.source.runnerProfileJson, + remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId: f.leaseId, + providerLeaseId: lease!.providerLeaseId!, remoteCwd: "/fixture", epoch: 4, state: "saved", snapshotId } } }) + .where(eq(heartbeatRuns.id, f.source.id)); + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID(), actorSource: "local_implicit" }); + expect(receipt).toMatchObject({ accepted: true }); + await heartbeatService(db).resumeTaskRetry(f.companyId, receipt.requestId!); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake.payload?.executionWait).toBeUndefined(); + expect(wake).toMatchObject({ status: "coalesced", runId: expect.any(String) }); + }); + it.each(["foreign_hold", "provider_event", "unstopped_lease", "unsaved_remote_workspace"] as const)( "keeps the Retry held when the stopped preparation is not proven: %s", async gate => { const f = await seed(); diff --git a/server/src/services/remote-workspace-abandoned-preparation.ts b/server/src/services/remote-workspace-abandoned-preparation.ts index dfc94a92cd2..ac7b43dfeae 100644 --- a/server/src/services/remote-workspace-abandoned-preparation.ts +++ b/server/src/services/remote-workspace-abandoned-preparation.ts @@ -2,6 +2,7 @@ import { and, eq, sql } from "drizzle-orm"; import { executionWorkspaces, heartbeatRuns, type Db } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared"; import { REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA } from "./remote-workspace-epoch-abandonment.js"; +import { readUnchangedBaseline } from "./remote-workspace-unchanged-baseline.js"; type Run = Pick; @@ -35,3 +36,25 @@ export function abandonedPreparedRemoteWorkspaceExists() { and receipt->>'schema' = ${REMOTE_WORKSPACE_ABANDONED_EPOCH_SCHEMA} and receipt->>'runId' = ${heartbeatRuns.id}::text and receipt->'epoch' = ${reference}->'epoch'))`; } + +/** + * The host handed the workspace on at its saved version because this run + * ended without changing it (`releaseUnchangedRemoteWorkspaceWriter`): the + * run's reference was turned to `saved` at that exact version and the + * workspace's unchanged-baseline receipt names the run, its epoch and that + * version, with the release time. The run holds no files beyond the saved + * version, so there is nothing to copy back or repair before a successor + * prepares the workspace again. A later writer that replaced the receipt + * leaves this unproven, and the run stays with its other recovery. + */ +export async function isReleasedUnchangedRemoteWorkspace(db: Db, run: Run): Promise { + if (run.runnerProfileJson?.nativeWorkspaceSync != null) return false; + const parsed = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace); + if (!parsed.success || parsed.data.state !== "saved" || !parsed.data.snapshotId) return false; + const reference = parsed.data; + const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, run.companyId), eq(executionWorkspaces.id, reference.workspaceId))); + const baseline = readUnchangedBaseline(workspace?.metadata); + return Boolean(baseline?.releasedAt && baseline.runId === run.id && baseline.epoch === reference.epoch + && baseline.snapshotId === reference.snapshotId); +} diff --git a/server/src/services/remote-workspace-unchanged-baseline.ts b/server/src/services/remote-workspace-unchanged-baseline.ts new file mode 100644 index 00000000000..ed52fe0e3c8 --- /dev/null +++ b/server/src/services/remote-workspace-unchanged-baseline.ts @@ -0,0 +1,29 @@ +// Leaf module: the unchanged-baseline receipt that remote workspace writers +// record. Recovery proofs read it without loading the unchanged-proof module. + +const object = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +/** + * A writer's prepared files equal its saved version only while nothing ran in + * them. Preparation records that starting point; everything after it leaves a + * durable record (an operation, a runner checkpoint, a run event or a + * process) before it can write. A run with none of those changed nothing. + */ +export const UNCHANGED_BASELINE_KEY = "remoteWorkspaceUnchangedBaseline"; +export const UNCHANGED_BASELINE_SCHEMA = "paperclip.remote-workspace-unchanged-baseline.v1"; + +export type UnchangedBaseline = { + schema: typeof UNCHANGED_BASELINE_SCHEMA; epoch: number; runId: string; snapshotId: string; + /** Set once the writer was released without a new save. */ + releasedAt?: string; +}; + +/** Missing or malformed fields (an older or partial record) never prove anything. */ +export function readUnchangedBaseline(metadata: Record | null | undefined): UnchangedBaseline | null { + const value = object(metadata?.[UNCHANGED_BASELINE_KEY]); + if (value.schema !== UNCHANGED_BASELINE_SCHEMA || !Number.isSafeInteger(value.epoch) || (value.epoch as number) < 1 + || typeof value.runId !== "string" || !value.runId || typeof value.snapshotId !== "string" || !value.snapshotId + || (value.releasedAt !== undefined && typeof value.releasedAt !== "string")) return null; + return value as UnchangedBaseline; +} diff --git a/server/src/services/remote-workspace-unchanged-proof.ts b/server/src/services/remote-workspace-unchanged-proof.ts index 4fe08424857..8fa001237de 100644 --- a/server/src/services/remote-workspace-unchanged-proof.ts +++ b/server/src/services/remote-workspace-unchanged-proof.ts @@ -8,24 +8,13 @@ import { executionStartedEventPredicate } from "./pre-dispatch-execution-evidenc import { runOwnerFenceHeldSql } from "./remote-workspace-writer-hold.js"; import { carriesInheritedRunnerIdentity } from "./native-runtime/native-inherited-runner-retry.js"; import { readFailedBeforeProviderTurnRecord } from "./native-runtime/failed-before-provider-turn.js"; +import { readUnchangedBaseline, UNCHANGED_BASELINE_KEY, UNCHANGED_BASELINE_SCHEMA, + type UnchangedBaseline } from "./remote-workspace-unchanged-baseline.js"; const PROVIDER_TURN_EVENT_TYPES = ["turn.started", "turn.completed", "turn.failed", "provider.event", "provider.rpc_result", "tool.execution.started", "item.started", "item.completed"]; -/** - * A writer's prepared files equal its saved version only while nothing ran in - * them. Preparation records that starting point; everything after it leaves a - * durable record (an operation, a runner checkpoint, a run event or a - * process) before it can write. A run with none of those changed nothing. - */ -export const UNCHANGED_BASELINE_KEY = "remoteWorkspaceUnchangedBaseline"; -export const UNCHANGED_BASELINE_SCHEMA = "paperclip.remote-workspace-unchanged-baseline.v1"; - -export type UnchangedBaseline = { - schema: typeof UNCHANGED_BASELINE_SCHEMA; epoch: number; runId: string; snapshotId: string; - /** Set once the writer was released without a new save. */ - releasedAt?: string; -}; +export { readUnchangedBaseline, UNCHANGED_BASELINE_KEY, UNCHANGED_BASELINE_SCHEMA, type UnchangedBaseline }; type Workspace = Pick; type Repository = { id: string; relativePath: string; repoUrl: string; ref: string | null; pinnedCommit?: string }; @@ -33,14 +22,6 @@ type Repository = { id: string; relativePath: string; repoUrl: string; ref: stri const object = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; -/** Missing or malformed fields (an older or partial record) never prove anything. */ -export function readUnchangedBaseline(metadata: Record | null | undefined): UnchangedBaseline | null { - const value = object(metadata?.[UNCHANGED_BASELINE_KEY]); - if (value.schema !== UNCHANGED_BASELINE_SCHEMA || !Number.isSafeInteger(value.epoch) || (value.epoch as number) < 1 - || typeof value.runId !== "string" || !value.runId || typeof value.snapshotId !== "string" || !value.snapshotId - || (value.releasedAt !== undefined && typeof value.releasedAt !== "string")) return null; - return value as UnchangedBaseline; -} /** Operations that cannot change the saved files: a restore writes exactly the * saved version, and finalize and recovery rows are receipts that wrap a save diff --git a/server/src/services/workspace-recovery-receipt.ts b/server/src/services/workspace-recovery-receipt.ts index 33e402fece0..c86d07dce50 100644 --- a/server/src/services/workspace-recovery-receipt.ts +++ b/server/src/services/workspace-recovery-receipt.ts @@ -7,7 +7,7 @@ import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-m import { readSnapshot } from "./workspace-recovery-snapshot.js"; import { exhaustedSystemRecoveryAction, isCapturePolicyRearmable, readSystemRecovery } from "./system-recovery-state.js"; import { verifiedUnchangedRestoredSnapshot, verifiedWorkspaceSnapshotReuse } from "./remote-workspace-snapshot-reuse-receipt.js"; -import { isAbandonedPreparedRemoteWorkspace } from "./remote-workspace-abandoned-preparation.js"; +import { isAbandonedPreparedRemoteWorkspace, isReleasedUnchangedRemoteWorkspace } from "./remote-workspace-abandoned-preparation.js"; export const workspaceRecoverySchema = z.object({ schema: z.literal("paperclip.workspace-recovery.v1"), @@ -198,7 +198,8 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick) { - return (await verifiedWorkspaceRepairReceipt(...args)) !== null || await isAbandonedPreparedRemoteWorkspace(...args); + return (await verifiedWorkspaceRepairReceipt(...args)) !== null || await isAbandonedPreparedRemoteWorkspace(...args) + || await isReleasedUnchangedRemoteWorkspace(...args); } /** Read-only historical bytes have no execution authority. Their immutable