Preserve descriptor-free recovery identities through migration

This commit is contained in:
Dotta committed 2026-11-03 15:28:38 -06:00
1 parent 1c172fc057
commit 9ab6a38fa4
7 files changed
+149 -45

No files matched your search

@@ -384,7 +384,7 @@ export async function admitExplicitNativeContinuation(input: {
// before admitting new user input; never apply or replay the old result.
const legacyReference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
const migratedSource = coordinator && !coordinator.resultId && !coordinator.leaseOwner
&& ["observed", "retryable_failure"].includes(coordinator.phase) && legacyReference
&& ["observed", "retryable_failure"].includes(coordinator.phase)
&& await hasVerifiedWorkspaceRepair(db, run)
? await readMigratedLegacyWorkspaceSource(db, { companyId, runId: run.id, reference: legacyReference }) : null;
// Offline migration fenced this exact unfinished turn and preserved its
@@ -1,11 +1,12 @@
import { isDeepStrictEqual } from "node:util";
import { and, eq, sql } from "drizzle-orm";
import { executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db";
import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import type { NativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
export type LegacyWorkspaceRecoverySource = {
runId: string; issueId: string; reference: NativeWorkspaceSyncReference | null;
recoveryAllocations?: { leaseId: string; providerLeaseId: string; remoteCwd: string }[] | null;
unresolvedFinalization?: { phase: string; leaseOwner: string | null; leaseExpiresAt: string | null };
};
export const LEGACY_WORKSPACE_RECOVERY_MIGRATION = "paperclip.legacy-workspace-recovery-migration.v1";
@@ -34,34 +35,64 @@ export async function isLegacyWorkspaceTaskRelated(db: Pick<Db, "select">, input
return false;
}
/** Record every declared source, or none: partial preservation cannot release
* the task's recovery hold. The recovery owner separately verifies provenance,
* provider termination and archive bytes before writing its receipts. */
export async function readLegacyWorkspaceRecoveryAllocations(db: Pick<Db, "select">, run: {
id: string; companyId: string; nativeIssueId: string | null; resultJson: Record<string, unknown> | null;
}): Promise<LegacyWorkspaceRecoverySource["recoveryAllocations"]> {
const recovery = run.resultJson?.workspaceRestoreRecovery as { schema?: unknown; leaseIds?: unknown } | undefined;
const ids = recovery?.leaseIds;
if (recovery?.schema !== "paperclip.workspace-restore-recovery.v1" || !Array.isArray(ids) || !ids.length
|| ids.some(id => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))
|| new Set(ids).size !== ids.length || !run.nativeIssueId) return null;
const allocations = [];
for (const id of [...ids].sort()) {
const [lease] = await db.select({ providerLeaseId: environmentLeases.providerLeaseId, metadata: environmentLeases.metadata })
.from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, id),
eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.issueId, run.nativeIssueId)));
if (!lease?.providerLeaseId || typeof lease.metadata?.remoteCwd !== "string" || !lease.metadata.remoteCwd.startsWith("/")) return null;
allocations.push({ leaseId: id as string, providerLeaseId: lease.providerLeaseId, remoteCwd: lease.metadata.remoteCwd });
}
return allocations;
}
/** This receipt changes where recovery may save files; it never certifies that
* a provider stopped or that its files were recovered. The recovery owner must
* still validate allocation provenance and termination before exporting. */
export async function readMigratedLegacyWorkspaceSource(db: Pick<Db, "select">, input: {
companyId: string; runId: string; reference: NativeWorkspaceSyncReference;
companyId: string; runId: string; reference: NativeWorkspaceSyncReference | null;
}): Promise<LegacyWorkspaceRecoverySource | null> {
if (input.reference.state !== "prepared") return null;
if (input.reference && input.reference.state !== "prepared") return null;
const [run] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
));
const runMarker = run?.runnerProfileJson?.legacyWorkspaceRecoveryMigration as { workspaceId?: unknown } | undefined;
const workspaceId = input.reference?.workspaceId ?? runMarker?.workspaceId;
if (typeof workspaceId !== "string") return null;
const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.reference.workspaceId),
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, workspaceId),
));
if (!readRemoteWorkspaceAuthority(workspace?.metadata)) return null;
const marker = workspace?.metadata?.legacyWorkspaceRecoveryMigration as { schema?: unknown; operationId?: unknown } | undefined;
if (marker?.schema !== LEGACY_WORKSPACE_RECOVERY_MIGRATION || typeof marker.operationId !== "string")
throw new Error("legacy_workspace_migration_receipt_invalid");
const [run] = await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
));
if (!isDeepStrictEqual(run?.profile?.legacyWorkspaceRecoveryMigration, {
schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId: input.reference.workspaceId,
if (!isDeepStrictEqual(runMarker, {
schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId,
})) throw new Error("legacy_workspace_migration_receipt_invalid");
const [operation] = await db.select({ metadata: workspaceOperations.metadata }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.reference.workspaceId),
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, workspaceId),
eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration"),
eq(workspaceOperations.status, "succeeded"),
));
const sources = operation?.metadata?.legacyRecoverySources;
const source = Array.isArray(sources) ? sources.find(source => source && typeof source === "object"
&& source.runId === input.runId && typeof source.issueId === "string" && isDeepStrictEqual(source.reference, input.reference)) : null;
if (source && source.issueId !== run?.nativeIssueId) throw new Error("legacy_workspace_migration_receipt_invalid");
if (source && !input.reference && (run?.runnerProfileJson?.nativeWorkspaceSync != null
|| run?.runnerProfileJson?.remoteWorkspace != null
|| !isDeepStrictEqual(source.recoveryAllocations ?? null, run ? await readLegacyWorkspaceRecoveryAllocations(db, run) : null)))
throw new Error("legacy_workspace_migration_receipt_invalid");
if (!source) throw new Error("legacy_workspace_migration_receipt_invalid");
return source as LegacyWorkspaceRecoverySource;
}
@@ -8,7 +8,7 @@ import { createServer, type Server } from "node:http";
import { pathToFileURL } from "node:url";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { activityLog, agents, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
import { activityLog, agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
@@ -267,6 +267,25 @@ describe.skipIf(!support.supported)("remote workspace migration", () => {
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true })).rejects.toThrow("remote_workspace_migration_source_changed");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
});
it("rejects a descriptor-free allocation changed during upload", async () => {
const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
await db.insert(environmentLeases).values({ id: f.reference.leaseId, companyId: f.companyId,
issueId: f.issueId, heartbeatRunId: f.runId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId,
status: "released", metadata: { remoteCwd: f.reference.remoteCwd } });
await db.update(heartbeatRuns).set({ runnerProfileJson: {}, resultJson: {
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.reference.leaseId] },
} }).where(eq(heartbeatRuns.id, f.runId));
await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId));
const put = storage.putFile.bind(storage);
storage.putFile = async input => {
const result = await put(input);
await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.reference.leaseId));
return result;
};
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }))
.rejects.toThrow("remote_workspace_migration_source_changed");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
});
it("does not release a coordinator whose expired owner changed during upload", async () => {
const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: "journaled-owner",
@@ -4,7 +4,7 @@ import os from "node:os";
import path from "node:path";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
import {
executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
environmentLeases, executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db,
} from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
@@ -18,7 +18,7 @@ import { materializeTaskDirectorySnapshot, removeTaskDirectoryTree } from "./iso
import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, validateTaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js";
import { inspectArchivedGitMigrationSource, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js";
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
import { isLegacyWorkspaceTaskRelated, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js";
import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js";
import { offlineWorkspaceMigrationBlockers, prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js";
const SCHEMA = "paperclip.remote-workspace-migration.v1";
@@ -161,6 +161,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri
if (stopped && row.run.runnerProfileJson?.nativeWorkspaceSync == null && row.run.runnerProfileJson?.remoteWorkspace == null
&& row.resultId === null && ["observed", "retryable_failure"].includes(row.phase)) {
legacyRecoverySources.push({ runId: row.id, issueId: row.run.nativeIssueId!, reference: null,
recoveryAllocations: await readLegacyWorkspaceRecoveryAllocations(db, row.run),
unresolvedFinalization: { phase: row.phase, leaseOwner: row.leaseOwner,
leaseExpiresAt: row.leaseExpiresAt?.toISOString() ?? null } });
continue;
@@ -317,6 +318,9 @@ export async function migrateRemoteWorkspace(input: {
.where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, source.runId))).for("update");
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, source.runId))).for("update");
for (const allocation of source.recoveryAllocations ?? []) await tx.select({ id: environmentLeases.id })
.from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
eq(environmentLeases.id, allocation.leaseId))).for("update");
}
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
@@ -3,7 +3,7 @@ import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueR
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared";
import { z } from "zod";
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
import { hasMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js";
import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js";
import { readSnapshot } from "./workspace-recovery-snapshot.js";
export const workspaceRecoverySchema = z.object({
@@ -45,29 +45,36 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick<typeof he
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
if ((run.runnerProfileJson?.remoteWorkspace != null && !parsedRemote.success) || reference && parsedRemote.success) return null;
if (typeof issueId === "string" && reference?.state === "prepared") {
let migrated: boolean;
try { migrated = await hasMigratedLegacyWorkspaceSource(db, { companyId: run.companyId, runId: run.id, reference }); }
if (typeof issueId === "string" && (reference?.state === "prepared" || !reference && !parsedRemote.success)) {
let migrated;
try { migrated = await readMigratedLegacyWorkspaceSource(db, { companyId: run.companyId, runId: run.id, reference }); }
catch (error) { if (error instanceof Error && error.message === "legacy_workspace_migration_receipt_invalid") return null; throw error; }
if (migrated) {
// Migration retires the local merge destination. A verified immutable
// attachment is the repair receipt; the old copyback remains unresolved.
// Provider stop and current execution ownership are checked separately.
const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and(
eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId),
sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`,
)).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"),
eq(workspaceOperations.status, "succeeded"),
sql`${workspaceOperations.metadata}->>'leaseId' = ${reference.leaseId}`,
sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${reference.providerLeaseId}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`,
)).orderBy(desc(workspaceOperations.createdAt)).limit(1);
if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null;
try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; }
return saved.id;
const allocations = reference ? [{ leaseId: reference.leaseId, providerLeaseId: reference.providerLeaseId }]
: migrated.recoveryAllocations;
if (!allocations?.length) return null;
let receiptId: string | null = null;
for (const allocation of allocations) {
// Migration retires the local merge destination. A verified immutable
// attachment is the repair receipt; the old copyback remains unresolved.
// Provider stop and current execution ownership are checked separately.
const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and(
eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId),
sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`,
)).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"),
eq(workspaceOperations.status, "succeeded"),
sql`${workspaceOperations.metadata}->>'leaseId' = ${allocation.leaseId}`,
sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${allocation.providerLeaseId}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`,
)).orderBy(desc(workspaceOperations.createdAt)).limit(1);
if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null;
try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; }
receiptId = saved.id;
}
return receiptId;
}
}
if (typeof issueId !== "string" || (!remote && (!reference || reference.state !== "finalized" || !reference.finalHostSha256))) return null;
+46 -9
View File
@@ -184,7 +184,7 @@ describe("automatic retained workspace recovery", () => {
ownerType: "board", cause: mode === "native" ? "native_continuation_requires_reconciliation" : "legacy_execution_requires_reconciliation",
fingerprint: runId, nextAction: "Manual repair required", evidence: { runId, workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery } });
const runtime = { resumeRunLease: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, metadata: { remoteCwd: remote } })),
retryPendingSandboxTeardown: vi.fn(async () => ({ providerLeaseId, state: "stopped" })) };
retryPendingSandboxTeardown: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, state: "stopped" })) };
const environmentRuntime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
acquireRunLease: vi.fn(), releaseRunLease: vi.fn(), ...runtime,
resolveCapabilities: async () => classifyEnvironmentCapabilities({ verifiedMethods: [], declared: null, narrowing: null }),
@@ -195,10 +195,10 @@ describe("automatic retained workspace recovery", () => {
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
it.each(["retained", "removed", "child", "expired-owner"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => {
it.each(["retained", "removed", "child", "expired-owner", "no-descriptor"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => {
vi.stubEnv("DATABASE_URL", connectionString);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32));
const f = await seed();
const f = await seed("native", mode !== "no-descriptor");
// The fixture's command runner supports native copyback; the provider
// snapshot boundary separately needs its real-file syncOut capability.
targets.get(f.lease.id)!.runner!.syncIn = async operations => {
@@ -211,6 +211,25 @@ describe("automatic retained workspace recovery", () => {
await cp(file.sourcePath, file.targetPath, { recursive: true, dereference: false });
return { filesTransferred: 2, bytesTransferred: 0, operations: [] };
};
if (mode === "no-descriptor") {
const secondId = randomUUID(), secondProviderId = randomUUID(), secondRemote = path.join(root, secondId);
await cp(f.remote, secondRemote, { recursive: true });
const second: typeof f.lease = { ...f.lease, id: secondId, providerLeaseId: secondProviderId,
metadata: { ...f.lease.metadata, remoteCwd: secondRemote } };
second.metadata = { ...second.metadata,
remoteExecutionTermination: remoteTerminationReceipt(second, { providerLeaseId: secondProviderId, state: "stopped" }) };
await db.insert(environmentLeases).values(second);
targets.set(secondId, { ...targets.get(f.lease.id)!, leaseId: secondId, remoteCwd: secondRemote });
f.runtime.retryPendingSandboxTeardown.mockImplementation(async ({ lease }) => ({
providerLeaseId: lease.providerLeaseId!, state: "stopped",
}));
f.runtime.resumeRunLease.mockImplementation(async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!,
metadata: { remoteCwd: lease.metadata!.remoteCwd as string } }));
const source = await run(f.runId);
await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id, secondId] },
} }).where(eq(heartbeatRuns.id, f.runId));
}
const sourceIssueId = mode === "child" ? randomUUID() : f.issueId;
if (mode === "child") {
await db.insert(issues).values({ id: sourceIssueId, companyId: f.companyId, title: "Ancestor workspace owner" });
@@ -219,7 +238,7 @@ describe("automatic retained workspace recovery", () => {
await db.insert(executionWorkspaces).values({ id: f.issueId, companyId: f.companyId, sourceIssueId,
name: "Existing files", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.host });
await db.update(issues).set({ executionWorkspaceId: f.issueId }).where(eq(issues.id, f.issueId));
if (mode === "expired-owner") await db.update(nativeRunFinalizations).set({ phase: "observed",
if (mode === "expired-owner" || mode === "no-descriptor") await db.update(nativeRunFinalizations).set({ phase: "observed",
leaseOwner: "stopped-old-controller", leaseExpiresAt: new Date(Date.now() - 60_000) })
.where(eq(nativeRunFinalizations.runId, f.runId));
vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
@@ -231,17 +250,35 @@ describe("automatic retained workspace recovery", () => {
await recoverRetainedWorkspace(f.input);
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
expect(await readFile(path.join(f.host, "saved.txt")).catch(() => null)).toBeNull();
expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: { nativeWorkspaceSync: { state: "prepared" } } });
expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: mode === "no-descriptor" ? {} : { nativeWorkspaceSync: { state: "prepared" } } });
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ workspaceRepairRequired: false });
const restored = path.join(root, randomUUID()); await mkdir(restored);
const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId,
sourceRunIds: [f.runId], cwd: restored });
expect(copies).toHaveLength(1);
expect(copies).toHaveLength(mode === "no-descriptor" ? 2 : 1);
expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
if (mode === "child" || mode === "expired-owner") {
if (mode === "no-descriptor") {
const source = await run(f.runId);
await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] },
} }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
await db.update(heartbeatRuns).set({ resultJson: source.resultJson }).where(eq(heartbeatRuns.id, f.runId));
for (const copy of copies) {
await db.update(workspaceOperations).set({ status: "running" }).where(eq(workspaceOperations.id, copy.operationId));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, copy.operationId));
}
const [allocation] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.lease.id));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
await db.update(environmentLeases).set({ providerLeaseId: allocation.providerLeaseId }).where(eq(environmentLeases.id, f.lease.id));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
}
if (mode === "child" || mode === "expired-owner" || mode === "no-descriptor") {
// The parent must not receive the child's task-scoped recovery copy.
if (mode === "child") expect(await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: sourceIssueId,
sourceRunIds: [f.runId], cwd: restored })).toHaveLength(0);
@@ -252,7 +289,7 @@ describe("automatic retained workspace recovery", () => {
const continuationInput = { db, companyId: f.companyId, issueId: f.issueId,
agentId: sourceRun.agentId, actorType: "user", actorId: "operator", reason: "issue_commented",
commentId: comment.id, successorRunId: nextRunId };
if (mode === "expired-owner") {
if (mode === "expired-owner" || mode === "no-descriptor") {
for (const changed of [{ phase: "retryable_failure", leaseOwner: null }, { phase: "observed", leaseOwner: "new-controller" }]) {
await db.update(nativeRunFinalizations).set(changed).where(eq(nativeRunFinalizations.runId, f.runId));
expect(await admitExplicitNativeContinuation({ ...continuationInput, dryRun: true })).toBeNull();
@@ -281,7 +318,7 @@ describe("automatic retained workspace recovery", () => {
expect(await readFile(path.join(prepared.remoteRoot, ".paperclip-recovered", copies[0]!.operationId, "files", "saved.txt"), "utf8"))
.toBe("finished work\n");
}
if (mode === "retained") {
if (mode === "retained" || mode === "no-descriptor") {
const [receipt] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, copies[0]!.operationId));
for (const metadata of [{ ...receipt.metadata, leaseId: randomUUID() }, { ...receipt.metadata, providerLeaseId: randomUUID() },
{ ...receipt.metadata, recoveryActionId: randomUUID() }, { ...receipt.metadata, archive: { broken: true } }]) {