Preserve descriptor-free recovery identities through migration

This commit is contained in:
Dotta committed 2026-11-03 15:28:38 -06:00
1 parent 1c172fc057
commit 9ab6a38fa4
7 files changed
+149 -45

No files matched your search

@@ -76,6 +76,12 @@ use the retained sources and remote workspace. This is not successful recovery,
provider termination evidence, or permission to delete the original allocation.
After independently verified sandbox preservation, the task accepts the exact
migrated snapshot receipt for continuation without requiring a fresh workspace.
For older runs without a synchronization descriptor, the migration journals
all declared recovery lease/allocation/directory identities under row locks.
Continuation requires the unchanged complete set and a verified snapshot for
every source. Missing or ambiguous source identity remains held; a partial
archive cannot certify recovery. The recovery journey covers two allocations,
removal of the old filesystem, and the files appearing in the next remote turn.
Accepted results with unfinished copyback remain blockers. Once the original
sync reference is finalized and its successful `workspace_finalize` operation
receipt exists, assessment and status effects can continue from their
@@ -384,7 +384,7 @@ export async function admitExplicitNativeContinuation(input: {
// before admitting new user input; never apply or replay the old result.
const legacyReference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
const migratedSource = coordinator && !coordinator.resultId && !coordinator.leaseOwner
&& ["observed", "retryable_failure"].includes(coordinator.phase) && legacyReference
&& ["observed", "retryable_failure"].includes(coordinator.phase)
&& await hasVerifiedWorkspaceRepair(db, run)
? await readMigratedLegacyWorkspaceSource(db, { companyId, runId: run.id, reference: legacyReference }) : null;
// Offline migration fenced this exact unfinished turn and preserved its
@@ -1,11 +1,12 @@
import { isDeepStrictEqual } from "node:util";
import { and, eq, sql } from "drizzle-orm";
import { executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db";
import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import type { NativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
export type LegacyWorkspaceRecoverySource = {
runId: string; issueId: string; reference: NativeWorkspaceSyncReference | null;
recoveryAllocations?: { leaseId: string; providerLeaseId: string; remoteCwd: string }[] | null;
unresolvedFinalization?: { phase: string; leaseOwner: string | null; leaseExpiresAt: string | null };
};
export const LEGACY_WORKSPACE_RECOVERY_MIGRATION = "paperclip.legacy-workspace-recovery-migration.v1";
@@ -34,34 +35,64 @@ export async function isLegacyWorkspaceTaskRelated(db: Pick<Db, "select">, input
return false;
}
/** Record every declared source, or none: partial preservation cannot release
* the task's recovery hold. The recovery owner separately verifies provenance,
* provider termination and archive bytes before writing its receipts. */
export async function readLegacyWorkspaceRecoveryAllocations(db: Pick<Db, "select">, run: {
id: string; companyId: string; nativeIssueId: string | null; resultJson: Record<string, unknown> | null;
}): Promise<LegacyWorkspaceRecoverySource["recoveryAllocations"]> {
const recovery = run.resultJson?.workspaceRestoreRecovery as { schema?: unknown; leaseIds?: unknown } | undefined;
const ids = recovery?.leaseIds;
if (recovery?.schema !== "paperclip.workspace-restore-recovery.v1" || !Array.isArray(ids) || !ids.length
|| ids.some(id => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))
|| new Set(ids).size !== ids.length || !run.nativeIssueId) return null;
const allocations = [];
for (const id of [...ids].sort()) {
const [lease] = await db.select({ providerLeaseId: environmentLeases.providerLeaseId, metadata: environmentLeases.metadata })
.from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, id),
eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.issueId, run.nativeIssueId)));
if (!lease?.providerLeaseId || typeof lease.metadata?.remoteCwd !== "string" || !lease.metadata.remoteCwd.startsWith("/")) return null;
allocations.push({ leaseId: id as string, providerLeaseId: lease.providerLeaseId, remoteCwd: lease.metadata.remoteCwd });
}
return allocations;
}
/** This receipt changes where recovery may save files; it never certifies that
* a provider stopped or that its files were recovered. The recovery owner must
* still validate allocation provenance and termination before exporting. */
export async function readMigratedLegacyWorkspaceSource(db: Pick<Db, "select">, input: {
companyId: string; runId: string; reference: NativeWorkspaceSyncReference;
companyId: string; runId: string; reference: NativeWorkspaceSyncReference | null;
}): Promise<LegacyWorkspaceRecoverySource | null> {
if (input.reference.state !== "prepared") return null;
if (input.reference && input.reference.state !== "prepared") return null;
const [run] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
));
const runMarker = run?.runnerProfileJson?.legacyWorkspaceRecoveryMigration as { workspaceId?: unknown } | undefined;
const workspaceId = input.reference?.workspaceId ?? runMarker?.workspaceId;
if (typeof workspaceId !== "string") return null;
const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.reference.workspaceId),
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, workspaceId),
));
if (!readRemoteWorkspaceAuthority(workspace?.metadata)) return null;
const marker = workspace?.metadata?.legacyWorkspaceRecoveryMigration as { schema?: unknown; operationId?: unknown } | undefined;
if (marker?.schema !== LEGACY_WORKSPACE_RECOVERY_MIGRATION || typeof marker.operationId !== "string")
throw new Error("legacy_workspace_migration_receipt_invalid");
const [run] = await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
));
if (!isDeepStrictEqual(run?.profile?.legacyWorkspaceRecoveryMigration, {
schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId: input.reference.workspaceId,
if (!isDeepStrictEqual(runMarker, {
schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId,
})) throw new Error("legacy_workspace_migration_receipt_invalid");
const [operation] = await db.select({ metadata: workspaceOperations.metadata }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.reference.workspaceId),
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, workspaceId),
eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration"),
eq(workspaceOperations.status, "succeeded"),
));
const sources = operation?.metadata?.legacyRecoverySources;
const source = Array.isArray(sources) ? sources.find(source => source && typeof source === "object"
&& source.runId === input.runId && typeof source.issueId === "string" && isDeepStrictEqual(source.reference, input.reference)) : null;
if (source && source.issueId !== run?.nativeIssueId) throw new Error("legacy_workspace_migration_receipt_invalid");
if (source && !input.reference && (run?.runnerProfileJson?.nativeWorkspaceSync != null
|| run?.runnerProfileJson?.remoteWorkspace != null
|| !isDeepStrictEqual(source.recoveryAllocations ?? null, run ? await readLegacyWorkspaceRecoveryAllocations(db, run) : null)))
throw new Error("legacy_workspace_migration_receipt_invalid");
if (!source) throw new Error("legacy_workspace_migration_receipt_invalid");
return source as LegacyWorkspaceRecoverySource;
}
@@ -8,7 +8,7 @@ import { createServer, type Server } from "node:http";
import { pathToFileURL } from "node:url";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { activityLog, agents, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
import { activityLog, agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
@@ -267,6 +267,25 @@ describe.skipIf(!support.supported)("remote workspace migration", () => {
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true })).rejects.toThrow("remote_workspace_migration_source_changed");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
});
it("rejects a descriptor-free allocation changed during upload", async () => {
const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
await db.insert(environmentLeases).values({ id: f.reference.leaseId, companyId: f.companyId,
issueId: f.issueId, heartbeatRunId: f.runId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId,
status: "released", metadata: { remoteCwd: f.reference.remoteCwd } });
await db.update(heartbeatRuns).set({ runnerProfileJson: {}, resultJson: {
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.reference.leaseId] },
} }).where(eq(heartbeatRuns.id, f.runId));
await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId));
const put = storage.putFile.bind(storage);
storage.putFile = async input => {
const result = await put(input);
await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.reference.leaseId));
return result;
};
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }))
.rejects.toThrow("remote_workspace_migration_source_changed");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
});
it("does not release a coordinator whose expired owner changed during upload", async () => {
const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: "journaled-owner",
@@ -4,7 +4,7 @@ import os from "node:os";
import path from "node:path";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
import {
executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
environmentLeases, executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db,
} from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
@@ -18,7 +18,7 @@ import { materializeTaskDirectorySnapshot, removeTaskDirectoryTree } from "./iso
import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, validateTaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js";
import { inspectArchivedGitMigrationSource, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js";
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
import { isLegacyWorkspaceTaskRelated, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js";
import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js";
import { offlineWorkspaceMigrationBlockers, prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js";
const SCHEMA = "paperclip.remote-workspace-migration.v1";
@@ -161,6 +161,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri
if (stopped && row.run.runnerProfileJson?.nativeWorkspaceSync == null && row.run.runnerProfileJson?.remoteWorkspace == null
&& row.resultId === null && ["observed", "retryable_failure"].includes(row.phase)) {
legacyRecoverySources.push({ runId: row.id, issueId: row.run.nativeIssueId!, reference: null,
recoveryAllocations: await readLegacyWorkspaceRecoveryAllocations(db, row.run),
unresolvedFinalization: { phase: row.phase, leaseOwner: row.leaseOwner,
leaseExpiresAt: row.leaseExpiresAt?.toISOString() ?? null } });
continue;
@@ -317,6 +318,9 @@ export async function migrateRemoteWorkspace(input: {
.where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, source.runId))).for("update");
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, source.runId))).for("update");
for (const allocation of source.recoveryAllocations ?? []) await tx.select({ id: environmentLeases.id })
.from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
eq(environmentLeases.id, allocation.leaseId))).for("update");
}
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
@@ -3,7 +3,7 @@ import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueR
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared";
import { z } from "zod";
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
import { hasMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js";
import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js";
import { readSnapshot } from "./workspace-recovery-snapshot.js";
export const workspaceRecoverySchema = z.object({
@@ -45,29 +45,36 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick<typeof he
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
if ((run.runnerProfileJson?.remoteWorkspace != null && !parsedRemote.success) || reference && parsedRemote.success) return null;
if (typeof issueId === "string" && reference?.state === "prepared") {
let migrated: boolean;
try { migrated = await hasMigratedLegacyWorkspaceSource(db, { companyId: run.companyId, runId: run.id, reference }); }
if (typeof issueId === "string" && (reference?.state === "prepared" || !reference && !parsedRemote.success)) {
let migrated;
try { migrated = await readMigratedLegacyWorkspaceSource(db, { companyId: run.companyId, runId: run.id, reference }); }
catch (error) { if (error instanceof Error && error.message === "legacy_workspace_migration_receipt_invalid") return null; throw error; }
if (migrated) {
// Migration retires the local merge destination. A verified immutable
// attachment is the repair receipt; the old copyback remains unresolved.
// Provider stop and current execution ownership are checked separately.
const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and(
eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId),
sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`,
)).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"),
eq(workspaceOperations.status, "succeeded"),
sql`${workspaceOperations.metadata}->>'leaseId' = ${reference.leaseId}`,
sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${reference.providerLeaseId}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`,
)).orderBy(desc(workspaceOperations.createdAt)).limit(1);
if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null;
try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; }
return saved.id;
const allocations = reference ? [{ leaseId: reference.leaseId, providerLeaseId: reference.providerLeaseId }]
: migrated.recoveryAllocations;
if (!allocations?.length) return null;
let receiptId: string | null = null;
for (const allocation of allocations) {
// Migration retires the local merge destination. A verified immutable
// attachment is the repair receipt; the old copyback remains unresolved.
// Provider stop and current execution ownership are checked separately.
const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and(
eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId),
sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`,
)).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"),
eq(workspaceOperations.status, "succeeded"),
sql`${workspaceOperations.metadata}->>'leaseId' = ${allocation.leaseId}`,
sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${allocation.providerLeaseId}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`,
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`,
)).orderBy(desc(workspaceOperations.createdAt)).limit(1);
if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null;
try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; }
receiptId = saved.id;
}
return receiptId;
}
}
if (typeof issueId !== "string" || (!remote && (!reference || reference.state !== "finalized" || !reference.finalHostSha256))) return null;
+46 -9
View File
@@ -184,7 +184,7 @@ describe("automatic retained workspace recovery", () => {
ownerType: "board", cause: mode === "native" ? "native_continuation_requires_reconciliation" : "legacy_execution_requires_reconciliation",
fingerprint: runId, nextAction: "Manual repair required", evidence: { runId, workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery } });
const runtime = { resumeRunLease: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, metadata: { remoteCwd: remote } })),
retryPendingSandboxTeardown: vi.fn(async () => ({ providerLeaseId, state: "stopped" })) };
retryPendingSandboxTeardown: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, state: "stopped" })) };
const environmentRuntime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
acquireRunLease: vi.fn(), releaseRunLease: vi.fn(), ...runtime,
resolveCapabilities: async () => classifyEnvironmentCapabilities({ verifiedMethods: [], declared: null, narrowing: null }),
@@ -195,10 +195,10 @@ describe("automatic retained workspace recovery", () => {
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
it.each(["retained", "removed", "child", "expired-owner"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => {
it.each(["retained", "removed", "child", "expired-owner", "no-descriptor"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => {
vi.stubEnv("DATABASE_URL", connectionString);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32));
const f = await seed();
const f = await seed("native", mode !== "no-descriptor");
// The fixture's command runner supports native copyback; the provider
// snapshot boundary separately needs its real-file syncOut capability.
targets.get(f.lease.id)!.runner!.syncIn = async operations => {
@@ -211,6 +211,25 @@ describe("automatic retained workspace recovery", () => {
await cp(file.sourcePath, file.targetPath, { recursive: true, dereference: false });
return { filesTransferred: 2, bytesTransferred: 0, operations: [] };
};
if (mode === "no-descriptor") {
const secondId = randomUUID(), secondProviderId = randomUUID(), secondRemote = path.join(root, secondId);
await cp(f.remote, secondRemote, { recursive: true });
const second: typeof f.lease = { ...f.lease, id: secondId, providerLeaseId: secondProviderId,
metadata: { ...f.lease.metadata, remoteCwd: secondRemote } };
second.metadata = { ...second.metadata,
remoteExecutionTermination: remoteTerminationReceipt(second, { providerLeaseId: secondProviderId, state: "stopped" }) };
await db.insert(environmentLeases).values(second);
targets.set(secondId, { ...targets.get(f.lease.id)!, leaseId: secondId, remoteCwd: secondRemote });
f.runtime.retryPendingSandboxTeardown.mockImplementation(async ({ lease }) => ({
providerLeaseId: lease.providerLeaseId!, state: "stopped",
}));
f.runtime.resumeRunLease.mockImplementation(async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!,
metadata: { remoteCwd: lease.metadata!.remoteCwd as string } }));
const source = await run(f.runId);
await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id, secondId] },
} }).where(eq(heartbeatRuns.id, f.runId));
}
const sourceIssueId = mode === "child" ? randomUUID() : f.issueId;
if (mode === "child") {
await db.insert(issues).values({ id: sourceIssueId, companyId: f.companyId, title: "Ancestor workspace owner" });
@@ -219,7 +238,7 @@ describe("automatic retained workspace recovery", () => {
await db.insert(executionWorkspaces).values({ id: f.issueId, companyId: f.companyId, sourceIssueId,
name: "Existing files", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.host });
await db.update(issues).set({ executionWorkspaceId: f.issueId }).where(eq(issues.id, f.issueId));
if (mode === "expired-owner") await db.update(nativeRunFinalizations).set({ phase: "observed",
if (mode === "expired-owner" || mode === "no-descriptor") await db.update(nativeRunFinalizations).set({ phase: "observed",
leaseOwner: "stopped-old-controller", leaseExpiresAt: new Date(Date.now() - 60_000) })
.where(eq(nativeRunFinalizations.runId, f.runId));
vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
@@ -231,17 +250,35 @@ describe("automatic retained workspace recovery", () => {
await recoverRetainedWorkspace(f.input);
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
expect(await readFile(path.join(f.host, "saved.txt")).catch(() => null)).toBeNull();
expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: { nativeWorkspaceSync: { state: "prepared" } } });
expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: mode === "no-descriptor" ? {} : { nativeWorkspaceSync: { state: "prepared" } } });
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ workspaceRepairRequired: false });
const restored = path.join(root, randomUUID()); await mkdir(restored);
const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId,
sourceRunIds: [f.runId], cwd: restored });
expect(copies).toHaveLength(1);
expect(copies).toHaveLength(mode === "no-descriptor" ? 2 : 1);
expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n");
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
if (mode === "child" || mode === "expired-owner") {
if (mode === "no-descriptor") {
const source = await run(f.runId);
await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] },
} }).where(eq(heartbeatRuns.id, f.runId));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
await db.update(heartbeatRuns).set({ resultJson: source.resultJson }).where(eq(heartbeatRuns.id, f.runId));
for (const copy of copies) {
await db.update(workspaceOperations).set({ status: "running" }).where(eq(workspaceOperations.id, copy.operationId));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, copy.operationId));
}
const [allocation] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.lease.id));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
await db.update(environmentLeases).set({ providerLeaseId: allocation.providerLeaseId }).where(eq(environmentLeases.id, f.lease.id));
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
}
if (mode === "child" || mode === "expired-owner" || mode === "no-descriptor") {
// The parent must not receive the child's task-scoped recovery copy.
if (mode === "child") expect(await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: sourceIssueId,
sourceRunIds: [f.runId], cwd: restored })).toHaveLength(0);
@@ -252,7 +289,7 @@ describe("automatic retained workspace recovery", () => {
const continuationInput = { db, companyId: f.companyId, issueId: f.issueId,
agentId: sourceRun.agentId, actorType: "user", actorId: "operator", reason: "issue_commented",
commentId: comment.id, successorRunId: nextRunId };
if (mode === "expired-owner") {
if (mode === "expired-owner" || mode === "no-descriptor") {
for (const changed of [{ phase: "retryable_failure", leaseOwner: null }, { phase: "observed", leaseOwner: "new-controller" }]) {
await db.update(nativeRunFinalizations).set(changed).where(eq(nativeRunFinalizations.runId, f.runId));
expect(await admitExplicitNativeContinuation({ ...continuationInput, dryRun: true })).toBeNull();
@@ -281,7 +318,7 @@ describe("automatic retained workspace recovery", () => {
expect(await readFile(path.join(prepared.remoteRoot, ".paperclip-recovered", copies[0]!.operationId, "files", "saved.txt"), "utf8"))
.toBe("finished work\n");
}
if (mode === "retained") {
if (mode === "retained" || mode === "no-descriptor") {
const [receipt] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, copies[0]!.operationId));
for (const metadata of [{ ...receipt.metadata, leaseId: randomUUID() }, { ...receipt.metadata, providerLeaseId: randomUUID() },
{ ...receipt.metadata, recoveryActionId: randomUUID() }, { ...receipt.metadata, archive: { broken: true } }]) {