mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
Preserve descriptor-free recovery identities through migration
This commit is contained in:
1 parent
1c172fc057
commit
9ab6a38fa4
7 files changed
+149
-45
No files matched your search
@@ -76,6 +76,12 @@ use the retained sources and remote workspace. This is not successful recovery,
|
||||
provider termination evidence, or permission to delete the original allocation.
|
||||
After independently verified sandbox preservation, the task accepts the exact
|
||||
migrated snapshot receipt for continuation without requiring a fresh workspace.
|
||||
For older runs without a synchronization descriptor, the migration journals
|
||||
all declared recovery lease/allocation/directory identities under row locks.
|
||||
Continuation requires the unchanged complete set and a verified snapshot for
|
||||
every source. Missing or ambiguous source identity remains held; a partial
|
||||
archive cannot certify recovery. The recovery journey covers two allocations,
|
||||
removal of the old filesystem, and the files appearing in the next remote turn.
|
||||
Accepted results with unfinished copyback remain blockers. Once the original
|
||||
sync reference is finalized and its successful `workspace_finalize` operation
|
||||
receipt exists, assessment and status effects can continue from their
|
||||
|
||||
@@ -384,7 +384,7 @@ export async function admitExplicitNativeContinuation(input: {
|
||||
// before admitting new user input; never apply or replay the old result.
|
||||
const legacyReference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
|
||||
const migratedSource = coordinator && !coordinator.resultId && !coordinator.leaseOwner
|
||||
&& ["observed", "retryable_failure"].includes(coordinator.phase) && legacyReference
|
||||
&& ["observed", "retryable_failure"].includes(coordinator.phase)
|
||||
&& await hasVerifiedWorkspaceRepair(db, run)
|
||||
? await readMigratedLegacyWorkspaceSource(db, { companyId, runId: run.id, reference: legacyReference }) : null;
|
||||
// Offline migration fenced this exact unfinished turn and preserved its
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import type { NativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
|
||||
|
||||
export type LegacyWorkspaceRecoverySource = {
|
||||
runId: string; issueId: string; reference: NativeWorkspaceSyncReference | null;
|
||||
recoveryAllocations?: { leaseId: string; providerLeaseId: string; remoteCwd: string }[] | null;
|
||||
unresolvedFinalization?: { phase: string; leaseOwner: string | null; leaseExpiresAt: string | null };
|
||||
};
|
||||
export const LEGACY_WORKSPACE_RECOVERY_MIGRATION = "paperclip.legacy-workspace-recovery-migration.v1";
|
||||
@@ -34,34 +35,64 @@ export async function isLegacyWorkspaceTaskRelated(db: Pick<Db, "select">, input
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Record every declared source, or none: partial preservation cannot release
|
||||
* the task's recovery hold. The recovery owner separately verifies provenance,
|
||||
* provider termination and archive bytes before writing its receipts. */
|
||||
export async function readLegacyWorkspaceRecoveryAllocations(db: Pick<Db, "select">, run: {
|
||||
id: string; companyId: string; nativeIssueId: string | null; resultJson: Record<string, unknown> | null;
|
||||
}): Promise<LegacyWorkspaceRecoverySource["recoveryAllocations"]> {
|
||||
const recovery = run.resultJson?.workspaceRestoreRecovery as { schema?: unknown; leaseIds?: unknown } | undefined;
|
||||
const ids = recovery?.leaseIds;
|
||||
if (recovery?.schema !== "paperclip.workspace-restore-recovery.v1" || !Array.isArray(ids) || !ids.length
|
||||
|| ids.some(id => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id))
|
||||
|| new Set(ids).size !== ids.length || !run.nativeIssueId) return null;
|
||||
const allocations = [];
|
||||
for (const id of [...ids].sort()) {
|
||||
const [lease] = await db.select({ providerLeaseId: environmentLeases.providerLeaseId, metadata: environmentLeases.metadata })
|
||||
.from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, id),
|
||||
eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.issueId, run.nativeIssueId)));
|
||||
if (!lease?.providerLeaseId || typeof lease.metadata?.remoteCwd !== "string" || !lease.metadata.remoteCwd.startsWith("/")) return null;
|
||||
allocations.push({ leaseId: id as string, providerLeaseId: lease.providerLeaseId, remoteCwd: lease.metadata.remoteCwd });
|
||||
}
|
||||
return allocations;
|
||||
}
|
||||
|
||||
/** This receipt changes where recovery may save files; it never certifies that
|
||||
* a provider stopped or that its files were recovered. The recovery owner must
|
||||
* still validate allocation provenance and termination before exporting. */
|
||||
export async function readMigratedLegacyWorkspaceSource(db: Pick<Db, "select">, input: {
|
||||
companyId: string; runId: string; reference: NativeWorkspaceSyncReference;
|
||||
companyId: string; runId: string; reference: NativeWorkspaceSyncReference | null;
|
||||
}): Promise<LegacyWorkspaceRecoverySource | null> {
|
||||
if (input.reference.state !== "prepared") return null;
|
||||
if (input.reference && input.reference.state !== "prepared") return null;
|
||||
const [run] = await db.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
|
||||
));
|
||||
const runMarker = run?.runnerProfileJson?.legacyWorkspaceRecoveryMigration as { workspaceId?: unknown } | undefined;
|
||||
const workspaceId = input.reference?.workspaceId ?? runMarker?.workspaceId;
|
||||
if (typeof workspaceId !== "string") return null;
|
||||
const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.reference.workspaceId),
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, workspaceId),
|
||||
));
|
||||
if (!readRemoteWorkspaceAuthority(workspace?.metadata)) return null;
|
||||
const marker = workspace?.metadata?.legacyWorkspaceRecoveryMigration as { schema?: unknown; operationId?: unknown } | undefined;
|
||||
if (marker?.schema !== LEGACY_WORKSPACE_RECOVERY_MIGRATION || typeof marker.operationId !== "string")
|
||||
throw new Error("legacy_workspace_migration_receipt_invalid");
|
||||
const [run] = await db.select({ profile: heartbeatRuns.runnerProfileJson }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
|
||||
));
|
||||
if (!isDeepStrictEqual(run?.profile?.legacyWorkspaceRecoveryMigration, {
|
||||
schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId: input.reference.workspaceId,
|
||||
if (!isDeepStrictEqual(runMarker, {
|
||||
schema: LEGACY_WORKSPACE_RECOVERY_MIGRATION, operationId: marker.operationId, workspaceId,
|
||||
})) throw new Error("legacy_workspace_migration_receipt_invalid");
|
||||
const [operation] = await db.select({ metadata: workspaceOperations.metadata }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.reference.workspaceId),
|
||||
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, workspaceId),
|
||||
eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration"),
|
||||
eq(workspaceOperations.status, "succeeded"),
|
||||
));
|
||||
const sources = operation?.metadata?.legacyRecoverySources;
|
||||
const source = Array.isArray(sources) ? sources.find(source => source && typeof source === "object"
|
||||
&& source.runId === input.runId && typeof source.issueId === "string" && isDeepStrictEqual(source.reference, input.reference)) : null;
|
||||
if (source && source.issueId !== run?.nativeIssueId) throw new Error("legacy_workspace_migration_receipt_invalid");
|
||||
if (source && !input.reference && (run?.runnerProfileJson?.nativeWorkspaceSync != null
|
||||
|| run?.runnerProfileJson?.remoteWorkspace != null
|
||||
|| !isDeepStrictEqual(source.recoveryAllocations ?? null, run ? await readLegacyWorkspaceRecoveryAllocations(db, run) : null)))
|
||||
throw new Error("legacy_workspace_migration_receipt_invalid");
|
||||
if (!source) throw new Error("legacy_workspace_migration_receipt_invalid");
|
||||
return source as LegacyWorkspaceRecoverySource;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import { createServer, type Server } from "node:http";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { activityLog, agents, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
|
||||
import { activityLog, agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, completionContracts, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
|
||||
executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
@@ -267,6 +267,25 @@ describe.skipIf(!support.supported)("remote workspace migration", () => {
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true })).rejects.toThrow("remote_workspace_migration_source_changed");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
});
|
||||
it("rejects a descriptor-free allocation changed during upload", async () => {
|
||||
const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
|
||||
await db.insert(environmentLeases).values({ id: f.reference.leaseId, companyId: f.companyId,
|
||||
issueId: f.issueId, heartbeatRunId: f.runId, provider: "daytona", providerLeaseId: f.reference.providerLeaseId,
|
||||
status: "released", metadata: { remoteCwd: f.reference.remoteCwd } });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: {}, resultJson: {
|
||||
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.reference.leaseId] },
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.update(nativeRunFinalizations).set({ phase: "observed" }).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
const put = storage.putFile.bind(storage);
|
||||
storage.putFile = async input => {
|
||||
const result = await put(input);
|
||||
await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.reference.leaseId));
|
||||
return result;
|
||||
};
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true, writersStopped: true }))
|
||||
.rejects.toThrow("remote_workspace_migration_source_changed");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
});
|
||||
it("does not release a coordinator whose expired owner changed during upload", async () => {
|
||||
const f = await failedCopybackFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
|
||||
await db.update(nativeRunFinalizations).set({ phase: "observed", leaseOwner: "journaled-owner",
|
||||
|
||||
@@ -4,7 +4,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
import {
|
||||
executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
|
||||
environmentLeases, executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
|
||||
heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
|
||||
@@ -18,7 +18,7 @@ import { materializeTaskDirectorySnapshot, removeTaskDirectoryTree } from "./iso
|
||||
import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, validateTaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js";
|
||||
import { inspectArchivedGitMigrationSource, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
|
||||
import { isLegacyWorkspaceTaskRelated, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js";
|
||||
import { isLegacyWorkspaceTaskRelated, readLegacyWorkspaceRecoveryAllocations, LEGACY_WORKSPACE_RECOVERY_MIGRATION, type LegacyWorkspaceRecoverySource } from "./legacy-workspace-recovery-migration.js";
|
||||
import { offlineWorkspaceMigrationBlockers, prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js";
|
||||
|
||||
const SCHEMA = "paperclip.remote-workspace-migration.v1";
|
||||
@@ -161,6 +161,7 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri
|
||||
if (stopped && row.run.runnerProfileJson?.nativeWorkspaceSync == null && row.run.runnerProfileJson?.remoteWorkspace == null
|
||||
&& row.resultId === null && ["observed", "retryable_failure"].includes(row.phase)) {
|
||||
legacyRecoverySources.push({ runId: row.id, issueId: row.run.nativeIssueId!, reference: null,
|
||||
recoveryAllocations: await readLegacyWorkspaceRecoveryAllocations(db, row.run),
|
||||
unresolvedFinalization: { phase: row.phase, leaseOwner: row.leaseOwner,
|
||||
leaseExpiresAt: row.leaseExpiresAt?.toISOString() ?? null } });
|
||||
continue;
|
||||
@@ -317,6 +318,9 @@ export async function migrateRemoteWorkspace(input: {
|
||||
.where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, source.runId))).for("update");
|
||||
await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, source.runId))).for("update");
|
||||
for (const allocation of source.recoveryAllocations ?? []) await tx.select({ id: environmentLeases.id })
|
||||
.from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
|
||||
eq(environmentLeases.id, allocation.leaseId))).for("update");
|
||||
}
|
||||
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
|
||||
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
|
||||
|
||||
@@ -3,7 +3,7 @@ import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueR
|
||||
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared";
|
||||
import { z } from "zod";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
|
||||
import { hasMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js";
|
||||
import { readMigratedLegacyWorkspaceSource } from "./legacy-workspace-recovery-migration.js";
|
||||
import { readSnapshot } from "./workspace-recovery-snapshot.js";
|
||||
|
||||
export const workspaceRecoverySchema = z.object({
|
||||
@@ -45,29 +45,36 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick<typeof he
|
||||
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
|
||||
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
|
||||
if ((run.runnerProfileJson?.remoteWorkspace != null && !parsedRemote.success) || reference && parsedRemote.success) return null;
|
||||
if (typeof issueId === "string" && reference?.state === "prepared") {
|
||||
let migrated: boolean;
|
||||
try { migrated = await hasMigratedLegacyWorkspaceSource(db, { companyId: run.companyId, runId: run.id, reference }); }
|
||||
if (typeof issueId === "string" && (reference?.state === "prepared" || !reference && !parsedRemote.success)) {
|
||||
let migrated;
|
||||
try { migrated = await readMigratedLegacyWorkspaceSource(db, { companyId: run.companyId, runId: run.id, reference }); }
|
||||
catch (error) { if (error instanceof Error && error.message === "legacy_workspace_migration_receipt_invalid") return null; throw error; }
|
||||
if (migrated) {
|
||||
// Migration retires the local merge destination. A verified immutable
|
||||
// attachment is the repair receipt; the old copyback remains unresolved.
|
||||
// Provider stop and current execution ownership are checked separately.
|
||||
const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
|
||||
evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and(
|
||||
eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId),
|
||||
sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`,
|
||||
)).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
|
||||
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"),
|
||||
eq(workspaceOperations.status, "succeeded"),
|
||||
sql`${workspaceOperations.metadata}->>'leaseId' = ${reference.leaseId}`,
|
||||
sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${reference.providerLeaseId}`,
|
||||
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`,
|
||||
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`,
|
||||
)).orderBy(desc(workspaceOperations.createdAt)).limit(1);
|
||||
if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null;
|
||||
try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; }
|
||||
return saved.id;
|
||||
const allocations = reference ? [{ leaseId: reference.leaseId, providerLeaseId: reference.providerLeaseId }]
|
||||
: migrated.recoveryAllocations;
|
||||
if (!allocations?.length) return null;
|
||||
let receiptId: string | null = null;
|
||||
for (const allocation of allocations) {
|
||||
// Migration retires the local merge destination. A verified immutable
|
||||
// attachment is the repair receipt; the old copyback remains unresolved.
|
||||
// Provider stop and current execution ownership are checked separately.
|
||||
const [saved] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
|
||||
evidence: issueRecoveryActions.evidence }).from(workspaceOperations).innerJoin(issueRecoveryActions, and(
|
||||
eq(issueRecoveryActions.companyId, workspaceOperations.companyId), eq(issueRecoveryActions.sourceIssueId, workspaceOperations.issueId),
|
||||
sql`${issueRecoveryActions.id}::text = ${workspaceOperations.metadata}->>'recoveryActionId'`,
|
||||
)).where(and(eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
|
||||
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery_snapshot"),
|
||||
eq(workspaceOperations.status, "succeeded"),
|
||||
sql`${workspaceOperations.metadata}->>'leaseId' = ${allocation.leaseId}`,
|
||||
sql`${workspaceOperations.metadata}->>'providerLeaseId' = ${allocation.providerLeaseId}`,
|
||||
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,runId}' = ${run.id}`,
|
||||
sql`${issueRecoveryActions.evidence}#>>'{workspaceRecovery,status}' = 'recovered'`,
|
||||
)).orderBy(desc(workspaceOperations.createdAt)).limit(1);
|
||||
if (!saved || readWorkspaceRecovery(saved.evidence)?.status !== "recovered") return null;
|
||||
try { readSnapshot(saved.metadata, run.companyId, run.id); } catch { return null; }
|
||||
receiptId = saved.id;
|
||||
}
|
||||
return receiptId;
|
||||
}
|
||||
}
|
||||
if (typeof issueId !== "string" || (!remote && (!reference || reference.state !== "finalized" || !reference.finalHostSha256))) return null;
|
||||
|
||||
@@ -184,7 +184,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
ownerType: "board", cause: mode === "native" ? "native_continuation_requires_reconciliation" : "legacy_execution_requires_reconciliation",
|
||||
fingerprint: runId, nextAction: "Manual repair required", evidence: { runId, workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: recovery } });
|
||||
const runtime = { resumeRunLease: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, metadata: { remoteCwd: remote } })),
|
||||
retryPendingSandboxTeardown: vi.fn(async () => ({ providerLeaseId, state: "stopped" })) };
|
||||
retryPendingSandboxTeardown: vi.fn(async (_input: { lease: EnvironmentLease }) => ({ providerLeaseId, state: "stopped" })) };
|
||||
const environmentRuntime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
|
||||
acquireRunLease: vi.fn(), releaseRunLease: vi.fn(), ...runtime,
|
||||
resolveCapabilities: async () => classifyEnvironmentCapabilities({ verifiedMethods: [], declared: null, narrowing: null }),
|
||||
@@ -195,10 +195,10 @@ describe("automatic retained workspace recovery", () => {
|
||||
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
|
||||
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
|
||||
|
||||
it.each(["retained", "removed", "child", "expired-owner"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => {
|
||||
it.each(["retained", "removed", "child", "expired-owner", "no-descriptor"])("preserves an old sandbox after storage migration without writing its old host (%s)", async mode => {
|
||||
vi.stubEnv("DATABASE_URL", connectionString);
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "63".repeat(32));
|
||||
const f = await seed();
|
||||
const f = await seed("native", mode !== "no-descriptor");
|
||||
// The fixture's command runner supports native copyback; the provider
|
||||
// snapshot boundary separately needs its real-file syncOut capability.
|
||||
targets.get(f.lease.id)!.runner!.syncIn = async operations => {
|
||||
@@ -211,6 +211,25 @@ describe("automatic retained workspace recovery", () => {
|
||||
await cp(file.sourcePath, file.targetPath, { recursive: true, dereference: false });
|
||||
return { filesTransferred: 2, bytesTransferred: 0, operations: [] };
|
||||
};
|
||||
if (mode === "no-descriptor") {
|
||||
const secondId = randomUUID(), secondProviderId = randomUUID(), secondRemote = path.join(root, secondId);
|
||||
await cp(f.remote, secondRemote, { recursive: true });
|
||||
const second: typeof f.lease = { ...f.lease, id: secondId, providerLeaseId: secondProviderId,
|
||||
metadata: { ...f.lease.metadata, remoteCwd: secondRemote } };
|
||||
second.metadata = { ...second.metadata,
|
||||
remoteExecutionTermination: remoteTerminationReceipt(second, { providerLeaseId: secondProviderId, state: "stopped" }) };
|
||||
await db.insert(environmentLeases).values(second);
|
||||
targets.set(secondId, { ...targets.get(f.lease.id)!, leaseId: secondId, remoteCwd: secondRemote });
|
||||
f.runtime.retryPendingSandboxTeardown.mockImplementation(async ({ lease }) => ({
|
||||
providerLeaseId: lease.providerLeaseId!, state: "stopped",
|
||||
}));
|
||||
f.runtime.resumeRunLease.mockImplementation(async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!,
|
||||
metadata: { remoteCwd: lease.metadata!.remoteCwd as string } }));
|
||||
const source = await run(f.runId);
|
||||
await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
|
||||
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id, secondId] },
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
}
|
||||
const sourceIssueId = mode === "child" ? randomUUID() : f.issueId;
|
||||
if (mode === "child") {
|
||||
await db.insert(issues).values({ id: sourceIssueId, companyId: f.companyId, title: "Ancestor workspace owner" });
|
||||
@@ -219,7 +238,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
await db.insert(executionWorkspaces).values({ id: f.issueId, companyId: f.companyId, sourceIssueId,
|
||||
name: "Existing files", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.host });
|
||||
await db.update(issues).set({ executionWorkspaceId: f.issueId }).where(eq(issues.id, f.issueId));
|
||||
if (mode === "expired-owner") await db.update(nativeRunFinalizations).set({ phase: "observed",
|
||||
if (mode === "expired-owner" || mode === "no-descriptor") await db.update(nativeRunFinalizations).set({ phase: "observed",
|
||||
leaseOwner: "stopped-old-controller", leaseExpiresAt: new Date(Date.now() - 60_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
|
||||
@@ -231,17 +250,35 @@ describe("automatic retained workspace recovery", () => {
|
||||
await recoverRetainedWorkspace(f.input);
|
||||
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete");
|
||||
expect(await readFile(path.join(f.host, "saved.txt")).catch(() => null)).toBeNull();
|
||||
expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: { nativeWorkspaceSync: { state: "prepared" } } });
|
||||
expect(await run(f.runId)).toMatchObject({ status: "failed", runnerProfileJson: mode === "no-descriptor" ? {} : { nativeWorkspaceSync: { state: "prepared" } } });
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ workspaceRepairRequired: false });
|
||||
const restored = path.join(root, randomUUID()); await mkdir(restored);
|
||||
const copies = await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: f.issueId,
|
||||
sourceRunIds: [f.runId], cwd: restored });
|
||||
expect(copies).toHaveLength(1);
|
||||
expect(copies).toHaveLength(mode === "no-descriptor" ? 2 : 1);
|
||||
expect(await readFile(path.join(copies[0]!.path, "files", "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
|
||||
if (mode === "child" || mode === "expired-owner") {
|
||||
if (mode === "no-descriptor") {
|
||||
const source = await run(f.runId);
|
||||
await db.update(heartbeatRuns).set({ resultJson: { ...source.resultJson,
|
||||
workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] },
|
||||
} }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
|
||||
await db.update(heartbeatRuns).set({ resultJson: source.resultJson }).where(eq(heartbeatRuns.id, f.runId));
|
||||
for (const copy of copies) {
|
||||
await db.update(workspaceOperations).set({ status: "running" }).where(eq(workspaceOperations.id, copy.operationId));
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
|
||||
await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, copy.operationId));
|
||||
}
|
||||
const [allocation] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
||||
await db.update(environmentLeases).set({ providerLeaseId: randomUUID() }).where(eq(environmentLeases.id, f.lease.id));
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
|
||||
await db.update(environmentLeases).set({ providerLeaseId: allocation.providerLeaseId }).where(eq(environmentLeases.id, f.lease.id));
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
||||
}
|
||||
if (mode === "child" || mode === "expired-owner" || mode === "no-descriptor") {
|
||||
// The parent must not receive the child's task-scoped recovery copy.
|
||||
if (mode === "child") expect(await attachPreservedWorkspaceSnapshots({ db, companyId: f.companyId, issueId: sourceIssueId,
|
||||
sourceRunIds: [f.runId], cwd: restored })).toHaveLength(0);
|
||||
@@ -252,7 +289,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
const continuationInput = { db, companyId: f.companyId, issueId: f.issueId,
|
||||
agentId: sourceRun.agentId, actorType: "user", actorId: "operator", reason: "issue_commented",
|
||||
commentId: comment.id, successorRunId: nextRunId };
|
||||
if (mode === "expired-owner") {
|
||||
if (mode === "expired-owner" || mode === "no-descriptor") {
|
||||
for (const changed of [{ phase: "retryable_failure", leaseOwner: null }, { phase: "observed", leaseOwner: "new-controller" }]) {
|
||||
await db.update(nativeRunFinalizations).set(changed).where(eq(nativeRunFinalizations.runId, f.runId));
|
||||
expect(await admitExplicitNativeContinuation({ ...continuationInput, dryRun: true })).toBeNull();
|
||||
@@ -281,7 +318,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
expect(await readFile(path.join(prepared.remoteRoot, ".paperclip-recovered", copies[0]!.operationId, "files", "saved.txt"), "utf8"))
|
||||
.toBe("finished work\n");
|
||||
}
|
||||
if (mode === "retained") {
|
||||
if (mode === "retained" || mode === "no-descriptor") {
|
||||
const [receipt] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, copies[0]!.operationId));
|
||||
for (const metadata of [{ ...receipt.metadata, leaseId: randomUUID() }, { ...receipt.metadata, providerLeaseId: randomUUID() },
|
||||
{ ...receipt.metadata, recoveryActionId: randomUUID() }, { ...receipt.metadata, archive: { broken: true } }]) {
|
||||
|
||||
Reference in new issue
Block a user