mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
Preserve published workspace paths across cold restores
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
17c9f7e091
commit
b5b0dbb03f
3 files changed
+46
-7
No files matched your search
@@ -387,7 +387,8 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem
|
|||||||
}
|
}
|
||||||
}, 60_000);
|
}, 60_000);
|
||||||
|
|
||||||
it("prepares, saves and restores task files without a controller checkout or original allocation", async () => {
|
it.each(["same root", "different root", "destroyed allocation", "occupied root"] as const)(
|
||||||
|
"preserves the published workspace path and files on a cold restore (%s)", async mode => {
|
||||||
const reserved = await reserve({ kind: "empty" }), first = await target("sandbox-A");
|
const reserved = await reserve({ kind: "empty" }), first = await target("sandbox-A");
|
||||||
const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first });
|
const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first });
|
||||||
expect(readRemoteWorkspaceAuthority(prepared.workspace.metadata)?.snapshotId).toBeTruthy();
|
expect(readRemoteWorkspaceAuthority(prepared.workspace.metadata)?.snapshotId).toBeTruthy();
|
||||||
@@ -399,8 +400,16 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem
|
|||||||
lease: prepared.lease, target: first, assertOwned: async () => undefined });
|
lease: prepared.lease, target: first, assertOwned: async () => undefined });
|
||||||
const current = await workspace(reserved.id);
|
const current = await workspace(reserved.id);
|
||||||
expect(readRemoteWorkspaceAuthority(current.metadata)).toMatchObject({ dirty: false, writerRunId: null, snapshotId: saved.snapshotId });
|
expect(readRemoteWorkspaceAuthority(current.metadata)).toMatchObject({ dirty: false, writerRunId: null, snapshotId: saved.snapshotId });
|
||||||
await fs.rm(path.dirname(prepared.remoteRoot), { recursive: true, force: true });
|
if (mode === "destroyed allocation") {
|
||||||
const second = await target("sandbox-B");
|
const destroyed: EnvironmentLease = { ...prepared.lease, status: "released", releasedAt: new Date(), cleanupStatus: "success",
|
||||||
|
metadata: { ...prepared.lease.metadata, remoteExecutionTermination: remoteTerminationReceipt(prepared.lease,
|
||||||
|
{ providerLeaseId: prepared.lease.providerLeaseId, state: "destroyed" }) } };
|
||||||
|
await db.transaction(tx => clearRemoteWorkspaceLocationForReleasedLease({ db: tx, lease: destroyed }));
|
||||||
|
expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata)?.location).toBeNull();
|
||||||
|
}
|
||||||
|
await fs.rm(path.join(root, "sandbox-A"), { recursive: true, force: true });
|
||||||
|
const second = await target(mode === "different root" ? "sandbox-B" : "sandbox-A");
|
||||||
|
const expectedRemoteRoot = path.join(second.remoteCwd, ".paperclip-workspaces", path.basename(prepared.remoteRoot));
|
||||||
const nextRun = randomUUID(), nextLease = randomUUID();
|
const nextRun = randomUUID(), nextLease = randomUUID();
|
||||||
await db.insert(heartbeatRuns).values({ id: nextRun, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, runtimeMode: "native", status: "running" });
|
await db.insert(heartbeatRuns).values({ id: nextRun, companyId: ids.company, agentId: ids.agent, nativeIssueId: ids.issue, runtimeMode: "native", status: "running" });
|
||||||
await db.insert(environmentLeases).values({ id: nextLease, companyId: ids.company, environmentId: ids.environment,
|
await db.insert(environmentLeases).values({ id: nextLease, companyId: ids.company, environmentId: ids.environment,
|
||||||
@@ -408,7 +417,19 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem
|
|||||||
const reservedAgain = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId: nextRun,
|
const reservedAgain = await reserveRemoteWorkspace({ db, storage, companyId: ids.company, issueId: ids.issue, runId: nextRun,
|
||||||
agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: reserved.id, source: { kind: "empty" } });
|
agentId: ids.agent, environmentId: ids.environment, projectId: null, projectWorkspaceId: null, existingWorkspaceId: reserved.id, source: { kind: "empty" } });
|
||||||
const newLease = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, nextLease)))[0]! as EnvironmentLease;
|
const newLease = (await db.select().from(environmentLeases).where(eq(environmentLeases.id, nextLease)))[0]! as EnvironmentLease;
|
||||||
|
if (mode === "occupied root") {
|
||||||
|
await fs.mkdir(expectedRemoteRoot, { recursive: true });
|
||||||
|
await fs.writeFile(path.join(expectedRemoteRoot, "unpublished.txt"), "preserve unknown preparation\n");
|
||||||
|
await expect(prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second }))
|
||||||
|
.rejects.toThrow("remote_workspace_operation_failed");
|
||||||
|
expect(await fs.readFile(path.join(expectedRemoteRoot, "unpublished.txt"), "utf8")).toBe("preserve unknown preparation\n");
|
||||||
|
expect(readRemoteWorkspaceAuthority((await workspace(reserved.id)).metadata))
|
||||||
|
.toEqual(readRemoteWorkspaceAuthority(reservedAgain.metadata));
|
||||||
|
await fs.rename(expectedRemoteRoot, path.join(root, "retained-unknown-preparation"));
|
||||||
|
}
|
||||||
const restored = await prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second });
|
const restored = await prepareRemoteWorkspace({ db, storage, workspace: reservedAgain, runId: nextRun, lease: newLease, target: second });
|
||||||
|
expect(restored.remoteRoot).toBe(expectedRemoteRoot);
|
||||||
|
if (mode !== "different root") expect(restored.remoteRoot).toBe(prepared.remoteRoot);
|
||||||
expect(await fs.readFile(path.join(restored.remoteRoot, "report.md"), "utf8")).toBe("useful saved result\n");
|
expect(await fs.readFile(path.join(restored.remoteRoot, "report.md"), "utf8")).toBe("useful saved result\n");
|
||||||
expect(await fs.readFile(path.join(restored.remoteRoot, "binary.bin"))).toEqual(Buffer.from([0, 1, 255]));
|
expect(await fs.readFile(path.join(restored.remoteRoot, "binary.bin"))).toEqual(Buffer.from([0, 1, 255]));
|
||||||
expect((await fs.stat(path.join(restored.remoteRoot, "empty-folder"))).isDirectory()).toBe(true);
|
expect((await fs.stat(path.join(restored.remoteRoot, "empty-folder"))).isDirectory()).toBe(true);
|
||||||
@@ -619,8 +640,7 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem
|
|||||||
existingWorkspaceId: reserved.id, source: { kind: "empty" } });
|
existingWorkspaceId: reserved.id, source: { kind: "empty" } });
|
||||||
const nextTarget = await target("restore-cleanup-destination");
|
const nextTarget = await target("restore-cleanup-destination");
|
||||||
const sandbox = nextTarget as Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
|
const sandbox = nextTarget as Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
|
||||||
const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces",
|
const expectedRemoteRoot = path.join(sandbox.remoteCwd, ".paperclip-workspaces", path.basename(prepared.remoteRoot));
|
||||||
`${reserved.id}-${readRemoteWorkspaceAuthority(nextWorkspace.metadata)!.epoch}-${nextLease}`);
|
|
||||||
const execute = sandbox.runner!.execute.bind(sandbox.runner!);
|
const execute = sandbox.runner!.execute.bind(sandbox.runner!);
|
||||||
let failCleanup = true, cleanupAttempts = 0;
|
let failCleanup = true, cleanupAttempts = 0;
|
||||||
sandbox.runner!.execute = async input => {
|
sandbox.runner!.execute = async input => {
|
||||||
|
|||||||
@@ -321,8 +321,22 @@ function isSavedWorkspaceRepository(workspace: Workspace, authority: RemoteWorks
|
|||||||
&& (saved.readOnly ?? false) === (repository.readOnly ?? false));
|
&& (saved.readOnly ?? false) === (repository.readOnly ?? false));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function publishedWorkspaceDirectory(workspace: Workspace & Pick<typeof executionWorkspaces.$inferSelect, "providerRef">,
|
||||||
|
authority: RemoteWorkspaceAuthority) {
|
||||||
|
const publishedRoot = authority.snapshotId && (authority.location?.remoteCwd ?? workspace.providerRef);
|
||||||
|
if (!publishedRoot) return null;
|
||||||
|
const directory = path.posix.basename(publishedRoot);
|
||||||
|
const suffix = directory.slice(workspace.id.length);
|
||||||
|
if (!path.posix.isAbsolute(publishedRoot) || path.posix.basename(path.posix.dirname(publishedRoot)) !== ".paperclip-workspaces"
|
||||||
|
|| !directory.startsWith(workspace.id) || (suffix !== "" && !/^-[1-9][0-9]*-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/.test(suffix))) {
|
||||||
|
throw new Error("remote_workspace_published_path_invalid");
|
||||||
|
}
|
||||||
|
return directory;
|
||||||
|
}
|
||||||
|
|
||||||
export async function prepareRemoteWorkspace(input: {
|
export async function prepareRemoteWorkspace(input: {
|
||||||
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
|
db: Db; workspace: Workspace & Pick<typeof executionWorkspaces.$inferSelect, "providerRef">;
|
||||||
|
runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
|
||||||
source?: RemoteWorkspaceSource; assertOwned?: () => Promise<void>;
|
source?: RemoteWorkspaceSource; assertOwned?: () => Promise<void>;
|
||||||
provisionCommand?: string | null; runtimeProvisionCommand?: string | null; storage?: StorageService;
|
provisionCommand?: string | null; runtimeProvisionCommand?: string | null; storage?: StorageService;
|
||||||
}) {
|
}) {
|
||||||
@@ -339,7 +353,7 @@ export async function prepareRemoteWorkspace(input: {
|
|||||||
&& initial.location?.environmentId === input.lease.environmentId;
|
&& initial.location?.environmentId === input.lease.environmentId;
|
||||||
if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation.");
|
if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation.");
|
||||||
const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces",
|
const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces",
|
||||||
`${input.workspace.id}-${initial.epoch}-${input.lease.id}`);
|
publishedWorkspaceDirectory(input.workspace, initial) ?? `${input.workspace.id}-${initial.epoch}-${input.lease.id}`);
|
||||||
const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id,
|
const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id,
|
||||||
input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex");
|
input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex");
|
||||||
let restoredExport: Awaited<ReturnType<typeof restoreRemoteWorkspaceDirectory>> | null = null;
|
let restoredExport: Awaited<ReturnType<typeof restoreRemoteWorkspaceDirectory>> | null = null;
|
||||||
|
|||||||
@@ -116,6 +116,11 @@ The observer rechecks the original live owner immediately before switching
|
|||||||
traffic and fails if that turn already settled. A completed turn cannot count
|
traffic and fails if that turn already settled. A completed turn cannot count
|
||||||
as active takeover. Per-turn file inspection requires the exact committed saved
|
as active takeover. Per-turn file inspection requires the exact committed saved
|
||||||
snapshot; the warm lane still requires the live workspace.
|
snapshot; the warm lane still requires the live workspace.
|
||||||
|
Cold restores retain the published workspace directory name, including after
|
||||||
|
the old allocation is destroyed, because provider recovery binds the working
|
||||||
|
directory into session identity. Unpublished preparations retain distinct
|
||||||
|
epoch/lease directories, and a restore refuses to overwrite an occupied path
|
||||||
|
owned by another preparation.
|
||||||
|
|
||||||
The ordinary Everyday matrix retains its 125-second active outage default. The
|
The ordinary Everyday matrix retains its 125-second active outage default. The
|
||||||
two explicit continuity suites require 300 seconds and reject a shorter override.
|
two explicit continuity suites require 300 seconds and reject a shorter override.
|
||||||
|
|||||||
Reference in new issue
Block a user