Merge qualified remote workspace storage assembly for finite boot acceptance

* commit '343f1e57122f061cb229b8403fe2f522df178aea':
  feat(workspaces): keep remote files off the webserver disk (#18403)
This commit is contained in:
Dotta committed 2026-10-31 15:28:01 -05:00
commit af5492a39f
99 files changed
+9874 -238

No files matched your search

@@ -0,0 +1,28 @@
import { closeRegisteredClients, createDb } from "@paperclipai/db";
import { getStorageService } from "../src/storage/index.js";
import { migrateRemoteWorkspace } from "../src/services/remote-workspace-migration.js";
import { assertRemoteWorkspacePersistence } from "../src/services/remote-workspaces.js";
const args = process.argv.slice(2);
const allowed = new Set(["--company-id", "--workspace-id", "--dry-run", "--apply"]);
function identifier(name: string) {
const value = args[args.indexOf(name) + 1];
if (!args.includes(name) || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value ?? "")) throw new Error(`${name} requires a UUID`);
return value!;
}
try {
for (let index = 0; index < args.length; index++) {
const arg = args[index]!; if (!allowed.has(arg)) throw new Error(`Unknown argument: ${arg}`);
if (arg === "--company-id" || arg === "--workspace-id") index++;
}
if (args.includes("--dry-run") && args.includes("--apply")) throw new Error("Select either --dry-run or --apply");
const companyId = identifier("--company-id"), workspaceId = identifier("--workspace-id");
const storage = getStorageService();
const { databaseUrl } = assertRemoteWorkspacePersistence(storage);
const db = createDb(databaseUrl);
try { process.stdout.write(`${JSON.stringify(await migrateRemoteWorkspace({ db, storage, companyId, workspaceId, apply: args.includes("--apply") }), null, 2)}\n`); }
finally { await closeRegisteredClients(databaseUrl, { scope: "database" }); }
} catch (error) {
process.stderr.write(`${error instanceof Error ? error.message : "Remote workspace migration failed"}\n`);
process.exitCode = 1;
}
@@ -1,4 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { randomUUID } from "node:crypto";
import * as remoteWorkspaces from "../services/remote-workspaces.js";
// ---------------------------------------------------------------------------
// Hoisted mocks — must be declared before any imports that reference them
@@ -9,6 +11,7 @@ const mockAdapterExecutionTargetToRemoteSpec = vi.hoisted(() => vi.fn());
const mockBuildWorkspaceRealizationRequest = vi.hoisted(() => vi.fn());
const mockUpdateLeaseMetadata = vi.hoisted(() => vi.fn());
const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn());
const mockGetStoredWorkspace = vi.hoisted(() => vi.fn());
const mockLogActivity = vi.hoisted(() => vi.fn());
const mockLoggerInfo = vi.hoisted(() => vi.fn());
const mockGetEnvironment = vi.hoisted(() => vi.fn());
@@ -41,6 +44,7 @@ vi.mock("../services/environments.js", () => ({
vi.mock("../services/execution-workspaces.js", () => ({
executionWorkspaceService: vi.fn(() => ({
update: mockUpdateExecutionWorkspace,
getStoredById: mockGetStoredWorkspace,
})),
}));
@@ -272,6 +276,37 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
);
});
it("realizes an enrolled remote workspace in place and persists its authoritative remote root", async () => {
const environment = { ...makeEnvironment("sandbox"), id: randomUUID() };
const lease = makeLease({ id: randomUUID(), environmentId: environment.id, provider: "daytona", providerLeaseId: "sandbox-allocation" });
const workspace = makePersistedExecutionWorkspace({ id: randomUUID(), providerType: "cloud_sandbox", metadata: {
remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, source: { kind: "empty" },
writerRunId: randomUUID(), dirty: false, snapshotId: null, location: null },
} });
const remoteRoot = `/sandbox/.paperclip-workspaces/${workspace.id}`;
const savedLease = { ...lease, metadata: { remoteCwd: remoteRoot } };
const target = { kind: "remote", transport: "sandbox", remoteCwd: "/sandbox", runner: { execute: vi.fn() } };
mockResolveEnvironmentExecutionTarget.mockResolvedValue(target);
mockGetStoredWorkspace.mockResolvedValue({ ...workspace, cwd: remoteRoot });
mockUpdateLeaseMetadata.mockResolvedValue(savedLease);
const prepare = vi.spyOn(remoteWorkspaces, "prepareRemoteWorkspace").mockResolvedValue({
workspace, lease: savedLease, remoteRoot, reference: {} as never,
} as never);
try {
const runtime = makeMockRuntime();
const input = { ...makeRealizeInput({ environment, lease, persistedExecutionWorkspace: workspace }), adapterType: "paperclip_runner" };
const result = await environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }).realizeForRun(input);
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ workspace, target, source: { kind: "empty" } }));
expect(mockBuildWorkspaceRealizationRequest).not.toHaveBeenCalled();
expect(runtime.realizeWorkspace).not.toHaveBeenCalled();
expect(result.executionTarget).toMatchObject({ remoteCwd: remoteRoot,
workspaceRealization: { mode: "in_place", authoritativeRoot: remoteRoot, outboundRestorePaths: [] } });
expect(input.executionWorkspace.cwd).toBe(remoteRoot);
expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(lease.id, expect.objectContaining({ remoteCwd: remoteRoot }));
expect(result.lease.metadata?.remoteCwd).toBe(remoteRoot);
} finally { prepare.mockRestore(); }
});
it("happy path: returns lease, executionTarget, and remoteExecution on successful realization", async () => {
const executionTarget = { kind: "local", environmentId: "env-1", leaseId: "lease-1" };
const remoteExecution = { kind: "local", environmentId: "env-1", leaseId: "lease-1" };
@@ -6,7 +6,7 @@ import path from "node:path";
import { PassThrough } from "node:stream";
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
// This unit suite isolates runtime lifecycle behavior; privacy is covered with a
// real database by privacy-production-review.test.ts.
@@ -37,6 +37,7 @@ const mockAssertCanManageProjectWorkspaceRuntimeServices = vi.hoisted(() => vi.f
const mockStartRuntimeServices = vi.hoisted(() => vi.fn());
const mockStopRuntimeServicesForExecutionWorkspace = vi.hoisted(() => vi.fn());
const mockEnsurePersistedExecutionWorkspaceAvailable = vi.hoisted(() => vi.fn());
const mockRunWorkspaceJobForControl = vi.hoisted(() => vi.fn());
const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn());
const mockListConfiguredRuntimeServiceEntries = vi.hoisted(() => vi.fn());
// The integrated control path also takes the durable runtime-control lease (PAP-17205). This
@@ -45,6 +46,10 @@ const mockListConfiguredRuntimeServiceEntries = vi.hoisted(() => vi.fn());
const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null));
const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined));
const mockSpawn = vi.hoisted(() => vi.fn());
const mockPublishRuntimeOperation = vi.hoisted(() => vi.fn());
const mockLifecycleLock = vi.hoisted(() => vi.fn(async (_db: unknown, _id: string, run: (tx: unknown) => Promise<unknown>) => run({})));
vi.mock("../services/execution-workspace-lifecycle-lock.js", () => ({ withExecutionWorkspaceLifecycleLock: mockLifecycleLock }));
vi.mock("node:child_process", async () => ({
...(await vi.importActual<typeof import("node:child_process")>("node:child_process")),
@@ -74,7 +79,7 @@ vi.mock("../services/workspace-runtime.js", () => ({
cleanupExecutionWorkspaceArtifacts: vi.fn(),
ensurePersistedExecutionWorkspaceAvailable: mockEnsurePersistedExecutionWorkspaceAvailable,
listConfiguredRuntimeServiceEntries: mockListConfiguredRuntimeServiceEntries,
runWorkspaceJobForControl: vi.fn(),
runWorkspaceJobForControl: mockRunWorkspaceJobForControl,
startRuntimeServicesForWorkspaceControl: mockStartRuntimeServices,
stopRuntimeServicesForExecutionWorkspace: mockStopRuntimeServicesForExecutionWorkspace,
stopRuntimeServicesForProjectWorkspace: vi.fn(),
@@ -129,7 +134,7 @@ function buildExecutionWorkspace(overrides: Record<string, unknown> = {}) {
};
}
async function createApp() {
async function createApp(actorType: "agent" | "board" = "agent") {
const [{ executionWorkspaceRoutes }, { errorHandler }] = await Promise.all([
import("../routes/execution-workspaces.js"),
import("../middleware/index.js"),
@@ -137,7 +142,9 @@ async function createApp() {
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
(req as any).actor = {
(req as any).actor = actorType === "board" ? {
type: "board", userId: "board-1", companyIds: ["company-1"], source: "local_implicit", isInstanceAdmin: true,
} : {
type: "agent",
agentId: "agent-1",
companyId: "company-1",
@@ -246,9 +253,15 @@ function mockVerifiedReseed(
* stopped and retryable instead of "desired running" with residue.
*/
describe("execution workspace runtime control conflict and failure reconciliation", () => {
beforeAll(async () => {
// Route-module transformation is fixture setup, outside the HTTP control
// operation whose latency each test measures.
await Promise.all([import("../routes/execution-workspaces.js"), import("../middleware/index.js")]);
}, 30_000);
beforeEach(() => {
vi.clearAllMocks();
mockSpawn.mockReset();
mockLifecycleLock.mockImplementation(async (_db, _id, run) => run({}));
registeredProjectWorkspace = null;
mockAccessService.decide.mockResolvedValue({
allowed: true,
@@ -281,6 +294,8 @@ describe("execution workspace runtime control conflict and failure reconciliatio
// Mirror the real recorder: a throwing `run` becomes a terminal failed operation and
// the error still propagates to the caller.
try {
if (input.admit) await input.admit(async () => { mockPublishRuntimeOperation(); });
else mockPublishRuntimeOperation();
await input.run(async () => undefined);
} catch (error) {
(error as any).recordedOperationStatus = "failed";
@@ -291,6 +306,35 @@ describe("execution workspace runtime control conflict and failure reconciliatio
});
});
it.each([
{ route: "runtime-commands", action: "run", source: "empty" },
{ route: "runtime-commands", action: "run", source: "git" },
{ route: "runtime-commands", action: "repair", source: "empty" },
{ route: "runtime-services", action: "start", source: "git" },
] as const)("rejects remote $source $route/$action before controller file or job operations", async ({ route, action, source }) => {
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({
cwd: "/workspaces/remote-only", providerType: "cloud_sandbox", providerRef: "/workspaces/remote-only",
metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1,
snapshotId: null, source: source === "empty" ? { kind: "empty" }
: { kind: "git", repoUrl: "https://github.com/example/work.git", ref: "main" },
location: null, writerRunId: null, dirty: false } },
config: { workspaceRuntime: { jobs: [{ name: "verify", command: "node verify.js" }],
services: [{ name: "app", command: "node app.js" }] } },
}));
const response = await request(await createApp())
.post(`/api/execution-workspaces/${executionWorkspaceId}/${route}/${action}`)
.send(action === "run" ? { workspaceCommandId: "job:verify" } : {});
expect(response.status, JSON.stringify(response.body)).toBe(422);
expect(response.body).toMatchObject({ code: "remote_workspace_runtime_commands_unsupported" });
expect(mockEnsurePersistedExecutionWorkspaceAvailable).not.toHaveBeenCalled();
expect(mockWorkspaceOperationService.assertRuntimeControlAvailable).not.toHaveBeenCalled();
expect(mockWorkspaceOperationService.createRecorder).not.toHaveBeenCalled();
expect(mockRunWorkspaceJobForControl).not.toHaveBeenCalled();
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockSpawn).not.toHaveBeenCalled();
expect(mockClaimRuntimeLease).not.toHaveBeenCalled();
});
it("returns a stable 409 while a managed control is genuinely live", async () => {
const { conflict } = await import("../errors.js");
mockWorkspaceOperationService.assertRuntimeControlAvailable.mockRejectedValue(
@@ -320,6 +364,29 @@ describe("execution workspace runtime control conflict and failure reconciliatio
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
});
it.each(["migrating", "remote", "path_changed"] as const)("revalidates a board command at operation publication after the workspace becomes %s", async state => {
mockAssertCanManageExecutionWorkspaceRuntimeServices.mockResolvedValue({ actorType: "board" });
const stale = buildExecutionWorkspace();
mockExecutionWorkspaceService.getById.mockResolvedValueOnce(stale).mockResolvedValue(buildExecutionWorkspace(state === "migrating"
? { metadata: { remoteWorkspaceMigration: { schema: "paperclip.remote-workspace-migration.v1", state: "running" } } }
: state === "remote" ? { metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1,
snapshotId: null, source: { kind: "empty" }, location: null, writerRunId: null, dirty: false } } }
: { cwd: "/tmp/changed-authority" }));
const response = await request(await createApp("board"))
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-services/start`).send({});
expect(response.status, JSON.stringify(response.body)).toBe(state === "remote" ? 422 : 409);
expect(response.body.code).toBe(state === "remote" ? "remote_workspace_runtime_commands_unsupported"
: state === "migrating" ? "workspace_runtime_migration_in_progress" : "workspace_runtime_authority_changed");
expect(mockLifecycleLock).toHaveBeenCalled();
expect(mockPublishRuntimeOperation).not.toHaveBeenCalled();
expect(mockClaimRuntimeLease).not.toHaveBeenCalled();
expect(mockEnsurePersistedExecutionWorkspaceAvailable).not.toHaveBeenCalled();
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).not.toHaveBeenCalled();
expect(mockExecutionWorkspaceService.update).not.toHaveBeenCalled();
expect(mockSpawn).not.toHaveBeenCalled();
});
it("checks authorization before recovering or claiming the workspace", async () => {
const { forbidden } = await import("../errors.js");
mockAssertCanManageExecutionWorkspaceRuntimeServices.mockRejectedValue(
@@ -655,6 +722,7 @@ describe("execution workspace runtime control conflict and failure reconciliatio
mockWorkspaceOperationService.createRecorder.mockReturnValue({
attachExecutionWorkspaceId: vi.fn(),
recordOperation: async (input: any) => {
await input.admit(async () => { mockPublishRuntimeOperation(); });
await input.run(async (entry: Record<string, unknown>) => {
progress.push(entry);
});
@@ -682,7 +750,8 @@ describe("execution workspace runtime control conflict and failure reconciliatio
// The recorder's ceiling fires outside `run`, so only the outer handler can reconcile.
mockWorkspaceOperationService.createRecorder.mockReturnValue({
attachExecutionWorkspaceId: vi.fn(),
recordOperation: async () => {
recordOperation: async (input: any) => {
await input.admit(async () => { mockPublishRuntimeOperation(); });
throw new WorkspaceOperationTimeoutError(1_000, "start");
},
});
@@ -22,6 +22,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({
}));
const mockAccessService = vi.hoisted(() => ({ decide: vi.fn(async () => ({ allowed: true })) }));
const mockRecordOperation = vi.hoisted(() => vi.fn());
const mockPublishOperation = vi.hoisted(() => vi.fn());
const mockReconcileStaleRuntimeControlOperations = vi.hoisted(() => vi.fn(async () => ({ reconciled: 0, operationIds: [] })));
// The control path recovers stranded operations and then refuses only a genuinely live one
// (PAP-17249). This suite is about the durable lease, so availability always resolves.
@@ -112,7 +113,8 @@ describeEmbeddedPostgres("execution workspace runtime control lease enforcement"
runId: null,
issueId: req.get("x-test-owner-issue") ?? null,
}));
mockRecordOperation.mockImplementation(async (input: { run: () => Promise<unknown> }) => {
mockRecordOperation.mockImplementation(async (input: { run: () => Promise<unknown>; admit: (publish: () => Promise<void>) => Promise<void> }) => {
await input.admit(async () => { mockPublishOperation(); });
await input.run();
return { id: randomUUID(), status: "succeeded" };
});
@@ -237,6 +239,7 @@ describeEmbeddedPostgres("execution workspace runtime control lease enforcement"
expect(mockStartRuntimeServices).toHaveBeenCalledTimes(2);
mockRecordOperation.mockClear();
mockPublishOperation.mockClear();
mockStartRuntimeServices.mockClear();
mockStopRuntimeServices.mockClear();
mockReconcileStaleRuntimeControlOperations.mockClear();
@@ -255,7 +258,7 @@ describeEmbeddedPostgres("execution workspace runtime control lease enforcement"
expect(typeof blocked.body.remediation).toBe("string");
// No workspace operation, no runtime-service mutation, no reconciliation side effect.
expect(mockRecordOperation).not.toHaveBeenCalled();
expect(mockPublishOperation).not.toHaveBeenCalled();
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockStopRuntimeServices).not.toHaveBeenCalled();
expect(mockReconcileStaleRuntimeControlOperations).not.toHaveBeenCalled();
+14 -5
View File
@@ -29,7 +29,8 @@ import {
startEmbeddedPostgresTestDatabase,
} from "./helpers/embedded-postgres.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const externalDatabase = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
const embeddedPostgresSupport = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
const execFileAsync = promisify(execFile);
let seedGraphSequence = 0;
@@ -205,11 +206,15 @@ describeEmbeddedPostgres("workspace file resources", () => {
beforeAll(async () => {
vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce");
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resources-");
db = createDb(tempDb.connectionString);
if (externalDatabase) db = createDb(externalDatabase);
else {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resources-");
db = createDb(tempDb.connectionString);
}
}, 60_000);
afterAll(async () => {
await db?.$client.end();
await tempDb?.cleanup();
vi.unstubAllEnvs();
});
@@ -1952,11 +1957,15 @@ describeEmbeddedPostgres("file resource route guards", () => {
let db: Db;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resource-guards-");
db = createDb(tempDb.connectionString);
if (externalDatabase) db = createDb(externalDatabase);
else {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resource-guards-");
db = createDb(tempDb.connectionString);
}
}, 60_000);
afterAll(async () => {
await db?.$client.end();
await tempDb?.cleanup();
});
@@ -1,12 +1,45 @@
import { describe, expect, it, vi } from "vitest";
import type { InstanceExperimentalSettings } from "@paperclipai/shared";
import { INSTANCE_FEATURE_CATALOG, instanceExperimentalSettingsSchema, type InstanceExperimentalSettings } from "@paperclipai/shared";
import {
applyExperimentalSettingsPatch,
applyCloudCatalogDefaults,
applyManagedExperimentalOverlay,
normalizeExperimentalSettings,
resolveWorktreeRunExecutionActivationState,
} from "../services/instance-settings.js";
import type { ManagedInstanceConfig } from "../services/managed-config.js";
describe("instance settings service", () => {
it("defaults remote workspace enrollment on while preserving explicit operator choices", () => {
expect(instanceExperimentalSettingsSchema.parse({}).remoteWorkspaceStorageEnabled).toBe(true);
expect(INSTANCE_FEATURE_CATALOG.remoteWorkspaceStorageEnabled).toMatchObject({ cloudDefault: true, selfHostedDefault: true });
for (const stored of [undefined, null, {}, { enableApps: true }, { remoteWorkspaceStorageEnabled: "invalid" }]) {
expect(normalizeExperimentalSettings(stored).remoteWorkspaceStorageEnabled).toBe(true);
}
const disabled = applyExperimentalSettingsPatch({}, { remoteWorkspaceStorageEnabled: false });
expect(normalizeExperimentalSettings(JSON.parse(JSON.stringify(disabled))).remoteWorkspaceStorageEnabled).toBe(false);
expect(normalizeExperimentalSettings({ remoteWorkspaceStorageEnabled: false, enableCases: "invalid" }).remoteWorkspaceStorageEnabled).toBe(false);
expect(applyExperimentalSettingsPatch(disabled, { enableStatusCards: true }).remoteWorkspaceStorageEnabled).toBe(false);
expect(applyExperimentalSettingsPatch(disabled, { remoteWorkspaceStorageEnabled: true }).remoteWorkspaceStorageEnabled).toBe(true);
});
it.each([
{ stored: {}, managed: undefined, expected: true },
{ stored: { remoteWorkspaceStorageEnabled: false }, managed: undefined, expected: false },
{ stored: { remoteWorkspaceStorageEnabled: false }, managed: true, expected: true },
{ stored: { remoteWorkspaceStorageEnabled: true }, managed: false, expected: false },
])("respects Cloud enrollment defaults and managed choices: $expected", ({ stored, managed, expected }) => {
const config: ManagedInstanceConfig = {
v: 1, mode: "cloud", catalogVersion: "test", plugins: { autoInstall: [] }, environments: [],
features: managed === undefined ? {} : { remoteWorkspaceStorageEnabled: managed },
};
const normalized = normalizeExperimentalSettings(stored);
const result = applyManagedExperimentalOverlay(applyCloudCatalogDefaults(normalized, stored, config), config);
expect(result.experimental.remoteWorkspaceStorageEnabled).toBe(expected);
expect(normalized.remoteWorkspaceStorageEnabled).toBe("remoteWorkspaceStorageEnabled" in stored ? stored.remoteWorkspaceStorageEnabled : true);
expect(result.managedKeys.remoteWorkspaceStorageEnabled).toEqual(managed === undefined
? undefined : { managed: true, managedBy: "paperclip-cloud" });
});
it("keeps Notebook off for legacy or invalid settings and round-trips explicit operator choices", () => {
for (const stored of [undefined, null, {}, { enableApps: true },
{ enablePaperclipNotebook: "true" }, { enablePaperclipNotebook: 1 },
@@ -62,6 +95,7 @@ describe("instance settings service", () => {
})).toEqual({
enableEnvironments: true,
enableNativeRunner: false,
remoteWorkspaceStorageEnabled: true,
enableAiConnectionRouters: false,
enableManagedSandboxOnly: false,
enableIsolatedWorkspaces: true,
@@ -0,0 +1,707 @@
import express, { type Request } from "express";
import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
import request from "supertest";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, companySecrets, companySecretBindings, createDb, environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, plugins, projects, projectWorkspaces, secretAccessEvents, workspaceOperations, type Db } from "@paperclipai/db";
import { INSPECTION_AUDIENCE, INSPECTION_HEADER, INSPECTION_PERMIT_TYPE, parseInspectionQuery, type InspectionQuery, type InspectionPermit, type RemoteWorkspaceAuthority } from "@paperclipai/shared";
import { errorHandler } from "../middleware/index.js";
import { notFound, unprocessable } from "../errors.js";
import { fileResourceRoutes } from "../routes/file-resources.js";
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
import { createRemoteWorkspaceFiles, REMOTE_WORKSPACE_FILES_SCRIPT } from "../services/remote-workspace-files.js";
import { workspaceFileResourceService } from "../services/workspace-file-resources.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { customerSuccessRoutes } from "../routes/customer-success.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { secretService } from "../services/secrets.js";
import { captureRemoteWorkspaceDirectory } from "../services/remote-workspace-snapshot-archive.js";
import { beginWorkspaceSnapshot, uploadWorkspaceSnapshot } from "../services/remote-workspace-snapshot-storage.js";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
import { createStorageService } from "../storage/service.js";
import { createSavedWorkspaceLoader } from "../services/remote-workspaces.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const execFileAsync = promisify(execFile);
const decryptCleanupFault = vi.hoisted(() => ({ error: undefined as Error | undefined, path: undefined as string | undefined }));
vi.mock("node:fs/promises", async importOriginal => {
const actual = await importOriginal<typeof import("node:fs/promises")>();
return { ...actual, async rm(target: Parameters<typeof actual.rm>[0], options: Parameters<typeof actual.rm>[1]) {
if (decryptCleanupFault.error && String(target).includes("pc-workspace-decrypt-")) {
const error = decryptCleanupFault.error; decryptCleanupFault.error = undefined; decryptCleanupFault.path = String(target); throw error;
}
return actual.rm(target, options);
} };
});
const externalDatabase = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeDatabase = support.supported ? describe : describe.skip;
describe("confined workspace file helper", () => {
let root: string;
beforeAll(async () => {
root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "workspace-helper-")));
await fs.mkdir(path.join(root, "folder"));
await fs.writeFile(path.join(root, "folder", "report.md"), "Saved result\n");
await fs.writeFile(path.join(root, ".env"), "hidden");
await fs.symlink(os.tmpdir(), path.join(root, "escape"));
await fs.writeFile(path.join(root, "binary.bin"), Buffer.from([0, 1, 2]));
});
afterAll(async () => { await fs.rm(root, { recursive: true, force: true }); });
async function helper(action: string, args: Record<string, unknown>) {
const { stdout } = await execFileAsync(process.execPath, ["--input-type=commonjs", "-e", REMOTE_WORKSPACE_FILES_SCRIPT,
JSON.stringify({ root, action, ...args })], { maxBuffer: 2 * 1024 * 1024 });
return JSON.parse(stdout);
}
it("lists ordinary children without exposing secrets or symlinks", async () => {
const result = await helper("list", { options: { mode: "all", limit: 25, offset: 0, maxScannedEntries: 5000, maxDepth: 20 } });
expect(result.entries.map((entry: { relativePath: string }) => entry.relativePath)).toEqual(["folder", "binary.bin"]);
expect(result.truncated).toBe(false);
const report = await helper("stat", { relativePath: "folder/report.md", withHash: true });
expect(report).toMatchObject({ kind: "file", byteSize: 13, text: true });
expect(report.sha256).toMatch(/^[a-f0-9]{64}$/);
expect(await helper("stat", { relativePath: "binary.bin" })).toMatchObject({ text: false });
});
it.each(["../outside", "/etc/passwd", "escape/file", ".env", "folder/../.env", "file:///etc/passwd", "folder\\report.md"])(
"rejects a confined or policy-denied path %s", async relativePath => {
await expect(helper("stat", { relativePath })).rejects.toThrow();
},
);
it("uses bounded recursive search and pagination", async () => {
const result = await helper("list", { options: { mode: "all", query: "report", limit: 1, offset: 0, maxScannedEntries: 5000, maxDepth: 20 } });
expect(result.entries.map((entry: { relativePath: string }) => entry.relativePath)).toEqual(["folder/report.md"]);
const bounded = await helper("list", { options: { mode: "recent", limit: 25, offset: 0, maxScannedEntries: 1, maxDepth: 20 } });
expect(bounded.scannedCount).toBe(1);
expect(bounded.truncated).toBe(true);
});
it("gets changed files from the workspace repository without revealing Git metadata or secrets", async () => {
await execFileAsync("git", ["-C", root, "init", "--initial-branch=main"]);
await execFileAsync("git", ["-C", root, "add", "folder/report.md"]);
await execFileAsync("git", ["-C", root, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "commit", "-m", "initial"]);
await fs.writeFile(path.join(root, "folder", "report.md"), "Changed useful result\n");
const result = await helper("list", { options: { mode: "changed", limit: 25, offset: 0, maxScannedEntries: 5000, maxDepth: 20 } });
expect(result.entries.map((entry: { relativePath: string }) => entry.relativePath)).toEqual(["binary.bin", "folder/report.md"]);
});
});
describeDatabase("remote workspace ordinary file routes", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
let db: Db;
const directories: string[] = [];
beforeAll(async () => {
if (externalDatabase) db = createDb(externalDatabase);
else {
temporary = await startEmbeddedPostgresTestDatabase("remote-workspace-files-");
db = createDb(temporary.connectionString);
}
}, 60_000);
afterAll(async () => {
await temporary?.cleanup();
await db?.$client.end();
await Promise.all(directories.map(directory => fs.rm(directory, { recursive: true, force: true })));
});
async function fixture() {
const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "remote-file-test-")));
directories.push(root);
await fs.mkdir(path.join(root, "docs"));
await fs.writeFile(path.join(root, "docs", "result.md"), "Remote useful result\n");
await fs.writeFile(path.join(root, ".env"), "private-key");
const ids = { company: randomUUID(), workspace: randomUUID(), issue: randomUUID(), environment: randomUUID(), lease: randomUUID() };
const authority: RemoteWorkspaceAuthority = {
schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, writerRunId: null, dirty: false,
source: { kind: "empty" }, location: { environmentId: ids.environment, leaseId: ids.lease,
providerLeaseId: `allocation-${ids.lease}`, remoteCwd: root },
};
await db.insert(companies).values({ id: ids.company, name: "File access", issuePrefix: `RF${ids.company.slice(0, 6)}` });
await db.insert(executionWorkspaces).values({ id: ids.workspace, companyId: ids.company, mode: "isolated_workspace",
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Remote task", metadata: { remoteWorkspace: authority } });
await db.insert(issues).values({ id: ids.issue, companyId: ids.company, title: "Read remote result", executionWorkspaceId: ids.workspace });
await db.update(executionWorkspaces).set({ sourceIssueId: ids.issue }).where(eq(executionWorkspaces.id, ids.workspace));
await db.insert(environments).values({ id: ids.environment, name: `Remote file ${ids.environment}`, driver: "sandbox" });
await db.insert(environmentLeases).values({ id: ids.lease, companyId: ids.company, environmentId: ids.environment,
executionWorkspaceId: ids.workspace, issueId: ids.issue, provider: "fake", providerLeaseId: authority.location!.providerLeaseId,
metadata: { remoteCwd: root } });
let beforeCopy: (() => Promise<void>) | undefined;
const execute = vi.fn(async (input: Parameters<EnvironmentRuntimeService["execute"]>[0]) => {
expect(input.lease.id).toBe(ids.lease);
expect(input.bypassSession).toBe(true);
try {
const output = await execFileAsync(process.execPath, input.args!, { cwd: input.cwd, maxBuffer: 2 * 1024 * 1024 });
return { ...output, exitCode: 0, timedOut: false };
} catch (error) {
const failure = error as { stdout: string; stderr: string };
return { stdout: failure.stdout, stderr: failure.stderr, exitCode: 1, timedOut: false };
}
});
const syncOut = vi.fn(async (input: Parameters<EnvironmentRuntimeService["syncOut"]>[0]) => {
await beforeCopy?.();
for (const operation of input.operations) for (const mapping of operation.files) await fs.copyFile(mapping.sourcePath, mapping.targetPath);
return { operations: input.operations.map(operation => ({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred: 0 })) };
});
const runtime = { execute, syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: runtime });
const app = express();
app.use(express.json());
app.use((req, _res, next) => {
req.actor = { type: "board", userId: "local-board", companyIds: [ids.company], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
next();
});
app.use("/api", fileResourceRoutes(db, { remoteFiles }));
app.use(errorHandler);
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
}
it("lists, resolves, previews, and downloads live bytes without a controller checkout", async () => {
const f = await fixture();
const base = `/api/issues/${f.ids.issue}/file-resources`;
const listed = await request(f.app).get(`${base}/list`).query({ workspace: "execution" }).expect(200);
expect(listed.body).toMatchObject({ state: "available", workspace: { source: { kind: "live" } } });
expect(listed.body.items.map((item: { relativePath: string }) => item.relativePath)).toEqual(["docs"]);
const resolved = await request(f.app).get(`${base}/resolve`).query({ path: "docs/result.md", workspace: "execution" }).expect(200);
expect(resolved.body).toMatchObject({ kind: "file", source: { kind: "live" }, capabilities: { preview: true, download: true } });
const preview = await request(f.app).get(`${base}/content`).query({ path: "docs/result.md", workspace: "execution" }).expect(200);
expect(preview.body.content).toEqual({ encoding: "utf8", data: "Remote useful result\n" });
const download = await request(f.app).get(`${base}/content`).query({ path: "docs/result.md", workspace: "execution", download: true }).expect(200);
expect(download.text).toBe("Remote useful result\n");
expect(f.syncOut).toHaveBeenCalledTimes(2);
expect(await fs.readdir(path.dirname(f.syncOut.mock.calls[0]![0].operations[0]!.files[0]!.targetPath)).catch(() => [])).toEqual([]);
});
it("serves signed support reads from secret-bound live and saved workspaces inside its read-only transaction", async () => {
const f = await fixture();
const other = await fixture();
const keyDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "inspection-workspace-key-"));
directories.push(keyDirectory);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(keyDirectory, "master.key"));
vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_ENABLED", "true");
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "remote-inspection-fixture");
vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_CLOUD_ORIGIN", "https://cloud.example.test");
const keys = generateKeyPairSync("ed25519");
vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_JWKS", JSON.stringify({ keys: [{ ...keys.publicKey.export({ format: "jwk" }), kid: "cloud" }] }));
const consumed = vi.fn(async () => Response.json({ consumed: true }));
vi.stubGlobal("fetch", consumed);
try {
const pluginId = randomUUID();
const provider = `inspection-${pluginId}`;
const pluginKey = `paperclip.${provider}`;
await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: `@paperclipai/${provider}`, version: "1.0.0", apiVersion: 1,
categories: ["automation"], status: "ready", manifestJson: {
id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Inspection fixture", description: "Synthetic inspection provider", author: "Paperclip",
categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" },
environmentDrivers: [{ driverKey: provider, kind: "sandbox_provider", displayName: "Inspection fixture",
configSchema: { type: "object", properties: { apiKey: { type: "string", format: "secret-ref" } } } }],
} });
const secret = await secretService(db).create(f.ids.company, { name: "synthetic inspection provider", provider: "local_encrypted", value: "synthetic-provider-credential" });
await secretService(db).createBinding({ companyId: f.ids.company, secretId: secret.id,
targetType: "environment", targetId: f.ids.environment, configPath: "apiKey" });
await db.update(environments).set({ config: { provider, apiKey: secret.id } }).where(eq(environments.id, f.ids.environment));
await db.update(environmentLeases).set({ provider, metadata: { remoteCwd: f.root, provider, apiKey: secret.id,
pluginId, sandboxProviderPlugin: true } }).where(eq(environmentLeases.id, f.ids.lease));
const storage = createStorageService(createLocalDiskStorageProvider(path.join(keyDirectory, "objects")));
const archivePath = path.join(keyDirectory, "saved.pcws");
const archive = await captureRemoteWorkspaceDirectory({ cwd: f.root, archivePath });
const [operation] = await db.insert(workspaceOperations).values({ companyId: f.ids.company,
executionWorkspaceId: f.ids.workspace, phase: "synthetic_saved_fixture" }).returning();
const snapshot = await beginWorkspaceSnapshot(db, { companyId: f.ids.company, workspaceId: f.ids.workspace,
operationId: operation.id, generation: 1, epoch: 1 });
await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
await fs.writeFile(path.join(f.root, "docs", "result.md"), "Updated live result\n");
await fs.symlink(other.root, path.join(f.root, "escape"));
const call = vi.fn(async (_pluginId: string, method: string, params: any) => {
expect(_pluginId).toBe(pluginId);
expect(params.companyId).toBe(f.ids.company);
expect(params.lease.providerLeaseId).toBe(f.authority.location!.providerLeaseId);
expect(params.config.apiKey).toBe("synthetic-provider-credential");
if (method === "environmentExecute") {
expect(params.bypassSession).toBe(true);
try {
const result = await execFileAsync(process.execPath, params.args, { cwd: params.cwd, maxBuffer: 2 * 1024 * 1024 });
return { ...result, exitCode: 0, timedOut: false, signal: null };
} catch (error) {
const result = error as { stdout: string; stderr: string };
return { stdout: result.stdout, stderr: result.stderr, exitCode: 1, timedOut: false, signal: null };
}
}
if (method === "environmentSyncOut") {
for (const operation of params.operations) for (const mapping of operation.files) await fs.copyFile(mapping.sourcePath, mapping.targetPath);
return { operations: [] };
}
throw new Error(`Inspection must not allocate or resume: ${method}`);
});
const workerManager = { call, isRunning: (id: string) => id === pluginId } as unknown as PluginWorkerManager;
const app = express();
app.use(express.json());
app.use("/api/customer-success/v1", customerSuccessRoutes(db, storage, { pluginWorkerManager: workerManager }));
app.use(errorHandler);
const inspect = (value: InspectionQuery) => {
const query = parseInspectionQuery(value);
const now = Math.floor(Date.now() / 1000);
const claims: InspectionPermit = { v: 1, iss: "paperclip-cloud", aud: INSPECTION_AUDIENCE, sub: "remote-inspection-fixture",
iat: now, exp: now + 60, jti: randomUUID(), bindingVersion: 1, grantId: randomUUID(), requestId: randomUUID(),
agentId: randomUUID(), keyId: "sha256:synthetic-fixture", runId: randomUUID(), query };
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", typ: INSPECTION_PERMIT_TYPE, kid: "cloud" })).toString("base64url");
const body = Buffer.from(JSON.stringify(claims)).toString("base64url");
const payload = `${header}.${body}`;
return request(app).post("/api/customer-success/v1/read").set(INSPECTION_HEADER,
`${payload}.${sign(null, Buffer.from(payload), keys.privateKey).toString("base64url")}`).send(query);
};
const base = { companyId: f.ids.company, resourceId: f.ids.issue, context: { workspace: "execution" as const } };
const beforeSecret = await db.select().from(companySecrets).where(eq(companySecrets.id, secret.id));
const beforeAudit = await db.select().from(secretAccessEvents).where(eq(secretAccessEvents.companyId, f.ids.company));
const transaction = db.transaction.bind(db);
const transactions: string[] = [];
const transactionSpy = vi.spyOn(db, "transaction").mockImplementation((async (callback, options) => transaction(async tx => {
const [row] = await tx.execute(sql`show transaction_read_only`);
transactions.push(String(row.transaction_read_only));
return callback(tx);
}, options)) as typeof db.transaction);
// A reader accidentally closed over the outer Db fails here; nested
// provider/config/secret reads must use the inspection transaction.
const selectSpy = vi.spyOn(db, "select").mockImplementation(() => { throw new Error("Inspection escaped its transaction"); });
try {
const listed = await inspect({ ...base, operation: "files.list" }).expect(200);
expect(listed.body).toMatchObject({ state: "available", workspace: { source: { kind: "live" } } });
expect(JSON.stringify(listed.body)).not.toContain(".env");
const read = await inspect({ ...base, operation: "files.read", path: "docs/result.md" }).expect(200);
expect(read.body.content.data).toBe("Updated live result\n");
const downloaded = await inspect({ ...base, operation: "files.download", path: "docs/result.md", range: { start: 2, end: 7 } }).expect(200);
expect(Buffer.from(downloaded.body.content, "base64").toString()).toBe("dated ");
expect(JSON.stringify([listed.body, read.body, downloaded.body])).not.toContain("synthetic-provider-credential");
const beforeDenied = call.mock.calls.length;
await inspect({ ...base, companyId: other.ids.company, operation: "files.read", path: "docs/result.md" }).expect(404);
await inspect({ ...base, operation: "files.read", path: ".env" }).expect(403);
await inspect({ ...base, context: { workspace: "execution", workspaceId: other.ids.workspace }, operation: "files.read", path: "docs/result.md" }).expect(422);
expect(call).toHaveBeenCalledTimes(beforeDenied);
await inspect({ ...base, operation: "files.read", path: "escape/docs/result.md" }).expect(403);
} finally { selectSpy.mockRestore(); transactionSpy.mockRestore(); }
expect(transactions.length).toBeGreaterThan(0);
expect(new Set(transactions)).toEqual(new Set(["on"]));
expect(await db.select().from(companySecrets).where(eq(companySecrets.id, secret.id))).toEqual(beforeSecret);
expect(await db.select().from(secretAccessEvents).where(eq(secretAccessEvents.companyId, f.ids.company))).toEqual(beforeAudit);
await db.delete(companySecretBindings).where(eq(companySecretBindings.secretId, secret.id));
const beforeUnbound = call.mock.calls.length;
await inspect({ ...base, operation: "files.read", path: "docs/result.md" }).expect(422);
expect(call).toHaveBeenCalledTimes(beforeUnbound);
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
call.mockClear();
const savedList = await inspect({ ...base, operation: "files.list" }).expect(200);
expect(savedList.body).toMatchObject({ workspace: { source: { kind: "saved", snapshotId: snapshot.id } } });
const savedRead = await inspect({ ...base, operation: "files.read", path: "docs/result.md" }).expect(200);
expect(savedRead.body.content.data).toBe("Remote useful result\n");
const savedDownload = await inspect({ ...base, operation: "files.download", path: "docs/result.md" }).expect(200);
expect(Buffer.from(savedDownload.body.content, "base64").toString()).toBe("Remote useful result\n");
expect(call).not.toHaveBeenCalled();
expect(consumed).toHaveBeenCalled();
} finally { vi.unstubAllEnvs(); vi.unstubAllGlobals(); }
});
it("rejects denied files before provider access and prevents changed transfer bytes", async () => {
const f = await fixture();
await request(f.app).get(`/api/issues/${f.ids.issue}/file-resources/content`).query({ workspace: "execution", path: ".env" }).expect(403);
expect(f.execute).not.toHaveBeenCalled();
f.beforeCopy(async () => { await fs.writeFile(path.join(f.root, "docs", "result.md"), "Another changed text\n"); });
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: f.remoteFiles }).readContent(f.ids.issue,
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422, details: { code: "file_changed" } });
});
async function savedFileFixture() {
const f = await fixture(), storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "saved-file-budget-"));
directories.push(storageRoot);
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
const archivePath = path.join(storageRoot, "saved.pcws"), archive = await captureRemoteWorkspaceDirectory({ cwd: f.root, archivePath });
const [operation] = await db.insert(workspaceOperations).values({ companyId: f.ids.company,
executionWorkspaceId: f.ids.workspace, phase: "synthetic_file_budget" }).returning();
const snapshot = await beginWorkspaceSnapshot(db, { companyId: f.ids.company, workspaceId: f.ids.workspace,
operationId: operation!.id, generation: 1, epoch: 1 });
const saved = await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
return { ...f, storage, saved, input: { companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id } };
}
it("shares fail-fast saved Files admission across loader instances until owned cleanup completes", async () => {
const f = await savedFileFixture(), availableBytes = async () => 8n * 1024n ** 3n;
const first = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
const second = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
const third = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
const opened: Array<{ root: string; cleanup(): Promise<void> }> = [];
const getObject = vi.spyOn(f.storage, "getObject");
try {
opened.push(await first(f.input)); opened.push(await second(f.input));
expect(opened[0]!.root).not.toBe(opened[1]!.root);
getObject.mockClear();
await expect(third(f.input)).rejects.toMatchObject({ status: 422, details: { code: "workspace_unavailable" } });
expect(getObject).not.toHaveBeenCalled();
await opened[0]!.cleanup(); await opened[0]!.cleanup();
await expect(fs.stat(opened[0]!.root)).rejects.toMatchObject({ code: "ENOENT" });
opened.push(await third(f.input));
expect(getObject).toHaveBeenCalledTimes(1);
} finally { await Promise.all(opened.map(access => access.cleanup())); getObject.mockRestore(); }
});
it("bounds reserved scratch bytes across saved Files loader instances", async () => {
const f = await savedFileFixture(), options = { availableBytes: async () => 8n * 1024n ** 3n,
limits: { maxScratchBytes: 180 * 1024 * 1024 } };
const first = createSavedWorkspaceLoader(db, f.storage, options), second = createSavedWorkspaceLoader(db, f.storage, options);
const access = await first(f.input), getObject = vi.spyOn(f.storage, "getObject");
try {
await expect(second(f.input)).rejects.toMatchObject({ status: 422, details: { code: "workspace_unavailable" } });
expect(getObject).not.toHaveBeenCalled();
} finally { await access.cleanup(); getObject.mockRestore(); }
const next = await second(f.input); await next.cleanup();
});
it.each(["archive", "disk"] as const)("rejects saved Files %s bounds before object retrieval and releases admission", async bound => {
const f = await savedFileFixture(), getObject = vi.spyOn(f.storage, "getObject");
if (bound === "archive") await db.update(executionWorkspaceSnapshots).set({ byteSize: 512 * 1024 * 1024 + 1 })
.where(eq(executionWorkspaceSnapshots.id, f.saved.id));
const load = createSavedWorkspaceLoader(db, f.storage, { availableBytes: async () => bound === "disk" ? 0n : 8n * 1024n ** 3n });
try {
await expect(load(f.input)).rejects.toMatchObject({ status: 422, details: { code: bound === "archive" ? "too_large" : "workspace_unavailable" } });
expect(getObject).not.toHaveBeenCalled();
} finally {
await db.update(executionWorkspaceSnapshots).set({ byteSize: f.saved.byteSize }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
getObject.mockRestore();
}
const next = await createSavedWorkspaceLoader(db, f.storage, { availableBytes: async () => 8n * 1024n ** 3n })(f.input);
await next.cleanup();
});
it.each(["entries", "manifest"] as const)("rejects excessive saved Files %s before extraction and cleans the reserved archive", async bound => {
const f = await savedFileFixture(), options = { availableBytes: async () => 8n * 1024n ** 3n };
await expect(createSavedWorkspaceLoader(db, f.storage, { ...options, limits: bound === "entries" ? { maxEntries: 1 } : { maxManifestBytes: 1 } })(f.input))
.rejects.toMatchObject({ status: 422, details: { code: "too_large" } });
const opened: Array<{ root: string; cleanup(): Promise<void> }> = [];
try {
const load = createSavedWorkspaceLoader(db, f.storage, options);
opened.push(await load(f.input)); opened.push(await load(f.input));
expect(await fs.readFile(path.join(opened[0]!.root, "docs", "result.md"), "utf8")).toBe("Remote useful result\n");
} finally { await Promise.all(opened.map(access => access.cleanup())); }
});
it("preserves object failures while releasing saved Files admission for the next copies", async () => {
const f = await savedFileFixture(), original = new Error("synthetic object retrieval failed");
const options = { availableBytes: async () => 8n * 1024n ** 3n };
await expect(createSavedWorkspaceLoader(db, { ...f.storage, getObject: async () => { throw original; } }, options)(f.input)).rejects.toBe(original);
const opened: Array<{ cleanup(): Promise<void> }> = [];
try {
const load = createSavedWorkspaceLoader(db, f.storage, options);
opened.push(await load(f.input)); opened.push(await load(f.input));
} finally { await Promise.all(opened.map(access => access.cleanup())); }
});
it("owns failed decrypt cleanup beneath the Files reservation before releasing capacity", async () => {
const f = await savedFileFixture(), original = new Error("synthetic decrypt cleanup failed");
const load = createSavedWorkspaceLoader(db, f.storage, { availableBytes: async () => 8n * 1024n ** 3n });
const opened: Array<{ cleanup(): Promise<void> }> = [];
decryptCleanupFault.error = original; decryptCleanupFault.path = undefined;
try {
await expect(load(f.input)).rejects.toBe(original);
const failed = decryptCleanupFault.path!;
expect(path.basename(path.dirname(failed))).toMatch(/^pc-workspace-files-/);
await expect(fs.stat(failed)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.stat(path.dirname(failed))).rejects.toMatchObject({ code: "ENOENT" });
opened.push(await load(f.input)); opened.push(await load(f.input));
} finally {
decryptCleanupFault.error = undefined;
await Promise.all(opened.map(access => access.cleanup()));
if (decryptCleanupFault.path) await fs.rm(decryptCleanupFault.path, { recursive: true, force: true });
decryptCleanupFault.path = undefined;
}
});
it("reads saved content after explicit location release, with no provider call", async () => {
const f = await fixture();
const snapshotId = randomUUID();
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
const cleanup = vi.fn(async () => {});
const loadSavedWorkspace = vi.fn(async () => ({ root: f.root, savedAt: "2026-10-30T12:00:00.000Z", cleanup }));
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">, loadSavedWorkspace });
const service = workspaceFileResourceService(db, undefined, { remoteFiles });
const result = await service.readContent(f.ids.issue, { workspace: "execution", path: "docs/result.md" });
expect(result).toMatchObject({ resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } }, content: { data: "Remote useful result\n" } });
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
expect(cleanup).toHaveBeenCalledTimes(1);
expect(f.execute).not.toHaveBeenCalled();
expect(f.syncOut).not.toHaveBeenCalled();
});
it("shares one verified saved extraction across 100 availability paths and removes it after the request", async () => {
const f = await fixture();
const storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "availability-saved-"));
directories.push(storageRoot);
const paths = Array.from({ length: 100 }, (_, index) => `docs/result-${index}.md`);
await Promise.all(paths.map(file => fs.writeFile(path.join(f.root, file), `Result ${file}\n`)));
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
const archivePath = path.join(storageRoot, "saved.pcws");
const archive = await captureRemoteWorkspaceDirectory({ cwd: f.root, archivePath });
const [operation] = await db.insert(workspaceOperations).values({ companyId: f.ids.company,
executionWorkspaceId: f.ids.workspace, phase: "synthetic_saved_fixture" }).returning();
const snapshot = await beginWorkspaceSnapshot(db, { companyId: f.ids.company, workspaceId: f.ids.workspace,
operationId: operation!.id, generation: 1, epoch: 1 });
await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
const loadSavedWorkspace = vi.fn(createSavedWorkspaceLoader(db, storage));
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
const service = workspaceFileResourceService(db, undefined, { remoteFiles });
const result = await service.availability(f.ids.issue, { queries: paths.map(file => ({ workspace: "execution", path: file })) });
expect(result.results).toHaveLength(100);
expect(result.results.every(item => item.openable && item.resource?.source?.kind === "saved")).toBe(true);
expect(loadSavedWorkspace).toHaveBeenCalledTimes(1);
for (const loaded of loadSavedWorkspace.mock.results) {
const saved = await loaded.value;
await expect(fs.stat(saved.root)).rejects.toMatchObject({ code: "ENOENT" });
}
await service.availability(f.ids.issue, { queries: [{ workspace: "execution", path: paths[0]! }] });
expect(loadSavedWorkspace).toHaveBeenCalledTimes(2);
expect(f.execute).not.toHaveBeenCalled();
}, 60_000);
it.each(["null", "rejected", "unavailable"] as const)("shares a %s remote open across availability paths", async state => {
const f = await fixture();
const original = new Error("saved storage unavailable");
const open = vi.fn(async () => {
if (state === "rejected") throw original;
if (state === "unavailable") throw unprocessable("Workspace files are unavailable", { code: "workspace_unavailable" });
return null;
});
const result = workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
{ queries: Array.from({ length: 20 }, (_, index) => ({ workspace: "execution", path: `result-${index}.md` })) });
if (state === "rejected") await expect(result).rejects.toBe(original);
else {
const results = (await result).results;
expect(results).toHaveLength(20);
if (state === "unavailable") expect(results.every(item => item.unavailableReason === "workspace_unavailable")).toBe(true);
}
expect(open).toHaveBeenCalledTimes(1);
});
it("drains started availability checks before closing shared access and preserves the original error", async () => {
const f = await fixture();
let releaseSlow!: () => void;
let announceSlow!: () => void;
let announceError!: () => void;
const slow = new Promise<void>(resolve => { releaseSlow = resolve; });
const slowStarted = new Promise<void>(resolve => { announceSlow = resolve; });
const errorThrown = new Promise<void>(resolve => { announceError = resolve; });
const original = new Error("availability read failed");
const close = vi.fn(async () => { throw new Error("cleanup also failed"); });
let slowFinished = false;
const open = vi.fn(async () => ({
source: { kind: "saved" as const, snapshotId: randomUUID(), savedAt: "2026-10-30T12:00:00.000Z" },
stat: async (relativePath: string) => {
if (relativePath === "error.md") {
await slowStarted;
announceError();
throw original;
}
announceSlow();
await slow;
slowFinished = true;
return { kind: "file" as const, relativePath, byteSize: 1, modifiedAt: null, text: true };
},
read: vi.fn(), list: vi.fn(), download: vi.fn(), close,
}));
const pending = workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
{ queries: [{ workspace: "execution", path: "error.md" }, { workspace: "execution", path: "slow.md" }] });
const observed = pending.catch(error => error);
try {
await errorThrown;
await new Promise(resolve => setImmediate(resolve));
expect(close).not.toHaveBeenCalled();
expect(slowFinished).toBe(false);
} finally { releaseSlow(); }
expect(await observed).toBe(original);
expect(slowFinished).toBe(true);
expect(open).toHaveBeenCalledTimes(1);
expect(close).toHaveBeenCalledTimes(1);
});
it("keeps denied availability paths ahead of opening a remote workspace", async () => {
const f = await fixture();
const open = vi.fn();
const result = await workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
{ queries: [".env", ".git/config", "../escape"].map(file => ({ workspace: "execution", path: file })) });
expect(result.results.every(item => !item.openable)).toBe(true);
expect(open).not.toHaveBeenCalled();
});
it("isolates availability task access from a project workspace with the same UUID", async () => {
const f = await fixture();
const projectId = randomUUID();
await db.insert(projects).values({ id: projectId, companyId: f.ids.company, name: "Remote project" });
await db.insert(projectWorkspaces).values({ id: f.ids.workspace, companyId: f.ids.company, projectId,
name: "Project root", sourceType: "remote_managed", isPrimary: true });
await db.update(issues).set({ projectId, projectWorkspaceId: f.ids.workspace }).where(eq(issues.id, f.ids.issue));
const close = vi.fn(async () => {});
const stat = vi.fn(async (relativePath: string) => ({ kind: "file" as const, relativePath, byteSize: 1, modifiedAt: null, text: true }));
const open = vi.fn(async () => ({ source: { kind: "live" as const, snapshotId: null, savedAt: null },
stat, read: vi.fn(), list: vi.fn(), download: vi.fn(), close }));
const result = await workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
{ queries: [{ workspace: "execution", path: "result.md" }, { workspace: "project", path: "result.md" }] });
expect(result.results.map(item => item.resource?.kind)).toEqual(["file", "remote_resource"]);
expect(open).toHaveBeenCalledTimes(1);
expect(stat).toHaveBeenCalledTimes(1);
expect(close).toHaveBeenCalledTimes(1);
});
it("attempts every availability accessor close even when an earlier close fails", async () => {
const f = await fixture();
const otherWorkspace = randomUUID();
await db.insert(executionWorkspaces).values({ id: otherWorkspace, companyId: f.ids.company, mode: "isolated_workspace",
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Other task root", sourceIssueId: f.ids.issue });
const cleanupFailure = new Error("first scratch cleanup failed");
const closeFirst = vi.fn(async () => { throw cleanupFailure; });
const closeOther = vi.fn(async () => {});
const open = vi.fn(async ({ workspaceId }: { workspaceId: string }) => ({
source: { kind: "live" as const, snapshotId: null, savedAt: null },
stat: async (relativePath: string) => {
if (workspaceId === f.ids.workspace) throw notFound("Workspace file not found");
return { kind: "file" as const, relativePath, byteSize: 1, modifiedAt: null, text: true };
},
read: vi.fn(), list: vi.fn(), download: vi.fn(), close: workspaceId === f.ids.workspace ? closeFirst : closeOther,
}));
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
{ queries: [{ workspace: "auto", path: "one.md" }, { workspace: "auto", path: "two.md" }] })).rejects.toBe(cleanupFailure);
expect(open).toHaveBeenCalledTimes(2);
expect(closeFirst).toHaveBeenCalledTimes(1);
expect(closeOther).toHaveBeenCalledTimes(1);
});
it("rechecks saved authority for each availability stat on a shared accessor", async () => {
const f = await fixture();
const snapshotId = randomUUID();
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
const cleanup = vi.fn(async () => {});
const loadSavedWorkspace = vi.fn(async () => ({ root: f.root, savedAt: "2026-10-30T12:00:00.000Z", cleanup }));
const actual = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
let changed!: () => void;
const authorityChanged = new Promise<void>(resolve => { changed = resolve; });
const open = vi.fn(async (input: Parameters<typeof actual.open>[0]) => {
const access = (await actual.open(input))!;
return { ...access, stat: async (relativePath: string) => {
if (relativePath === "docs/after.md") await authorityChanged;
const entry = await access.stat(relativePath);
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId, epoch: 2 } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
changed();
return entry;
} };
});
const result = await workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
{ queries: [{ workspace: "execution", path: "docs/result.md" }, { workspace: "execution", path: "docs/after.md" }] });
expect(result.results[0]!.openable).toBe(true);
expect(result.results[1]!.unavailableReason).toBe("workspace_changed");
expect(loadSavedWorkspace).toHaveBeenCalledTimes(1);
expect(cleanup).toHaveBeenCalledTimes(1);
});
it.each(["confirmed_stop", "unknown_stop", "dirty", "foreign_receipt"])(
"uses saved bytes only for a clean workspace with an exact stopped receipt (%s)", async state => {
const f = await fixture();
const snapshotId = randomUUID();
const agentId = randomUUID();
const runId = randomUUID();
await db.insert(agents).values({ id: agentId, companyId: f.ids.company, name: "Stopped workspace reader" });
await db.insert(heartbeatRuns).values({ id: runId, companyId: f.ids.company, agentId, status: "succeeded" });
const termination = remoteTerminationReceipt({ id: f.ids.lease, companyId: f.ids.company, heartbeatRunId: runId,
provider: "fake", providerLeaseId: f.authority.location!.providerLeaseId },
{ state: "stopped", providerLeaseId: f.authority.location!.providerLeaseId })!;
await db.update(environmentLeases).set({ heartbeatRunId: runId, status: "released", releasedAt: new Date(), cleanupStatus: "success",
metadata: { remoteCwd: f.root, ...(state !== "unknown_stop" ? { remoteExecutionTermination:
state === "foreign_receipt" ? { ...termination, leaseId: randomUUID() } : termination } : {}) } })
.where(eq(environmentLeases.id, f.ids.lease));
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId, dirty: state === "dirty" } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
const savedRoot = await fs.mkdtemp(path.join(os.tmpdir(), "stopped-workspace-saved-"));
directories.push(savedRoot);
await fs.mkdir(path.join(savedRoot, "docs"));
await fs.writeFile(path.join(savedRoot, "docs", "result.md"), "Saved stopped result\n");
const cleanup = vi.fn(async () => {});
const loadSavedWorkspace = vi.fn(async () => ({ root: savedRoot, savedAt: "2026-10-30T12:00:00.000Z", cleanup }));
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
const content = workspaceFileResourceService(db, undefined, { remoteFiles }).readContent(f.ids.issue,
{ workspace: "execution", path: "docs/result.md" });
if (state === "confirmed_stop") {
await expect(content).resolves.toMatchObject({ content: { data: "Saved stopped result\n" },
resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } } });
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
expect(cleanup).toHaveBeenCalledOnce();
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.ids.workspace));
expect(workspace!.metadata?.remoteWorkspace).toMatchObject({ location: f.authority.location });
} else {
await expect(content).rejects.toMatchObject({ status: 422, details: { code: "workspace_unavailable" } });
expect(loadSavedWorkspace).not.toHaveBeenCalled();
}
expect(f.execute).not.toHaveBeenCalled();
expect(f.syncOut).not.toHaveBeenCalled();
},
);
it("does not fall back to a saved version for an unreachable live allocation", async () => {
const f = await fixture();
f.execute.mockRejectedValueOnce(new Error("provider unavailable"));
const loadSavedWorkspace = vi.fn();
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">, loadSavedWorkspace });
await expect(workspaceFileResourceService(db, undefined, { remoteFiles }).resolve(f.ids.issue,
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422 });
expect(loadSavedWorkspace).not.toHaveBeenCalled();
});
it("rejects cross-company and stale allocation bindings before provider access", async () => {
const f = await fixture();
await expect(f.remoteFiles.open({ companyId: randomUUID(), workspaceId: f.ids.workspace })).rejects.toMatchObject({ status: 404 });
await db.update(environmentLeases).set({ providerLeaseId: "replacement" }).where(and(eq(environmentLeases.id, f.ids.lease), eq(environmentLeases.companyId, f.ids.company)));
await expect(f.remoteFiles.open({ companyId: f.ids.company, workspaceId: f.ids.workspace })).rejects.toMatchObject({ status: 422 });
expect(f.execute).not.toHaveBeenCalled();
});
it("rejects a result after the durable workspace location changes", async () => {
const f = await fixture();
const original = f.execute.getMockImplementation()!;
f.execute.mockImplementationOnce(async input => {
const result = await original(input);
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, epoch: 2 } } })
.where(eq(executionWorkspaces.id, f.ids.workspace));
return result;
});
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: f.remoteFiles }).resolve(f.ids.issue,
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ details: { code: "workspace_changed" } });
});
it("rejects corrupt provider metadata without exposing its output", async () => {
const f = await fixture();
f.execute.mockResolvedValueOnce({ stdout: '{"kind":"file","relativePath":"../../private","byteSize":1}',
stderr: "private-provider-details", exitCode: 0, timedOut: false });
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: f.remoteFiles }).resolve(f.ids.issue,
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ message: "Workspace files are unavailable", details: { code: "workspace_unavailable" } });
});
});
@@ -1,4 +1,4 @@
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { applyPendingMigrations, closeRegisteredClients, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { execFile as execFileCallback } from "node:child_process";
import { promisify } from "node:util";
import { randomUUID } from "node:crypto";
@@ -7,10 +7,12 @@ import { tmpdir } from "node:os";
import path from "node:path";
import { constants as fsConstants } from "node:fs";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { and, eq } from "drizzle-orm";
import { and, eq, sql } from "drizzle-orm";
import { agents, companies, createDb, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issues } from "@paperclipai/db";
import { executionWorkspaceService } from "../services/execution-workspaces.js";
import { executionWorkspaceRepositoryService } from "../services/execution-workspace-repositories.js";
import * as remoteWorkspaces from "../services/remote-workspaces.js";
import type { EnvironmentLease } from "@paperclipai/shared";
import { listLocalProjectRepositories } from "@paperclipai/adapter-utils/ssh";
import { readManagedWorkspaceRepositories, disposeGitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync";
import { captureDirectorySnapshot, disposeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
@@ -18,7 +20,11 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro
const catalog = vi.hoisted(() => ({ available: true }));
vi.mock("../services/tool-access.js", () => ({ toolAccessService: () => ({ listProjectRepositories: async () => ({ repositories: catalog.available ? [{ id: "123", fullName: "team/source", url: "https://github.com/team/source", connections: [] }] : [] }) }) }));
const support = await getEmbeddedPostgresTestSupport();
const externalDatabaseUrl = process.env.PAPERCLIP_TASK_REPOSITORIES_TEST_DATABASE_URL;
if (externalDatabaseUrl && new URL(externalDatabaseUrl).pathname !== "/remote_workspace_repositories_test") {
throw new Error("Remote repository tests require the disposable remote_workspace_repositories_test database");
}
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("durable task repository receipts", () => {
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
@@ -26,8 +32,13 @@ const support = await getEmbeddedPostgresTestSupport();
const actor = { type: "board" as const, source: "local_implicit" as const, userId: "local-board" };
beforeAll(async () => {
vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce");
temporary = await startEmbeddedPostgresTestDatabase("paperclip-task-repositories-");
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
temporary = externalDatabaseUrl ? { connectionString: externalDatabaseUrl,
cleanup: () => closeRegisteredClients(externalDatabaseUrl) }
: await startEmbeddedPostgresTestDatabase("paperclip-task-repositories-");
db = createDb(temporary.connectionString);
await db.execute(sql`set client_min_messages = warning`);
if (externalDatabaseUrl) await db.execute(sql`truncate table companies restart identity cascade`);
cwd = await mkdtemp(path.join(tmpdir(), "paperclip-task-repositories-files-"));
[companyId] = (await db.insert(companies).values({ name: "Task repositories", issuePrefix: "TREPO" }).returning()).map(row => row.id);
[agentId] = (await db.insert(agents).values({ companyId, name: "Repository worker", adapterType: "process", status: "idle" }).returning()).map(row => row.id);
@@ -41,6 +52,39 @@ const support = await getEmbeddedPostgresTestSupport();
return executionWorkspaceRepositoryService(db).prepareForAdmission(input);
};
const request = (requestKey: string, ref?: string) => executionWorkspaceRepositoryService(db).request({ companyId, issueId, actor, request: { repository: { kind: "catalog", id: "123" }, requestKey, ...(ref ? { ref } : {}) } });
it("prepares a remote task repository through its current writer without opening a controller path", async () => {
const runId = randomUUID(), taskId = randomUUID(), remoteWorkspaceId = randomUUID(), leaseId = randomUUID(), environmentId = randomUUID();
const remoteCwd = `/workspaces/${remoteWorkspaceId}`;
await db.insert(issues).values({ id: taskId, companyId, title: "Remote repositories" });
await db.insert(executionWorkspaces).values({ id: remoteWorkspaceId, companyId, sourceIssueId: taskId,
name: "Remote task", cwd: remoteCwd, mode: "isolated_workspace", strategyType: "project_primary", providerType: "cloud_sandbox",
metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null,
source: { kind: "empty" }, writerRunId: runId, dirty: true,
location: { leaseId, environmentId, providerLeaseId: "remote-allocation", remoteCwd } } } });
await db.update(issues).set({ executionWorkspaceId: remoteWorkspaceId }).where(eq(issues.id, taskId));
const intent = await executionWorkspaceRepositoryService(db).request({ companyId, issueId: taskId, actor,
request: { repository: { kind: "catalog", id: "123" }, requestKey: "remote-repository" } });
const pin = "a".repeat(40);
const prepare = vi.spyOn(remoteWorkspaces, "prepareRemoteWorkspaceRepository").mockResolvedValue({
cwd: `${remoteCwd}/${intent.repository.relativePath}`, pinnedCommit: pin, branchName: "main" });
try {
const prepared = await prepareForAdmission({ companyId, issueId: taskId, workspaceId: remoteWorkspaceId,
cwd: remoteCwd, agentId, runId, responsibleUserId: "local-board",
remoteTarget: { kind: "remote", transport: "sandbox", remoteCwd } as never,
remoteLease: { id: leaseId, environmentId, companyId, providerLeaseId: "remote-allocation" } as EnvironmentLease });
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ runId,
repository: expect.objectContaining({ id: intent.repository.id, relativePath: intent.repository.relativePath }) }));
expect(prepared).toMatchObject([{ id: intent.repository.id, pinnedCommit: pin }]);
const [receipt] = await db.select().from(executionWorkspaceRepositories).where(eq(executionWorkspaceRepositories.id, intent.repository.id));
expect(receipt).toMatchObject({ state: "ready", pinnedCommit: pin, branchName: "main" });
expect(await lstat(remoteCwd).catch(() => null)).toBeNull();
await expect(prepareForAdmission({ companyId, issueId: taskId, workspaceId: remoteWorkspaceId,
cwd: remoteCwd, agentId, runId: randomUUID(), responsibleUserId: "local-board",
remoteTarget: { kind: "remote", transport: "sandbox", remoteCwd } as never,
remoteLease: { id: leaseId } as EnvironmentLease })).rejects.toThrow("publication binding");
expect(prepare).toHaveBeenCalledTimes(1);
} finally { prepare.mockRestore(); }
});
it("publishes a requested repository from an absent path through real admission locking and cloning", async () => {
const root = await mkdtemp(path.join(tmpdir(), "paperclip-fresh-repository-"));
const previousPath = process.env.PATH;
+10 -3
View File
@@ -112,6 +112,8 @@ import { caseRoutes } from "./routes/cases.js";
import { caseHandoffRoutes } from "./routes/case-handoffs.js";
import { workflowRoutes } from "./routes/workflows.js";
import { fileResourceRoutes } from "./routes/file-resources.js";
import { createRemoteWorkspaceFiles } from "./services/remote-workspace-files.js";
import { createSavedWorkspaceLoader } from "./services/remote-workspaces.js";
import { routineRoutes } from "./routes/routines.js";
import { environmentRoutes } from "./routes/environments.js";
import { executionWorkspaceRoutes } from "./routes/execution-workspaces.js";
@@ -537,7 +539,10 @@ export async function createApp(
// A signed claim above commits identity before any normal request can seed
// company data. Unclaimed probes bypass session resolution as well as SQL.
app.use(cloudWarmStandbyMiddleware(isWarmStandby, health, staticUi));
app.use("/api/customer-success/v1", customerSuccessRoutes(db, opts.storageService));
const workerManager = opts.pluginWorkerManager ?? createPluginWorkerManager();
app.use("/api/customer-success/v1", customerSuccessRoutes(db, opts.storageService, {
pluginWorkerManager: workerManager,
}));
app.use(museTransportRoutes(db, opts.authPublicBaseUrl));
app.use(publicMcpIngress);
// Connection-intent tools carry their own short-lived, run-bound bearer and
@@ -545,7 +550,6 @@ export async function createApp(
// agent API key. Every request revalidates the active heartbeat row.
app.use(runtimeConnectionIntentRoutes(db));
const hostServicesDisposers = new Map<string, () => void>();
const workerManager = opts.pluginWorkerManager ?? createPluginWorkerManager();
let lifecyclePluginsReady = false;
const agentLifecycle = startAgentLifecycle(db, createLifecycleDriver(db, workerManager), () => lifecyclePluginsReady && !isWarmStandby() && !isIdleTaskDrainActive());
const connectionIntentHeartbeat = heartbeatService(db, {
@@ -772,7 +776,10 @@ export async function createApp(
api.use(caseHandoffRoutes(db));
api.use(workflowRoutes(db));
api.use(issueTreeControlRoutes(db, { pluginWorkerManager: workerManager }));
api.use(fileResourceRoutes(db));
api.use(fileResourceRoutes(db, { remoteFiles: createRemoteWorkspaceFiles(db, {
environmentRuntime: environmentRuntimeService(db, { pluginWorkerManager: workerManager }),
loadSavedWorkspace: createSavedWorkspaceLoader(db, opts.storageService),
}) }));
api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));
api.use(
environmentRoutes(db, {
+6
View File
@@ -42,6 +42,7 @@ import {
} from "./services/native-runtime/native-harness-backup-expiration.js";
import { registerNativeRunnerEvidenceRetentionWork } from "./services/native-runtime/native-runner-evidence-retention-work.js";
import { gitRestorePointRetentionService, registerGitRestorePointWork } from "./services/git-restore-point-ledger.js";
import { remoteWorkspaceSnapshotRetentionService, registerRemoteWorkspaceSnapshotRetentionWork } from "./services/remote-workspace-snapshot-retention.js";
import {
nativeWorkspaceSyncRetentionService,
registerNativeWorkspaceSyncRetentionWork,
@@ -1499,6 +1500,7 @@ async function startServerWithDatabaseTeardown(
wakeup: heartbeat.wakeup,
});
const terminalWorkspaces = executionWorkspaceService(db as any, {
destroyRemoteWorkspaceLease: input => cleanupInspectionRuntime.destroyRemoteWorkspaceLease(input),
pluginWorkerManager,
gitCacheExpirationEnabled: config.gitCacheExpirationEnabled,
gitBaseExpirationEnabled: config.gitBaseExpirationEnabled,
@@ -1535,6 +1537,10 @@ async function startServerWithDatabaseTeardown(
logger.info(result, "git restore point retention");
}
});
registerRemoteWorkspaceSnapshotRetentionWork(deliveryWork,
remoteWorkspaceSnapshotRetentionService(db as any, { storage: storageService }), result => {
if (result.deleted || result.failed) logger.info(result, "remote workspace snapshot retention");
});
if (config.nativeWorkspaceSyncRetentionEnabled) {
unsubscribeNativeWorkspaceSyncRetention = registerNativeWorkspaceSyncRetentionWork(deliveryWork, nativeWorkspaceSyncRetentionService(db as any, {
retentionDays: config.nativeWorkspaceSyncRetentionDays,
+3 -3
View File
@@ -12,7 +12,7 @@ import {
import type { StorageService } from "../storage/types.js";
import { getCloudRuntimeIdentity } from "../services/cloud-runtime-identity.js";
import { customerSuccessRunAuthority } from "../services/customer-success-authority.js";
import { readCustomerSuccessResource } from "../services/customer-success-inspection.js";
import { readCustomerSuccessResource, type CustomerSuccessInspectionOptions } from "../services/customer-success-inspection.js";
import { forbidden, unprocessable } from "../errors.js";
export function verifyInspectionPermit(
@@ -70,7 +70,7 @@ export function verifyInspectionPermit(
}
/** Mounted before actor middleware: none of these reads creates tenant state. */
export function customerSuccessRoutes(db: Db, storage: StorageService) {
export function customerSuccessRoutes(db: Db, storage: StorageService, options: CustomerSuccessInspectionOptions = {}) {
const router = Router();
router.use((_req, res, next) => {
res.setHeader("Cache-Control", "no-store");
@@ -122,7 +122,7 @@ export function customerSuccessRoutes(db: Db, storage: StorageService) {
signal: AbortSignal.timeout(10000),
});
if (!consumed.ok) throw forbidden("Inspection permit could not be consumed");
const result = await readCustomerSuccessResource(db, storage, query);
const result = await readCustomerSuccessResource(db, storage, query, options);
const body = JSON.stringify(result);
if (Buffer.byteLength(body) > 2 * 1024 * 1024)
throw unprocessable("Inspection response is too large; use pagination or a byte range");
+39 -24
View File
@@ -11,6 +11,7 @@ import type { Db } from "@paperclipai/db";
import { issues, projects, projectWorkspaces } from "@paperclipai/db";
import {
findWorkspaceCommandDefinition,
readRemoteWorkspaceAuthority,
matchWorkspaceRuntimeServiceToCommand,
reconcileExecutionWorkspaceBranchSchema,
updateExecutionWorkspaceSchema,
@@ -58,6 +59,7 @@ import { appendWithCap } from "../adapters/utils.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js";
import { withExecutionWorkspaceLifecycleLock } from "../services/execution-workspace-lifecycle-lock.js";
import { resolveManagedWorkspaceInstanceId } from "../services/managed-workspace-identity.js";
import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
import {
@@ -89,16 +91,15 @@ function resolveFallbackSeedSourceConfigPath(baseWorkspaceCwd: string): string |
export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
const router = Router();
const svc = executionWorkspaceService(db, { pluginWorkerManager: opts.pluginWorkerManager });
const environmentRuntime = environmentRuntimeService(db, { pluginWorkerManager: opts.pluginWorkerManager });
const svc = executionWorkspaceService(db, { pluginWorkerManager: opts.pluginWorkerManager,
destroyRemoteWorkspaceLease: input => environmentRuntime.destroyRemoteWorkspaceLease(input) });
const access = accessService(db);
const workspaceOperationsSvc = workspaceOperationService(db);
const runtimeLeases = workspaceRuntimeLeaseService(db);
const heartbeat = heartbeatService(db, {
pluginWorkerManager: opts.pluginWorkerManager,
});
const environmentRuntime = environmentRuntimeService(db, {
pluginWorkerManager: opts.pluginWorkerManager,
});
async function assertWorkspacePrivacy(req: Request, res: Response, id: string) {
if (await canActorReadExecutionWorkspace(db, req.actor, id)) return true;
@@ -302,6 +303,10 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
executionWorkspaceId: existing.id,
sourceIssueId: existing.sourceIssueId,
});
if (readRemoteWorkspaceAuthority(existing.metadata)) {
throw unprocessable("Workspace commands are not supported for remote workspaces yet. Run the command through an agent task.",
{ code: "remote_workspace_runtime_commands_unsupported" });
}
const workspaceCwd = existing.cwd;
if (!workspaceCwd) {
@@ -479,6 +484,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
let runtimeServiceCount = existing.runtimeServices?.length ?? 0;
let stdout = "";
let stderr = "";
let leaseClaimResult: WorkspaceRuntimeLeaseClaim | null = null;
let runtimeControlAdmitted = false;
/**
* Bring runtime rows, local listeners and the recorded desired state back to a consistent
@@ -531,6 +538,32 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
}
const recordRuntimeControlOperation = () => recorder.recordOperation({
admit: async (publish) => {
await withExecutionWorkspaceLifecycleLock(db, existing.id, async () => {
const fresh = await svc.getById(existing.id);
if (!fresh || fresh.companyId !== existing.companyId) throw conflict("Execution workspace changed before command admission.");
if (readRemoteWorkspaceAuthority(fresh.metadata)) {
throw unprocessable("Workspace commands are not supported for remote workspaces yet. Run the command through an agent task.",
{ code: "remote_workspace_runtime_commands_unsupported" });
}
if (fresh.metadata?.remoteWorkspaceMigration) {
throw conflict("The workspace is being moved to remote storage. Retry after migration completes.",
{ code: "workspace_runtime_migration_in_progress" });
}
if (fresh.cwd !== existing.cwd || fresh.providerRef !== existing.providerRef || fresh.providerType !== existing.providerType) {
throw conflict("Execution workspace changed before command admission.", { code: "workspace_runtime_authority_changed" });
}
if (LEASED_WORKSPACE_RUNTIME_ACTIONS.includes(action)) {
leaseClaimResult = await runtimeLeases.claim({ companyId: existing.companyId,
executionWorkspaceId: existing.id, action, owner: authorization });
}
await workspaceOperationsSvc.assertRuntimeControlAvailable({ executionWorkspaceId: existing.id, action });
// Migration uses this lifecycle fence and refuses every running operation.
// Publish before unlocking; host I/O starts only after the row is visible.
await publish();
});
runtimeControlAdmitted = true;
},
phase: action === "stop"
? "workspace_teardown"
: action === "repair"
@@ -989,25 +1022,6 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
executionWorkspaceId: existing.id,
action,
run: async () => {
// Claim the durable exclusivity lease before anything mutates the workspace or
// its runtime services. A competing issue/run throws 409 here, leaving no
// workspace operation row and no runtime-service change behind.
let leaseClaimResult: WorkspaceRuntimeLeaseClaim | null = null;
if (LEASED_WORKSPACE_RUNTIME_ACTIONS.includes(action)) {
leaseClaimResult = await runtimeLeases.claim({
companyId: existing.companyId,
executionWorkspaceId: existing.id,
action,
owner: authorization,
});
}
// Re-check inside the claim: recovery of stranded operations plus a refusal if one
// is genuinely still live. The same assertion ran before the lease was taken, but a
// row can be stranded in that window, and only the recovering path may proceed.
await workspaceOperationsSvc.assertRuntimeControlAvailable({
executionWorkspaceId: existing.id,
action,
});
try {
return {
operation: await recordRuntimeControlOperation(),
@@ -1017,7 +1031,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
// The operation is already terminal here. This also catches the recorder's own time
// budget expiring on a start that never settles, which is the one failure the inner
// handler cannot see — reconcile there too so no listener or desired state is left over.
if (action === "start" || action === "restart" || action === "repair") {
if (runtimeControlAdmitted && (action === "start" || action === "restart" || action === "repair")) {
await reconcileFailedRuntimeStart(error);
}
throw error;
@@ -1300,6 +1314,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
workspaceId: id,
capturedGeneration,
destroy: async () => {
if (readRemoteWorkspaceAuthority(workspace.metadata)) return svc.cleanupRemoteStorage(workspace);
await environmentRuntime.destroyReusableSandboxLeases({
companyId: existing.companyId,
executionWorkspaceId: existing.id,
+38 -32
View File
@@ -14,7 +14,9 @@ import {
type WorkspaceFileListResponse,
} from "@paperclipai/shared";
import { badRequest, HttpError, notFound, unprocessable } from "../errors.js";
import { workspaceFileResourceService } from "../services/index.js";
import { workspaceFileResourceService } from "../services/workspace-file-resources.js";
import type { PreparedWorkspaceFileDownload } from "../services/workspace-file-resources.js";
import type { RemoteWorkspaceFiles } from "../services/remote-workspace-files.js";
import { assertBoard, getActorInfo, hasCompanyAccess } from "./authz.js";
import { logActivity } from "../services/activity-log.js";
import {
@@ -56,7 +58,7 @@ export type WorkspaceFileResourceService = {
issueId: string,
input: { path: string; workspace?: "auto" | "execution" | "project" | null; projectId?: string | null; workspaceId?: string | null },
opts?: { issue?: Awaited<ReturnType<WorkspaceFileResourceService["getIssue"]>> },
): Promise<{ resource: ResolvedWorkspaceResource; realPath: string }>;
): Promise<PreparedWorkspaceFileDownload>;
};
type FileResourceLimiter = {
@@ -353,9 +355,10 @@ export function fileResourceRoutes(db: Db, opts: {
limiter?: FileResourceLimiter;
listLimiter?: FileResourceLimiter;
availabilityLimiter?: FileResourceLimiter;
remoteFiles?: RemoteWorkspaceFiles;
} = {}) {
const router = Router();
const serviceFor = (req: Request) => opts.service ?? workspaceFileResourceService(db, req.actor);
const serviceFor = (req: Request) => opts.service ?? workspaceFileResourceService(db, req.actor, { remoteFiles: opts.remoteFiles });
const limiter = opts.limiter ?? createFileResourceLimiter();
const listLimiter = opts.listLimiter ?? createFileResourceListLimiter();
const availabilityLimiter = opts.availabilityLimiter ?? createFileResourceAvailabilityLimiter();
@@ -857,36 +860,39 @@ export function fileResourceRoutes(db: Db, opts: {
throw error;
}
await logActivity(db, {
companyId: issue.companyId,
actorType: actor.actorType,
actorId: actor.actorId,
action: "issue.file_resource_download",
entityType: "issue",
entityId: req.params.issueId,
agentId: actor.agentId,
runId: actor.runId,
agentApiKeyId: actor.agentApiKeyId,
details: activityDetails({
outcome: "success",
workspaceKind: result.resource.workspaceKind,
workspaceId: result.resource.workspaceId,
projectId: result.resource.projectId ?? null,
projectName: result.resource.projectName ?? null,
displayPath: result.resource.displayPath,
byteSize: result.resource.byteSize ?? null,
contentType: result.resource.contentType ?? null,
}),
});
try {
await logActivity(db, {
companyId: issue.companyId,
actorType: actor.actorType,
actorId: actor.actorId,
action: "issue.file_resource_download",
entityType: "issue",
entityId: req.params.issueId,
agentId: actor.agentId,
runId: actor.runId,
agentApiKeyId: actor.agentApiKeyId,
details: activityDetails({
outcome: "success",
workspaceKind: result.resource.workspaceKind,
workspaceId: result.resource.workspaceId,
projectId: result.resource.projectId ?? null,
projectName: result.resource.projectName ?? null,
displayPath: result.resource.displayPath,
byteSize: result.resource.byteSize ?? null,
contentType: result.resource.contentType ?? null,
}),
});
res.setHeader("Content-Type", result.resource.contentType ?? "application/octet-stream");
if (result.resource.byteSize != null) {
res.setHeader("Content-Length", String(result.resource.byteSize));
}
res.setHeader("Cache-Control", "private, max-age=60");
res.setHeader("X-Content-Type-Options", "nosniff");
res.setHeader("Content-Disposition", `attachment; filename="${safeAttachmentFilename(result.resource.title)}"`);
await pipeline(createReadStream(result.realPath), res);
res.setHeader("Content-Type", result.resource.contentType ?? "application/octet-stream");
if (result.resource.byteSize != null) {
res.setHeader("Content-Length", String(result.resource.byteSize));
}
res.setHeader("Cache-Control", "private, max-age=60");
res.setHeader("X-Content-Type-Options", "nosniff");
res.setHeader("Content-Disposition", `attachment; filename="${safeAttachmentFilename(result.resource.title)}"`);
if ("stream" in result) await pipeline(result.stream, res);
else await pipeline(createReadStream(result.realPath), res);
} finally { if ("stream" in result) await result.cleanup(); }
return;
}
@@ -21,6 +21,14 @@ import { deriveBundleState } from "./agent-instructions.js";
import { readInstructionBytes } from "./agent-instruction-files.js";
import { skillPackageStore } from "./skill-package-store.js";
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
import { createRemoteWorkspaceFiles } from "./remote-workspace-files.js";
import { createSavedWorkspaceLoader } from "./remote-workspaces.js";
import { environmentRuntimeService } from "./environment-runtime.js";
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
export interface CustomerSuccessInspectionOptions {
pluginWorkerManager?: PluginWorkerManager;
}
// This is a reviewed catalog of readers, not a proxy for existing GET routes.
// Select credentials out of queries; never load encrypted identity records.
@@ -136,6 +144,7 @@ export async function readCustomerSuccessResource(
db: Db,
storage: StorageService,
q: InspectionQuery,
options: CustomerSuccessInspectionOptions = {},
): Promise<unknown> {
return db.transaction(
async (rawTx) => {
@@ -328,7 +337,11 @@ export async function readCustomerSuccessResource(
if (q.operation.startsWith("files.")) {
try {
const issue = await ownRow(tx, schema.issues, q.companyId!, q.resourceId!);
const files = workspaceFileResourceService(tx);
const remoteFiles = createRemoteWorkspaceFiles(tx, {
environmentRuntime: environmentRuntimeService(tx, { pluginWorkerManager: options.pluginWorkerManager, readOnlyInspection: true }),
loadSavedWorkspace: createSavedWorkspaceLoader(tx, storage),
});
const files = workspaceFileResourceService(tx, undefined, { remoteFiles });
const input = { path: q.path ?? "", ...q.context, limit: q.limit, offset: q.offset };
const opts = {
issue: issue as unknown as NonNullable<Parameters<typeof files.list>[2]>["issue"],
@@ -337,6 +350,31 @@ export async function readCustomerSuccessResource(
if (q.operation === "files.read")
return await files.readContent(q.resourceId!, input, opts);
const resolved = await files.prepareDownload(q.resourceId!, input, opts);
if ("stream" in resolved) {
try {
const size = resolved.resource.byteSize;
if (size == null) return unavailable("file_size_unavailable");
if (size === 0 && !q.range) return binary(Buffer.alloc(0), 0);
const start = q.range?.start ?? 0;
const end = q.range?.end ?? size - 1;
if (start >= size || end >= size || end - start + 1 > MAX_INSPECTION_BYTES) {
throw unprocessable("Request a bounded byte range");
}
const chunks: Buffer[] = [];
let position = 0;
let received = 0;
for await (const chunk of resolved.stream) {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
const from = Math.max(0, start - position);
const to = Math.min(bytes.length, end + 1 - position);
if (to > from) { chunks.push(bytes.subarray(from, to)); received += to - from; }
position += bytes.length;
if (position > end) break;
}
if (received !== end - start + 1) return unavailable("file_changed_during_read");
return binary(Buffer.concat(chunks), size, start);
} finally { resolved.stream.destroy(); await resolved.cleanup(); }
}
const file = await fs.open(resolved.realPath, constants.O_RDONLY | constants.O_NOFOLLOW);
try {
const stat = await file.stat();
@@ -133,6 +133,7 @@ export const DELIVERY_QUEUES = {
connectionEvents: "connection-events",
connectionContinuity: "connection-continuity",
gitRestorePoints: "git-restore-points",
remoteWorkspaceSnapshots: "remote-workspace-snapshots",
nativeWorkspaceSyncRetention: "native-workspace-sync-retention",
question: "question-responses",
pluginSchedules: "plugin-schedules",
+7 -1
View File
@@ -294,6 +294,7 @@ async function resolveConfigSecretRefsForRuntime(input: {
companyId: string;
config: Record<string, unknown>;
schema: Record<string, unknown> | null;
readOnlyInspection?: boolean;
context: {
consumerId: string;
issueId?: string | null;
@@ -313,7 +314,10 @@ async function resolveConfigSecretRefsForRuntime(input: {
nextConfig = writeConfigValueAtPath(
nextConfig,
path,
await secrets.resolveSecretValue(input.companyId, trimmed, "latest", {
input.readOnlyInspection ? await secrets.resolveSecretValueForReadOnlyEnvironmentInspection(input.companyId, trimmed, "latest", {
environmentId: input.context.consumerId, configPath: path,
issueId: input.context.issueId, heartbeatRunId: input.context.heartbeatRunId,
}) : await secrets.resolveSecretValue(input.companyId, trimmed, "latest", {
consumerType: "environment",
consumerId: input.context.consumerId,
actorType: "system",
@@ -659,6 +663,7 @@ export async function resolveEnvironmentDriverConfigForRuntime(
// configuration and tooling so the test reflects what real agent runs use.
applyCustomImageTemplate?: boolean;
readOnlyCustomImageTemplate?: boolean;
readOnlyInspection?: boolean;
onResolvedCustomImageTemplate?: (identity: { id: string; capturedAt: string | null; templateKind: string; bindingField: string }) => void;
},
): Promise<ParsedEnvironmentConfig> {
@@ -704,6 +709,7 @@ export async function resolveEnvironmentDriverConfigForRuntime(
companyId,
config: parsed.config as Record<string, unknown>,
schema,
readOnlyInspection: context?.readOnlyInspection,
context: {
consumerId: environmentId!,
issueId: context?.issueId ?? null,
@@ -1,4 +1,5 @@
import { freshWorkspaceSchema } from "@paperclipai/plugin-sdk";
import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRepositoryPath } from "./remote-workspaces.js";
/**
* Centralized environment run orchestrator.
*
@@ -412,6 +413,7 @@ export function environmentRunOrchestrator(
effectiveExecutionWorkspaceMode: string | null;
persistedExecutionWorkspace: ExecutionWorkspace | null;
runnerTaskProjectIds?: readonly string[];
assertOwned?: () => Promise<void>;
/**
* The host duplex observability recorder for this run. The orchestrator threads
* it to `resolveEnvironmentExecutionTarget`, which stamps it on the sandbox
@@ -429,6 +431,51 @@ export function environmentRunOrchestrator(
effectiveExecutionWorkspaceMode,
} = input;
let { lease, persistedExecutionWorkspace } = input;
const remoteAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata);
if (remoteAuthority && persistedExecutionWorkspace) {
if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") {
throw new EnvironmentRunError("unsupported_adapter_environment", "Remote workspace storage requires a qualified native sandbox");
}
const target = await resolveEnvironmentExecutionTarget({
db, companyId, adapterType, environment, leaseId: lease.id, leaseMetadata: lease.metadata,
lease, environmentRuntime, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null,
});
if (!target || target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) {
throw new EnvironmentRunError("transport_resolution_failed", "Remote workspace storage requires a sandbox file transport");
}
const prepared = await prepareRemoteWorkspace({
db, workspace: persistedExecutionWorkspace, runId: heartbeatRunId, lease, target,
source: remoteAuthority.source,
assertOwned: input.assertOwned,
provisionCommand: persistedExecutionWorkspace.config?.provisionCommand,
runtimeProvisionCommand: persistedExecutionWorkspace.config?.runtimeProvisionCommand,
});
lease = prepared.lease;
const workspaceRealization = {
version: 1, mode: "in_place", authoritativeRoot: prepared.remoteRoot,
pathAliases: [], outboundRestorePaths: [], provider: lease.provider,
environmentId: environment.id, leaseId: lease.id, providerLeaseId: lease.providerLeaseId,
additional: (remoteAuthority.source.repositories ?? []).filter(repository => repository.readOnly)
.map(repository => ({ path: remoteWorkspaceRepositoryPath(prepared.remoteRoot, repository),
projectId: repository.projectId ?? null, projectWorkspaceId: repository.projectWorkspaceId ?? null,
repoUrl: repository.repoUrl, repoRef: repository.ref })),
};
const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, {
...lease.metadata, remoteCwd: prepared.remoteRoot, workspaceRealization,
});
if (!updatedLease) throw new Error("Remote workspace lease disappeared during preparation");
lease = updatedLease;
persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(companyId, prepared.workspace.id);
if (!persistedExecutionWorkspace) throw new Error("Remote workspace disappeared during preparation");
input.executionWorkspace.cwd = prepared.remoteRoot;
const executionTarget: AdapterExecutionTarget = {
...target, remoteCwd: prepared.remoteRoot,
workspaceRealization: { mode: "in_place", authoritativeRoot: prepared.remoteRoot,
pathAliases: [], outboundRestorePaths: [] },
};
return { lease, workspaceRealization, executionTarget,
remoteExecution: adapterExecutionTargetToRemoteSpec(executionTarget), persistedExecutionWorkspace };
}
if (executionWorkspace.providerBinding) {
const binding = executionWorkspace.providerBinding;
+26 -3
View File
@@ -1268,6 +1268,7 @@ function createSandboxEnvironmentDriver(
db: Db,
options: {
pluginWorkerManager?: PluginWorkerManager;
readOnlyInspection?: boolean;
pluginWorkerReadyTimeoutMs?: number;
pluginWorkerReadyPollMs?: number;
deferredOrphanCleanupBufferLimit?: number;
@@ -1498,7 +1499,7 @@ function createSandboxEnvironmentDriver(
id: input.environment.id,
driver: "sandbox",
config: sandboxConfigForLeaseMetadata(metadataConfig),
});
}, { readOnlyInspection: options.readOnlyInspection });
if (parsed.driver === "sandbox") {
return dropInternalPluginSandboxConfigKeys(parsed.config as unknown as Record<string, unknown>);
}
@@ -1510,16 +1511,19 @@ function createSandboxEnvironmentDriver(
db,
input.lease.companyId,
input.environment,
{ readOnlyInspection: options.readOnlyInspection },
);
if (parsed.driver === "sandbox" && parsed.config.provider === input.provider) {
return dropInternalPluginSandboxConfigKeys(parsed.config as unknown as Record<string, unknown>);
}
} catch {
} catch (error) {
if (options.readOnlyInspection) throw error;
// Lease metadata below is intentionally kept sufficient for cleanup
// after the environment config changes or becomes invalid.
}
}
if (options.readOnlyInspection) throw new Error("Inspection requires a validated sandbox configuration.");
return {
provider: input.provider,
...dropInternalPluginSandboxConfigKeys(input.lease.metadata),
@@ -3586,6 +3590,7 @@ export function environmentRuntimeService(
options: {
drivers?: EnvironmentRuntimeDriver[];
pluginWorkerManager?: PluginWorkerManager;
readOnlyInspection?: boolean;
pluginWorkerReadyTimeoutMs?: number;
pluginWorkerReadyPollMs?: number;
deferredOrphanCleanupBufferLimit?: number;
@@ -3602,6 +3607,7 @@ export function environmentRuntimeService(
createSshEnvironmentDriver(db),
createSandboxEnvironmentDriver(db, {
pluginWorkerManager: options.pluginWorkerManager,
readOnlyInspection: options.readOnlyInspection,
pluginWorkerReadyTimeoutMs: options.pluginWorkerReadyTimeoutMs,
pluginWorkerReadyPollMs: options.pluginWorkerReadyPollMs,
deferredOrphanCleanupBufferLimit: options.deferredOrphanCleanupBufferLimit,
@@ -3647,6 +3653,7 @@ export function environmentRuntimeService(
scopeCondition?: ReturnType<typeof and>;
runId?: string;
failureReason: string;
anyLeasePolicy?: boolean;
}): Promise<EnvironmentLease | null> {
if (await nativeWorkspaceExportPendingForLease(db, input.leaseRow)) return null;
if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.leaseRow))) return null;
@@ -3671,7 +3678,7 @@ export function environmentRuntimeService(
eq(environmentLeases.companyId, input.leaseRow.companyId),
eq(environmentLeases.environmentId, input.environment.id),
input.runId ? eq(environmentLeases.heartbeatRunId, input.runId)
: eq(environmentLeases.leasePolicy, "reuse_by_environment"),
: input.anyLeasePolicy ? undefined : eq(environmentLeases.leasePolicy, "reuse_by_environment"),
eq(environmentLeases.status, input.leaseRow.status),
input.scopeCondition,
sql`coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true'`,
@@ -3732,6 +3739,22 @@ export function environmentRuntimeService(
return {
getDriver,
async destroyRemoteWorkspaceLease(input: {
companyId: string; workspaceId: string; leaseId: string; environmentId: string; providerLeaseId: string;
}): Promise<EnvironmentLease | null> {
const scope = and(eq(environmentLeases.executionWorkspaceId, input.workspaceId),
eq(environmentLeases.providerLeaseId, input.providerLeaseId));
const [leaseRow] = await db.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, input.leaseId),
eq(environmentLeases.environmentId, input.environmentId), scope,
));
if (!leaseRow) throw new Error("remote_workspace_cleanup_lease_changed");
const environment = await environmentsSvc.getById(input.environmentId);
if (!environment || environment.driver !== "sandbox") throw new Error("remote_workspace_cleanup_environment_unavailable");
return destroyClaimedSandboxLease({ environment, leaseRow, scopeCondition: scope, anyLeasePolicy: true,
failureReason: "execution_workspace_closed" });
},
/**
* Read the sandbox duplex bridge kill switch for a new run. The host calls it
* per run before it selects the callback bridge transport. It reads the
+15 -2
View File
@@ -1,4 +1,5 @@
import { createReusableLeaseHandoff, preserveReusableLeaseHandoff, withoutReusableLeaseHandoff } from "./environment-lease-handoff.js";
import { clearRemoteWorkspaceLocationForReleasedLease } from "./remote-workspace-snapshot-retention.js";
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js";
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
@@ -7,7 +8,7 @@ import { withoutCleanupVerification } from "./cleanup-verification-proof.js";
import { randomUUID } from "node:crypto";
import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "./environment-custom-image-runtime.js";
import { and, desc, eq, inArray, isNull, ne, sql, type SQL } from "drizzle-orm";
import { and, asc, desc, eq, inArray, isNull, ne, sql, type SQL } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import {
agents,
@@ -1695,6 +1696,16 @@ export function environmentService(db: Db) {
const cleanupSettled = options?.expectedPendingCleanupAttemptId && options.cleanupStatus === "success"
&& status !== "pending_cleanup";
const row = await db.transaction(async tx => {
if (options?.remoteExecutionTermination?.state === "destroyed") {
// Workspace preparation locks its authority before the lease. Keep
// that order when confirmed destruction clears the same authority.
const [identity] = await tx.select({ companyId: environmentLeases.companyId }).from(environmentLeases)
.where(eq(environmentLeases.id, id));
if (identity) await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, identity.companyId),
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->'location'->>'leaseId' = ${id}`,
)).orderBy(asc(executionWorkspaces.id)).for("update");
}
const condition = and(eq(environmentLeases.id, id), options?.expectedPendingCleanupAttemptId
? and(eq(environmentLeases.status, "pending_cleanup"),
sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${options.expectedPendingCleanupAttemptId}`)
@@ -1724,7 +1735,7 @@ export function environmentService(db: Db) {
.where(and(eq(sandboxCreationOperations.id, previous.creationOperationId), eq(sandboxCreationOperations.companyId, previous.companyId),
eq(sandboxCreationOperations.leaseId, id), eq(sandboxCreationOperations.state, "cleanup_pending")));
}
return tx.update(environmentLeases)
const released = await tx.update(environmentLeases)
.set({
status,
releasedAt: status === "retained" ? null : now,
@@ -1741,6 +1752,8 @@ export function environmentService(db: Db) {
.where(condition)
.returning()
.then((rows) => rows[0] ?? null);
if (released) await clearRemoteWorkspaceLocationForReleasedLease({ db: tx, lease: toEnvironmentLease(released) });
return released;
});
return row ? toEnvironmentLease(row) : null;
},
@@ -20,7 +20,8 @@ export async function notifyExecutionRunWork(transaction: object, scope?: Execut
await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
}
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors,
DELIVERY_QUEUES.remoteWorkspaceSnapshots]) {
await notifyDeliveryWork(transaction, queue);
}
}
@@ -1,4 +1,7 @@
import { measureRunTransaction } from "./execution-transaction-timing.js";
import { prepareRemoteWorkspaceRepository, readRemoteWorkspaceAuthority } from "./remote-workspaces.js";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import type { EnvironmentLease } from "@paperclipai/shared";
import { beginRunPhase, measureRunPhase } from "./execution-preparation-timing.js";
import { createHash, randomUUID } from "node:crypto";
import { execFile as execFileCallback } from "node:child_process";
@@ -196,9 +199,12 @@ export function executionWorkspaceRepositoryService(db: Db) {
return measureRunPhase("workspace.repository.binding", async () => {
const [issue] = await measureRunPhase("workspace.repository.db.issue", () => reader.select().from(issues).where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId))));
const [workspace] = await measureRunPhase("workspace.repository.db.workspace", () => reader.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, input.workspaceId), eq(executionWorkspaces.companyId, input.companyId))));
const remoteAuthority = readRemoteWorkspaceAuthority(workspace?.metadata);
if (!issue || !workspace || issue.executionWorkspaceId !== workspace.id || !workspace.cwd
|| (verifyDirectory && await measureRunPhase("workspace.repository.paths", async () =>
await canonicalRepositoryRoot(workspace.cwd!) !== await canonicalRepositoryRoot(input.cwd)))) {
|| (verifyDirectory && (remoteAuthority
? workspace.cwd !== input.cwd || remoteAuthority.writerRunId !== input.runId
: await measureRunPhase("workspace.repository.paths", async () =>
await canonicalRepositoryRoot(workspace.cwd!) !== await canonicalRepositoryRoot(input.cwd))))) {
throw forbidden("Task repository publication binding is unavailable");
}
const acquisitionRestricted = await requiresAnonymousAcquisition(reader, { issue, workspace, agentId: input.agentId, runId: input.runId });
@@ -318,7 +324,8 @@ export function executionWorkspaceRepositoryService(db: Db) {
if (pin && pin !== candidatePin) throw conflict("Shared repository publication receipts disagree; repair is required before reuse");
return candidatePin;
}
async function prepareForAdmission(input: { companyId: string; issueId: string; workspaceId: string; cwd: string; agentId: string; runId: string; responsibleUserId: string | null }) {
async function prepareForAdmission(input: { companyId: string; issueId: string; workspaceId: string; cwd: string; agentId: string; runId: string; responsibleUserId: string | null;
remoteTarget?: AdapterExecutionTarget | null; remoteLease?: EnvironmentLease; assertOwned?: () => Promise<void> }) {
const { acquisitionRestricted, workspace } = await publicationBinding(db, input, false);
const rows = await list(input.companyId, input.workspaceId);
if (!rows.length) return [];
@@ -326,6 +333,54 @@ export function executionWorkspaceRepositoryService(db: Db) {
if (acquisitionRestricted && rows.some(row => row.state !== "ready" || !row.pinnedCommit || !/^[a-f0-9]{40,64}$/.test(row.pinnedCommit))) {
throw forbidden(controllerAcquisitionMessage);
}
if (readRemoteWorkspaceAuthority(workspace.metadata)) {
if (!input.remoteTarget || !input.remoteLease) throw conflict("Remote repository preparation requires its bound sandbox");
const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board");
const resolveAuth = createGitRemoteAuthProvider(db, input.companyId, { issueId: input.issueId, heartbeatRunId: input.runId,
agentId: input.agentId, responsibleUserId: input.responsibleUserId });
const prepared: Array<{ id: string; cwd: string; repoUrl: string; relativePath: string; pinnedCommit: string; branchName: string | null }> = [];
for (const row of rows) {
assertAcquisitionReceipt(row, acquisitionRestricted);
const source = resolveAcquisitionSource(row.catalogRepositoryId ? { kind: "catalog", id: row.catalogRepositoryId }
: { kind: "url", url: row.repoUrl }, available, acquisitionRestricted);
if (source.repositoryIdentity !== row.repositoryIdentity || row.relativePath !== taskRepositoryRelativePath(row.repositoryIdentity)) {
throw conflict("Repository preparation receipt does not match its source");
}
const verify = async () => {
await input.assertOwned?.();
await publicationBinding(db, input);
if (!acquisitionRestricted) await assertCurrentControllerAcquisition(input);
if (source.catalogRepositoryId) {
const current = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board");
const authorized = resolveAcquisitionSource({ kind: "catalog", id: source.catalogRepositoryId }, current, false);
if (authorized.repositoryIdentity !== row.repositoryIdentity) throw conflict("Repository catalog authorization changed");
}
};
await verify();
const anonymous = buildGitAuthInvocation({ token: "", source: "managed_connection", secretName: null });
anonymous.env = { ...anonymous.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_SYSTEM: "/dev/null" };
try {
const result = await prepareRemoteWorkspaceRepository({
db, workspace, runId: input.runId, lease: input.remoteLease, target: input.remoteTarget,
repository: { id: row.id, relativePath: row.relativePath, repoUrl: row.repoUrl,
ref: row.requestedRef === "HEAD" ? null : row.requestedRef,
...(row.pinnedCommit ? { pinnedCommit: row.pinnedCommit } : {}) },
assertOwned: verify, verifyOnly: acquisitionRestricted,
resolveGitAuth: source.catalogRepositoryId ? resolveAuth : async () => anonymous,
});
await verify();
await db.update(executionWorkspaceRepositories).set({ state: "ready", pinnedCommit: result.pinnedCommit,
branchName: result.branchName, failureCode: null, updatedAt: new Date() })
.where(and(eq(executionWorkspaceRepositories.id, row.id), eq(executionWorkspaceRepositories.companyId, input.companyId)));
prepared.push({ ...result, id: row.id, repoUrl: row.repoUrl, relativePath: row.relativePath });
} catch (error) {
await db.update(executionWorkspaceRepositories).set({ state: "needs_repair", failureCode: "repository_preparation_failed", updatedAt: new Date() })
.where(and(eq(executionWorkspaceRepositories.id, row.id), eq(executionWorkspaceRepositories.companyId, input.companyId)));
throw error;
}
}
return prepared;
}
const root = await measureRunPhase("workspace.repository.paths", () => fs.realpath(input.cwd));
const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board");
const resolveAuth = createGitRemoteAuthProvider(db, input.companyId, { issueId: input.issueId, heartbeatRunId: input.runId, agentId: input.agentId, responsibleUserId: input.responsibleUserId });
+90 -11
View File
@@ -1,5 +1,7 @@
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { cleanupRemoteWorkspaceAllocation, inspectRemoteWorkspacePersistence, type DestroyRemoteWorkspaceLease } from "./remote-workspace-lifecycle.js";
import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, pluginWorkspaceSupportsMergeAncestry, type WorkspaceWorkers } from "./plugin-workspaces.js";
import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
import { measureRunTransaction } from "./execution-transaction-timing.js";
@@ -264,12 +266,25 @@ function preserveWorkspaceMetadataForPatch(
current: typeof executionWorkspaces.$inferSelect, patch: Partial<typeof executionWorkspaces.$inferInsert>,
) {
const nextPatch = { ...patch };
if (readRemoteWorkspaceAuthority(current.metadata)) {
for (const key of workspaceSourceKeys) {
if (patch[key] !== undefined && patch[key] !== current[key]) throw conflict("Remote workspace coordinates belong to its sandbox storage");
}
}
const remoteAuthorityKeys = ["remoteWorkspace", "remoteWorkspaceMigration", "remoteWorkspaceRecoverySourceRunIds", "remoteWorkspaceRecoveredFiles"] as const;
if (patch.metadata !== undefined && remoteAuthorityKeys.some(key => current.metadata?.[key] !== undefined || patch.metadata?.[key] !== undefined)) {
nextPatch.metadata = { ...patch.metadata };
for (const key of remoteAuthorityKeys) {
if (current.metadata?.[key] !== undefined) nextPatch.metadata[key] = current.metadata[key];
else delete nextPatch.metadata[key];
}
}
if (current.providerType === "plugin") {
for (const key of workspaceSourceKeys) {
if (patch[key] !== undefined && patch[key] !== current[key]) throw conflict("Managed workspace coordinates belong to its storage plugin");
}
if (patch.metadata !== undefined) {
nextPatch.metadata = { ...patch.metadata };
nextPatch.metadata = { ...nextPatch.metadata };
for (const key of ["workspaceProvider", "workspaceProviderOwner", "workspaceProviderSource", "workspaceProviderBaseSha", "workspaceProviderRemoved", "workspaceProviderBranchMode", "workspaceProviderRemoval", "workspaceProviderRetired", "workspaceProviderRestoration", "lifecycleGeneration", "reopenPendingConsumption", "reopenPendingConsumptionSince"] as const) {
if (current.metadata?.[key] !== undefined) nextPatch.metadata[key] = current.metadata[key];
else delete nextPatch.metadata[key];
@@ -313,6 +328,7 @@ export type ReopenClosedIsolatedExecutionWorkspaceResult =
export type ExecutionWorkspaceServiceOptions = {
pluginWorkerManager?: WorkspaceWorkers;
destroyRemoteWorkspaceLease?: DestroyRemoteWorkspaceLease;
gitCacheExpirationEnabled?: boolean;
gitBaseExpirationEnabled?: boolean;
gitCacheExpirationDryRun?: boolean;
@@ -1424,7 +1440,14 @@ const WORKSPACE_COMPACTION_TRASH_STALE_MS = 60 * 60 * 1000;
const WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS = 30 * 60 * 1000;
export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServiceOptions = {}) {
const destroyRemoteLease: DestroyRemoteWorkspaceLease = input => opts.destroyRemoteWorkspaceLease
? opts.destroyRemoteWorkspaceLease(input)
: import("./environment-runtime.js").then(({ environmentRuntimeService }) => environmentRuntimeService(db).destroyRemoteWorkspaceLease(input));
async function inspectWorkspace(workspace: ExecutionWorkspace) {
if (readRemoteWorkspaceAuthority(workspace.metadata)) {
const state = await inspectRemoteWorkspacePersistence(db, workspace);
return { statusInspectionSucceeded: state.ready, warnings: state.reason ? [state.reason] : [], git: null };
}
if (workspace.providerType !== "plugin") return (opts.inspectGitCloseReadiness ?? inspectGitCloseReadiness)(workspace);
const [row] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, workspace.id), eq(executionWorkspaces.companyId, workspace.companyId)));
if (!row) throw notFound("Workspace is unavailable");
@@ -2439,6 +2462,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
async function runTerminalWorkspaceCleanup(workspace: ExecutionWorkspaceRow, expectedHeadSha: string | null) {
if (readRemoteWorkspaceAuthority(workspace.metadata)) {
if (await workspaceHasActiveRun(workspace)) throw conflict("Workspace still has active execution");
return cleanupRemoteWorkspaceAllocation({ db, workspace, destroyLease: destroyRemoteLease });
}
if (workspace.providerType === "plugin") {
if (await workspaceHasActiveRun(workspace)) throw conflict("Workspace still has active execution");
await removePluginWorkspace(db, opts.pluginWorkerManager, workspace, expectedHeadSha);
@@ -3020,6 +3047,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
return toExecutionWorkspace(row, (runtimeServices?.get(row.id) ?? []).map(toRuntimeService));
},
cleanupRemoteStorage: async (workspace: ExecutionWorkspace) => cleanupRemoteWorkspaceAllocation({
db, workspace, destroyLease: destroyRemoteLease,
}),
getById: async (id: string) => {
const row = await db
.select()
@@ -3116,8 +3147,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
const { deliveryState } = await assessDelivery(workspace, git);
const warnings = [...gitWarnings];
const blockingReasons: string[] = [];
const remoteWorkspace = readRemoteWorkspaceAuthority(workspace.metadata);
if (!statusInspectionSucceeded) {
blockingReasons.push("Paperclip could not verify the workspace git status. Retry before destructive cleanup.");
blockingReasons.push(remoteWorkspace ? gitWarnings[0] ?? "The remote workspace save is incomplete."
: "Paperclip could not verify the workspace git status. Retry before destructive cleanup.");
}
const isSharedWorkspace = executionWorkspace.mode === "shared_workspace";
const workspacePath = readNullableString(executionWorkspace.providerRef) ?? readNullableString(executionWorkspace.cwd);
@@ -3148,7 +3181,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
}
if (isSharedWorkspace) {
if (isSharedWorkspace && !remoteWorkspace) {
warnings.push("This shared workspace session points at project workspace infrastructure. Archiving it only removes the session record.");
}
@@ -3230,12 +3263,12 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
},
];
for (const action of configuredCleanupCommands) {
if (!action.command || executionWorkspace.providerType === "plugin") continue;
if (!action.command || executionWorkspace.providerType === "plugin" || remoteWorkspace) continue;
plannedActions.push(action);
}
const teardownCommand = config?.teardownCommand ?? projectPolicy?.workspaceStrategy?.teardownCommand ?? null;
if (teardownCommand && executionWorkspace.providerType !== "plugin") {
if (teardownCommand && executionWorkspace.providerType !== "plugin" && !remoteWorkspace) {
plannedActions.push({
kind: "teardown_command",
label: "Run teardown command",
@@ -3248,6 +3281,13 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
plannedActions.push({ kind: "remove_provider_workspace", label: "Remove provider workspace",
description: "The storage plugin will verify remote recovery and remove the workspace.", command: null });
}
if (remoteWorkspace) {
plannedActions.push({ kind: "remove_provider_workspace", label: "Release remote workspace compute",
description: "Release the sandbox and keep the saved workspace files.", command: null });
if (configuredCleanupCommands.some(action => action.command) || teardownCommand) {
blockingReasons.push("Remote workspace close does not support custom cleanup or teardown commands.");
}
}
if (executionWorkspace.providerType === "git_worktree" && workspacePath) {
plannedActions.push({
@@ -3258,7 +3298,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
});
}
if (executionWorkspace.providerType !== "plugin" && git?.createdByRuntime && executionWorkspace.branchName) {
if (executionWorkspace.providerType !== "plugin" && !remoteWorkspace && git?.createdByRuntime && executionWorkspace.branchName) {
plannedActions.push({
kind: "git_branch_delete",
label: "Delete runtime-created branch",
@@ -3347,6 +3387,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
or(and(inArray(executionWorkspaces.status, ["active", "idle", "in_review"]), isNull(executionWorkspaces.closedAt)),
and(eq(executionWorkspaces.providerType, "plugin"), inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
sql`(${executionWorkspaces.metadata}->>'workspaceProviderRemoved') IS DISTINCT FROM 'true'`),
and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->>'schema' = 'paperclip.remote-workspace.v1'`,
sql`jsonb_typeof(${executionWorkspaces.metadata}->'remoteWorkspace'->'location') = 'object'`),
opts.gitCacheExpirationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
sql`${executionWorkspaces.metadata}->'gitCacheExpiration'->>'completed' = 'false'`) : undefined,
opts.nestedCloneMigrationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
@@ -3493,7 +3536,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
if (isClosedExecutionWorkspaceStatus(workspace.status) && !hasPendingGitExpiration(workspace)) continue;
}
if (opts.nestedCloneMigrationEnabled && workspace.providerType !== "plugin"
if (opts.nestedCloneMigrationEnabled && workspace.providerType !== "plugin" && !readRemoteWorkspaceAuthority(workspace.metadata)
&& (isNestedCloneMigrationCandidate(workspace) || hasPendingNestedCloneMigration(workspace))) {
const nestedOnly = isClosedExecutionWorkspaceStatus(workspace.status) && !hasPendingGitExpiration(workspace)
&& !hasPendingIsolatedRepositoryRehomeCleanup(workspace);
@@ -3551,7 +3594,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
);
if (reopenPending) terminalWorkspaceDeadlines.set(workspace.id, now().getTime());
const treeOpen = issueTree.some(issue => !TERMINAL_ISSUE_STATUSES.has(issue.status));
const gitExpirationCandidate = workspace.providerType !== "plugin" && Boolean(opts.gitCacheExpirationEnabled)
const remoteWorkspace = readRemoteWorkspaceAuthority(workspace.metadata);
const gitExpirationCandidate = workspace.providerType !== "plugin" && !remoteWorkspace && Boolean(opts.gitCacheExpirationEnabled)
&& (hasPendingGitExpiration(workspace) || (workspace.mode === "isolated_workspace" && workspace.strategyType === "git_worktree"));
const usingIssueIds = sourceIssue && gitExpirationCandidate && !reopenPending
&& TERMINAL_ISSUE_STATUSES.has(sourceIssue.status)
@@ -3566,8 +3610,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
});
}
const idleMs = opts.gitRestorePoints?.idleMs;
const pendingExpiration = opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin" && hasPendingGitExpiration(workspace);
if (pendingExpiration || (sourceIssue && idleMs && !reopenPending && opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin"
const pendingExpiration = opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin" && !remoteWorkspace && hasPendingGitExpiration(workspace);
if (pendingExpiration || (sourceIssue && idleMs && !reopenPending && opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin" && !remoteWorkspace
&& !isClosedExecutionWorkspaceStatus(workspace.status)
&& workspace.mode === "isolated_workspace" && workspace.strategyType === "git_worktree")) {
const lastActivityAt = pendingExpiration ? null : await gitWorkspaceLastActivityAt(db, workspace);
@@ -3667,6 +3711,21 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
else result.skippedActiveRun += 1;
continue;
}
if (remoteWorkspace && isClosedExecutionWorkspaceStatus(workspace.status)) {
const generation = readExecutionWorkspaceLifecycleGeneration(workspace.metadata);
try {
const cleanup = await cleanupTerminalWorkspace(workspace, null, generation);
if (cleanup.skippedReopened) result.skippedReopened += 1;
else if (!cleanup.cleaned) result.cleanupFailed += 1;
else { result.archived += 1; terminalWorkspaceDeadlines.delete(workspace.id); }
} catch (error) {
result.cleanupFailed += 1;
terminalWorkspaceDeadlines.set(workspace.id, now().getTime() + TERMINAL_WORKSPACE_RETRY_MS);
await markTerminalCleanupFailedFenced({ workspaceId: workspace.id, capturedGeneration: generation,
cleanupReason: `${ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON} | ${error instanceof Error ? error.message : String(error)}` });
}
continue;
}
let deliveryReferences: GitHubPullRequestReference[] | null = null;
if (workspace.providerType === "plugin" && !isClosedExecutionWorkspaceStatus(workspace.status)
&& !reopenPending && !workspace.metadata?.workspaceProviderRemoval && !workspace.metadata?.workspaceProviderRestoration) {
@@ -3727,6 +3786,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
const { git, statusInspectionSucceeded } = await inspection;
if (!statusInspectionSucceeded) {
if (remoteWorkspace) terminalWorkspaceDeadlines.delete(workspace.id);
result.skippedUndelivered += 1;
continue;
}
@@ -3755,6 +3815,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
assessment.deliveryState !== "merged_via_pr"
&& assessment.deliveryState !== "merged_by_ancestry"
) {
if (remoteWorkspace) terminalWorkspaceDeadlines.delete(workspace.id);
result.skippedUndelivered += 1;
continue;
}
@@ -3978,7 +4039,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
else result.archived += 1;
} catch (error) {
result.cleanupFailed += 1;
if (archived.providerType === "plugin") terminalWorkspaceDeadlines.set(archived.id, now().getTime() + TERMINAL_WORKSPACE_RETRY_MS);
if (archived.providerType === "plugin" || remoteWorkspace) terminalWorkspaceDeadlines.set(archived.id, now().getTime() + TERMINAL_WORKSPACE_RETRY_MS);
const failure = error instanceof Error ? error.message : String(error);
// Mark cleanup_failed only while the row is still closed at the
// generation this sweep captured. A reopen that raced after the failure
@@ -4112,6 +4173,24 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
if (!freshSnapshot || freshSnapshot.sourceIssueId !== sourceIssueId) {
return { ok: false, code: "not_reopenable", message: "Execution workspace source changed during reopen" };
}
if (readRemoteWorkspaceAuthority(freshSnapshot.metadata)) {
return withExecutionWorkspaceLifecycleLock(db, freshSnapshot.id, async tx => {
const [row] = await tx.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, issue.companyId), eq(executionWorkspaces.id, freshSnapshot.id)));
if (!row || row.sourceIssueId !== sourceIssueId) return { ok: false, code: "not_reopenable", message: "Workspace source changed" };
await assertTaskWorkspaceExecutionAccess(tx, input.authorizationActor, issue.companyId, { issueId: issue.id, workspaceId: row.id });
const persistence = await inspectRemoteWorkspacePersistence(tx, row);
if (!persistence.ready) return { ok: false, code: "rebuild_failed", message: persistence.reason ?? "Saved workspace is unavailable" };
const generation = readExecutionWorkspaceLifecycleGeneration(row.metadata);
if (!isClosedExecutionWorkspaceStatus(row.status)) return { ok: true, reopened: false, workspace: toExecutionWorkspace(row), generation };
const [active] = await tx.update(executionWorkspaces).set({ status: "active", closedAt: null,
cleanupReason: null, cleanupEligibleAt: null, lastUsedAt: now(), updatedAt: now(),
metadata: setMetadataReopenPendingConsumption(bumpExecutionWorkspaceLifecycleGeneration(row.metadata), now()),
}).where(eq(executionWorkspaces.id, row.id)).returning();
await notifyTerminalWorkspaceWork(tx, `workspace:${row.id}`);
return { ok: true, reopened: true, workspace: toExecutionWorkspace(active!), generation: generation + 1 };
});
}
const snapshot = freshSnapshot.providerType === "plugin" ? freshSnapshot : await rehomeExpiredIsolatedRestoration(db, freshSnapshot);
if (snapshot.providerType === "plugin") {
return withExecutionWorkspaceLifecycleLock(db, snapshot.id, async tx => {
@@ -0,0 +1,153 @@
import { asc, and, eq } from "drizzle-orm";
import { projectWorkspaces, type Db } from "@paperclipai/db";
import type { TaskWorkspaceIntent } from "@paperclipai/shared";
import { assertTaskWorkspaceExecutionAccess } from "../task-workspace-access.js";
import { managedProjectCheckoutName, projectRepositoryIdentity } from "../managed-project-checkout-name.js";
import { issueService } from "../issues.js";
import { projectService } from "../projects.js";
import { authorizationService } from "../authorization.js";
import { readRemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "../remote-workspaces.js";
import { isMultiProjectWorkspaceSyncEnabled, isMultiProjectWorkspaceSyncRemoteEnabled,
resolveRunReferencedProjects, WorkspaceValidationFailure, type ResolvedWorkspaceForRun } from "./workspaces.js";
import type { HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";
function remoteRepositorySource(repoUrl: string): boolean {
const value = repoUrl.trim();
if (!value || /[\x00-\x1f\x7f]/.test(value)) throw new WorkspaceValidationFailure(
"The repository source is invalid.", { reason: "remote_workspace_source_invalid" });
if (/^[a-z]:/i.test(value) || /^file:/i.test(value)) return false;
try {
const parsed = new URL(value);
if (["https:", "http:", "ssh:", "git:", "git+ssh:"].includes(parsed.protocol) && parsed.hostname) return true;
} catch { /* Git also accepts scp-style SSH sources and local paths. */ }
if (!value.includes("://") && /^(?:[^@:/\s]+@)?(?:\[[a-f\d:]+\]|[a-z\d.-]+):[^:\s][^\r\n]*$/i.test(value)) return true;
if (!/^[a-z][a-z\d+.-]*:/i.test(value)) return false;
throw new WorkspaceValidationFailure("The repository source is invalid.", { reason: "remote_workspace_source_invalid" });
}
type RemoteWorkspaceSourceResolution = { authority: "remote"; source: RemoteWorkspaceSource;
projectId: string | null; projectWorkspaceId: string | null; warnings?: string[];
referencedProjectFailures?: ResolvedWorkspaceForRun["referencedProjectFailures"] } | { authority: "host" };
/** Resolve an authorized source without materializing any controller directory. */
export async function resolveRemoteWorkspaceSource(db: Db, input: {
companyId: string;
issueId: string;
agentId: string;
runId: string;
responsibleUserId: string | null;
projectId: string | null;
projectWorkspaceId: string | null;
selection?: TaskWorkspaceIntent | null;
existingWorkspace?: HeartbeatWorkspacePreparationInput["reuse"]["existingExecutionWorkspace"];
taskDirectory?: boolean;
configuredCwd?: string | null;
automaticEnrollment?: boolean;
ref?: string | null;
branchName?: string | null;
branchMode?: "create" | "existing";
}): Promise<RemoteWorkspaceSourceResolution> {
const selected = await assertTaskWorkspaceExecutionAccess(db, {
type: "agent", source: "agent_jwt", companyId: input.companyId,
agentId: input.agentId, runId: input.runId,
onBehalfOfUserId: input.responsibleUserId === "local-board" ? null : input.responsibleUserId,
}, input.companyId, { issueId: input.issueId,
...(!input.selection && input.existingWorkspace ? { workspaceId: input.existingWorkspace.id } : {}),
...(input.selection ? { persistedSelection: input.selection.selection } : {}),
});
const hostSource = (): { authority: "host" } => {
if (input.automaticEnrollment) return { authority: "host" };
throw new WorkspaceValidationFailure("Import local repository folders before using them as remote workspace sources.",
{ reason: "remote_workspace_local_source_requires_import" });
};
const withReferences = async (base: RemoteWorkspaceSource, anchorProjectId: string | null,
sourceId: string | null, savedSource = false): Promise<RemoteWorkspaceSourceResolution> => {
const extra: NonNullable<RemoteWorkspaceSource["repositories"]> =
(base.repositories ?? []).filter(repository => !repository.readOnly);
// A verified saved archive owns these files after import. Its original Git
// source remains an identity and need not be reachable from the new sandbox.
if (!savedSource) {
if (base.kind === "git" && !remoteRepositorySource(base.repoUrl)) return hostSource();
for (const repository of extra) if (!remoteRepositorySource(repository.repoUrl)) return hostSource();
}
const referenced = isMultiProjectWorkspaceSyncEnabled() ? await resolveRunReferencedProjects(input.issueId, anchorProjectId, {
companyId: input.companyId,
actor: { type: "agent", source: "agent_jwt", companyId: input.companyId,
agentId: input.agentId, runId: input.runId,
onBehalfOfUserId: input.responsibleUserId === "local-board" ? null : input.responsibleUserId },
issues: issueService(db), projects: projectService(db), access: authorizationService(db),
}) : { additional: [], warnings: [], failures: [] };
if (referenced.additional.length > 0) {
if (!isMultiProjectWorkspaceSyncRemoteEnabled()) throw new WorkspaceValidationFailure(
"Referenced remote workspace access is disabled.", { reason: "remote_referenced_workspace_disabled" });
// Native sandbox providers currently run with external sandbox authority.
// They cannot enforce a separate read-only filesystem root for a project.
throw new WorkspaceValidationFailure("This remote runtime cannot safely provide read-only project folders.",
{ reason: "remote_workspace_readonly_sources_unsupported" });
}
return { authority: "remote", source: { ...base, ...(extra.length ? { repositories: extra } : { repositories: undefined }) },
projectId: anchorProjectId, projectWorkspaceId: sourceId,
warnings: referenced.warnings, referencedProjectFailures: referenced.failures };
};
const existing = readRemoteWorkspaceAuthority(input.existingWorkspace?.metadata);
if (existing) return withReferences(existing.source, input.existingWorkspace!.projectId,
input.existingWorkspace!.projectWorkspaceId, existing.snapshotId !== null);
if (input.existingWorkspace && input.automaticEnrollment) return { authority: "host" };
if (input.existingWorkspace) throw new WorkspaceValidationFailure(
"The existing workspace must be imported before it can use remote workspace storage.",
{ reason: "remote_workspace_migration_required" });
if (input.taskDirectory || input.selection?.selection.kind === "task_directory") {
return withReferences({ kind: "empty" }, null, null);
}
if (input.configuredCwd) return hostSource();
const projectId = selected.task.projectId;
const sourceId = input.selection?.selection.kind === "configured_source"
? input.selection.selection.projectWorkspaceId : selected.source?.id ?? selected.task.projectWorkspaceId;
const [source] = sourceId ? await db.select().from(projectWorkspaces).where(and(
eq(projectWorkspaces.id, sourceId), eq(projectWorkspaces.companyId, input.companyId),
)) : projectId ? await db.select().from(projectWorkspaces).where(and(
eq(projectWorkspaces.projectId, projectId), eq(projectWorkspaces.companyId, input.companyId),
)).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)).limit(1) : [];
if (!source) {
if (sourceId || projectId) throw new WorkspaceValidationFailure(
"The selected project has no repository source for a remote workspace.",
{ reason: "remote_workspace_source_unavailable" });
return withReferences({ kind: "empty" }, null, null);
}
// Recheck authorization against the concrete configured source; the earliest
// project default may have been selected after the initial task read.
await assertTaskWorkspaceExecutionAccess(db, {
type: "agent", source: "agent_jwt", companyId: input.companyId,
agentId: input.agentId, runId: input.runId,
onBehalfOfUserId: input.responsibleUserId === "local-board" ? null : input.responsibleUserId,
}, input.companyId, { issueId: input.issueId,
...(selected.source ? { projectWorkspaceId: source.id } : {}),
});
if (source.sourceType === "remote_managed") return hostSource();
if (source.cwd && source.cwd !== "/__paperclip_repo_only__") return hostSource();
if (!source.repoUrl) throw new WorkspaceValidationFailure(
"The selected workspace has no repository URL.", { reason: "remote_workspace_source_unavailable" });
if (!remoteRepositorySource(source.repoUrl)) return hostSource();
const sameProject = await db.select().from(projectWorkspaces).where(and(
eq(projectWorkspaces.companyId, input.companyId), eq(projectWorkspaces.projectId, source.projectId),
)).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id));
const repositories: NonNullable<RemoteWorkspaceSource["repositories"]> = [];
const identities = new Set([projectRepositoryIdentity(source.repoUrl)]);
for (const repository of sameProject) {
if (!repository.repoUrl || identities.has(projectRepositoryIdentity(repository.repoUrl))) continue;
if (repository.sourceType === "remote_managed") return hostSource();
if (repository.cwd && repository.cwd !== "/__paperclip_repo_only__") return hostSource();
if (!remoteRepositorySource(repository.repoUrl)) return hostSource();
identities.add(projectRepositoryIdentity(repository.repoUrl));
repositories.push({ id: repository.id, relativePath: `.paperclip-repositories/${managedProjectCheckoutName({
id: repository.id, repoUrl: repository.repoUrl, repoRef: repository.repoRef,
})}`, repoUrl: repository.repoUrl, ref: repository.repoRef, projectId: source.projectId,
projectWorkspaceId: repository.id });
}
return withReferences({ kind: "git", repoUrl: source.repoUrl,
ref: input.ref ?? source.repoRef ?? source.defaultRef ?? null,
...(input.branchName ? { branchName: input.branchName } : {}),
...(input.branchMode ? { branchMode: input.branchMode } : {}),
...(repositories.length ? { repositories } : {}),
}, source.projectId, source.id);
}
@@ -2,6 +2,8 @@ import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-c
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js";
import { environmentService } from "../environments.js";
import { writeExecutionControl } from "../execution-control-notifications.js";
import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js";
import { AsyncLocalStorage } from "node:async_hooks";
@@ -235,6 +237,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
const { issueRef, issueContext, context, issueId, taskKey, executionContinuation } = input.task;
const { nativeRuntimeResolution, nativeExecution, nativeRunnerInstanceId, adapter, runtimeForAdapter, getNativeFreshSessionHandoff, getFreshSessionHandoff, runOptions } = input.runtime;
const { persistedExecutionWorkspace, executionWorkspace, executionTarget, workspaceOperationRecorder, nativeWorkspaceSync, workspaceRestoreSource, remoteExecution } = input.workspace;
const remoteWorkspaceAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata);
const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, observability, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config;
const { runtimeSessionParamsForAdapter, configuredModel, sessionConfigMetadata, goalCheckpointSession } = input.session;
const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions;
@@ -290,7 +293,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
pluginWorkspaceFinalization = workspaceRecord
? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id, run.id)
: null;
if (pluginWorkspaceFinalization || workspaceRecord?.strategyType !== "git_worktree") return null;
if (pluginWorkspaceFinalization || readRemoteWorkspaceAuthority(workspaceRecord?.metadata)
|| workspaceRecord?.strategyType !== "git_worktree") return null;
const worktreePath =
readNonEmptyString(workspaceRecord.providerRef) ??
@@ -960,6 +964,19 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
// finalize row.
const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => {
try {
if (remoteWorkspaceAuthority && persistedExecutionWorkspace) {
if (!ownership || !executionTarget) throw new Error("remote_workspace_ownership_required");
const current = await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id);
const authority = readRemoteWorkspaceAuthority(current?.metadata);
const lease = authority?.location
? await environmentService(db).getLeaseById(authority.location.leaseId) : null;
if (!current || !lease || lease.companyId !== run.companyId || lease.heartbeatRunId !== run.id) {
throw new Error("workspace_sync_out_unrecoverable");
}
await recordWorkspaceFinalize("succeeded", { owningService: "native_workspace_finalizer", storage: "remote" },
async () => { await saveRemoteWorkspace({ db, workspace: current, runId: run.id, lease,
target: executionTarget, assertOwned: ownership.assertHeld }); });
}
if (nativeWorkspaceSync) {
const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, run.companyId),
@@ -994,7 +1011,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
db,
runId: run.id,
workspaceFinalizeStatus,
preserveProviderAttempt: Boolean(nativeWorkspaceSync),
preserveProviderAttempt: Boolean(nativeWorkspaceSync || remoteWorkspaceAuthority),
}));
await dispatchPendingNativeStatusWakeups({
companyId: run.companyId,
@@ -1017,7 +1034,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
throw error;
}
};
if (nativeWorkspaceSync) {
if (nativeWorkspaceSync || remoteWorkspaceAuthority) {
const owned = await measureRunPhase("execution.workspace_ownership", () => withNativeWorkspaceFinalizationOwnership({
db, companyId: run.companyId, runId: run.id,
}, completeWorkspace));
@@ -1051,7 +1068,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
input.effects.onNativeOwnershipHeld();
throw adapterErr;
}
if (nativeRuntimeResolution.kind === "native" && nativeWorkspaceSync?.reference.state === "prepared"
if (nativeRuntimeResolution.kind === "native" && (nativeWorkspaceSync?.reference.state === "prepared" || remoteWorkspaceAuthority?.dirty)
&& !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) {
input.effects.onProviderResourceDisposition("stop_and_retain");
}
@@ -1144,7 +1161,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
.where(eq(nativeRunFinalizations.runId, run.id))
.limit(1)
.then((rows) => rows[0]?.resultId ?? null);
if (proposedResult && nativeWorkspaceSync) {
if (proposedResult && (nativeWorkspaceSync || remoteWorkspaceAuthority)) {
const workspaceFailureMessage =
adapterErr instanceof Error ? adapterErr.message : "";
const unrecoverable =
@@ -1,4 +1,5 @@
import { museRunnerBroker } from "../muse-runner-broker.js";
import { readRemoteWorkspaceAuthority } from "../remote-workspaces.js";
import { measureRunTransaction } from "../execution-transaction-timing.js";
import { measureRunPhase } from "../execution-preparation-timing.js";
import {
@@ -118,7 +119,7 @@ export interface HeartbeatRuntimeSelectionInput {
getFreshSessionHandoff: (() => Promise<string | null>) | undefined;
};
workspace: {
persistedExecutionWorkspace: { id: string } | null;
persistedExecutionWorkspace: { id: string; metadata?: Record<string, unknown> | null } | null;
executionWorkspace: Omit<Parameters<typeof buildNativeExecutionInput>[0]["workspace"], "id">;
executionTarget: Parameters<typeof prepareNativeWorkspaceSync>[0]["target"];
activeEnvironmentLease: Pick<Parameters<typeof prepareNativeWorkspaceSync>[0], "lease">;
@@ -1047,7 +1048,8 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
});
if (!nativeSelected) return { selected: false };
stopControllerLease();
nativeWorkspaceSync = await measureRunPhase("runtime_selection.workspace_sync", () => prepareNativeWorkspaceSync({
nativeWorkspaceSync = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata) ? null
: await measureRunPhase("runtime_selection.workspace_sync", () => prepareNativeWorkspaceSync({
db,
runId: run.id,
companyId: agent.companyId,
@@ -120,10 +120,33 @@ postgresTests("upgrade readiness persisted execution evidence", () => {
expect((await readUpgradeReadiness(db, input)).ready).toBe(false);
await db.update(nativeRunFinalizations).set({ ...lease, failureCode: null }).where(eq(nativeRunFinalizations.runId, runId));
}
// Remote-authoritative runs retain their logical workspace identity without a host sync record.
const remoteWorkspace = { schema: "paperclip.remote-workspace-run.v1", workspaceId: workspace.workspaceId,
leaseId: workspace.leaseId, providerLeaseId: workspace.providerLeaseId, remoteCwd: workspace.remoteCwd,
epoch: 1, state: "prepared", snapshotId: null };
const remoteProfile = { remoteWorkspace };
await db.update(heartbeatRuns).set({ runnerProfileJson: remoteProfile }).where(eq(heartbeatRuns.id, runId));
expect(await readUpgradeReadiness(db, input)).toMatchObject({ ready: true, recoverableRuns: 1, blockingRuns: 0 });
for (const remote of [{ ...remoteWorkspace, state: "saved", snapshotId: randomUUID() },
remoteWorkspace]) {
await db.update(heartbeatRuns).set({ runnerProfileJson: { remoteWorkspace: remote,
nativeWorkspaceSync: { ...workspace, workspaceId: randomUUID() } } }).where(eq(heartbeatRuns.id, runId));
expect((await readUpgradeReadiness(db, input)).ready).toBe(true); // remote identity takes precedence
}
for (const remote of [null, {}, { ...remoteWorkspace, schema: "unknown" }, { ...remoteWorkspace, epoch: 0 },
{ ...remoteWorkspace, workspaceId: randomUUID() }, { ...remoteWorkspace, leaseId: "invalid" },
{ ...remoteWorkspace, remoteCwd: "" }, { ...remoteWorkspace, snapshotId: "invalid" }]) {
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceSync: workspace,
remoteWorkspace: remote } }).where(eq(heartbeatRuns.id, runId));
expect((await readUpgradeReadiness(db, input)).ready).toBe(false); // no fallback to the legacy identity
}
await db.update(heartbeatRuns).set({ runnerProfileJson: execution.runnerProfileJson }).where(eq(heartbeatRuns.id, runId));
// Plugin task admission owns an environment lease identity and deliberately has no workspace-sync record.
const pluginLeaseId = randomUUID();
await db.insert(environmentLeases).values({ id: pluginLeaseId, companyId, heartbeatRunId: runId, issueId,
provider: "plugin", providerLeaseId: "synthetic-task", metadata: { agentId, runnerTaskLease: true, runnerTaskDispatched: true } });
await db.update(heartbeatRuns).set({ runnerProfileJson: remoteProfile }).where(eq(heartbeatRuns.id, runId));
expect((await readUpgradeReadiness(db, input)).ready).toBe(false); // contradictory task authorities cannot be adopted
await db.update(heartbeatRuns).set({ runnerProfileJson: {} }).where(eq(heartbeatRuns.id, runId));
expect((await readUpgradeReadiness(db, input)).ready).toBe(false); // workspace identity is not plugin task authority
const pluginHeader = JSON.stringify({ ...JSON.parse(header), state: { ...JSON.parse(header).state,
@@ -1,5 +1,6 @@
import type { Db } from "@paperclipai/db";
import { sql } from "drizzle-orm";
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runner-checkpoint.js";
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
@@ -23,10 +24,12 @@ export async function readUpgradeReadiness(db: Db, input: {
const owner = await currentNativeControllerIdentity();
const rows = await db.transaction(async tx => {
await tx.execute(sql`set local statement_timeout = '2s'`);
return tx.execute<Candidate & { companyId: string; issueId: string; runnerId: string; sessionId: string; workspace: unknown; pluginLeaseId: string | null; material: Record<string, unknown> | null }>(sql`
return tx.execute<Candidate & { companyId: string; issueId: string; runnerId: string; sessionId: string; workspace: unknown; remoteWorkspace: unknown; hasRemoteWorkspace: boolean; pluginLeaseId: string | null; material: Record<string, unknown> | null }>(sql`
select r.id, r.company_id as "companyId", r.native_issue_id as "issueId",
r.runner_instance_id as "runnerId", r.native_session_id as "sessionId",
r.runner_profile_json->'nativeWorkspaceSync' as workspace, p.id as "pluginLeaseId",
r.runner_profile_json->'nativeWorkspaceSync' as workspace,
r.runner_profile_json->'remoteWorkspace' as "remoteWorkspace",
r.runner_profile_json ? 'remoteWorkspace' as "hasRemoteWorkspace", p.id as "pluginLeaseId",
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608 then c.material else null end as material, (
r.runtime_mode = 'native' and a.adapter_type = 'paperclip_runner'
and r.runner_instance_id is not null and r.native_session_id is not null
@@ -54,12 +57,17 @@ export async function readUpgradeReadiness(db: Db, input: {
limit ${MAX_UPGRADE_RUNS + 1}`);
}, { accessMode: "read only" });
const candidates = rows.map(row => {
const workspace = readNativeWorkspaceSyncReference(row.workspace);
const remote = remoteWorkspaceRunReferenceSchema.safeParse(row.remoteWorkspace);
// A remote reference owns the logical workspace identity. Corruption or conflicting
// plugin authority must not fall back to a different recoverable-looking record.
const workspaceId = row.hasRemoteWorkspace
? remote.success && !row.pluginLeaseId ? remote.data.workspaceId : null
: row.pluginLeaseId ?? readNativeWorkspaceSyncReference(row.workspace)?.workspaceId;
const identity = row.recoverable && row.material
? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.id }, row.material) : null;
return { id: row.id, recoverable: Boolean((row.pluginLeaseId || workspace) && identity && identity.runId === row.id
return { id: row.id, recoverable: Boolean(workspaceId && identity && identity.runId === row.id
&& identity.runnerInstanceId === row.runnerId && identity.normalizedSessionId === row.sessionId
&& identity.environmentLeaseId === (row.pluginLeaseId ?? workspace?.workspaceId)) };
&& identity.environmentLeaseId === workspaceId) };
});
// Additive evidence does not strengthen or redefine the existing ready flag.
const retained = await readNativeControllerRecoveryInventory(db).catch(() => ({
@@ -7,9 +7,9 @@ import { pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { captureDirectorySnapshot, directorySnapshotSha256, serializeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, plugins, projects, projectWorkspaces, type Db } from "@paperclipai/db";
import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, type Db } from "@paperclipai/db";
import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { heartbeatService } from "../heartbeat.js";
@@ -25,12 +25,14 @@ import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } fr
import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js";
import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js";
import * as managedRepositoryCheckout from "../managed-repository-checkout.js";
import * as freshWorkspaceSeed from "../fresh-workspace-seed.js";
import { requestTaskRetry } from "../task-retry.js";
import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js";
import { createSystemRecovery } from "../system-recovery-state.js";
import { isolatedRemoteSourcePath } from "../isolated-remote-workspace-source.js";
import { cleanupExecutionWorkspaceArtifacts } from "../workspace-runtime.js";
import { buildProjectMentionHref } from "@paperclipai/shared";
import { buildProjectMentionHref, readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { migrateRemoteWorkspace } from "../remote-workspace-migration.js";
import { trace } from "@opentelemetry/api";
import { effectiveProjectWorkspacePolicy } from "../project-workspace-provider.js";
import { resolveExecutionWorkspaceMode, buildExecutionWorkspaceAdapterConfig } from "../execution-workspace-policy.js";
@@ -38,6 +40,8 @@ import { createLocalDiskStorageProvider } from "../../storage/local-disk-provide
import { createStorageService } from "../../storage/service.js";
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
import { selectHeartbeatEnvironment } from "./environment-selection.js";
import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js";
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
const execFile = promisify(execFileCallback);
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_PREPARATION_TEST_DATABASE_URL");
@@ -47,6 +51,10 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: Db;
let home: string;
const persistenceKeys = ["DATABASE_URL", "PAPERCLIP_STORAGE_PROVIDER", "PAPERCLIP_SECRETS_MASTER_KEY", "PAPERCLIP_SECRETS_MASTER_KEY_FILE"];
let persistenceBefore = new Map<string, string | undefined>();
const configureRemotePersistence = () => Object.assign(process.env, { DATABASE_URL: database.connectionString,
PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_SECRETS_MASTER_KEY: "remote-workspace-preparation-fixture" });
beforeAll(async () => {
home = await mkdtemp(join(tmpdir(), "paperclip-workspace-preparation-"));
@@ -57,6 +65,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
: await startEmbeddedPostgresTestDatabase("paperclip-workspace-preparation");
db = createDb(database.connectionString);
await db.execute(sql`set client_min_messages = warning`);
if (external) await db.execute(sql`truncate table environments restart identity cascade`);
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
afterAll(async () => {
await db?.$client.end();
@@ -65,11 +74,18 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
if (home) await rm(home, { recursive: true, force: true });
});
beforeEach(async () => {
persistenceBefore = new Map(persistenceKeys.map(key => [key, process.env[key]]));
vi.restoreAllMocks();
await db.execute(sql`truncate table companies, plugins restart identity cascade`);
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
});
afterEach(() => {
for (const [key, value] of persistenceBefore) {
if (value === undefined) delete process.env[key]; else process.env[key] = value;
}
});
async function fixture(isolated = false) {
const [company] = await db.insert(companies).values({ name: "Workspace preparation", issuePrefix: "WSP" }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Workspace agent", status: "running", adapterType: "process" }).returning();
@@ -108,6 +124,464 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
};
return { input, source, issue, run, envOrchestrator };
}
it.each(["unconfigured", "operator-disabled"] as const)("auto-enrollment keeps %s native sandboxes on their existing host path", async mode => {
const { input, source, envOrchestrator } = await fixture();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
if (mode === "operator-disabled") {
configureRemotePersistence();
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false;
} else {
Object.assign(process.env, { DATABASE_URL: database.connectionString, PAPERCLIP_STORAGE_PROVIDER: "local_disk" });
}
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("host authority retained"));
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("host authority retained");
expect(resolve).toHaveBeenCalledOnce(); expect(acquire).not.toHaveBeenCalled();
expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
});
it("auto-enrollment preserves an existing host workspace even when persistence is ready", async () => {
const { input, source, issue, envOrchestrator } = await fixture();
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: issue.companyId,
sourceIssueId: issue.id, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
name: "Existing host workspace", cwd: source, providerType: "local_fs", strategyType: "project_primary",
mode: "shared_workspace", status: "active" }).returning();
await writeFile(join(source, "unpublished.txt"), "keep existing work\n");
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(workspace);
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(workspace);
input.reuse.requestedShouldReuseExisting = true;
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
configureRemotePersistence();
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("host authority retained"));
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("host authority retained");
expect(resolve).toHaveBeenCalledOnce(); expect(acquire).not.toHaveBeenCalled();
expect(await readFile(join(source, "unpublished.txt"), "utf8")).toBe("keep existing work\n");
expect((await db.select().from(executionWorkspaces))[0]?.metadata?.remoteWorkspace).toBeUndefined();
});
it.each([
{ kind: "empty", repoUrl: null, branchMode: null, fresh: false },
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: null, fresh: false },
{ kind: "git", repoUrl: "ssh://git@github.com/example/remote.git", branchMode: null, fresh: false },
{ kind: "git", repoUrl: "git@github.com:example/remote.git", branchMode: null, fresh: false },
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "existing", fresh: false },
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: false },
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true },
] as const)("reserves a $kind remote workspace ($repoUrl, branch: $branchMode, fresh: $fresh) without materializing a controller checkout", async ({ kind, repoUrl, branchMode, fresh }) => {
const { input, source, issue, envOrchestrator } = await fixture();
const [savedEnvironment] = await db.insert(environments).values({ name: `Remote ${randomUUID()}`,
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
const environment = { ...input.environment.localEnvironment, id: savedEnvironment.id,
driver: "sandbox" as const, config: { provider: "daytona" } };
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, input.agent.id));
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = environment;
input.environment.selectedEnvironmentId = environment.id;
expect(input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled).toBe(true);
input.policy.requestedExecutionWorkspaceMode = "agent_default";
input.policy.useIsolatedTaskDirectory = kind === "empty";
if (branchMode) {
input.policy.requestedExecutionWorkspaceMode = "isolated_workspace";
input.policy.effectiveExecutionWorkspaceMode = "isolated_workspace";
input.config.mergedConfig = { workspaceStrategy: { type: "git_worktree", baseRef: "main",
...(branchMode === "existing" ? { existingBranch: "feature/diverged" } : { branchTemplate: "test-{{issue.identifier}}" }) } };
}
if (kind === "git") {
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl, repoRef: "main" })
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
}
if (fresh) {
const sourceRunId = randomUUID(), retryIntentId = randomUUID();
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: issue.companyId, agentId: input.agent.id,
issueId: issue.id, status: "failed", finishedAt: new Date() });
await db.insert(agentWakeupRequests).values({ id: retryIntentId, companyId: issue.companyId,
agentId: input.agent.id, reason: "task_retry", source: "on_demand", status: "coalesced", runId: input.run.id,
payload: { issueId: issue.id } });
const readiness = { mode: "fresh_isolated_workspace" as const, retryIntentId, sourceRunId, retainedSourceIds: [] };
await db.insert(issueRecoveryActions).values({ companyId: issue.companyId, sourceIssueId: issue.id,
kind: "active_run_watchdog", status: "resolved", ownerType: "system", nextAction: "Synthetic committed fresh attempt",
cause: "native_continuation_requires_reconciliation", fingerprint: randomUUID(), evidence: { runId: sourceRunId,
explicitUserContinuation: { runId: input.run.id, readiness }, systemRecovery: {
...createSystemRecovery(sourceRunId, retryIntentId), state: "exhausted", freshAttemptRunId: input.run.id,
} } });
input.task.freshWorkspace = await readFreshWorkspaceReadiness(db, input.run);
expect(input.task.freshWorkspace).toEqual(readiness);
}
await rm(source, { recursive: true });
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("host source resolver called"));
const hostSeed = vi.spyOn(freshWorkspaceSeed, "materializeFreshWorkspaceSeed");
const repositoryPreparation = vi.spyOn(input.services.executionWorkspacesSvc, "prepareTaskRepositoriesForAdmission");
vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
expect(args.persistedExecutionWorkspace).toMatchObject({ providerType: "cloud_sandbox" });
expect(args.persistedExecutionWorkspace).toMatchObject({ cwd: expect.stringMatching(/^\/workspaces\//) });
throw new Error("reached remote acquisition");
});
const configured = { DATABASE_URL: database.connectionString, PAPERCLIP_STORAGE_PROVIDER: "s3",
PAPERCLIP_SECRETS_MASTER_KEY: "remote-workspace-preparation-fixture" };
const previous = new Map(Object.keys(configured).map(key => [key, process.env[key]]));
Object.assign(process.env, configured);
try {
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached remote acquisition");
expect(resolve).not.toHaveBeenCalled();
expect(repositoryPreparation).not.toHaveBeenCalled();
expect(hostSeed).not.toHaveBeenCalled();
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.sourceIssueId, issue.id));
expect(workspace.mode).toBe(branchMode ? "isolated_workspace" : "shared_workspace");
expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { kind }, writerRunId: input.run.id });
if (branchMode) expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: {
ref: "main", branchMode, branchName: branchMode === "existing" ? "feature/diverged" : "test-WSP-1",
} });
await expect(access(workspace.cwd!)).rejects.toThrow();
} finally {
for (const [key, value] of previous) {
if (value === undefined) delete process.env[key]; else process.env[key] = value;
}
}
});
it("rejects automatic Core services before remote reservation or acquisition", async () => {
const { input, envOrchestrator } = await fixture();
configureRemotePersistence();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
input.policy.useIsolatedTaskDirectory = true;
input.config.mergedConfig = { workspaceRuntime: { services: [{ name: "web", command: "node server.js" }] } };
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_runtime_services_unsupported" },
});
expect(acquire).not.toHaveBeenCalled();
expect(resolve).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
});
it("auto-enrollment leaves a configured local folder on the host authority", async () => {
const { input, envOrchestrator } = await fixture();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
configureRemotePersistence();
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("local folder retained"));
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("local folder retained");
expect(resolve).toHaveBeenCalledOnce();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
expect(acquire).not.toHaveBeenCalled();
});
it.each(["configured-folder", "explicit-task-directory", "enrolled-remote"] as const)(
"preserves effective configured cwd authority for %s", async scenario => {
const { input, source, issue, envOrchestrator } = await fixture();
configureRemotePersistence();
await writeFile(join(source, "unpublished.txt"), "configured folder work\n");
await db.update(issues).set({ projectId: null, projectWorkspaceId: null }).where(eq(issues.id, issue.id));
input.task.issueRef = { ...issue, projectId: null, projectWorkspaceId: null };
input.task.context = { issueId: issue.id };
input.config.mergedConfig = { cwd: source };
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, input.agent.id));
input.agent.adapterType = "paperclip_runner";
const [savedEnvironment] = await db.insert(environments).values({ name: `Configured cwd ${randomUUID()}`,
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
id: savedEnvironment.id, driver: "sandbox", config: { provider: "daytona" } };
input.environment.selectedEnvironmentId = savedEnvironment.id;
expect(input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled).toBe(true);
if (scenario === "explicit-task-directory") {
input.task.workspaceSelection = { version: 1, source: "explicit", selection: { kind: "task_directory" } };
await db.update(issues).set({ workspaceSelection: input.task.workspaceSelection }).where(eq(issues.id, issue.id));
input.policy.useIsolatedTaskDirectory = true;
}
let enrolledId: string | undefined;
if (scenario === "enrolled-remote") {
enrolledId = randomUUID();
const [workspace] = await db.insert(executionWorkspaces).values({ id: enrolledId, companyId: issue.companyId,
sourceIssueId: issue.id, name: "Existing remote workspace", cwd: `/workspaces/${enrolledId}`,
providerType: "cloud_sandbox", strategyType: "project_primary", mode: "shared_workspace", status: "active",
metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null,
source: { kind: "empty" }, location: null, writerRunId: null, dirty: false } } }).returning();
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
input.task.issueRef.executionWorkspaceId = workspace.id;
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(workspace);
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(workspace);
input.reuse.requestedShouldReuseExisting = true;
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false;
}
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
const acquire = vi.spyOn(envOrchestrator, "acquireForRun").mockRejectedValue(new Error("configured cwd acquisition boundary"));
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("configured cwd acquisition boundary");
expect(acquire).toHaveBeenCalledOnce();
const [workspace] = await db.select().from(executionWorkspaces);
if (scenario === "configured-folder") {
expect(resolve).toHaveBeenCalledOnce();
expect(workspace).toMatchObject({ providerType: "local_fs", cwd: source });
expect(workspace.metadata?.remoteWorkspace).toBeUndefined();
} else {
expect(resolve).not.toHaveBeenCalled();
expect(workspace).toMatchObject({ providerType: "cloud_sandbox" });
expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { kind: "empty" }, writerRunId: input.run.id });
if (enrolledId) expect(workspace.id).toBe(enrolledId);
}
expect(await readFile(join(source, "unpublished.txt"), "utf8")).toBe("configured folder work\n");
},
);
it.each(["path", "file-url"] as const)("resumes an imported %s Git source from saved authority after its host files disappear", async sourceKind => {
const { input, source, issue, envOrchestrator } = await fixture();
configureRemotePersistence();
process.env.PAPERCLIP_SECRETS_MASTER_KEY = "72".repeat(32);
const upstream = join(home, `import-upstream-${randomUUID()}`);
await execFile("git", ["clone", "--bare", source, upstream]);
const repoUrl = sourceKind === "path" ? upstream : pathToFileURL(upstream).href;
await execFile("git", ["-C", source, "remote", "add", "origin", repoUrl]);
await writeFile(join(source, "unsaved.txt"), "imported untracked work\n");
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.run.id));
const [original] = await db.insert(executionWorkspaces).values({ companyId: issue.companyId,
sourceIssueId: issue.id, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
name: "Imported Git workspace", cwd: source, providerRef: source, repoUrl, baseRef: "main", branchName: "main",
providerType: "local_fs", strategyType: "project_primary", mode: "shared_workspace", status: "active" }).returning();
await db.update(issues).set({ executionWorkspaceId: original.id }).where(eq(issues.id, issue.id));
const storage = createStorageService(createLocalDiskStorageProvider(join(home, `import-objects-${randomUUID()}`)));
await expect(migrateRemoteWorkspace({ db, storage, companyId: issue.companyId, workspaceId: original.id, apply: true }))
.resolves.toMatchObject({ applied: true });
const [imported] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, original.id));
const saved = readRemoteWorkspaceAuthority(imported.metadata)!;
expect(saved).toMatchObject({ source: { kind: "git", repoUrl, ref: "main", branchName: "main" },
snapshotId: expect.any(String), location: null, dirty: false, writerRunId: null });
await rm(source, { recursive: true });
await rm(upstream, { recursive: true });
const [next] = await db.insert(heartbeatRuns).values({ companyId: issue.companyId, agentId: input.agent.id,
issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();
input.run = next;
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, input.agent.id));
input.agent.adapterType = "paperclip_runner";
const [environment] = await db.insert(environments).values({ name: `Imported source ${randomUUID()}`,
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
id: environment.id, driver: "sandbox", config: { provider: "daytona" } };
input.environment.selectedEnvironmentId = environment.id;
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false;
input.task.issueRef = { ...issue, executionWorkspaceId: imported.id };
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(imported);
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(imported);
input.reuse.requestedShouldReuseExisting = true;
input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: imported.id,
issueExecutionWorkspacePreference: null, existingExecutionWorkspaceStatus: imported.status });
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
const hostSeed = vi.spyOn(freshWorkspaceSeed, "materializeFreshWorkspaceSeed");
const acquire = vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
expect(args.persistedExecutionWorkspace).toMatchObject({ id: imported.id, providerType: "cloud_sandbox" });
const [reserved] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, imported.id));
expect(readRemoteWorkspaceAuthority(reserved.metadata)).toMatchObject({
snapshotId: saved.snapshotId, source: saved.source,
});
throw new Error("imported saved authority reached remote acquisition");
});
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("imported saved authority reached remote acquisition");
expect(acquire).toHaveBeenCalledOnce();
expect(resolve).not.toHaveBeenCalled();
expect(hostSeed).not.toHaveBeenCalled();
const [after] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, imported.id));
expect(readRemoteWorkspaceAuthority(after.metadata)).toMatchObject({ snapshotId: saved.snapshotId, source: saved.source });
await expect(access(source)).rejects.toThrow();
await expect(access(upstream)).rejects.toThrow();
});
it.each(["file:///tmp/repository", "/tmp/repository", "../repository", "C:\\work\\repository", "C:repository"])(
"auto-enrollment leaves local repository source %s on the host authority", async repoUrl => {
const { input, issue, envOrchestrator } = await fixture();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl })
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
configureRemotePersistence();
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("local repository retained"));
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("local repository retained");
expect(resolve).toHaveBeenCalledOnce();
expect(acquire).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
},
);
it.each(["https://", "ftp://example.com/repo", "javascript://invalid", "https://example.com/repo\ninvalid"])(
"rejects malformed or unsupported repository source %s without a host fallback", async repoUrl => {
const { input, issue, envOrchestrator } = await fixture();
configureRemotePersistence();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl })
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_source_invalid" },
});
expect(resolve).not.toHaveBeenCalled(); expect(acquire).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
},
);
it("does not turn an unauthorized configured-source selection into a host fallback", async () => {
const { input, envOrchestrator } = await fixture();
configureRemotePersistence();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.task.workspaceSelection = { version: 1, source: "explicit",
selection: { kind: "configured_source", projectWorkspaceId: randomUUID(), mode: "shared" } };
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ status: 403 });
expect(resolve).not.toHaveBeenCalled(); expect(acquire).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
});
it.each(["source", "effective-policy"] as const)("keeps plugin repository ownership via %s out of automatic Core enrollment", async ownership => {
const { input, issue, envOrchestrator } = await fixture();
configureRemotePersistence();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
if (ownership === "effective-policy") input.policy.projectExecutionWorkspacePolicy = {
enabled: true, storageProviderPluginId: randomUUID(), defaultMode: "isolated_workspace",
};
await db.update(projectWorkspaces).set({ cwd: null, sourceType: ownership === "source" ? "remote_managed" : "git", repoUrl: "https://github.com/example/managed.git" })
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("managed ownership retained"));
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("managed ownership retained");
expect(resolve).toHaveBeenCalledOnce(); expect(acquire).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
});
it.each(["shared_workspace", "agent_default"] as const)("rejects concurrent shared writers resolved from %s before resolving a remote source", async requestedMode => {
const { input, envOrchestrator } = await fixture();
configureRemotePersistence();
input.policy.useIsolatedTaskDirectory = true;
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
input.policy.requestedExecutionWorkspaceMode = requestedMode;
input.policy.issueExecutionWorkspaceSettings = { sharedWorkspaceConcurrency: "allow" };
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("one active writer");
expect(acquire).not.toHaveBeenCalled();
});
it.each([false, true])("holds execution while the selected workspace is migrating (remote: %s)", async remote => {
const { input, source, issue, envOrchestrator } = await fixture();
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: issue.companyId,
sourceIssueId: issue.id, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
name: "Migrating workspace", cwd: source, providerType: "local_fs", strategyType: "project_primary",
mode: "shared_workspace", status: "active", metadata: { remoteWorkspaceMigration: { state: "running" } } }).returning();
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
// The admission snapshot may predate the operator starting the import.
input.reuse.existingExecutionWorkspace = toExecutionWorkspace({ ...workspace, metadata: {} });
if (remote) {
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
}
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_migration_in_progress" },
});
expect(resolve).not.toHaveBeenCalled();
expect(acquire).not.toHaveBeenCalled();
expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");
});
it("rejects read-only project references before reserving or acquiring a remote workspace", async () => {
const { input, source, issue, envOrchestrator } = await fixture();
const previous = new Map(["PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE"]
.map(key => [key, process.env[key]]));
vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "true");
vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE", "true");
configureRemotePersistence();
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl: "https://github.com/example/remote.git" })
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
const [reference] = await db.insert(projects).values({ companyId: issue.companyId, name: "Referenced project" }).returning();
await db.insert(projectWorkspaces).values({ companyId: issue.companyId, projectId: reference.id,
name: "Referenced repository", sourceType: "git", repoUrl: "https://github.com/example/reference.git" });
await db.update(issues).set({ description: `[project](${buildProjectMentionHref(reference.id)})` }).where(eq(issues.id, issue.id));
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
try {
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_readonly_sources_unsupported" },
});
expect(acquire).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
expect(await readdir(source)).not.toContain(".paperclip-repositories");
} finally {
for (const [key, value] of previous) {
if (value === undefined) delete process.env[key]; else process.env[key] = value;
}
}
});
it("fences recovered native preparation when its controller claim changes or expires", async () => {
const { input, issue, envOrchestrator } = await fixture();
const controller = await currentNativeControllerIdentity();
const leaseOwner = `recovery:${randomUUID()}`;
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, input.run.id));
input.run.runtimeMode = "native";
configureRemotePersistence();
input.policy.useIsolatedTaskDirectory = true;
input.agent.adapterType = "paperclip_runner";
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
driver: "sandbox", config: { provider: "daytona" } };
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
input.reuse.runOptions = { nativeLeaseOwner: leaseOwner };
await db.insert(nativeRunFinalizations).values({ runId: input.run.id, companyId: issue.companyId,
issueId: issue.id, phase: "preparing", attempt: 2, controllerGeneration: 3, leaseOwner,
leaseExpiresAt: new Date(Date.now() + 60_000), controllerBootId: controller.bootId,
controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt });
const authority = { db, companyId: issue.companyId, runId: input.run.id, leaseOwner,
controllerGeneration: 3, providerAttempt: 2 };
await expect(assertNativeSessionPreparationAuthority(authority)).resolves.toBeUndefined();
await expect(assertNativeSessionPreparationAuthority({ ...authority, controllerGeneration: 2 }))
.rejects.toThrow("native_session_lease_lost");
await db.update(nativeRunFinalizations).set({ leaseOwner: "successor-controller" })
.where(eq(nativeRunFinalizations.runId, input.run.id));
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("native_session_lease_lost");
expect(resolve).not.toHaveBeenCalled();
expect(acquire).not.toHaveBeenCalled();
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
await db.update(nativeRunFinalizations).set({ leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) })
.where(eq(nativeRunFinalizations.runId, input.run.id));
await expect(assertNativeSessionPreparationAuthority(authority)).rejects.toThrow("native_session_lease_lost");
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60_000) })
.where(eq(nativeRunFinalizations.runId, input.run.id));
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, input.run.id));
await expect(assertNativeSessionPreparationAuthority(authority)).rejects.toThrow("native_session_lease_lost");
});
it.each([
{ inherited: false, mode: "isolated_workspace" as const, fresh: false },
{ inherited: false, mode: "isolated_workspace" as const, fresh: true },
@@ -1,3 +1,7 @@
import { readRemoteWorkspaceAuthority, reserveRemoteWorkspace, shouldUseRemoteWorkspaces } from "../remote-workspaces.js";
import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js";
import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js";
import { resolveRemoteWorkspaceSource } from "./remote-workspace-source.js";
import { reserveFreshPluginTaskWorkspace, stageProviderFreshWorkspace } from "../provider-fresh-workspace.js";
import { freshWorkspaceSchema, type PluginFreshWorkspace } from "@paperclipai/plugin-sdk";
import { pluginRegistryService } from "../plugin-registry.js";
@@ -55,6 +59,7 @@ import {
} from "../workspace-instance-cleanup.js";
import { isRuntimeOwnedGitBranch } from "../execution-workspace-branch-ownership.js";
import {
resolveSharedWorkspaceConcurrency,
issueExecutionWorkspaceModeForPersistedWorkspace,
resolveEffectiveWorkspaceStrategyType,
} from "../execution-workspace-policy.js";
@@ -139,7 +144,7 @@ export interface HeartbeatWorkspacePreparationInput {
nativeRecoveryExecutionWorkspaceId: string | null;
persistedNativeExecutionInput: NativeExecutionInput | null;
isDotRun: boolean;
runOptions: { nativeRestartRecovery?: NativeRestartRecoveryClaim };
runOptions: { nativeLeaseOwner?: string; nativeRestartRecovery?: NativeRestartRecoveryClaim };
};
services: {
workspaceWorkers?: WorkspaceWorkers;
@@ -220,6 +225,34 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
appendRunEvent,
} = input.services;
const pluginTaskWorkspace = selectedEnvironmentForConfig?.driver === "plugin" && agent.adapterType === "paperclip_runner";
for (const workspaceId of new Set([existingExecutionWorkspace?.id, reusableExistingExecutionWorkspace?.id].filter(Boolean))) {
const [workspace] = await db.select({ metadata: executionWorkspacesTable.metadata }).from(executionWorkspacesTable)
.where(and(eq(executionWorkspacesTable.companyId, agent.companyId), eq(executionWorkspacesTable.id, workspaceId!)));
if (parseObject(workspace?.metadata?.remoteWorkspaceMigration).state === "running") {
throw new WorkspaceValidationFailure("The workspace is being moved to remote storage. Try again after the move completes.",
{ reason: "remote_workspace_migration_in_progress" });
}
}
const enrolledRemoteAuthority = readRemoteWorkspaceAuthority((reusableExistingExecutionWorkspace ?? existingExecutionWorkspace)?.metadata) !== null;
let remoteAuthority = !isDotRun && shouldUseRemoteWorkspaces({
adapterType: agent.adapterType,
environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
experimentalEnabled: resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled && Boolean(issueId && issueRef)
&& !projectExecutionWorkspacePolicy?.storageProviderPluginId,
existingWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace,
});
const assertRemotePreparationOwned = run.runtimeMode === "native" ? () => assertNativeSessionPreparationAuthority({
db, companyId: agent.companyId, runId: run.id,
leaseOwner: runOptions.nativeLeaseOwner ?? runOptions.nativeRestartRecovery?.leaseOwner,
...(runOptions.nativeRestartRecovery ? {
controllerGeneration: runOptions.nativeRestartRecovery.controllerGeneration,
providerAttempt: runOptions.nativeRestartRecovery.providerAttempt,
} : {}),
}) : () => controllerLease.assertOwned();
if (enrolledRemoteAuthority) await assertRemotePreparationOwned();
if (remoteAuthority && (!issueId || !issueRef)) throw new WorkspaceValidationFailure(
"Remote workspace storage requires a task.", { reason: "remote_workspace_task_required" });
const remoteSourceState: { value: Extract<Awaited<ReturnType<typeof resolveRemoteWorkspaceSource>>, { authority: "remote" }> | null } = { value: null };
const pluginConfig = parseObject(selectedEnvironmentForConfig?.config);
if (fresh && !isDotRun && pluginTaskWorkspace) {
const plugin = typeof pluginConfig.pluginKey === "string" ? await pluginRegistryService(db).getByKey(pluginConfig.pluginKey) : null;
@@ -269,6 +302,46 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
return preflightEnvironment.driver;
},
resolveWorkspace: async () => {
if (remoteAuthority) {
const strategy = parseObject(mergedConfig.workspaceStrategy);
const sourceState = await resolveRemoteWorkspaceSource(db, {
companyId: agent.companyId, issueId: issueId!, agentId: agent.id, runId: run.id,
responsibleUserId, projectId: issueRef!.projectId,
projectWorkspaceId: issueRef!.projectWorkspaceId, selection: input.task.workspaceSelection,
existingWorkspace: fresh ? null : reusableExistingExecutionWorkspace ?? existingExecutionWorkspace,
taskDirectory: useIsolatedTaskDirectory, automaticEnrollment: !enrolledRemoteAuthority,
configuredCwd: readNonEmptyString(mergedConfig.cwd),
ref: readNonEmptyString(strategy.baseRef),
branchMode: readNonEmptyString(strategy.existingBranch) ? "existing" : "create",
branchName: readNonEmptyString(strategy.existingBranch) ??
(resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, mergedConfig) === "git_worktree"
? sanitizeBranchName(renderWorkspaceTemplate(String(strategy.branchTemplate ?? "{{issue.identifier}}-{{slug}}"), {
issue: issueRef, agent, projectId: issueRef!.projectId, repoRef: null,
})) : null),
});
if (sourceState.authority === "host") remoteAuthority = false;
else {
remoteSourceState.value = sourceState;
if (!enrolledRemoteAuthority) await assertRemotePreparationOwned();
if (effectiveExecutionWorkspaceMode === "shared_workspace" && resolveSharedWorkspaceConcurrency({
projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings,
}) === "allow") throw new WorkspaceValidationFailure(
"Remote workspace storage requires one active writer per workspace.", { reason: "remote_workspace_concurrent_writers_unsupported" });
if (Array.isArray(workspaceServices) && workspaceServices.length > 0) throw new WorkspaceValidationFailure(
"Remote workspace storage does not support Core workspace services.", { reason: "remote_workspace_runtime_services_unsupported" });
const source = sourceState.source;
return {
cwd: reusableExistingExecutionWorkspace?.cwd ?? `/workspaces/${run.id}`,
source: "task_session" as const,
projectId: remoteSourceState.value!.projectId, workspaceId: remoteSourceState.value!.projectWorkspaceId,
repoUrl: source.kind === "git" ? source.repoUrl : null,
repoRef: source.kind === "git" ? source.ref : null,
workspaceHints: [], warnings: remoteSourceState.value.warnings ?? [], baseCwdFallback: false,
materializationFailures: [], additionalWorkspaces: [],
referencedProjectFailures: remoteSourceState.value.referencedProjectFailures ?? [],
};
}
}
if (isDotRun) {
// This is private controller storage, never a provider filesystem.
// External providers receive workspace.access=none and cwd=null.
@@ -396,6 +469,18 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
: workspace;
},
}));
const remoteFresh = remoteAuthority && fresh ? await readFreshWorkspaceReadiness(db, run) : null;
if (remoteAuthority && fresh && (remoteFresh?.mode !== "fresh_isolated_workspace"
|| input.task.freshWorkspace?.mode !== "fresh_isolated_workspace"
|| remoteFresh.retryIntentId !== input.task.freshWorkspace.retryIntentId
|| remoteFresh.sourceRunId !== input.task.freshWorkspace.sourceRunId)) {
throw new WorkspaceValidationFailure("The committed fresh workspace recovery authorization changed.",
{ reason: "remote_workspace_fresh_authorization_changed" });
}
const remoteFreshBinding = remoteFresh?.mode === "fresh_isolated_workspace"
? (await db.select({ workspaceId: issues.executionWorkspaceId, revision: issues.workspaceBindingRevision }).from(issues)
.where(and(eq(issues.companyId, agent.companyId), eq(issues.id, issueId!))))[0] : null;
if (remoteFresh && !remoteFreshBinding) throw new Error("remote_workspace_task_binding_missing");
const hostProvisionDriver = selectedEnvironmentForConfig?.driver ?? lowTrustPreflightEnvironmentDriver;
let providerFresh: PluginFreshWorkspace | undefined;
let previousProviderWorkspaceId: string | null = null;
@@ -416,7 +501,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
previousWorkspaceId: previousProviderWorkspaceId });
}
}
const pluginPrepared = isDotRun ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
const pluginPrepared = isDotRun || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId,
issueId, runId: run.id, agentId: agent.id, responsibleUserId,
existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
@@ -439,7 +524,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
source: { projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId,
repoUrl: resolvedWorkspace.repoUrl, baseRef: resolvedWorkspace.repoRef } }) : null;
const freshTaskReservation = freshTaskReservationId ? await executionWorkspacesSvc.getStoredById(agent.companyId, freshTaskReservationId) : null;
const copyOnlyProvisioning = pluginPrepared ? true : await timeWorkspace("workspace_provision_policy", () => workspaceRequiresCopyOnlyProvisioning(db, {
const copyOnlyProvisioning = pluginPrepared || remoteAuthority ? true : await timeWorkspace("workspace_provision_policy", () => workspaceRequiresCopyOnlyProvisioning(db, {
companyId: agent.companyId,
task: issueRef ? { ...issueRef, workspaceSelection: input.task.workspaceSelection } : null,
sourceProjectId: resolvedWorkspace.projectId,
@@ -466,7 +551,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
throw new WorkspaceValidationFailure("The selected plugin environment does not support Core Git worktree preparation.",
{ reason: "plugin_workspace_strategy_unsupported" });
}
if (!pluginPrepared) await timeWorkspace("workspace_base_validation", () => assertGitWorktreeBaseWorkspaceReady({
if (!pluginPrepared && !remoteAuthority) await timeWorkspace("workspace_base_validation", () => assertGitWorktreeBaseWorkspaceReady({
requestedExecutionWorkspaceMode,
config: hostExecutionWorkspaceConfig,
issue: issueRef,
@@ -598,7 +683,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
agentId: agent.id,
},
);
if (freshNeedsSeed && issueId && !pluginPrepared && !pluginTaskWorkspace) {
if (freshNeedsSeed && issueId && !pluginPrepared && !pluginTaskWorkspace && !remoteAuthority) {
await controllerLease.assertOwned();
const verifySourceAccess = async () => {
await assertTaskWorkspaceExecutionAccess(db, {
@@ -629,7 +714,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
warnings: [], created: false, branchCreatedByRuntime: false,
baseRefSha: pluginPrepared.metadata?.workspaceProviderBaseSha as string | null,
} as RealizedExecutionWorkspace, reusedExecutionWorkspace: Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy }
: isDotRun || pluginTaskWorkspace ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: pluginTaskWorkspace && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
: isDotRun || pluginTaskWorkspace || remoteAuthority ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace || remoteAuthority ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: remoteSourceState.value?.source.kind === "git" ? remoteSourceState.value!.source.branchName ?? null : null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: (pluginTaskWorkspace || remoteAuthority) && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
{
requestedShouldReuseExisting,
existingExecutionWorkspaceId:
@@ -733,7 +818,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
}),
},
));
if (input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && issueId) {
if (!remoteAuthority && input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && issueId) {
const recoveredFiles = providerFresh ? providerFresh.recoverySnapshots.map(snapshot => ({
operationId: snapshot.operationId, sourceRunId: snapshot.sourceRunId,
path: `.paperclip-recovered/${snapshot.operationId}`,
@@ -855,7 +940,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
]
: null;
if (
!pluginPrepared && !persistedWorktreeInstanceRoot &&
!pluginPrepared && !remoteAuthority && !persistedWorktreeInstanceRoot &&
executionWorkspace.strategy === "git_worktree" &&
executionWorkspace.worktreePath
) {
@@ -897,7 +982,33 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
pendingForwardBranchReconcile?.recordedBranchName ??
executionWorkspace.branchName;
try {
persistedExecutionWorkspace =
if (remoteAuthority) {
if (!remoteSourceState.value || !issueId) throw new Error("remote_workspace_source_missing");
await assertRemotePreparationOwned();
const reserved = await reserveRemoteWorkspace({
db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id,
environmentId: selectedEnvironmentId,
mode: effectiveExecutionWorkspaceMode === "shared_workspace" ? "shared_workspace"
: effectiveExecutionWorkspaceMode === "operator_branch" ? "operator_branch" : "isolated_workspace",
projectId: remoteSourceState.value!.projectId, projectWorkspaceId: remoteSourceState.value!.projectWorkspaceId,
existingWorkspaceId: reusableExistingExecutionWorkspace?.id ?? existingExecutionWorkspace?.id ?? null,
source: remoteSourceState.value!.source,
...(remoteFresh?.mode === "fresh_isolated_workspace" ? {
fresh: { attemptId: remoteFresh.retryIntentId, previousWorkspaceId: remoteFreshBinding!.workspaceId,
expectedBindingRevision: remoteFreshBinding!.revision },
recoverySourceRunIds: [remoteFresh.sourceRunId],
} : {}),
});
persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(agent.companyId, reserved.id);
if (!persistedExecutionWorkspace?.cwd) throw new Error("remote_workspace_reservation_missing");
persistedExecutionWorkspace = await executionWorkspacesSvc.update(reserved.id, {
metadata: { ...nextExecutionWorkspaceMetadata, ...reserved.metadata,
remoteWorkspace: reserved.metadata?.remoteWorkspace },
});
if (!persistedExecutionWorkspace?.cwd) throw new Error("remote_workspace_reservation_missing");
executionWorkspace.cwd = persistedExecutionWorkspace.cwd;
resolvedWorkspace.cwd = executionWorkspace.cwd;
} else persistedExecutionWorkspace =
pluginPrepared ? await executionWorkspacesSvc.update(pluginPrepared.id, { metadata: nextExecutionWorkspaceMetadata }) :
freshTaskReservation ? await executionWorkspacesSvc.update(freshTaskReservation.id, { metadata: nextExecutionWorkspaceMetadata }) :
resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&
@@ -1047,14 +1158,16 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
}
await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
const projectRepositoryPaths: string[] = [];
if (!pluginPrepared && !pluginTaskWorkspace && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
if (remoteAuthority) projectRepositoryPaths.push(...(remoteSourceState.value?.source.repositories ?? [])
.filter(repository => !repository.readOnly).map(repository => repository.relativePath));
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
const taskRepositories = await timeWorkspace("workspace_repository_admission", () =>
executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId,
issueId, workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd,
agentId: agent.id, runId: run.id, responsibleUserId }));
projectRepositoryPaths.push(...taskRepositories.map(repository => repository.relativePath));
}
if (!pluginPrepared && !pluginTaskWorkspace && executionWorkspace.projectId
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && executionWorkspace.projectId
&& (resolvedWorkspace.source === "project_primary" || Boolean(reusableExistingExecutionWorkspace?.projectWorkspaceId))
&& !resolvedWorkspace.baseCwdFallback && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
const repositoryRows = await db.select().from(projectWorkspaces).where(and(
@@ -1112,9 +1225,13 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
const sandboxRecovery = remoteRecovery && remoteRecovery.remote.kind !== "plugin_task" ? remoteRecovery.remote : null;
const recoveryWorkspace = sandboxRecovery
? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null;
if (remoteRecovery && (remoteRecovery.runId !== run.id || (sandboxRecovery && (!recoveryWorkspace ||
recoveryWorkspace.providerLeaseId !== sandboxRecovery.providerLeaseId ||
recoveryWorkspace.remoteCwd !== sandboxRecovery.remoteCwd)))) {
const remoteRecoveryLocation = remoteAuthority
? readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata)?.location ?? null : null;
if (remoteRecovery && (remoteRecovery.runId !== run.id || (sandboxRecovery && (remoteAuthority
? !remoteRecoveryLocation || remoteRecoveryLocation.providerLeaseId !== sandboxRecovery.providerLeaseId
|| remoteRecoveryLocation.remoteCwd !== sandboxRecovery.remoteCwd
: !recoveryWorkspace || recoveryWorkspace.providerLeaseId !== sandboxRecovery.providerLeaseId
|| recoveryWorkspace.remoteCwd !== sandboxRecovery.remoteCwd)))) {
throw new Error("native_remote_recovery_lease_mismatch");
}
acquiredEnvironment = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({
@@ -1134,8 +1251,8 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
agentId: agent.id,
persistedExecutionWorkspace,
executionWorkspaceSettings: environmentExecutionWorkspaceSettings,
...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : sandboxRecovery && recoveryWorkspace ? { reattachRemoteLease: {
leaseId: recoveryWorkspace.leaseId,
...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : sandboxRecovery && (recoveryWorkspace || remoteRecoveryLocation) ? { reattachRemoteLease: {
leaseId: (remoteRecoveryLocation ?? recoveryWorkspace)!.leaseId,
providerLeaseId: sandboxRecovery.providerLeaseId,
remoteCwd: sandboxRecovery.remoteCwd,
} } : {}),
@@ -1226,6 +1343,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
effectiveExecutionWorkspaceMode,
persistedExecutionWorkspace,
runnerTaskProjectIds: resolvedWorkspace.runnerTaskProjectIds,
assertOwned: remoteAuthority ? assertRemotePreparationOwned : controllerLease.assertOwned,
duplexObservabilityRecorder,
}));
nativeRunnerPreparationSpans.push({
@@ -1260,7 +1378,34 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
const workspaceRealization = realizationResult.workspaceRealization;
const executionTarget = realizationResult.executionTarget;
if (!pluginPrepared && !pluginTaskWorkspace && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId)
if (remoteAuthority && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
const repositories = await timeWorkspace("workspace_repository_admission", () =>
executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId, issueId,
workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd, agentId: agent.id, runId: run.id,
responsibleUserId, remoteTarget: executionTarget, remoteLease: activeEnvironmentLease.lease,
assertOwned: assertRemotePreparationOwned }));
projectRepositoryPaths.push(...repositories.map(repository => repository.relativePath));
persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id);
}
if (remoteFresh?.mode === "fresh_isolated_workspace") {
const recorded = persistedExecutionWorkspace?.metadata?.remoteWorkspaceRecoveredFiles;
const recoveredFiles = (Array.isArray(recorded) ? recorded : []).map(value => {
const file = parseObject(value);
if (typeof file.operationId !== "string" || typeof file.sourceRunId !== "string"
|| file.path !== `.paperclip-recovered/${file.operationId}` || !/^[a-f0-9-]{36}$/.test(file.operationId)) {
throw new Error("remote_workspace_recovered_files_invalid");
}
return { operationId: file.operationId, sourceRunId: file.sourceRunId, path: file.path as string };
});
context.recoveredWorkspaceFiles = recoveredFiles;
const continuation = context.executionContinuation as import("@paperclipai/shared").ExecutionContinuationEnvelope | null;
if (continuation) {
continuation.recoveredFiles = recoveredFiles;
const wake = context.paperclipWake as Record<string, unknown> | undefined;
if (wake) wake.executionContinuation = continuation;
}
}
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId)
&& !(executionTarget?.kind === "remote" && (executionTarget.transport === "ssh"
|| (executionTarget.transport === "sandbox" && executionTarget.runnerTask)))) {
const workspaceId = persistedNativeExecutionInput?.binding.executionWorkspaceId ?? nativeRecoveryExecutionWorkspaceId!;
+6
View File
@@ -249,6 +249,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
enableAiConnectionRouters: parsed.data.enableAiConnectionRouters ?? false,
enableManagedSandboxOnly: parsed.data.enableManagedSandboxOnly ?? false,
enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false,
remoteWorkspaceStorageEnabled: parsed.data.remoteWorkspaceStorageEnabled ?? true,
enableIsolatedWorkspacesByDefault: parsed.data.enableIsolatedWorkspacesByDefault ?? false,
enableStreamlinedLeftNavigation: parsed.data.enableStreamlinedLeftNavigation ?? true,
enableStreamlinedUi: parsed.data.enableStreamlinedUi ?? true,
@@ -303,6 +304,11 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
enableAiConnectionRouters: false,
enableManagedSandboxOnly: false,
enableIsolatedWorkspaces: false,
// An unrelated malformed field must not erase an explicit enrollment opt-out.
remoteWorkspaceStorageEnabled: !(
raw !== null && typeof raw === "object" && !Array.isArray(raw)
&& "remoteWorkspaceStorageEnabled" in raw && raw.remoteWorkspaceStorageEnabled === false
),
enableIsolatedWorkspacesByDefault: false,
enableStreamlinedLeftNavigation: true,
enableStreamlinedUi: true,
@@ -1,6 +1,6 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import type { Db } from "@paperclipai/db";
import { livenessDb, nativeRunFinalizations } from "@paperclipai/db";
import { heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db";
import { and, eq, gt, sql } from "drizzle-orm";
import { currentNativeControllerIdentity, type NativeControllerIdentity } from "./native-restart-recovery.js";
import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js";
@@ -9,6 +9,27 @@ export const NATIVE_SESSION_EXECUTION_LEASE_TTL_MS = 20 * 60_000;
const NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS = 5 * 60_000;
const RENEWAL_STATEMENT_TIMEOUT_MS = 10_000;
/** Preparation can perform provider I/O only under its current recovery claim. */
export async function assertNativeSessionPreparationAuthority(input: {
db: Db; companyId: string; runId: string; leaseOwner: string | undefined;
controllerGeneration?: number; providerAttempt?: number;
}) {
if (!input.leaseOwner) throw new Error("native_session_lease_lost");
const controller = await currentNativeControllerIdentity();
const [owned] = await input.db.select({ runId: nativeRunFinalizations.runId })
.from(nativeRunFinalizations).innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeRunFinalizations.runId))
.where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId),
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.status, "running"),
eq(nativeRunFinalizations.leaseOwner, input.leaseOwner),
eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid),
eq(nativeRunFinalizations.controllerProcessStartedAt, controller.processStartedAt),
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`),
...(input.controllerGeneration === undefined ? [] : [eq(nativeRunFinalizations.controllerGeneration, input.controllerGeneration)]),
...(input.providerAttempt === undefined ? [] : [eq(nativeRunFinalizations.attempt, input.providerAttempt)]),
)).limit(1);
if (!owned) throw new Error("native_session_lease_lost");
}
export async function renewNativeSessionExecutionLease(input: {
db: Db;
runId: string;
@@ -1,3 +1,4 @@
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
import type { CancelledWorkspacePreservationScope } from "../cancelled-workspace-preservation-notifications.js";
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
@@ -13,11 +14,18 @@ import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finaliz
import { hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
import { logger } from "../../middleware/logger.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js";
function workspaceRecoveryReference(profile: Record<string, unknown> | null | undefined) {
const remote = remoteWorkspaceRunReferenceSchema.safeParse(profile?.remoteWorkspace);
return readNativeWorkspaceSyncReference(profile?.nativeWorkspaceSync) ?? (remote.success ? remote.data : null);
}
export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeof heartbeatRuns.$inferSelect,
"id" | "companyId" | "nativeIssueId" | "issueId" | "contextSnapshot" | "runnerProfileJson">,
options: { allowPreterminal?: boolean; requireCancelledStop?: boolean; restartRecovery?: NativeRestartRecoveryClaim } = {}) {
if (readNativeWorkspaceSyncReference(selected.runnerProfileJson?.nativeWorkspaceSync)?.state !== "prepared") return;
if (workspaceRecoveryReference(selected.runnerProfileJson)?.state !== "prepared") return;
const issueId = selected.nativeIssueId ?? selected.issueId ?? selected.contextSnapshot?.issueId;
if (typeof issueId !== "string") return;
return db.transaction(async tx => {
@@ -32,7 +40,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
)).for("update");
if (options.requireCancelledStop && coordinator?.phase !== "terminal_failure") return;
const claim = options.restartRecovery;
const reference = readNativeWorkspaceSyncReference(selected.runnerProfileJson?.nativeWorkspaceSync)!;
const reference = workspaceRecoveryReference(selected.runnerProfileJson)!;
const ownsClaim = Boolean(options.allowPreterminal && coordinator?.phase === "observed" && claim
&& claim.runId === selected.id && claim.leaseOwner === coordinator.leaseOwner
&& claim.controllerGeneration === coordinator.controllerGeneration && claim.providerAttempt === coordinator.attempt
@@ -47,7 +55,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
)).for("update");
if (!run || run.runtimeMode !== "native" || (!options.allowPreterminal && run.status !== "failed")) return;
if (options.requireCancelledStop && (run.status !== "cancelled" || !hasAcknowledgedNativeStopIntent(run))) return;
const currentReference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
const currentReference = workspaceRecoveryReference(run.runnerProfileJson);
if (!currentReference || currentReference.state !== "prepared"
|| (claim?.kind === "reattach_remote_runner" && (claim.remote.kind === "plugin_task" || claim.remote.providerLeaseId !== currentReference.providerLeaseId
|| claim.remote.remoteCwd !== currentReference.remoteCwd))) return;
@@ -95,10 +103,19 @@ export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof
if (!lease.heartbeatRunId) return false;
const [run] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId),
eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "failed"),
inArray(heartbeatRuns.status, ["failed", "cancelled", "timed_out", "interrupted"]),
)).limit(1);
if (!run || hasRequiredWorkspaceRecovery(run.resultJson)) return false;
try {
if (run.runtimeMode === "legacy") {
const parsed = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
if (!parsed.success) return false;
const [source] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId),
eq(environmentLeases.id, parsed.data.leaseId), eq(environmentLeases.heartbeatRunId, run.id))).limit(1);
const retained = source && await preserveRemoteWorkspacePreparationFailure(db, source);
return Boolean(retained && hasRequiredWorkspaceRecovery(retained.resultJson));
}
if (run.status !== "failed") return false;
const retained = await preserveNativeFailedWorkspace(db, run);
return Boolean(retained && hasRequiredWorkspaceRecovery(retained.resultJson));
} catch (error) {
@@ -137,7 +154,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) {
)), eq(heartbeatRuns.runtimeMode, "native"),
eq(heartbeatRuns.status, "cancelled"), eq(nativeRunFinalizations.phase, "terminal_failure"),
isNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner),
sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'`,
sql`(${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')`,
sql`${heartbeatRuns.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged'`,
cursor ? gt(heartbeatRuns.id, cursor) : undefined,
)).orderBy(asc(heartbeatRuns.id)).limit(5);
@@ -163,7 +180,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) {
function pendingCancelledWorkspace() {
return sql`${heartbeatRuns.runtimeMode} = 'native' AND ${heartbeatRuns.status} = 'cancelled'
AND ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'
AND (${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')
AND ${heartbeatRuns.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged'
AND (${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'
AND ${heartbeatRuns.resultJson}->>'workspaceRestoreFailure' IN (${workspaceRestoreFailureCodesSql})) IS NOT TRUE`;
@@ -176,7 +193,7 @@ export function registerCancelledWorkspacePreservationWork(db: Db,
type Candidate = Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "agentId" | "issueId" | "nativeIssueId"
| "runtimeMode" | "driverKind" | "status" | "resultJson" | "runnerProfileJson">;
const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeStopIntent(run)
&& readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync)?.state === "prepared"
&& workspaceRecoveryReference(run.runnerProfileJson)?.state === "prepared"
&& !hasRequiredWorkspaceRecovery(run.resultJson) && !nativeRunHasNoWorkspaceAccess(run);
const scopeCondition = (scope?: CancelledWorkspacePreservationScope) => scope ? and(
eq(heartbeatRuns.companyId, scope.companyId), scope.runId ? eq(heartbeatRuns.id, scope.runId) : undefined) : undefined;
@@ -1,6 +1,7 @@
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
import { notifyUnsafeWorkspaceExportRunChange } from "../unsafe-workspace-export-notifications.js";
@@ -629,12 +630,14 @@ export async function claimNativeRestartRecoveries(input: {
}
const profile = row.run.runnerProfileJson ?? {};
const remoteWorkspace = readNativeWorkspaceSyncReference(profile.nativeWorkspaceSync);
const storedRemoteWorkspace = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace);
const remoteWorkspace = readNativeWorkspaceSyncReference(profile.nativeWorkspaceSync)
?? (storedRemoteWorkspace.success ? storedRemoteWorkspace.data : null);
const [pluginTask] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, row.run.companyId), eq(environmentLeases.heartbeatRunId, row.run.id),
sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`,
)).orderBy(desc(environmentLeases.createdAt)).limit(1);
const remoteSandbox = profile.nativeWorkspaceSync != null;
const remoteSandbox = profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null;
const remoteProcess = remoteSandbox || !!pluginTask;
const durableAuthority = (remoteWorkspace || pluginTask) && await hasNativeRunnerCheckpoint(tx, {
companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id,
@@ -9,19 +9,24 @@ import { and, eq, inArray, isNull, sql } from "drizzle-orm";
import {
agentWakeupRequests,
environmentLeases,
executionWorkspaces,
executionWorkspaceSnapshots,
heartbeatRunEvents,
heartbeatRuns,
issues,
issueRecoveryActions,
nativeRunFinalizations,
toolInvocations,
workspaceOperations,
type Db,
} from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
import { decideNativeReplacement } from "./native-replacement-evidence.js";
import { issueService } from "../issues.js";
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { buildExecutionContinuation } from "../execution-continuation.js";
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js";
export const NATIVE_SAFE_REPLACEMENT_REASON = "native_safe_replacement";
const record = (v: unknown): Record<string, unknown> =>
@@ -38,6 +43,72 @@ const processAlive = (pid: number | null) => {
}
};
type ReplacementDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
/** A saved remote source is proved by its current ledger, never a controller path. */
async function remoteWorkspaceReplacementEvidence(
db: ReplacementDb,
run: typeof heartbeatRuns.$inferSelect,
issueId: string,
leases: Array<typeof environmentLeases.$inferSelect>,
lock = false,
) {
const [task] = await db.select({ workspaceId: issues.executionWorkspaceId }).from(issues).where(and(
eq(issues.companyId, run.companyId), eq(issues.id, issueId),
));
const query = task?.workspaceId ? db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, run.companyId), eq(executionWorkspaces.id, task.workspaceId),
)) : null;
const [workspace] = query ? await (lock ? query.for("update") : query) : [];
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
if (run.runnerProfileJson?.remoteWorkspace == null && !authority) return null;
if (lock) {
const [currentRun] = await db.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id),
)).for("update");
if (!currentRun || currentRun.status !== "failed") return { workspacePreserved: false, providerStopped: false };
run = currentRun;
leases = await db.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
)).for("update");
}
const providerStopped = leases.length > 0 && leases.every(hasRemoteTerminationReceipt);
const denied = { workspacePreserved: false, providerStopped };
const parsed = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
if (!workspace || !authority || !parsed.success || parsed.data.state !== "saved" || !parsed.data.snapshotId
|| run.runnerProfileJson?.nativeWorkspaceSync != null || ["archived", "cleanup_failed"].includes(workspace.status)) return denied;
const reference = parsed.data;
const executionWorkspace = record(record(run.runnerProfileJson?.nativeExecutionInput).workspace);
if (workspace.id !== reference.workspaceId || executionWorkspace.cwd !== reference.remoteCwd
|| authority.dirty || authority.writerRunId || authority.epoch !== reference.epoch
|| authority.snapshotId !== reference.snapshotId) return denied;
const lease = leases.find(row => row.id === reference.leaseId && row.providerLeaseId === reference.providerLeaseId
&& (!row.executionWorkspaceId || row.executionWorkspaceId === workspace.id)
&& row.metadata?.remoteWorkspaceId === workspace.id && row.metadata?.remoteCwd === reference.remoteCwd);
if (!lease || !providerStopped) return denied;
if (authority.location ? authority.location.leaseId !== lease.id || authority.location.environmentId !== lease.environmentId
|| authority.location.providerLeaseId !== reference.providerLeaseId || authority.location.remoteCwd !== reference.remoteCwd
: record(lease.metadata?.remoteExecutionTermination).state !== "destroyed") return denied;
const snapshotQuery = db.select().from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, run.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
eq(executionWorkspaceSnapshots.id, reference.snapshotId!), eq(executionWorkspaceSnapshots.status, "committed"),
eq(executionWorkspaceSnapshots.epoch, reference.epoch), eq(executionWorkspaceSnapshots.sourceRunId, run.id),
));
const [snapshot] = await (lock ? snapshotQuery.for("update") : snapshotQuery);
if (!snapshot?.sha256 || !snapshot.objectKey || !snapshot.key || !snapshot.manifestDigest || !snapshot.byteSize) return denied;
const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
eq(workspaceOperations.id, snapshot.sourceOperationId), eq(workspaceOperations.companyId, run.companyId),
eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.heartbeatRunId, run.id),
eq(workspaceOperations.issueId, issueId), eq(workspaceOperations.phase, "remote_workspace_snapshot"),
eq(workspaceOperations.status, "succeeded"),
sql`${workspaceOperations.metadata}->>'boundary' = 'final'`,
sql`${workspaceOperations.metadata}->>'epoch' = ${String(reference.epoch)}`,
sql`${workspaceOperations.metadata}->>'snapshotId' = ${snapshot.id}`,
sql`${workspaceOperations.metadata}->>'sha256' = ${snapshot.sha256}`,
));
return { workspacePreserved: Boolean(operation), providerStopped };
}
export function nativeReplacementPending() {
return and(
eq(heartbeatRuns.status, "failed"),
@@ -208,6 +279,7 @@ export async function reconcileSafeNativeReplacements(
} catch {
/* The named operator outcome retains incomplete context. */
}
const remoteWorkspace = await remoteWorkspaceReplacementEvidence(db, run, coordinator.issueId, leases);
const decision = decideNativeReplacement({
failedSession: true,
// A facade transport failure can conceal an authoritative protocol
@@ -223,8 +295,9 @@ export async function reconcileSafeNativeReplacements(
coordinator.leaseOwner === null && run.status === "failed",
providerStopped:
!processAlive(run.processPid) &&
!processAlive(run.processGroupId ? -run.processGroupId : null),
workspacePreserved:
!processAlive(run.processGroupId ? -run.processGroupId : null) &&
(remoteWorkspace?.providerStopped ?? true),
workspacePreserved: remoteWorkspace ? remoteWorkspace.workspacePreserved :
typeof workspace.cwd === "string" &&
(await stat(workspace.cwd).then(
(s) => s.isDirectory(),
@@ -333,6 +406,7 @@ export async function reconcileSafeNativeReplacements(
(task.checkoutRunId !== null && task.checkoutRunId !== run.id) ||
!current ||
current.phase !== "terminal_failure" ||
current.leaseOwner !== null ||
current.failureCode !== coordinator.failureCode ||
current.failureDetail?.successorRunId ||
current.failureDetail?.replacementDenied ||
@@ -353,6 +427,10 @@ export async function reconcileSafeNativeReplacements(
});
if (!ownsBlock) return false;
}
if (remoteWorkspace) {
const currentWorkspace = await remoteWorkspaceReplacementEvidence(tx, run, coordinator.issueId, leases, true);
if (!currentWorkspace?.workspacePreserved || !currentWorkspace.providerStopped) return false;
}
if (stoppedSession) {
const [currentRun] = await tx.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
@@ -1,13 +1,91 @@
import { environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { and, eq, exists, sql } from "drizzle-orm";
import { nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js";
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js";
import { recordRequiredWorkspaceRestoreFailure } from "../legacy-execution-recovery.js";
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
type WorkspaceSourceLease = Pick<typeof environmentLeases.$inferSelect,
"id" | "companyId" | "heartbeatRunId" | "provider" | "providerLeaseId">;
/** Preparation publishes file authority before native execution is selected.
* Neither runtimeMode nor a later execution-input projection owns these files. */
async function preparedRemoteWorkspaceSource(db: Db, lease: WorkspaceSourceLease) {
if (!lease.heartbeatRunId || !lease.provider || lease.provider === "local" || !lease.providerLeaseId) return null;
const [bound] = await db.select({ lease: environmentLeases, run: {
id: heartbeatRuns.id, runtimeMode: heartbeatRuns.runtimeMode,
controllerBootId: heartbeatRuns.controllerBootId,
issueId: sql<string | null>`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text,
${heartbeatRuns.contextSnapshot}->>'issueId')`,
reference: sql<unknown>`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'`,
legacyReference: sql<unknown>`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'`,
} }).from(environmentLeases)
.innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId),
eq(heartbeatRuns.companyId, environmentLeases.companyId)))
.where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId),
eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId), eq(environmentLeases.provider, lease.provider),
eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared'`)).limit(1);
if (!bound) return null;
const parsed = remoteWorkspaceRunReferenceSchema.safeParse(bound.run.reference);
if (!parsed.success || parsed.data.state !== "prepared" || bound.run.legacyReference != null) return null;
const reference = parsed.data;
const issueId = bound.run.issueId;
if (typeof issueId !== "string" || bound.lease.issueId !== issueId || reference.leaseId !== bound.lease.id
|| reference.providerLeaseId !== bound.lease.providerLeaseId || reference.remoteCwd !== bound.lease.metadata?.remoteCwd
|| bound.lease.executionWorkspaceId && bound.lease.executionWorkspaceId !== reference.workspaceId) return null;
const [workspace] = await db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, lease.companyId), eq(executionWorkspaces.id, reference.workspaceId))).limit(1);
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
if (!workspace || workspace.sourceIssueId !== issueId && workspace.mode !== "shared_workspace"
|| !authority || !authority.dirty || authority.writerRunId !== bound.run.id || authority.epoch !== reference.epoch
|| authority.snapshotId !== reference.snapshotId || !authority.location
|| authority.location.environmentId !== bound.lease.environmentId || authority.location.leaseId !== bound.lease.id
|| authority.location.providerLeaseId !== bound.lease.providerLeaseId || authority.location.remoteCwd !== reference.remoteCwd) return null;
return { ...bound, workspace: workspace!, reference, issueId };
}
/** File-only recovery for setup that never reached native admission. The live
* preparer must still own its controller; orphan repair accepts terminal runs
* only. This records no provider result and never starts a provider turn. */
export async function preserveRemoteWorkspacePreparationFailure(db: Db, lease: WorkspaceSourceLease,
options: { assertOwned?: () => Promise<void> } = {}) {
const candidate = await preparedRemoteWorkspaceSource(db, lease);
if (!candidate || candidate.run.runtimeMode !== "legacy") return null;
// This callback renews the run on the liveness connection. Finish it before
// locking that row; the captured owner is rechecked under the lock below.
await options.assertOwned?.();
return db.transaction(async tx => {
await tx.execute(sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`);
const [task] = await tx.select({ id: issues.id }).from(issues).where(and(
eq(issues.companyId, lease.companyId), eq(issues.id, candidate.issueId))).for("update");
const [run] = await tx.select().from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, candidate.run.id))).for("update");
if (!task || !run || run.runtimeMode !== "legacy" || run.runnerProfileJson?.nativeExecutionInput != null
|| options.assertOwned && (run.controllerBootId !== candidate.run.controllerBootId
|| run.controllerBootId !== null && (!run.controllerLeaseExpiresAt || run.controllerLeaseExpiresAt.getTime() <= Date.now()))
|| (!options.assertOwned && (!["failed", "cancelled", "timed_out", "interrupted"].includes(run.status)
|| run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt.getTime() > Date.now()))) return null;
const [coordinator] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
.where(and(eq(nativeRunFinalizations.companyId, lease.companyId), eq(nativeRunFinalizations.runId, run.id)));
if (coordinator) return null;
await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, lease.companyId), eq(executionWorkspaces.id, candidate.reference.workspaceId))).for("update");
await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
eq(environmentLeases.companyId, lease.companyId), eq(environmentLeases.id, lease.id))).for("update");
const current = await preparedRemoteWorkspaceSource(tx as unknown as Db, lease);
if (!current || current.issueId !== task.id || current.run.id !== run.id) return null;
if (options.assertOwned && run.controllerBootId !== null
&& (!run.controllerLeaseExpiresAt || run.controllerLeaseExpiresAt.getTime() <= Date.now())) return null;
if (hasRequiredWorkspaceRecovery(run.resultJson)) return run;
return await recordRequiredWorkspaceRestoreFailure(tx as unknown as Db, run,
{ workspaceRestoreFailure: "restore_failed" }, current.lease);
});
}
export async function nativeWorkspaceExportPendingForLease(db: Db, lease: WorkspaceSourceLease): Promise<boolean> {
if (!lease.heartbeatRunId || !lease.provider || lease.provider === "local" || !lease.providerLeaseId) return false;
if (await preparedRemoteWorkspaceSource(db, lease)) return true;
const input = sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'`;
const [run] = await db.select({
id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
@@ -1,4 +1,5 @@
import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js";
import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js";
import { recordNativeFinalizationFailure } from "./native-run-finalizer.js";
import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js";
import type { WorkspaceWorkers } from "../plugin-workspaces.js";
@@ -157,6 +158,42 @@ export async function attemptNativeWorkspaceFinalization(input: {
},
run: async () => {
await ownership.assertHeld();
const remoteAuthority = readRemoteWorkspaceAuthority(workspace?.metadata);
if (remoteAuthority && workspace) {
if (!input.environmentRuntime || !remoteAuthority.location) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
const location = remoteAuthority.location;
const environmentsSvc = environmentService(input.db);
const lease = await environmentsSvc.getLeaseById(location.leaseId);
const environment = lease?.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
if (!lease || !environment || lease.companyId !== bound.companyId
|| lease.heartbeatRunId !== input.runId || lease.executionWorkspaceId !== workspace.id
|| lease.providerLeaseId !== location.providerLeaseId || environment.id !== location.environmentId
|| ["expired", "failed", "pending_cleanup"].includes(lease.status)) {
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
}
const target = await resolveEnvironmentExecutionTarget({
db: input.db, companyId: bound.companyId, adapterType: "paperclip_runner",
environment, leaseId: lease.id, leaseMetadata: lease.metadata, lease,
environmentRuntime: input.environmentRuntime,
});
if (!target) return workspaceSyncFailure("workspace_sync_out_unrecoverable");
try {
const saved = await saveRemoteWorkspace({
db: input.db, workspace, runId: input.runId, lease, target,
assertOwned: ownership.assertHeld,
});
await ownership.assertHeld();
return { status: "succeeded", exitCode: 0,
system: "Native workspace files were saved to durable storage.\n",
metadata: { remoteWorkspace: saved } };
} catch (error) {
await ownership.assertHeld();
return workspaceSyncFailure(classifyNativeWorkspaceFailure(error).code === "workspace_sync_out_unrecoverable"
? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed");
}
}
if (nativeWorkspaceSync) {
if (!input.environmentRuntime) {
return {
@@ -1342,6 +1342,19 @@ export class PaperclipRunnerToolAuthority {
return { bytes, filename: basename(file.path!), contentType: "application/octet-stream" };
}
const resolved = await workspaceFileResourceService(this.db).prepareDownload(this.binding.issueId, { path: file.path!, workspace: "auto" });
if ("stream" in resolved) {
try {
const chunks: Buffer[] = [];
let length = 0;
for await (const chunk of resolved.stream) {
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
length += bytes.length;
if (length > RUNNER_API_MAX_BYTES) throw badRequest("Workspace file exceeds API transfer limit");
chunks.push(bytes);
}
return { bytes: Buffer.concat(chunks), filename: resolved.resource.title, contentType: resolved.resource.contentType ?? "application/octet-stream" };
} finally { await resolved.cleanup(); }
}
const handle = await openRunnerApiWorkspaceFile(resolved.realPath);
try {
const stat = await handle.stat();
+10 -2
View File
@@ -6,6 +6,7 @@ import { execFile as execFileCallback } from "node:child_process";
import { promisify } from "node:util";
import { and, eq, sql } from "drizzle-orm";
import { executionWorkspaces, projectWorkspaces, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync";
import { acquireGitExpirationLocks, recoverGitExpirationLocks, type GitExpirationLockLocation } from "./git-workspace-expiration-locks.js";
import { allocatedBytes, withAnonymousGitFallback, type GitExpirationFailure } from "./git-workspace-expiration.js";
@@ -75,17 +76,24 @@ async function canonicalPath(target: string) {
return path.join(await fs.realpath(path.dirname(target)), path.basename(target));
}
function hasRemoteStorageFence(workspace: Pick<Workspace, "metadata">) {
const migration = workspace.metadata?.remoteWorkspaceMigration;
return Boolean(readRemoteWorkspaceAuthority(workspace.metadata)) || Boolean(migration && typeof migration === "object"
&& !Array.isArray(migration) && (migration as Record<string, unknown>).state === "running");
}
export function isNestedCloneMigrationCandidate(workspace: Workspace) {
return workspace.providerType !== "plugin" && OPEN_STATUSES.includes(workspace.status) && !workspace.closedAt
return !hasRemoteStorageFence(workspace) && workspace.providerType !== "plugin" && OPEN_STATUSES.includes(workspace.status) && !workspace.closedAt
&& Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId);
}
function canRecover(workspace: Workspace, name: string) {
return workspace.providerType !== "plugin" && Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId)
return !hasRemoteStorageFence(workspace) && workspace.providerType !== "plugin" && Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId)
&& pendingRecords(workspace)[name] !== undefined;
}
function pendingRecords(workspace: Pick<Workspace, "metadata">) {
if (hasRemoteStorageFence(workspace)) return {};
const value = workspace.metadata?.[NESTED_CLONE_MIGRATION_PENDING_KEY];
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
}
@@ -0,0 +1,390 @@
import { createHash, randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import type { Readable } from "node:stream";
import { and, eq } from "drizzle-orm";
import { environments, environmentLeases, executionWorkspaces, type Db } from "@paperclipai/db";
import type { Environment, EnvironmentLease, WorkspaceFileListMode, WorkspaceFileSource } from "@paperclipai/shared";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { HttpError, notFound, unprocessable } from "../errors.js";
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
const execFileAsync = promisify(execFile);
const READ_TIMEOUT_MS = 30_000;
const MAX_HELPER_OUTPUT_BYTES = 2 * 1024 * 1024;
export interface RemoteWorkspaceFileEntry {
kind: "file" | "directory";
relativePath: string;
byteSize: number | null;
modifiedAt: string | null;
text?: boolean;
sha256?: string;
}
export interface RemoteWorkspaceFileListInput {
relativePath?: string;
mode: WorkspaceFileListMode;
query?: string | null;
limit: number;
offset: number;
maxScannedEntries: number;
maxDepth: number;
}
export interface RemoteWorkspaceDownload {
stream: Readable;
byteSize: number;
cleanup(): Promise<void>;
}
export interface RemoteWorkspaceFileAccess {
source: WorkspaceFileSource;
stat(relativePath: string): Promise<RemoteWorkspaceFileEntry>;
read(relativePath: string, options: { maxBytes: number }): Promise<Buffer>;
list(input: RemoteWorkspaceFileListInput): Promise<{
entries: RemoteWorkspaceFileEntry[];
scannedCount: number;
truncated: boolean;
}>;
download(relativePath: string): Promise<RemoteWorkspaceDownload>;
close(): Promise<void>;
}
export interface SavedWorkspaceFiles {
root: string;
savedAt: string;
cleanup(): Promise<void>;
}
export interface RemoteWorkspaceFileOptions {
environmentRuntime?: Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
loadSavedWorkspace?(input: {
companyId: string;
workspaceId: string;
snapshotId: string;
signal?: AbortSignal;
}): Promise<SavedWorkspaceFiles>;
}
export interface RemoteWorkspaceFiles {
open(input: { companyId: string; workspaceId: string; signal?: AbortSignal }): Promise<RemoteWorkspaceFileAccess | null>;
}
// Run the same confined helper in the execution environment and in a verified,
// disposable saved-version directory. No absolute path comes from a file caller.
export const REMOTE_WORKSPACE_FILES_SCRIPT = String.raw`
const fs = require('node:fs/promises');
const { constants } = require('node:fs');
const path = require('node:path');
const { createHash } = require('node:crypto');
const { execFile } = require('node:child_process');
const { promisify } = require('node:util');
const input = JSON.parse(process.argv[1]);
const denied = new Set(['.git','.paperclip','.paperclip-runtime','node_modules','.pnpm-store','.yarn','.cache','.turbo','.next','.vite','.vercel','dist','build','coverage','runtime-services','.runtime']);
const fail = code => { const error = new Error(code); error.code = code; throw error; };
const deniedPath = relative => {
const parts = relative.toLowerCase().split('/');
const file = parts.at(-1) || '';
return parts.some(part => denied.has(part) || part === '.aws' || part === '.ssh') ||
file === '.env' || file.startsWith('.env.') || /\.(pem|key|p12|pfx)$/.test(file) ||
['id_rsa','id_ed25519','.npmrc','.pypirc','.netrc','kubeconfig'].includes(file) ||
(parts.at(-2) === '.docker' && file === 'config.json') || (parts.at(-2) === '.kube' && file === 'config');
};
const normalize = (relative, allowRoot = false) => {
if (allowRoot && !relative) return '';
if (typeof relative !== 'string' || !relative || relative.length > 4096 || /[\\\u0000-\u001f\u007f]/u.test(relative) ||
path.isAbsolute(relative) || /^[a-z][a-z0-9+.-]*:/iu.test(relative) || relative.startsWith('~') ||
relative.split('/').some(part => part === '..' || part === '.' || part === '')) fail('invalid_path');
if (deniedPath(relative)) fail('denied_path');
return relative;
};
const same = (a,b) => ['dev','ino','size','mtimeNs','ctimeNs'].every(key => a[key] === b[key]);
const within = (root,target) => { const relative = path.relative(root,target); return relative === '' || (relative !== '..' && !relative.startsWith('../') && !path.isAbsolute(relative)); };
(async () => {
const rootEntry = await fs.lstat(input.root);
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) fail('workspace_unavailable');
const root = await fs.realpath(input.root);
const confined = async (relative, allowRoot = false) => {
relative = normalize(relative, allowRoot);
let current = root;
for (const part of relative ? relative.split('/') : []) {
current = path.join(current,part);
const stat = await fs.lstat(current);
if (stat.isSymbolicLink()) fail('symlink_denied');
}
const canonical = await fs.realpath(current);
if (!within(root,canonical) || canonical !== current) fail('outside_workspace');
return canonical;
};
const entry = async (relative, withHash = false) => {
const target = await confined(relative);
const stat = await fs.lstat(target);
if (!stat.isDirectory() && !stat.isFile()) fail('not_regular_file');
const result = {kind:stat.isDirectory()?'directory':'file',relativePath:relative,byteSize:stat.isFile()?stat.size:null,modifiedAt:stat.mtime.toISOString()};
if (stat.isFile() && input.action === 'stat') {
const handle = await fs.open(target,constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
try {
const before = await handle.stat({bigint:true});
if (!before.isFile() || before.nlink !== 1n) fail('not_regular_file');
const sample = Buffer.alloc(4096);
const {bytesRead} = await handle.read(sample,0,sample.length,0);
let controls = 0;
let nul = false;
for (const byte of sample.subarray(0,bytesRead)) { if (!byte) nul = true; if (byte < 32 && ![9,10,13].includes(byte)) controls++; }
result.text = !nul && (!bytesRead || controls / bytesRead < .02);
if (withHash) { const hash = createHash('sha256'); for await (const bytes of handle.createReadStream({start:0,autoClose:false})) hash.update(bytes); result.sha256=hash.digest('hex'); }
const after = await handle.stat({bigint:true});
const current = await fs.lstat(target,{bigint:true});
if (!same(before,after) || !same(after,current) || await confined(relative) !== target) fail('file_changed');
} finally { await handle.close(); }
}
return result;
};
let result;
if (input.action === 'stat') result=await entry(input.relativePath,input.withHash === true);
else if (input.action === 'list') {
const requested = input.options;
const start = normalize(requested.relativePath || '',true);
const startPath = await confined(start,true);
if (!(await fs.stat(startPath)).isDirectory()) fail('not_directory');
const maxScanned = Math.min(5000,Math.max(1,requested.maxScannedEntries));
const maxDepth = Math.min(20,Math.max(0,requested.maxDepth));
const limit = Math.min(100,Math.max(1,requested.limit));
const offset = Math.min(10000,Math.max(0,requested.offset));
const query = (requested.query || '').toLowerCase();
let scannedCount=0,truncated=false;
const entries=[];
const matches = relative => !query || relative.toLowerCase().includes(query);
const add = async relative => {
try { const value=await entry(relative); if(matches(relative)) entries.push(value); }
catch(error) { if(!['ENOENT','ENOTDIR','denied_path','symlink_denied','not_regular_file'].includes(error.code)) throw error; }
};
if (requested.mode === 'changed') {
let stdout;
try { ({stdout}=await promisify(execFile)('git',['-c','core.fsmonitor=false','-c','core.hooksPath=/dev/null','-C',root,'status','--porcelain=v1','-z','--untracked-files=all'],{encoding:'utf8',timeout:10000,maxBuffer:1048576,env:{...process.env,GIT_OPTIONAL_LOCKS:'0',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null'}})); }
catch { fail('changed_unavailable'); }
const tokens=stdout.split('\0').filter(Boolean);
for(let i=0;i<tokens.length;i++) {
if(++scannedCount>maxScanned) {truncated=true;break;}
const token=tokens[i];
const status=token.slice(0,2),relative=token.slice(3);
if(status.includes('R')||status.includes('C')) i++;
if(relative && !deniedPath(relative)) await add(relative);
}
} else {
const immediate=requested.mode==='all'&&!query;
const queue=[{relative:start,depth:0}];
scan: while(queue.length) {
const current=queue.shift();
const directory=await confined(current.relative,true);
const children=await fs.readdir(directory,{withFileTypes:true});
children.sort((a,b)=>immediate&&a.isDirectory()!==b.isDirectory()?(a.isDirectory()?-1:1):a.name.localeCompare(b.name));
for(const child of children) {
if(++scannedCount>maxScanned) {truncated=true;break scan;}
const relative=current.relative?current.relative+'/'+child.name:child.name;
if(deniedPath(relative)||child.isSymbolicLink()||/[\\\u0000-\u001f\u007f]/u.test(relative)) continue;
if(child.isDirectory()) {
if(immediate) await add(relative);
else if(current.depth<maxDepth) queue.push({relative,depth:current.depth+1});
else truncated=true;
} else if(child.isFile()) await add(relative);
if(immediate&&entries.length>offset+limit) {truncated=true;break scan;}
}
}
}
if(requested.mode==='recent') entries.sort((a,b)=>(b.modifiedAt||'').localeCompare(a.modifiedAt||'')||a.relativePath.localeCompare(b.relativePath));
else if(requested.mode==='changed') entries.sort((a,b)=>a.relativePath.localeCompare(b.relativePath));
result={entries:entries.slice(offset,offset+limit),scannedCount:Math.min(scannedCount,maxScanned),truncated:truncated||entries.length>offset+limit};
} else fail('invalid_operation');
if(await fs.realpath(input.root)!==root) fail('workspace_unavailable');
process.stdout.write(JSON.stringify(result));
})().catch(error => {
const allowed=new Set(['invalid_path','denied_path','symlink_denied','outside_workspace','workspace_unavailable','not_regular_file','not_directory','file_changed','changed_unavailable','ENOENT','ENOTDIR']);
process.stderr.write(allowed.has(error.code)?error.code:'remote_read_failed');
process.exitCode=1;
});
`;
function fileFailure(code: string): HttpError {
if (["ENOENT", "ENOTDIR"].includes(code)) return notFound("Workspace file not found", { code: "not_found" });
if (["denied_path", "symlink_denied", "outside_workspace", "invalid_path"].includes(code)) {
return new HttpError(403, "Workspace file path is denied", { code });
}
const safe = new Set(["workspace_unavailable", "workspace_changed", "not_regular_file", "not_directory", "file_changed", "changed_unavailable", "too_large"]);
return unprocessable("Workspace files are unavailable", { code: safe.has(code) ? code : "workspace_unavailable" });
}
function parseFileEntry(value: unknown): RemoteWorkspaceFileEntry {
const entry = value as Partial<RemoteWorkspaceFileEntry> | null;
if (!entry || !["file", "directory"].includes(entry.kind ?? "") || typeof entry.relativePath !== "string" ||
!entry.relativePath || entry.relativePath.length > 4096 || /[\\\u0000-\u001f\u007f]/u.test(entry.relativePath) ||
path.posix.isAbsolute(entry.relativePath) || entry.relativePath.split("/").some(part => !part || part === "." || part === "..") ||
(entry.kind === "file" ? !Number.isSafeInteger(entry.byteSize) || entry.byteSize! < 0 : entry.byteSize !== null) ||
(entry.modifiedAt !== null && (typeof entry.modifiedAt !== "string" || !Number.isFinite(Date.parse(entry.modifiedAt)))) ||
(entry.text !== undefined && typeof entry.text !== "boolean") ||
(entry.sha256 !== undefined && !/^[a-f0-9]{64}$/.test(entry.sha256))) throw fileFailure("workspace_unavailable");
return entry as RemoteWorkspaceFileEntry;
}
function parseHelperResult(action: "stat" | "list", value: unknown, withHash: boolean): unknown {
if (action === "stat") {
const entry = parseFileEntry(value);
if (withHash && entry.kind === "file" && !entry.sha256) throw fileFailure("workspace_unavailable");
return entry;
}
const list = value as { entries?: unknown; scannedCount?: unknown; truncated?: unknown } | null;
if (!list || !Array.isArray(list.entries) || list.entries.length > 100 ||
!Number.isSafeInteger(list.scannedCount) || Number(list.scannedCount) < 0 || Number(list.scannedCount) > 5000 ||
typeof list.truncated !== "boolean") throw fileFailure("workspace_unavailable");
return { entries: list.entries.map(parseFileEntry), scannedCount: list.scannedCount, truncated: list.truncated };
}
async function verifyDownload(file: string, expected: RemoteWorkspaceFileEntry): Promise<number> {
const handle = await fs.open(file, "r");
try {
const stat = await handle.stat();
if (!stat.isFile() || stat.size !== expected.byteSize) throw fileFailure("file_changed");
const hash = createHash("sha256");
for await (const bytes of handle.createReadStream({ autoClose: false })) hash.update(bytes);
if (expected.sha256 && hash.digest("hex") !== expected.sha256) throw fileFailure("file_changed");
return stat.size;
} finally { await handle.close(); }
}
export function createRemoteWorkspaceFiles(db: Db, options: RemoteWorkspaceFileOptions = {}): RemoteWorkspaceFiles {
return {
async open(input) {
input.signal?.throwIfAborted();
const [workspace] = await db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId),
)).limit(1);
if (!workspace) throw notFound("Workspace not found");
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
if (!authority) return null;
const initialAuthority = JSON.stringify(authority);
const checkCurrent = async () => {
input.signal?.throwIfAborted();
const [current] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId),
)).limit(1);
if (!current || JSON.stringify(readRemoteWorkspaceAuthority(current.metadata)) !== initialAuthority) {
throw fileFailure("workspace_changed");
}
};
let saved: SavedWorkspaceFiles | undefined;
let environment: Environment | undefined;
let lease: EnvironmentLease | undefined;
let source: WorkspaceFileSource;
let root: string;
const loadSaved = async () => {
if (!authority.snapshotId || authority.dirty || authority.writerRunId || !options.loadSavedWorkspace) {
throw fileFailure("workspace_unavailable");
}
saved = await options.loadSavedWorkspace({ companyId: input.companyId, workspaceId: input.workspaceId, snapshotId: authority.snapshotId,
...(input.signal ? { signal: input.signal } : {}) });
return { root: saved.root, source: { kind: "saved", savedAt: saved.savedAt, snapshotId: authority.snapshotId } as WorkspaceFileSource };
};
if (authority.location) {
const [leaseRow] = await db.select().from(environmentLeases).where(and(
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, authority.location.leaseId),
)).limit(1);
const [environmentRow] = await db.select().from(environments).where(eq(environments.id, authority.location.environmentId)).limit(1);
if (!leaseRow || !environmentRow || leaseRow.environmentId !== environmentRow.id ||
leaseRow.providerLeaseId !== authority.location.providerLeaseId ||
leaseRow.metadata?.remoteCwd !== authority.location.remoteCwd) throw fileFailure("workspace_unavailable");
if (hasRemoteTerminationReceipt(leaseRow)
&& (leaseRow.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "stopped") {
({ root, source } = await loadSaved());
} else {
if (!options.environmentRuntime || !["active", "retained", "pending_cleanup"].includes(leaseRow.status)) {
throw fileFailure("workspace_unavailable");
}
lease = leaseRow as EnvironmentLease;
environment = environmentRow as Environment;
root = authority.location.remoteCwd;
source = { kind: "live", savedAt: null, snapshotId: null };
}
} else {
({ root, source } = await loadSaved());
}
let closed = false;
const close = async () => { if (closed) return; closed = true; await saved?.cleanup(); };
const execute = async <T>(action: "stat" | "list", args: Record<string, unknown>): Promise<T> => {
if (closed) throw fileFailure("workspace_unavailable");
await checkCurrent();
const payload = JSON.stringify({ root, action, ...args });
let stdout: string;
if (saved) {
try {
({ stdout } = await execFileAsync(process.execPath, ["--input-type=commonjs", "-e", REMOTE_WORKSPACE_FILES_SCRIPT, payload], {
cwd: root, timeout: READ_TIMEOUT_MS, maxBuffer: MAX_HELPER_OUTPUT_BYTES,
signal: input.signal, env: { ...process.env, NODE_OPTIONS: "", NODE_PATH: "" },
}));
} catch (error) { throw fileFailure(String((error as { stderr?: string }).stderr ?? "workspace_unavailable").trim()); }
} else {
const result = await options.environmentRuntime!.execute({
environment: environment!, lease: lease!, command: "node",
args: ["--input-type=commonjs", "-e", REMOTE_WORKSPACE_FILES_SCRIPT, payload],
cwd: root, timeoutMs: READ_TIMEOUT_MS, bypassSession: true, env: { NODE_OPTIONS: "", NODE_PATH: "" },
}).catch(() => { throw fileFailure("workspace_unavailable"); });
if (result.timedOut || result.exitCode !== 0) throw fileFailure(result.timedOut ? "workspace_unavailable" : result.stderr.trim());
stdout = result.stdout;
}
await checkCurrent();
try { return parseHelperResult(action, JSON.parse(stdout), args.withHash === true) as T; }
catch { throw fileFailure("workspace_unavailable"); }
};
const stat = (relativePath: string) => execute<RemoteWorkspaceFileEntry>("stat", { relativePath });
const download = async (relativePath: string): Promise<RemoteWorkspaceDownload> => {
const expected = await execute<RemoteWorkspaceFileEntry>("stat", { relativePath, withHash: true });
if (expected.kind !== "file") throw fileFailure("not_regular_file");
if (saved) {
await checkCurrent();
const file = path.join(root, ...relativePath.split("/"));
return { stream: createReadStream(file), byteSize: expected.byteSize!, cleanup: async () => {} };
}
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-file-"));
const cleanup = async () => { await fs.rm(directory, { recursive: true, force: true }); };
try {
await options.environmentRuntime!.syncOut({ environment: environment!, lease: lease!, operations: [{
operationId: `workspace-file:${randomUUID()}`,
files: [{ sourcePath: path.posix.join(root, relativePath), targetPath: path.join(directory, "file"), kind: "file", mode: 0o600 }],
}] }).catch(() => { throw fileFailure("workspace_unavailable"); });
await verifyDownload(path.join(directory, "file"), expected);
await checkCurrent();
return { stream: createReadStream(path.join(directory, "file")), byteSize: expected.byteSize!, cleanup };
} catch (error) { await cleanup(); throw error; }
};
return {
source, stat, close,
list: listInput => execute("list", { options: listInput }),
async read(relativePath, readOptions) {
const expected = await stat(relativePath);
if (expected.kind !== "file") throw fileFailure("not_regular_file");
if (expected.byteSize! > readOptions.maxBytes) throw fileFailure("too_large");
const transfer = await download(relativePath);
try {
const chunks: Buffer[] = [];
let size = 0;
for await (const chunk of transfer.stream) {
input.signal?.throwIfAborted();
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
size += bytes.length;
if (size > readOptions.maxBytes) throw fileFailure("too_large");
chunks.push(bytes);
}
await checkCurrent();
return Buffer.concat(chunks);
} finally { transfer.stream.destroy(); await transfer.cleanup(); }
},
download,
};
},
};
}
@@ -0,0 +1,266 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, environmentLeases, environments,
executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, projects, workspaceOperations, workspaceRuntimeServices, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, type RemoteWorkspaceAuthority } from "@paperclipai/shared";
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
import { environmentRuntimeService } from "./environment-runtime.js";
import { environmentService } from "./environments.js";
import { executionWorkspaceService, toExecutionWorkspace } from "./execution-workspaces.js";
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
import { cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, restartDesiredRuntimeServicesOnStartup, startRuntimeServicesForWorkspaceControl } from "./workspace-runtime.js";
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
import { workspaceOperationService } from "./workspace-operations.js";
import { inventoryRemoteWorkspaceMigration } from "./remote-workspace-migration.js";
import { hasPendingNestedCloneMigration, isNestedCloneMigrationCandidate, listNestedCloneMigrationNames,
migrateNestedRepositoryClone, pendingNestedCloneMigrationNames } from "./nested-clone-migration.js";
const url = resolveExternalTestDatabaseUrl("PAPERCLIP_REMOTE_WORKSPACE_LIFECYCLE_TEST_DATABASE_URL");
describe.skipIf(!url)("remote workspace close and allocation cleanup", () => {
let db: Db;
const directories: string[] = [];
beforeAll(async () => { await applyPendingMigrations(url!); db = createDb(url!); }, 120_000);
beforeEach(async () => {
await db.execute(sql`set client_min_messages = warning`);
await db.execute(sql`truncate table companies, execution_workspace_snapshots, environments restart identity cascade`);
});
afterEach(async () => { vi.restoreAllMocks(); await Promise.all(directories.splice(0).map(path => rm(path, { recursive: true, force: true }))); });
afterAll(async () => { await closeRegisteredClients(url!, { scope: "database" }); });
async function fixture(receiptState: "destroyed" | "stopped" | "invalid" = "destroyed") {
const [company] = await db.insert(companies).values({ name: "Remote close", issuePrefix: "RC" }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Native worker", adapterType: "paperclip_runner" }).returning();
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Saved work", status: "done",
completedAt: new Date(0), assigneeAgentId: agent.id }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id,
nativeIssueId: issue.id, runtimeMode: "native", status: "succeeded", finishedAt: new Date() }).returning();
const [environment] = await db.insert(environments).values({ name: `Remote close ${randomUUID()}`, driver: "sandbox", config: { provider: "daytona" } }).returning();
const cwd = await mkdtemp(join(tmpdir(), "paperclip-remote-close-")); directories.push(cwd);
await writeFile(join(cwd, "host-sentinel.txt"), "Host files must never be cleaned as remote files.");
const workspaceId = randomUUID(), leaseId = randomUUID(), snapshotId = randomUUID(), providerLeaseId = `allocation-${randomUUID()}`;
const authority: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1,
snapshotId, source: { kind: "empty" }, writerRunId: null, dirty: false,
location: { environmentId: environment.id, leaseId, providerLeaseId, remoteCwd: cwd } };
const [workspace] = await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: company.id, sourceIssueId: issue.id,
name: "Remote files", cwd, providerRef: cwd, providerType: "cloud_sandbox", mode: "isolated_workspace",
strategyType: "git_worktree", status: "active", metadata: { remoteWorkspace: authority } }).returning();
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
await db.insert(executionWorkspaceSnapshots).values({ id: snapshotId, companyId: company.id,
executionWorkspaceId: workspace.id, sourceOperationId: randomUUID(), sourceRunId: run.id,
generation: 1, epoch: 1, status: "committed", objectKey: "saved-workspace-object", sha256: "a".repeat(64), byteSize: 1 });
await db.insert(environmentLeases).values({ id: leaseId, companyId: company.id, environmentId: environment.id,
heartbeatRunId: run.id, issueId: issue.id, executionWorkspaceId: workspace.id, provider: "daytona", providerLeaseId,
leasePolicy: "ephemeral", status: "active", metadata: { driver: "sandbox", remoteCwd: cwd } });
const environmentsSvc = environmentService(db);
const base = environmentRuntimeService(db).getDriver("sandbox")!;
const destroyProvider = vi.fn(async (input: Parameters<NonNullable<typeof base.destroyRunLease>>[0]) => environmentsSvc.releaseLease(input.lease.id, "released", {
cleanupStatus: "success", remoteExecutionTermination: remoteTerminationReceipt(input.lease, {
state: receiptState === "stopped" ? "stopped" : "destroyed",
providerLeaseId: receiptState === "invalid" ? "wrong-allocation" : input.lease.providerLeaseId,
}),
}));
const runtime = environmentRuntimeService(db, { drivers: [{ ...base, destroyRunLease: destroyProvider }] });
const inspectLocal = vi.fn(async () => { throw new Error("Controller Git inspection must not run"); });
const service = executionWorkspaceService(db, { destroyRemoteWorkspaceLease: input => runtime.destroyRemoteWorkspaceLease(input),
inspectGitCloseReadiness: inspectLocal, workspaceReaperCooldownDays: 0, gitCacheExpirationEnabled: true,
gitRestorePoints: undefined });
return { company, issue, run, workspace, snapshotId, authority, service, destroyProvider, inspectLocal, cwd, environmentsSvc, leaseId };
}
const current = async (id: string) => (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!;
it.each([true, false])("rejects automatic Core services before controller access (skip provision: %s)", async skipHostProvision => {
const f = await fixture();
const inspect = vi.spyOn(fs, "lstat");
const log = vi.fn();
await expect(ensureRuntimeServicesForRun({ db, runId: f.run.id,
agent: { id: f.run.agentId, companyId: f.company.id, name: "Native worker" }, issue: null,
workspace: { cwd: f.cwd, baseCwd: f.cwd, source: "task_session", projectId: null, workspaceId: null,
repoUrl: null, repoRef: null, strategy: "git_worktree", branchName: null, worktreePath: f.cwd,
warnings: [], created: false, branchCreatedByRuntime: false }, executionWorkspaceId: f.workspace.id,
config: { workspaceRuntime: { services: [{ name: "web", command: "/__paperclip_missing_remote_service__", cwd: "." }] } },
adapterEnv: {}, skipHostProvision, onLog: log,
})).rejects.toMatchObject({ code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_runtime_services_unsupported" } });
expect(inspect).not.toHaveBeenCalled();
expect(log).not.toHaveBeenCalled();
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
});
it.each(["remote", "import"])("does not inspect or recover nested controller clones under a %s storage fence", async mode => {
const f = await fixture();
const [project] = await db.insert(projects).values({ companyId: f.company.id, name: "Remote project" }).returning();
await db.update(executionWorkspaces).set({ projectId: project.id,
...(mode === "import" ? { providerType: "local_fs" } : {}),
metadata: { ...(mode === "remote" ? { remoteWorkspace: f.authority } : {
remoteWorkspaceMigration: { schema: "paperclip.remote-workspace-migration.v1", state: "running", operationId: randomUUID() },
}),
nestedCloneMigrationPending: { "repo-123456789abc": { phase: "cleanup" } } } }).where(eq(executionWorkspaces.id, f.workspace.id));
const workspace = await current(f.workspace.id);
const inspect = vi.spyOn(fs, "lstat");
expect(isNestedCloneMigrationCandidate(workspace)).toBe(false);
expect(hasPendingNestedCloneMigration(workspace)).toBe(false);
expect(pendingNestedCloneMigrationNames(workspace)).toEqual([]);
expect(await listNestedCloneMigrationNames(db, workspace)).toEqual([]);
expect(await migrateNestedRepositoryClone(db, workspace, "repo-123456789abc")).toMatchObject({ outcome: "skipped", reason: "not_candidate" });
expect(inspect).not.toHaveBeenCalled();
});
it("closes remote compute under the lifecycle fence and keeps saved files and host paths intact", async () => {
const f = await fixture();
await f.service.update(f.workspace.id, { metadata: { userNote: "Keep", remoteWorkspace: { ...f.authority, dirty: false, location: null } } });
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)?.location).toEqual(f.authority.location);
await expect(f.service.update(f.workspace.id, { providerType: "local_fs" })).rejects.toThrow("sandbox storage");
const readiness = await f.service.getCloseReadiness(f.workspace.id);
expect(readiness).toMatchObject({ state: "ready", git: null, isDestructiveCloseAllowed: true });
expect(readiness!.plannedActions.map(action => action.kind)).toEqual(["archive_record", "remove_provider_workspace"]);
const archived = await f.service.archiveWorkspaceUnderLifecycleLock({ id: f.workspace.id, patch: {}, closedAt: new Date() });
if (archived?.outcome !== "archived") throw new Error("Expected a close receipt");
await f.service.fenceClosedWorkspaceDestruction({ workspaceId: f.workspace.id, capturedGeneration: archived.capturedGeneration,
destroy: () => f.service.cleanupRemoteStorage(archived.workspace) });
expect(f.destroyProvider).toHaveBeenCalledTimes(1);
expect(f.destroyProvider.mock.calls[0]![0].lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "ephemeral" });
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)).toMatchObject({ location: null, snapshotId: f.snapshotId });
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.snapshotId)))[0]!.status).toBe("committed");
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
expect(f.inspectLocal).not.toHaveBeenCalled();
await expect(cleanupExecutionWorkspaceArtifacts({ workspace: f.workspace })).rejects.toThrow("sandbox lifecycle");
});
it.each(["stopped", "invalid"] as const)("retains location until destruction is proved (%s)", async state => {
const f = await fixture(state);
await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date() }).where(eq(executionWorkspaces.id, f.workspace.id));
await expect(f.service.cleanupRemoteStorage(toExecutionWorkspace({ ...f.workspace, status: "archived" }))).rejects.toThrow("remote_workspace_cleanup_unconfirmed");
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)?.location).toEqual(f.authority.location);
});
it("blocks close while a save is pending and leaves the allocation available for recovery", async () => {
const f = await fixture();
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, dirty: true, writerRunId: f.run.id } } })
.where(eq(executionWorkspaces.id, f.workspace.id));
expect(await f.service.getCloseReadiness(f.workspace.id)).toMatchObject({ state: "blocked", isDestructiveCloseAllowed: false });
expect(f.destroyProvider).not.toHaveBeenCalled();
});
it("preserves the automatic delivery gate and retries an explicitly closed remote allocation", async () => {
const f = await fixture();
const active = await f.service.sweepTerminalWorkspaces();
expect(active).toMatchObject({ archived: 0, skippedUndelivered: 1 });
expect(f.destroyProvider).not.toHaveBeenCalled();
await db.update(executionWorkspaces).set({ status: "cleanup_failed", closedAt: new Date() }).where(eq(executionWorkspaces.id, f.workspace.id));
const closed = await f.service.sweepTerminalWorkspaces();
expect(closed).toMatchObject({ archived: 1, cleanupFailed: 0 });
expect(f.destroyProvider).toHaveBeenCalledTimes(1);
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)?.location).toBeNull();
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
expect(f.inspectLocal).not.toHaveBeenCalled();
});
it("reopens saved remote work by changing its lifecycle without rebuilding a controller folder", async () => {
const f = await fixture();
await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date(),
metadata: { remoteWorkspace: { ...f.authority, location: null } } }).where(eq(executionWorkspaces.id, f.workspace.id));
const reopened = await f.service.reopenClosedIsolatedExecutionWorkspaceForIssue({ workspaceId: f.workspace.id,
issue: { id: f.issue.id, companyId: f.company.id, projectId: null }, actor: { actorType: "user", agentId: null },
authorizationActor: { type: "board", source: "local_implicit", isInstanceAdmin: true } });
expect(reopened).toMatchObject({ ok: true, reopened: true, workspace: { status: "active", closedAt: null } });
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)).toMatchObject({ snapshotId: f.snapshotId, location: null });
expect(f.destroyProvider).not.toHaveBeenCalled();
expect(f.inspectLocal).not.toHaveBeenCalled();
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
});
it.each([
{ caller: "direct", state: "remote" }, { caller: "startup", state: "remote" },
{ caller: "direct", state: "migrating" }, { caller: "startup", state: "migrating" },
] as const)("rechecks $caller control authority after waiting for $state migration publication", async ({ caller, state }) => {
const f = await fixture();
const escaped = join(f.cwd, "unexpected-start.txt");
const command = `node -e 'require("fs").writeFileSync(${JSON.stringify(escaped)},"unexpected")'`;
const config = { desiredState: "running", runtimeProvisionCommand: command,
workspaceRuntime: { services: [{ name: "web", command, cwd: "." }] } };
await db.update(executionWorkspaces).set({ providerType: "local_fs", strategyType: "project_primary", metadata: { config } })
.where(eq(executionWorkspaces.id, f.workspace.id));
let release!: () => void;
const released = new Promise<void>(resolve => { release = resolve; });
let ready!: (pid: number) => void;
const locked = new Promise<number>(resolve => { ready = resolve; });
const migration = withExecutionWorkspaceLifecycleLock(db, f.workspace.id, async tx => {
const [row] = await tx.execute(sql`select pg_backend_pid() as pid`);
ready(Number(row.pid));
await released;
await db.update(executionWorkspaces).set({ providerType: state === "remote" ? "cloud_sandbox" : "local_fs",
metadata: state === "remote" ? { config, remoteWorkspace: f.authority } : { config,
remoteWorkspaceMigration: { schema: "paperclip.remote-workspace-migration.v1", state: "running" } } })
.where(eq(executionWorkspaces.id, f.workspace.id));
});
const blocker = await locked;
const starting = caller === "startup" ? restartDesiredRuntimeServicesOnStartup(db) : startRuntimeServicesForWorkspaceControl({
db, actor: { id: null, companyId: f.company.id, name: "Board" }, issue: null, executionWorkspaceId: f.workspace.id,
workspace: { cwd: f.cwd, baseCwd: f.cwd, source: "task_session", projectId: null, workspaceId: null,
repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null,
warnings: [], created: false, branchCreatedByRuntime: false }, config, adapterEnv: {},
});
const settled = starting.then(value => ({ value }), error => ({ error }));
try {
// Prove the stale request is waiting on the actual PostgreSQL lifecycle
// fence, then switch authority before admitting it. No timing sleep.
await vi.waitFor(async () => {
const [row] = await db.execute(sql`select count(*)::int as waiting from pg_stat_activity
where ${blocker} = any(pg_blocking_pids(pid))`);
expect(Number(row.waiting)).toBeGreaterThan(0);
});
} finally { release(); }
await migration;
const result = await settled;
if (caller === "startup") expect(result).toMatchObject({ value: { restarted: 0, failed: 1 } });
else expect(result).toMatchObject({ error: { status: state === "remote" ? 422 : 409,
details: { code: state === "remote" ? "remote_workspace_runtime_commands_unsupported" : "workspace_runtime_migration_in_progress" } } });
await expect(fs.access(escaped)).rejects.toMatchObject({ code: "ENOENT" });
expect(await db.select().from(workspaceRuntimeServices)).toEqual([]);
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
});
it("publishes runtime admission before releasing the migration fence and keeps migration operations blocking", async () => {
const f = await fixture();
await db.update(executionWorkspaces).set({ providerType: "local_fs", strategyType: "project_primary", metadata: {} })
.where(eq(executionWorkspaces.id, f.workspace.id));
const operations = workspaceOperationService(db);
let finish!: () => void;
const finished = new Promise<void>(resolve => { finish = resolve; });
let admitted!: () => void;
const admission = new Promise<void>(resolve => { admitted = resolve; });
const running = operations.createRecorder({ companyId: f.company.id, executionWorkspaceId: f.workspace.id }).recordOperation({
phase: "workspace_provision", metadata: { action: "run" },
admit: publish => withExecutionWorkspaceLifecycleLock(db, f.workspace.id, async () => { await publish(); }),
run: async () => { admitted(); await finished; return {}; },
});
try {
await admission;
const inventory = await withExecutionWorkspaceLifecycleLock(db, f.workspace.id,
() => inventoryRemoteWorkspaceMigration(db, { companyId: f.company.id, workspaceId: f.workspace.id }));
expect(inventory.reasons).toContain("workspace_operation_unresolved");
} finally { finish(); }
await running;
const [migration] = await db.insert(workspaceOperations).values({ companyId: f.company.id, executionWorkspaceId: f.workspace.id,
phase: "remote_workspace_migration", status: "running", metadata: {}, startedAt: new Date(0), updatedAt: new Date(0) }).returning();
await expect(operations.assertRuntimeControlAvailable({ executionWorkspaceId: f.workspace.id, action: "start" }))
.rejects.toMatchObject({ status: 409, details: { activeOperationId: migration.id } });
expect((await operations.getById(migration.id))?.status).toBe("running");
});
it("does not resolve a controller folder when a remote control has no selected services", async () => {
const f = await fixture();
await expect(startRuntimeServicesForWorkspaceControl({ db,
actor: { id: null, companyId: f.company.id, name: "Board" }, issue: null, executionWorkspaceId: f.workspace.id,
workspace: { get cwd(): string { throw new Error("Empty control read a controller folder"); },
baseCwd: f.cwd, source: "task_session", projectId: null, workspaceId: null, repoUrl: null, repoRef: null,
strategy: "git_worktree", branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false },
config: { desiredState: "stopped", workspaceRuntime: { services: [{ name: "web", command: "node web.js" }] } },
adapterEnv: {}, respectDesiredStates: true,
})).resolves.toEqual([]);
});
});
@@ -0,0 +1,44 @@
import { and, eq } from "drizzle-orm";
import { executionWorkspaces, executionWorkspaceSnapshots, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, type EnvironmentLease } from "@paperclipai/shared";
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
type Workspace = Pick<typeof executionWorkspaces.$inferSelect, "id" | "companyId" | "metadata">;
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
export type DestroyRemoteWorkspaceLease = (input: {
companyId: string; workspaceId: string; leaseId: string; environmentId: string; providerLeaseId: string;
}) => Promise<EnvironmentLease | null>;
export async function inspectRemoteWorkspacePersistence(db: Db | Transaction, workspace: Workspace) {
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
if (!authority || authority.dirty || authority.writerRunId || !authority.snapshotId) {
return { ready: false, reason: "The workspace still has unsaved work or an unfinished writer.", authority };
}
const [snapshot] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.status, "committed"),
));
return { ready: Boolean(snapshot), reason: snapshot ? null : "The saved workspace is unavailable. Its remote files were retained.", authority };
}
/** Called within the workspace lifecycle fence; the saved workspace remains durable. */
export async function cleanupRemoteWorkspaceAllocation(input: {
db: Db; workspace: Workspace; destroyLease: DestroyRemoteWorkspaceLease;
}) {
const [workspace] = await input.db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.workspace.companyId), eq(executionWorkspaces.id, input.workspace.id),
));
if (!workspace || !["archived", "cleanup_failed"].includes(workspace.status)) throw new Error("remote_workspace_cleanup_binding_changed");
const persistence = await inspectRemoteWorkspacePersistence(input.db, workspace);
if (!persistence.ready || !persistence.authority) throw new Error(persistence.reason ?? "remote_workspace_saved_state_required");
const location = persistence.authority.location;
if (location) {
const released = await input.destroyLease({ companyId: workspace.companyId, workspaceId: workspace.id,
leaseId: location.leaseId, environmentId: location.environmentId, providerLeaseId: location.providerLeaseId });
if (!released || !hasRemoteTerminationReceipt(released)
|| (released.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state !== "destroyed") {
throw new Error("remote_workspace_cleanup_unconfirmed");
}
}
return { cleaned: true, warnings: [] as string[] };
}
@@ -0,0 +1,210 @@
import { randomUUID } from "node:crypto";
import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { activityLog, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
import { createStorageService } from "../storage/service.js";
import type { StorageService } from "../storage/types.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js";
import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js";
import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js";
const execute = promisify(execFile);
const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C", cwd, ...args], { env: { ...process.env,
GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_AUTHOR_NAME: "Migration", GIT_COMMITTER_NAME: "Migration",
GIT_AUTHOR_EMAIL: "migration@example.test", GIT_COMMITTER_EMAIL: "migration@example.test" } })).stdout.trim();
const url = process.env.PAPERCLIP_TEST_DATABASE_URL;
if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required");
const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("remote workspace migration", () => {
let db: ReturnType<typeof createDb>, root: string, storage: StorageService;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
beforeAll(async () => {
if (url) { await applyPendingMigrations(url); db = createDb(url); }
else { temporary = await startEmbeddedPostgresTestDatabase("workspace-migration-"); db = createDb(temporary.connectionString); }
}, 90_000);
afterAll(async () => { if (url) await closeRegisteredClients(url, { scope: "database" }); else await temporary?.cleanup(); });
beforeEach(async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32));
root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-migration-test-")); storage = createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))); });
afterEach(async () => {
vi.unstubAllEnvs(); await db.delete(activityLog); await db.delete(workspaceOperations); await db.delete(executionWorkspaceSnapshots);
await db.delete(workspaceRuntimeServices); await db.delete(executionWorkspaceRuntimeLeases);
await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects);
await db.delete(companies); await fs.rm(root, { recursive: true, force: true });
});
async function seed() {
const companyId = randomUUID(), workspaceId = randomUUID(), cwd = path.join(root, "original"); await fs.mkdir(cwd);
await fs.writeFile(path.join(cwd, "saved.txt"), "original saved work\n");
await db.insert(companies).values({ id: companyId, name: "Migration", issuePrefix: `M${companyId.slice(0, 6)}` });
const [issue] = await db.insert(issues).values({ companyId, title: "Existing task" }).returning();
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issue!.id, mode: "isolated_workspace",
strategyType: "project_primary", providerType: "local_fs", name: "Existing workspace", cwd, providerRef: cwd, metadata: { retainedNote: "keep" } });
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issue!.id));
return { companyId, workspaceId, issueId: issue!.id, cwd };
}
async function workspace(id: string) { return (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!; }
it("defaults to dry run and preserves both database authority and local files", async () => {
const f = await seed(); expect(await migrateRemoteWorkspace({ db, storage, ...f })).toMatchObject({ eligible: true, applied: false, alreadyMigrated: false });
expect((await workspace(f.workspaceId)).providerType).toBe("local_fs");
expect(await db.select().from(workspaceOperations)).toHaveLength(0);
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
});
it("verifies saved files, switches authority once, and retains the original folder", async () => {
const f = await seed();
expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ applied: true, alreadyMigrated: true });
const row = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(row.metadata);
expect(authority).toMatchObject({ epoch: 1, location: null, dirty: false, writerRunId: null });
expect(authority?.snapshotId).toMatch(/^[a-f0-9-]{36}$/);
expect(row.metadata?.retainedNote).toBe("keep"); expect(row.metadata?.remoteWorkspaceMigration).toBeUndefined();
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ replayed: true });
expect(await db.select().from(executionWorkspaceSnapshots)).toHaveLength(1);
expect(await db.select().from(activityLog)).toHaveLength(1);
});
it.each([false, true])("removes plaintext verification files with read-only folders after import (source changed: %s)", async sourceChanged => {
const f = await seed(), protectedDir = path.join(f.cwd, "protected"), scratchRoot = path.join(root, "scratch");
await fs.mkdir(protectedDir); await fs.mkdir(scratchRoot);
await fs.writeFile(path.join(protectedDir, "saved.txt"), "read-only saved work\n", { mode: 0o444 });
await fs.chmod(protectedDir, 0o555);
vi.stubEnv("TMPDIR", scratchRoot);
const get = storage.getObject.bind(storage); let reads = 0;
if (sourceChanged) storage.getObject = async (...args) => {
if (++reads === 2) await fs.writeFile(path.join(f.cwd, "saved.txt"), "new local work\n");
return get(...args);
};
try {
const migration = migrateRemoteWorkspace({ db, storage, ...f, apply: true });
if (sourceChanged) {
await expect(migration).rejects.toThrow("remote_workspace_migration_source_changed");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
} else {
await expect(migration).resolves.toMatchObject({ applied: true, alreadyMigrated: true });
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)?.snapshotId).toBeTruthy();
}
expect(await fs.readdir(scratchRoot)).toEqual([]);
expect((await fs.stat(protectedDir)).mode & 0o777).toBe(0o555);
expect((await fs.stat(path.join(protectedDir, "saved.txt"))).mode & 0o777).toBe(0o444);
expect(await fs.readFile(path.join(protectedDir, "saved.txt"), "utf8")).toBe("read-only saved work\n");
} finally {
await fs.chmod(protectedDir, 0o755);
await removeTaskDirectoryTree(scratchRoot);
}
});
it("resumes an interrupted upload with the original operation identity", async () => {
const f = await seed(), put = storage.putFile.bind(storage); let fail = true;
storage.putFile = async input => { const saved = await put(input); if (fail) { fail = false; throw new Error("upload reply lost"); } return saved; };
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("upload reply lost");
const before = await inventoryRemoteWorkspaceMigration(db, f); expect(before.operationId).not.toBeNull();
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ applied: true });
const operations = await db.select().from(workspaceOperations); expect(operations).toHaveLength(1); expect(operations[0]?.id).toBe(before.operationId);
});
it("imports dirty unpushed Git work and restores it after the original folder disappears", async () => {
const f = await seed(); await git(f.cwd, "init", "-q", "-b", "feature");
await git(f.cwd, "add", "."); await git(f.cwd, "commit", "-qm", "unpushed work");
await git(f.cwd, "remote", "add", "origin", "https://example.invalid/repository.git");
const head = await git(f.cwd, "rev-parse", "HEAD");
await fs.writeFile(path.join(f.cwd, "saved.txt"), "staged edit\n"); await git(f.cwd, "add", "saved.txt");
await fs.writeFile(path.join(f.cwd, "saved.txt"), "unstaged edit\n");
await db.update(executionWorkspaces).set({ repoUrl: "https://example.invalid/repository.git", branchName: "feature" }).where(eq(executionWorkspaces.id, f.workspaceId));
await migrateRemoteWorkspace({ db, storage, ...f, apply: true });
const authority = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!;
const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! });
await fs.rm(f.cwd, { recursive: true });
const archivePath = path.join(root, "cold.pcws"), targetDir = path.join(root, "cold");
await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath });
await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest });
expect(await git(targetDir, "rev-parse", "HEAD")).toBe(head);
expect(await git(targetDir, "show", ":saved.txt")).toBe("staged edit");
expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("unstaged edit\n");
});
it("blocks unresolved writers and explicitly parallel shared workspaces", async () => {
const f = await seed();
await db.update(executionWorkspaces).set({ mode: "shared_workspace" }).where(eq(executionWorkspaces.id, f.workspaceId));
await db.update(issues).set({ executionWorkspaceSettings: { sharedWorkspaceConcurrency: "allow" } }).where(eq(issues.id, f.issueId));
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["parallel_shared_writers_unsupported"] });
await db.update(issues).set({ executionWorkspaceSettings: { sharedWorkspaceConcurrency: "serialize" } }).where(eq(issues.id, f.issueId));
await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId, phase: "workspace_command", status: "running" });
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_operation_unresolved");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
});
it.each(["running", "starting", "unknown", "failed"])("blocks a %s local service after its control lease expires", async status => {
const f = await seed(), serviceId = randomUUID();
await db.insert(executionWorkspaceRuntimeLeases).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId,
ownerKey: `issue:${f.issueId}`, ownerIssueId: f.issueId, lastAction: "start", expiresAt: new Date(Date.now() - 60_000) });
await db.insert(workspaceRuntimeServices).values({ id: serviceId, companyId: f.companyId, executionWorkspaceId: f.workspaceId,
scopeType: "execution_workspace", scopeId: f.workspaceId, serviceName: "Synthetic preview", status,
lifecycle: "shared", provider: "local_process", command: "node synthetic-preview.js", cwd: f.cwd });
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["workspace_runtime_service_unsettled"] });
expect(await migrateRemoteWorkspace({ db, storage, ...f })).toMatchObject({ applied: false, eligible: false });
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled");
expect((await workspace(f.workspaceId)).providerType).toBe("local_fs");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
expect(await db.select().from(workspaceOperations)).toHaveLength(0);
expect(await db.select().from(executionWorkspaceSnapshots)).toHaveLength(0);
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
await db.$client`update workspace_runtime_services set status = 'stopped' where id = ${serviceId}::uuid`;
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true, reasons: [] });
});
it.each([{ repository: { phase: "cleanup" } }, "unrecognized-state"])("blocks an unresolved or malformed nested clone migration marker: %j", async marker => {
const f = await seed();
await db.update(executionWorkspaces).set({ metadata: { retainedNote: "keep", nestedCloneMigrationPending: marker } }).where(eq(executionWorkspaces.id, f.workspaceId));
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["nested_clone_migration_unresolved"] });
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("nested_clone_migration_unresolved");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
});
it("blocks a running project service inherited by a shared workspace", async () => {
const f = await seed(), projectId = randomUUID(), projectWorkspaceId = randomUUID();
await db.insert(projects).values({ id: projectId, companyId: f.companyId, name: "Shared synthetic project" });
await db.insert(projectWorkspaces).values({ id: projectWorkspaceId, companyId: f.companyId, projectId,
name: "Shared synthetic source", sourceType: "local", cwd: f.cwd });
await db.update(executionWorkspaces).set({ mode: "shared_workspace", projectId, projectWorkspaceId }).where(eq(executionWorkspaces.id, f.workspaceId));
await db.insert(workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId, projectWorkspaceId,
scopeType: "project_workspace", scopeId: projectWorkspaceId, serviceName: "Inherited preview", status: "running",
lifecycle: "shared", provider: "local_process", cwd: f.cwd });
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["workspace_runtime_service_unsettled"] });
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
});
it("rechecks a service that becomes unresolved during snapshot verification", async () => {
const f = await seed(), get = storage.getObject.bind(storage); let reads = 0;
storage.getObject = async (...args) => {
if (++reads === 2) await db.insert(workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId,
executionWorkspaceId: f.workspaceId, scopeType: "execution_workspace", scopeId: f.workspaceId,
serviceName: "Late synthetic preview", status: "running", lifecycle: "shared", provider: "local_process", cwd: f.cwd });
return get(...args);
};
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
});
it("rechecks a writer that appears while the restored copy is being verified", async () => {
const f = await seed(), get = storage.getObject.bind(storage); let reads = 0;
storage.getObject = async (...args) => {
if (++reads === 2) await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId,
phase: "workspace_command", status: "running" });
return get(...args);
};
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_operation_unresolved");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
});
it("preserves a local edit made during verification and refuses activation", async () => {
const f = await seed(), get = storage.getObject.bind(storage); let reads = 0;
storage.getObject = async (...args) => { if (++reads === 2) await fs.writeFile(path.join(f.cwd, "saved.txt"), "new local work\n"); return get(...args); };
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("source_changed");
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("new local work\n");
});
});
@@ -0,0 +1,205 @@
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
import {
executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db,
} from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
import type { StorageService } from "../storage/types.js";
import { persistActivity, publishActivity } from "./activity-log.js";
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
import { beginWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot } from "./remote-workspace-snapshot-storage.js";
import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js";
import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js";
const SCHEMA = "paperclip.remote-workspace-migration.v1";
type Workspace = typeof executionWorkspaces.$inferSelect;
type ReadDb = Pick<Db, "select">;
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
function canonical(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
if (value && typeof value === "object") return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`).join(",")}}`;
return JSON.stringify(value) ?? "null";
}
function fingerprint(workspace: Workspace) {
const metadata = { ...workspace.metadata }; delete metadata.remoteWorkspaceMigration;
return createHash("sha256").update(canonical({ companyId: workspace.companyId, id: workspace.id, projectId: workspace.projectId,
projectWorkspaceId: workspace.projectWorkspaceId, sourceIssueId: workspace.sourceIssueId, mode: workspace.mode, strategyType: workspace.strategyType,
providerType: workspace.providerType, cwd: workspace.cwd, providerRef: workspace.providerRef, repoUrl: workspace.repoUrl,
baseRef: workspace.baseRef, branchName: workspace.branchName, status: workspace.status, closedAt: workspace.closedAt?.toISOString(), metadata })).digest("hex");
}
async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string }) {
const [workspace] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId)));
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
const marker = object(workspace.metadata?.remoteWorkspaceMigration);
const allowedOperation = input.operationId ?? (marker.schema === SCHEMA && typeof marker.operationId === "string" ? marker.operationId : undefined);
const reasons: string[] = [];
if (authority) return { workspace, reasons, authority, marker, source: authority.source };
if (!["local_fs", "git_worktree"].includes(workspace.providerType)) reasons.push("workspace_provider_unsupported");
if (!["active", "idle", "in_review"].includes(workspace.status)) reasons.push("workspace_not_active");
if (!workspace.sourceIssueId) reasons.push("task_binding_required");
const nestedMigration = workspace.metadata?.nestedCloneMigrationPending;
if (nestedMigration != null && (typeof nestedMigration !== "object" || Array.isArray(nestedMigration)
|| Object.keys(nestedMigration).length > 0)) reasons.push("nested_clone_migration_unresolved");
const cwd = workspace.cwd ?? workspace.providerRef;
if (!cwd || !path.isAbsolute(cwd)) reasons.push("local_source_missing");
else {
const entry = await fs.lstat(cwd).catch(() => null);
if (!entry?.isDirectory() || entry.isSymbolicLink()) reasons.push("local_source_unsafe_or_missing");
}
const [issue] = workspace.sourceIssueId ? await db.select().from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))) : [];
if (workspace.sourceIssueId && (!issue || issue.executionWorkspaceId !== workspace.id)) reasons.push("task_binding_changed");
const [project] = workspace.projectId ? await db.select({ policy: projects.executionWorkspacePolicy }).from(projects)
.where(and(eq(projects.companyId, workspace.companyId), eq(projects.id, workspace.projectId))) : [];
const issuePolicy = object(issue?.executionWorkspaceSettings), projectPolicy = object(project?.policy);
if (workspace.mode === "shared_workspace" && (issuePolicy.sharedWorkspaceConcurrency ?? (projectPolicy.enabled === true ? projectPolicy.sharedWorkspaceConcurrency : undefined)) === "allow") reasons.push("parallel_shared_writers_unsupported");
if (issue?.executionRunId || issue?.checkoutRunId) reasons.push("task_writer_bound");
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, workspace.companyId),
inArray(heartbeatRuns.status, ["queued", "running"]), sql`(${heartbeatRuns.nativeIssueId} = ${workspace.sourceIssueId}::uuid
or ${heartbeatRuns.issueId} = ${workspace.sourceIssueId}::uuid
or ${heartbeatRuns.contextSnapshot}->>'executionWorkspaceId' = ${workspace.id}
or ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'workspaceId' = ${workspace.id})`,
)).limit(1);
if (run) reasons.push("active_execution");
const [unexported] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, workspace.companyId),
sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'workspaceId' = ${workspace.id}`,
sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'`,
)).limit(1);
if (unexported) reasons.push("remote_copyback_unresolved");
const [finalization] = workspace.sourceIssueId ? await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, workspace.companyId), eq(nativeRunFinalizations.issueId, workspace.sourceIssueId),
sql`${nativeRunFinalizations.phase} not in ('committed', 'terminal_failure')`,
)).limit(1) : [];
if (finalization) reasons.push("execution_finalization_unresolved");
const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
allowedOperation ? ne(workspaceOperations.id, allowedOperation) : undefined,
sql`(${workspaceOperations.status} in ('running', 'pending') or ${workspaceOperations.metadata}->>'uncertain' = 'true')`,
)).limit(1);
if (operation) reasons.push("workspace_operation_unresolved");
const [lease] = await db.select({ id: executionWorkspaceRuntimeLeases.id }).from(executionWorkspaceRuntimeLeases).where(and(
eq(executionWorkspaceRuntimeLeases.companyId, workspace.companyId), eq(executionWorkspaceRuntimeLeases.executionWorkspaceId, workspace.id),
sql`${executionWorkspaceRuntimeLeases.expiresAt} > clock_timestamp()`,
)).limit(1);
if (lease) reasons.push("workspace_runtime_owned");
// A control lease can expire while the service process remains alive. Require
// settled service state independently, including inherited shared services.
const runtime = object(workspace.metadata?.config).workspaceRuntime;
const inheritedSourceId = workspace.mode === "shared_workspace" && workspace.projectWorkspaceId
&& (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) ? workspace.projectWorkspaceId : null;
const [service] = await db.select({ id: workspaceRuntimeServices.id }).from(workspaceRuntimeServices).where(and(
eq(workspaceRuntimeServices.companyId, workspace.companyId), ne(workspaceRuntimeServices.status, "stopped"),
or(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id), inheritedSourceId ? and(
eq(workspaceRuntimeServices.projectWorkspaceId, inheritedSourceId), eq(workspaceRuntimeServices.scopeType, "project_workspace"),
) : undefined),
)).limit(1);
if (service) reasons.push("workspace_runtime_service_unsettled");
const [recovery] = workspace.sourceIssueId ? await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, workspace.companyId), eq(issueRecoveryActions.sourceIssueId, workspace.sourceIssueId),
inArray(issueRecoveryActions.status, ["active", "escalated"]),
)).limit(1) : [];
if (recovery) reasons.push("workspace_recovery_unresolved");
const repositories = await db.select().from(executionWorkspaceRepositories).where(and(eq(executionWorkspaceRepositories.companyId, workspace.companyId),
eq(executionWorkspaceRepositories.executionWorkspaceId, workspace.id)));
if (repositories.some(repository => repository.state !== "ready")) reasons.push("repository_preparation_unresolved");
const source: RemoteWorkspaceSource = workspace.repoUrl ? { kind: "git", repoUrl: workspace.repoUrl,
ref: workspace.baseRef, branchName: workspace.branchName } : { kind: "empty" };
if (repositories.length) source.repositories = repositories.map(repository => ({ id: repository.id, relativePath: repository.relativePath,
repoUrl: repository.repoUrl, ref: repository.requestedRef, ...(repository.pinnedCommit ? { pinnedCommit: repository.pinnedCommit } : {}) }));
return { workspace, reasons: [...new Set(reasons)], authority, marker, source };
}
export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string }) {
const result = await inspect(db, input);
return { companyId: input.companyId, workspaceId: input.workspaceId, eligible: result.reasons.length === 0,
alreadyMigrated: Boolean(result.authority), reasons: result.reasons, localPath: result.workspace.cwd ?? result.workspace.providerRef,
source: result.source, operationId: typeof result.marker.operationId === "string" ? result.marker.operationId : null,
snapshotId: result.authority?.snapshotId ?? null };
}
/** Administrative import. The CLI enforces external deployment prerequisites. */
export async function migrateRemoteWorkspace(input: {
db: Db; storage: StorageService; companyId: string; workspaceId: string; apply?: boolean; source?: RemoteWorkspaceSource;
}) {
if (!input.apply) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: false };
return withExecutionWorkspaceLifecycleLock(input.db, input.workspaceId, async lock => {
const reserved = await input.db.transaction(async tx => {
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
if (workspace.sourceIssueId) await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))).for("update");
const current = await inspect(tx as unknown as Db, input);
if (current.authority) return { workspace, authority: current.authority, operation: null, source: current.source, sourceFingerprint: "" };
if (current.reasons.length) throw new Error(`remote_workspace_migration_blocked:${current.reasons.join(",")}`);
const source = remoteWorkspaceSourceSchema.parse(input.source ?? current.source);
const sourceFingerprint = fingerprint(workspace);
const marker = current.marker;
if (marker.schema === SCHEMA && typeof marker.operationId === "string") {
if (marker.sourceFingerprint !== sourceFingerprint || canonical(marker.source) !== canonical(source)) throw new Error("remote_workspace_migration_source_changed");
const [operation] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, workspace.companyId),
eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration")));
if (!operation) throw new Error("remote_workspace_migration_operation_missing");
return { workspace, authority: null, operation, source, sourceFingerprint };
}
const [operation] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id,
issueId: workspace.sourceIssueId, cwd: workspace.cwd ?? workspace.providerRef, phase: "remote_workspace_migration", status: "running",
metadata: { schema: SCHEMA, sourceFingerprint, source } }).returning();
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspaceMigration: {
schema: SCHEMA, operationId: operation!.id, state: "running", sourceFingerprint, source,
} }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id));
return { workspace, authority: null, operation: operation!, source, sourceFingerprint };
});
if (reserved.authority) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: true };
const operation = reserved.operation!;
const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.companyId, workspaceId: input.workspaceId,
operationId: operation.id, generation: 1, epoch: 1 });
const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-migration-"));
try {
const archivePath = path.join(scratch, "workspace.pcws");
const captured = await captureRemoteWorkspaceDirectory({ cwd: reserved.workspace.cwd ?? reserved.workspace.providerRef!, archivePath });
const ref = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage, snapshotId: snapshot.id,
companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest });
const restoredArchive = path.join(scratch, "verified.pcws"), restored = path.join(scratch, "restored");
await loadWorkspaceSnapshotArchive({ storage: input.storage, reference: ref, targetPath: restoredArchive });
await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest });
const currentArchive = path.join(scratch, "current.pcws");
const current = await captureRemoteWorkspaceDirectory({ cwd: reserved.workspace.cwd ?? reserved.workspace.providerRef!, archivePath: currentArchive });
if (current.manifestDigest !== captured.manifestDigest) throw new Error("remote_workspace_migration_source_changed");
await lock.execute(sql`select 1`);
const publication = await input.db.transaction(async tx => {
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
const existing = readRemoteWorkspaceAuthority(workspace.metadata);
if (existing?.snapshotId === ref.id) return null;
if (existing || fingerprint(workspace) !== reserved.sourceFingerprint
|| object(workspace.metadata?.remoteWorkspaceMigration).operationId !== operation.id) throw new Error("remote_workspace_migration_publication_changed");
if (workspace.sourceIssueId) await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, workspace.sourceIssueId)).for("update");
const validation = await inspect(tx as unknown as Db, { ...input, operationId: operation.id });
if (validation.reasons.length) throw new Error(`remote_workspace_migration_blocked:${validation.reasons.join(",")}`);
const [uploaded] = await tx.select().from(executionWorkspaceSnapshots).where(and(eq(executionWorkspaceSnapshots.companyId, workspace.companyId),
eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), eq(executionWorkspaceSnapshots.id, ref.id))).for("update");
if (!uploaded || uploaded.status !== "uploaded" || uploaded.sourceOperationId !== operation.id) throw new Error("remote_workspace_migration_snapshot_changed");
const authority: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: ref.id,
source: reserved.source, location: null, writerRunId: null, dirty: false };
const metadata: Record<string, unknown> = { ...workspace.metadata, remoteWorkspace: authority }; delete metadata.remoteWorkspaceMigration;
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
await tx.update(executionWorkspaceSnapshots).set({ status: "committed", deleteAfter: null, updatedAt: new Date() }).where(eq(executionWorkspaceSnapshots.id, ref.id));
await tx.update(executionWorkspaces).set({ providerType: "cloud_sandbox", providerRef: null, cwd: `/workspaces/${workspace.id}`, metadata,
updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id));
await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(),
metadata: { ...operation.metadata, snapshotId: ref.id, manifestDigest: ref.manifestDigest } }).where(eq(workspaceOperations.id, operation.id));
return (await persistActivity(tx as unknown as Db, { companyId: workspace.companyId, actorType: "system", actorId: "workspace-migration",
action: "workspace.remote_storage_migrated", entityType: "execution_workspace", entityId: workspace.id,
details: { snapshotId: ref.id, operationId: operation.id, manifestDigest: ref.manifestDigest } })).publication;
});
if (publication) publishActivity(publication);
return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: false };
} finally { await removeTaskDirectoryTree(scratch); }
});
}
@@ -0,0 +1,326 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { captureRemoteWorkspaceDirectory, remoteWorkspaceArchiveScript, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
const archiveIo = vi.hoisted(() => ({ afterChmod: undefined as ((target: string, mode: number) => void) | undefined }));
vi.mock("node:fs/promises", async importOriginal => {
const actual = await importOriginal<typeof import("node:fs/promises")>();
return { ...actual, async chmod(target: string, mode: number) {
await actual.chmod(target, mode); archiveIo.afterChmod?.(target, mode);
} };
});
const execute = promisify(execFile);
const env = { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull,
GIT_AUTHOR_NAME: "Snapshot", GIT_COMMITTER_NAME: "Snapshot", GIT_AUTHOR_EMAIL: "snapshot@example.test", GIT_COMMITTER_EMAIL: "snapshot@example.test" };
const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C", cwd, ...args], { env })).stdout.trim();
describe("portable remote workspace archives", () => {
let root: string;
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-archive-test-")); });
afterEach(async () => { archiveIo.afterChmod = undefined; await fs.rm(root, { recursive: true, force: true }); });
async function source() { const cwd = path.join(root, "source"); await fs.mkdir(cwd); return cwd; }
async function capture(cwd: string) { const archivePath = path.join(root, "snapshot.pcws"); return { archivePath, ...await captureRemoteWorkspaceDirectory({ cwd, archivePath }) }; }
async function restore(snapshot: Awaited<ReturnType<typeof capture>>) {
const targetDir = path.join(root, "restored");
await restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir, expectedManifestDigest: snapshot.manifestDigest });
return targetDir;
}
async function sourceBytes(directory: string): Promise<string> {
const entries: unknown[] = [];
for (const name of (await fs.readdir(directory)).sort()) {
const target = path.join(directory, name), stat = await fs.lstat(target);
entries.push([name, stat.mode & 0o777, stat.isDirectory() ? await sourceBytes(target)
: stat.isSymbolicLink() ? await fs.readlink(target) : createHash("sha256").update(await fs.readFile(target)).digest("hex")]);
}
return JSON.stringify(entries);
}
it("restores binary files, empty entries, executable modes and confined symlinks without its source", async () => {
const cwd = await source();
await fs.mkdir(path.join(cwd, "empty")); await fs.writeFile(path.join(cwd, "empty-file"), "");
const bytes = Buffer.from([0, 255, 1, 128]); await fs.writeFile(path.join(cwd, "data.bin"), bytes);
await fs.writeFile(path.join(cwd, "run"), "#!/bin/sh\n", { mode: 0o755 });
await fs.symlink("data.bin", path.join(cwd, "link"));
const snapshot = await capture(cwd); await fs.rm(cwd, { recursive: true });
const restored = await restore(snapshot);
expect(await fs.readFile(path.join(restored, "data.bin"))).toEqual(bytes);
expect(await fs.readFile(path.join(restored, "empty-file"))).toHaveLength(0);
expect(await fs.readdir(path.join(restored, "empty"))).toEqual([]);
expect((await fs.stat(path.join(restored, "run"))).mode & 0o777).toBe(0o755);
expect(await fs.readlink(path.join(restored, "link"))).toBe("data.bin");
});
it("round trips an empty workspace", async () => {
const snapshot = await capture(await source()); const restored = await restore(snapshot);
expect(await fs.readdir(restored)).toEqual([]);
});
it("rejects a Files entry limit before extraction while ordinary restores keep their full contents", async () => {
const cwd = await source(); await fs.writeFile(path.join(cwd, "first.txt"), "first\n"); await fs.writeFile(path.join(cwd, "second.txt"), "second\n");
const snapshot = await capture(cwd), targetDir = path.join(root, "limited");
await expect(restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir,
expectedManifestDigest: snapshot.manifestDigest, maxEntries: 1 })).rejects.toThrow("remote_workspace_archive_entry_limit");
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
const restored = await restore(snapshot);
expect(await fs.readFile(path.join(restored, "first.txt"), "utf8")).toBe("first\n");
expect(await fs.readFile(path.join(restored, "second.txt"), "utf8")).toBe("second\n");
});
it("checks a Files manifest byte cap before allocating or reading its body", async () => {
const archivePath = path.join(root, "oversized-manifest.pcws"), targetDir = path.join(root, "limited-manifest");
const length = Buffer.alloc(4); length.writeUInt32BE(1024);
await fs.writeFile(archivePath, Buffer.concat([Buffer.from("PCWS1\n"), length]));
await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: "a".repeat(64), maxManifestBytes: 16 }))
.rejects.toThrow("remote_workspace_archive_manifest_too_large");
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: "a".repeat(64) }))
.rejects.toThrow("remote_workspace_archive_truncated");
});
it("does not publish an empty archive when restoration was already canceled", async () => {
const snapshot = await capture(await source()), targetDir = path.join(root, "canceled");
const controller = new AbortController(), reason = new Error("restore canceled"); controller.abort(reason);
await expect(restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir,
expectedManifestDigest: snapshot.manifestDigest, signal: controller.signal })).rejects.toBe(reason);
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
});
it("removes private staging after cancellation during read-only directory mode restoration", async () => {
const cwd = await source(), directory = path.join(cwd, "readonly"); await fs.mkdir(directory);
await fs.writeFile(path.join(directory, "saved.txt"), "saved content"); await fs.chmod(directory, 0o555);
const snapshot = await capture(cwd), targetDir = path.join(root, "interrupted");
const controller = new AbortController(), reason = new Error("restore canceled while applying modes");
let appliedReadOnlyModes = 0;
archiveIo.afterChmod = (target, mode) => {
if (target.includes(".workspace-restore-") && mode === 0o555) { appliedReadOnlyModes++; controller.abort(reason); }
};
try {
await expect(restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir,
expectedManifestDigest: snapshot.manifestDigest, signal: controller.signal })).rejects.toBe(reason);
expect(appliedReadOnlyModes).toBe(1);
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
} finally { archiveIo.afterChmod = undefined; await fs.chmod(directory, 0o700); }
});
it("preserves read-only directory modes without making capture scratch unremovable", async () => {
const cwd = await source(), directory = path.join(cwd, "readonly"); await fs.mkdir(directory);
await fs.writeFile(path.join(directory, "file"), "retained\n"); await fs.chmod(directory, 0o555);
let restored: string | undefined;
try {
restored = await restore(await capture(cwd));
expect((await fs.stat(path.join(restored, "readonly"))).mode & 0o777).toBe(0o555);
expect(await fs.readFile(path.join(restored, "readonly", "file"), "utf8")).toBe("retained\n");
} finally { await fs.chmod(directory, 0o700); if (restored) await fs.chmod(path.join(restored, "readonly"), 0o700); }
});
it.each(["ordinary", "linked", "borrowed"])("restores %s Git with staged/unstaged/unpushed data after all sources disappear", async kind => {
const base = await source(); await git(base, "init", "-q", "-b", "main");
await fs.writeFile(path.join(base, "work.txt"), "base\n"); await git(base, "add", "."); await git(base, "commit", "-qm", "base");
let cwd = base;
if (kind === "linked") { cwd = path.join(root, "linked"); await git(base, "worktree", "add", "-q", "-b", "feature", cwd); }
if (kind === "borrowed") { cwd = path.join(root, "borrowed"); await git(root, "clone", "-q", "--shared", base, cwd); }
await fs.writeFile(path.join(cwd, "work.txt"), "unpushed\n"); await git(cwd, "commit", "-qam", "unpushed");
const head = await git(cwd, "rev-parse", "HEAD");
await fs.writeFile(path.join(cwd, "work.txt"), "staged\n"); await git(cwd, "add", "work.txt");
await fs.writeFile(path.join(cwd, "work.txt"), "unstaged\n"); await fs.writeFile(path.join(cwd, "untracked"), "work\n");
const snapshot = await capture(cwd); await fs.rm(base, { recursive: true, force: true }); if (cwd !== base) await fs.rm(cwd, { recursive: true });
const restored = await restore(snapshot);
expect(await git(restored, "rev-parse", "HEAD")).toBe(head);
expect(await git(restored, "show", ":work.txt")).toBe("staged");
expect(await fs.readFile(path.join(restored, "work.txt"), "utf8")).toBe("unstaged\n");
expect(await fs.readFile(path.join(restored, "untracked"), "utf8")).toBe("work\n");
expect((await fs.lstat(path.join(restored, ".git"))).isDirectory()).toBe(true);
await expect(fs.access(path.join(restored, ".git", "objects", "info", "alternates"))).rejects.toMatchObject({ code: "ENOENT" });
await git(restored, "fsck", "--full", "--no-dangling");
});
it.each(["rebase-merge", "rebase-apply", "linked-rebase", "sequencer", "MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD"])(
"rejects a real unfinished %s operation without changing its source or publishing an archive", async operation => {
let cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
await fs.writeFile(path.join(cwd, "work.txt"), "base\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
await git(cwd, "checkout", "-qb", "feature");
await fs.writeFile(path.join(cwd, "work.txt"), "feature\n"); await git(cwd, "commit", "-qam", "feature change");
const first = await git(cwd, "rev-parse", "HEAD");
await fs.writeFile(path.join(cwd, "later.txt"), "remaining sequence step\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "later change");
const second = await git(cwd, "rev-parse", "HEAD");
await git(cwd, "checkout", "-q", "main");
await fs.writeFile(path.join(cwd, "work.txt"), "main\n"); await git(cwd, "commit", "-qam", "main change");
let command: string[];
if (operation === "linked-rebase") {
const linked = path.join(root, "linked"); await git(cwd, "worktree", "add", "-q", linked, "feature"); cwd = linked;
}
if (operation.includes("rebase")) {
await git(cwd, "checkout", "-q", "feature");
command = ["rebase", operation === "rebase-apply" ? "--apply" : "--merge", "main"];
} else if (operation === "sequencer") command = ["cherry-pick", first, second];
else if (operation === "CHERRY_PICK_HEAD") command = ["cherry-pick", first];
else if (operation === "REVERT_HEAD") command = ["revert", "--no-edit", first];
else command = ["merge", "--no-edit", "feature"];
await expect(git(cwd, ...command)).rejects.toThrow();
const gitDir = await git(cwd, "rev-parse", "--absolute-git-dir");
const stateName = operation === "linked-rebase" ? "rebase-merge" : operation;
await fs.access(path.join(gitDir, stateName));
const before = await git(cwd, "status", "--porcelain=v2");
const index = await fs.readFile(path.join(gitDir, "index"));
const work = await fs.readFile(path.join(cwd, "work.txt"));
const state = (await fs.readdir(gitDir)).sort();
const bytes = await sourceBytes(cwd), administration = await sourceBytes(gitDir);
await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_git_operation_in_progress");
expect(await sourceBytes(cwd)).toBe(bytes);
expect(await sourceBytes(gitDir)).toBe(administration);
expect(await git(cwd, "status", "--porcelain=v2")).toBe(before);
expect(await fs.readFile(path.join(gitDir, "index"))).toEqual(index);
expect(await fs.readFile(path.join(cwd, "work.txt"))).toEqual(work);
expect((await fs.readdir(gitDir)).sort()).toEqual(state);
await fs.access(path.join(gitDir, stateName));
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
if (operation === "sequencer") {
const helper = path.join(root, "helper.cjs"); await fs.writeFile(helper, remoteWorkspaceArchiveScript());
await expect(execute(process.execPath, [helper, "capture", cwd, path.join(root, "remote.pcws")], { env }))
.rejects.toMatchObject({ stderr: expect.stringContaining("remote_workspace_archive_git_operation_in_progress") });
await expect(fs.access(path.join(root, "remote.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
expect(await sourceBytes(cwd)).toBe(bytes);
}
const abort = operation.includes("rebase") ? "rebase" : operation === "MERGE_HEAD" ? "merge"
: operation === "REVERT_HEAD" ? "revert" : "cherry-pick";
await git(cwd, abort, "--abort");
const head = await git(cwd, "rev-parse", "HEAD"), status = await git(cwd, "status", "--porcelain=v2");
const restored = await restore(await capture(cwd));
expect(await git(restored, "rev-parse", "HEAD")).toBe(head);
expect(await git(restored, "status", "--porcelain=v2")).toBe(status);
},
);
it("rejects an unfinished bisect without discarding its portable selection history", async () => {
const cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
for (const value of ["base", "middle", "bad"]) {
await fs.writeFile(path.join(cwd, "work.txt"), `${value}\n`); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", value);
}
await git(cwd, "bisect", "start", "HEAD", "HEAD~2");
const bytes = await sourceBytes(cwd);
await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_git_operation_in_progress");
expect(await sourceBytes(cwd)).toBe(bytes);
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
});
it("rejects an external symlink before creating an archive", async () => {
const cwd = await source(); await fs.symlink("../outside", path.join(cwd, "unsafe"));
await expect(capture(cwd)).rejects.toThrow("unsafe_symlink");
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
});
it("preserves shallow repository boundaries and a nested managed checkout", async () => {
const base = await source(); await git(base, "init", "-q", "-b", "main");
for (const value of ["one", "two"]) {
await fs.writeFile(path.join(base, "file"), value); await git(base, "add", "."); await git(base, "commit", "-qm", value);
}
const cwd = path.join(root, "task"), checkouts = path.join(cwd, ".paperclip-repositories"); await fs.mkdir(checkouts, { recursive: true });
const repository = path.join(checkouts, "repo");
await git(root, "clone", "-q", "--depth=1", `file://${base}`, repository);
const snapshot = await capture(cwd); await fs.rm(base, { recursive: true }); await fs.rm(cwd, { recursive: true });
const restored = path.join(await restore(snapshot), ".paperclip-repositories", "repo");
expect(await git(restored, "rev-parse", "--is-shallow-repository")).toBe("true");
expect(await git(restored, "rev-list", "--count", "HEAD")).toBe("1");
expect(await fs.readFile(path.join(restored, "file"), "utf8")).toBe("two");
});
it("produces the same saved contents for repeated captures of an unchanged Git clone", async () => {
const base = await source(); await git(base, "init", "-q", "-b", "main");
await fs.writeFile(path.join(base, "file"), "stable\n"); await git(base, "add", "."); await git(base, "commit", "-qm", "base");
const cwd = path.join(root, "clone"); await git(root, "clone", "-q", base, cwd);
const first = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "first.pcws") });
await new Promise(resolve => setTimeout(resolve, 1100));
const second = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "second.pcws") });
expect(second.manifestDigest).toBe(first.manifestDigest); expect(second.sha256).toBe(first.sha256);
});
it("preserves portable secondary remotes and tracking settings without local paths or executable configuration", async () => {
const cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
await fs.writeFile(path.join(cwd, "file"), "portable tracking\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
const head = await git(cwd, "rev-parse", "HEAD");
await git(cwd, "remote", "add", "origin", "https://github.com/synthetic/work.git");
await git(cwd, "remote", "add", "upstream", "ssh://git@example.invalid/team/base.git");
await git(cwd, "config", "--add", "remote.upstream.url", "https://backup.example.invalid/team/base.git");
await git(cwd, "config", "remote.upstream.pushurl", "git@example.invalid:team/publish.git");
await git(cwd, "update-ref", "refs/remotes/upstream/main", head);
await git(cwd, "config", "branch.main.remote", "upstream");
await git(cwd, "config", "branch.main.merge", "refs/heads/main");
await git(cwd, "config", "branch.main.rebase", "true");
await git(cwd, "config", "user.name", "Synthetic repository author");
await git(cwd, "config", "user.email", "repository-author@example.invalid");
await git(cwd, "remote", "add", "local-cache", path.join(root, "old-host-cache.git"));
await git(cwd, "config", "branch.cache.remote", "local-cache");
await git(cwd, "config", "branch.cache.merge", "refs/heads/main");
for (const [key, value] of [["core.hooksPath", path.join(root, "old-host-hooks")], ["core.worktree", cwd],
["filter.synthetic.smudge", "echo SYNTHETIC_SENSITIVE"], ["credential.helper", "echo SYNTHETIC_SENSITIVE"],
["include.path", path.join(root, "old-host-gitconfig")]]) await git(cwd, "config", key!, value!);
const snapshot = await capture(cwd); await fs.rm(cwd, { recursive: true });
const restored = await restore(snapshot);
expect(await git(restored, "remote")).toBe("origin\nupstream");
expect(await git(restored, "config", "--get-all", "remote.upstream.url")).toBe("ssh://git@example.invalid/team/base.git\nhttps://backup.example.invalid/team/base.git");
expect(await git(restored, "remote", "get-url", "--push", "upstream")).toBe("git@example.invalid:team/publish.git");
expect(await git(restored, "config", "remote.upstream.fetch")).toBe("+refs/heads/*:refs/remotes/upstream/*");
expect(await git(restored, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}")).toBe("upstream/main");
expect(await git(restored, "config", "branch.main.rebase")).toBe("true");
const configuration = await fs.readFile(path.join(restored, ".git", "config"), "utf8");
expect(configuration).not.toContain(root); expect(configuration).not.toContain("SYNTHETIC_SENSITIVE");
expect(configuration).not.toContain("local-cache"); expect(configuration).not.toContain('branch "cache"');
await fs.writeFile(path.join(restored, "next-file"), "later work\n"); await git(restored, "add", "next-file");
await execute("git", ["-C", restored, "-c", `core.hooksPath=${os.devNull}`, "-c", "commit.gpgsign=false", "commit", "-qm", "restored workspace continues"], {
env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_CONFIG_SYSTEM: os.devNull, GIT_CONFIG_COUNT: "0" },
});
expect(await git(restored, "log", "-1", "--format=%an <%ae>")).toBe("Synthetic repository author <repository-author@example.invalid>");
await git(restored, "fsck", "--full", "--no-dangling");
});
it.each(["https://synthetic-token@github.com/team/repo.git", "https://github.com/team/repo.git?access_token=SYNTHETIC_ONLY"])(
"rejects credentials in any saved remote URL: %s", async remoteUrl => {
const cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
await fs.writeFile(path.join(cwd, "file"), "source\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
await git(cwd, "remote", "add", "upstream", remoteUrl);
await expect(capture(cwd)).rejects.toThrow("credential_in_git_remote");
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
},
);
it("omits reinstallable dependencies while preserving deliberately tracked files", async () => {
const cwd = await source(); await fs.writeFile(path.join(cwd, "package.json"), "{}\n");
await fs.mkdir(path.join(cwd, "node_modules")); await fs.writeFile(path.join(cwd, "node_modules", "dependency"), "reinstall\n");
const first = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "dependencies.pcws") });
const targetDir = path.join(root, "without-dependencies");
await restoreRemoteWorkspaceArchive({ archivePath: path.join(root, "dependencies.pcws"), targetDir, expectedManifestDigest: first.manifestDigest });
await expect(fs.access(path.join(targetDir, "node_modules"))).rejects.toMatchObject({ code: "ENOENT" });
await git(cwd, "init", "-q", "-b", "main"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "deliberately tracked dependency");
const second = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "tracked.pcws") });
const trackedTarget = path.join(root, "with-tracked-dependencies");
await restoreRemoteWorkspaceArchive({ archivePath: path.join(root, "tracked.pcws"), targetDir: trackedTarget, expectedManifestDigest: second.manifestDigest });
expect(await fs.readFile(path.join(trackedTarget, "node_modules", "dependency"), "utf8")).toBe("reinstall\n");
});
it("rejects a symlink source root", async () => {
const cwd = await source(), linked = path.join(root, "linked-source"); await fs.symlink(cwd, linked);
await expect(capture(linked)).rejects.toThrow("source_invalid");
});
it("rejects path traversal before creating a restored directory", async () => {
const manifest = { schema: "paperclip.remote-workspace-archive.v1", entries: [{ path: "../escape", type: "file", mode: 0o600,
size: 0, sha256: createHash("sha256").digest("hex") }] };
const bytes = Buffer.from(JSON.stringify(manifest)), length = Buffer.alloc(4); length.writeUInt32BE(bytes.length);
const archivePath = path.join(root, "unsafe.pcws"), targetDir = path.join(root, "target");
await fs.writeFile(archivePath, Buffer.concat([Buffer.from("PCWS1\n"), length, bytes]));
await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: createHash("sha256").update(bytes).digest("hex") })).rejects.toThrow("invalid_entry");
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.access(path.join(root, "escape"))).rejects.toMatchObject({ code: "ENOENT" });
});
it("rejects a corrupt file without publishing a partial restored directory", async () => {
const cwd = await source(); await fs.writeFile(path.join(cwd, "file"), "saved"); const snapshot = await capture(cwd);
const file = await fs.open(snapshot.archivePath, "r+"); try { const size = (await file.stat()).size; await file.write(Buffer.from("X"), 0, 1, size - 1); } finally { await file.close(); }
await expect(restore(snapshot)).rejects.toThrow("file_hash_mismatch");
await expect(fs.access(path.join(root, "restored"))).rejects.toMatchObject({ code: "ENOENT" });
});
it("executes the generated helper in a separate Node process", async () => {
const cwd = await source(); await fs.writeFile(path.join(cwd, "file"), "remote helper\n");
await git(cwd, "init", "-q", "-b", "main"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
await git(cwd, "remote", "add", "upstream", "https://example.invalid/team/base.git");
await git(cwd, "config", "branch.main.remote", "upstream"); await git(cwd, "config", "branch.main.merge", "refs/heads/main");
const script = path.join(root, "helper.cjs"), archivePath = path.join(root, "remote.pcws"), target = path.join(root, "remote-restored");
await fs.writeFile(script, remoteWorkspaceArchiveScript());
const receipt = JSON.parse((await execute(process.execPath, [script, "capture", cwd, archivePath])).stdout);
await fs.rm(cwd, { recursive: true });
await execute(process.execPath, [script, "restore", archivePath, target, receipt.manifestDigest]);
expect(await fs.readFile(path.join(target, "file"), "utf8")).toBe("remote helper\n");
expect(await git(target, "remote", "get-url", "upstream")).toBe("https://example.invalid/team/base.git");
expect(await git(target, "config", "branch.main.remote")).toBe("upstream");
});
});
@@ -0,0 +1,352 @@
/**
* A portable, streaming workspace archive. This function is intentionally
* self-contained: the same implementation runs locally and in a sandbox.
*/
async function workspaceArchive(input: {
kind: "capture" | "restore";
cwd?: string;
archivePath: string;
manifestPath?: string;
targetDir?: string;
expectedManifestDigest?: string;
exclude?: string[];
signal?: AbortSignal;
maxEntries?: number;
maxManifestBytes?: number;
}): Promise<{ schema: "paperclip.remote-workspace-archive.v1"; manifestDigest: string; byteSize: number; sha256: string; entries: number }> {
const fs = await import("node:fs/promises");
const { createReadStream } = await import("node:fs");
const path = await import("node:path");
const os = await import("node:os");
const crypto = await import("node:crypto");
const child = await import("node:child_process");
const schema = "paperclip.remote-workspace-archive.v1" as const;
const magic = Buffer.from("PCWS1\n");
type Entry = { path: string; type: "directory" | "file" | "symlink"; mode?: number; size?: number; sha256?: string; target?: string };
const fail = (reason: string): never => { throw new Error(`remote_workspace_archive_${reason}`); };
const check = () => input.signal?.throwIfAborted();
check();
const hashFile = async (file: string) => {
check();
const hash = crypto.createHash("sha256"); let size = 0;
for await (const bytes of createReadStream(file, { signal: input.signal })) { check(); hash.update(bytes); size += bytes.length; }
return { sha256: hash.digest("hex"), byteSize: size };
};
const safeRelative = (value: unknown): value is string => typeof value === "string" && value.length > 0
&& !value.includes("\\") && !/[\u0000-\u001f\u007f]/u.test(value) && !path.posix.isAbsolute(value)
&& value.split("/").every(part => part !== "" && part !== "." && part !== "..");
const safeLink = (relative: string, target: string) => {
if (typeof target !== "string" || !target || target.includes("\\") || /[\u0000-\u001f\u007f]/u.test(target) || path.posix.isAbsolute(target)) return false;
const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(relative), target));
return resolved !== ".." && !resolved.startsWith("../");
};
const git = (cwd: string, args: string[], stdin?: string) => new Promise<string>((resolve, reject) => {
check();
const command = child.execFile("git", ["-c", "core.fsmonitor=false", "-c", "pack.threads=1", "-c", `core.hooksPath=${os.devNull}`, "-C", cwd, ...args], {
signal: input.signal, timeout: 300_000, maxBuffer: 64 * 1024 * 1024,
env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0",
GIT_AUTHOR_NAME: "Paperclip", GIT_COMMITTER_NAME: "Paperclip", GIT_AUTHOR_EMAIL: "workspace@paperclip.invalid", GIT_COMMITTER_EMAIL: "workspace@paperclip.invalid",
GIT_AUTHOR_DATE: "@0 +0000", GIT_COMMITTER_DATE: "@0 +0000" },
}, (error, stdout) => error ? reject(error) : resolve(stdout));
if (stdin !== undefined) command.stdin?.end(stdin);
});
const portableRemoteUrl = (value: string) => {
if (!value || /[\u0000-\u0020\u007f\\]/u.test(value)) return false;
if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(value)) {
let url: URL; try { url = new URL(value); } catch { return false; }
if (!["http:", "https:", "ssh:", "git:"].includes(url.protocol)) return false;
if (url.password || (url.username && url.protocol !== "ssh:") || url.search || url.hash) fail("credential_in_git_remote");
return Boolean(url.hostname);
}
if (/^[a-z]:[\/\\]/iu.test(value)) return false;
return /^(?:[a-z0-9._-]+@)?(?:[a-z0-9._-]+|\[[a-f0-9:]+\]):[^\s]+$/iu.test(value);
};
const portableGitSettings = async (repo: string) => {
// Read only the repository's own allowlisted values. Includes, execution
// hooks, filters, credential helpers, and machine-specific paths stay out.
const raw = await git(repo, ["config", "--null", "--local", "--no-includes", "--get-regexp",
"^(remote\\..*\\.(url|pushurl|fetch)|branch\\..*\\.(remote|merge|rebase)|user\\.(name|email))$"])
.catch((error: { code?: number }) => { if (error.code === 1) return ""; throw error; });
const settings = raw.split("\0").filter(Boolean).map(row => {
const split = row.indexOf("\n"), key = row.slice(0, split), value = row.slice(split + 1);
if (split < 1 || /[\u0000-\u0020\u007f\\]/u.test(key) || /[\u0000-\u001f\u007f\\]/u.test(value)) fail("unsupported_git_setting");
return { key, value, remote: /^remote\.(.+)\.(url|pushurl|fetch)$/.exec(key), branch: /^branch\.(.+)\.(remote|merge|rebase)$/.exec(key), identity: /^user\.(name|email)$/.test(key) };
});
const remotes = new Set(settings.flatMap(item => item.remote?.[2] === "url" ? [item.remote[1]!] : []));
for (const setting of settings) if (setting.remote && ["url", "pushurl"].includes(setting.remote[2]!)) {
if (!portableRemoteUrl(setting.value)) remotes.delete(setting.remote[1]!);
}
const branches = new Map<string, boolean>();
for (const setting of settings) if (setting.branch?.[2] === "remote") {
const name = setting.branch[1]!;
branches.set(name, (branches.get(name) ?? true) && (setting.value === "." || remotes.has(setting.value)));
}
const retained: Array<{ key: string; value: string }> = [];
for (const setting of settings) {
if (setting.remote) {
if (!remotes.has(setting.remote[1]!)) continue;
if (setting.remote[2] === "fetch") {
const refs = setting.value.replace(/^[+^]/u, "").split(":");
if (refs.length > 2 || !refs.some(Boolean)) fail("unsupported_git_setting");
for (const ref of refs.filter(Boolean)) {
if (ref.startsWith("-")) fail("unsupported_git_setting");
await git(repo, ["check-ref-format", "--allow-onelevel", "--refspec-pattern", ref]).catch(() => fail("unsupported_git_setting"));
}
}
} else if (setting.branch) {
if (!(branches.get(setting.branch[1]!) ?? remotes.has("origin"))) continue;
if (setting.branch[2] === "merge") {
if (setting.value.startsWith("-")) fail("unsupported_git_setting");
await git(repo, ["check-ref-format", "--allow-onelevel", setting.value]).catch(() => fail("unsupported_git_setting"));
}
if (setting.branch[2] === "rebase" && !["true", "false", "merges", "interactive", "preserve", "yes", "no", "on", "off", "1", "0", ""].includes(setting.value)) fail("unsupported_git_setting");
} else if (!setting.identity) fail("unsupported_git_setting");
retained.push({ key: setting.key, value: setting.value });
}
return retained;
};
const assertGitOperationComplete = async (repo: string) => {
const gitDir = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim();
// A mirror plus the selected index is not a portable continuation of an
// unfinished Git operation. Never acknowledge a copy that drops its todo,
// scripts, autostash, or operation-only object references. Inspect the
// selected worktree's administration directory without following links.
for (const name of ["rebase-merge", "rebase-apply", "sequencer", "MERGE_HEAD", "CHERRY_PICK_HEAD",
"REVERT_HEAD", "REBASE_HEAD", "MERGE_AUTOSTASH", "BISECT_START"]) {
const state = await fs.lstat(path.join(gitDir, name)).catch((error: NodeJS.ErrnoException) => {
if (error.code !== "ENOENT") throw error;
return null;
});
if (state) fail("git_operation_in_progress: finish or abort the Git operation before saving the workspace");
}
};
const directories: string[] = [];
const excludedPaths = new Set<string>();
const walk = async (root: string, copyTo?: string, skipGit = false) => {
const entries: Entry[] = [];
const visit = async (relative: string, parentIsRepository = false) => {
const directory = relative ? path.join(root, relative) : root;
const directoryStat = await fs.lstat(directory);
if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink() || await fs.realpath(directory) !== directory) fail("changed");
const names = (await fs.readdir(directory)).sort();
const inRepository = parentIsRepository || names.includes(".git");
for (const name of names) {
check();
const next = relative ? `${relative}/${name}` : name;
if (!safeRelative(next)) fail("unsupported_filename");
if (input.exclude?.some(excluded => next === excluded || next.startsWith(`${excluded}/`)) || next === ".paperclip-runtime" || next.startsWith(".paperclip-runtime/")) {
excludedPaths.add(next); continue;
}
const source = path.join(root, next), target = copyTo ? path.join(copyTo, next) : null;
const stat = await fs.lstat(source);
if (skipGit && name === ".git") { if (stat.isSymbolicLink()) fail("unsafe_git_link"); directories.push(relative); continue; }
if (stat.isDirectory() && name === "node_modules") {
const packageManifest = await fs.lstat(path.join(directory, "package.json")).catch(() => null);
if (packageManifest?.isFile() && !packageManifest.isSymbolicLink()) {
// A failing Git read cannot establish that data is untracked.
const tracked = inRepository ? await git(directory, ["ls-files", "-z", "--", name]) : "";
if (!tracked) { excludedPaths.add(next); continue; }
}
}
if (stat.isDirectory()) {
entries.push({ path: next, type: "directory", mode: stat.mode & 0o777 });
if (target) await fs.mkdir(target, { mode: 0o700 });
await visit(next, inRepository);
} else if (stat.isSymbolicLink()) {
const link = await fs.readlink(source);
if (!safeLink(next, link)) fail("unsafe_symlink");
entries.push({ path: next, type: "symlink", target: link });
if (target) await fs.symlink(link, target);
} else if (stat.isFile()) {
const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
let output: Awaited<ReturnType<typeof fs.open>> | undefined;
try {
const before = await handle.stat({ bigint: true });
if (!before.isFile() || BigInt(stat.dev) !== before.dev || BigInt(stat.ino) !== before.ino || await fs.realpath(source) !== source) fail("changed");
if (target) output = await fs.open(target, "wx", 0o600);
const digest = crypto.createHash("sha256"); let bytes = 0;
for await (const part of handle.createReadStream({ autoClose: false, signal: input.signal })) {
check(); digest.update(part); bytes += part.length;
await output?.writeFile(part);
}
const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true });
if (bytes !== Number(before.size) || await fs.realpath(source) !== source || ["dev", "ino", "size", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed");
entries.push({ path: next, type: "file", mode: stat.mode & 0o777, size: bytes, sha256: digest.digest("hex") });
if (target) await fs.chmod(target, stat.mode & 0o777);
} finally { await output?.close(); await handle.close(); }
} else fail("unsupported_entry");
}
};
await visit("");
return entries;
};
const manifestHash = (manifest: { schema: string; entries: Entry[] }) => crypto.createHash("sha256").update(JSON.stringify(manifest)).digest("hex");
if (input.kind === "capture") {
if (!input.cwd) fail("source_missing");
const sourceEntry = await fs.lstat(input.cwd!);
if (!sourceEntry.isDirectory() || sourceEntry.isSymbolicLink()) fail("source_invalid");
const root = await fs.realpath(input.cwd!);
if (!(await fs.lstat(root)).isDirectory()) fail("source_invalid");
const scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-capture-")));
try {
const tree = path.join(scratch, "tree"); await fs.mkdir(tree, { mode: 0o700 });
const before = await walk(root, tree, true);
const repositories = [...new Set(directories)];
const gitSignatures = new Map<string, string>();
const signature = async (repo: string, settings?: Awaited<ReturnType<typeof portableGitSettings>>) => {
await assertGitOperationComplete(repo);
const index = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-path", "index"])).trim();
const indexHash = await hashFile(index).catch((error: NodeJS.ErrnoException) => error.code === "ENOENT" ? null : Promise.reject(error));
return JSON.stringify([await git(repo, ["status", "--porcelain=v2", "-z", "--untracked-files=no"]), await git(repo, ["for-each-ref", "--format=%(objectname) %(refname)"]), await git(repo, ["symbolic-ref", "-q", "HEAD"]).catch(() => git(repo, ["rev-parse", "HEAD"])), indexHash, settings ?? await portableGitSettings(repo)]);
};
for (let index = 0; index < repositories.length; index++) {
const relative = repositories[index]!, repo = path.join(root, relative), normalized = path.join(scratch, `repo-${index}.git`);
const settings = await portableGitSettings(repo);
gitSignatures.set(relative, await signature(repo, settings));
// Mirror copies all refs; dissociation brings borrowed objects into this
// private repository. The selected worktree's index and HEAD are separate.
await git(root, ["clone", "--mirror", "--no-hardlinks", "--dissociate", "--", repo, normalized]);
const destination = path.join(tree, relative, ".git");
await fs.rename(normalized, destination);
const sourceGit = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim();
const sourceCommon = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim();
for (const name of ["HEAD", "index", "ORIG_HEAD", "MERGE_HEAD", "MERGE_MSG", "CHERRY_PICK_HEAD", "REVERT_HEAD"]) {
await fs.copyFile(path.join(sourceGit, name), path.join(destination, name)).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; });
}
for (const directory of new Set([sourceGit, sourceCommon])) {
for (const name of await fs.readdir(directory)) if (/^sharedindex\.[a-f0-9]+$/.test(name)) await fs.copyFile(path.join(directory, name), path.join(destination, name));
}
for (const name of ["exclude", "attributes"]) {
await fs.mkdir(path.join(destination, "info"), { recursive: true });
await fs.copyFile(path.join(sourceCommon, "info", name), path.join(destination, "info", name)).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; });
}
const restoredRepo = path.join(tree, relative);
await git(restoredRepo, ["config", "core.bare", "false"]);
await git(restoredRepo, ["config", "--unset-all", "remote.origin.mirror"]).catch(() => undefined);
await git(restoredRepo, ["remote", "remove", "origin"]).catch(() => undefined);
for (const setting of settings) await git(restoredRepo, ["config", "--add", setting.key, setting.value]);
// Removing the source-only remote also removes its refs. Restore every
// source ref without using the network; objects are already self-contained.
const refs = await git(repo, ["for-each-ref", "--format=%(objectname) %(refname)"]);
const updates = refs.split("\n").filter(Boolean).map(row => {
const [object, ref] = row.split(" "); return `update ${ref} ${object}\n`;
}).join("");
if (updates) await git(restoredRepo, ["update-ref", "--stdin"], updates);
await git(restoredRepo, ["repack", "-a", "-d"]);
await fs.rm(path.join(destination, "objects", "info", "alternates"), { force: true });
await git(restoredRepo, ["fsck", "--connectivity-only", "--no-dangling"]);
if (await signature(repo) !== gitSignatures.get(relative)) fail("changed");
}
directories.length = 0;
const after = await walk(root, undefined, true);
if (JSON.stringify(before) !== JSON.stringify(after)) fail("changed");
for (const relative of repositories) await assertGitOperationComplete(path.join(root, relative));
const entries = await walk(tree);
// Keep private staging writable for cleanup; the manifest preserves the
// source modes and restoration applies them only after writing children.
const originals = new Map(before.map(entry => [entry.path, entry]));
for (const entry of entries) if (entry.type === "directory") {
const original = originals.get(entry.path); if (original) entry.mode = original.mode;
}
const manifest = { schema, exclusionPolicy: "paperclip.remote-workspace-exclusions.v1", excluded: [...excludedPaths].sort(), entries }, body = Buffer.from(JSON.stringify(manifest));
if (body.length > 64 * 1024 * 1024) fail("manifest_too_large");
const digest = manifestHash(manifest), length = Buffer.alloc(4); length.writeUInt32BE(body.length);
const output = await fs.open(input.archivePath, "wx", 0o600);
try {
await output.writeFile(Buffer.concat([magic, length, body]));
for (const entry of entries) if (entry.type === "file") {
for await (const bytes of createReadStream(path.join(tree, entry.path), { signal: input.signal })) { check(); await output.writeFile(bytes); }
}
} finally { await output.close(); }
if (input.manifestPath) await fs.writeFile(input.manifestPath, body, { flag: "wx", mode: 0o600 });
return { schema, manifestDigest: digest, ...await hashFile(input.archivePath), entries: entries.length };
} finally { await fs.rm(scratch, { recursive: true, force: true }); }
}
if (!input.targetDir || !/^[a-f0-9]{64}$/.test(input.expectedManifestDigest ?? "")) fail("restore_binding_invalid");
if (input.maxEntries !== undefined && (!Number.isSafeInteger(input.maxEntries) || input.maxEntries < 0)) fail("restore_binding_invalid");
if (input.maxManifestBytes !== undefined && (!Number.isSafeInteger(input.maxManifestBytes) || input.maxManifestBytes < 0)) fail("restore_binding_invalid");
const archive = await fs.open(input.archivePath, "r");
let staging: string | undefined;
const stagedDirectories: string[] = [];
try {
const prefix = Buffer.alloc(magic.length + 4);
if ((await archive.read(prefix, 0, prefix.length, 0)).bytesRead !== prefix.length || !prefix.subarray(0, magic.length).equals(magic)) fail("invalid_header");
check();
const manifestSize = prefix.readUInt32BE(magic.length);
if (manifestSize > 64 * 1024 * 1024 || input.maxManifestBytes !== undefined && manifestSize > input.maxManifestBytes) fail("manifest_too_large");
const bytes = Buffer.alloc(manifestSize);
if ((await archive.read(bytes, 0, manifestSize, prefix.length)).bytesRead !== manifestSize) fail("truncated");
check();
const manifest = JSON.parse(bytes.toString("utf8")) as { schema: string; entries: Entry[] };
if (manifest.schema !== schema || !Array.isArray(manifest.entries) || manifestHash(manifest) !== input.expectedManifestDigest) fail("manifest_mismatch");
if (input.maxEntries !== undefined && manifest.entries.length > input.maxEntries) fail("entry_limit");
const kinds = new Map<string, string>(); let bodySize = 0;
for (const entry of manifest.entries) {
check();
if (!entry || !safeRelative(entry.path) || kinds.has(entry.path)) fail("invalid_entry");
const parent = path.posix.dirname(entry.path);
if (parent !== "." && kinds.get(parent) !== "directory") fail("invalid_parent");
if (entry.type === "file") {
if (!Number.isSafeInteger(entry.size) || entry.size! < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256 ?? "")) fail("invalid_file");
bodySize += entry.size!;
} else if (entry.type === "symlink") { if (!safeLink(entry.path, entry.target!)) fail("unsafe_symlink"); }
else if (entry.type !== "directory") fail("invalid_type");
if (entry.type !== "symlink" && (!Number.isInteger(entry.mode) || entry.mode! < 0 || entry.mode! > 0o777)) fail("invalid_mode");
kinds.set(entry.path, entry.type);
}
if (!Number.isSafeInteger(bodySize) || (await archive.stat()).size !== prefix.length + manifestSize + bodySize) fail("size_mismatch");
const parent = await fs.realpath(path.dirname(input.targetDir!));
const existing = await fs.lstat(input.targetDir!).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; return null; });
if (existing && (!existing.isDirectory() || existing.isSymbolicLink() || (await fs.readdir(input.targetDir!)).length)) fail("target_not_empty");
check();
staging = await fs.mkdtemp(path.join(parent, ".workspace-restore-"));
let offset = prefix.length + manifestSize;
const buffer = Buffer.alloc(1024 * 1024);
for (const entry of manifest.entries) {
check(); const target = path.join(staging, entry.path);
if (entry.type === "directory") { await fs.mkdir(target, { mode: 0o700 }); stagedDirectories.push(target); }
else if (entry.type === "symlink") await fs.symlink(entry.target!, target);
else {
const output = await fs.open(target, "wx", 0o600), digest = crypto.createHash("sha256");
try {
let remaining = entry.size!;
while (remaining) {
check(); const read = await archive.read(buffer, 0, Math.min(buffer.length, remaining), offset);
if (!read.bytesRead) fail("truncated");
const part = buffer.subarray(0, read.bytesRead); await output.writeFile(part); digest.update(part);
offset += read.bytesRead; remaining -= read.bytesRead;
}
if (digest.digest("hex") !== entry.sha256) fail("file_hash_mismatch");
} finally { await output.close(); }
await fs.chmod(target, entry.mode!);
}
}
for (const entry of manifest.entries.filter(entry => entry.type === "directory").reverse()) {
check(); await fs.chmod(path.join(staging, entry.path), entry.mode!);
}
const content = await hashFile(input.archivePath);
check();
await fs.rename(staging, input.targetDir!); staging = undefined;
return { schema, manifestDigest: input.expectedManifestDigest!, ...content, entries: manifest.entries.length };
} finally {
await archive.close();
if (staging) {
for (const directory of stagedDirectories) await fs.chmod(directory, 0o700).catch(() => undefined);
await fs.rm(staging, { recursive: true, force: true });
}
}
}
export function captureRemoteWorkspaceDirectory(input: { cwd: string; archivePath: string; manifestPath?: string; exclude?: string[]; signal?: AbortSignal }) {
return workspaceArchive({ ...input, kind: "capture" });
}
export function restoreRemoteWorkspaceArchive(input: { archivePath: string; targetDir: string; expectedManifestDigest: string; signal?: AbortSignal; maxEntries?: number; maxManifestBytes?: number }) {
return workspaceArchive({ ...input, kind: "restore" });
}
export function remoteWorkspaceArchiveScript(): string {
// tsx's function-name preservation helper may appear in Function#toString.
const source = workspaceArchive.toString().replaceAll("__vite_ssr_dynamic_import__", "import");
return `const __name = (value) => value;\nconst archive = ${source};\nconst [kind, first, second, digest] = process.argv.slice(2);\narchive(kind === 'capture' ? {kind,cwd:first,archivePath:second} : {kind,archivePath:first,targetDir:second,expectedManifestDigest:digest}).then(value=>process.stdout.write(JSON.stringify(value))).catch(error=>{process.stderr.write(error.message);process.exitCode=1;});\n`;
}
@@ -0,0 +1,324 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
executionWorkspaces, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
import { createStorageService } from "../storage/service.js";
import { createS3StorageProvider } from "../storage/s3-provider.js";
import type { StorageService } from "../storage/types.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { issueRecoveryActionService } from "./issue-recovery-actions.js";
import { notifyRemoteWorkspaceSnapshots, registerRemoteWorkspaceSnapshotRetentionWork, remoteWorkspaceSnapshotRetentionService } from "./remote-workspace-snapshot-retention.js";
const url = process.env.PAPERCLIP_TEST_DATABASE_URL;
if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required");
const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport();
async function versionedS3Fixture() {
vi.stubEnv("AWS_ACCESS_KEY_ID", "synthetic-access-key");
vi.stubEnv("AWS_SECRET_ACCESS_KEY", "synthetic-secret-key");
vi.stubEnv("AWS_SESSION_TOKEN", "");
const versions: { key: string; versionId: string; marker: boolean; size: number }[] = [];
const requests: { method: string; key: string; versionId: string | null; keyMarker: string | null; versionMarker: string | null }[] = [];
const faults = { denied: false, listDenied: false, missingCursor: false, repeatCursor: false, preserveVersions: false, hangList: false, abortedLists: 0 };
let serial = 0;
const add = (key: string, marker = false, versionId = `version-${++serial}`) => {
versions.unshift({ key, versionId, marker, size: 14 });
};
const xml = (value: string) => value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
const server = createServer((req, res) => {
req.resume();
req.on("end", () => {
const parsed = new URL(req.url!, "http://fixture.invalid");
const key = decodeURIComponent(parsed.pathname.replace(/^\/bucket\/?/, ""));
const versionId = parsed.searchParams.get("versionId");
const keyMarker = parsed.searchParams.get("key-marker"), versionMarker = parsed.searchParams.get("version-id-marker");
requests.push({ method: req.method!, key, versionId, keyMarker, versionMarker });
if (parsed.searchParams.has("versions")) {
if (faults.listDenied) {
res.writeHead(403, { "content-type": "application/xml" });
res.end("<Error><Code>AccessDenied</Code><Message>Version listing permission is required</Message></Error>");
return;
}
if (faults.hangList) { res.on("close", () => { faults.abortedLists++; }); return; }
const prefix = parsed.searchParams.get("prefix")!;
const matches = versions.filter(value => value.key.startsWith(prefix)).sort((a, b) => a.key.localeCompare(b.key));
const start = keyMarker ? matches.findIndex(value => value.key === keyMarker && value.versionId === versionMarker) + 1 : 0;
const page = matches.slice(start, start + 2), truncated = matches.length > start + page.length;
const last = faults.repeatCursor && keyMarker ? { key: keyMarker, versionId: versionMarker! } : page.at(-1);
res.setHeader("content-type", "application/xml");
res.end(`<?xml version="1.0"?><ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">` +
`<IsTruncated>${truncated}</IsTruncated>` +
(truncated && !faults.missingCursor ? `<NextKeyMarker>${xml(last!.key)}</NextKeyMarker><NextVersionIdMarker>${xml(last!.versionId)}</NextVersionIdMarker>` : "") +
page.map(value => `<${value.marker ? "DeleteMarker" : "Version"}><Key>${xml(value.key)}</Key><VersionId>${xml(value.versionId)}</VersionId>` +
`<IsLatest>false</IsLatest><LastModified>2026-10-30T12:00:00Z</LastModified>${value.marker ? "" : `<Size>${value.size}</Size>`}</${value.marker ? "DeleteMarker" : "Version"}>`).join("") +
"</ListVersionsResult>");
} else if (req.method === "PUT") {
add(key); res.setHeader("x-amz-version-id", versions[0]!.versionId); res.end();
} else if (req.method === "DELETE") {
if (versionId && faults.denied) {
res.writeHead(403, { "content-type": "application/xml" });
res.end("<Error><Code>AccessDenied</Code><Message>Object retention forbids deletion</Message></Error>");
return;
}
if (versionId) {
if (!faults.preserveVersions) {
const index = versions.findIndex(value => value.key === key && value.versionId === versionId);
if (index >= 0) versions.splice(index, 1);
}
} else add(key, true);
res.writeHead(204); res.end();
} else if (req.method === "HEAD") {
const latest = versions.find(value => value.key === key);
if (!latest || latest.marker) { res.writeHead(404); res.end(); }
else { res.setHeader("content-length", latest.size); res.end(); }
} else { res.writeHead(400); res.end(); }
});
});
await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
const endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
const provider = createS3StorageProvider({ bucket: "bucket", region: "us-east-1", prefix: "scoped-prefix", endpoint, forcePathStyle: true });
return { provider, versions, requests, faults, add,
close: async () => { server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve())); } };
}
describe.skipIf(!support.supported)("remote workspace snapshot retention", () => {
let db: ReturnType<typeof createDb>, root: string, storage: StorageService;
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
const now = new Date("2026-10-30T12:00:00Z");
beforeAll(async () => {
if (url) { await applyPendingMigrations(url); db = createDb(url); }
else { temporary = await startEmbeddedPostgresTestDatabase("workspace-snapshot-retention-"); db = createDb(temporary.connectionString); }
}, 90_000);
afterAll(async () => { if (url) await closeRegisteredClients(url, { scope: "database" }); else await temporary?.cleanup(); });
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-retention-test-")); storage = createStorageService(createLocalDiskStorageProvider(root)); });
afterEach(async () => {
await db.delete(issueRecoveryActions); await db.delete(workspaceOperations); await db.delete(executionWorkspaceSnapshots);
await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(companies);
await fs.rm(root, { recursive: true, force: true });
vi.unstubAllEnvs();
});
async function seed(status: "superseded" | "pending" | "uploaded" | "committed" = "superseded", id = randomUUID()) {
const companyId = randomUUID(), workspaceId = randomUUID(), operationId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `R${companyId.slice(0, 6)}` });
const [issue] = await db.insert(issues).values({ companyId, title: "Retention source" }).returning();
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issue!.id,
mode: "isolated_workspace", strategyType: "project_primary", name: "Retained" });
const keys = ["winner", "ambiguous"].map(name => `${companyId}/workspace-snapshots/${workspaceId}/${id}/${name}.pcws`);
for (const objectKey of keys) await storage.putFile({ objectKey, companyId, namespace: "workspace-snapshots", originalFilename: null,
contentType: "application/octet-stream", body: Buffer.from("retained bytes") });
const [row] = await db.insert(executionWorkspaceSnapshots).values({ id, companyId, executionWorkspaceId: workspaceId,
sourceOperationId: operationId, generation: 1, epoch: 1, status, objectKey: keys[0],
metadata: { uploads: keys.map(objectKey => ({ objectKey })) },
deleteAfter: new Date(now.getTime() - 1000), createdAt: new Date(now.getTime() - 10 * 86400_000) }).returning();
return { row: row!, companyId, workspaceId, issueId: issue!.id, operationId, keys };
}
const service = () => remoteWorkspaceSnapshotRetentionService(db, { storage, now: () => now });
it("deletes every recorded attempt and winning object after grace", async () => {
const f = await seed(); const owner = service();
expect(await owner.tick()).toMatchObject({ deleted: 1, failed: 0 });
for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false);
expect(await owner.nextDeadlineAt()).toBeNull();
});
it("purges exact-key S3 versions and delete markers before settling retention", async () => {
const s3 = await versionedS3Fixture();
try {
storage = createStorageService(s3.provider);
const f = await seed();
await storage.putFile({ objectKey: f.keys[0]!, companyId: f.companyId, namespace: "workspace-snapshots", originalFilename: null,
contentType: "application/octet-stream", body: Buffer.from("older retry bytes") });
await storage.deleteObject(f.companyId, f.keys[0]!);
expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false);
expect(s3.requests.filter(value => value.method === "GET")).toHaveLength(0);
const neighbor = `scoped-prefix/${f.keys[0]}-neighbor`;
s3.add(neighbor);
s3.add(`scoped-prefix/${f.keys[1]}`, false, "null");
const owner = service();
expect(await owner.tick()).toMatchObject({ deleted: 1, failed: 0 });
expect(s3.versions).toEqual([expect.objectContaining({ key: neighbor })]);
const versionDeletes = s3.requests.filter(value => value.method === "DELETE" && value.versionId !== null);
expect(versionDeletes).toHaveLength(5);
expect(versionDeletes.every(value => f.keys.some(key => value.key === `scoped-prefix/${key}`))).toBe(true);
expect(s3.requests.some(value => value.keyMarker && value.versionMarker)).toBe(true);
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]!.status).toBe("deleted");
} finally { await s3.close(); }
});
it.each(["missingCursor", "repeatCursor"] as const)("does not settle S3 cleanup with an invalid %s listing", async fault => {
const s3 = await versionedS3Fixture();
try {
storage = createStorageService(s3.provider);
const f = await seed();
for (let index = 0; index < 5; index++) s3.add(`scoped-prefix/${f.keys[0]}`);
s3.faults[fault] = true;
expect(await service().tick()).toMatchObject({ deleted: 0, failed: 1 });
expect(s3.requests.filter(value => value.method === "DELETE")).toHaveLength(0);
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0])
.toMatchObject({ status: "delete_requested", lastError: "snapshot_cleanup_failed" });
} finally { await s3.close(); }
});
it.each(["denied", "listDenied", "preserveVersions"] as const)("keeps retryable S3 cleanup for %s and settles after verified recovery", async fault => {
const s3 = await versionedS3Fixture();
try {
storage = createStorageService(s3.provider);
const f = await seed();
s3.faults[fault] = true;
let at = now;
const owner = remoteWorkspaceSnapshotRetentionService(db, { storage, now: () => at });
expect(await owner.tick()).toMatchObject({ deleted: 0, failed: 1 });
expect(s3.versions.length).toBeGreaterThan(0);
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0])
.toMatchObject({ status: "delete_requested", lastError: "snapshot_cleanup_failed" });
s3.faults[fault] = false;
at = new Date(now.getTime() + 16 * 60_000);
expect(await owner.tick()).toMatchObject({ deleted: 1, failed: 0 });
expect(s3.versions).toEqual([]);
} finally { await s3.close(); }
});
it("cancels an in-flight S3 version listing and leaves its retention obligation intact", async () => {
const s3 = await versionedS3Fixture();
try {
storage = createStorageService(s3.provider);
const f = await seed();
const preAborted = new AbortController(); preAborted.abort();
const before = s3.requests.length;
await expect(storage.deleteObject(f.companyId, f.keys[0]!, { allVersions: true, signal: preAborted.signal }))
.rejects.toMatchObject({ name: "AbortError" });
expect(s3.requests).toHaveLength(before);
await expect(storage.deleteObject(randomUUID(), f.keys[0]!, { allVersions: true })).rejects.toMatchObject({ status: 403 });
expect(s3.requests).toHaveLength(before);
s3.faults.hangList = true;
const controller = new AbortController();
const running = service().tick(controller.signal);
const observed = running.catch(error => error);
await vi.waitFor(() => expect(s3.requests.some(value => value.method === "GET")).toBe(true));
controller.abort();
expect(await observed).toMatchObject({ name: "AbortError" });
await vi.waitFor(() => expect(s3.faults.abortedLists).toBe(1));
expect(s3.requests.filter(value => value.method === "DELETE")).toHaveLength(0);
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]!.status)
.toBe("delete_requested");
} finally { await s3.close(); }
});
it("pins the current snapshot and leaves no idle deadline", async () => {
const f = await seed("committed");
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { snapshotId: f.row.id } } }).where(eq(executionWorkspaces.id, f.workspaceId));
const owner = service(); expect(await owner.tick()).toMatchObject({ pinned: 1, deleted: 0 });
expect(await owner.nextDeadlineAt()).toBeNull();
expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(true);
});
it("pins active restore operations and releases an already-due generation on settlement", async () => {
const f = await seed();
const [op] = await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId,
phase: "remote_workspace_restore", status: "running", metadata: { snapshotId: f.row.id } }).returning();
const owner = service(); expect(await owner.tick()).toMatchObject({ pinned: 1 }); expect(await owner.nextDeadlineAt()).toBeNull();
await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, op!.id));
expect(await owner.tick()).toMatchObject({ deleted: 1 });
});
it("leaves healthy current versions outside later unrelated maintenance passes", async () => {
const current = await seed("committed");
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { snapshotId: current.row.id } } })
.where(eq(executionWorkspaces.id, current.workspaceId));
const owner = service();
expect(await owner.tick()).toMatchObject({ checked: 1, pinned: 1, deleted: 0 });
const obsolete = await seed();
expect(await owner.tick()).toMatchObject({ checked: 1, pinned: 0, deleted: 1 });
expect((await storage.headObject(current.companyId, current.keys[0]!)).exists).toBe(true);
expect((await storage.headObject(obsolete.companyId, obsolete.keys[0]!)).exists).toBe(false);
expect(await owner.nextDeadlineAt()).toBeNull();
});
it("revisits a parent when cleanup of an unfinished child releases its last pin", async () => {
const parent = await seed("superseded", "10000000-0000-4000-8000-000000000001");
await db.insert(executionWorkspaceSnapshots).values({ id: "20000000-0000-4000-8000-000000000001",
companyId: parent.companyId, executionWorkspaceId: parent.workspaceId, sourceOperationId: randomUUID(),
parentSnapshotId: parent.row.id, generation: 2, epoch: 1, status: "pending",
deleteAfter: new Date(now.getTime() - 1000), createdAt: new Date(now.getTime() - 10 * 86400_000) });
const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
try {
await vi.waitFor(async () => expect((await storage.headObject(parent.companyId, parent.keys[0]!)).exists).toBe(false),
{ timeout: 4000, interval: 40 });
expect(await owner.nextDeadlineAt()).toBeNull();
} finally { await worker.stop(); await coordinator.stop(); }
});
it("pins an unfinished upload and an active recovery hold", async () => {
const f = await seed("pending");
await db.insert(workspaceOperations).values({ id: f.operationId, companyId: f.companyId, executionWorkspaceId: f.workspaceId,
phase: "remote_workspace_snapshot", status: "running" });
const owner = service(); expect(await owner.tick()).toMatchObject({ pinned: 1 });
await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.id, f.operationId));
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog",
ownerType: "board", cause: "workspace_recovery", fingerprint: randomUUID(), nextAction: "Recover saved files" });
expect(await owner.tick()).toMatchObject({ pinned: 1, deleted: 0 }); expect(await owner.nextDeadlineAt()).toBeNull();
});
it("rejects a foreign upload key without deleting another company's object", async () => {
const f = await seed(); const foreignKey = `${randomUUID()}/workspace-snapshots/foreign`;
await db.update(executionWorkspaceSnapshots).set({ metadata: { uploads: [{ objectKey: foreignKey }] } }).where(eq(executionWorkspaceSnapshots.id, f.row.id));
expect(await service().tick()).toMatchObject({ failed: 1, deleted: 0 });
for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(true);
expect(await service().nextDeadlineAt()).toBeGreaterThan(now.getTime());
});
it("settles without recurring reads and wakes on the first real producer", async () => {
const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const run = vi.spyOn(owner, "tick"); const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
try {
await vi.waitFor(() => expect(run).toHaveBeenCalledOnce());
await new Promise(resolve => setTimeout(resolve, 30)); expect(run).toHaveBeenCalledOnce();
const f = await seed();
await db.transaction(async tx => { await notifyRemoteWorkspaceSnapshots(tx, `workspace:${f.workspaceId}`); });
await vi.waitFor(async () => expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false));
expect(run).toHaveBeenCalledTimes(2);
} finally { await worker.stop(); await coordinator.stop(); }
});
it("finishes bounded pages and revisits a lower UUID produced during a pass", async () => {
const later = await seed("superseded", "20000000-0000-4000-8000-000000000001");
const last = await seed("superseded", "30000000-0000-4000-8000-000000000001");
let earlier: Awaited<ReturnType<typeof seed>> | undefined;
const remove = storage.deleteObject.bind(storage); let inserted = false;
storage.deleteObject = async (...args) => {
if (!inserted) {
inserted = true; earlier = await seed("superseded", "10000000-0000-4000-8000-000000000001");
await db.transaction(async tx => { await notifyRemoteWorkspaceSnapshots(tx, `workspace:${earlier!.workspaceId}`); });
}
await remove(...args);
};
const owner = remoteWorkspaceSnapshotRetentionService(db, { storage, now: () => now, batchSize: 1 });
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const run = vi.spyOn(owner, "tick"), worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
try {
await vi.waitFor(async () => {
expect(earlier).toBeDefined();
for (const fixture of [earlier!, later, last]) for (const key of fixture.keys) {
expect((await storage.headObject(fixture.companyId, key)).exists).toBe(false);
}
}, { timeout: 8000, interval: 40 });
expect(await owner.nextDeadlineAt()).toBeNull();
const completed = run.mock.calls.length; await new Promise(resolve => setTimeout(resolve, 100));
expect(run).toHaveBeenCalledTimes(completed);
} finally { await worker.stop(); await coordinator.stop(); }
});
it("becomes quiet while recovery pins a snapshot and wakes on the real recovery resolution", async () => {
const f = await seed();
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog",
ownerType: "board", cause: "workspace_recovery", fingerprint: randomUUID(), nextAction: "Recover saved files" });
const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
try {
await worker.ready;
expect(await owner.nextDeadlineAt()).toBeNull();
expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(true);
const resolved = await issueRecoveryActionService(db).resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: f.issueId,
status: "resolved", outcome: "restored", resolutionNote: "Verified saved files" });
expect(resolved?.status).toBe("resolved");
await vi.waitFor(async () => expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false));
expect(await owner.nextDeadlineAt()).toBeNull();
} finally { await worker.stop(); await coordinator.stop(); }
});
});
@@ -0,0 +1,202 @@
import { and, asc, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from "drizzle-orm";
import { executionWorkspaceSnapshots, executionWorkspaces, heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, type EnvironmentLease } from "@paperclipai/shared";
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
import type { StorageService } from "../storage/types.js";
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
export const REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS = 7 * 86400_000;
export const REMOTE_WORKSPACE_SNAPSHOT_RETRY_MS = 15 * 60_000;
type Snapshot = typeof executionWorkspaceSnapshots.$inferSelect;
type Tx = Parameters<Parameters<Db["transaction"]>[0]>[0];
type ReadDb = Pick<Db, "select"> | Pick<Tx, "select">;
const date = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? new Date(value) : null;
export function notifyRemoteWorkspaceSnapshots(transaction: object, hint?: string) {
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.remoteWorkspaceSnapshots, hint);
}
async function pinned(db: ReadDb, row: Snapshot) {
const [workspace] = await db.select({ id: executionWorkspaces.id, sourceIssueId: executionWorkspaces.sourceIssueId,
metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, row.companyId), eq(executionWorkspaces.id, row.executionWorkspaceId),
));
if ((workspace?.metadata?.remoteWorkspace as { snapshotId?: unknown } | undefined)?.snapshotId === row.id) return true;
const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, row.companyId), eq(workspaceOperations.executionWorkspaceId, row.executionWorkspaceId),
or(inArray(workspaceOperations.status, ["running", "pending"]),
sql`(${workspaceOperations.metadata}->'saveWork'->>'schema' = 'paperclip.remote-workspace-save-work.v1'
and (${workspaceOperations.metadata}#>'{saveWork,unknown}' is distinct from 'false'::jsonb
or case when jsonb_typeof(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 'array'
then jsonb_array_length(${workspaceOperations.metadata}#>'{saveWork,activeIds}') <> 0 else true end))`),
sql`(${workspaceOperations.id} = ${row.sourceOperationId}::uuid
or ${workspaceOperations.metadata}->>'snapshotId' = ${row.id}
or ${workspaceOperations.metadata}->>'parentSnapshotId' = ${row.id})`,
)).limit(1);
if (operation) return true;
const [child] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, row.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, row.executionWorkspaceId),
eq(executionWorkspaceSnapshots.parentSnapshotId, row.id), inArray(executionWorkspaceSnapshots.status, ["pending", "uploaded"]),
)).limit(1);
if (child) return true;
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, row.companyId),
inArray(heartbeatRuns.status, ["queued", "running"]),
sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'workspaceId' = ${row.executionWorkspaceId}`,
sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'snapshotId' = ${row.id}`,
)).limit(1);
if (run) return true;
const [recovery] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, row.companyId), inArray(issueRecoveryActions.status, ["active", "escalated"]),
sql`(${workspace?.sourceIssueId ? sql`${issueRecoveryActions.sourceIssueId} = ${workspace.sourceIssueId}::uuid` : sql`false`}
or ${issueRecoveryActions.evidence}->>'snapshotId' = ${row.id}
or ${issueRecoveryActions.evidence}->>'workspaceId' = ${row.executionWorkspaceId}
or ${issueRecoveryActions.evidence}->'remoteWorkspace'->>'snapshotId' = ${row.id})`,
)).limit(1);
return Boolean(recovery);
}
/** Producers wake this owner; pinned history carries no speculative retry. */
export function remoteWorkspaceSnapshotRetentionService(db: Db, options: {
storage: StorageService; now?: () => Date; batchSize?: number;
}) {
const now = options.now ?? (() => new Date()), batch = options.batchSize ?? 50;
let after: string | null = null;
let scanning = false;
let rescanRequested = false;
async function tick(signal?: AbortSignal) {
scanning = true;
try {
const at = now(), result = { checked: 0, pinned: 0, scheduled: 0, deleted: 0, failed: 0, truncated: false };
const rows = await db.select().from(executionWorkspaceSnapshots).where(and(
sql`${executionWorkspaceSnapshots.status} <> 'deleted'`,
// Ordinary execution notifications must not turn each healthy current
// version into a per-workspace maintenance read. A version with a stale
// deadline still needs one pass to clear it; orphaned versions need grace.
or(ne(executionWorkspaceSnapshots.status, "committed"), isNotNull(executionWorkspaceSnapshots.deleteAfter),
sql`not exists (select 1 from ${executionWorkspaces} w
where w.company_id = ${executionWorkspaceSnapshots.companyId}
and w.id = ${executionWorkspaceSnapshots.executionWorkspaceId}
and w.metadata->'remoteWorkspace'->>'snapshotId' = ${executionWorkspaceSnapshots.id}::text)`),
or(isNull(executionWorkspaceSnapshots.deleteAfter),
lte(executionWorkspaceSnapshots.deleteAfter, at),
eq(executionWorkspaceSnapshots.status, "committed")),
after ? sql`${executionWorkspaceSnapshots.id} > ${after}::uuid` : undefined,
)).orderBy(asc(executionWorkspaceSnapshots.id)).limit(batch + 1);
result.truncated = rows.length > batch;
for (const candidate of rows.slice(0, batch)) {
signal?.throwIfAborted(); result.checked++; after = candidate.id;
try {
const claimed = await db.transaction(async tx => {
const [row] = await tx.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, candidate.id)).for("update");
if (!row || row.status === "deleted") return null;
if (await pinned(tx, row)) {
result.pinned++;
const metadata = { ...row.metadata };
if (row.status === "superseded" && row.deleteAfter) metadata.retentionDueAt = row.deleteAfter.toISOString();
if (row.status === "committed") delete metadata.retentionDueAt;
if (row.deleteAfter) await tx.update(executionWorkspaceSnapshots).set({ deleteAfter: null, metadata }).where(eq(executionWorkspaceSnapshots.id, row.id));
return null;
}
const deadline = row.deleteAfter ?? (row.status === "superseded" ? date(row.metadata?.retentionDueAt) : null)
?? new Date((row.status === "pending" || row.status === "uploaded" ? row.createdAt.getTime() : at.getTime()) + REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS);
if (row.status !== "delete_requested" && deadline > at) {
result.scheduled++;
await tx.update(executionWorkspaceSnapshots).set({ deleteAfter: deadline,
...(row.status === "committed" ? { status: "superseded" as const } : {}) }).where(eq(executionWorkspaceSnapshots.id, row.id));
return null;
}
if (row.status === "delete_requested" && row.deleteAfter && row.deleteAfter > at) return null;
const [saved] = await tx.update(executionWorkspaceSnapshots).set({ status: "delete_requested", deleteAfter: at, lastError: null })
.where(eq(executionWorkspaceSnapshots.id, row.id)).returning();
return saved!;
});
if (!claimed) continue;
// New readers cannot adopt a delete_requested record. Recheck all
// durable references immediately before any irreversible object I/O.
if (await pinned(db, claimed)) {
await db.update(executionWorkspaceSnapshots).set({ deleteAfter: null }).where(eq(executionWorkspaceSnapshots.id, claimed.id));
result.pinned++; continue;
}
const uploads = Array.isArray(claimed.metadata?.uploads) ? claimed.metadata.uploads : [];
const keys = new Set<string>();
for (const value of uploads) {
if (!value || typeof value !== "object" || typeof (value as { objectKey?: unknown }).objectKey !== "string") throw new Error("snapshot_upload_reference_invalid");
keys.add((value as { objectKey: string }).objectKey);
}
if (claimed.objectKey) keys.add(claimed.objectKey);
const prefix = `${claimed.companyId}/workspace-snapshots/${claimed.executionWorkspaceId}/${claimed.id}/`;
for (const key of keys) {
if (!key.startsWith(prefix) || key.includes("..") || key.includes("\\")) throw new Error("snapshot_upload_scope_invalid");
}
for (const key of keys) {
signal?.throwIfAborted();
await options.storage.deleteObject(claimed.companyId, key, { signal, allVersions: true });
if ((await options.storage.headObject(claimed.companyId, key, { signal })).exists) throw new Error("snapshot_object_still_present");
}
await db.update(executionWorkspaceSnapshots).set({ status: "deleted", deleteAfter: null, lastError: null, updatedAt: at })
.where(and(eq(executionWorkspaceSnapshots.id, claimed.id), eq(executionWorkspaceSnapshots.status, "delete_requested")));
// A parent inspected earlier in this page can now lose its last pin.
// Revisit it once instead of waiting for an unrelated execution event.
if (claimed.parentSnapshotId) rescanRequested = true;
result.deleted++;
} catch (error) {
if (signal?.aborted) throw error;
result.failed++;
await db.update(executionWorkspaceSnapshots).set({ lastError: "snapshot_cleanup_failed",
deleteAfter: new Date(at.getTime() + REMOTE_WORKSPACE_SNAPSHOT_RETRY_MS) }).where(eq(executionWorkspaceSnapshots.id, candidate.id));
}
}
if (!result.truncated) {
after = null;
// Finish the current ordered scan before revisiting a producer that may
// have inserted a lower UUID. Repeated hints cannot starve later pages.
if (rescanRequested) { rescanRequested = false; result.truncated = true; }
}
return result;
} finally { scanning = false; }
}
async function nextDeadlineAt() {
const [row] = await db.select({ at: sql<Date | null>`min(${executionWorkspaceSnapshots.deleteAfter})` }).from(executionWorkspaceSnapshots)
.where(sql`${executionWorkspaceSnapshots.status} <> 'deleted'`);
return row?.at ? new Date(row.at).getTime() : null;
}
return { tick, nextDeadlineAt, resetCursor: () => { if (scanning || after !== null) rescanRequested = true; } };
}
export function registerRemoteWorkspaceSnapshotRetentionWork(
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, service: ReturnType<typeof remoteWorkspaceSnapshotRetentionService>,
onResult: (result: Awaited<ReturnType<typeof service.tick>>) => void,
) {
let next: number | null = null;
return coordinator.register(DELIVERY_QUEUES.remoteWorkspaceSnapshots, {
retryMs: REMOTE_WORKSPACE_SNAPSHOT_RETRY_MS, pausePassiveMaintenance: true,
onSettled: () => service.resetCursor(),
run: async signal => { next = null; const result = await service.tick(signal); onResult(result);
next = result.truncated ? Date.now() + 1000 : await service.nextDeadlineAt(); },
hasPending: async () => false, nextRunAt: () => next,
});
}
/** Called in the lease-release transaction; an absent PID/timeout grants no authority. */
export async function clearRemoteWorkspaceLocationForReleasedLease(input: { db: Db | Tx; lease: EnvironmentLease }) {
if (!hasRemoteTerminationReceipt(input.lease)
|| (input.lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state !== "destroyed") return;
const workspaces = await input.db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.lease.companyId),
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->'location'->>'leaseId' = ${input.lease.id}`,
)).for("update");
for (const workspace of workspaces) {
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
if (!authority?.location || authority.dirty || !authority.snapshotId || authority.writerRunId
|| authority.location.providerLeaseId !== input.lease.providerLeaseId || authority.location.environmentId !== input.lease.environmentId) continue;
const [saved] = await input.db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.status, "committed")));
if (!saved) continue;
await notifyRemoteWorkspaceSnapshots(input.db, `workspace:${workspace.id}`);
await input.db.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspace: { ...authority, location: null } }, updatedAt: new Date() })
.where(and(eq(executionWorkspaces.id, workspace.id),
sql`${executionWorkspaces.metadata}->'remoteWorkspace' = ${JSON.stringify(authority)}::jsonb`));
}
}
@@ -0,0 +1,147 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { Readable } from "node:stream";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { applyPendingMigrations, closeRegisteredClients, createDb, executionWorkspaceSnapshots } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
import { createStorageService } from "../storage/service.js";
import type { StorageService } from "../storage/types.js";
import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
import { beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot } from "./remote-workspace-snapshot-storage.js";
const externalDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL;
if (externalDatabaseUrl && !/^remote_workspace_[a-z_]*test$/.test(new URL(externalDatabaseUrl).pathname.slice(1))) {
throw new Error("Workspace storage tests require a disposable remote_workspace_*test database");
}
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const suite = support.supported ? describe : describe.skip;
if (!support.supported) console.warn(`Remote workspace snapshot DB tests unavailable: ${support.reason}`);
suite("remote workspace snapshot storage", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: ReturnType<typeof createDb>;
let root: string, storage: StorageService;
beforeAll(async () => {
if (externalDatabaseUrl) { await applyPendingMigrations(externalDatabaseUrl); db = createDb(externalDatabaseUrl); }
else { database = await startEmbeddedPostgresTestDatabase("workspace-snapshot-"); db = createDb(database.connectionString); }
}, 90_000);
afterAll(async () => { if (externalDatabaseUrl) await closeRegisteredClients(externalDatabaseUrl); else await database?.cleanup(); });
beforeEach(async () => {
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "41".repeat(32));
root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-store-test-"));
storage = createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")));
});
afterEach(async () => { vi.unstubAllEnvs(); await db.delete(executionWorkspaceSnapshots); await fs.rm(root, { recursive: true, force: true }); });
async function fixture() {
const companyId = randomUUID(), workspaceId = randomUUID(), operationId = randomUUID();
const cwd = path.join(root, "source"), archivePath = path.join(root, "archive.pcws"); await fs.mkdir(cwd);
await fs.writeFile(path.join(cwd, "saved.txt"), "acknowledged workspace content\n");
const captured = await captureRemoteWorkspaceDirectory({ cwd, archivePath });
const row = await beginWorkspaceSnapshot(db, { companyId, workspaceId, operationId, generation: 1, epoch: 3 });
const upload = { db, storage, snapshotId: row.id, companyId, workspaceId, archivePath, manifestDigest: captured.manifestDigest };
return { companyId, workspaceId, operationId, cwd, archivePath, captured, row, upload };
}
it("streams encrypted saved data through the real ledger and filesystem object backend", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
expect(ref.byteSize).toBeGreaterThan(f.captured.byteSize);
const object = await storage.getObject(f.companyId, ref.objectKey); const parts: Buffer[] = [];
for await (const part of object.stream) parts.push(part);
expect(Buffer.concat(parts).includes(Buffer.from("acknowledged workspace content"))).toBe(false);
await fs.rm(f.cwd, { recursive: true }); await fs.rm(f.archivePath);
const restoredArchive = path.join(root, "restored.pcws"), targetDir = path.join(root, "restored");
await loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath: restoredArchive });
await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir, expectedManifestDigest: ref.manifestDigest });
expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("acknowledged workspace content\n");
const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, ref.id));
expect(row?.status).toBe("uploaded"); // The owning service still controls atomic publication.
});
it("reconciles an upload whose reply was lost after object persistence, without the old scratch file", async () => {
const f = await fixture(); const put = storage.putFile.bind(storage); let loseReply = true;
storage.putFile = async input => { const result = await put(input); if (loseReply) { loseReply = false; throw new Error("lost upload reply"); } return result; };
await expect(uploadWorkspaceSnapshot(f.upload)).rejects.toThrow("lost upload reply");
await fs.rm(f.archivePath);
const ref = await uploadWorkspaceSnapshot(f.upload);
expect(ref.manifestDigest).toBe(f.captured.manifestDigest);
expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: ref.id })).toEqual(ref);
const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, ref.id));
expect((row?.metadata?.uploads as unknown[]).length).toBe(1);
});
it("retains an unsuccessful operation and rejects a content-changing retry", async () => {
const f = await fixture(); storage.putFile = async () => { throw new Error("storage unavailable"); };
await expect(uploadWorkspaceSnapshot(f.upload)).rejects.toThrow("storage unavailable");
const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id));
expect(row?.status).toBe("pending"); expect((row?.metadata?.uploads as unknown[]).length).toBe(1);
await expect(uploadWorkspaceSnapshot({ ...f.upload, manifestDigest: "b".repeat(64) })).rejects.toThrow("operation_content_changed");
});
it("keeps operation identity and company/workspace scope exact", async () => {
const f = await fixture();
expect((await beginWorkspaceSnapshot(db, { ...f, generation: 1, epoch: 3 })).id).toBe(f.row.id);
await expect(beginWorkspaceSnapshot(db, { ...f, generation: 1, epoch: 4 })).rejects.toThrow("operation_binding_changed");
await uploadWorkspaceSnapshot(f.upload);
expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: f.row.id, companyId: randomUUID() })).toBeNull();
expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: f.row.id, workspaceId: randomUUID() })).toBeNull();
});
it("rejects object corruption and leaves the requested plaintext path unpublished", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
const objectPath = path.join(root, "objects", ref.objectKey), object = await fs.open(objectPath, "r+");
try { await object.write(Buffer.from("X"), 0, 1, 15); } finally { await object.close(); }
const targetPath = path.join(root, "corrupted.pcws");
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toThrow("object_hash_mismatch");
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
});
it("rejects a foreign wrapped key even when ciphertext metadata is unchanged", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
const foreignId = randomUUID();
await expect(loadWorkspaceSnapshotArchive({ storage, reference: { ...ref, id: foreignId, objectKey: ref.objectKey.replace(ref.id, foreignId) },
targetPath: path.join(root, "foreign.pcws") })).rejects.toThrow("key_binding_invalid");
});
it("does not overwrite an existing caller file", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload), targetPath = path.join(root, "existing.pcws");
await fs.writeFile(targetPath, "existing");
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toMatchObject({ code: "EEXIST" });
expect(await fs.readFile(targetPath, "utf8")).toBe("existing");
});
it("does not start object retrieval for an already canceled snapshot load", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
const getObject = vi.spyOn(storage, "getObject");
const controller = new AbortController(), reason = new Error("request already canceled");
controller.abort(reason);
const targetPath = path.join(root, "canceled.pcws");
const error = await loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath, signal: controller.signal }).catch(error => error);
expect(getObject).not.toHaveBeenCalled();
expect(error).toBe(reason);
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
});
it("cancels an active object body and removes private decryption scratch before rejecting", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
const ciphertext = await fs.readFile(path.join(root, "objects", ref.objectKey));
const controller = new AbortController();
let produced = 0, closed = false;
const body = new Readable({
highWaterMark: 1,
read() {
if (produced === 2) { controller.abort(new Error("client disconnected")); return; }
this.push(ciphertext.subarray(produced, ++produced));
},
destroy(error, callback) { closed = true; callback(error); },
});
storage.getObject = vi.fn(async (_companyId, _objectKey, options) => {
expect(options?.signal).toBe(controller.signal);
return { stream: body, contentLength: ciphertext.length };
});
const scratchRoot = path.join(root, "private-scratch"); await fs.mkdir(scratchRoot);
const temp = vi.spyOn(os, "tmpdir").mockReturnValue(scratchRoot);
const targetPath = path.join(root, "interrupted.pcws");
try {
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath, signal: controller.signal }))
.rejects.toMatchObject({ name: "AbortError" });
expect(closed).toBe(true); expect(body.destroyed).toBe(true);
expect(produced).toBeGreaterThan(0); expect(produced).toBeLessThan(ciphertext.length);
const atRejection = produced; await new Promise(resolve => setImmediate(resolve)); expect(produced).toBe(atRejection);
expect(await fs.readdir(scratchRoot)).toEqual([]);
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
} finally { temp.mockRestore(); }
});
});
@@ -0,0 +1,219 @@
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto";
import { createReadStream, createWriteStream } from "node:fs";
import { link, mkdtemp, open, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { pipeline } from "node:stream/promises";
import { Transform } from "node:stream";
import { and, eq } from "drizzle-orm";
import { executionWorkspaceSnapshots, type Db } from "@paperclipai/db";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import type { StoredSecretVersionMaterial } from "../secrets/types.js";
import type { StorageService } from "../storage/types.js";
import { notifyRemoteWorkspaceSnapshots, REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS } from "./remote-workspace-snapshot-retention.js";
const SCHEMA = "paperclip.workspace-snapshot.v1";
const KEY_SCHEMA = "paperclip.workspace-snapshot-key.v1";
const MAGIC = Buffer.from("PCWSE1\n");
const FORMAT = "paperclip.remote-workspace-archive.v1";
type SnapshotRow = typeof executionWorkspaceSnapshots.$inferSelect;
export interface WorkspaceSnapshotRef {
schema: typeof SCHEMA;
id: string;
companyId: string;
workspaceId: string;
generation: number;
objectKey: string;
sha256: string;
byteSize: number;
key: StoredSecretVersionMaterial;
manifestDigest: string;
}
type UploadAttempt = Omit<WorkspaceSnapshotRef, "schema" | "companyId" | "workspaceId" | "generation" | "id">;
function failure(reason: string): never { throw new Error(`remote_workspace_snapshot_${reason}`); }
const aad = (ref: Pick<WorkspaceSnapshotRef, "companyId" | "workspaceId" | "id" | "generation">) =>
Buffer.from(JSON.stringify([SCHEMA, ref.companyId, ref.workspaceId, ref.id, ref.generation]));
const scope = (input: { companyId: string; workspaceId: string; snapshotId: string }) => and(
eq(executionWorkspaceSnapshots.id, input.snapshotId), eq(executionWorkspaceSnapshots.companyId, input.companyId),
eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspaceId),
);
function reference(row: SnapshotRow): WorkspaceSnapshotRef {
if (!row.objectKey || !row.sha256 || !row.byteSize || !row.key || !row.manifestDigest) failure("not_uploaded");
const value = { schema: SCHEMA, id: row.id, companyId: row.companyId, workspaceId: row.executionWorkspaceId,
generation: row.generation, objectKey: row.objectKey!, sha256: row.sha256!, byteSize: row.byteSize!, key: row.key!, manifestDigest: row.manifestDigest! } as WorkspaceSnapshotRef;
validateReference(value);
return value;
}
function validateReference(ref: WorkspaceSnapshotRef) {
if (ref.schema !== SCHEMA || !Number.isSafeInteger(ref.generation) || ref.generation < 1
|| !ref.objectKey.startsWith(`${ref.companyId}/workspace-snapshots/${ref.workspaceId}/${ref.id}/`)
|| !/^[a-f0-9]{64}$/.test(ref.sha256) || !/^[a-f0-9]{64}$/.test(ref.manifestDigest)
|| !Number.isSafeInteger(ref.byteSize) || ref.byteSize < MAGIC.length + 12 + 16
|| !ref.key || typeof ref.key !== "object") failure("reference_invalid");
}
async function digestFile(file: string, signal?: AbortSignal) {
const digest = createHash("sha256"); let byteSize = 0;
for await (const part of createReadStream(file, { signal })) { digest.update(part); byteSize += part.length; }
return { sha256: digest.digest("hex"), byteSize };
}
async function verifyObject(storage: StorageService, ref: WorkspaceSnapshotRef, signal?: AbortSignal) {
const object = await storage.getObject(ref.companyId, ref.objectKey, { signal });
const digest = createHash("sha256"); let size = 0;
try {
for await (const chunk of object.stream) {
signal?.throwIfAborted(); size += chunk.length;
if (size > ref.byteSize) failure("object_size_mismatch");
digest.update(chunk);
}
if (size !== ref.byteSize || digest.digest("hex") !== ref.sha256) failure("object_hash_mismatch");
} finally { object.stream.destroy(); }
}
export async function beginWorkspaceSnapshot(db: Db, input: {
companyId: string; workspaceId: string; operationId: string; runId?: string | null;
parentSnapshotId?: string | null; generation: number; epoch: number;
}) {
if (!Number.isSafeInteger(input.generation) || input.generation < 1 || !Number.isSafeInteger(input.epoch) || input.epoch < 0) failure("generation_invalid");
await db.transaction(async tx => {
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${input.workspaceId}`);
await tx.insert(executionWorkspaceSnapshots).values({ companyId: input.companyId, executionWorkspaceId: input.workspaceId,
sourceOperationId: input.operationId, sourceRunId: input.runId ?? null, parentSnapshotId: input.parentSnapshotId ?? null,
generation: input.generation, epoch: input.epoch, format: FORMAT,
deleteAfter: new Date(Date.now() + REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS) }).onConflictDoNothing();
});
const [row] = await db.select().from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, input.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspaceId),
eq(executionWorkspaceSnapshots.sourceOperationId, input.operationId),
));
if (!row || row.generation !== input.generation || row.epoch !== input.epoch || row.parentSnapshotId !== (input.parentSnapshotId ?? null)
|| row.sourceRunId !== (input.runId ?? null)) failure("operation_binding_changed");
return row!;
}
export async function getWorkspaceSnapshot(db: Db, input: { companyId: string; workspaceId: string; snapshotId: string }) {
const [row] = await db.select().from(executionWorkspaceSnapshots).where(scope(input));
return row && ["uploaded", "committed", "superseded"].includes(row.status) ? reference(row) : null;
}
/** Upload attempts are journaled before remote I/O. The caller owns publication. */
export async function uploadWorkspaceSnapshot(input: {
db: Db; storage: StorageService; snapshotId: string; companyId: string; workspaceId: string;
archivePath: string; manifestDigest: string; signal?: AbortSignal;
}): Promise<WorkspaceSnapshotRef> {
const [row] = await input.db.select().from(executionWorkspaceSnapshots).where(scope(input));
if (!row) failure("missing");
if (["uploaded", "committed", "superseded"].includes(row!.status)) {
const saved = reference(row!);
if (saved.manifestDigest !== input.manifestDigest) failure("operation_content_changed");
await verifyObject(input.storage, saved, input.signal); return saved;
}
if (row!.status !== "pending" || !/^[a-f0-9]{64}$/.test(input.manifestDigest)) failure("not_writable");
const identity = { id: row!.id, companyId: row!.companyId, workspaceId: row!.executionWorkspaceId, generation: row!.generation };
const publish = async (attempt: UploadAttempt) => {
const updated = await input.db.transaction(async tx => {
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${input.workspaceId}`);
return (await tx.update(executionWorkspaceSnapshots).set({ ...attempt, status: "uploaded", updatedAt: new Date() })
.where(and(scope(input), eq(executionWorkspaceSnapshots.status, "pending"))).returning())[0];
});
const winning = updated ?? (await input.db.select().from(executionWorkspaceSnapshots).where(scope(input)))[0];
if (!winning || !["uploaded", "committed", "superseded"].includes(winning.status)) failure("publication_changed");
const saved = reference(winning!);
if (saved.manifestDigest !== input.manifestDigest) failure("operation_content_changed");
return saved;
};
// Recover a completed upload even when the previous process lost its reply
// and its scratch directory. Failed attempts remain referenced for cleanup.
const attempts = Array.isArray(row!.metadata?.uploads) ? row!.metadata!.uploads as UploadAttempt[] : [];
for (const attempt of attempts) {
if (attempt.manifestDigest !== input.manifestDigest) failure("operation_content_changed");
const ref = { schema: SCHEMA, ...identity, ...attempt } as WorkspaceSnapshotRef;
validateReference(ref);
if ((await input.storage.headObject(ref.companyId, ref.objectKey, { signal: input.signal })).exists) {
await verifyObject(input.storage, ref, input.signal); return publish(attempt);
}
}
const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-encrypt-"));
try {
const encrypted = path.join(scratch, "archive.encrypted"), dataKey = randomBytes(32), iv = randomBytes(12);
const cipher = createCipheriv("aes-256-gcm", dataKey, iv); cipher.setAAD(aad(identity));
const plaintext = await digestFile(input.archivePath, input.signal);
const { material } = await localEncryptedProvider.createVersion({ value: JSON.stringify({ schema: KEY_SCHEMA,
...identity, dataKey: dataKey.toString("base64"), plaintextSha256: plaintext.sha256, plaintextBytes: plaintext.byteSize }) });
const header = await open(encrypted, "wx", 0o600);
try { await header.writeFile(Buffer.concat([MAGIC, iv])); } finally { await header.close(); }
const incomingDigest = createHash("sha256"); let incomingBytes = 0;
const observed = new Transform({ transform(bytes, _encoding, callback) { incomingDigest.update(bytes); incomingBytes += bytes.length; callback(null, bytes); } });
await pipeline(createReadStream(input.archivePath, { signal: input.signal }), observed, cipher, createWriteStream(encrypted, { flags: "a", mode: 0o600 }), { signal: input.signal });
if (incomingBytes !== plaintext.byteSize || incomingDigest.digest("hex") !== plaintext.sha256) failure("archive_changed");
const footer = await open(encrypted, "a"); try { await footer.writeFile(cipher.getAuthTag()); } finally { await footer.close(); }
dataKey.fill(0);
const content = await digestFile(encrypted, input.signal);
const attempt: UploadAttempt = { objectKey: `${identity.companyId}/workspace-snapshots/${identity.workspaceId}/${identity.id}/${randomUUID()}.pcws`,
...content, key: material, manifestDigest: input.manifestDigest };
const recorded = await input.db.transaction(async tx => {
const [current] = await tx.select().from(executionWorkspaceSnapshots).where(scope(input)).for("update");
if (!current || current.status !== "pending") return false;
const previous = Array.isArray(current.metadata?.uploads) ? current.metadata.uploads : [];
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${input.workspaceId}`);
await tx.update(executionWorkspaceSnapshots).set({ metadata: { ...current.metadata, uploads: [...previous, attempt] }, updatedAt: new Date() }).where(scope(input));
return true;
});
if (!recorded) {
const saved = await getWorkspaceSnapshot(input.db, input);
if (!saved || saved.manifestDigest !== input.manifestDigest) failure("publication_changed");
await verifyObject(input.storage, saved, input.signal); return saved;
}
// The object key is unique per attempt, so a late losing uploader cannot
// overwrite an object already used by a committed generation.
await input.storage.putFile({ companyId: identity.companyId, namespace: "workspace-snapshots", objectKey: attempt.objectKey,
originalFilename: null, contentType: "application/octet-stream", body: createReadStream(encrypted),
byteSize: attempt.byteSize, sha256: attempt.sha256, signal: input.signal });
await verifyObject(input.storage, { schema: SCHEMA, ...identity, ...attempt } as WorkspaceSnapshotRef, input.signal);
return await publish(attempt);
} finally { await rm(scratch, { recursive: true, force: true }); }
}
/** Authentication finishes in private scratch before any plaintext is published. */
export async function loadWorkspaceSnapshotArchive(input: {
storage: StorageService; reference: WorkspaceSnapshotRef; targetPath: string; signal?: AbortSignal; temporaryParent?: string;
}): Promise<void> {
input.signal?.throwIfAborted();
const ref = input.reference; validateReference(ref);
const wrapped = JSON.parse(await localEncryptedProvider.resolveVersion({ material: ref.key, externalRef: null }));
if (wrapped.schema !== KEY_SCHEMA || wrapped.companyId !== ref.companyId || wrapped.workspaceId !== ref.workspaceId
|| wrapped.id !== ref.id || wrapped.generation !== ref.generation || !/^[a-f0-9]{64}$/.test(wrapped.plaintextSha256 ?? "")
|| !Number.isSafeInteger(wrapped.plaintextBytes) || wrapped.plaintextBytes < 0) failure("key_binding_invalid");
const scratch = await mkdtemp(path.join(input.temporaryParent ?? os.tmpdir(), "pc-workspace-decrypt-"));
try {
input.signal?.throwIfAborted();
const ciphertext = path.join(scratch, "ciphertext"), plaintext = path.join(scratch, "plaintext");
const object = await input.storage.getObject(ref.companyId, ref.objectKey, { signal: input.signal });
let received = 0; const digest = createHash("sha256");
const bounded = new Transform({ transform(bytes, _encoding, callback) {
received += bytes.length; digest.update(bytes);
callback(received > ref.byteSize ? new Error("remote_workspace_snapshot_object_size_mismatch") : null, bytes);
} });
try { await pipeline(object.stream, bounded, createWriteStream(ciphertext, { flags: "wx", mode: 0o600 }), { signal: input.signal }); }
finally { object.stream.destroy(); }
if (received !== ref.byteSize || digest.digest("hex") !== ref.sha256) failure("object_hash_mismatch");
const file = await open(ciphertext, "r"), header = Buffer.alloc(MAGIC.length + 12), tag = Buffer.alloc(16);
try { await file.read(header, 0, header.length, 0); await file.read(tag, 0, tag.length, ref.byteSize - tag.length); } finally { await file.close(); }
if (!header.subarray(0, MAGIC.length).equals(MAGIC)) failure("ciphertext_header_invalid");
const key = Buffer.from(wrapped.dataKey, "base64"); if (key.length !== 32) failure("key_invalid");
const decipher = createDecipheriv("aes-256-gcm", key, header.subarray(MAGIC.length));
decipher.setAAD(aad(ref)); decipher.setAuthTag(tag);
try {
await pipeline(createReadStream(ciphertext, { start: header.length, end: ref.byteSize - tag.length - 1, signal: input.signal }),
decipher, createWriteStream(plaintext, { flags: "wx", mode: 0o600 }), { signal: input.signal });
} finally { key.fill(0); }
const actual = await digestFile(plaintext, input.signal);
if (actual.sha256 !== wrapped.plaintextSha256 || actual.byteSize !== wrapped.plaintextBytes) failure("plaintext_hash_mismatch");
// Publish a complete file without replacing an existing destination. A
// failed/cancelled relay cannot leave a plaintext cache that looks complete.
const staged = path.join(path.dirname(input.targetPath), `.workspace-snapshot-${randomUUID()}.tmp`);
try {
await pipeline(createReadStream(plaintext, { signal: input.signal }), createWriteStream(staged, { flags: "wx", mode: 0o600 }), { signal: input.signal });
input.signal?.throwIfAborted(); await link(staged, input.targetPath);
} finally { await rm(staged, { force: true }); }
} finally { await rm(scratch, { recursive: true, force: true }); }
}
@@ -0,0 +1,486 @@
import { execFile } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
import {
captureRemoteWorkspaceArchive,
cleanupRemoteWorkspaceArchive,
prepareRemoteRepository,
prepareRemoteWorkspaceDirectory,
restoreRemoteWorkspaceDirectory,
type RemoteWorkspaceTarget,
} from "./remote-workspace-transport.js";
const exec = promisify(execFile);
describe("remote workspace transport safety with real sandbox commands", () => {
let scratch: string;
let workspace: string;
beforeEach(async () => {
scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "remote-workspace-transport-")));
workspace = path.join(scratch, "workspace");
await fs.mkdir(workspace);
});
afterEach(async () => { await fs.rm(scratch, { recursive: true, force: true }); });
async function git(cwd: string, ...args: string[]) {
return exec("git", ["-C", cwd, "-c", "user.name=Synthetic fixture", "-c", "user.email=fixture@example.invalid",
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", ...args], {
env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1", GIT_TERMINAL_PROMPT: "0" },
});
}
async function repository(options: { filter?: boolean; revisions?: boolean } = {}) {
const root = path.join(scratch, "origin");
await fs.mkdir(root);
await git(root, "init", "--initial-branch=main");
if (options.filter) await fs.writeFile(path.join(root, ".gitattributes"), "*.txt filter=synthetic-probe\n");
await fs.writeFile(path.join(root, "source.txt"), "initial synthetic source\n");
await git(root, "add", ".");
await git(root, "commit", "-m", "Initial synthetic source");
if (options.revisions) {
await fs.writeFile(path.join(root, "source.txt"), "later synthetic source\n");
await git(root, "commit", "-am", "Later synthetic source");
}
return root;
}
function target(ambientEnv: Record<string, string> = {}) {
const execute = vi.fn<CommandManagedRuntimeRunner["execute"]>(async input => {
try {
const result = await exec(input.command, input.args ?? [], {
cwd: input.cwd ?? workspace, timeout: input.timeoutMs, maxBuffer: 4 * 1024 * 1024,
// Carry only generated fixture values across this execution boundary.
env: { PATH: process.env.PATH, ...ambientEnv, ...input.env },
});
return { ...result, exitCode: 0, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() };
} catch (error) {
const failure = error as { code?: number; stdout?: string; stderr?: string; signal?: string };
return { exitCode: typeof failure.code === "number" ? failure.code : 1, stdout: failure.stdout ?? "",
stderr: failure.stderr ?? "", timedOut: false, signal: failure.signal ?? null,
pid: process.pid, startedAt: new Date().toISOString() };
}
});
const transfer: NonNullable<CommandManagedRuntimeRunner["syncOut"]> = async operations => {
const results = [];
for (const operation of operations) {
let bytesTransferred = 0;
for (const file of operation.files) {
await fs.mkdir(path.dirname(file.targetPath), { recursive: true });
await fs.cp(file.sourcePath, file.targetPath, { recursive: file.kind === "directory" });
if (file.kind === "file") bytesTransferred += (await fs.stat(file.targetPath)).size;
}
results.push({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred });
}
return { operations: results };
};
const runner: CommandManagedRuntimeRunner = { execute, syncOut: transfer, syncIn: transfer };
return { remote: { kind: "remote", transport: "sandbox", remoteCwd: workspace, runner } as RemoteWorkspaceTarget, execute };
}
it("rejects a repository ancestor symlink before writing outside the workspace", async () => {
const origin = await repository();
const outside = path.join(scratch, "outside");
await fs.mkdir(outside);
await fs.writeFile(path.join(outside, "sentinel"), "untouched\n");
await fs.symlink(outside, path.join(workspace, "repos"));
const { remote } = target();
await expect(prepareRemoteRepository({ target: remote, remoteCwd: path.join(workspace, "repos", "clone"),
repoUrl: origin, ref: "HEAD", resolveGitAuth: async () => null })).rejects.toThrow();
expect(await fs.readdir(outside)).toEqual(["sentinel"]);
expect(await fs.readFile(path.join(outside, "sentinel"), "utf8")).toBe("untouched\n");
});
it.each(["existing", "create"] as const)("selects the %s remote branch without confusing its tip with the base ref", async branchMode => {
const origin = await repository();
await git(origin, "checkout", "-b", "feature/diverged");
await fs.writeFile(path.join(origin, "source.txt"), "existing feature work\n");
await git(origin, "commit", "-am", "Existing feature work");
const featureTip = (await git(origin, "rev-parse", "HEAD")).stdout.trim();
await git(origin, "checkout", "main");
await fs.writeFile(path.join(origin, "source.txt"), "new main work\n");
await git(origin, "commit", "-am", "New main work");
const mainTip = (await git(origin, "rev-parse", "HEAD")).stdout.trim();
expect(featureTip).not.toBe(mainTip);
const { remote } = target(), root = path.join(workspace, "selected-branch");
await prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root,
preparationIdentity: createHash("sha256").update(`synthetic-${branchMode}-branch`).digest("hex"),
source: { kind: "git", repoUrl: origin, ref: "main", branchName: "feature/diverged", branchMode },
resolveGitAuth: async () => null });
expect((await git(root, "rev-parse", "HEAD")).stdout.trim()).toBe(branchMode === "existing" ? featureTip : mainTip);
expect((await git(root, "symbolic-ref", "--short", "HEAD")).stdout.trim()).toBe("feature/diverged");
expect((await git(origin, "rev-parse", "refs/heads/feature/diverged")).stdout.trim()).toBe(featureTip);
});
it("rejects a missing existing remote branch without publishing a task folder", async () => {
const origin = await repository(), { remote } = target(), root = path.join(workspace, "missing-branch");
await expect(prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root,
preparationIdentity: createHash("sha256").update("synthetic-missing-existing-branch").digest("hex"),
source: { kind: "git", repoUrl: origin, ref: "main", branchName: "feature/missing", branchMode: "existing" },
resolveGitAuth: async () => null })).rejects.toThrow();
expect(await fs.lstat(root).catch(() => null)).toBeNull();
});
it("adopts an atomic restored folder after a lost reply without overwriting later edits", async () => {
await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic files\n");
const { remote, execute } = target();
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() });
try {
const root = path.join(scratch, "restored");
const original = execute.getMockImplementation()!;
let lost = false;
execute.mockImplementation(async input => {
const result = await original(input);
if (!lost && input.args?.[0] === "-e" && input.args[1]?.includes("workspace-restored.json") && result.exitCode === 0) {
lost = true; throw new Error("synthetic lost restore reply");
}
return result;
});
const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath,
manifestDigest: captured.manifestDigest, restorationIdentity: "a".repeat(64) };
await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow("synthetic lost restore reply");
await fs.writeFile(path.join(root, "newer.txt"), "retain later edits\n");
const restored = await restoreRemoteWorkspaceDirectory(input);
expect(restored.cleanupPending).toBe(true);
await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot, operationId: restored.operationId });
expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("saved synthetic files\n");
expect(await fs.readFile(path.join(root, "newer.txt"), "utf8")).toBe("retain later edits\n");
expect(await fs.readdir(path.join(scratch, ".paperclip-runtime", "workspace-snapshots"))).toEqual([]);
await expect(restoreRemoteWorkspaceDirectory({ ...input, restorationIdentity: "b".repeat(64) })).rejects.toThrow();
expect(await fs.readFile(path.join(root, "newer.txt"), "utf8")).toBe("retain later edits\n");
} finally { await captured.cleanup(); }
});
it("keeps a completed restore available when the staging cleanup reply is lost", async () => {
await fs.writeFile(path.join(workspace, "saved.txt"), "durable synthetic files\n");
const { remote, execute } = target();
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() });
try {
const actual = execute.getMockImplementation()!;
let loseCleanup = true;
execute.mockImplementation(async input => {
const result = await actual(input);
if (loseCleanup && input.args?.[1]?.includes("remote_workspace_export_cleanup_failed") && result.exitCode === 0) {
throw new Error("synthetic lost cleanup reply");
}
return result;
});
const root = path.join(scratch, "restored-after-cleanup-failure");
const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath,
manifestDigest: captured.manifestDigest, restorationIdentity: "c".repeat(64) };
const restored = await restoreRemoteWorkspaceDirectory(input);
expect(restored.cleanupPending).toBe(true);
expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("durable synthetic files\n");
await expect(cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot,
operationId: restored.operationId })).rejects.toThrow("synthetic lost cleanup reply");
await fs.writeFile(path.join(root, "later.txt"), "keep later work\n");
loseCleanup = false;
const retried = await restoreRemoteWorkspaceDirectory(input);
await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: retried.stagingRoot, operationId: retried.operationId });
expect(await fs.readFile(path.join(root, "later.txt"), "utf8")).toBe("keep later work\n");
} finally { await captured.cleanup(); }
});
it("publishes one complete capture pair when the same operation starts simultaneously", async () => {
await fs.writeFile(path.join(workspace, "saved.txt"), "simultaneously captured work\n");
const barrier = path.join(scratch, "capture-start-barrier"); await fs.mkdir(barrier);
const { remote, execute } = target(), run = execute.getMockImplementation()!;
execute.mockImplementation(async input => {
const args = input.args ?? [];
if (args[4]?.endsWith("archive.cjs") && args[5]) {
// Hold the actual codec subprocesses until both captures have started.
// The codec and all archive writes still run unchanged on real files.
const body = Buffer.from(args[5], "base64").toString("utf8");
const controlled = `const barrierFs=require('node:fs/promises');(async()=>{
const barrier=${JSON.stringify(barrier)};await barrierFs.writeFile(barrier+'/'+process.pid,'ready');
const until=Date.now()+10000;while((await barrierFs.readdir(barrier)).length<2){
if(Date.now()>until)throw Error('capture barrier timed out');await new Promise(resolve=>setTimeout(resolve,5));}
${body}
})().catch(error=>{process.stderr.write(error.message);process.exitCode=1});`;
return run({ ...input, args: [...args.slice(0, 5), Buffer.from(controlled).toString("base64")] });
}
return run(input);
});
const operationId = randomUUID();
const outcomes = await Promise.allSettled([0, 1].map(() => captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })));
const completed = outcomes.flatMap(result => result.status === "fulfilled" ? [result.value] : []);
try {
expect(outcomes.map(result => result.status)).toEqual(["fulfilled", "fulfilled"]);
expect(await fs.readdir(barrier)).toHaveLength(2);
expect(completed[0]!.manifestDigest).toBe(completed[1]!.manifestDigest);
expect(completed[0]!.sha256).toBe(completed[1]!.sha256);
for (const saved of completed) expect(createHash("sha256").update(await fs.readFile(saved.archivePath)).digest("hex")).toBe(saved.sha256);
const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
expect((await fs.readdir(directory)).filter(name => name.startsWith("capture-"))).toEqual([]);
await fs.writeFile(path.join(workspace, "saved.txt"), "later work must not replace a completed capture\n");
const replay = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try { expect(replay.sha256).toBe(completed[0]!.sha256); }
finally { await replay.cleanup(); }
} finally { await Promise.all(completed.map(saved => saved.cleanup())); }
}, 15_000);
it("recaptures after an abandoned partial attempt without unlinking unknown staging", async () => {
await fs.writeFile(path.join(workspace, "saved.txt"), "work after interrupted capture\n");
const operationId = randomUUID(), directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
const abandoned = path.join(directory, "capture-abandoned"); await fs.mkdir(abandoned, { recursive: true });
const legacy = path.join(directory, "workspace.pcws"), partial = path.join(abandoned, "workspace.pcws");
await fs.writeFile(legacy, "unknown old in-progress archive\n"); await fs.writeFile(partial, "abandoned partial archive\n");
const { remote } = target();
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try {
expect(await fs.readFile(legacy, "utf8")).toBe("unknown old in-progress archive\n");
expect(await fs.readFile(partial, "utf8")).toBe("abandoned partial archive\n");
expect((await fs.readdir(directory)).filter(name => name.startsWith("capture-"))).toEqual(["capture-abandoned"]);
await fs.writeFile(path.join(workspace, "saved.txt"), "newer work\n");
const replay = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try { expect(replay.sha256).toBe(captured.sha256); }
finally { await replay.cleanup(); }
} finally { await captured.cleanup(); }
});
it("prefers the canonical capture and reads a completed legacy pair without replacing it", async () => {
await fs.writeFile(path.join(workspace, "saved.txt"), "immutable completed capture\n");
const operationId = randomUUID(), directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
const { remote } = target();
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try {
const canonical = path.join(directory, "captured"), archive = path.join(directory, "workspace.pcws"), receipt = path.join(directory, "receipt.json");
const legacyReceipt = { ...JSON.parse(await fs.readFile(path.join(canonical, "receipt.json"), "utf8")),
layout: "untrusted-layout", archivePath: path.join(scratch, "untrusted-source") };
await fs.writeFile(receipt, JSON.stringify(legacyReceipt));
await fs.writeFile(archive, "corrupt legacy bytes ignored when canonical exists\n");
const canonicalReplay = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try { expect(canonicalReplay.sha256).toBe(captured.sha256); }
finally { await canonicalReplay.cleanup(); }
await fs.copyFile(path.join(canonical, "workspace.pcws"), archive);
await fs.rm(canonical, { recursive: true });
await fs.writeFile(path.join(workspace, "saved.txt"), "later uncaptured work\n");
const fallback = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try { expect(fallback.sha256).toBe(captured.sha256); }
finally { await fallback.cleanup(); }
expect(await fs.readFile(receipt, "utf8")).toBe(JSON.stringify(legacyReceipt));
expect(createHash("sha256").update(await fs.readFile(archive)).digest("hex")).toBe(captured.sha256);
expect(await fs.lstat(canonical).catch(() => null)).toBeNull();
await fs.writeFile(receipt, "invalid completed receipt\n");
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId }))
.rejects.toThrow("remote_workspace_operation_failed");
expect(await fs.readFile(receipt, "utf8")).toBe("invalid completed receipt\n");
expect(createHash("sha256").update(await fs.readFile(archive)).digest("hex")).toBe(captured.sha256);
expect(await fs.lstat(canonical).catch(() => null)).toBeNull();
} finally { await captured.cleanup(); }
});
it("rejects a preexisting canonical capture symlink without touching its target", async () => {
const operationId = randomUUID(), directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
const outside = path.join(scratch, "outside-capture"); await fs.mkdir(outside); await fs.mkdir(directory, { recursive: true });
await fs.writeFile(path.join(outside, "workspace.pcws"), "outside archive sentinel\n");
await fs.symlink(outside, path.join(directory, "captured"));
const { remote } = target();
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId }))
.rejects.toThrow("remote_workspace_operation_failed");
expect(await fs.readFile(path.join(outside, "workspace.pcws"), "utf8")).toBe("outside archive sentinel\n");
expect(await fs.readdir(outside)).toEqual(["workspace.pcws"]);
});
it("preserves only the fixed unfinished Git operation diagnostic across capture commands", async () => {
await git(workspace, "init", "--initial-branch=main");
await fs.writeFile(path.join(workspace, "work.txt"), "base\n"); await git(workspace, "add", "."); await git(workspace, "commit", "-m", "base");
await git(workspace, "checkout", "-b", "feature");
await fs.writeFile(path.join(workspace, "work.txt"), "feature\n"); await git(workspace, "commit", "-am", "feature");
await git(workspace, "checkout", "main");
await fs.writeFile(path.join(workspace, "work.txt"), "main\n"); await git(workspace, "commit", "-am", "main");
await expect(git(workspace, "merge", "--no-edit", "feature")).rejects.toThrow();
const { remote, execute } = target();
const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace";
const operationId = randomUUID();
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })).rejects.toThrow(gitOperation);
const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
for (const name of ["workspace.pcws", "receipt.json"]) await expect(fs.access(path.join(directory, name))).rejects.toMatchObject({ code: "ENOENT" });
const failure = { exitCode: 1, stdout: "", stderr: `${gitOperation}\nSYNTHETIC_PRIVATE_PATH_OR_CREDENTIAL`,
timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() };
execute.mockResolvedValueOnce(failure);
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }))
.rejects.toThrow(/^remote_workspace_operation_failed$/);
execute.mockResolvedValueOnce({ ...failure, timedOut: true, stderr: gitOperation });
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }))
.rejects.toThrow(/^remote_workspace_operation_failed$/);
});
it("never follows a preexisting snapshot receipt temporary symlink", async () => {
await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n");
const operationId = randomUUID();
const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
const outside = path.join(scratch, "outside-sentinel");
await fs.mkdir(directory, { recursive: true });
await fs.writeFile(outside, "untouched\n");
const oldTemporaryPath = path.join(directory, "receipt.json.tmp");
await fs.symlink(outside, oldTemporaryPath);
const { remote } = target();
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
try {
expect(captured.sha256).toMatch(/^[a-f0-9]{64}$/);
expect(await fs.readFile(outside, "utf8")).toBe("untouched\n");
expect((await fs.lstat(oldTemporaryPath)).isSymbolicLink()).toBe(true);
} finally { await captured.cleanup(); }
});
it("does not run ambient checkout filters with the Git transport credential", async () => {
const origin = await repository({ filter: true, revisions: true });
const globalConfig = path.join(scratch, "synthetic-gitconfig");
const sentinel = path.join(scratch, "filter-received-credential");
await git(origin, "config", "--file", globalConfig, "filter.synthetic-probe.smudge",
`sh -c 'printf %s "$PAPERCLIP_GIT_TOKEN" > "${sentinel}"; cat'`);
const calibration = path.join(scratch, "filter-calibration");
await git(scratch, "clone", "--no-hardlinks", "--", origin, calibration);
await exec("git", ["-C", calibration, "checkout", "--detach", "HEAD~1"], {
env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: globalConfig, GIT_CONFIG_NOSYSTEM: "1", PAPERCLIP_GIT_TOKEN: "SYNTHETIC_ONLY" },
});
expect(await fs.readFile(sentinel, "utf8")).toBe("SYNTHETIC_ONLY");
await fs.rm(sentinel);
const { remote } = target({ GIT_CONFIG_GLOBAL: globalConfig });
// Inject a fake credential for a local synthetic source so the real Git
// subprocess boundary is exercised without contacting a credentialed host.
await prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: path.join(workspace, "clone"),
preparationIdentity: createHash("sha256").update("synthetic-filter-preparation").digest("hex"),
source: { kind: "git", repoUrl: origin, ref: "HEAD~1" },
resolveGitAuth: async () => ({ configArgs: [], env: { PAPERCLIP_GIT_TOKEN: "SYNTHETIC_ONLY" },
source: "company_secret", secretName: "SYNTHETIC_FIXTURE" }) });
expect(await fs.readFile(path.join(workspace, "clone", "source.txt"), "utf8")).toBe("initial synthetic source\n");
expect(await fs.lstat(sentinel).catch(() => null)).toBeNull();
});
it("adopts only the bound completed preparation after its reply is lost", async () => {
const origin = await repository();
const { remote, execute } = target();
const actualExecute = execute.getMockImplementation()!;
let loseReply = true;
execute.mockImplementation(async input => {
const result = await actualExecute(input);
if (loseReply && result.exitCode === 0 && !JSON.parse(input.args?.[3] ?? "{}").probeOnly) {
loseReply = false; throw new Error("synthetic lost preparation reply");
}
return result;
});
const preparedRoot = path.join(workspace, "clone");
const input = { target: remote, remoteCwd: preparedRoot,
preparationIdentity: createHash("sha256").update(JSON.stringify({ run: "synthetic", workspace: "synthetic", epoch: 1, lease: "synthetic" })).digest("hex"),
source: { kind: "git" as const, repoUrl: origin, ref: "unused-base-ref", branchName: "main", branchMode: "existing" as const },
resolveGitAuth: async () => null };
await expect(prepareRemoteWorkspaceDirectory(input)).rejects.toThrow("synthetic lost preparation reply");
await fs.writeFile(path.join(preparedRoot, "in-flight.txt"), "preserve this unsaved work\n");
// A second clone is now impossible; successful retry must use its durable
// completed-preparation receipt and retain the existing folder.
await fs.rm(origin, { recursive: true, force: true });
const unavailableAuth = vi.fn(async () => { throw new Error("synthetic Git credential service unavailable"); });
await expect(prepareRemoteWorkspaceDirectory({ ...input, resolveGitAuth: unavailableAuth })).resolves.toBeUndefined();
expect(unavailableAuth).not.toHaveBeenCalled();
expect(await fs.readFile(path.join(preparedRoot, "source.txt"), "utf8")).toBe("initial synthetic source\n");
expect(await fs.readFile(path.join(preparedRoot, "in-flight.txt"), "utf8")).toBe("preserve this unsaved work\n");
await expect(prepareRemoteWorkspaceDirectory({ ...input, preparationIdentity: "f".repeat(64), resolveGitAuth: unavailableAuth })).rejects.toThrow("remote_workspace_operation_failed");
expect(unavailableAuth).not.toHaveBeenCalled();
expect(await fs.readFile(path.join(preparedRoot, "in-flight.txt"), "utf8")).toBe("preserve this unsaved work\n");
});
it("verifies a saved repository without resolving unavailable transport credentials", async () => {
const origin = await repository();
const { remote } = target();
const root = path.join(workspace, "saved-repository");
const prepared = await prepareRemoteRepository({ target: remote, remoteCwd: root,
repoUrl: origin, ref: "HEAD", resolveGitAuth: async () => null });
await fs.writeFile(path.join(root, "unsaved.txt"), "preserve local work\n");
await fs.rm(origin, { recursive: true, force: true });
const resolveGitAuth = vi.fn(async () => { throw new Error("synthetic Git credential service unavailable"); });
const input = { target: remote, remoteCwd: root, repoUrl: origin, ref: "HEAD",
pinnedCommit: prepared.pinnedCommit, verifyOnly: true, resolveGitAuth };
await expect(prepareRemoteRepository(input)).resolves.toEqual(prepared);
await expect(prepareRemoteRepository({ ...input, pinnedCommit: "0".repeat(40) })).rejects.toThrow("remote_workspace_operation_failed");
await expect(prepareRemoteRepository({ ...input, repoUrl: `${origin}-foreign` })).rejects.toThrow("remote_workspace_operation_failed");
expect(resolveGitAuth).not.toHaveBeenCalled();
expect(await fs.readFile(path.join(root, "unsaved.txt"), "utf8")).toBe("preserve local work\n");
});
it.each(["path", "file-url"] as const)("verifies an archived secondary %s repository after its local origin disappears", async sourceKind => {
const origin = await repository(), { remote } = target();
const repoUrl = sourceKind === "path" ? origin : pathToFileURL(origin).href;
const root = path.join(workspace, "secondary");
const prepared = await prepareRemoteRepository({ target: remote, remoteCwd: root, repoUrl,
ref: "HEAD", resolveGitAuth: async () => null });
await fs.writeFile(path.join(root, "source.txt"), "staged imported work\n");
await git(root, "add", "source.txt");
await fs.writeFile(path.join(root, "source.txt"), "unstaged imported work\n");
await fs.writeFile(path.join(root, "untracked.txt"), "untracked imported work\n");
const status = (await git(root, "status", "--porcelain=v2")).stdout;
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() });
try {
await fs.rm(root, { recursive: true });
await fs.rm(origin, { recursive: true });
const restoredRoot = path.join(scratch, "saved-root");
const restored = await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: restoredRoot,
archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, restorationIdentity: "c".repeat(64) });
await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot, operationId: restored.operationId });
const secondary = path.join(restoredRoot, "secondary");
expect((await git(secondary, "remote")).stdout.trim()).toBe("");
const resolveGitAuth = vi.fn(async () => { throw new Error("original Git source is unavailable"); });
const input = { target: remote, remoteCwd: secondary, repoUrl, ref: "HEAD",
pinnedCommit: prepared.pinnedCommit, savedArchive: true, resolveGitAuth };
await expect(prepareRemoteRepository(input)).resolves.toEqual({ ...prepared, cwd: secondary });
expect((await git(secondary, "status", "--porcelain=v2")).stdout).toBe(status);
expect(await fs.readFile(path.join(secondary, "source.txt"), "utf8")).toBe("unstaged imported work\n");
await expect(prepareRemoteRepository({ ...input, savedArchive: false })).rejects.toThrow("remote_workspace_operation_failed");
await expect(prepareRemoteRepository({ ...input, pinnedCommit: "0".repeat(40) })).rejects.toThrow("remote_workspace_operation_failed");
const missing = path.join(restoredRoot, "missing");
await expect(prepareRemoteRepository({ ...input, remoteCwd: missing })).rejects.toThrow("remote_workspace_operation_failed");
expect(await fs.lstat(missing).catch(() => null)).toBeNull();
await git(secondary, "remote", "add", "origin", "https://example.invalid/foreign.git");
await expect(prepareRemoteRepository(input)).rejects.toThrow("remote_workspace_operation_failed");
expect(resolveGitAuth).not.toHaveBeenCalled();
} finally { await captured.cleanup(); }
});
it("keeps the preparation pin and later agent commits across the next admission", async () => {
const origin = await repository(), { remote } = target(), root = path.join(workspace, "managed");
const prepared = await prepareRemoteRepository({ target: remote, remoteCwd: root,
repoUrl: origin, ref: "HEAD", resolveGitAuth: async () => null });
await fs.writeFile(path.join(root, "source.txt"), "agent committed work\n");
await git(root, "commit", "-am", "Agent progress");
const head = (await git(root, "rev-parse", "HEAD")).stdout.trim();
expect(head).not.toBe(prepared.pinnedCommit);
await fs.writeFile(path.join(root, "untracked.txt"), "still in progress\n");
await fs.rm(origin, { recursive: true, force: true });
const resolveGitAuth = vi.fn(async () => { throw new Error("origin unavailable"); });
expect(await prepareRemoteRepository({ target: remote, remoteCwd: root, repoUrl: origin,
ref: "HEAD", pinnedCommit: prepared.pinnedCommit, resolveGitAuth })).toEqual(prepared);
expect((await git(root, "rev-parse", "HEAD")).stdout.trim()).toBe(head);
expect(await fs.readFile(path.join(root, "untracked.txt"), "utf8")).toBe("still in progress\n");
expect(resolveGitAuth).not.toHaveBeenCalled();
});
it("uses trusted SSH rewrites only for clone and ignores ambient Git configuration", async () => {
const origin = await repository({ revisions: true });
const remoteUrl = "ssh://git@127.0.0.1:1/synthetic-repository";
const rewrite = { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: `url.${pathToFileURL(origin).href}.insteadOf`,
GIT_CONFIG_VALUE_0: remoteUrl };
const resolveGitAuth = async () => ({ configArgs: [], env: rewrite,
source: "company_secret" as const, secretName: "SYNTHETIC_FIXTURE" });
const { remote } = target({ GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "remote.origin.url",
GIT_CONFIG_VALUE_0: "synthetic-wrong-ambient-origin" });
const root = path.join(workspace, "rewritten-root");
await prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root,
preparationIdentity: "a".repeat(64), source: { kind: "git", repoUrl: remoteUrl, ref: "HEAD~1" }, resolveGitAuth });
expect(await fs.readFile(path.join(root, "source.txt"), "utf8")).toBe("initial synthetic source\n");
const nested = path.join(workspace, "rewritten-repository");
const receipt = await prepareRemoteRepository({ target: remote, remoteCwd: nested, repoUrl: remoteUrl,
ref: "HEAD~1", resolveGitAuth });
expect(await fs.readFile(path.join(nested, "source.txt"), "utf8")).toBe("initial synthetic source\n");
expect((await git(nested, "config", "--get", "remote.origin.url")).stdout.trim()).toBe(remoteUrl);
const unavailableAuth = vi.fn(async () => { throw new Error("unexpected credential lookup"); });
await expect(prepareRemoteRepository({ target: remote, remoteCwd: nested, repoUrl: remoteUrl,
ref: "HEAD~1", pinnedCommit: receipt.pinnedCommit, resolveGitAuth: unavailableAuth })).resolves.toEqual(receipt);
expect(unavailableAuth).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,270 @@
import { createHash, randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import type { RemoteWorkspaceSource } from "@paperclipai/shared";
import { remoteWorkspaceArchiveScript } from "./remote-workspace-snapshot-archive.js";
import type { GitRemoteAuthProvider } from "./git-credentials.js";
export type RemoteWorkspaceTarget = Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
export function requireRemoteWorkspaceTarget(target: AdapterExecutionTarget | null): RemoteWorkspaceTarget {
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask
|| !target.runner.syncIn || !target.runner.syncOut) throw new Error("remote_workspace_transport_unsupported");
return target;
}
async function command(target: RemoteWorkspaceTarget, args: string[], env?: Record<string, string>, timeoutMs = 300_000) {
const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd,
env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true });
if (result.exitCode !== 0 || result.timedOut) {
const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace";
throw new Error(!result.timedOut && result.stderr.trim() === gitOperation ? gitOperation : "remote_workspace_operation_failed");
}
return result.stdout;
}
const PREPARE = String.raw`
const fs = require('node:fs/promises'), path = require('node:path'), cp = require('node:child_process');
const {promisify}=require('node:util'); const exec=promisify(cp.execFile);
(async()=>{
const [root,raw]=process.argv.slice(1), input=JSON.parse(raw);
if (!path.isAbsolute(root)||!/^[a-f0-9]{64}$/.test(input.preparationIdentity)) throw Error('invalid_root');
const noConfig={GIT_TERMINAL_PROMPT:'0',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_CONFIG_SYSTEM:'/dev/null',GIT_CONFIG_COUNT:'0',GIT_CONFIG_PARAMETERS:'',GIT_TEMPLATE_DIR:'',GIT_ASKPASS:'',SSH_ASKPASS:''};
const isolated={...process.env,...noConfig};for(const key of Object.keys(isolated))if(/^GIT_CONFIG_(KEY|VALUE)_\d+$/.test(key))delete isolated[key];
const cloneEnv={...isolated};for(const key of input.authEnvKeys)if(/^GIT_CONFIG_(COUNT|(KEY|VALUE)_\d+)$/.test(key)&&process.env[key]!==undefined)cloneEnv[key]=process.env[key];
const localEnv={...isolated};for(const key of input.authEnvKeys)delete localEnv[key];Object.assign(localEnv,noConfig);
let current='/'; for(const part of path.dirname(root).split('/').filter(Boolean)) {
current=path.join(current,part); await fs.mkdir(current,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});
const s=await fs.lstat(current); if(!s.isDirectory()||s.isSymbolicLink())throw Error('unsafe_root');
}
const prior=await fs.lstat(root).catch(e=>{if(e.code==='ENOENT')return null;throw e});
if(prior){
if(!prior.isDirectory()||prior.isSymbolicLink()||await fs.realpath(root)!==root)throw Error('unsafe_root');
const runtime=path.join(root,'.paperclip-runtime'),marker=path.join(runtime,'workspace-ready.json');
const dir=await fs.lstat(runtime);if(!dir.isDirectory()||dir.isSymbolicLink())throw Error('unsafe_receipt');
const file=await fs.lstat(marker);if(!file.isFile()||file.isSymbolicLink()||file.nlink!==1||file.size>16384)throw Error('unsafe_receipt');
if(JSON.parse(await fs.readFile(marker,'utf8')).preparationIdentity!==input.preparationIdentity)throw Error('workspace_owner_changed');
process.stdout.write(JSON.stringify({cwd:root}));return;
}
if(input.probeOnly){process.stdout.write(JSON.stringify({cloneRequired:true}));return;}
const staged=await fs.mkdtemp(root+'.preparing-');
if(input.kind==='git'){
await exec('git',['-c','core.hooksPath=/dev/null','-c','init.templateDir=',...input.authArgs,'clone','--no-hardlinks','--',input.repoUrl,staged],{env:cloneEnv});
const git=async args=>exec('git',['-c','core.hooksPath=/dev/null','-C',staged,...args],{env:localEnv});
if(input.branchMode==='existing'){
if(!input.branchName)throw Error('existing_branch_required');
await git(['check-ref-format','--branch',input.branchName]);
const remoteRef='refs/remotes/origin/'+input.branchName;
await git(['rev-parse','--verify','--quiet',remoteRef+'^{commit}']);
const local=await git(['show-ref','--verify','--quiet','refs/heads/'+input.branchName]).then(()=>true,()=>false);
await git(local?['checkout',input.branchName]:['checkout','-b',input.branchName,remoteRef]);
}else{
if(input.ref)await git(['checkout','--detach',input.ref]);
if(input.branchName)await git(['checkout','-B',input.branchName]);
}
await fs.mkdir(path.join(staged,'.git','info'),{recursive:true,mode:0o700});
await fs.appendFile(path.join(staged,'.git','info','exclude'),'\n/.paperclip-runtime/\n/.paperclip-repositories/\n');
}else{
await fs.writeFile(path.join(staged,'.gitignore'),'/.paperclip-runtime/\n/.paperclip-repositories/\n',{flag:'wx',mode:0o600});
}
const runtime=path.join(staged,'.paperclip-runtime');await fs.mkdir(runtime,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});
const runtimeStat=await fs.lstat(runtime);if(!runtimeStat.isDirectory()||runtimeStat.isSymbolicLink())throw Error('unsafe_receipt');
await fs.writeFile(path.join(runtime,'workspace-ready.json'),JSON.stringify({preparationIdentity:input.preparationIdentity}),{flag:'wx',mode:0o600});
if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root');
await fs.rename(staged,root);
process.stdout.write(JSON.stringify({cwd:await fs.realpath(root)}));
})().catch(()=>{process.stderr.write('remote_workspace_prepare_failed');process.exitCode=1});`;
export async function prepareRemoteWorkspaceDirectory(input: {
target: RemoteWorkspaceTarget; remoteCwd: string; source: RemoteWorkspaceSource; resolveGitAuth: GitRemoteAuthProvider;
preparationIdentity: string;
}) {
if (input.source.kind === "git") {
const prior = JSON.parse(await command(input.target, ["-e", PREPARE, input.remoteCwd,
JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity,
authArgs: [], authEnvKeys: [], probeOnly: true })])) as { cwd?: string; cloneRequired?: boolean };
if (prior.cwd === input.remoteCwd) return;
if (prior.cloneRequired !== true) throw new Error("remote_workspace_root_changed");
}
const auth = input.source.kind === "git" ? await input.resolveGitAuth(input.source.repoUrl) : null;
const output = await command(input.target, ["-e", PREPARE, input.remoteCwd,
JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity,
authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env);
const result = JSON.parse(output) as { cwd: string };
if (result.cwd !== input.remoteCwd) throw new Error("remote_workspace_root_changed");
}
export function remoteWorkspaceRepositoryPath(remoteRoot: string, repository: {
id: string; relativePath: string; readOnly?: boolean;
}) {
if (!/^[a-zA-Z0-9_-]+$/.test(repository.id) || path.posix.isAbsolute(repository.relativePath)
|| repository.relativePath.split("/").some(part => !part || part === "." || part === "..")
|| /[\\\u0000-\u001f]/u.test(repository.relativePath)) throw new Error("remote_workspace_repository_path_invalid");
return repository.readOnly
? path.posix.join(path.posix.dirname(remoteRoot), ".paperclip-referenced", path.posix.basename(remoteRoot), repository.id)
: path.posix.join(remoteRoot, repository.relativePath);
}
export async function prepareRemoteRepository(input: {
target: RemoteWorkspaceTarget; remoteCwd: string; repoUrl: string; ref: string | null;
pinnedCommit?: string; verifyOnly?: boolean; savedArchive?: boolean; resolveGitAuth: GitRemoteAuthProvider;
}) {
const script = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');const {promisify}=require('node:util');const exec=promisify(cp.execFile);
(async()=>{const [root,raw]=process.argv.slice(1),input=JSON.parse(raw);const noConfig={GIT_TERMINAL_PROMPT:'0',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_CONFIG_SYSTEM:'/dev/null',GIT_CONFIG_COUNT:'0',GIT_CONFIG_PARAMETERS:'',GIT_TEMPLATE_DIR:'',GIT_ASKPASS:'',SSH_ASKPASS:''};
const env={...process.env,...noConfig};for(const key of Object.keys(env))if(/^GIT_CONFIG_(KEY|VALUE)_\d+$/.test(key))delete env[key];
const cloneEnv={...env};for(const key of input.authEnvKeys)if(/^GIT_CONFIG_(COUNT|(KEY|VALUE)_\d+)$/.test(key)&&process.env[key]!==undefined)cloneEnv[key]=process.env[key];
const localEnv={...env};for(const key of input.authEnvKeys)delete localEnv[key];Object.assign(localEnv,noConfig);const opts={env:localEnv};
let parent='/';for(const part of path.dirname(root).split('/').filter(Boolean)){parent=path.join(parent,part);
await fs.mkdir(parent,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const entry=await fs.lstat(parent);
if(!entry.isDirectory()||entry.isSymbolicLink()||await fs.realpath(parent)!==parent)throw Error('unsafe_repository_parent');}
const existing=await fs.lstat(root).catch(e=>{if(e.code==='ENOENT')return null;throw e});
if(existing){if(!existing.isDirectory()||existing.isSymbolicLink())throw Error('unsafe_repository')}
else{if(input.verifyOnly||input.savedArchive)throw Error('repository_missing');if(input.probeOnly){process.stdout.write(JSON.stringify({cloneRequired:true}));return;}await exec('git',['-c','core.hooksPath=/dev/null','-c','init.templateDir=',...input.authArgs,'clone','--no-hardlinks','--',input.repoUrl,root],{env:cloneEnv});
if(input.ref)await exec('git',['-c','core.hooksPath=/dev/null','-C',root,'checkout','--detach',input.ref],opts);}
const git=async args=>(await exec('git',['-c','core.hooksPath=/dev/null','-C',root,...args],opts)).stdout.trim();
if(await fs.realpath(root)!==root)throw Error('unsafe_repository');
const hasOrigin=(await git(['remote'])).split('\n').includes('origin');
const origin=hasOrigin?await git(['remote','get-url','origin']):null;
if(origin!==input.repoUrl&&!(input.savedArchive&&origin===null))throw Error('repository_origin_changed');
const head=await git(['rev-parse','HEAD']);if(input.pinnedCommit)await git(['cat-file','-e',input.pinnedCommit+'^{commit}']);
const branch=await git(['symbolic-ref','--short','-q','HEAD']).catch(()=>null);process.stdout.write(JSON.stringify({cwd:root,pinnedCommit:input.pinnedCommit||head,branchName:branch}));
})().catch(()=>{process.stderr.write('remote_workspace_repository_failed');process.exitCode=1});`;
const payload = { ...input, target: undefined, resolveGitAuth: undefined };
const prior = JSON.parse(await command(input.target, ["-e", script, input.remoteCwd,
JSON.stringify({ ...payload, authArgs: [], authEnvKeys: [], probeOnly: true })])) as {
cwd?: string; pinnedCommit?: string; branchName?: string | null; cloneRequired?: boolean;
};
if (prior.cloneRequired !== true) {
if (prior.cwd !== input.remoteCwd || !/^[a-f0-9]{40,64}$/.test(prior.pinnedCommit ?? "")) throw new Error("remote_workspace_repository_receipt_invalid");
return { cwd: prior.cwd, pinnedCommit: prior.pinnedCommit!, branchName: prior.branchName ?? null };
}
const auth = await input.resolveGitAuth(input.repoUrl);
const output = await command(input.target, ["-e", script, input.remoteCwd,
JSON.stringify({ ...payload, authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env);
const value = JSON.parse(output) as { cwd: string; pinnedCommit: string; branchName: string | null };
if (value.cwd !== input.remoteCwd || !/^[a-f0-9]{40,64}$/.test(value.pinnedCommit)) throw new Error("remote_workspace_repository_receipt_invalid");
return value;
}
async function installArchiveHelper(target: RemoteWorkspaceTarget, remoteCwd: string, operationId: string) {
if (!/^[a-f0-9-]{36}$/.test(operationId)) throw new Error("remote_workspace_operation_identity_invalid");
const directory = path.posix.join(remoteCwd, ".paperclip-runtime", "workspace-snapshots", operationId);
const helper = path.posix.join(directory, "archive.cjs");
const install = String.raw`const fs=require('node:fs/promises'),path=require('node:path');(async()=>{
const [root,dir,file,body]=process.argv.slice(1);let p='/';
for(const s of root.split('/').filter(Boolean)){p=path.join(p,s);await fs.mkdir(p,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const st=await fs.lstat(p);if(st.isSymbolicLink()||!st.isDirectory())throw Error('unsafe');}
for(const s of path.relative(root,dir).split('/')){p=path.join(p,s);await fs.mkdir(p,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const st=await fs.lstat(p);if(st.isSymbolicLink()||!st.isDirectory())throw Error('unsafe');}
const existing=await fs.lstat(file).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(existing&&(!existing.isFile()||existing.isSymbolicLink()))throw Error('unsafe');
const temp=file+'.'+require('node:crypto').randomUUID()+'.tmp';await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file);
})().catch(()=>{process.exitCode=1});`;
await command(target, ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]);
return { directory, helper, archive: path.posix.join(directory, "workspace.pcws"), receipt: path.posix.join(directory, "receipt.json") };
}
/** Only immutable operation staging is removed; working files are never a target. */
export async function cleanupRemoteWorkspaceArchive(input: {
target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string;
}) {
if (!/^[a-f0-9-]{36}$/.test(input.operationId)) throw new Error("remote_workspace_operation_identity_invalid");
const script = String.raw`const fs=require('node:fs/promises'),path=require('node:path');(async()=>{
const [root,id]=process.argv.slice(1);if(!path.isAbsolute(root)||!/^[a-f0-9-]{36}$/.test(id))throw Error('invalid');
const dir=path.join(root,'.paperclip-runtime','workspace-snapshots',id);let current='/';
for(const part of dir.split('/').filter(Boolean)){current=path.join(current,part);
const entry=await fs.lstat(current).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(!entry)return;
if(!entry.isDirectory()||entry.isSymbolicLink())throw Error('unsafe');}
await fs.rm(dir,{recursive:true,force:true});
})().catch(()=>{process.stderr.write('remote_workspace_export_cleanup_failed');process.exitCode=1});`;
await command(input.target, ["-e", script, input.remoteCwd, input.operationId], undefined, 10_000);
}
export async function captureRemoteWorkspaceArchive(input: {
target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string;
}) {
const staged = await installArchiveHelper(input.target, input.remoteCwd, input.operationId);
// Publish the archive and receipt as one immutable directory. Concurrent
// invocations own separate attempts; no invocation unlinks another's archive.
const capture = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');const {promisify}=require('node:util');(async()=>{
const [helper,root,directory]=process.argv.slice(1),canonical=path.join(directory,'captured');
const readPair=async(dir,layout,required)=>{
const st=await fs.lstat(dir).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(!st)return null;
if(!st.isDirectory()||st.isSymbolicLink()||await fs.realpath(dir)!==dir)throw Error('unsafe_capture');
const receipt=path.join(dir,'receipt.json'),archive=path.join(dir,'workspace.pcws');
const receiptStat=await fs.lstat(receipt).catch(e=>{if(!required&&e.code==='ENOENT')return null;throw e});if(!receiptStat)return null;
const archiveStat=await fs.lstat(archive);
if(!receiptStat.isFile()||receiptStat.isSymbolicLink()||receiptStat.nlink!==1||receiptStat.size>16384
||!archiveStat.isFile()||archiveStat.isSymbolicLink()||archiveStat.nlink!==1)throw Error('unsafe_capture');
const result=JSON.parse(await fs.readFile(receipt,'utf8'));if(archiveStat.size!==result.byteSize)throw Error('invalid_capture');
return {...result,layout};
};
let result=await readPair(canonical,'captured',true)??await readPair(directory,'legacy',false),attempt;
try{if(!result){
attempt=await fs.mkdtemp(path.join(directory,'capture-'));
const archive=path.join(attempt,'workspace.pcws'),receipt=path.join(attempt,'receipt.json');
const captured=await promisify(cp.execFile)(process.execPath,[helper,'capture',root,archive],{maxBuffer:1048576});
await fs.writeFile(receipt,JSON.stringify(JSON.parse(captured.stdout)),{mode:0o600,flag:'wx'});
await readPair(attempt,'captured',true);
try{await fs.rename(attempt,canonical);attempt=undefined}catch(e){if(!['EEXIST','ENOTEMPTY'].includes(e.code))throw e;}
result=await readPair(canonical,'captured',true);if(!result)throw Error('capture_missing');
}process.stdout.write(JSON.stringify(result));}finally{if(attempt)await fs.rm(attempt,{recursive:true,force:true});}
})().catch(error=>{
const gitOperation='remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace';
process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation:'remote_workspace_capture_failed');process.exitCode=1;
});`;
const { layout, ...receipt } = JSON.parse(await command(input.target, ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as {
manifestDigest: string; sha256: string; byteSize: number; entries: number; layout: "legacy" | "captured";
};
if (!["legacy", "captured"].includes(layout) || !/^[a-f0-9]{64}$/.test(receipt.manifestDigest) || !/^[a-f0-9]{64}$/.test(receipt.sha256)
|| !Number.isSafeInteger(receipt.byteSize) || receipt.byteSize <= 0) throw new Error("remote_workspace_capture_receipt_invalid");
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-upload-"));
const archivePath = path.join(root, "workspace.pcws");
try {
await input.target.runner!.syncOut!([{ operationId: input.operationId,
files: [{ kind: "file", sourcePath: layout === "captured" ? path.posix.join(staged.directory, "captured", "workspace.pcws") : staged.archive, targetPath: archivePath }],
}]);
const hash = createHash("sha256"); let bytes = 0;
for await (const chunk of createReadStream(archivePath)) { hash.update(chunk); bytes += chunk.length; }
if (hash.digest("hex") !== receipt.sha256 || bytes !== receipt.byteSize) throw new Error("remote_workspace_capture_transfer_mismatch");
return { ...receipt, archivePath, cleanup: () => rm(root, { recursive: true, force: true }) };
} catch (error) { await rm(root, { recursive: true, force: true }); throw error; }
}
export async function restoreRemoteWorkspaceDirectory(input: {
target: RemoteWorkspaceTarget; remoteCwd: string; archivePath: string; manifestDigest: string; restorationIdentity: string;
}) {
if (!/^[a-f0-9]{64}$/.test(input.restorationIdentity)) throw new Error("remote_workspace_restore_identity_invalid");
const id = input.restorationIdentity;
const operationId = `${id.slice(0, 8)}-${id.slice(8, 12)}-${id.slice(12, 16)}-${id.slice(16, 20)}-${id.slice(20, 32)}`;
// Stage outside the destination; the restore validator requires an empty target.
const parent = path.posix.dirname(input.remoteCwd);
const staged = await installArchiveHelper(input.target, parent, operationId);
await input.target.runner!.syncIn!([{ operationId,
files: [{ kind: "file", sourcePath: input.archivePath, targetPath: staged.archive }],
}]);
// Publish the restored folder and its identity together. A lost successful
// reply can then adopt the same files without repeating a destructive restore.
const restore = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');
(async()=>{const [helper,archive,root,digest,identity]=process.argv.slice(1);
if(!/^[a-f0-9]{64}$/.test(identity))throw Error('invalid_identity');
const prior=await fs.lstat(root).catch(e=>{if(e.code==='ENOENT')return null;throw e});
if(prior){
if(!prior.isDirectory()||prior.isSymbolicLink()||await fs.realpath(root)!==root)throw Error('unsafe_root');
const runtime=path.join(root,'.paperclip-runtime'),marker=path.join(runtime,'workspace-restored.json');
const dir=await fs.lstat(runtime);if(!dir.isDirectory()||dir.isSymbolicLink())throw Error('unsafe_receipt');
const file=await fs.lstat(marker);if(!file.isFile()||file.isSymbolicLink()||file.nlink!==1||file.size>16384)throw Error('unsafe_receipt');
const receipt=JSON.parse(await fs.readFile(marker,'utf8'));
if(receipt.identity!==identity||receipt.manifestDigest!==digest)throw Error('workspace_owner_changed');return;
}
const staged=await fs.mkdtemp(root+'.restoring-');
await require('node:util').promisify(cp.execFile)(process.execPath,[helper,'restore',archive,staged,digest],{maxBuffer:1048576});
const runtime=path.join(staged,'.paperclip-runtime');await fs.mkdir(runtime,{mode:0o700});
await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify({identity,manifestDigest:digest}),{flag:'wx',mode:0o600});
if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root');await fs.rename(staged,root);
})().catch(()=>{process.stderr.write('remote_workspace_restore_failed');process.exitCode=1});`;
await command(input.target, ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity]);
// Publish the restored location before cleanup can fail, stall, or lose its
// reply. The owner journals this receipt and then reclaims disposable staging.
return { operationId, stagingRoot: parent, cleanupPending: true };
}
File diff suppressed because it is too large. Load diff
+710
View File
@@ -0,0 +1,710 @@
import { createHash, randomUUID } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdtemp, rm, statfs } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { and, eq, inArray, sql } from "drizzle-orm";
import {
environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db,
} from "@paperclipai/db";
import {
readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema,
type RemoteWorkspaceAuthority, type RemoteWorkspaceRunReference, type RemoteWorkspaceSource,
type EnvironmentLease,
} from "@paperclipai/shared";
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { getStorageService, type StorageService } from "../storage/index.js";
import { loadConfig } from "../config.js";
import { conflict, forbidden, unprocessable } from "../errors.js";
import { logger } from "../middleware/logger.js";
import { createGitRemoteAuthProvider } from "./git-credentials.js";
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js";
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
import { persistActivity } from "./activity-log.js";
import {
captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath,
requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory,
} from "./remote-workspace-transport.js";
import {
beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot,
} from "./remote-workspace-snapshot-storage.js";
import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js";
import { attachPreservedWorkspaceSnapshots } from "./workspace-recovery-snapshot.js";
import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js";
import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js";
import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js";
export { readRemoteWorkspaceAuthority, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
export { remoteWorkspaceRepositoryPath } from "./remote-workspace-transport.js";
export { clearRemoteWorkspaceLocationForReleasedLease } from "./remote-workspace-snapshot-retention.js";
type Workspace = Pick<typeof executionWorkspaces.$inferSelect, "id" | "companyId" | "metadata" | "sourceIssueId" | "projectId" | "projectWorkspaceId">;
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
const SAVE_WORK_SCHEMA = "paperclip.remote-workspace-save-work.v1";
const RETIRED_SAVE_SCHEMA = "paperclip.remote-workspace-retired-save.v1";
type SaveWork = { schema: typeof SAVE_WORK_SCHEMA; activeIds: string[]; unknown: boolean; quiescedAt: string | null };
function readSaveWork(metadata: Record<string, unknown> | null | undefined): SaveWork | null {
const work = object(metadata?.saveWork);
return work.schema === SAVE_WORK_SCHEMA && Array.isArray(work.activeIds)
&& work.activeIds.every(id => typeof id === "string") && new Set(work.activeIds).size === work.activeIds.length
&& typeof work.unknown === "boolean" && (work.quiescedAt === null
|| typeof work.quiescedAt === "string" && Number.isFinite(Date.parse(work.quiescedAt)))
? work as SaveWork : null;
}
function sameExportLocation(value: unknown, location: NonNullable<RemoteWorkspaceAuthority["location"]>) {
const recorded = object(value);
return (["environmentId", "leaseId", "providerLeaseId", "remoteCwd"] as const).every(key => recorded[key] === location[key]);
}
export function shouldUseRemoteWorkspaces(input: {
adapterType: string; environmentDriver: string | null; experimentalEnabled: boolean; existingWorkspace?: { metadata?: Record<string, unknown> | null } | null;
}) {
const enrolled = readRemoteWorkspaceAuthority(input.existingWorkspace?.metadata) !== null;
if (enrolled && (input.adapterType !== "paperclip_runner" || input.environmentDriver !== "sandbox")) {
throw conflict("This workspace requires its native sandbox execution environment.", { code: "remote_workspace_execution_required" });
}
if (enrolled) return true;
// Existing host authority changes only through the explicit import workflow.
if (input.existingWorkspace || !input.experimentalEnabled || input.adapterType !== "paperclip_runner"
|| input.environmentDriver !== "sandbox") return false;
return configuredRemoteWorkspacePersistence() !== null;
}
function configuredRemoteWorkspacePersistence(storage?: StorageService) {
const config = loadConfig();
const databaseUrl = config.databaseUrl?.trim();
const keyFilePath = process.env.PAPERCLIP_SECRETS_MASTER_KEY?.trim() ? null : config.secretsMasterKeyFilePath;
if ((storage?.provider ?? config.storageProvider) !== "s3" || !databaseUrl || !/^postgres(?:ql)?:\/\//.test(databaseUrl)
|| (keyFilePath !== null && !existsSync(keyFilePath))) {
return null;
}
return { databaseUrl, keyFilePath };
}
export function assertRemoteWorkspacePersistence(storage: StorageService = getStorageService()) {
const persistence = configuredRemoteWorkspacePersistence(storage);
if (!persistence) throw conflict("Remote workspace storage requires external Postgres, object storage, and a stable configured encryption key.",
{ code: "remote_workspace_persistence_unavailable" });
return persistence;
}
function reference(workspaceId: string, authority: RemoteWorkspaceAuthority, state: "prepared" | "saved"): RemoteWorkspaceRunReference {
if (!authority.location) throw new Error("remote_workspace_location_missing");
return { schema: "paperclip.remote-workspace-run.v1", workspaceId, epoch: authority.epoch,
leaseId: authority.location.leaseId, providerLeaseId: authority.location.providerLeaseId,
remoteCwd: authority.location.remoteCwd, state, snapshotId: authority.snapshotId };
}
async function currentWorkspace(tx: Transaction, companyId: string, workspaceId: string) {
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, workspaceId),
eq(executionWorkspaces.companyId, companyId))).for("update");
if (!workspace) throw new Error("remote_workspace_missing");
return workspace;
}
export async function reserveRemoteWorkspace(input: {
db: Db; companyId: string; issueId: string; runId: string; agentId: string; environmentId: string;
projectId: string | null; projectWorkspaceId: string | null; existingWorkspaceId: string | null;
source: RemoteWorkspaceSource; storage?: StorageService;
mode?: string;
fresh?: { attemptId: string; previousWorkspaceId: string | null; expectedBindingRevision: number };
recoverySourceRunIds?: string[];
}) {
assertRemoteWorkspacePersistence(input.storage ?? getStorageService());
const source = remoteWorkspaceSourceSchema.parse(input.source);
return input.db.transaction(async tx => {
const [issue] = await tx.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update");
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
if (!issue || !run || run.agentId !== input.agentId || run.status !== "running" ||
![run.nativeIssueId, run.issueId, object(run.contextSnapshot).issueId].includes(input.issueId)) throw forbidden("Workspace preparation requires its current task execution.");
if (input.fresh && (issue.executionWorkspaceId !== input.fresh.previousWorkspaceId
|| issue.workspaceBindingRevision !== input.fresh.expectedBindingRevision)) throw conflict("The task workspace selection changed.");
let selected = input.fresh ? input.fresh.attemptId : input.existingWorkspaceId ?? issue.executionWorkspaceId;
let selectedSharedSource = false;
if (!selected && input.mode === "shared_workspace" && input.projectWorkspaceId) {
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`remote-shared-workspace:${input.companyId}:${input.projectWorkspaceId}`}, 0))`);
const [shared] = await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.projectWorkspaceId, input.projectWorkspaceId),
eq(executionWorkspaces.mode, "shared_workspace"), eq(executionWorkspaces.status, "active"),
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->>'schema' = 'paperclip.remote-workspace.v1'`,
)).limit(1);
selected = shared?.id ?? null;
selectedSharedSource = selected !== null;
}
const [selectedRow] = selected ? await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, selected))).for("update") : [];
const existing = selectedRow ?? null;
if (selected && !existing && !input.fresh) throw new Error("remote_workspace_missing");
const authority = readRemoteWorkspaceAuthority(existing?.metadata);
if (existing && !authority) throw conflict("Import this existing workspace before enabling remote storage.", { code: "remote_workspace_migration_required" });
if (existing && authority) {
if ((existing.sourceIssueId !== input.issueId && existing.mode !== "shared_workspace") || ["archived", "cleanup_failed"].includes(existing.status)) throw conflict("Workspace ownership changed.");
if (existing.sourceIssueId !== input.issueId) {
const actor = { type: "agent", source: "agent_jwt", companyId: input.companyId,
agentId: input.agentId, runId: input.runId,
onBehalfOfUserId: run.responsibleUserId === "local-board" ? null : run.responsibleUserId } as const;
await assertTaskWorkspaceExecutionAccess(tx, actor, input.companyId, selectedSharedSource
? { issueId: input.issueId, projectWorkspaceId: input.projectWorkspaceId }
: { issueId: input.issueId, workspaceId: existing.id });
await assertExecutionWorkspaceRecordAccess(tx, actor, input.companyId, existing.id);
}
const [uncertain] = await tx.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, existing.id),
eq(workspaceOperations.phase, "remote_workspace_provision"), sql`${workspaceOperations.metadata}->>'uncertain' = 'true'`,
sql`${workspaceOperations.heartbeatRunId} is distinct from ${input.runId}::uuid`,
)).limit(1);
if (uncertain) throw conflict("The prior provisioning operation needs verification before this workspace can run again.", { code: "remote_workspace_provision_unverified" });
if (authority.writerRunId && authority.writerRunId !== input.runId) {
const [prior] = await tx.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, authority.writerRunId)));
const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
eq(environmentLeases.heartbeatRunId, authority.writerRunId)));
const unusedReservation = !authority.dirty && prior && ["failed", "cancelled", "interrupted", "timed_out", "succeeded"].includes(prior.status)
&& leases.every(hasRemoteTerminationReceipt);
if (!unusedReservation) throw conflict("The workspace still has an unfinished writer or save operation.", { code: "remote_workspace_writer_pending" });
}
if (authority.writerRunId === input.runId) return existing;
const next: RemoteWorkspaceAuthority = { ...authority, source, epoch: authority.epoch + 1, writerRunId: input.runId };
const [saved] = await tx.update(executionWorkspaces).set({ metadata: { ...existing.metadata, remoteWorkspace: next }, updatedAt: new Date() })
.where(eq(executionWorkspaces.id, existing.id)).returning();
if (issue.executionWorkspaceId !== existing.id) await tx.update(issues).set({ executionWorkspaceId: existing.id,
executionWorkspacePreference: "reuse_existing", workspaceBindingRevision: issue.workspaceBindingRevision + 1,
updatedAt: new Date() }).where(eq(issues.id, issue.id));
return saved!;
}
const id = input.fresh?.attemptId ?? randomUUID();
const initial: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1,
snapshotId: null, source, location: null, writerRunId: input.runId, dirty: false };
const [workspace] = await tx.insert(executionWorkspaces).values({ id, companyId: input.companyId, sourceIssueId: input.issueId,
projectId: input.projectId, projectWorkspaceId: input.projectWorkspaceId, mode: input.mode ?? "isolated_workspace",
strategyType: source.kind === "git" ? "git_worktree" : "project_primary", name: "Task workspace", status: "active",
providerType: "cloud_sandbox", cwd: `/workspaces/${id}`, repoUrl: source.kind === "git" ? source.repoUrl : null,
baseRef: source.kind === "git" ? source.ref : null, branchName: source.kind === "git" ? source.branchName ?? null : null,
metadata: { remoteWorkspace: initial, ...(input.recoverySourceRunIds?.length
? { remoteWorkspaceRecoverySourceRunIds: input.recoverySourceRunIds } : {}) },
}).returning();
await tx.update(issues).set({ executionWorkspaceId: id, executionWorkspacePreference: "reuse_existing",
workspaceBindingRevision: issue.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, issue.id));
await persistActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", actorId: "remote-workspaces",
action: "workspace.remote_created", entityType: "execution_workspace", entityId: id, runId: input.runId,
issueId: input.issueId, details: { authority: "sandbox", epoch: initial.epoch } });
return workspace!;
});
}
async function persistRunReference(tx: Transaction, companyId: string, runId: string, value: RemoteWorkspaceRunReference) {
const [run] = await tx.select({ profile: heartbeatRuns.runnerProfileJson, nativeIssueId: heartbeatRuns.nativeIssueId,
issueId: heartbeatRuns.issueId, contextIssueId: sql<unknown>`${heartbeatRuns.contextSnapshot}->'issueId'` }).from(heartbeatRuns)
.where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))).for("update");
if (!run) throw new Error("remote_workspace_run_missing");
await notifyCancelledWorkspacePreservation(tx, { companyId, runId });
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...run.profile, remoteWorkspace: value }, updatedAt: new Date() })
.where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
return run.nativeIssueId ?? run.issueId ?? (typeof run.contextIssueId === "string" ? run.contextIssueId : null);
}
/** Retry acknowledged or explicitly retired exports, never active captures. */
async function cleanupSavedExports(input: { db: Db; companyId: string; workspaceId: string;
location: NonNullable<RemoteWorkspaceAuthority["location"]>; target: ReturnType<typeof requireRemoteWorkspaceTarget>;
assertOwned: () => Promise<void> }) {
for (;;) {
const rows = await input.db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
phase: workspaceOperations.phase }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.workspaceId),
inArray(workspaceOperations.phase, ["remote_workspace_snapshot", "remote_workspace_restore"]),
sql`(${workspaceOperations.status} = 'succeeded' or (${workspaceOperations.status} = 'failed'
and ${workspaceOperations.metadata}->'retiredSave'->>'schema' = ${RETIRED_SAVE_SCHEMA}))`,
sql`(${workspaceOperations.metadata}->'saveWork'->>'schema' is distinct from ${SAVE_WORK_SCHEMA}
or (${workspaceOperations.metadata}#>'{saveWork,unknown}' = 'false'::jsonb and case when jsonb_typeof(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 'array'
then jsonb_array_length(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 0 else false end))`,
sql`${workspaceOperations.metadata}->'exportCleanup'->>'state' = 'pending'`,
sql`${workspaceOperations.metadata}->'exportCleanup'->>'environmentId' = ${input.location.environmentId}`,
sql`${workspaceOperations.metadata}->'exportCleanup'->>'providerLeaseId' = ${input.location.providerLeaseId}`,
sql`${workspaceOperations.metadata}->'exportCleanup'->>'remoteCwd' = ${input.location.remoteCwd}`,
)).limit(20);
if (!rows.length) return true;
for (const row of rows) {
await input.assertOwned();
let cleaned = false;
try {
await cleanupRemoteWorkspaceArchive({ target: input.target,
remoteCwd: row.phase === "remote_workspace_restore" ? path.posix.dirname(input.location.remoteCwd) : input.location.remoteCwd,
operationId: row.id });
cleaned = true;
} catch { /* Keep a durable retry obligation; the acknowledged snapshot is safe. */ }
await input.assertOwned();
await input.db.update(workspaceOperations).set({ metadata: sql`jsonb_set(coalesce(${workspaceOperations.metadata}, '{}'::jsonb),
'{exportCleanup}', coalesce(${workspaceOperations.metadata}->'exportCleanup', '{}'::jsonb) ||
${JSON.stringify({ state: cleaned ? "cleaned" : "pending", lastError: cleaned ? null : "remote_workspace_export_cleanup_failed" })}::jsonb)`,
updatedAt: new Date() })
.where(and(eq(workspaceOperations.id, row.id), eq(workspaceOperations.companyId, input.companyId),
inArray(workspaceOperations.status, ["succeeded", "failed"])));
if (!cleaned) return false;
}
}
}
/** The caller retains execution admission and provider ownership; this owns file authority. */
function isSavedWorkspaceRepository(authority: RemoteWorkspaceAuthority,
repository: NonNullable<RemoteWorkspaceSource["repositories"]>[number]) {
return authority.snapshotId !== null && (authority.source.repositories ?? []).some(saved =>
saved.id === repository.id && saved.relativePath === repository.relativePath && saved.repoUrl === repository.repoUrl
&& (saved.ref ?? "HEAD") === (repository.ref ?? "HEAD") && saved.pinnedCommit === repository.pinnedCommit
&& (saved.readOnly ?? false) === (repository.readOnly ?? false));
}
export async function prepareRemoteWorkspace(input: {
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
source?: RemoteWorkspaceSource; assertOwned?: () => Promise<void>;
provisionCommand?: string | null; runtimeProvisionCommand?: string | null; storage?: StorageService;
}) {
const target = requireRemoteWorkspaceTarget(input.target);
const initial = readRemoteWorkspaceAuthority(input.workspace.metadata);
if ((input.source ?? initial?.source)?.repositories?.some(repository => repository.readOnly)) {
throw conflict("The selected native sandbox cannot enforce read-only referenced project roots.",
{ code: "remote_workspace_readonly_sources_unsupported" });
}
if (!initial || initial.writerRunId !== input.runId || !input.lease.environmentId || !input.lease.providerLeaseId
|| input.lease.companyId !== input.workspace.companyId || input.lease.heartbeatRunId !== input.runId) throw new Error("remote_workspace_binding_mismatch");
await input.assertOwned?.();
const sameAllocation = initial.location?.providerLeaseId === input.lease.providerLeaseId
&& initial.location?.environmentId === input.lease.environmentId;
if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation.");
const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", input.workspace.id);
const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id,
input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex");
let restoredExport: Awaited<ReturnType<typeof restoreRemoteWorkspaceDirectory>> | null = null;
if (sameAllocation && !await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId,
workspaceId: input.workspace.id, location: initial.location!, target,
assertOwned: input.assertOwned ?? (async () => undefined) })) throw new Error("remote_workspace_export_cleanup_pending");
if (!sameAllocation) {
if (initial.snapshotId) {
const snapshot = await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id, snapshotId: initial.snapshotId });
if (!snapshot) throw new Error("remote_workspace_snapshot_missing");
const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-restore-"));
try {
const archivePath = path.join(scratch, "workspace.pcws");
await loadWorkspaceSnapshotArchive({ storage: input.storage ?? getStorageService(), reference: snapshot, targetPath: archivePath });
restoredExport = await restoreRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, archivePath, manifestDigest: snapshot.manifestDigest,
restorationIdentity: preparationIdentity });
} finally { await rm(scratch, { recursive: true, force: true }); }
} else {
await prepareRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, source: initial.source,
preparationIdentity,
resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) });
}
}
for (const repository of (input.source ?? initial.source).repositories ?? []) {
await input.assertOwned?.();
// Only repositories already owned by the saved authority may rely on the
// archive when their old local origin was intentionally omitted. New source
// descriptors still follow normal acquisition and origin verification.
await prepareRemoteRepository({ target, remoteCwd: remoteWorkspaceRepositoryPath(remoteRoot, repository),
repoUrl: repository.repoUrl, ref: repository.ref, pinnedCommit: repository.pinnedCommit,
savedArchive: isSavedWorkspaceRepository(initial, repository),
resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) });
}
const recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string }> = [];
const recoverySourceRunIds = input.workspace.metadata?.remoteWorkspaceRecoverySourceRunIds;
if (!sameAllocation && Array.isArray(recoverySourceRunIds) && recoverySourceRunIds.length && input.workspace.sourceIssueId) {
const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-recovery-import-"));
try {
const recovered = await attachPreservedWorkspaceSnapshots({ db: input.db, companyId: input.workspace.companyId,
issueId: input.workspace.sourceIssueId, sourceRunIds: recoverySourceRunIds as string[], cwd: scratch });
for (const item of recovered) {
const relative = `.paperclip-recovered/${item.operationId}`;
await target.runner!.syncIn!([{ operationId: randomUUID(), files: [{
kind: "directory", sourcePath: item.path, targetPath: path.posix.join(remoteRoot, relative),
}] }]);
recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: relative });
}
} finally { await rm(scratch, { recursive: true, force: true }); }
}
await input.assertOwned?.();
const prepared = await input.db.transaction(async tx => {
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
const current = readRemoteWorkspaceAuthority(workspace.metadata);
if (!current || current.epoch !== initial.epoch || current.writerRunId !== input.runId) throw new Error("remote_workspace_authority_changed");
// The same run can retry with a different allocation and stale preparation
// input. Its controller lease alone cannot replace already-published dirty files.
if (current.dirty && current.location && (current.location.environmentId !== input.lease.environmentId
|| current.location.providerLeaseId !== input.lease.providerLeaseId || current.location.leaseId !== input.lease.id
|| current.location.remoteCwd !== remoteRoot)) throw new Error("remote_workspace_authority_changed");
const authority: RemoteWorkspaceAuthority = { ...current, dirty: true, location: { environmentId: input.lease.environmentId!,
leaseId: input.lease.id, providerLeaseId: input.lease.providerLeaseId!, remoteCwd: remoteRoot } };
const [lease] = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.id, input.lease.id),
eq(environmentLeases.companyId, workspace.companyId))).for("update");
if (!lease || lease.status !== "active" || lease.environmentId !== input.lease.environmentId
|| lease.providerLeaseId !== authority.location!.providerLeaseId || lease.heartbeatRunId !== input.runId
|| (lease.executionWorkspaceId && lease.executionWorkspaceId !== workspace.id)) throw new Error("remote_workspace_lease_changed");
const [savedLease] = await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteCwd: remoteRoot, remoteWorkspaceId: workspace.id }, updatedAt: new Date() })
.where(eq(environmentLeases.id, lease.id)).returning();
const [savedWorkspace] = await tx.update(executionWorkspaces).set({ cwd: remoteRoot, providerRef: remoteRoot,
metadata: { ...workspace.metadata, remoteWorkspace: authority,
...(recoveredFiles.length ? { remoteWorkspaceRecoveredFiles: recoveredFiles } : {}) }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)).returning();
const runReference = reference(workspace.id, authority, "prepared");
const issueId = await persistRunReference(tx, workspace.companyId, input.runId, runReference);
if (restoredExport) {
const [existing] = await tx.select().from(workspaceOperations).where(eq(workspaceOperations.id, restoredExport.operationId)).for("update");
if (existing && (existing.companyId !== workspace.companyId || existing.executionWorkspaceId !== workspace.id
|| existing.heartbeatRunId !== input.runId || existing.phase !== "remote_workspace_restore")) throw new Error("remote_workspace_restore_receipt_changed");
const metadata = { epoch: authority.epoch, snapshotId: initial.snapshotId,
exportCleanup: { ...authority.location, stagingRoot: restoredExport.stagingRoot,
state: restoredExport.cleanupPending ? "pending" : "cleaned" } };
if (existing) await tx.update(workspaceOperations).set({ status: "succeeded", metadata, finishedAt: new Date(), updatedAt: new Date() })
.where(eq(workspaceOperations.id, existing.id));
else await tx.insert(workspaceOperations).values({ id: restoredExport.operationId, companyId: workspace.companyId,
executionWorkspaceId: workspace.id, heartbeatRunId: input.runId, issueId,
phase: "remote_workspace_restore", status: "succeeded", metadata, finishedAt: new Date() });
}
return { workspace: savedWorkspace!, lease: savedLease! as EnvironmentLease, remoteRoot, reference: runReference };
});
target.remoteCwd = remoteRoot;
if (restoredExport?.cleanupPending) await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId,
workspaceId: input.workspace.id, location: readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.location!, target,
assertOwned: input.assertOwned ?? (async () => undefined) });
try {
await provisionRemoteWorkspace({ db: input.db, companyId: input.workspace.companyId, workspaceId: input.workspace.id,
runId: input.runId, epoch: initial.epoch, remoteCwd: remoteRoot, target, assertOwned: input.assertOwned,
commands: [!initial.snapshotId ? input.provisionCommand : null, input.runtimeProvisionCommand] });
if (!initial.snapshotId) {
await saveRemoteWorkspace({ ...input, workspace: prepared.workspace, lease: prepared.lease, target,
assertOwned: input.assertOwned ?? (async () => undefined), boundary: "initial" });
const [workspace] = await input.db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, input.workspace.id), eq(executionWorkspaces.companyId, input.workspace.companyId)));
prepared.workspace = workspace!;
prepared.reference = reference(workspace!.id, readRemoteWorkspaceAuthority(workspace!.metadata)!, "prepared");
}
} catch (error) {
// A first save can fail before runtime selection. Its already-published
// authority is still a file obligation, independent of native admission.
await preserveRemoteWorkspacePreparationFailure(input.db, prepared.lease, { assertOwned: input.assertOwned })
.catch(preservationError => logger.error({ err: preservationError, runId: input.runId }, "Remote workspace preparation recovery was deferred; source remains protected"));
throw error;
}
return prepared;
}
/** The workspace lock fences new invocations while completed final authority
* retires only attempts whose every registered invocation has settled. */
async function retireQuiescedSaves(tx: Transaction, workspace: Workspace, runId: string, epoch: number,
location: NonNullable<RemoteWorkspaceAuthority["location"]>) {
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
if (!authority || authority.epoch !== epoch || authority.dirty || authority.writerRunId !== null
|| !authority.snapshotId || !sameExportLocation(authority.location, location)) return;
const [final] = await tx.select({ id: executionWorkspaceSnapshots.id, generation: executionWorkspaceSnapshots.generation,
operationId: workspaceOperations.id, metadata: workspaceOperations.metadata }).from(executionWorkspaceSnapshots)
.innerJoin(workspaceOperations, and(eq(workspaceOperations.id, executionWorkspaceSnapshots.sourceOperationId),
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
eq(workspaceOperations.heartbeatRunId, runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"),
eq(workspaceOperations.status, "succeeded"), sql`${workspaceOperations.metadata}->>'boundary' = 'final'`,
sql`${workspaceOperations.metadata}->>'epoch' = ${String(epoch)}`))
.where(and(eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.sourceRunId, runId),
eq(executionWorkspaceSnapshots.epoch, epoch), eq(executionWorkspaceSnapshots.status, "committed"))).limit(1);
if (!final || final.metadata?.snapshotId !== final.id || !sameExportLocation(final.metadata?.exportCleanup, location)) return;
const candidates = await tx.select().from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
eq(workspaceOperations.heartbeatRunId, runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"),
inArray(workspaceOperations.status, ["running", "pending"]), sql`${workspaceOperations.metadata}->>'epoch' = ${String(epoch)}`)).for("update");
for (const candidate of candidates) {
const work = readSaveWork(candidate.metadata), generation = Number(candidate.metadata?.generation);
if (!work || work.unknown || work.activeIds.length || !work.quiescedAt || candidate.metadata?.boundary === "final"
|| !Number.isSafeInteger(generation) || generation >= final.generation
|| !sameExportLocation(candidate.metadata?.exportCleanup, location)) continue;
const [snapshot] = await tx.select({ status: executionWorkspaceSnapshots.status, generation: executionWorkspaceSnapshots.generation,
epoch: executionWorkspaceSnapshots.epoch, runId: executionWorkspaceSnapshots.sourceRunId }).from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
eq(executionWorkspaceSnapshots.sourceOperationId, candidate.id))).for("update");
if (snapshot && (!["pending", "uploaded"].includes(snapshot.status) || snapshot.generation !== generation
|| snapshot.epoch !== epoch || snapshot.runId !== runId)) continue;
const at = new Date();
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
await tx.update(workspaceOperations).set({ status: "failed", finishedAt: at, updatedAt: at,
metadata: { ...candidate.metadata, retiredSave: { schema: RETIRED_SAVE_SCHEMA, snapshotId: final.id,
operationId: final.operationId, retiredAt: at.toISOString() } } }).where(eq(workspaceOperations.id, candidate.id));
}
}
async function settleSaveInvocation(input: { db: Db; workspace: Workspace; runId: string; epoch: number;
location: NonNullable<RemoteWorkspaceAuthority["location"]>; operationId: string; invocationId: string }) {
return input.db.transaction(async tx => {
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
const [operation] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.id, input.operationId),
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
eq(workspaceOperations.heartbeatRunId, input.runId), sql`${workspaceOperations.metadata}->>'epoch' = ${String(input.epoch)}`)).for("update");
const work = readSaveWork(operation?.metadata);
if (!operation || !work || !work.activeIds.includes(input.invocationId)) return false;
const activeIds = work.activeIds.filter(id => id !== input.invocationId);
if (!activeIds.length && !work.unknown) await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
await tx.update(workspaceOperations).set({ metadata: { ...operation.metadata,
saveWork: { ...work, activeIds, quiescedAt: activeIds.length ? null : new Date().toISOString() } }, updatedAt: new Date() })
.where(eq(workspaceOperations.id, operation.id));
await retireQuiescedSaves(tx, workspace, input.runId, input.epoch, input.location);
return true;
});
}
export async function saveRemoteWorkspace(input: {
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
assertOwned: () => Promise<void>; storage?: StorageService; boundary?: string;
}) {
const target = requireRemoteWorkspaceTarget(input.target), boundary = input.boundary ?? "final";
const expected = readRemoteWorkspaceAuthority(input.workspace.metadata);
if (!expected?.location || expected.location.providerLeaseId !== input.lease.providerLeaseId || expected.location.leaseId !== input.lease.id) throw new Error("remote_workspace_save_binding_mismatch");
await input.assertOwned();
const invocationId = randomUUID();
const operation = await input.db.transaction(async tx => {
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
const current = readRemoteWorkspaceAuthority(workspace.metadata);
if (!current || current.epoch !== expected.epoch || (current.writerRunId !== input.runId && current.dirty)) throw new Error("remote_workspace_authority_changed");
const [previous] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, workspace.companyId),
eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.heartbeatRunId, input.runId),
eq(workspaceOperations.phase, "remote_workspace_snapshot"), sql`${workspaceOperations.metadata}->>'boundary' = ${boundary}`,
sql`${workspaceOperations.metadata}->>'epoch' = ${String(current.epoch)}`)).limit(1).for("update");
if (previous) {
if (object(previous.metadata?.retiredSave).schema === RETIRED_SAVE_SCHEMA) throw new Error("remote_workspace_snapshot_retired");
const work = readSaveWork(previous.metadata);
const [registered] = await tx.update(workspaceOperations).set({ metadata: { ...previous.metadata,
saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [...(work?.activeIds ?? []), invocationId],
unknown: work?.unknown ?? true, quiescedAt: null } }, updatedAt: new Date() }).where(eq(workspaceOperations.id, previous.id)).returning();
return registered!;
}
const [run] = await tx.select({ nativeIssueId: heartbeatRuns.nativeIssueId, issueId: heartbeatRuns.issueId,
contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(and(
eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, input.runId)));
if (!run) throw new Error("remote_workspace_run_missing");
const issueId = run.nativeIssueId ?? run.issueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);
// The workspace lock serializes generation reservations. Failed/deleted
// snapshots still own their generation, and an operation can be committed
// before beginWorkspaceSnapshot creates its ledger row. Count both journals
// across epochs while retaining the last committed snapshot as the parent.
const [reservation] = await tx.execute(sql`select greatest(
coalesce((select max(${executionWorkspaceSnapshots.generation}) from ${executionWorkspaceSnapshots}
where ${executionWorkspaceSnapshots.companyId} = ${workspace.companyId}
and ${executionWorkspaceSnapshots.executionWorkspaceId} = ${workspace.id}), 0),
coalesce((select max(case when ${workspaceOperations.metadata}->>'generation' ~ '^[0-9]+$'
then (${workspaceOperations.metadata}->>'generation')::numeric else 0 end) from ${workspaceOperations}
where ${workspaceOperations.companyId} = ${workspace.companyId}
and ${workspaceOperations.executionWorkspaceId} = ${workspace.id}
and ${workspaceOperations.phase} = 'remote_workspace_snapshot'), 0)
) as generation`);
const generation = Number(reservation?.generation) + 1;
if (!Number.isSafeInteger(generation) || generation < 1) throw new Error("remote_workspace_snapshot_generation_invalid");
const [created] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id,
heartbeatRunId: input.runId, issueId, phase: "remote_workspace_snapshot", status: "running",
metadata: { boundary, epoch: current.epoch, parentSnapshotId: current.snapshotId, generation,
saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [invocationId], unknown: false, quiescedAt: null },
exportCleanup: { ...current.location, state: "pending" } },
}).returning();
return created!;
});
let captured: Awaited<ReturnType<typeof captureRemoteWorkspaceArchive>> | null = null;
try {
const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id,
operationId: operation.id, runId: input.runId, parentSnapshotId: operation.metadata?.parentSnapshotId as string | null,
generation: operation.metadata?.generation as number, epoch: expected.epoch });
if (operation.status === "succeeded" && ["committed", "superseded"].includes(snapshot.status)) {
return { snapshotId: snapshot.id, generation: snapshot.generation, sha256: snapshot.sha256!, savedAt: snapshot.updatedAt.toISOString() };
}
const existingUpload = snapshot.status === "uploaded" ? await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId,
workspaceId: input.workspace.id, snapshotId: snapshot.id }) : null;
captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ target, remoteCwd: expected.location.remoteCwd, operationId: operation.id });
await input.assertOwned();
const saved = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage ?? getStorageService(), snapshotId: snapshot.id,
companyId: input.workspace.companyId, workspaceId: input.workspace.id, archivePath: captured?.archivePath ?? "",
manifestDigest: existingUpload?.manifestDigest ?? captured!.manifestDigest });
await input.assertOwned();
const at = new Date();
await input.db.transaction(async tx => {
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
const current = readRemoteWorkspaceAuthority(workspace.metadata);
if (!current || current.epoch !== expected.epoch || current.writerRunId !== input.runId || current.snapshotId !== snapshot.parentSnapshotId) {
if (current?.epoch === expected.epoch && current.snapshotId === saved.id) return;
throw new Error("remote_workspace_publication_fenced");
}
const authority: RemoteWorkspaceAuthority = { ...current, snapshotId: saved.id,
dirty: boundary !== "final", writerRunId: boundary !== "final" ? input.runId : null };
const [committed] = await tx.update(executionWorkspaceSnapshots).set({ status: "committed", updatedAt: at }).where(and(
eq(executionWorkspaceSnapshots.id, saved.id), eq(executionWorkspaceSnapshots.companyId, workspace.companyId),
eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), eq(executionWorkspaceSnapshots.status, "uploaded"))).returning({ id: executionWorkspaceSnapshots.id });
if (!committed) throw new Error("remote_workspace_snapshot_publication_changed");
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
if (boundary === "final") await notifyTerminalWorkspaceWork(tx, `workspace:${workspace.id}`);
await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspace: authority }, updatedAt: at }).where(eq(executionWorkspaces.id, workspace.id));
await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: at, updatedAt: at,
metadata: sql`coalesce(${workspaceOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ snapshotId: saved.id, sha256: saved.sha256 })}::jsonb` })
.where(eq(workspaceOperations.id, operation.id));
await persistRunReference(tx, workspace.companyId, input.runId, reference(workspace.id, authority, boundary === "final" ? "saved" : "prepared"));
await persistActivity(tx as unknown as Db, { companyId: workspace.companyId, actorType: "system", actorId: "remote-workspaces",
action: "workspace.snapshot_saved", entityType: "execution_workspace", entityId: workspace.id, runId: input.runId,
details: { snapshotId: saved.id, generation: saved.generation, epoch: current.epoch, boundary } });
if (snapshot.parentSnapshotId) await tx.update(executionWorkspaceSnapshots).set({ status: "superseded",
deleteAfter: new Date(at.getTime() + 7 * 86400_000), updatedAt: at }).where(and(eq(executionWorkspaceSnapshots.id, snapshot.parentSnapshotId),
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id)));
});
return { snapshotId: saved.id, generation: saved.generation, sha256: saved.sha256, savedAt: at.toISOString() };
} finally {
// Every upload/capture promise has settled before its durable token clears.
// Missing completion evidence remains pinned; cleanup never masks the save error.
try { await captured?.cleanup(); }
catch { logger.warn({ runId: input.runId, operationId: operation.id }, "Remote workspace local archive cleanup failed"); }
try {
if (await settleSaveInvocation({ db: input.db, workspace: input.workspace, runId: input.runId, epoch: expected.epoch,
location: expected.location, operationId: operation.id, invocationId })) {
await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId, workspaceId: input.workspace.id,
location: expected.location, target, assertOwned: input.assertOwned });
}
} catch { logger.warn({ runId: input.runId, operationId: operation.id }, "Remote workspace save settlement or export cleanup remains pending"); }
}
}
export async function prepareRemoteWorkspaceRepository(input: {
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
repository: { id: string; relativePath: string; repoUrl: string; ref: string | null; pinnedCommit?: string; readOnly?: boolean };
assertOwned?: () => Promise<void>; resolveGitAuth?: ReturnType<typeof createGitRemoteAuthProvider>; verifyOnly?: boolean; storage?: StorageService;
}) {
const authority = readRemoteWorkspaceAuthority(input.workspace.metadata);
if (!authority?.location || authority.writerRunId !== input.runId || authority.location.leaseId !== input.lease.id
|| authority.location.providerLeaseId !== input.lease.providerLeaseId) throw new Error("remote_workspace_repository_binding_mismatch");
await input.assertOwned?.();
const target = requireRemoteWorkspaceTarget(input.target);
const result = await prepareRemoteRepository({ target, remoteCwd: remoteWorkspaceRepositoryPath(authority.location.remoteCwd, input.repository),
...input.repository, verifyOnly: input.verifyOnly, savedArchive: isSavedWorkspaceRepository(authority, input.repository),
resolveGitAuth: input.resolveGitAuth ?? await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) });
await input.assertOwned?.();
await saveRemoteWorkspace({ ...input, target, boundary: `repository:${input.repository.id}:${result.pinnedCommit}`,
assertOwned: input.assertOwned ?? (async () => undefined) });
return result;
}
type SavedWorkspaceFileLimits = {
maxOpen: number; maxArchiveBytes: number; maxScratchBytes: number; maxEntries: number; maxManifestBytes: number; freeSpaceMarginBytes: number;
};
const SAVED_WORKSPACE_FILE_LIMITS: SavedWorkspaceFileLimits = {
maxOpen: 2, maxArchiveBytes: 512 * 1024 * 1024, maxScratchBytes: 2 * 1024 * 1024 * 1024,
maxEntries: 10_000, maxManifestBytes: 4 * 1024 * 1024, freeSpaceMarginBytes: 256 * 1024 * 1024,
};
let savedWorkspaceFileOpen = 0, savedWorkspaceFileReservedBytes = 0;
export function createSavedWorkspaceLoader(db: Db, storage: StorageService = getStorageService(), options: {
limits?: Partial<SavedWorkspaceFileLimits>; availableBytes?: () => Promise<bigint>;
} = {}) {
const limits = { ...SAVED_WORKSPACE_FILE_LIMITS, ...options.limits };
for (const key of Object.keys(limits) as Array<keyof SavedWorkspaceFileLimits>) {
if (!Number.isSafeInteger(limits[key]) || limits[key] < (key === "freeSpaceMarginBytes" ? 0 : 1)
|| limits[key] > SAVED_WORKSPACE_FILE_LIMITS[key]) throw new Error("invalid_saved_workspace_file_limits");
}
const availableBytes = options.availableBytes ?? (async () => {
const disk = await statfs(os.tmpdir(), { bigint: true });
return disk.bavail * disk.bsize;
});
return async (input: { companyId: string; workspaceId: string; snapshotId: string; signal?: AbortSignal }) => {
input.signal?.throwIfAborted();
const [row] = await db.select().from(executionWorkspaceSnapshots).where(and(eq(executionWorkspaceSnapshots.companyId, input.companyId),
eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspaceId), eq(executionWorkspaceSnapshots.id, input.snapshotId),
eq(executionWorkspaceSnapshots.status, "committed")));
const saved = row ? await getWorkspaceSnapshot(db, input) : null;
input.signal?.throwIfAborted();
if (!saved) throw new Error("remote_workspace_snapshot_unavailable");
if (saved.byteSize > limits.maxArchiveBytes) throw unprocessable("Saved workspace is too large for Files", { code: "too_large" });
// Decryption can hold ciphertext, plaintext, and its atomic publication at
// once. Bound filesystem allocation overhead even for many tiny entries.
const reservation = 3 * saved.byteSize + limits.maxEntries * 16 * 1024;
if (savedWorkspaceFileOpen >= limits.maxOpen || reservation > limits.maxScratchBytes - savedWorkspaceFileReservedBytes) {
throw unprocessable("Saved workspace file extraction capacity is unavailable", { code: "workspace_unavailable" });
}
savedWorkspaceFileOpen++; savedWorkspaceFileReservedBytes += reservation;
let scratch: string | undefined, released = false, cleaning: Promise<void> | undefined;
// Source modes stay intact while reading. Cleanup changes only private directories and skips symlinks.
const cleanup = async () => {
if (released) return;
if (!cleaning) cleaning = (async () => {
if (scratch) await removeTaskDirectoryTree(scratch);
released = true; savedWorkspaceFileOpen--; savedWorkspaceFileReservedBytes -= reservation;
})();
try { await cleaning; } catch (error) { cleaning = undefined; throw error; }
};
try {
// Existing reservations may not have written their bytes yet. Keep them
// deducted from free space until their owned directories are removed.
if (await availableBytes() < BigInt(savedWorkspaceFileReservedBytes + limits.freeSpaceMarginBytes)) {
throw unprocessable("Insufficient temporary disk space for saved workspace Files", { code: "workspace_unavailable" });
}
input.signal?.throwIfAborted();
scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-files-"));
const archive = path.join(scratch, "archive.pcws"), root = path.join(scratch, "files");
await loadWorkspaceSnapshotArchive({ storage, reference: saved, targetPath: archive, signal: input.signal, temporaryParent: scratch });
await restoreRemoteWorkspaceArchive({ archivePath: archive, targetDir: root, expectedManifestDigest: saved.manifestDigest,
signal: input.signal, maxEntries: limits.maxEntries, maxManifestBytes: limits.maxManifestBytes });
return { root, savedAt: row!.updatedAt.toISOString(), cleanup };
} catch (error) {
try { await cleanup(); } catch {
logger.warn({ workspaceId: input.workspaceId }, "Failed to remove saved workspace extraction; capacity remains reserved");
}
if (error instanceof Error && ["remote_workspace_archive_entry_limit", "remote_workspace_archive_manifest_too_large"].includes(error.message)) {
throw unprocessable("Saved workspace manifest exceeds Files extraction limits", { code: "too_large" });
}
throw error;
}
};
}
/** Bind credentials to the current run's identity, including shared workspace users. */
export async function createRemoteWorkspaceGitAuth(db: Db, companyId: string, runId: string,
create: typeof createGitRemoteAuthProvider = createGitRemoteAuthProvider) {
const [run] = await db.select({ agentId: heartbeatRuns.agentId, responsibleUserId: heartbeatRuns.responsibleUserId,
nativeIssueId: heartbeatRuns.nativeIssueId, issueId: heartbeatRuns.issueId, status: heartbeatRuns.status,
contextIssueId: sql<unknown>`${heartbeatRuns.contextSnapshot}->'issueId'`,
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
const issueId = run?.nativeIssueId ?? run?.issueId ?? run?.contextIssueId;
if (!run || run.status !== "running" || typeof issueId !== "string"
|| !/^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/i.test(issueId)) throw new Error("remote_workspace_git_run_unavailable");
return create(db, companyId, { issueId, heartbeatRunId: runId, agentId: run.agentId,
responsibleUserId: run.responsibleUserId === "local-board" ? null : run.responsibleUserId });
}
/** Arbitrary provisioning commands cannot be replayed after an ambiguous reply. */
export async function provisionRemoteWorkspace(input: {
db: Db; companyId: string; workspaceId: string; runId: string; epoch: number;
remoteCwd: string; target: ReturnType<typeof requireRemoteWorkspaceTarget>; commands: Array<string | null | undefined>;
assertOwned?: () => Promise<void>;
}) {
for (const [index, command] of input.commands.entries()) {
if (!command?.trim()) continue;
await input.assertOwned?.();
const digest = createHash("sha256").update(command).digest("hex");
const operation = await input.db.transaction(async tx => {
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
const [run] = await tx.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
if (!authority || authority.epoch !== input.epoch || authority.writerRunId !== input.runId || run?.status !== "running") throw new Error("remote_workspace_provision_fenced");
const [existing] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, input.companyId),
eq(workspaceOperations.executionWorkspaceId, input.workspaceId), eq(workspaceOperations.heartbeatRunId, input.runId),
eq(workspaceOperations.phase, "remote_workspace_provision"), sql`${workspaceOperations.metadata}->>'index' = ${String(index)}`,
sql`${workspaceOperations.metadata}->>'epoch' = ${String(input.epoch)}`)).limit(1);
if (existing) {
if (existing.metadata?.commandSha256 !== digest || existing.status !== "succeeded") throw new Error("remote_workspace_provision_unverified");
return null;
}
const [created] = await tx.insert(workspaceOperations).values({ companyId: input.companyId, executionWorkspaceId: input.workspaceId,
heartbeatRunId: input.runId, issueId: workspace!.sourceIssueId, phase: "remote_workspace_provision", status: "running",
metadata: { epoch: input.epoch, index, commandSha256: digest, uncertain: true } }).returning();
return created!;
});
if (!operation) continue;
await input.assertOwned?.();
const result = await input.target.runner!.execute({ command: "bash", args: ["-lc", command], cwd: input.remoteCwd,
timeoutMs: 300_000, bypassSession: true });
if (result.exitCode !== 0 || result.timedOut) throw new Error("remote_workspace_provision_unverified");
await input.assertOwned?.();
await input.db.transaction(async tx => {
const [workspace] = await tx.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces)
.where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
if (authority?.epoch !== input.epoch || authority.writerRunId !== input.runId) throw new Error("remote_workspace_provision_fenced");
await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(),
metadata: { ...operation.metadata, uncertain: false } }).where(eq(workspaceOperations.id, operation.id));
});
}
}
+21 -2
View File
@@ -1358,6 +1358,7 @@ export function secretService(db: Db | DbTransaction) {
secretId: string,
version: number | "latest",
options?: SecretResolutionOptions,
readOnlyEnvironmentInspection = false,
): Promise<RuntimeSecretResolution> {
const bindingContext = options?.bindingContext;
const accessContext = options?.accessContext ?? bindingContext;
@@ -1406,7 +1407,7 @@ export function secretService(db: Db | DbTransaction) {
version: resolvedVersion,
},
});
await Promise.all([
if (!readOnlyEnvironmentInspection) await Promise.all([
db
.update(companySecrets)
.set({ lastResolvedAt: new Date(), updatedAt: new Date() })
@@ -1442,7 +1443,7 @@ export function secretService(db: Db | DbTransaction) {
};
} catch (err) {
const errorCode = secretResolutionErrorCode(err);
await recordAccessEvent({
if (!readOnlyEnvironmentInspection) await recordAccessEvent({
companyId,
secretId: secret.id,
userSecretDefinitionId: secret.userSecretDefinitionId ?? null,
@@ -1478,6 +1479,23 @@ export function secretService(db: Db | DbTransaction) {
return (await resolveSecretValueInternal(companyId, secretId, version, options)).value;
}
// Cloud consumes and audits the signed inspection permit before the tenant
// opens its read-only transaction. Keep the ordinary environment binding and
// version checks, without attempting tenant audit writes inside that transaction.
async function resolveSecretValueForReadOnlyEnvironmentInspection(
companyId: string,
secretId: string,
version: number | "latest",
context: { environmentId: string; configPath: string; issueId?: string | null; heartbeatRunId?: string | null },
): Promise<string> {
if (!context.environmentId || !context.configPath) throw unprocessable("Environment secret binding required");
return (await resolveSecretValueInternal(companyId, secretId, version, {
bindingContext: { consumerType: "environment", consumerId: context.environmentId,
actorType: "system", actorId: null, configPath: context.configPath,
issueId: context.issueId ?? null, heartbeatRunId: context.heartbeatRunId ?? null },
}, true)).value;
}
async function resolveSecretValueForEphemeralAccess(
companyId: string,
secretId: string,
@@ -4618,6 +4636,7 @@ export function secretService(db: Db | DbTransaction) {
resolveSecretValueForAgentAccess,
listAgentSecretAccess,
resolveSecretValueForEphemeralAccess,
resolveSecretValueForReadOnlyEnvironmentInspection,
resolveSecretValueForSandboxCleanup,
resolveSecretValueForDeviceLoginCheck,
@@ -154,7 +154,7 @@ describe.skipIf(!support.supported)("automatic skill package work", () => {
let signalStarted: () => void = () => {};
const started = new Promise<void>(resolve => { signalStarted = resolve; });
let pendingSignal: AbortSignal | undefined;
vi.spyOn(provider, method).mockImplementation(async input => {
vi.spyOn(provider, method).mockImplementation(async (input: { signal?: AbortSignal }) => {
pendingSignal = input.signal;
signalStarted();
return new Promise<never>(() => {});
@@ -5,6 +5,9 @@ export type UnsafeWorkspaceExportScope = { companyId: string; issueId?: string }
export async function notifyUnsafeWorkspaceExportRecovery(transaction: object, scope: UnsafeWorkspaceExportScope) {
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.unsafeWorkspaceExportRecovery,
JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
// Recovery settlement can release a retained remote workspace snapshot.
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.remoteWorkspaceSnapshots,
scope.issueId ? `issue:${scope.issueId}` : `company:${scope.companyId}`);
}
export async function notifyUnsafeWorkspaceExportRunChange(transaction: object, run: {
+169 -10
View File
@@ -1,6 +1,7 @@
import { issueReadSqlCondition, projectReadSqlCondition, type AuthorizationActor } from "./authorization.js";
import fs from "node:fs/promises";
import path from "node:path";
import type { Readable } from "node:stream";
import { and, desc, eq, inArray, isNull } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
import { executionWorkspaces, issues, projects, projectWorkspaces } from "@paperclipai/db";
@@ -17,7 +18,9 @@ import type {
WorkspaceFilePreviewKind,
WorkspaceFileSelector,
WorkspaceFileWorkspaceKind,
WorkspaceFileSource,
} from "@paperclipai/shared";
import type { RemoteWorkspaceFiles, RemoteWorkspaceFileAccess, RemoteWorkspaceFileEntry } from "./remote-workspace-files.js";
import { HttpError, notFound, unprocessable } from "../errors.js";
import {
isWorkspaceGitScanError,
@@ -31,6 +34,7 @@ export const WORKSPACE_FILE_LIST_DEFAULT_LIMIT = 25;
export const WORKSPACE_FILE_LIST_MAX_LIMIT = 100;
export const WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES = 5_000;
export const WORKSPACE_FILE_AVAILABILITY_CONCURRENCY = 8;
const MAX_AVAILABILITY_REMOTE_WORKSPACES = 8;
const MAX_RELATIVE_PATH_BYTES = 4096;
const TEXT_SNIFF_BYTES = 4096;
const MAX_LIST_DEPTH = 20;
@@ -122,6 +126,8 @@ type WorkspaceCandidate = {
label: string;
rootPath: string | null;
remote: boolean;
remoteAuthority?: boolean;
source?: WorkspaceFileSource;
};
type NormalizedPath = {
@@ -134,6 +140,12 @@ type LocalResolvedFile = {
realPath: string;
};
export type PreparedWorkspaceFileDownload = LocalResolvedFile | {
resource: ResolvedWorkspaceResource;
stream: Readable;
cleanup(): Promise<void>;
};
type LocalResolvedDirectory = {
resource: ResolvedWorkspaceResource;
realPath: string;
@@ -266,14 +278,21 @@ function unavailableAvailabilityResult(
async function mapWithConcurrency<T, R>(items: T[], concurrency: number, mapper: (item: T) => Promise<R>) {
const results = new Array<R>(items.length);
let nextIndex = 0;
let failure: { error: unknown } | undefined;
async function worker() {
while (nextIndex < items.length) {
while (nextIndex < items.length && !failure) {
const index = nextIndex;
nextIndex += 1;
results[index] = await mapper(items[index]!);
try {
results[index] = await mapper(items[index]!);
} catch (error) {
failure ??= { error };
}
}
}
// Every started mapper must settle before its request-owned resources close.
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
if (failure) throw failure.error;
return results;
}
@@ -370,7 +389,7 @@ function listItemFromStat(input: {
candidate: WorkspaceCandidate;
relativePath: string;
stat: { size: number; mtime: Date };
}): WorkspaceFileListItem | null {
}): WorkspaceFileListItem {
const contentType = contentTypeForPath(input.relativePath);
const previewKind = previewKindForKnownContentType(contentType) ?? "unsupported";
const previewable = previewKind !== "unsupported" && input.stat.size <= previewCapForKind(previewKind);
@@ -473,7 +492,7 @@ function remoteResource(candidate: WorkspaceCandidate, relativePath: string): Re
function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceCandidate {
const provider = row.providerType || row.strategyType || "local_fs";
const rootPath = row.cwd || row.providerRef || null;
const remote = !["local_fs", "git_worktree"].includes(provider) || !rootPath || row.status !== "active" || row.closedAt != null;
const remote = Boolean(row.metadata?.remoteWorkspace) || !["local_fs", "git_worktree"].includes(provider) || !rootPath || row.status !== "active" || row.closedAt != null;
return {
companyId: row.companyId,
workspaceKind: "execution_workspace",
@@ -482,6 +501,7 @@ function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceC
label: row.name,
rootPath,
remote,
remoteAuthority: Boolean(row.metadata?.remoteWorkspace),
};
}
@@ -680,6 +700,7 @@ function unavailableFileList(input: {
unavailableReason: input.reason,
workspace: input.candidate
? {
...(input.candidate.source ? { source: input.candidate.source } : {}),
provider: input.candidate.provider,
workspaceLabel: input.candidate.label,
workspaceKind: input.candidate.workspaceKind,
@@ -718,6 +739,7 @@ function availableFileList(input: {
kind: "workspace_file_list",
state: "available",
workspace: {
...(input.candidate.source ? { source: input.candidate.source } : {}),
provider: input.candidate.provider,
workspaceLabel: input.candidate.label,
workspaceKind: input.candidate.workspaceKind,
@@ -1041,7 +1063,42 @@ async function listChangedWorkspaceFiles(input: {
};
}
export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor) {
export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor, options: { remoteFiles?: RemoteWorkspaceFiles } = {}) {
async function remoteAccess(candidate: WorkspaceCandidate, signal?: AbortSignal) {
if (!options.remoteFiles || candidate.workspaceKind !== "execution_workspace") return null;
return options.remoteFiles.open({ companyId: candidate.companyId, workspaceId: candidate.workspaceId, signal });
}
function remoteEntryResource(candidate: WorkspaceCandidate, entry: RemoteWorkspaceFileEntry, access: RemoteWorkspaceFileAccess): ResolvedWorkspaceResource {
if (entry.kind === "directory") return { ...directoryResource({ candidate, relativePath: entry.relativePath }), source: access.source };
const contentType = contentTypeForPath(entry.relativePath);
const previewKind = previewKindForKnownContentType(contentType) ?? (entry.text ? "text" : "unsupported");
const tooLarge = entry.byteSize! > previewCapForKind(previewKind);
return {
kind: "file", provider: candidate.provider, title: path.posix.basename(entry.relativePath),
displayPath: candidate.projectName ? `${candidate.projectName} / ${entry.relativePath}` : entry.relativePath,
workspaceLabel: candidate.label, workspaceKind: candidate.workspaceKind, workspaceId: candidate.workspaceId,
projectId: candidate.projectId ?? null, projectName: candidate.projectName ?? null,
contentType: contentType ?? (previewKind === "text" ? "text/plain; charset=utf-8" : "application/octet-stream"),
byteSize: entry.byteSize, previewKind, source: access.source,
denialReason: tooLarge ? "too_large" : previewKind === "unsupported" ? "unsupported_content" : null,
capabilities: { preview: !tooLarge && previewKind !== "unsupported", download: true, listChildren: false },
};
}
async function resolveRemote(candidate: WorkspaceCandidate, normalized: NormalizedPath, directory: boolean,
requestAccess?: (candidate: WorkspaceCandidate) => Promise<RemoteWorkspaceFileAccess | null>) {
throwIfDenied(normalized.segments);
const access = await (requestAccess ?? remoteAccess)(candidate);
if (!access) return null;
try {
const entry = await access.stat(normalized.relativePath);
if (directory && entry.kind !== "directory") throw unprocessable("Workspace path is not a directory", { code: "not_directory" });
if (!directory && entry.kind !== "file") throw unprocessable("Workspace file is not a regular file", { code: "not_regular_file" });
return remoteEntryResource(candidate, entry, access);
} finally { if (!requestAccess) await access.close(); }
}
async function visibleCandidates(candidates: WorkspaceCandidate[], skipDeniedSources = false, task?: IssueRow) {
if (!actor) return candidates;
const visible: WorkspaceCandidate[] = [];
@@ -1068,7 +1125,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
}
candidate.label = "Task workspace";
}
if (candidate.rootPath && (!candidate.remote || candidate.provider === "local_fs" || LOCAL_PROJECT_WORKSPACE_SOURCE_TYPES.has(candidate.provider))) {
if (candidate.rootPath && !candidate.remoteAuthority && (!candidate.remote || candidate.provider === "local_fs" || LOCAL_PROJECT_WORKSPACE_SOURCE_TYPES.has(candidate.provider))) {
try {
const canonicalCwd = await fs.realpath(candidate.rootPath);
@@ -1323,8 +1380,22 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
? await sameCompanyProjectWorkspaceCandidates(issue, new Set(initialCandidates.map((candidate) => candidate.workspaceId)))
: [];
const explicitTargets = await loadAvailabilityTargets(issue, queries);
const remoteAccesses = new Map<string, Promise<RemoteWorkspaceFileAccess | null>>();
const requestAccess = (candidate: WorkspaceCandidate) => {
// UUIDs can be equal across workspace tables; project roots never borrow task access.
if (!options.remoteFiles || candidate.workspaceKind !== "execution_workspace") return Promise.resolve(null);
const key = `${candidate.companyId}:${candidate.workspaceId}`;
const existing = remoteAccesses.get(key);
if (existing) return existing;
if (remoteAccesses.size >= MAX_AVAILABILITY_REMOTE_WORKSPACES) {
throw unprocessable("Too many remote workspaces in one availability request", { code: "workspace_unavailable" });
}
const access = remoteAccess(candidate);
remoteAccesses.set(key, access);
return access;
};
const results = await mapWithConcurrency(
const outcome = await mapWithConcurrency(
queries,
WORKSPACE_FILE_AVAILABILITY_CONCURRENCY,
async (item): Promise<WorkspaceFileAvailabilityResult> => {
@@ -1349,6 +1420,16 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
try {
const resource = await resolveRemote(candidate, item.normalizedPath, item.directory, requestAccess);
if (resource) return availabilityResult(item.query, resource);
} catch (error) {
if (!hasExplicitTarget && item.query.workspace === "auto" && isHttpStatus(error, 404)) {
lastNotFound = error;
continue;
}
throw error;
}
if (hasExplicitTarget || item.query.workspace !== "auto") {
return availabilityResult(item.query, remoteResource(candidate, item.normalizedPath.relativePath));
}
@@ -1393,9 +1474,16 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
return unavailableAvailabilityResult(item.query, reason);
}
},
);
).then(results => ({ results }), (error: unknown) => ({ error }));
return { kind: "workspace_file_availability", results };
const opened = await Promise.allSettled(remoteAccesses.values());
const closed = await Promise.allSettled(opened.map(async result => {
if (result.status === "fulfilled") await result.value?.close();
}));
if ("error" in outcome) throw outcome.error;
const cleanupFailure = closed.find(result => result.status === "rejected");
if (cleanupFailure?.status === "rejected") throw cleanupFailure.reason;
return { kind: "workspace_file_availability", results: outcome.results };
}
async function resolve(issueId: string, input: {
@@ -1417,6 +1505,16 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
try {
const resource = await resolveRemote(candidate, normalized, isDirectoryRequest);
if (resource) return resource;
} catch (error) {
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) {
lastNotFound = error;
continue;
}
throw error;
}
if (explicitTarget || selector !== "auto") return remoteResource(candidate, normalized.relativePath);
continue;
}
@@ -1475,6 +1573,33 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
if (normalizedPath) throwIfDenied(normalizedPath.segments);
const access = await remoteAccess(candidate, opts.scanContext?.signal);
if (access) {
try {
const listed = await access.list({ relativePath: normalizedPath?.relativePath, mode, query: normalizedQuery,
limit, offset, maxScannedEntries: WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES, maxDepth: MAX_LIST_DEPTH });
const remoteCandidate = { ...candidate, source: access.source };
const items = listed.entries.filter(entry => {
try { return !shouldPruneSegments(normalizeWorkspaceRelativePath(entry.relativePath).segments); }
catch { return false; }
}).map(entry => ({
...(entry.kind === "directory"
? listItemFromDirectory({ candidate, relativePath: entry.relativePath, stat: entry.modifiedAt ? { mtime: new Date(entry.modifiedAt) } : undefined })
: listItemFromStat({ candidate, relativePath: entry.relativePath, stat: { size: entry.byteSize!, mtime: new Date(entry.modifiedAt ?? 0) } })),
source: access.source,
}));
return availableFileList({ selector, mode, path: normalizedPath?.relativePath ?? null, q, limit, offset,
candidate: remoteCandidate, items, scannedCount: listed.scannedCount, truncated: listed.truncated });
} catch (error) {
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) { lastNotFound = error; continue; }
if (error instanceof HttpError && error.status === 422 && !normalizedPath) {
return unavailableFileList({ selector, mode, path: null, q, limit, offset,
candidate: { ...candidate, source: access.source }, reason: expectedAvailabilityReason(error) ?? "workspace_unavailable" });
}
throw error;
} finally { await access.close(); }
}
firstUnavailable ??= { candidate, reason: "remote_workspace" };
if (explicitTarget || selector !== "auto") {
return unavailableFileList({
@@ -1681,6 +1806,25 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
throwIfDenied(normalized.segments);
const access = await remoteAccess(candidate);
if (access) {
try {
const resource = remoteEntryResource(candidate, await access.stat(normalized.relativePath), access);
if (resource.kind !== "file" || !resource.capabilities.preview) {
throw unprocessable("Workspace file cannot be previewed", { code: resource.denialReason ?? "not_regular_file" });
}
const data = await access.read(normalized.relativePath, { maxBytes: previewCapForKind(resource.previewKind) });
if (resource.previewKind === "text" && !looksLikeText(data.subarray(0, Math.min(data.length, TEXT_SNIFF_BYTES)))) {
throw unprocessable("Workspace file is not a text file", { code: "binary_content" });
}
return { resource, content: { encoding: resource.previewKind === "text" ? "utf8" : "base64",
data: resource.previewKind === "text" ? data.toString("utf8") : data.toString("base64") } };
} catch (error) {
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) { lastNotFound = error; continue; }
throw error;
} finally { await access.close(); }
}
if (explicitTarget || selector !== "auto") {
throw unprocessable("Remote workspaces cannot be previewed by the server", { code: "remote_workspace" });
}
@@ -1752,7 +1896,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
workspace?: WorkspaceFileSelector | null;
projectId?: string | null;
workspaceId?: string | null;
}, opts: { issue?: IssueRow } = {}): Promise<LocalResolvedFile> {
}, opts: { issue?: IssueRow } = {}): Promise<PreparedWorkspaceFileDownload> {
const issue = opts.issue ?? await getIssue(issueId);
const selector = input.workspace ?? "auto";
const explicitTarget = Boolean(input.projectId || input.workspaceId);
@@ -1765,6 +1909,21 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.remote) {
throwIfDenied(normalized.segments);
const access = await remoteAccess(candidate);
if (access) {
try {
const resource = remoteEntryResource(candidate, await access.stat(normalized.relativePath), access);
if (resource.kind !== "file") throw unprocessable("Workspace file is not a regular file", { code: "not_regular_file" });
const transfer = await access.download(normalized.relativePath);
return { resource: { ...resource, byteSize: transfer.byteSize }, stream: transfer.stream,
cleanup: async () => { try { transfer.stream.destroy(); await transfer.cleanup(); } finally { await access.close(); } } };
} catch (error) {
await access.close();
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) { lastNotFound = error; continue; }
throw error;
}
}
if (explicitTarget || selector !== "auto") {
throw unprocessable("Remote workspaces cannot be downloaded by the server", { code: "remote_workspace" });
}
+14 -2
View File
@@ -254,6 +254,8 @@ export interface WorkspaceOperationRecorder {
* timeout error is rethrown, so a hung provider can never leave an active operation.
*/
timeoutMs?: number | null;
/** Publish the running row under a caller-owned admission fence before run starts. */
admit?: (publish: () => Promise<void>) => Promise<void>;
run: (reportProgress: (input: {
metadata?: Record<string, unknown> | null;
system?: string | null;
@@ -313,6 +315,12 @@ export function workspaceOperationService(db: Db) {
const live: WorkspaceOperation[] = [];
for (const candidate of candidates) {
// Migration owns the workspace until its explicit publication/repair. It
// has no runtime-control action stamp and must never be swept as abandoned.
if (candidate.phase === "remote_workspace_migration") {
live.push(toWorkspaceOperation(candidate));
continue;
}
const action = readRuntimeControlAction(candidate.metadata);
if (!action) continue;
@@ -543,7 +551,8 @@ export function workspaceOperationService(db: Db) {
if (runtimeControlAction) liveRuntimeControlOperationIds.add(id);
try {
await writeState(database => database.insert(workspaceOperations).values({
let published = false;
const publish = async () => { await writeState(database => database.insert(workspaceOperations).values({
id,
companyId: input.companyId,
executionWorkspaceId,
@@ -561,7 +570,10 @@ export function workspaceOperationService(db: Db) {
) as Record<string, unknown> | null,
startedAt,
updatedAt: startedAt,
}));
})); published = true; };
if (recordInput.admit) await recordInput.admit(publish);
else await publish();
if (!published) throw new Error("Workspace operation admission did not publish its running row");
} catch (insertError) {
liveRuntimeControlOperationIds.delete(id);
throw insertError;
@@ -1,6 +1,6 @@
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db";
import type { ExecutionBlocker } from "@paperclipai/shared";
import { and, eq, sql } from "drizzle-orm";
import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db";
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared";
import { z } from "zod";
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
@@ -40,16 +40,34 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick<typeof he
"id" | "companyId" | "nativeIssueId" | "issueId" | "contextSnapshot" | "runnerProfileJson">) {
const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId;
const reference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
if (typeof issueId !== "string" || !reference || reference.state !== "finalized" || !reference.finalHostSha256) return null;
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
if ((run.runnerProfileJson?.remoteWorkspace != null && !parsedRemote.success) || reference && parsedRemote.success) return null;
if (typeof issueId !== "string" || (!remote && (!reference || reference.state !== "finalized" || !reference.finalHostSha256))) return null;
const [receipt] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery"),
eq(workspaceOperations.status, "succeeded"),
remote ? sql`${workspaceOperations.metadata}->>'snapshotId' = ${remote.snapshotId}` : undefined,
)).limit(1);
const proof = receipt?.metadata;
if (!proof || proof.leaseId !== reference.leaseId || proof.providerLeaseId !== reference.providerLeaseId
|| proof.descriptorSha256 !== reference.descriptorSha256 || proof.finalHostSha256 !== reference.finalHostSha256
|| typeof proof.recoveryActionId !== "string") return null;
if (!proof || typeof proof.recoveryActionId !== "string") return null;
if (remote) {
const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, run.companyId), eq(executionWorkspaces.id, remote.workspaceId),
));
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
if (!authority || authority.dirty || authority.writerRunId || authority.epoch !== remote.epoch
|| authority.snapshotId !== remote.snapshotId || proof.storage !== "remote_workspace"
|| proof.leaseId !== remote.leaseId || proof.providerLeaseId !== remote.providerLeaseId || proof.epoch !== remote.epoch) return null;
const [snapshot] = await db.select().from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, run.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, remote.workspaceId),
eq(executionWorkspaceSnapshots.id, remote.snapshotId!), eq(executionWorkspaceSnapshots.status, "committed"),
eq(executionWorkspaceSnapshots.epoch, remote.epoch), eq(executionWorkspaceSnapshots.sourceRunId, run.id),
));
if (!snapshot || snapshot.sha256 !== proof.sha256 || snapshot.generation !== proof.generation) return null;
} else if (!reference || proof.leaseId !== reference.leaseId || proof.providerLeaseId !== reference.providerLeaseId
|| proof.descriptorSha256 !== reference.descriptorSha256 || proof.finalHostSha256 !== reference.finalHostSha256) return null;
const [action] = await db.select({ evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and(
eq(issueRecoveryActions.companyId, run.companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
eq(issueRecoveryActions.id, proof.recoveryActionId),
@@ -1,12 +1,12 @@
import { retiredReusableLeasePredecessors } from "./environment-lease-handoff.js";
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
import { environmentLeases, environments, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
import { adapterExecutionControls } from "./adapter-execution-control.js";
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
import { SystemRecoveryFailure } from "./system-recovery-state.js";
import type { NativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js";
import { readStopOnlyCleanup } from "./sandbox-stop-and-retain.js";
import { ENVIRONMENT_LEASE_POLICIES, type EnvironmentLeasePolicy,
import { ENVIRONMENT_LEASE_POLICIES, readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type EnvironmentLeasePolicy,
type WorkspaceSourceVerificationFailure } from "@paperclipai/shared";
export const workspaceExportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"];
@@ -81,19 +81,42 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
const rawReference = run.runnerProfileJson?.nativeWorkspaceSync;
const reference = readNativeWorkspaceSyncReference(rawReference);
if (rawReference != null && !reference) throw invalid("native_reference_invalid");
const rawRemoteReference = run.runnerProfileJson?.remoteWorkspace;
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(rawRemoteReference);
const remoteReference = parsedRemote.success ? parsedRemote.data : null;
if ((rawRemoteReference != null && !remoteReference) || (reference && remoteReference)) throw invalid("native_reference_invalid");
const recoveryReference = reference ?? remoteReference;
const [remoteWorkspace] = remoteReference ? await db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, remoteReference.workspaceId),
)) : [];
const remoteAuthority = readRemoteWorkspaceAuthority(remoteWorkspace?.metadata);
if (remoteReference && (!remoteWorkspace || !remoteAuthority?.location || remoteAuthority.epoch !== remoteReference.epoch
|| remoteAuthority.location.leaseId !== remoteReference.leaseId || remoteAuthority.location.providerLeaseId !== remoteReference.providerLeaseId
|| remoteAuthority.location.remoteCwd !== remoteReference.remoteCwd
|| (remoteReference.state === "prepared" ? !remoteAuthority.dirty || remoteAuthority.writerRunId !== run.id
: remoteAuthority.dirty || remoteAuthority.writerRunId !== null || remoteAuthority.snapshotId !== remoteReference.snapshotId))) {
throw invalid("workspace_reference_mismatch");
}
if (remoteReference?.state === "saved") {
const [snapshot] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
eq(executionWorkspaceSnapshots.companyId, input.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, remoteReference.workspaceId),
eq(executionWorkspaceSnapshots.id, remoteReference.snapshotId!), eq(executionWorkspaceSnapshots.status, "committed"),
));
if (!snapshot) throw invalid("workspace_reference_mismatch");
}
const recorded = run.resultJson?.workspaceRestoreRecovery as { schema?: unknown; leaseIds?: unknown } | undefined;
if (recorded != null && recorded.schema !== "paperclip.workspace-restore-recovery.v1") throw invalid("restore_reference_invalid");
const historical = action.evidence.historicalWorkspaceSource as { schema?: unknown; runId?: unknown; leaseIds?: unknown } | undefined;
if (historical != null && (historical.schema !== "paperclip.historical-workspace-source.v1" || historical.runId !== run.id)) throw invalid("historical_reference_invalid");
const leaseIds = recorded ? recorded.leaseIds
: reference && workspaceExportCauses.includes(action.cause) ? [reference.leaseId]
: recoveryReference && workspaceExportCauses.includes(action.cause) ? [recoveryReference.leaseId]
: historical?.leaseIds;
if (!Array.isArray(leaseIds) || leaseIds.length === 0)
throw new SystemRecoveryFailure("historical_source_missing", "use_alternative_strategy");
if (options.maximumSources !== undefined && leaseIds.length > options.maximumSources)
throw new WorkspaceSourceInspectionLimit();
if (leaseIds.some(id => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) ||
new Set(leaseIds).size !== leaseIds.length || (reference && !leaseIds.includes(reference.leaseId))) throw invalid("lease_reference_invalid");
new Set(leaseIds).size !== leaseIds.length || (recoveryReference && !leaseIds.includes(recoveryReference.leaseId))) throw invalid("lease_reference_invalid");
const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, run.id)));
if (coordinator?.leaseOwner || adapterExecutionControls.has(run.id) ||
@@ -102,7 +125,7 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
// Reusable allocations can still owe copyback for a completed native result.
// This authorizes only preservation of the exact descriptor, never reuse for
// agent execution. Stop verification and physical ownership checks remain below.
const acceptedExportResult = reference && workspaceExportCauses.includes(action.cause)
const acceptedExportResult = recoveryReference && workspaceExportCauses.includes(action.cause)
&& run.runtimeMode === "native" && coordinator?.issueId === input.issueId
&& coordinator.resultId && run.completionContractId
? (await db.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and(
@@ -116,14 +139,16 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
for (const leaseId of leaseIds as string[]) {
const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
eq(environmentLeases.id, leaseId), eq(environmentLeases.heartbeatRunId, run.id)));
const descriptor = reference?.leaseId === leaseId ? reference : null;
const legacyDescriptor = reference?.leaseId === leaseId ? reference : null;
const remoteDescriptor = remoteReference?.leaseId === leaseId ? remoteReference : null;
const descriptor = legacyDescriptor ?? remoteDescriptor;
const remoteCwd = lease?.metadata?.remoteCwd;
if (!lease) throw invalid("source_lease_not_bound");
if (lease.issueId !== input.issueId) throw invalid("lease_issue_mismatch");
if (!lease.environmentId) throw invalid("lease_environment_missing");
if (!lease.providerLeaseId) throw invalid("lease_allocation_missing");
if (!lease.provider || lease.provider === "local") throw invalid("lease_provider_invalid");
const reusableExport = descriptor && acceptedExportResult
const reusableExport = descriptor && (acceptedExportResult || remoteDescriptor && remoteAuthority)
&& ["reuse_by_environment", "reuse_by_execution_workspace"].includes(lease.leasePolicy);
if (!["retain_on_failure", "ephemeral"].includes(lease.leasePolicy) && !reusableExport)
throw invalid("lease_policy_not_retained", lease.leasePolicy);
@@ -132,11 +157,13 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
if (descriptor && remoteCwd !== descriptor.remoteCwd) throw invalid("cwd_reference_mismatch");
if (descriptor && lease.executionWorkspaceId && lease.executionWorkspaceId !== descriptor.workspaceId)
throw invalid("workspace_reference_mismatch");
if (remoteDescriptor && lease.environmentId !== remoteAuthority!.location!.environmentId)
throw invalid("workspace_reference_mismatch");
const history = await db.select().from(environmentLeases).where(and(
ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider),
eq(environmentLeases.providerLeaseId, lease.providerLeaseId)));
const predecessors = retiredReusableLeasePredecessors(lease, history);
if (!predecessors || (predecessors.length && (run.runtimeMode !== "native" || !descriptor || lease.metadata?.agentId !== run.agentId)))
if (!predecessors || (predecessors.length && ((run.runtimeMode !== "native" && !remoteDescriptor) || !descriptor || lease.metadata?.agentId !== run.agentId)))
throw new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required");
const predecessorRuns = predecessors.map(item => item.heartbeatRunId!);
const priors = predecessorRuns.length ? await db.select().from(heartbeatRuns).where(and(
@@ -148,14 +175,19 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
for (const predecessor of predecessors) {
const prior = priors.find(item => item.id === predecessor.heartbeatRunId);
const priorReference = readNativeWorkspaceSyncReference(prior?.runnerProfileJson?.nativeWorkspaceSync);
const parsedPriorRemote = remoteWorkspaceRunReferenceSchema.safeParse(prior?.runnerProfileJson?.remoteWorkspace);
const priorRemote = parsedPriorRemote.success ? parsedPriorRemote.data : null;
const priorDescriptor = priorReference ?? priorRemote;
const priorSaved = priorReference ? priorReference.state === "finalized" && Boolean(priorReference.finalHostSha256)
: priorRemote?.state === "saved" && Boolean(priorRemote.snapshotId) && Boolean(remoteDescriptor)
&& priorRemote.epoch < remoteDescriptor!.epoch;
const priorOwner = priorOwners.find(item => item.runId === predecessor.heartbeatRunId);
if (!prior || prior.runtimeMode !== "native" || prior.agentId !== run.agentId || prior.nativeIssueId !== input.issueId ||
!terminalWorkspaceRunStates.includes(prior.status) || adapterExecutionControls.has(prior.id) ||
(prior.controllerLeaseExpiresAt && prior.controllerLeaseExpiresAt > new Date()) ||
!priorOwner || priorOwner.leaseOwner || !["committed", "terminal_failure", "applied"].includes(priorOwner.phase) ||
!priorReference || priorReference.state !== "finalized" || !priorReference.finalHostSha256 ||
priorReference.leaseId !== predecessor.id || priorReference.providerLeaseId !== lease.providerLeaseId ||
priorReference.workspaceId !== descriptor!.workspaceId || priorReference.remoteCwd !== remoteCwd)
!priorDescriptor || !priorSaved || priorDescriptor.leaseId !== predecessor.id || priorDescriptor.providerLeaseId !== lease.providerLeaseId ||
priorDescriptor.workspaceId !== descriptor!.workspaceId || priorDescriptor.remoteCwd !== remoteCwd)
throw new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required");
}
// Snapshot export writes only a new staging directory. A native baseline
@@ -175,7 +207,8 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
}
const [environment] = await db.select().from(environments).where(eq(environments.id, lease.environmentId));
if (!environment) throw new SystemRecoveryFailure("source_unavailable", "service_required");
sources.push({ lease, environment, reference: descriptor, remoteCwd });
sources.push({ lease, environment, reference: legacyDescriptor, remoteReference: remoteDescriptor,
remoteWorkspace: remoteDescriptor ? remoteWorkspace : null, remoteCwd });
}
return { run, action, sources };
}
+224 -4
View File
@@ -9,7 +9,7 @@ import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wai
import { heartbeatRunEvents } from "@paperclipai/db";
import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "./heartbeat/queue.js";
import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js";
import { recordRequiredWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";
import { recordRequiredWorkspaceRestoreFailure, terminalizeLegacyExecution } from "./legacy-execution-recovery.js";
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
import { withQueuedCommentIdsInWakePayload } from "./issue-queued-comment-queue.js";
import * as recoverySources from "./workspace-recovery-source.js";
@@ -40,13 +40,13 @@ import { requestTaskRetry } from "./task-retry.js";
import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js";
import { execFile } from "node:child_process";
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { cp, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import express from "express";
import request from "supertest";
import { and, eq, inArray, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, approvals, budgetPolicies, costEvents, completionContracts, issueApprovals, issueTreeHolds, issueTreeHoldMembers, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, agentWakeupRequests, workspaceOperations } from "@paperclipai/db";
import { resumePreservedResultFinalization } from "./preserved-result-finalization.js";
import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
@@ -64,6 +64,10 @@ import { heartbeatService } from "./heartbeat.js";
import { hasVerifiedWorkspaceRepair, readWorkspaceRecovery } from "./workspace-recovery-receipt.js";
import { claimNativeSessionResumptions } from "./native-runtime/native-finalization-reconciler.js";
import { executionWorkspaces } from "@paperclipai/db";
import { executionWorkspaceSnapshots } from "@paperclipai/db";
import { createSavedWorkspaceLoader, readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "./remote-workspaces.js";
import { createReusableLeaseHandoff } from "./environment-lease-handoff.js";
import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js";
import { getExecutionBlocker } from "./execution-blocker.js";
import { issueRoutes } from "../routes/issues.js";
import { errorHandler } from "../middleware/index.js";
@@ -80,6 +84,7 @@ describe("automatic retained workspace recovery", () => {
let connectionString: string;
let db: ReturnType<typeof createDb>;
let root: string;
const fixtureCompanyIds = new Set<string>();
const priorHome = process.env.PAPERCLIP_HOME, priorInstance = process.env.PAPERCLIP_INSTANCE_ID;
beforeAll(async () => {
root = await mkdtemp(path.join(os.tmpdir(), "workspace-recovery-"));
@@ -98,9 +103,26 @@ describe("automatic retained workspace recovery", () => {
if (priorHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = priorHome;
if (priorInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = priorInstance;
});
afterEach(async () => {
// Discovery is intentionally global and bounded. An unresolved source from
// an earlier test must not consume the current fixture's recovery page.
// Keep all history within each test until its assertions have completed.
const companyIds = [...fixtureCompanyIds]; fixtureCompanyIds.clear();
if (!companyIds.length) return;
const leases = await db.select({ id: environmentLeases.id }).from(environmentLeases)
.where(inArray(environmentLeases.companyId, companyIds));
for (const lease of leases) targets.delete(lease.id);
await db.delete(environmentLeases).where(inArray(environmentLeases.companyId, companyIds));
// Historical adoption can reconstruct an action from either the lease or
// the run descriptor, so retire both fixture inputs before deleting it.
await db.update(heartbeatRuns).set({ resultJson: sql`${heartbeatRuns.resultJson} - 'workspaceRestoreRecovery' - 'workspaceRestoreFailure'` })
.where(inArray(heartbeatRuns.companyId, companyIds));
await db.delete(issueRecoveryActions).where(inArray(issueRecoveryActions.companyId, companyIds));
});
async function seed(mode = "native", nativeDescriptor = true, resourceDisposition: "destroy" | "keep_running" = "destroy", reusable = false, ephemeral = false) {
const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), actionId: string = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Workspace recovery", defaultResponsibleUserId: "board", issuePrefix: `R${companyId.slice(0, 6)}` });
fixtureCompanyIds.add(companyId);
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "idle", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } });
await db.insert(issues).values({ id: issueId, companyId, assigneeAgentId: agentId, status: "blocked", title: "Recover existing work" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, issueId, nativeIssueId: mode === "native" ? issueId : null,
@@ -166,6 +188,199 @@ describe("automatic retained workspace recovery", () => {
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
it.each([
{ status: "failed", runtimeMode: "native", uploadFailure: "none" },
{ status: "cancelled", runtimeMode: "native", uploadFailure: "none" },
{ status: "failed", runtimeMode: "legacy", uploadFailure: "final" },
{ status: "failed", runtimeMode: "legacy", uploadFailure: "initial" },
] as const)("saves the authoritative remote workspace after $runtimeMode $status without replaying a turn or certifying external effects ($uploadFailure upload failure)", async ({ status, runtimeMode, uploadFailure }) => {
const f = await seed(runtimeMode, false);
const remoteCwd = await realpath(f.remote);
targets.get(f.lease.id)!.remoteCwd = remoteCwd;
f.runtime.resumeRunLease.mockImplementation(async () => ({ providerLeaseId: f.providerLeaseId, metadata: { remoteCwd } }));
const workspaceId = randomUUID();
let authority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null as string | null,
source: { kind: "empty" }, writerRunId: f.runId, dirty: true,
location: { environmentId: f.lease.environmentId!, leaseId: f.lease.id,
providerLeaseId: f.providerLeaseId, remoteCwd } };
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId, sourceIssueId: f.issueId,
name: "Authoritative remote files", cwd: remoteCwd, providerRef: remoteCwd, providerType: "cloud_sandbox",
mode: "isolated_workspace", strategyType: "project_primary", status: "active", metadata: { remoteWorkspace: authority } });
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.issueId));
const [boundLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
await db.update(environmentLeases).set({ executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_execution_workspace",
metadata: { ...boundLease.metadata, remoteCwd, agentId: (await run(f.runId)).agentId } })
.where(eq(environmentLeases.id, f.lease.id));
const before = await run(f.runId);
if (uploadFailure === "final") {
// A reused allocation can fail during preparation, before native input or
// its finalization coordinator exists. Keep a real saved predecessor and
// the ordinary exact lease handoff; neither grants a new provider turn.
const [priorRun] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: before.agentId,
issueId: f.issueId, nativeIssueId: f.issueId, runtimeMode: "native", status: "succeeded",
createdAt: new Date(before.createdAt.getTime() - 2_000), finishedAt: new Date(before.createdAt.getTime() - 1_000),
contextSnapshot: { issueId: f.issueId } }).returning();
await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId,
runId: priorRun.id, phase: "committed" });
const [currentLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
const [priorLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId,
heartbeatRunId: priorRun.id, environmentId: currentLease.environmentId, executionWorkspaceId: workspaceId,
provider: currentLease.provider, providerLeaseId: f.providerLeaseId, leasePolicy: "reuse_by_environment", status: "active",
acquiredAt: new Date(currentLease.acquiredAt.getTime() - 1_000), metadata: currentLease.metadata }).returning();
const priorAuthority = { ...authority, writerRunId: priorRun.id,
location: { ...authority.location, leaseId: priorLease.id } };
const [priorWorkspace] = await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: priorAuthority } })
.where(eq(executionWorkspaces.id, workspaceId)).returning();
const priorSaved = await saveRemoteWorkspace({ db, workspace: priorWorkspace, runId: priorRun.id,
lease: priorLease as EnvironmentLease, target: { ...targets.get(f.lease.id)!, leaseId: priorLease.id },
assertOwned: async () => undefined });
const handoff = createReusableLeaseHandoff(priorLease, currentLease);
await db.update(environmentLeases).set({ status: "expired", cleanupStatus: "success", releasedAt: currentLease.acquiredAt,
metadata: { ...priorLease.metadata, reusableLeaseSuccessor: handoff } }).where(eq(environmentLeases.id, priorLease.id));
await db.update(environmentLeases).set({ metadata: { ...currentLease.metadata, reusableLeasePredecessor: handoff } })
.where(eq(environmentLeases.id, currentLease.id));
authority = { ...authority, epoch: 2, snapshotId: priorSaved.snapshotId };
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: authority } })
.where(eq(executionWorkspaces.id, workspaceId));
expect(before.runtimeMode).toBe("legacy");
expect(before.runnerProfileJson?.nativeExecutionInput).toBeUndefined();
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))).toEqual([]);
}
if (runtimeMode === "legacy") await writeFile(path.join(f.remote, "pre-native-change.txt"), "current preparation wrote these files\n");
await db.update(heartbeatRuns).set({ status, runnerProfileJson: { ...before.runnerProfileJson,
...(status === "failed" && runtimeMode === "native" ? { nativeResourceDisposition: "destroy" } : {}),
remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId: f.lease.id,
providerLeaseId: f.providerLeaseId, remoteCwd, epoch: authority.epoch, state: "prepared", snapshotId: authority.snapshotId } },
resultJson: { ...before.resultJson, ...(status === "cancelled" ? {
nativeCancellation: { dispatchState: "acknowledged", requestedAt: new Date().toISOString() },
} : {}) },
}).where(eq(heartbeatRuns.id, f.runId));
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...authority, epoch: authority.epoch + 1 } } })
.where(eq(executionWorkspaces.id, workspaceId));
await expect(recoverySources.inspectWorkspaceRecoverySource(db, f.input)).rejects.toMatchObject({ code: "source_binding_invalid" });
expect(f.runtime.resumeRunLease).not.toHaveBeenCalled();
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: authority } })
.where(eq(executionWorkspaces.id, workspaceId));
if (runtimeMode === "legacy") {
// Remote file ownership exists before native runtime selection.
expect((await recoverySources.inspectWorkspaceRecoverySource(db, f.input)).sources).toHaveLength(1);
const failedUpload = vi.spyOn(getStorageService(), "putFile").mockRejectedValueOnce(new Error("Synthetic saved-copy outage"));
try {
if (uploadFailure === "initial") {
// Exercise the original setup boundary, not a failed final save that
// retries the same operation. The initial ledger reservation survives.
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
await db.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null, resultJson: null })
.where(eq(heartbeatRuns.id, f.runId));
const [originalLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
const activeMetadata = { ...originalLease.metadata };
delete activeMetadata.remoteExecutionTermination;
const [activeLease] = await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null,
metadata: activeMetadata }).where(eq(environmentLeases.id, f.lease.id)).returning();
const [preparing] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
await expect(saveRemoteWorkspace({ db, workspace: preparing, runId: f.runId, lease: activeLease as EnvironmentLease,
target: targets.get(f.lease.id)!, assertOwned: async () => undefined, boundary: "initial" }))
.rejects.toThrow("Synthetic saved-copy outage");
const preserved = await preserveRemoteWorkspacePreparationFailure(db, activeLease, { assertOwned: async () => undefined });
expect(preserved?.resultJson).toMatchObject({ workspaceRestoreFailure: "restore_failed",
workspaceRestoreRecovery: { leaseIds: [f.lease.id] } });
await terminalizeLegacyExecution({ db, run: preserved!, status: "failed", patch: {
errorCode: "setup_failed", error: "Synthetic saved-copy outage", finishedAt: new Date(),
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
} });
const repairs = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId),
sql`${issueRecoveryActions.evidence}->>'runId' = ${f.runId}`));
expect(repairs).toHaveLength(1);
f.actionId = repairs[0].id;
f.input.actionId = repairs[0].id;
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))).toEqual([]);
expect((await run(f.runId)).runnerProfileJson).not.toHaveProperty("nativeExecutionInput");
} else await recoverRetainedWorkspace(f.input);
expect(failedUpload).toHaveBeenCalledTimes(1);
} finally { failedUpload.mockRestore(); }
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).not.toBe("complete");
const [unsaved] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
expect(readRemoteWorkspaceAuthority(unsaved.metadata)).toMatchObject({ dirty: true, writerRunId: f.runId,
epoch: authority.epoch, snapshotId: authority.snapshotId });
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n");
if (uploadFailure === "final") await makeRetryDue(f.actionId);
}
await recoverRetainedWorkspace(f.input);
const recoveredAction = await action(f.actionId);
expect(readSystemRecovery(recoveredAction.evidence)?.state).toBe("complete");
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
const saved = readRemoteWorkspaceAuthority(workspace.metadata);
expect(saved).toMatchObject({ dirty: false, writerRunId: null });
expect(saved?.snapshotId).toBeTruthy();
const snapshots = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.executionWorkspaceId, workspaceId));
expect(snapshots).toHaveLength(runtimeMode === "legacy" ? 2 : 1);
expect(snapshots.find(snapshot => snapshot.id === saved!.snapshotId))
.toMatchObject({ status: "committed", sourceRunId: f.runId, generation: runtimeMode === "legacy" ? 2 : 1 });
const extracted = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId, snapshotId: saved!.snapshotId! });
try {
expect(await readFile(path.join(extracted.root, "saved.txt"), "utf8")).toBe("finished work\n");
if (runtimeMode === "legacy") expect(await readFile(path.join(extracted.root, "pre-native-change.txt"), "utf8"))
.toBe("current preparation wrote these files\n");
}
finally { await extracted.cleanup(); }
expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n");
expect(await run(f.runId)).toMatchObject({ status, errorCode: uploadFailure === "initial" ? "setup_failed" : "execution_interrupted",
runnerProfileJson: { remoteWorkspace: { state: "saved", snapshotId: saved!.snapshotId } } });
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId))).toHaveLength(0);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.companyId))).toHaveLength(0);
expect(await db.select().from(workspaceOperations).where(and(eq(workspaceOperations.heartbeatRunId, f.runId),
eq(workspaceOperations.phase, "workspace_recovery_snapshot")))).toHaveLength(0);
expect(f.runtime.resumeRunLease).toHaveBeenCalledTimes(uploadFailure === "final" ? 2 : 1);
expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalledTimes(runtimeMode === "legacy" ? 2 : 1);
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
await scheduleRecoveredWorkspaceCleanup(db, await run(f.runId));
const [settledLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
if (runtimeMode === "legacy") {
expect(settledLease).toMatchObject({ status: "released",
leasePolicy: uploadFailure === "initial" ? "retain_on_failure" : "reuse_by_execution_workspace" });
expect((await run(f.runId)).runtimeMode).toBe("legacy");
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))).toEqual([]);
if (uploadFailure === "final") expect(snapshots.find(snapshot => snapshot.id === authority.snapshotId)).toMatchObject({ status: "superseded" });
else expect(snapshots.find(snapshot => snapshot.generation === 1)).toMatchObject({ status: "pending" });
// Explicit Retry may admit the preserved files. Saturate agent capacity
// so this tests admission without dispatching a new agent turn.
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: before.agentId, status: "running" });
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board",
actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() });
await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!);
const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));
expect(successor).toBeDefined();
expect(await readFreshWorkspaceReadiness(db, successor))
.toMatchObject({ mode: "resume_preserved_workspace", preservationReceiptId: expect.any(String) });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, successor.id))).toEqual([]);
expect(f.runtime.resumeRunLease).toHaveBeenCalledTimes(uploadFailure === "final" ? 2 : 1);
return;
}
if (status === "failed") {
expect(settledLease).toMatchObject({ status: "pending_cleanup", leasePolicy: "ephemeral",
metadata: { workspaceRecoveryCleanup: { runId: f.runId, snapshotId: saved!.snapshotId, epoch: authority.epoch } } });
const source = await run(f.runId);
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" });
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board",
actorSource: "local_implicit", observedRunId: source.id, idempotencyKey: randomUUID() });
await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!);
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0])
.toMatchObject({ status: "deferred_issue_execution", payload: { executionWait: { reason: "remote_cleanup" } } });
await environmentService(db).releaseLease(settledLease.id, "released", { cleanupStatus: "success",
remoteExecutionTermination: remoteTerminationReceipt(settledLease, { providerLeaseId: settledLease.providerLeaseId, state: "destroyed" }) });
await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!);
const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));
expect(successor).toBeDefined();
const readiness = await readFreshWorkspaceReadiness(db, successor);
expect(readiness).toMatchObject({ mode: "resume_preserved_workspace", preservationReceiptId: expect.any(String) });
} else {
expect(settledLease).toMatchObject({ status: "released", leasePolicy: "reuse_by_execution_workspace" });
expect(settledLease.metadata?.workspaceRecoveryCleanup).toBeUndefined();
}
}, 30_000);
async function warmHandoffSource(hops = 1, beforeStop?: (lease: typeof environmentLeases.$inferSelect) => Promise<void>) {
const prior = await seed("native", true, "keep_running", true), source = await run(prior.runId);
let previousTarget = targets.get(prior.lease.id)!, previousLeaseId = prior.lease.id;
@@ -216,8 +431,13 @@ describe("automatic retained workspace recovery", () => {
let isolatedRoot: string | undefined;
beforeAll(async () => {
original = { db, root, connectionString, home: process.env.PAPERCLIP_HOME };
isolated = await startEmbeddedPostgresTestDatabase("workspace-warm-handoff-");
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_RECOVERY_HANDOFF_TEST_DATABASE_URL");
if (external) {
await applyPendingMigrations(external);
isolated = { connectionString: external, cleanup: async () => {} };
} else isolated = await startEmbeddedPostgresTestDatabase("workspace-warm-handoff-");
isolatedDb = createDb(isolated.connectionString);
if (external) await isolatedDb.execute(sql`TRUNCATE TABLE companies CASCADE`);
isolatedRoot = await mkdtemp(path.join(os.tmpdir(), "workspace-warm-handoff-files-"));
db = isolatedDb; root = isolatedRoot; connectionString = isolated.connectionString;
process.env.PAPERCLIP_HOME = path.join(root, "home");
+41 -8
View File
@@ -10,8 +10,8 @@ import { beginIdleTrackedWork } from "./task-admission.js";
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
import { resumePreservedResultFinalization } from "./preserved-result-finalization.js";
import { and, asc, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm";
import { agentWakeupRequests, environmentLeases, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, type Db } from "@paperclipai/db";
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
import { agentWakeupRequests, environmentLeases, executionWorkspaces, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, type Db } from "@paperclipai/db";
import { remoteWorkspaceRunReferenceSchema, type Environment, type EnvironmentLease } from "@paperclipai/shared";
import { conflict } from "../errors.js";
import { logger } from "../middleware/logger.js";
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
@@ -31,6 +31,7 @@ import { SYSTEM_RECOVERY_MAX_RETRIES, createSystemRecovery, failSystemRecovery,
import { inspectWorkspaceRecoverySource, workspaceRecoverySourceEligible, workspaceSourceBindingFailure } from "./workspace-recovery-source.js";
import { preserveWorkspaceSnapshot } from "./workspace-recovery-snapshot.js";
import { adoptHistoricalWorkspaceRecoveries } from "./workspace-recovery-backfill.js";
import { saveRemoteWorkspace } from "./remote-workspaces.js";
const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"];
const terminal = ["failed", "interrupted", "timed_out", "cancelled", "succeeded"];
@@ -278,7 +279,29 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ
await inspectWorkspaceRecoverySource(db, input);
await progress({ phase: "preserve_files" });
let operationId: string;
if (reference) {
if (source.remoteReference && source.remoteWorkspace) {
const [workspace] = await db.select().from(executionWorkspaces).where(and(
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, source.remoteWorkspace.id),
));
if (!workspace) throw workspaceSourceBindingFailure("workspace_reference_mismatch");
const saved = await saveRemoteWorkspace({ db, workspace, runId: input.runId,
lease: cleanupLease as EnvironmentLease, target, assertOwned: ownership.assertHeld });
await ownership.assertHeld();
const [existing] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.heartbeatRunId, input.runId),
eq(workspaceOperations.phase, "workspace_recovery"), eq(workspaceOperations.status, "succeeded"),
sql`${workspaceOperations.metadata}->>'leaseId' = ${lease.id}`,
sql`${workspaceOperations.metadata}->>'snapshotId' = ${saved.snapshotId}`,
)).limit(1);
const operation = existing ?? (await db.insert(workspaceOperations).values({ companyId: input.companyId,
issueId: input.issueId, heartbeatRunId: input.runId, executionWorkspaceId: workspace.id,
phase: "workspace_recovery", status: "succeeded", finishedAt: new Date(), metadata: {
recoveryActionId: input.actionId, leaseId: lease.id, providerLeaseId: lease.providerLeaseId,
snapshotId: saved.snapshotId, generation: saved.generation, sha256: saved.sha256,
storage: "remote_workspace", epoch: source.remoteReference.epoch,
} }).returning({ id: workspaceOperations.id }))[0]!;
operationId = operation.id;
} else if (reference) {
const copied = await resumeNativeWorkspaceSync({ db, runId: input.runId, target, assertOwnership: ownership.assertHeld });
if (!copied) throw new SystemRecoveryFailure("snapshot_verification_failed", "service_required");
const after = await inspectWorkspaceRecoverySource(db, input);
@@ -384,8 +407,12 @@ const unfinishedCursors = new WeakMap<Db, string>();
* saved-file receipt and the original recorded disposition. */
export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof heartbeatRuns.$inferSelect) {
if (!await hasVerifiedWorkspaceRepair(db, run)) return;
const ref = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
if (ref?.resourceDisposition !== "destroy") return;
const legacy = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
const ref = legacy ?? remote;
const disposition = legacy?.resourceDisposition ?? (remote ? run.runnerProfileJson?.nativeResourceDisposition : undefined);
if (!ref || disposition !== "destroy") return;
const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, ref.leaseId)));
if (!lease || !hasRemoteTerminationReceipt(lease)) return;
const [other] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
@@ -394,7 +421,8 @@ export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof hear
if (other) return;
await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", leasePolicy: "ephemeral",
updatedAt: new Date(), metadata: sql`(${environmentLeases.metadata} - 'workspaceRestoreRecovery' - 'remoteExecutionTermination') ||
${JSON.stringify({ workspaceRecoveryCleanup: { runId: run.id, descriptorSha256: ref.descriptorSha256 },
${JSON.stringify({ workspaceRecoveryCleanup: { runId: run.id, ...(legacy ? { descriptorSha256: legacy.descriptorSha256 }
: { snapshotId: remote!.snapshotId, epoch: remote!.epoch }) },
pendingCleanupInFlight: false, pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupLeaseExpiresAtMs: 0 })}::jsonb`,
}).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, ref.leaseId),
eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.providerLeaseId, ref.providerLeaseId),
@@ -403,10 +431,15 @@ export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof hear
sql`${environmentLeases.metadata}->>'workspaceRecoveryCleanup' is null`)), { companyId: run.companyId, runId: run.id });
}
export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedOnly = false, obligationsOnly = false, inFlightRunIds: string[] = []) {
const cleanup = sql`(${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,resourceDisposition}' = 'destroy' and exists (
const cleanup = sql`(((${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,resourceDisposition}' = 'destroy'
and ${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,leaseId}' is not null)
or (${heartbeatRuns.runnerProfileJson}->>'nativeResourceDisposition' = 'destroy'
and ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,schema}' = 'paperclip.remote-workspace-run.v1'
and ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,state}' = 'saved')) and exists (
select 1 from ${environmentLeases} where ${environmentLeases.companyId} = ${heartbeatRuns.companyId}
and ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id}
and ${environmentLeases.id}::text = ${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,leaseId}'
and ${environmentLeases.id}::text = coalesce(${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,leaseId}',
${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,leaseId}')
and ${environmentLeases.status} = 'released' and ${environmentLeases.cleanupStatus} = 'success'
and ${environmentLeases.metadata}->'workspaceRecoveryCleanup' is null))`;
const waitingInput = sql`exists (select 1 from ${agentWakeupRequests}
+47 -10
View File
@@ -21,6 +21,7 @@ import type { Db } from "@paperclipai/db";
import { executionWorkspaces, issueComments, issues, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db";
import {
DEFAULT_TAILSCALE_HTTPS_EXPOSURE,
readRemoteWorkspaceAuthority,
deriveViteHmrPort,
forceLoopbackBindInCommand,
isRuntimeExposureAppPort,
@@ -45,6 +46,7 @@ import {
import { and, desc, eq, gte, inArray, isNull, lte, ne, or, sql } from "drizzle-orm";
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
import { conflict, unprocessable } from "../errors.js";
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
import { resolveHomeAwarePath } from "../home-paths.js";
import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
import {
@@ -4279,6 +4281,7 @@ export async function cleanupExecutionWorkspaceArtifacts(input: {
forceWorktreeRemoval?: boolean;
}) {
if (input.workspace.providerType === "plugin") throw new Error("Managed workspace cleanup requires its storage plugin");
if (readRemoteWorkspaceAuthority(input.workspace.metadata)) throw new Error("Remote workspace cleanup requires its sandbox lifecycle");
const warnings: string[] = [];
const workspacePath = input.workspace.providerRef ?? input.workspace.cwd;
const isolatedSource = await verifiedRecordedIsolatedRemoteSource(input.workspace);
@@ -7538,14 +7541,14 @@ function selectRuntimeServiceEntries(input: {
});
}
async function isPersistedIsolatedExecutionWorkspace(input: {
async function inspectPersistedRuntimeWorkspace(input: {
db?: Db;
companyId: string;
executionWorkspaceId?: string | null;
}) {
if (!input.db || !input.executionWorkspaceId) return false;
if (!input.db || !input.executionWorkspaceId) return { isolated: false, remote: false };
const row = await input.db
.select({ mode: executionWorkspaces.mode })
.select({ mode: executionWorkspaces.mode, metadata: executionWorkspaces.metadata })
.from(executionWorkspaces)
.where(
and(
@@ -7555,7 +7558,7 @@ async function isPersistedIsolatedExecutionWorkspace(input: {
)
.limit(1)
.then((rows) => rows[0] ?? null);
return row?.mode === "isolated_workspace";
return { isolated: row?.mode === "isolated_workspace", remote: Boolean(readRemoteWorkspaceAuthority(row?.metadata)) };
}
type EnsureRuntimeServicesForRunInput = {
@@ -7583,16 +7586,24 @@ async function ensureRuntimeServicesForRunInvocation(
});
const acquiredServiceIds: string[] = [];
const refs: RuntimeServiceRef[] = [];
const persistedWorkspace = await inspectPersistedRuntimeWorkspace({
db: input.db,
companyId: input.agent.companyId,
executionWorkspaceId: input.executionWorkspaceId,
});
if (persistedWorkspace.remote) {
if (rawServices.length > 0) {
throw new WorkspaceRuntimeValidationFailure("Remote workspace storage does not support Core workspace services.",
{ reason: "remote_workspace_runtime_services_unsupported" });
}
return [];
}
const allowFixedPortFallback = persistedWorkspace.isolated;
const runtimeProvision = input.skipHostProvision
? { command: "", kind: null }
: resolveRuntimeProvision(input);
const runtimeProvisionCommand = runtimeProvision.command;
const provisionCoordinator = createRuntimeProvisionCoordinator();
const allowFixedPortFallback = await isPersistedIsolatedExecutionWorkspace({
db: input.db,
companyId: input.agent.companyId,
executionWorkspaceId: input.executionWorkspaceId,
});
runtimeServiceLeasesByRun.set(input.runId, acquiredServiceIds);
try {
@@ -8212,6 +8223,7 @@ export async function startRuntimeServicesForWorkspaceControl(
defaultDesiredState: readDesiredRuntimeState(input.config.desiredState) ?? "stopped",
serviceStates: readConfiguredServiceStates(input.config),
});
if (services.length === 0) return [];
const invocationId = input.invocationId ?? "workspace_control";
const mutexPlan = resolveRuntimeStartMutexPlan({
services,
@@ -8225,7 +8237,32 @@ export async function startRuntimeServicesForWorkspaceControl(
try {
return await withRuntimeStartMutexes(
mutexPlan.ownerKeys,
() => startRuntimeServicesForWorkspaceControlInvocation(input),
async () => {
if (!input.db || !input.executionWorkspaceId) {
return startRuntimeServicesForWorkspaceControlInvocation(input);
}
// Startup and direct callers can bypass the HTTP operation recorder.
// Keep migration fenced through provision/bind/persistence, including the
// HTTPS path which intentionally commits its reservation before binding.
return withExecutionWorkspaceLifecycleLock(input.db, input.executionWorkspaceId, async (tx) => {
const [fresh] = await tx.select({ cwd: executionWorkspaces.cwd, metadata: executionWorkspaces.metadata })
.from(executionWorkspaces).where(and(eq(executionWorkspaces.id, input.executionWorkspaceId!),
eq(executionWorkspaces.companyId, input.actor.companyId)));
if (!fresh) throw new Error("Execution workspace not found before starting runtime services");
if (readRemoteWorkspaceAuthority(fresh.metadata)) {
throw unprocessable("Workspace commands are not supported for remote workspaces yet. Run the command through an agent task.",
{ code: "remote_workspace_runtime_commands_unsupported" });
}
if (fresh.metadata?.remoteWorkspaceMigration) {
throw conflict("The workspace is being moved to remote storage. Retry after migration completes.",
{ code: "workspace_runtime_migration_in_progress" });
}
if (fresh.cwd !== input.workspace.cwd) {
throw conflict("Execution workspace changed before command admission.", { code: "workspace_runtime_authority_changed" });
}
return startRuntimeServicesForWorkspaceControlInvocation(input);
});
},
);
} finally {
releaseRuntimeReplacementClaims(mutexPlan.replacementReuseKeys);
+40
View File
@@ -5,6 +5,7 @@ import {
HeadObjectCommand,
PutObjectCommand,
ListObjectsV2Command,
ListObjectVersionsCommand,
} from "@aws-sdk/client-s3";
import { putS3Multipart } from "./s3-multipart.js";
import { withS3FailureDiagnostics } from "./s3-failure.js";
@@ -79,6 +80,32 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid
forcePathStyle: Boolean(config.forcePathStyle),
});
async function listExactVersions(key: string, signal?: AbortSignal) {
const versions = new Set<string>(), cursors = new Set<string>();
let keyMarker: string | undefined, versionMarker: string | undefined;
while (true) {
signal?.throwIfAborted();
const output = await client.send(new ListObjectVersionsCommand({ Bucket: bucket, Prefix: key,
KeyMarker: keyMarker, VersionIdMarker: versionMarker, MaxKeys: 1000 }), { abortSignal: signal });
if (typeof output.IsTruncated !== "boolean") throw new Error("s3_version_listing_invalid");
for (const entry of [...(output.Versions ?? []), ...(output.DeleteMarkers ?? [])]) {
if (typeof entry.Key !== "string" || !entry.Key.startsWith(key) ||
typeof entry.VersionId !== "string" || !entry.VersionId) throw new Error("s3_version_listing_invalid");
// Prefix is the narrowest S3 listing filter, but neighboring keys are never ours to delete.
if (entry.Key === key) versions.add(entry.VersionId);
}
if (!output.IsTruncated) return versions;
const nextKey = output.NextKeyMarker, nextVersion = output.NextVersionIdMarker;
if (!nextKey || !nextKey.startsWith(key) || !nextVersion ||
(keyMarker && Buffer.compare(Buffer.from(nextKey), Buffer.from(keyMarker)) < 0)) {
throw new Error("s3_version_listing_cursor_invalid");
}
const cursor = JSON.stringify([nextKey, nextVersion]);
if (cursors.has(cursor)) throw new Error("s3_version_listing_cursor_invalid");
cursors.add(cursor); keyMarker = nextKey; versionMarker = nextVersion;
}
}
return {
id: "s3",
@@ -182,6 +209,19 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid
async deleteObject(input): Promise<void> {
const key = buildKey(prefix, input.objectKey);
if (input.allVersions) {
await withS3FailureDiagnostics("delete_object", async () => {
// Finish listing before deleting: removing a cursor version must not skip later pages.
const versions = await listExactVersions(key, input.signal);
for (const versionId of versions) {
input.signal?.throwIfAborted();
await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: versionId }),
{ abortSignal: input.signal });
}
if ((await listExactVersions(key, input.signal)).size) throw new Error("s3_object_versions_still_present");
});
return;
}
await withS3FailureDiagnostics("delete_object", () => client.send(
new DeleteObjectCommand({
Bucket: bucket,
+2 -1
View File
@@ -151,7 +151,8 @@ export function createStorageService(provider: StorageProvider): StorageService
async deleteObject(companyId: string, objectKey: string, options) {
ensureCompanyPrefix(companyId, objectKey);
await cancellable(options?.signal, () => provider.deleteObject({ objectKey, signal: options?.signal }));
await cancellable(options?.signal, () => provider.deleteObject({ objectKey, signal: options?.signal,
...(options?.allVersions ? { allVersions: true } : {}) }));
},
};
}
+9 -2
View File
@@ -21,6 +21,13 @@ export interface GetObjectInput {
};
}
export interface DeleteObjectInput {
objectKey: string;
signal?: AbortSignal;
/** Remove every version and delete marker of this exact key before returning. */
allVersions?: boolean;
}
export interface GetObjectResult {
stream: Readable;
contentType?: string;
@@ -42,7 +49,7 @@ export interface StorageProvider {
putObject(input: PutObjectInput): Promise<void>;
getObject(input: GetObjectInput): Promise<GetObjectResult>;
headObject(input: GetObjectInput): Promise<HeadObjectResult>;
deleteObject(input: GetObjectInput): Promise<void>;
deleteObject(input: DeleteObjectInput): Promise<void>;
putObjectConditional?(input: PutObjectInput & (
| { ifNoneMatch: "*"; ifMatch?: never }
@@ -79,5 +86,5 @@ export interface StorageService {
putFile(input: PutFileInput): Promise<PutFileResult>;
getObject(companyId: string, objectKey: string, options?: Pick<GetObjectInput, "range" | "signal">): Promise<GetObjectResult>;
headObject(companyId: string, objectKey: string, options?: Pick<GetObjectInput, "signal">): Promise<HeadObjectResult>;
deleteObject(companyId: string, objectKey: string, options?: Pick<GetObjectInput, "signal">): Promise<void>;
deleteObject(companyId: string, objectKey: string, options?: Pick<DeleteObjectInput, "signal" | "allVersions">): Promise<void>;
}