mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-03 01:27:09 +01:00
Merge qualified remote workspace storage assembly for finite boot acceptance
* commit '343f1e57122f061cb229b8403fe2f522df178aea': feat(workspaces): keep remote files off the webserver disk (#18403)
This commit is contained in:
commit
af5492a39f
99 files changed
+9874
-238
No files matched your search
@@ -0,0 +1,28 @@
|
||||
import { closeRegisteredClients, createDb } from "@paperclipai/db";
|
||||
import { getStorageService } from "../src/storage/index.js";
|
||||
import { migrateRemoteWorkspace } from "../src/services/remote-workspace-migration.js";
|
||||
import { assertRemoteWorkspacePersistence } from "../src/services/remote-workspaces.js";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const allowed = new Set(["--company-id", "--workspace-id", "--dry-run", "--apply"]);
|
||||
function identifier(name: string) {
|
||||
const value = args[args.indexOf(name) + 1];
|
||||
if (!args.includes(name) || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value ?? "")) throw new Error(`${name} requires a UUID`);
|
||||
return value!;
|
||||
}
|
||||
try {
|
||||
for (let index = 0; index < args.length; index++) {
|
||||
const arg = args[index]!; if (!allowed.has(arg)) throw new Error(`Unknown argument: ${arg}`);
|
||||
if (arg === "--company-id" || arg === "--workspace-id") index++;
|
||||
}
|
||||
if (args.includes("--dry-run") && args.includes("--apply")) throw new Error("Select either --dry-run or --apply");
|
||||
const companyId = identifier("--company-id"), workspaceId = identifier("--workspace-id");
|
||||
const storage = getStorageService();
|
||||
const { databaseUrl } = assertRemoteWorkspacePersistence(storage);
|
||||
const db = createDb(databaseUrl);
|
||||
try { process.stdout.write(`${JSON.stringify(await migrateRemoteWorkspace({ db, storage, companyId, workspaceId, apply: args.includes("--apply") }), null, 2)}\n`); }
|
||||
finally { await closeRegisteredClients(databaseUrl, { scope: "database" }); }
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : "Remote workspace migration failed"}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import * as remoteWorkspaces from "../services/remote-workspaces.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hoisted mocks — must be declared before any imports that reference them
|
||||
@@ -9,6 +11,7 @@ const mockAdapterExecutionTargetToRemoteSpec = vi.hoisted(() => vi.fn());
|
||||
const mockBuildWorkspaceRealizationRequest = vi.hoisted(() => vi.fn());
|
||||
const mockUpdateLeaseMetadata = vi.hoisted(() => vi.fn());
|
||||
const mockUpdateExecutionWorkspace = vi.hoisted(() => vi.fn());
|
||||
const mockGetStoredWorkspace = vi.hoisted(() => vi.fn());
|
||||
const mockLogActivity = vi.hoisted(() => vi.fn());
|
||||
const mockLoggerInfo = vi.hoisted(() => vi.fn());
|
||||
const mockGetEnvironment = vi.hoisted(() => vi.fn());
|
||||
@@ -41,6 +44,7 @@ vi.mock("../services/environments.js", () => ({
|
||||
vi.mock("../services/execution-workspaces.js", () => ({
|
||||
executionWorkspaceService: vi.fn(() => ({
|
||||
update: mockUpdateExecutionWorkspace,
|
||||
getStoredById: mockGetStoredWorkspace,
|
||||
})),
|
||||
}));
|
||||
|
||||
@@ -272,6 +276,37 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("realizes an enrolled remote workspace in place and persists its authoritative remote root", async () => {
|
||||
const environment = { ...makeEnvironment("sandbox"), id: randomUUID() };
|
||||
const lease = makeLease({ id: randomUUID(), environmentId: environment.id, provider: "daytona", providerLeaseId: "sandbox-allocation" });
|
||||
const workspace = makePersistedExecutionWorkspace({ id: randomUUID(), providerType: "cloud_sandbox", metadata: {
|
||||
remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, source: { kind: "empty" },
|
||||
writerRunId: randomUUID(), dirty: false, snapshotId: null, location: null },
|
||||
} });
|
||||
const remoteRoot = `/sandbox/.paperclip-workspaces/${workspace.id}`;
|
||||
const savedLease = { ...lease, metadata: { remoteCwd: remoteRoot } };
|
||||
const target = { kind: "remote", transport: "sandbox", remoteCwd: "/sandbox", runner: { execute: vi.fn() } };
|
||||
mockResolveEnvironmentExecutionTarget.mockResolvedValue(target);
|
||||
mockGetStoredWorkspace.mockResolvedValue({ ...workspace, cwd: remoteRoot });
|
||||
mockUpdateLeaseMetadata.mockResolvedValue(savedLease);
|
||||
const prepare = vi.spyOn(remoteWorkspaces, "prepareRemoteWorkspace").mockResolvedValue({
|
||||
workspace, lease: savedLease, remoteRoot, reference: {} as never,
|
||||
} as never);
|
||||
try {
|
||||
const runtime = makeMockRuntime();
|
||||
const input = { ...makeRealizeInput({ environment, lease, persistedExecutionWorkspace: workspace }), adapterType: "paperclip_runner" };
|
||||
const result = await environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }).realizeForRun(input);
|
||||
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ workspace, target, source: { kind: "empty" } }));
|
||||
expect(mockBuildWorkspaceRealizationRequest).not.toHaveBeenCalled();
|
||||
expect(runtime.realizeWorkspace).not.toHaveBeenCalled();
|
||||
expect(result.executionTarget).toMatchObject({ remoteCwd: remoteRoot,
|
||||
workspaceRealization: { mode: "in_place", authoritativeRoot: remoteRoot, outboundRestorePaths: [] } });
|
||||
expect(input.executionWorkspace.cwd).toBe(remoteRoot);
|
||||
expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(lease.id, expect.objectContaining({ remoteCwd: remoteRoot }));
|
||||
expect(result.lease.metadata?.remoteCwd).toBe(remoteRoot);
|
||||
} finally { prepare.mockRestore(); }
|
||||
});
|
||||
|
||||
it("happy path: returns lease, executionTarget, and remoteExecution on successful realization", async () => {
|
||||
const executionTarget = { kind: "local", environmentId: "env-1", leaseId: "lease-1" };
|
||||
const remoteExecution = { kind: "local", environmentId: "env-1", leaseId: "lease-1" };
|
||||
|
||||
@@ -6,7 +6,7 @@ import path from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
// This unit suite isolates runtime lifecycle behavior; privacy is covered with a
|
||||
// real database by privacy-production-review.test.ts.
|
||||
@@ -37,6 +37,7 @@ const mockAssertCanManageProjectWorkspaceRuntimeServices = vi.hoisted(() => vi.f
|
||||
const mockStartRuntimeServices = vi.hoisted(() => vi.fn());
|
||||
const mockStopRuntimeServicesForExecutionWorkspace = vi.hoisted(() => vi.fn());
|
||||
const mockEnsurePersistedExecutionWorkspaceAvailable = vi.hoisted(() => vi.fn());
|
||||
const mockRunWorkspaceJobForControl = vi.hoisted(() => vi.fn());
|
||||
const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn());
|
||||
const mockListConfiguredRuntimeServiceEntries = vi.hoisted(() => vi.fn());
|
||||
// The integrated control path also takes the durable runtime-control lease (PAP-17205). This
|
||||
@@ -45,6 +46,10 @@ const mockListConfiguredRuntimeServiceEntries = vi.hoisted(() => vi.fn());
|
||||
const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null));
|
||||
const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined));
|
||||
const mockSpawn = vi.hoisted(() => vi.fn());
|
||||
const mockPublishRuntimeOperation = vi.hoisted(() => vi.fn());
|
||||
const mockLifecycleLock = vi.hoisted(() => vi.fn(async (_db: unknown, _id: string, run: (tx: unknown) => Promise<unknown>) => run({})));
|
||||
|
||||
vi.mock("../services/execution-workspace-lifecycle-lock.js", () => ({ withExecutionWorkspaceLifecycleLock: mockLifecycleLock }));
|
||||
|
||||
vi.mock("node:child_process", async () => ({
|
||||
...(await vi.importActual<typeof import("node:child_process")>("node:child_process")),
|
||||
@@ -74,7 +79,7 @@ vi.mock("../services/workspace-runtime.js", () => ({
|
||||
cleanupExecutionWorkspaceArtifacts: vi.fn(),
|
||||
ensurePersistedExecutionWorkspaceAvailable: mockEnsurePersistedExecutionWorkspaceAvailable,
|
||||
listConfiguredRuntimeServiceEntries: mockListConfiguredRuntimeServiceEntries,
|
||||
runWorkspaceJobForControl: vi.fn(),
|
||||
runWorkspaceJobForControl: mockRunWorkspaceJobForControl,
|
||||
startRuntimeServicesForWorkspaceControl: mockStartRuntimeServices,
|
||||
stopRuntimeServicesForExecutionWorkspace: mockStopRuntimeServicesForExecutionWorkspace,
|
||||
stopRuntimeServicesForProjectWorkspace: vi.fn(),
|
||||
@@ -129,7 +134,7 @@ function buildExecutionWorkspace(overrides: Record<string, unknown> = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
async function createApp() {
|
||||
async function createApp(actorType: "agent" | "board" = "agent") {
|
||||
const [{ executionWorkspaceRoutes }, { errorHandler }] = await Promise.all([
|
||||
import("../routes/execution-workspaces.js"),
|
||||
import("../middleware/index.js"),
|
||||
@@ -137,7 +142,9 @@ async function createApp() {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
(req as any).actor = {
|
||||
(req as any).actor = actorType === "board" ? {
|
||||
type: "board", userId: "board-1", companyIds: ["company-1"], source: "local_implicit", isInstanceAdmin: true,
|
||||
} : {
|
||||
type: "agent",
|
||||
agentId: "agent-1",
|
||||
companyId: "company-1",
|
||||
@@ -246,9 +253,15 @@ function mockVerifiedReseed(
|
||||
* stopped and retryable instead of "desired running" with residue.
|
||||
*/
|
||||
describe("execution workspace runtime control conflict and failure reconciliation", () => {
|
||||
beforeAll(async () => {
|
||||
// Route-module transformation is fixture setup, outside the HTTP control
|
||||
// operation whose latency each test measures.
|
||||
await Promise.all([import("../routes/execution-workspaces.js"), import("../middleware/index.js")]);
|
||||
}, 30_000);
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockSpawn.mockReset();
|
||||
mockLifecycleLock.mockImplementation(async (_db, _id, run) => run({}));
|
||||
registeredProjectWorkspace = null;
|
||||
mockAccessService.decide.mockResolvedValue({
|
||||
allowed: true,
|
||||
@@ -281,6 +294,8 @@ describe("execution workspace runtime control conflict and failure reconciliatio
|
||||
// Mirror the real recorder: a throwing `run` becomes a terminal failed operation and
|
||||
// the error still propagates to the caller.
|
||||
try {
|
||||
if (input.admit) await input.admit(async () => { mockPublishRuntimeOperation(); });
|
||||
else mockPublishRuntimeOperation();
|
||||
await input.run(async () => undefined);
|
||||
} catch (error) {
|
||||
(error as any).recordedOperationStatus = "failed";
|
||||
@@ -291,6 +306,35 @@ describe("execution workspace runtime control conflict and failure reconciliatio
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ route: "runtime-commands", action: "run", source: "empty" },
|
||||
{ route: "runtime-commands", action: "run", source: "git" },
|
||||
{ route: "runtime-commands", action: "repair", source: "empty" },
|
||||
{ route: "runtime-services", action: "start", source: "git" },
|
||||
] as const)("rejects remote $source $route/$action before controller file or job operations", async ({ route, action, source }) => {
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({
|
||||
cwd: "/workspaces/remote-only", providerType: "cloud_sandbox", providerRef: "/workspaces/remote-only",
|
||||
metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1,
|
||||
snapshotId: null, source: source === "empty" ? { kind: "empty" }
|
||||
: { kind: "git", repoUrl: "https://github.com/example/work.git", ref: "main" },
|
||||
location: null, writerRunId: null, dirty: false } },
|
||||
config: { workspaceRuntime: { jobs: [{ name: "verify", command: "node verify.js" }],
|
||||
services: [{ name: "app", command: "node app.js" }] } },
|
||||
}));
|
||||
const response = await request(await createApp())
|
||||
.post(`/api/execution-workspaces/${executionWorkspaceId}/${route}/${action}`)
|
||||
.send(action === "run" ? { workspaceCommandId: "job:verify" } : {});
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(422);
|
||||
expect(response.body).toMatchObject({ code: "remote_workspace_runtime_commands_unsupported" });
|
||||
expect(mockEnsurePersistedExecutionWorkspaceAvailable).not.toHaveBeenCalled();
|
||||
expect(mockWorkspaceOperationService.assertRuntimeControlAvailable).not.toHaveBeenCalled();
|
||||
expect(mockWorkspaceOperationService.createRecorder).not.toHaveBeenCalled();
|
||||
expect(mockRunWorkspaceJobForControl).not.toHaveBeenCalled();
|
||||
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
|
||||
expect(mockSpawn).not.toHaveBeenCalled();
|
||||
expect(mockClaimRuntimeLease).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns a stable 409 while a managed control is genuinely live", async () => {
|
||||
const { conflict } = await import("../errors.js");
|
||||
mockWorkspaceOperationService.assertRuntimeControlAvailable.mockRejectedValue(
|
||||
@@ -320,6 +364,29 @@ describe("execution workspace runtime control conflict and failure reconciliatio
|
||||
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["migrating", "remote", "path_changed"] as const)("revalidates a board command at operation publication after the workspace becomes %s", async state => {
|
||||
mockAssertCanManageExecutionWorkspaceRuntimeServices.mockResolvedValue({ actorType: "board" });
|
||||
const stale = buildExecutionWorkspace();
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValueOnce(stale).mockResolvedValue(buildExecutionWorkspace(state === "migrating"
|
||||
? { metadata: { remoteWorkspaceMigration: { schema: "paperclip.remote-workspace-migration.v1", state: "running" } } }
|
||||
: state === "remote" ? { metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1,
|
||||
snapshotId: null, source: { kind: "empty" }, location: null, writerRunId: null, dirty: false } } }
|
||||
: { cwd: "/tmp/changed-authority" }));
|
||||
const response = await request(await createApp("board"))
|
||||
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-services/start`).send({});
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(state === "remote" ? 422 : 409);
|
||||
expect(response.body.code).toBe(state === "remote" ? "remote_workspace_runtime_commands_unsupported"
|
||||
: state === "migrating" ? "workspace_runtime_migration_in_progress" : "workspace_runtime_authority_changed");
|
||||
expect(mockLifecycleLock).toHaveBeenCalled();
|
||||
expect(mockPublishRuntimeOperation).not.toHaveBeenCalled();
|
||||
expect(mockClaimRuntimeLease).not.toHaveBeenCalled();
|
||||
expect(mockEnsurePersistedExecutionWorkspaceAvailable).not.toHaveBeenCalled();
|
||||
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
|
||||
expect(mockStopRuntimeServicesForExecutionWorkspace).not.toHaveBeenCalled();
|
||||
expect(mockExecutionWorkspaceService.update).not.toHaveBeenCalled();
|
||||
expect(mockSpawn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("checks authorization before recovering or claiming the workspace", async () => {
|
||||
const { forbidden } = await import("../errors.js");
|
||||
mockAssertCanManageExecutionWorkspaceRuntimeServices.mockRejectedValue(
|
||||
@@ -655,6 +722,7 @@ describe("execution workspace runtime control conflict and failure reconciliatio
|
||||
mockWorkspaceOperationService.createRecorder.mockReturnValue({
|
||||
attachExecutionWorkspaceId: vi.fn(),
|
||||
recordOperation: async (input: any) => {
|
||||
await input.admit(async () => { mockPublishRuntimeOperation(); });
|
||||
await input.run(async (entry: Record<string, unknown>) => {
|
||||
progress.push(entry);
|
||||
});
|
||||
@@ -682,7 +750,8 @@ describe("execution workspace runtime control conflict and failure reconciliatio
|
||||
// The recorder's ceiling fires outside `run`, so only the outer handler can reconcile.
|
||||
mockWorkspaceOperationService.createRecorder.mockReturnValue({
|
||||
attachExecutionWorkspaceId: vi.fn(),
|
||||
recordOperation: async () => {
|
||||
recordOperation: async (input: any) => {
|
||||
await input.admit(async () => { mockPublishRuntimeOperation(); });
|
||||
throw new WorkspaceOperationTimeoutError(1_000, "start");
|
||||
},
|
||||
});
|
||||
|
||||
@@ -22,6 +22,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({
|
||||
}));
|
||||
const mockAccessService = vi.hoisted(() => ({ decide: vi.fn(async () => ({ allowed: true })) }));
|
||||
const mockRecordOperation = vi.hoisted(() => vi.fn());
|
||||
const mockPublishOperation = vi.hoisted(() => vi.fn());
|
||||
const mockReconcileStaleRuntimeControlOperations = vi.hoisted(() => vi.fn(async () => ({ reconciled: 0, operationIds: [] })));
|
||||
// The control path recovers stranded operations and then refuses only a genuinely live one
|
||||
// (PAP-17249). This suite is about the durable lease, so availability always resolves.
|
||||
@@ -112,7 +113,8 @@ describeEmbeddedPostgres("execution workspace runtime control lease enforcement"
|
||||
runId: null,
|
||||
issueId: req.get("x-test-owner-issue") ?? null,
|
||||
}));
|
||||
mockRecordOperation.mockImplementation(async (input: { run: () => Promise<unknown> }) => {
|
||||
mockRecordOperation.mockImplementation(async (input: { run: () => Promise<unknown>; admit: (publish: () => Promise<void>) => Promise<void> }) => {
|
||||
await input.admit(async () => { mockPublishOperation(); });
|
||||
await input.run();
|
||||
return { id: randomUUID(), status: "succeeded" };
|
||||
});
|
||||
@@ -237,6 +239,7 @@ describeEmbeddedPostgres("execution workspace runtime control lease enforcement"
|
||||
expect(mockStartRuntimeServices).toHaveBeenCalledTimes(2);
|
||||
|
||||
mockRecordOperation.mockClear();
|
||||
mockPublishOperation.mockClear();
|
||||
mockStartRuntimeServices.mockClear();
|
||||
mockStopRuntimeServices.mockClear();
|
||||
mockReconcileStaleRuntimeControlOperations.mockClear();
|
||||
@@ -255,7 +258,7 @@ describeEmbeddedPostgres("execution workspace runtime control lease enforcement"
|
||||
expect(typeof blocked.body.remediation).toBe("string");
|
||||
|
||||
// No workspace operation, no runtime-service mutation, no reconciliation side effect.
|
||||
expect(mockRecordOperation).not.toHaveBeenCalled();
|
||||
expect(mockPublishOperation).not.toHaveBeenCalled();
|
||||
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
|
||||
expect(mockStopRuntimeServices).not.toHaveBeenCalled();
|
||||
expect(mockReconcileStaleRuntimeControlOperations).not.toHaveBeenCalled();
|
||||
|
||||
@@ -29,7 +29,8 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabase = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
|
||||
const embeddedPostgresSupport = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
const execFileAsync = promisify(execFile);
|
||||
let seedGraphSequence = 0;
|
||||
@@ -205,11 +206,15 @@ describeEmbeddedPostgres("workspace file resources", () => {
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce");
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resources-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (externalDatabase) db = createDb(externalDatabase);
|
||||
else {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resources-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await tempDb?.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
@@ -1952,11 +1957,15 @@ describeEmbeddedPostgres("file resource route guards", () => {
|
||||
let db: Db;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resource-guards-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (externalDatabase) db = createDb(externalDatabase);
|
||||
else {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-file-resource-guards-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,12 +1,45 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { InstanceExperimentalSettings } from "@paperclipai/shared";
|
||||
import { INSTANCE_FEATURE_CATALOG, instanceExperimentalSettingsSchema, type InstanceExperimentalSettings } from "@paperclipai/shared";
|
||||
import {
|
||||
applyExperimentalSettingsPatch,
|
||||
applyCloudCatalogDefaults,
|
||||
applyManagedExperimentalOverlay,
|
||||
normalizeExperimentalSettings,
|
||||
resolveWorktreeRunExecutionActivationState,
|
||||
} from "../services/instance-settings.js";
|
||||
import type { ManagedInstanceConfig } from "../services/managed-config.js";
|
||||
|
||||
describe("instance settings service", () => {
|
||||
it("defaults remote workspace enrollment on while preserving explicit operator choices", () => {
|
||||
expect(instanceExperimentalSettingsSchema.parse({}).remoteWorkspaceStorageEnabled).toBe(true);
|
||||
expect(INSTANCE_FEATURE_CATALOG.remoteWorkspaceStorageEnabled).toMatchObject({ cloudDefault: true, selfHostedDefault: true });
|
||||
for (const stored of [undefined, null, {}, { enableApps: true }, { remoteWorkspaceStorageEnabled: "invalid" }]) {
|
||||
expect(normalizeExperimentalSettings(stored).remoteWorkspaceStorageEnabled).toBe(true);
|
||||
}
|
||||
const disabled = applyExperimentalSettingsPatch({}, { remoteWorkspaceStorageEnabled: false });
|
||||
expect(normalizeExperimentalSettings(JSON.parse(JSON.stringify(disabled))).remoteWorkspaceStorageEnabled).toBe(false);
|
||||
expect(normalizeExperimentalSettings({ remoteWorkspaceStorageEnabled: false, enableCases: "invalid" }).remoteWorkspaceStorageEnabled).toBe(false);
|
||||
expect(applyExperimentalSettingsPatch(disabled, { enableStatusCards: true }).remoteWorkspaceStorageEnabled).toBe(false);
|
||||
expect(applyExperimentalSettingsPatch(disabled, { remoteWorkspaceStorageEnabled: true }).remoteWorkspaceStorageEnabled).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ stored: {}, managed: undefined, expected: true },
|
||||
{ stored: { remoteWorkspaceStorageEnabled: false }, managed: undefined, expected: false },
|
||||
{ stored: { remoteWorkspaceStorageEnabled: false }, managed: true, expected: true },
|
||||
{ stored: { remoteWorkspaceStorageEnabled: true }, managed: false, expected: false },
|
||||
])("respects Cloud enrollment defaults and managed choices: $expected", ({ stored, managed, expected }) => {
|
||||
const config: ManagedInstanceConfig = {
|
||||
v: 1, mode: "cloud", catalogVersion: "test", plugins: { autoInstall: [] }, environments: [],
|
||||
features: managed === undefined ? {} : { remoteWorkspaceStorageEnabled: managed },
|
||||
};
|
||||
const normalized = normalizeExperimentalSettings(stored);
|
||||
const result = applyManagedExperimentalOverlay(applyCloudCatalogDefaults(normalized, stored, config), config);
|
||||
expect(result.experimental.remoteWorkspaceStorageEnabled).toBe(expected);
|
||||
expect(normalized.remoteWorkspaceStorageEnabled).toBe("remoteWorkspaceStorageEnabled" in stored ? stored.remoteWorkspaceStorageEnabled : true);
|
||||
expect(result.managedKeys.remoteWorkspaceStorageEnabled).toEqual(managed === undefined
|
||||
? undefined : { managed: true, managedBy: "paperclip-cloud" });
|
||||
});
|
||||
it("keeps Notebook off for legacy or invalid settings and round-trips explicit operator choices", () => {
|
||||
for (const stored of [undefined, null, {}, { enableApps: true },
|
||||
{ enablePaperclipNotebook: "true" }, { enablePaperclipNotebook: 1 },
|
||||
@@ -62,6 +95,7 @@ describe("instance settings service", () => {
|
||||
})).toEqual({
|
||||
enableEnvironments: true,
|
||||
enableNativeRunner: false,
|
||||
remoteWorkspaceStorageEnabled: true,
|
||||
enableAiConnectionRouters: false,
|
||||
enableManagedSandboxOnly: false,
|
||||
enableIsolatedWorkspaces: true,
|
||||
|
||||
@@ -0,0 +1,707 @@
|
||||
import express, { type Request } from "express";
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import request from "supertest";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, companySecrets, companySecretBindings, createDb, environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, plugins, projects, projectWorkspaces, secretAccessEvents, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { INSPECTION_AUDIENCE, INSPECTION_HEADER, INSPECTION_PERMIT_TYPE, parseInspectionQuery, type InspectionQuery, type InspectionPermit, type RemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { notFound, unprocessable } from "../errors.js";
|
||||
import { fileResourceRoutes } from "../routes/file-resources.js";
|
||||
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { createRemoteWorkspaceFiles, REMOTE_WORKSPACE_FILES_SCRIPT } from "../services/remote-workspace-files.js";
|
||||
import { workspaceFileResourceService } from "../services/workspace-file-resources.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { customerSuccessRoutes } from "../routes/customer-success.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { secretService } from "../services/secrets.js";
|
||||
import { captureRemoteWorkspaceDirectory } from "../services/remote-workspace-snapshot-archive.js";
|
||||
import { beginWorkspaceSnapshot, uploadWorkspaceSnapshot } from "../services/remote-workspace-snapshot-storage.js";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import { createStorageService } from "../storage/service.js";
|
||||
import { createSavedWorkspaceLoader } from "../services/remote-workspaces.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const decryptCleanupFault = vi.hoisted(() => ({ error: undefined as Error | undefined, path: undefined as string | undefined }));
|
||||
vi.mock("node:fs/promises", async importOriginal => {
|
||||
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
||||
return { ...actual, async rm(target: Parameters<typeof actual.rm>[0], options: Parameters<typeof actual.rm>[1]) {
|
||||
if (decryptCleanupFault.error && String(target).includes("pc-workspace-decrypt-")) {
|
||||
const error = decryptCleanupFault.error; decryptCleanupFault.error = undefined; decryptCleanupFault.path = String(target); throw error;
|
||||
}
|
||||
return actual.rm(target, options);
|
||||
} };
|
||||
});
|
||||
const externalDatabase = process.env.PAPERCLIP_TEST_DATABASE_URL?.trim();
|
||||
const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeDatabase = support.supported ? describe : describe.skip;
|
||||
|
||||
describe("confined workspace file helper", () => {
|
||||
let root: string;
|
||||
beforeAll(async () => {
|
||||
root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "workspace-helper-")));
|
||||
await fs.mkdir(path.join(root, "folder"));
|
||||
await fs.writeFile(path.join(root, "folder", "report.md"), "Saved result\n");
|
||||
await fs.writeFile(path.join(root, ".env"), "hidden");
|
||||
await fs.symlink(os.tmpdir(), path.join(root, "escape"));
|
||||
await fs.writeFile(path.join(root, "binary.bin"), Buffer.from([0, 1, 2]));
|
||||
});
|
||||
afterAll(async () => { await fs.rm(root, { recursive: true, force: true }); });
|
||||
|
||||
async function helper(action: string, args: Record<string, unknown>) {
|
||||
const { stdout } = await execFileAsync(process.execPath, ["--input-type=commonjs", "-e", REMOTE_WORKSPACE_FILES_SCRIPT,
|
||||
JSON.stringify({ root, action, ...args })], { maxBuffer: 2 * 1024 * 1024 });
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
it("lists ordinary children without exposing secrets or symlinks", async () => {
|
||||
const result = await helper("list", { options: { mode: "all", limit: 25, offset: 0, maxScannedEntries: 5000, maxDepth: 20 } });
|
||||
expect(result.entries.map((entry: { relativePath: string }) => entry.relativePath)).toEqual(["folder", "binary.bin"]);
|
||||
expect(result.truncated).toBe(false);
|
||||
const report = await helper("stat", { relativePath: "folder/report.md", withHash: true });
|
||||
expect(report).toMatchObject({ kind: "file", byteSize: 13, text: true });
|
||||
expect(report.sha256).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(await helper("stat", { relativePath: "binary.bin" })).toMatchObject({ text: false });
|
||||
});
|
||||
|
||||
it.each(["../outside", "/etc/passwd", "escape/file", ".env", "folder/../.env", "file:///etc/passwd", "folder\\report.md"])(
|
||||
"rejects a confined or policy-denied path %s", async relativePath => {
|
||||
await expect(helper("stat", { relativePath })).rejects.toThrow();
|
||||
},
|
||||
);
|
||||
|
||||
it("uses bounded recursive search and pagination", async () => {
|
||||
const result = await helper("list", { options: { mode: "all", query: "report", limit: 1, offset: 0, maxScannedEntries: 5000, maxDepth: 20 } });
|
||||
expect(result.entries.map((entry: { relativePath: string }) => entry.relativePath)).toEqual(["folder/report.md"]);
|
||||
const bounded = await helper("list", { options: { mode: "recent", limit: 25, offset: 0, maxScannedEntries: 1, maxDepth: 20 } });
|
||||
expect(bounded.scannedCount).toBe(1);
|
||||
expect(bounded.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("gets changed files from the workspace repository without revealing Git metadata or secrets", async () => {
|
||||
await execFileAsync("git", ["-C", root, "init", "--initial-branch=main"]);
|
||||
await execFileAsync("git", ["-C", root, "add", "folder/report.md"]);
|
||||
await execFileAsync("git", ["-C", root, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "commit.gpgsign=false", "commit", "-m", "initial"]);
|
||||
await fs.writeFile(path.join(root, "folder", "report.md"), "Changed useful result\n");
|
||||
const result = await helper("list", { options: { mode: "changed", limit: 25, offset: 0, maxScannedEntries: 5000, maxDepth: 20 } });
|
||||
expect(result.entries.map((entry: { relativePath: string }) => entry.relativePath)).toEqual(["binary.bin", "folder/report.md"]);
|
||||
});
|
||||
});
|
||||
|
||||
describeDatabase("remote workspace ordinary file routes", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: Db;
|
||||
const directories: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalDatabase) db = createDb(externalDatabase);
|
||||
else {
|
||||
temporary = await startEmbeddedPostgresTestDatabase("remote-workspace-files-");
|
||||
db = createDb(temporary.connectionString);
|
||||
}
|
||||
}, 60_000);
|
||||
afterAll(async () => {
|
||||
await temporary?.cleanup();
|
||||
await db?.$client.end();
|
||||
await Promise.all(directories.map(directory => fs.rm(directory, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "remote-file-test-")));
|
||||
directories.push(root);
|
||||
await fs.mkdir(path.join(root, "docs"));
|
||||
await fs.writeFile(path.join(root, "docs", "result.md"), "Remote useful result\n");
|
||||
await fs.writeFile(path.join(root, ".env"), "private-key");
|
||||
const ids = { company: randomUUID(), workspace: randomUUID(), issue: randomUUID(), environment: randomUUID(), lease: randomUUID() };
|
||||
const authority: RemoteWorkspaceAuthority = {
|
||||
schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, writerRunId: null, dirty: false,
|
||||
source: { kind: "empty" }, location: { environmentId: ids.environment, leaseId: ids.lease,
|
||||
providerLeaseId: `allocation-${ids.lease}`, remoteCwd: root },
|
||||
};
|
||||
await db.insert(companies).values({ id: ids.company, name: "File access", issuePrefix: `RF${ids.company.slice(0, 6)}` });
|
||||
await db.insert(executionWorkspaces).values({ id: ids.workspace, companyId: ids.company, mode: "isolated_workspace",
|
||||
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Remote task", metadata: { remoteWorkspace: authority } });
|
||||
await db.insert(issues).values({ id: ids.issue, companyId: ids.company, title: "Read remote result", executionWorkspaceId: ids.workspace });
|
||||
await db.update(executionWorkspaces).set({ sourceIssueId: ids.issue }).where(eq(executionWorkspaces.id, ids.workspace));
|
||||
await db.insert(environments).values({ id: ids.environment, name: `Remote file ${ids.environment}`, driver: "sandbox" });
|
||||
await db.insert(environmentLeases).values({ id: ids.lease, companyId: ids.company, environmentId: ids.environment,
|
||||
executionWorkspaceId: ids.workspace, issueId: ids.issue, provider: "fake", providerLeaseId: authority.location!.providerLeaseId,
|
||||
metadata: { remoteCwd: root } });
|
||||
let beforeCopy: (() => Promise<void>) | undefined;
|
||||
const execute = vi.fn(async (input: Parameters<EnvironmentRuntimeService["execute"]>[0]) => {
|
||||
expect(input.lease.id).toBe(ids.lease);
|
||||
expect(input.bypassSession).toBe(true);
|
||||
try {
|
||||
const output = await execFileAsync(process.execPath, input.args!, { cwd: input.cwd, maxBuffer: 2 * 1024 * 1024 });
|
||||
return { ...output, exitCode: 0, timedOut: false };
|
||||
} catch (error) {
|
||||
const failure = error as { stdout: string; stderr: string };
|
||||
return { stdout: failure.stdout, stderr: failure.stderr, exitCode: 1, timedOut: false };
|
||||
}
|
||||
});
|
||||
const syncOut = vi.fn(async (input: Parameters<EnvironmentRuntimeService["syncOut"]>[0]) => {
|
||||
await beforeCopy?.();
|
||||
for (const operation of input.operations) for (const mapping of operation.files) await fs.copyFile(mapping.sourcePath, mapping.targetPath);
|
||||
return { operations: input.operations.map(operation => ({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred: 0 })) };
|
||||
});
|
||||
const runtime = { execute, syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: runtime });
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = { type: "board", userId: "local-board", companyIds: [ids.company], source: "local_implicit", isInstanceAdmin: true } as Request["actor"];
|
||||
next();
|
||||
});
|
||||
app.use("/api", fileResourceRoutes(db, { remoteFiles }));
|
||||
app.use(errorHandler);
|
||||
return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise<void>) => { beforeCopy = callback; } };
|
||||
}
|
||||
|
||||
it("lists, resolves, previews, and downloads live bytes without a controller checkout", async () => {
|
||||
const f = await fixture();
|
||||
const base = `/api/issues/${f.ids.issue}/file-resources`;
|
||||
const listed = await request(f.app).get(`${base}/list`).query({ workspace: "execution" }).expect(200);
|
||||
expect(listed.body).toMatchObject({ state: "available", workspace: { source: { kind: "live" } } });
|
||||
expect(listed.body.items.map((item: { relativePath: string }) => item.relativePath)).toEqual(["docs"]);
|
||||
const resolved = await request(f.app).get(`${base}/resolve`).query({ path: "docs/result.md", workspace: "execution" }).expect(200);
|
||||
expect(resolved.body).toMatchObject({ kind: "file", source: { kind: "live" }, capabilities: { preview: true, download: true } });
|
||||
const preview = await request(f.app).get(`${base}/content`).query({ path: "docs/result.md", workspace: "execution" }).expect(200);
|
||||
expect(preview.body.content).toEqual({ encoding: "utf8", data: "Remote useful result\n" });
|
||||
const download = await request(f.app).get(`${base}/content`).query({ path: "docs/result.md", workspace: "execution", download: true }).expect(200);
|
||||
expect(download.text).toBe("Remote useful result\n");
|
||||
expect(f.syncOut).toHaveBeenCalledTimes(2);
|
||||
expect(await fs.readdir(path.dirname(f.syncOut.mock.calls[0]![0].operations[0]!.files[0]!.targetPath)).catch(() => [])).toEqual([]);
|
||||
});
|
||||
|
||||
it("serves signed support reads from secret-bound live and saved workspaces inside its read-only transaction", async () => {
|
||||
const f = await fixture();
|
||||
const other = await fixture();
|
||||
const keyDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "inspection-workspace-key-"));
|
||||
directories.push(keyDirectory);
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", path.join(keyDirectory, "master.key"));
|
||||
vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_ENABLED", "true");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "remote-inspection-fixture");
|
||||
vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_CLOUD_ORIGIN", "https://cloud.example.test");
|
||||
const keys = generateKeyPairSync("ed25519");
|
||||
vi.stubEnv("PAPERCLIP_CUSTOMER_SUCCESS_INSPECTION_JWKS", JSON.stringify({ keys: [{ ...keys.publicKey.export({ format: "jwk" }), kid: "cloud" }] }));
|
||||
const consumed = vi.fn(async () => Response.json({ consumed: true }));
|
||||
vi.stubGlobal("fetch", consumed);
|
||||
try {
|
||||
const pluginId = randomUUID();
|
||||
const provider = `inspection-${pluginId}`;
|
||||
const pluginKey = `paperclip.${provider}`;
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey, packageName: `@paperclipai/${provider}`, version: "1.0.0", apiVersion: 1,
|
||||
categories: ["automation"], status: "ready", manifestJson: {
|
||||
id: pluginKey, apiVersion: 1, version: "1.0.0", displayName: "Inspection fixture", description: "Synthetic inspection provider", author: "Paperclip",
|
||||
categories: ["automation"], capabilities: ["environment.drivers.register"], entrypoints: { worker: "dist/worker.js" },
|
||||
environmentDrivers: [{ driverKey: provider, kind: "sandbox_provider", displayName: "Inspection fixture",
|
||||
configSchema: { type: "object", properties: { apiKey: { type: "string", format: "secret-ref" } } } }],
|
||||
} });
|
||||
const secret = await secretService(db).create(f.ids.company, { name: "synthetic inspection provider", provider: "local_encrypted", value: "synthetic-provider-credential" });
|
||||
await secretService(db).createBinding({ companyId: f.ids.company, secretId: secret.id,
|
||||
targetType: "environment", targetId: f.ids.environment, configPath: "apiKey" });
|
||||
await db.update(environments).set({ config: { provider, apiKey: secret.id } }).where(eq(environments.id, f.ids.environment));
|
||||
await db.update(environmentLeases).set({ provider, metadata: { remoteCwd: f.root, provider, apiKey: secret.id,
|
||||
pluginId, sandboxProviderPlugin: true } }).where(eq(environmentLeases.id, f.ids.lease));
|
||||
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(path.join(keyDirectory, "objects")));
|
||||
const archivePath = path.join(keyDirectory, "saved.pcws");
|
||||
const archive = await captureRemoteWorkspaceDirectory({ cwd: f.root, archivePath });
|
||||
const [operation] = await db.insert(workspaceOperations).values({ companyId: f.ids.company,
|
||||
executionWorkspaceId: f.ids.workspace, phase: "synthetic_saved_fixture" }).returning();
|
||||
const snapshot = await beginWorkspaceSnapshot(db, { companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
operationId: operation.id, generation: 1, epoch: 1 });
|
||||
await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
|
||||
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
|
||||
await fs.writeFile(path.join(f.root, "docs", "result.md"), "Updated live result\n");
|
||||
await fs.symlink(other.root, path.join(f.root, "escape"));
|
||||
|
||||
const call = vi.fn(async (_pluginId: string, method: string, params: any) => {
|
||||
expect(_pluginId).toBe(pluginId);
|
||||
expect(params.companyId).toBe(f.ids.company);
|
||||
expect(params.lease.providerLeaseId).toBe(f.authority.location!.providerLeaseId);
|
||||
expect(params.config.apiKey).toBe("synthetic-provider-credential");
|
||||
if (method === "environmentExecute") {
|
||||
expect(params.bypassSession).toBe(true);
|
||||
try {
|
||||
const result = await execFileAsync(process.execPath, params.args, { cwd: params.cwd, maxBuffer: 2 * 1024 * 1024 });
|
||||
return { ...result, exitCode: 0, timedOut: false, signal: null };
|
||||
} catch (error) {
|
||||
const result = error as { stdout: string; stderr: string };
|
||||
return { stdout: result.stdout, stderr: result.stderr, exitCode: 1, timedOut: false, signal: null };
|
||||
}
|
||||
}
|
||||
if (method === "environmentSyncOut") {
|
||||
for (const operation of params.operations) for (const mapping of operation.files) await fs.copyFile(mapping.sourcePath, mapping.targetPath);
|
||||
return { operations: [] };
|
||||
}
|
||||
throw new Error(`Inspection must not allocate or resume: ${method}`);
|
||||
});
|
||||
const workerManager = { call, isRunning: (id: string) => id === pluginId } as unknown as PluginWorkerManager;
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use("/api/customer-success/v1", customerSuccessRoutes(db, storage, { pluginWorkerManager: workerManager }));
|
||||
app.use(errorHandler);
|
||||
const inspect = (value: InspectionQuery) => {
|
||||
const query = parseInspectionQuery(value);
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const claims: InspectionPermit = { v: 1, iss: "paperclip-cloud", aud: INSPECTION_AUDIENCE, sub: "remote-inspection-fixture",
|
||||
iat: now, exp: now + 60, jti: randomUUID(), bindingVersion: 1, grantId: randomUUID(), requestId: randomUUID(),
|
||||
agentId: randomUUID(), keyId: "sha256:synthetic-fixture", runId: randomUUID(), query };
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", typ: INSPECTION_PERMIT_TYPE, kid: "cloud" })).toString("base64url");
|
||||
const body = Buffer.from(JSON.stringify(claims)).toString("base64url");
|
||||
const payload = `${header}.${body}`;
|
||||
return request(app).post("/api/customer-success/v1/read").set(INSPECTION_HEADER,
|
||||
`${payload}.${sign(null, Buffer.from(payload), keys.privateKey).toString("base64url")}`).send(query);
|
||||
};
|
||||
const base = { companyId: f.ids.company, resourceId: f.ids.issue, context: { workspace: "execution" as const } };
|
||||
const beforeSecret = await db.select().from(companySecrets).where(eq(companySecrets.id, secret.id));
|
||||
const beforeAudit = await db.select().from(secretAccessEvents).where(eq(secretAccessEvents.companyId, f.ids.company));
|
||||
const transaction = db.transaction.bind(db);
|
||||
const transactions: string[] = [];
|
||||
const transactionSpy = vi.spyOn(db, "transaction").mockImplementation((async (callback, options) => transaction(async tx => {
|
||||
const [row] = await tx.execute(sql`show transaction_read_only`);
|
||||
transactions.push(String(row.transaction_read_only));
|
||||
return callback(tx);
|
||||
}, options)) as typeof db.transaction);
|
||||
// A reader accidentally closed over the outer Db fails here; nested
|
||||
// provider/config/secret reads must use the inspection transaction.
|
||||
const selectSpy = vi.spyOn(db, "select").mockImplementation(() => { throw new Error("Inspection escaped its transaction"); });
|
||||
try {
|
||||
const listed = await inspect({ ...base, operation: "files.list" }).expect(200);
|
||||
expect(listed.body).toMatchObject({ state: "available", workspace: { source: { kind: "live" } } });
|
||||
expect(JSON.stringify(listed.body)).not.toContain(".env");
|
||||
const read = await inspect({ ...base, operation: "files.read", path: "docs/result.md" }).expect(200);
|
||||
expect(read.body.content.data).toBe("Updated live result\n");
|
||||
const downloaded = await inspect({ ...base, operation: "files.download", path: "docs/result.md", range: { start: 2, end: 7 } }).expect(200);
|
||||
expect(Buffer.from(downloaded.body.content, "base64").toString()).toBe("dated ");
|
||||
expect(JSON.stringify([listed.body, read.body, downloaded.body])).not.toContain("synthetic-provider-credential");
|
||||
const beforeDenied = call.mock.calls.length;
|
||||
await inspect({ ...base, companyId: other.ids.company, operation: "files.read", path: "docs/result.md" }).expect(404);
|
||||
await inspect({ ...base, operation: "files.read", path: ".env" }).expect(403);
|
||||
await inspect({ ...base, context: { workspace: "execution", workspaceId: other.ids.workspace }, operation: "files.read", path: "docs/result.md" }).expect(422);
|
||||
expect(call).toHaveBeenCalledTimes(beforeDenied);
|
||||
await inspect({ ...base, operation: "files.read", path: "escape/docs/result.md" }).expect(403);
|
||||
} finally { selectSpy.mockRestore(); transactionSpy.mockRestore(); }
|
||||
expect(transactions.length).toBeGreaterThan(0);
|
||||
expect(new Set(transactions)).toEqual(new Set(["on"]));
|
||||
expect(await db.select().from(companySecrets).where(eq(companySecrets.id, secret.id))).toEqual(beforeSecret);
|
||||
expect(await db.select().from(secretAccessEvents).where(eq(secretAccessEvents.companyId, f.ids.company))).toEqual(beforeAudit);
|
||||
|
||||
await db.delete(companySecretBindings).where(eq(companySecretBindings.secretId, secret.id));
|
||||
const beforeUnbound = call.mock.calls.length;
|
||||
await inspect({ ...base, operation: "files.read", path: "docs/result.md" }).expect(422);
|
||||
expect(call).toHaveBeenCalledTimes(beforeUnbound);
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
call.mockClear();
|
||||
const savedList = await inspect({ ...base, operation: "files.list" }).expect(200);
|
||||
expect(savedList.body).toMatchObject({ workspace: { source: { kind: "saved", snapshotId: snapshot.id } } });
|
||||
const savedRead = await inspect({ ...base, operation: "files.read", path: "docs/result.md" }).expect(200);
|
||||
expect(savedRead.body.content.data).toBe("Remote useful result\n");
|
||||
const savedDownload = await inspect({ ...base, operation: "files.download", path: "docs/result.md" }).expect(200);
|
||||
expect(Buffer.from(savedDownload.body.content, "base64").toString()).toBe("Remote useful result\n");
|
||||
expect(call).not.toHaveBeenCalled();
|
||||
expect(consumed).toHaveBeenCalled();
|
||||
} finally { vi.unstubAllEnvs(); vi.unstubAllGlobals(); }
|
||||
});
|
||||
|
||||
it("rejects denied files before provider access and prevents changed transfer bytes", async () => {
|
||||
const f = await fixture();
|
||||
await request(f.app).get(`/api/issues/${f.ids.issue}/file-resources/content`).query({ workspace: "execution", path: ".env" }).expect(403);
|
||||
expect(f.execute).not.toHaveBeenCalled();
|
||||
f.beforeCopy(async () => { await fs.writeFile(path.join(f.root, "docs", "result.md"), "Another changed text\n"); });
|
||||
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: f.remoteFiles }).readContent(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422, details: { code: "file_changed" } });
|
||||
});
|
||||
|
||||
async function savedFileFixture() {
|
||||
const f = await fixture(), storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "saved-file-budget-"));
|
||||
directories.push(storageRoot);
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
|
||||
const archivePath = path.join(storageRoot, "saved.pcws"), archive = await captureRemoteWorkspaceDirectory({ cwd: f.root, archivePath });
|
||||
const [operation] = await db.insert(workspaceOperations).values({ companyId: f.ids.company,
|
||||
executionWorkspaceId: f.ids.workspace, phase: "synthetic_file_budget" }).returning();
|
||||
const snapshot = await beginWorkspaceSnapshot(db, { companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
operationId: operation!.id, generation: 1, epoch: 1 });
|
||||
const saved = await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
|
||||
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
return { ...f, storage, saved, input: { companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id } };
|
||||
}
|
||||
|
||||
it("shares fail-fast saved Files admission across loader instances until owned cleanup completes", async () => {
|
||||
const f = await savedFileFixture(), availableBytes = async () => 8n * 1024n ** 3n;
|
||||
const first = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
|
||||
const second = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
|
||||
const third = createSavedWorkspaceLoader(db, f.storage, { availableBytes });
|
||||
const opened: Array<{ root: string; cleanup(): Promise<void> }> = [];
|
||||
const getObject = vi.spyOn(f.storage, "getObject");
|
||||
try {
|
||||
opened.push(await first(f.input)); opened.push(await second(f.input));
|
||||
expect(opened[0]!.root).not.toBe(opened[1]!.root);
|
||||
getObject.mockClear();
|
||||
await expect(third(f.input)).rejects.toMatchObject({ status: 422, details: { code: "workspace_unavailable" } });
|
||||
expect(getObject).not.toHaveBeenCalled();
|
||||
await opened[0]!.cleanup(); await opened[0]!.cleanup();
|
||||
await expect(fs.stat(opened[0]!.root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
opened.push(await third(f.input));
|
||||
expect(getObject).toHaveBeenCalledTimes(1);
|
||||
} finally { await Promise.all(opened.map(access => access.cleanup())); getObject.mockRestore(); }
|
||||
});
|
||||
|
||||
it("bounds reserved scratch bytes across saved Files loader instances", async () => {
|
||||
const f = await savedFileFixture(), options = { availableBytes: async () => 8n * 1024n ** 3n,
|
||||
limits: { maxScratchBytes: 180 * 1024 * 1024 } };
|
||||
const first = createSavedWorkspaceLoader(db, f.storage, options), second = createSavedWorkspaceLoader(db, f.storage, options);
|
||||
const access = await first(f.input), getObject = vi.spyOn(f.storage, "getObject");
|
||||
try {
|
||||
await expect(second(f.input)).rejects.toMatchObject({ status: 422, details: { code: "workspace_unavailable" } });
|
||||
expect(getObject).not.toHaveBeenCalled();
|
||||
} finally { await access.cleanup(); getObject.mockRestore(); }
|
||||
const next = await second(f.input); await next.cleanup();
|
||||
});
|
||||
|
||||
it.each(["archive", "disk"] as const)("rejects saved Files %s bounds before object retrieval and releases admission", async bound => {
|
||||
const f = await savedFileFixture(), getObject = vi.spyOn(f.storage, "getObject");
|
||||
if (bound === "archive") await db.update(executionWorkspaceSnapshots).set({ byteSize: 512 * 1024 * 1024 + 1 })
|
||||
.where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
const load = createSavedWorkspaceLoader(db, f.storage, { availableBytes: async () => bound === "disk" ? 0n : 8n * 1024n ** 3n });
|
||||
try {
|
||||
await expect(load(f.input)).rejects.toMatchObject({ status: 422, details: { code: bound === "archive" ? "too_large" : "workspace_unavailable" } });
|
||||
expect(getObject).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await db.update(executionWorkspaceSnapshots).set({ byteSize: f.saved.byteSize }).where(eq(executionWorkspaceSnapshots.id, f.saved.id));
|
||||
getObject.mockRestore();
|
||||
}
|
||||
const next = await createSavedWorkspaceLoader(db, f.storage, { availableBytes: async () => 8n * 1024n ** 3n })(f.input);
|
||||
await next.cleanup();
|
||||
});
|
||||
|
||||
it.each(["entries", "manifest"] as const)("rejects excessive saved Files %s before extraction and cleans the reserved archive", async bound => {
|
||||
const f = await savedFileFixture(), options = { availableBytes: async () => 8n * 1024n ** 3n };
|
||||
await expect(createSavedWorkspaceLoader(db, f.storage, { ...options, limits: bound === "entries" ? { maxEntries: 1 } : { maxManifestBytes: 1 } })(f.input))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "too_large" } });
|
||||
const opened: Array<{ root: string; cleanup(): Promise<void> }> = [];
|
||||
try {
|
||||
const load = createSavedWorkspaceLoader(db, f.storage, options);
|
||||
opened.push(await load(f.input)); opened.push(await load(f.input));
|
||||
expect(await fs.readFile(path.join(opened[0]!.root, "docs", "result.md"), "utf8")).toBe("Remote useful result\n");
|
||||
} finally { await Promise.all(opened.map(access => access.cleanup())); }
|
||||
});
|
||||
|
||||
it("preserves object failures while releasing saved Files admission for the next copies", async () => {
|
||||
const f = await savedFileFixture(), original = new Error("synthetic object retrieval failed");
|
||||
const options = { availableBytes: async () => 8n * 1024n ** 3n };
|
||||
await expect(createSavedWorkspaceLoader(db, { ...f.storage, getObject: async () => { throw original; } }, options)(f.input)).rejects.toBe(original);
|
||||
const opened: Array<{ cleanup(): Promise<void> }> = [];
|
||||
try {
|
||||
const load = createSavedWorkspaceLoader(db, f.storage, options);
|
||||
opened.push(await load(f.input)); opened.push(await load(f.input));
|
||||
} finally { await Promise.all(opened.map(access => access.cleanup())); }
|
||||
});
|
||||
|
||||
it("owns failed decrypt cleanup beneath the Files reservation before releasing capacity", async () => {
|
||||
const f = await savedFileFixture(), original = new Error("synthetic decrypt cleanup failed");
|
||||
const load = createSavedWorkspaceLoader(db, f.storage, { availableBytes: async () => 8n * 1024n ** 3n });
|
||||
const opened: Array<{ cleanup(): Promise<void> }> = [];
|
||||
decryptCleanupFault.error = original; decryptCleanupFault.path = undefined;
|
||||
try {
|
||||
await expect(load(f.input)).rejects.toBe(original);
|
||||
const failed = decryptCleanupFault.path!;
|
||||
expect(path.basename(path.dirname(failed))).toMatch(/^pc-workspace-files-/);
|
||||
await expect(fs.stat(failed)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(fs.stat(path.dirname(failed))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
opened.push(await load(f.input)); opened.push(await load(f.input));
|
||||
} finally {
|
||||
decryptCleanupFault.error = undefined;
|
||||
await Promise.all(opened.map(access => access.cleanup()));
|
||||
if (decryptCleanupFault.path) await fs.rm(decryptCleanupFault.path, { recursive: true, force: true });
|
||||
decryptCleanupFault.path = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
it("reads saved content after explicit location release, with no provider call", async () => {
|
||||
const f = await fixture();
|
||||
const snapshotId = randomUUID();
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
const cleanup = vi.fn(async () => {});
|
||||
const loadSavedWorkspace = vi.fn(async () => ({ root: f.root, savedAt: "2026-10-30T12:00:00.000Z", cleanup }));
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">, loadSavedWorkspace });
|
||||
const service = workspaceFileResourceService(db, undefined, { remoteFiles });
|
||||
const result = await service.readContent(f.ids.issue, { workspace: "execution", path: "docs/result.md" });
|
||||
expect(result).toMatchObject({ resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } }, content: { data: "Remote useful result\n" } });
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
|
||||
expect(cleanup).toHaveBeenCalledTimes(1);
|
||||
expect(f.execute).not.toHaveBeenCalled();
|
||||
expect(f.syncOut).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shares one verified saved extraction across 100 availability paths and removes it after the request", async () => {
|
||||
const f = await fixture();
|
||||
const storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "availability-saved-"));
|
||||
directories.push(storageRoot);
|
||||
const paths = Array.from({ length: 100 }, (_, index) => `docs/result-${index}.md`);
|
||||
await Promise.all(paths.map(file => fs.writeFile(path.join(f.root, file), `Result ${file}\n`)));
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects")));
|
||||
const archivePath = path.join(storageRoot, "saved.pcws");
|
||||
const archive = await captureRemoteWorkspaceDirectory({ cwd: f.root, archivePath });
|
||||
const [operation] = await db.insert(workspaceOperations).values({ companyId: f.ids.company,
|
||||
executionWorkspaceId: f.ids.workspace, phase: "synthetic_saved_fixture" }).returning();
|
||||
const snapshot = await beginWorkspaceSnapshot(db, { companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
operationId: operation!.id, generation: 1, epoch: 1 });
|
||||
await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace,
|
||||
snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest });
|
||||
await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id));
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
const loadSavedWorkspace = vi.fn(createSavedWorkspaceLoader(db, storage));
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
|
||||
const service = workspaceFileResourceService(db, undefined, { remoteFiles });
|
||||
|
||||
const result = await service.availability(f.ids.issue, { queries: paths.map(file => ({ workspace: "execution", path: file })) });
|
||||
|
||||
expect(result.results).toHaveLength(100);
|
||||
expect(result.results.every(item => item.openable && item.resource?.source?.kind === "saved")).toBe(true);
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledTimes(1);
|
||||
for (const loaded of loadSavedWorkspace.mock.results) {
|
||||
const saved = await loaded.value;
|
||||
await expect(fs.stat(saved.root)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
}
|
||||
await service.availability(f.ids.issue, { queries: [{ workspace: "execution", path: paths[0]! }] });
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledTimes(2);
|
||||
expect(f.execute).not.toHaveBeenCalled();
|
||||
}, 60_000);
|
||||
|
||||
it.each(["null", "rejected", "unavailable"] as const)("shares a %s remote open across availability paths", async state => {
|
||||
const f = await fixture();
|
||||
const original = new Error("saved storage unavailable");
|
||||
const open = vi.fn(async () => {
|
||||
if (state === "rejected") throw original;
|
||||
if (state === "unavailable") throw unprocessable("Workspace files are unavailable", { code: "workspace_unavailable" });
|
||||
return null;
|
||||
});
|
||||
const result = workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
|
||||
{ queries: Array.from({ length: 20 }, (_, index) => ({ workspace: "execution", path: `result-${index}.md` })) });
|
||||
if (state === "rejected") await expect(result).rejects.toBe(original);
|
||||
else {
|
||||
const results = (await result).results;
|
||||
expect(results).toHaveLength(20);
|
||||
if (state === "unavailable") expect(results.every(item => item.unavailableReason === "workspace_unavailable")).toBe(true);
|
||||
}
|
||||
expect(open).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("drains started availability checks before closing shared access and preserves the original error", async () => {
|
||||
const f = await fixture();
|
||||
let releaseSlow!: () => void;
|
||||
let announceSlow!: () => void;
|
||||
let announceError!: () => void;
|
||||
const slow = new Promise<void>(resolve => { releaseSlow = resolve; });
|
||||
const slowStarted = new Promise<void>(resolve => { announceSlow = resolve; });
|
||||
const errorThrown = new Promise<void>(resolve => { announceError = resolve; });
|
||||
const original = new Error("availability read failed");
|
||||
const close = vi.fn(async () => { throw new Error("cleanup also failed"); });
|
||||
let slowFinished = false;
|
||||
const open = vi.fn(async () => ({
|
||||
source: { kind: "saved" as const, snapshotId: randomUUID(), savedAt: "2026-10-30T12:00:00.000Z" },
|
||||
stat: async (relativePath: string) => {
|
||||
if (relativePath === "error.md") {
|
||||
await slowStarted;
|
||||
announceError();
|
||||
throw original;
|
||||
}
|
||||
announceSlow();
|
||||
await slow;
|
||||
slowFinished = true;
|
||||
return { kind: "file" as const, relativePath, byteSize: 1, modifiedAt: null, text: true };
|
||||
},
|
||||
read: vi.fn(), list: vi.fn(), download: vi.fn(), close,
|
||||
}));
|
||||
const pending = workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
|
||||
{ queries: [{ workspace: "execution", path: "error.md" }, { workspace: "execution", path: "slow.md" }] });
|
||||
const observed = pending.catch(error => error);
|
||||
try {
|
||||
await errorThrown;
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
expect(slowFinished).toBe(false);
|
||||
} finally { releaseSlow(); }
|
||||
expect(await observed).toBe(original);
|
||||
expect(slowFinished).toBe(true);
|
||||
expect(open).toHaveBeenCalledTimes(1);
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps denied availability paths ahead of opening a remote workspace", async () => {
|
||||
const f = await fixture();
|
||||
const open = vi.fn();
|
||||
const result = await workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
|
||||
{ queries: [".env", ".git/config", "../escape"].map(file => ({ workspace: "execution", path: file })) });
|
||||
expect(result.results.every(item => !item.openable)).toBe(true);
|
||||
expect(open).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("isolates availability task access from a project workspace with the same UUID", async () => {
|
||||
const f = await fixture();
|
||||
const projectId = randomUUID();
|
||||
await db.insert(projects).values({ id: projectId, companyId: f.ids.company, name: "Remote project" });
|
||||
await db.insert(projectWorkspaces).values({ id: f.ids.workspace, companyId: f.ids.company, projectId,
|
||||
name: "Project root", sourceType: "remote_managed", isPrimary: true });
|
||||
await db.update(issues).set({ projectId, projectWorkspaceId: f.ids.workspace }).where(eq(issues.id, f.ids.issue));
|
||||
const close = vi.fn(async () => {});
|
||||
const stat = vi.fn(async (relativePath: string) => ({ kind: "file" as const, relativePath, byteSize: 1, modifiedAt: null, text: true }));
|
||||
const open = vi.fn(async () => ({ source: { kind: "live" as const, snapshotId: null, savedAt: null },
|
||||
stat, read: vi.fn(), list: vi.fn(), download: vi.fn(), close }));
|
||||
const result = await workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
|
||||
{ queries: [{ workspace: "execution", path: "result.md" }, { workspace: "project", path: "result.md" }] });
|
||||
expect(result.results.map(item => item.resource?.kind)).toEqual(["file", "remote_resource"]);
|
||||
expect(open).toHaveBeenCalledTimes(1);
|
||||
expect(stat).toHaveBeenCalledTimes(1);
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("attempts every availability accessor close even when an earlier close fails", async () => {
|
||||
const f = await fixture();
|
||||
const otherWorkspace = randomUUID();
|
||||
await db.insert(executionWorkspaces).values({ id: otherWorkspace, companyId: f.ids.company, mode: "isolated_workspace",
|
||||
strategyType: "project_primary", providerType: "cloud_sandbox", name: "Other task root", sourceIssueId: f.ids.issue });
|
||||
const cleanupFailure = new Error("first scratch cleanup failed");
|
||||
const closeFirst = vi.fn(async () => { throw cleanupFailure; });
|
||||
const closeOther = vi.fn(async () => {});
|
||||
const open = vi.fn(async ({ workspaceId }: { workspaceId: string }) => ({
|
||||
source: { kind: "live" as const, snapshotId: null, savedAt: null },
|
||||
stat: async (relativePath: string) => {
|
||||
if (workspaceId === f.ids.workspace) throw notFound("Workspace file not found");
|
||||
return { kind: "file" as const, relativePath, byteSize: 1, modifiedAt: null, text: true };
|
||||
},
|
||||
read: vi.fn(), list: vi.fn(), download: vi.fn(), close: workspaceId === f.ids.workspace ? closeFirst : closeOther,
|
||||
}));
|
||||
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
|
||||
{ queries: [{ workspace: "auto", path: "one.md" }, { workspace: "auto", path: "two.md" }] })).rejects.toBe(cleanupFailure);
|
||||
expect(open).toHaveBeenCalledTimes(2);
|
||||
expect(closeFirst).toHaveBeenCalledTimes(1);
|
||||
expect(closeOther).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("rechecks saved authority for each availability stat on a shared accessor", async () => {
|
||||
const f = await fixture();
|
||||
const snapshotId = randomUUID();
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
const cleanup = vi.fn(async () => {});
|
||||
const loadSavedWorkspace = vi.fn(async () => ({ root: f.root, savedAt: "2026-10-30T12:00:00.000Z", cleanup }));
|
||||
const actual = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
|
||||
let changed!: () => void;
|
||||
const authorityChanged = new Promise<void>(resolve => { changed = resolve; });
|
||||
const open = vi.fn(async (input: Parameters<typeof actual.open>[0]) => {
|
||||
const access = (await actual.open(input))!;
|
||||
return { ...access, stat: async (relativePath: string) => {
|
||||
if (relativePath === "docs/after.md") await authorityChanged;
|
||||
const entry = await access.stat(relativePath);
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId, epoch: 2 } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
changed();
|
||||
return entry;
|
||||
} };
|
||||
});
|
||||
const result = await workspaceFileResourceService(db, undefined, { remoteFiles: { open } }).availability(f.ids.issue,
|
||||
{ queries: [{ workspace: "execution", path: "docs/result.md" }, { workspace: "execution", path: "docs/after.md" }] });
|
||||
expect(result.results[0]!.openable).toBe(true);
|
||||
expect(result.results[1]!.unavailableReason).toBe("workspace_changed");
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledTimes(1);
|
||||
expect(cleanup).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each(["confirmed_stop", "unknown_stop", "dirty", "foreign_receipt"])(
|
||||
"uses saved bytes only for a clean workspace with an exact stopped receipt (%s)", async state => {
|
||||
const f = await fixture();
|
||||
const snapshotId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
const runId = randomUUID();
|
||||
await db.insert(agents).values({ id: agentId, companyId: f.ids.company, name: "Stopped workspace reader" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId: f.ids.company, agentId, status: "succeeded" });
|
||||
const termination = remoteTerminationReceipt({ id: f.ids.lease, companyId: f.ids.company, heartbeatRunId: runId,
|
||||
provider: "fake", providerLeaseId: f.authority.location!.providerLeaseId },
|
||||
{ state: "stopped", providerLeaseId: f.authority.location!.providerLeaseId })!;
|
||||
await db.update(environmentLeases).set({ heartbeatRunId: runId, status: "released", releasedAt: new Date(), cleanupStatus: "success",
|
||||
metadata: { remoteCwd: f.root, ...(state !== "unknown_stop" ? { remoteExecutionTermination:
|
||||
state === "foreign_receipt" ? { ...termination, leaseId: randomUUID() } : termination } : {}) } })
|
||||
.where(eq(environmentLeases.id, f.ids.lease));
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId, dirty: state === "dirty" } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
const savedRoot = await fs.mkdtemp(path.join(os.tmpdir(), "stopped-workspace-saved-"));
|
||||
directories.push(savedRoot);
|
||||
await fs.mkdir(path.join(savedRoot, "docs"));
|
||||
await fs.writeFile(path.join(savedRoot, "docs", "result.md"), "Saved stopped result\n");
|
||||
const cleanup = vi.fn(async () => {});
|
||||
const loadSavedWorkspace = vi.fn(async () => ({ root: savedRoot, savedAt: "2026-10-30T12:00:00.000Z", cleanup }));
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace });
|
||||
const content = workspaceFileResourceService(db, undefined, { remoteFiles }).readContent(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" });
|
||||
if (state === "confirmed_stop") {
|
||||
await expect(content).resolves.toMatchObject({ content: { data: "Saved stopped result\n" },
|
||||
resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } } });
|
||||
expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId });
|
||||
expect(cleanup).toHaveBeenCalledOnce();
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
expect(workspace!.metadata?.remoteWorkspace).toMatchObject({ location: f.authority.location });
|
||||
} else {
|
||||
await expect(content).rejects.toMatchObject({ status: 422, details: { code: "workspace_unavailable" } });
|
||||
expect(loadSavedWorkspace).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(f.execute).not.toHaveBeenCalled();
|
||||
expect(f.syncOut).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not fall back to a saved version for an unreachable live allocation", async () => {
|
||||
const f = await fixture();
|
||||
f.execute.mockRejectedValueOnce(new Error("provider unavailable"));
|
||||
const loadSavedWorkspace = vi.fn();
|
||||
const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } as unknown as Pick<EnvironmentRuntimeService, "execute" | "syncOut">, loadSavedWorkspace });
|
||||
await expect(workspaceFileResourceService(db, undefined, { remoteFiles }).resolve(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422 });
|
||||
expect(loadSavedWorkspace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects cross-company and stale allocation bindings before provider access", async () => {
|
||||
const f = await fixture();
|
||||
await expect(f.remoteFiles.open({ companyId: randomUUID(), workspaceId: f.ids.workspace })).rejects.toMatchObject({ status: 404 });
|
||||
await db.update(environmentLeases).set({ providerLeaseId: "replacement" }).where(and(eq(environmentLeases.id, f.ids.lease), eq(environmentLeases.companyId, f.ids.company)));
|
||||
await expect(f.remoteFiles.open({ companyId: f.ids.company, workspaceId: f.ids.workspace })).rejects.toMatchObject({ status: 422 });
|
||||
expect(f.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a result after the durable workspace location changes", async () => {
|
||||
const f = await fixture();
|
||||
const original = f.execute.getMockImplementation()!;
|
||||
f.execute.mockImplementationOnce(async input => {
|
||||
const result = await original(input);
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, epoch: 2 } } })
|
||||
.where(eq(executionWorkspaces.id, f.ids.workspace));
|
||||
return result;
|
||||
});
|
||||
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: f.remoteFiles }).resolve(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ details: { code: "workspace_changed" } });
|
||||
});
|
||||
|
||||
it("rejects corrupt provider metadata without exposing its output", async () => {
|
||||
const f = await fixture();
|
||||
f.execute.mockResolvedValueOnce({ stdout: '{"kind":"file","relativePath":"../../private","byteSize":1}',
|
||||
stderr: "private-provider-details", exitCode: 0, timedOut: false });
|
||||
await expect(workspaceFileResourceService(db, undefined, { remoteFiles: f.remoteFiles }).resolve(f.ids.issue,
|
||||
{ workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ message: "Workspace files are unavailable", details: { code: "workspace_unavailable" } });
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { applyPendingMigrations, closeRegisteredClients, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -7,10 +7,12 @@ import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, companies, createDb, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issues } from "@paperclipai/db";
|
||||
import { executionWorkspaceService } from "../services/execution-workspaces.js";
|
||||
import { executionWorkspaceRepositoryService } from "../services/execution-workspace-repositories.js";
|
||||
import * as remoteWorkspaces from "../services/remote-workspaces.js";
|
||||
import type { EnvironmentLease } from "@paperclipai/shared";
|
||||
import { listLocalProjectRepositories } from "@paperclipai/adapter-utils/ssh";
|
||||
import { readManagedWorkspaceRepositories, disposeGitWorkspaceSnapshot } from "@paperclipai/adapter-utils/git-workspace-sync";
|
||||
import { captureDirectorySnapshot, disposeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
@@ -18,7 +20,11 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro
|
||||
|
||||
const catalog = vi.hoisted(() => ({ available: true }));
|
||||
vi.mock("../services/tool-access.js", () => ({ toolAccessService: () => ({ listProjectRepositories: async () => ({ repositories: catalog.available ? [{ id: "123", fullName: "team/source", url: "https://github.com/team/source", connections: [] }] : [] }) }) }));
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_TASK_REPOSITORIES_TEST_DATABASE_URL;
|
||||
if (externalDatabaseUrl && new URL(externalDatabaseUrl).pathname !== "/remote_workspace_repositories_test") {
|
||||
throw new Error("Remote repository tests require the disposable remote_workspace_repositories_test database");
|
||||
}
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("durable task repository receipts", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
@@ -26,8 +32,13 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
const actor = { type: "board" as const, source: "local_implicit" as const, userId: "local-board" };
|
||||
beforeAll(async () => {
|
||||
vi.stubEnv("PAPERCLIP_ISSUE_PRIVACY_MODE", "enforce");
|
||||
temporary = await startEmbeddedPostgresTestDatabase("paperclip-task-repositories-");
|
||||
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
|
||||
temporary = externalDatabaseUrl ? { connectionString: externalDatabaseUrl,
|
||||
cleanup: () => closeRegisteredClients(externalDatabaseUrl) }
|
||||
: await startEmbeddedPostgresTestDatabase("paperclip-task-repositories-");
|
||||
db = createDb(temporary.connectionString);
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
if (externalDatabaseUrl) await db.execute(sql`truncate table companies restart identity cascade`);
|
||||
cwd = await mkdtemp(path.join(tmpdir(), "paperclip-task-repositories-files-"));
|
||||
[companyId] = (await db.insert(companies).values({ name: "Task repositories", issuePrefix: "TREPO" }).returning()).map(row => row.id);
|
||||
[agentId] = (await db.insert(agents).values({ companyId, name: "Repository worker", adapterType: "process", status: "idle" }).returning()).map(row => row.id);
|
||||
@@ -41,6 +52,39 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
return executionWorkspaceRepositoryService(db).prepareForAdmission(input);
|
||||
};
|
||||
const request = (requestKey: string, ref?: string) => executionWorkspaceRepositoryService(db).request({ companyId, issueId, actor, request: { repository: { kind: "catalog", id: "123" }, requestKey, ...(ref ? { ref } : {}) } });
|
||||
it("prepares a remote task repository through its current writer without opening a controller path", async () => {
|
||||
const runId = randomUUID(), taskId = randomUUID(), remoteWorkspaceId = randomUUID(), leaseId = randomUUID(), environmentId = randomUUID();
|
||||
const remoteCwd = `/workspaces/${remoteWorkspaceId}`;
|
||||
await db.insert(issues).values({ id: taskId, companyId, title: "Remote repositories" });
|
||||
await db.insert(executionWorkspaces).values({ id: remoteWorkspaceId, companyId, sourceIssueId: taskId,
|
||||
name: "Remote task", cwd: remoteCwd, mode: "isolated_workspace", strategyType: "project_primary", providerType: "cloud_sandbox",
|
||||
metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null,
|
||||
source: { kind: "empty" }, writerRunId: runId, dirty: true,
|
||||
location: { leaseId, environmentId, providerLeaseId: "remote-allocation", remoteCwd } } } });
|
||||
await db.update(issues).set({ executionWorkspaceId: remoteWorkspaceId }).where(eq(issues.id, taskId));
|
||||
const intent = await executionWorkspaceRepositoryService(db).request({ companyId, issueId: taskId, actor,
|
||||
request: { repository: { kind: "catalog", id: "123" }, requestKey: "remote-repository" } });
|
||||
const pin = "a".repeat(40);
|
||||
const prepare = vi.spyOn(remoteWorkspaces, "prepareRemoteWorkspaceRepository").mockResolvedValue({
|
||||
cwd: `${remoteCwd}/${intent.repository.relativePath}`, pinnedCommit: pin, branchName: "main" });
|
||||
try {
|
||||
const prepared = await prepareForAdmission({ companyId, issueId: taskId, workspaceId: remoteWorkspaceId,
|
||||
cwd: remoteCwd, agentId, runId, responsibleUserId: "local-board",
|
||||
remoteTarget: { kind: "remote", transport: "sandbox", remoteCwd } as never,
|
||||
remoteLease: { id: leaseId, environmentId, companyId, providerLeaseId: "remote-allocation" } as EnvironmentLease });
|
||||
expect(prepare).toHaveBeenCalledWith(expect.objectContaining({ runId,
|
||||
repository: expect.objectContaining({ id: intent.repository.id, relativePath: intent.repository.relativePath }) }));
|
||||
expect(prepared).toMatchObject([{ id: intent.repository.id, pinnedCommit: pin }]);
|
||||
const [receipt] = await db.select().from(executionWorkspaceRepositories).where(eq(executionWorkspaceRepositories.id, intent.repository.id));
|
||||
expect(receipt).toMatchObject({ state: "ready", pinnedCommit: pin, branchName: "main" });
|
||||
expect(await lstat(remoteCwd).catch(() => null)).toBeNull();
|
||||
await expect(prepareForAdmission({ companyId, issueId: taskId, workspaceId: remoteWorkspaceId,
|
||||
cwd: remoteCwd, agentId, runId: randomUUID(), responsibleUserId: "local-board",
|
||||
remoteTarget: { kind: "remote", transport: "sandbox", remoteCwd } as never,
|
||||
remoteLease: { id: leaseId } as EnvironmentLease })).rejects.toThrow("publication binding");
|
||||
expect(prepare).toHaveBeenCalledTimes(1);
|
||||
} finally { prepare.mockRestore(); }
|
||||
});
|
||||
it("publishes a requested repository from an absent path through real admission locking and cloning", async () => {
|
||||
const root = await mkdtemp(path.join(tmpdir(), "paperclip-fresh-repository-"));
|
||||
const previousPath = process.env.PATH;
|
||||
|
||||
+10
-3
@@ -112,6 +112,8 @@ import { caseRoutes } from "./routes/cases.js";
|
||||
import { caseHandoffRoutes } from "./routes/case-handoffs.js";
|
||||
import { workflowRoutes } from "./routes/workflows.js";
|
||||
import { fileResourceRoutes } from "./routes/file-resources.js";
|
||||
import { createRemoteWorkspaceFiles } from "./services/remote-workspace-files.js";
|
||||
import { createSavedWorkspaceLoader } from "./services/remote-workspaces.js";
|
||||
import { routineRoutes } from "./routes/routines.js";
|
||||
import { environmentRoutes } from "./routes/environments.js";
|
||||
import { executionWorkspaceRoutes } from "./routes/execution-workspaces.js";
|
||||
@@ -537,7 +539,10 @@ export async function createApp(
|
||||
// A signed claim above commits identity before any normal request can seed
|
||||
// company data. Unclaimed probes bypass session resolution as well as SQL.
|
||||
app.use(cloudWarmStandbyMiddleware(isWarmStandby, health, staticUi));
|
||||
app.use("/api/customer-success/v1", customerSuccessRoutes(db, opts.storageService));
|
||||
const workerManager = opts.pluginWorkerManager ?? createPluginWorkerManager();
|
||||
app.use("/api/customer-success/v1", customerSuccessRoutes(db, opts.storageService, {
|
||||
pluginWorkerManager: workerManager,
|
||||
}));
|
||||
app.use(museTransportRoutes(db, opts.authPublicBaseUrl));
|
||||
app.use(publicMcpIngress);
|
||||
// Connection-intent tools carry their own short-lived, run-bound bearer and
|
||||
@@ -545,7 +550,6 @@ export async function createApp(
|
||||
// agent API key. Every request revalidates the active heartbeat row.
|
||||
app.use(runtimeConnectionIntentRoutes(db));
|
||||
const hostServicesDisposers = new Map<string, () => void>();
|
||||
const workerManager = opts.pluginWorkerManager ?? createPluginWorkerManager();
|
||||
let lifecyclePluginsReady = false;
|
||||
const agentLifecycle = startAgentLifecycle(db, createLifecycleDriver(db, workerManager), () => lifecyclePluginsReady && !isWarmStandby() && !isIdleTaskDrainActive());
|
||||
const connectionIntentHeartbeat = heartbeatService(db, {
|
||||
@@ -772,7 +776,10 @@ export async function createApp(
|
||||
api.use(caseHandoffRoutes(db));
|
||||
api.use(workflowRoutes(db));
|
||||
api.use(issueTreeControlRoutes(db, { pluginWorkerManager: workerManager }));
|
||||
api.use(fileResourceRoutes(db));
|
||||
api.use(fileResourceRoutes(db, { remoteFiles: createRemoteWorkspaceFiles(db, {
|
||||
environmentRuntime: environmentRuntimeService(db, { pluginWorkerManager: workerManager }),
|
||||
loadSavedWorkspace: createSavedWorkspaceLoader(db, opts.storageService),
|
||||
}) }));
|
||||
api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));
|
||||
api.use(
|
||||
environmentRoutes(db, {
|
||||
|
||||
@@ -42,6 +42,7 @@ import {
|
||||
} from "./services/native-runtime/native-harness-backup-expiration.js";
|
||||
import { registerNativeRunnerEvidenceRetentionWork } from "./services/native-runtime/native-runner-evidence-retention-work.js";
|
||||
import { gitRestorePointRetentionService, registerGitRestorePointWork } from "./services/git-restore-point-ledger.js";
|
||||
import { remoteWorkspaceSnapshotRetentionService, registerRemoteWorkspaceSnapshotRetentionWork } from "./services/remote-workspace-snapshot-retention.js";
|
||||
import {
|
||||
nativeWorkspaceSyncRetentionService,
|
||||
registerNativeWorkspaceSyncRetentionWork,
|
||||
@@ -1499,6 +1500,7 @@ async function startServerWithDatabaseTeardown(
|
||||
wakeup: heartbeat.wakeup,
|
||||
});
|
||||
const terminalWorkspaces = executionWorkspaceService(db as any, {
|
||||
destroyRemoteWorkspaceLease: input => cleanupInspectionRuntime.destroyRemoteWorkspaceLease(input),
|
||||
pluginWorkerManager,
|
||||
gitCacheExpirationEnabled: config.gitCacheExpirationEnabled,
|
||||
gitBaseExpirationEnabled: config.gitBaseExpirationEnabled,
|
||||
@@ -1535,6 +1537,10 @@ async function startServerWithDatabaseTeardown(
|
||||
logger.info(result, "git restore point retention");
|
||||
}
|
||||
});
|
||||
registerRemoteWorkspaceSnapshotRetentionWork(deliveryWork,
|
||||
remoteWorkspaceSnapshotRetentionService(db as any, { storage: storageService }), result => {
|
||||
if (result.deleted || result.failed) logger.info(result, "remote workspace snapshot retention");
|
||||
});
|
||||
if (config.nativeWorkspaceSyncRetentionEnabled) {
|
||||
unsubscribeNativeWorkspaceSyncRetention = registerNativeWorkspaceSyncRetentionWork(deliveryWork, nativeWorkspaceSyncRetentionService(db as any, {
|
||||
retentionDays: config.nativeWorkspaceSyncRetentionDays,
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { getCloudRuntimeIdentity } from "../services/cloud-runtime-identity.js";
|
||||
import { customerSuccessRunAuthority } from "../services/customer-success-authority.js";
|
||||
import { readCustomerSuccessResource } from "../services/customer-success-inspection.js";
|
||||
import { readCustomerSuccessResource, type CustomerSuccessInspectionOptions } from "../services/customer-success-inspection.js";
|
||||
import { forbidden, unprocessable } from "../errors.js";
|
||||
|
||||
export function verifyInspectionPermit(
|
||||
@@ -70,7 +70,7 @@ export function verifyInspectionPermit(
|
||||
}
|
||||
|
||||
/** Mounted before actor middleware: none of these reads creates tenant state. */
|
||||
export function customerSuccessRoutes(db: Db, storage: StorageService) {
|
||||
export function customerSuccessRoutes(db: Db, storage: StorageService, options: CustomerSuccessInspectionOptions = {}) {
|
||||
const router = Router();
|
||||
router.use((_req, res, next) => {
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
@@ -122,7 +122,7 @@ export function customerSuccessRoutes(db: Db, storage: StorageService) {
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!consumed.ok) throw forbidden("Inspection permit could not be consumed");
|
||||
const result = await readCustomerSuccessResource(db, storage, query);
|
||||
const result = await readCustomerSuccessResource(db, storage, query, options);
|
||||
const body = JSON.stringify(result);
|
||||
if (Buffer.byteLength(body) > 2 * 1024 * 1024)
|
||||
throw unprocessable("Inspection response is too large; use pagination or a byte range");
|
||||
|
||||
@@ -11,6 +11,7 @@ import type { Db } from "@paperclipai/db";
|
||||
import { issues, projects, projectWorkspaces } from "@paperclipai/db";
|
||||
import {
|
||||
findWorkspaceCommandDefinition,
|
||||
readRemoteWorkspaceAuthority,
|
||||
matchWorkspaceRuntimeServiceToCommand,
|
||||
reconcileExecutionWorkspaceBranchSchema,
|
||||
updateExecutionWorkspaceSchema,
|
||||
@@ -58,6 +59,7 @@ import { appendWithCap } from "../adapters/utils.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js";
|
||||
import { withExecutionWorkspaceLifecycleLock } from "../services/execution-workspace-lifecycle-lock.js";
|
||||
import { resolveManagedWorkspaceInstanceId } from "../services/managed-workspace-identity.js";
|
||||
import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
import {
|
||||
@@ -89,16 +91,15 @@ function resolveFallbackSeedSourceConfigPath(baseWorkspaceCwd: string): string |
|
||||
|
||||
export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
|
||||
const router = Router();
|
||||
const svc = executionWorkspaceService(db, { pluginWorkerManager: opts.pluginWorkerManager });
|
||||
const environmentRuntime = environmentRuntimeService(db, { pluginWorkerManager: opts.pluginWorkerManager });
|
||||
const svc = executionWorkspaceService(db, { pluginWorkerManager: opts.pluginWorkerManager,
|
||||
destroyRemoteWorkspaceLease: input => environmentRuntime.destroyRemoteWorkspaceLease(input) });
|
||||
const access = accessService(db);
|
||||
const workspaceOperationsSvc = workspaceOperationService(db);
|
||||
const runtimeLeases = workspaceRuntimeLeaseService(db);
|
||||
const heartbeat = heartbeatService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
const environmentRuntime = environmentRuntimeService(db, {
|
||||
pluginWorkerManager: opts.pluginWorkerManager,
|
||||
});
|
||||
|
||||
async function assertWorkspacePrivacy(req: Request, res: Response, id: string) {
|
||||
if (await canActorReadExecutionWorkspace(db, req.actor, id)) return true;
|
||||
@@ -302,6 +303,10 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
executionWorkspaceId: existing.id,
|
||||
sourceIssueId: existing.sourceIssueId,
|
||||
});
|
||||
if (readRemoteWorkspaceAuthority(existing.metadata)) {
|
||||
throw unprocessable("Workspace commands are not supported for remote workspaces yet. Run the command through an agent task.",
|
||||
{ code: "remote_workspace_runtime_commands_unsupported" });
|
||||
}
|
||||
|
||||
const workspaceCwd = existing.cwd;
|
||||
if (!workspaceCwd) {
|
||||
@@ -479,6 +484,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
let runtimeServiceCount = existing.runtimeServices?.length ?? 0;
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let leaseClaimResult: WorkspaceRuntimeLeaseClaim | null = null;
|
||||
let runtimeControlAdmitted = false;
|
||||
|
||||
/**
|
||||
* Bring runtime rows, local listeners and the recorded desired state back to a consistent
|
||||
@@ -531,6 +538,32 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
}
|
||||
|
||||
const recordRuntimeControlOperation = () => recorder.recordOperation({
|
||||
admit: async (publish) => {
|
||||
await withExecutionWorkspaceLifecycleLock(db, existing.id, async () => {
|
||||
const fresh = await svc.getById(existing.id);
|
||||
if (!fresh || fresh.companyId !== existing.companyId) throw conflict("Execution workspace changed before command admission.");
|
||||
if (readRemoteWorkspaceAuthority(fresh.metadata)) {
|
||||
throw unprocessable("Workspace commands are not supported for remote workspaces yet. Run the command through an agent task.",
|
||||
{ code: "remote_workspace_runtime_commands_unsupported" });
|
||||
}
|
||||
if (fresh.metadata?.remoteWorkspaceMigration) {
|
||||
throw conflict("The workspace is being moved to remote storage. Retry after migration completes.",
|
||||
{ code: "workspace_runtime_migration_in_progress" });
|
||||
}
|
||||
if (fresh.cwd !== existing.cwd || fresh.providerRef !== existing.providerRef || fresh.providerType !== existing.providerType) {
|
||||
throw conflict("Execution workspace changed before command admission.", { code: "workspace_runtime_authority_changed" });
|
||||
}
|
||||
if (LEASED_WORKSPACE_RUNTIME_ACTIONS.includes(action)) {
|
||||
leaseClaimResult = await runtimeLeases.claim({ companyId: existing.companyId,
|
||||
executionWorkspaceId: existing.id, action, owner: authorization });
|
||||
}
|
||||
await workspaceOperationsSvc.assertRuntimeControlAvailable({ executionWorkspaceId: existing.id, action });
|
||||
// Migration uses this lifecycle fence and refuses every running operation.
|
||||
// Publish before unlocking; host I/O starts only after the row is visible.
|
||||
await publish();
|
||||
});
|
||||
runtimeControlAdmitted = true;
|
||||
},
|
||||
phase: action === "stop"
|
||||
? "workspace_teardown"
|
||||
: action === "repair"
|
||||
@@ -989,25 +1022,6 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
executionWorkspaceId: existing.id,
|
||||
action,
|
||||
run: async () => {
|
||||
// Claim the durable exclusivity lease before anything mutates the workspace or
|
||||
// its runtime services. A competing issue/run throws 409 here, leaving no
|
||||
// workspace operation row and no runtime-service change behind.
|
||||
let leaseClaimResult: WorkspaceRuntimeLeaseClaim | null = null;
|
||||
if (LEASED_WORKSPACE_RUNTIME_ACTIONS.includes(action)) {
|
||||
leaseClaimResult = await runtimeLeases.claim({
|
||||
companyId: existing.companyId,
|
||||
executionWorkspaceId: existing.id,
|
||||
action,
|
||||
owner: authorization,
|
||||
});
|
||||
}
|
||||
// Re-check inside the claim: recovery of stranded operations plus a refusal if one
|
||||
// is genuinely still live. The same assertion ran before the lease was taken, but a
|
||||
// row can be stranded in that window, and only the recovering path may proceed.
|
||||
await workspaceOperationsSvc.assertRuntimeControlAvailable({
|
||||
executionWorkspaceId: existing.id,
|
||||
action,
|
||||
});
|
||||
try {
|
||||
return {
|
||||
operation: await recordRuntimeControlOperation(),
|
||||
@@ -1017,7 +1031,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
// The operation is already terminal here. This also catches the recorder's own time
|
||||
// budget expiring on a start that never settles, which is the one failure the inner
|
||||
// handler cannot see — reconcile there too so no listener or desired state is left over.
|
||||
if (action === "start" || action === "restart" || action === "repair") {
|
||||
if (runtimeControlAdmitted && (action === "start" || action === "restart" || action === "repair")) {
|
||||
await reconcileFailedRuntimeStart(error);
|
||||
}
|
||||
throw error;
|
||||
@@ -1300,6 +1314,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
workspaceId: id,
|
||||
capturedGeneration,
|
||||
destroy: async () => {
|
||||
if (readRemoteWorkspaceAuthority(workspace.metadata)) return svc.cleanupRemoteStorage(workspace);
|
||||
await environmentRuntime.destroyReusableSandboxLeases({
|
||||
companyId: existing.companyId,
|
||||
executionWorkspaceId: existing.id,
|
||||
|
||||
@@ -14,7 +14,9 @@ import {
|
||||
type WorkspaceFileListResponse,
|
||||
} from "@paperclipai/shared";
|
||||
import { badRequest, HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import { workspaceFileResourceService } from "../services/index.js";
|
||||
import { workspaceFileResourceService } from "../services/workspace-file-resources.js";
|
||||
import type { PreparedWorkspaceFileDownload } from "../services/workspace-file-resources.js";
|
||||
import type { RemoteWorkspaceFiles } from "../services/remote-workspace-files.js";
|
||||
import { assertBoard, getActorInfo, hasCompanyAccess } from "./authz.js";
|
||||
import { logActivity } from "../services/activity-log.js";
|
||||
import {
|
||||
@@ -56,7 +58,7 @@ export type WorkspaceFileResourceService = {
|
||||
issueId: string,
|
||||
input: { path: string; workspace?: "auto" | "execution" | "project" | null; projectId?: string | null; workspaceId?: string | null },
|
||||
opts?: { issue?: Awaited<ReturnType<WorkspaceFileResourceService["getIssue"]>> },
|
||||
): Promise<{ resource: ResolvedWorkspaceResource; realPath: string }>;
|
||||
): Promise<PreparedWorkspaceFileDownload>;
|
||||
};
|
||||
|
||||
type FileResourceLimiter = {
|
||||
@@ -353,9 +355,10 @@ export function fileResourceRoutes(db: Db, opts: {
|
||||
limiter?: FileResourceLimiter;
|
||||
listLimiter?: FileResourceLimiter;
|
||||
availabilityLimiter?: FileResourceLimiter;
|
||||
remoteFiles?: RemoteWorkspaceFiles;
|
||||
} = {}) {
|
||||
const router = Router();
|
||||
const serviceFor = (req: Request) => opts.service ?? workspaceFileResourceService(db, req.actor);
|
||||
const serviceFor = (req: Request) => opts.service ?? workspaceFileResourceService(db, req.actor, { remoteFiles: opts.remoteFiles });
|
||||
const limiter = opts.limiter ?? createFileResourceLimiter();
|
||||
const listLimiter = opts.listLimiter ?? createFileResourceListLimiter();
|
||||
const availabilityLimiter = opts.availabilityLimiter ?? createFileResourceAvailabilityLimiter();
|
||||
@@ -857,36 +860,39 @@ export function fileResourceRoutes(db: Db, opts: {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await logActivity(db, {
|
||||
companyId: issue.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
action: "issue.file_resource_download",
|
||||
entityType: "issue",
|
||||
entityId: req.params.issueId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
agentApiKeyId: actor.agentApiKeyId,
|
||||
details: activityDetails({
|
||||
outcome: "success",
|
||||
workspaceKind: result.resource.workspaceKind,
|
||||
workspaceId: result.resource.workspaceId,
|
||||
projectId: result.resource.projectId ?? null,
|
||||
projectName: result.resource.projectName ?? null,
|
||||
displayPath: result.resource.displayPath,
|
||||
byteSize: result.resource.byteSize ?? null,
|
||||
contentType: result.resource.contentType ?? null,
|
||||
}),
|
||||
});
|
||||
try {
|
||||
await logActivity(db, {
|
||||
companyId: issue.companyId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
action: "issue.file_resource_download",
|
||||
entityType: "issue",
|
||||
entityId: req.params.issueId,
|
||||
agentId: actor.agentId,
|
||||
runId: actor.runId,
|
||||
agentApiKeyId: actor.agentApiKeyId,
|
||||
details: activityDetails({
|
||||
outcome: "success",
|
||||
workspaceKind: result.resource.workspaceKind,
|
||||
workspaceId: result.resource.workspaceId,
|
||||
projectId: result.resource.projectId ?? null,
|
||||
projectName: result.resource.projectName ?? null,
|
||||
displayPath: result.resource.displayPath,
|
||||
byteSize: result.resource.byteSize ?? null,
|
||||
contentType: result.resource.contentType ?? null,
|
||||
}),
|
||||
});
|
||||
|
||||
res.setHeader("Content-Type", result.resource.contentType ?? "application/octet-stream");
|
||||
if (result.resource.byteSize != null) {
|
||||
res.setHeader("Content-Length", String(result.resource.byteSize));
|
||||
}
|
||||
res.setHeader("Cache-Control", "private, max-age=60");
|
||||
res.setHeader("X-Content-Type-Options", "nosniff");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${safeAttachmentFilename(result.resource.title)}"`);
|
||||
await pipeline(createReadStream(result.realPath), res);
|
||||
res.setHeader("Content-Type", result.resource.contentType ?? "application/octet-stream");
|
||||
if (result.resource.byteSize != null) {
|
||||
res.setHeader("Content-Length", String(result.resource.byteSize));
|
||||
}
|
||||
res.setHeader("Cache-Control", "private, max-age=60");
|
||||
res.setHeader("X-Content-Type-Options", "nosniff");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${safeAttachmentFilename(result.resource.title)}"`);
|
||||
if ("stream" in result) await pipeline(result.stream, res);
|
||||
else await pipeline(createReadStream(result.realPath), res);
|
||||
} finally { if ("stream" in result) await result.cleanup(); }
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,14 @@ import { deriveBundleState } from "./agent-instructions.js";
|
||||
import { readInstructionBytes } from "./agent-instruction-files.js";
|
||||
import { skillPackageStore } from "./skill-package-store.js";
|
||||
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
|
||||
import { createRemoteWorkspaceFiles } from "./remote-workspace-files.js";
|
||||
import { createSavedWorkspaceLoader } from "./remote-workspaces.js";
|
||||
import { environmentRuntimeService } from "./environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
|
||||
export interface CustomerSuccessInspectionOptions {
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
}
|
||||
|
||||
// This is a reviewed catalog of readers, not a proxy for existing GET routes.
|
||||
// Select credentials out of queries; never load encrypted identity records.
|
||||
@@ -136,6 +144,7 @@ export async function readCustomerSuccessResource(
|
||||
db: Db,
|
||||
storage: StorageService,
|
||||
q: InspectionQuery,
|
||||
options: CustomerSuccessInspectionOptions = {},
|
||||
): Promise<unknown> {
|
||||
return db.transaction(
|
||||
async (rawTx) => {
|
||||
@@ -328,7 +337,11 @@ export async function readCustomerSuccessResource(
|
||||
if (q.operation.startsWith("files.")) {
|
||||
try {
|
||||
const issue = await ownRow(tx, schema.issues, q.companyId!, q.resourceId!);
|
||||
const files = workspaceFileResourceService(tx);
|
||||
const remoteFiles = createRemoteWorkspaceFiles(tx, {
|
||||
environmentRuntime: environmentRuntimeService(tx, { pluginWorkerManager: options.pluginWorkerManager, readOnlyInspection: true }),
|
||||
loadSavedWorkspace: createSavedWorkspaceLoader(tx, storage),
|
||||
});
|
||||
const files = workspaceFileResourceService(tx, undefined, { remoteFiles });
|
||||
const input = { path: q.path ?? "", ...q.context, limit: q.limit, offset: q.offset };
|
||||
const opts = {
|
||||
issue: issue as unknown as NonNullable<Parameters<typeof files.list>[2]>["issue"],
|
||||
@@ -337,6 +350,31 @@ export async function readCustomerSuccessResource(
|
||||
if (q.operation === "files.read")
|
||||
return await files.readContent(q.resourceId!, input, opts);
|
||||
const resolved = await files.prepareDownload(q.resourceId!, input, opts);
|
||||
if ("stream" in resolved) {
|
||||
try {
|
||||
const size = resolved.resource.byteSize;
|
||||
if (size == null) return unavailable("file_size_unavailable");
|
||||
if (size === 0 && !q.range) return binary(Buffer.alloc(0), 0);
|
||||
const start = q.range?.start ?? 0;
|
||||
const end = q.range?.end ?? size - 1;
|
||||
if (start >= size || end >= size || end - start + 1 > MAX_INSPECTION_BYTES) {
|
||||
throw unprocessable("Request a bounded byte range");
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let position = 0;
|
||||
let received = 0;
|
||||
for await (const chunk of resolved.stream) {
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
const from = Math.max(0, start - position);
|
||||
const to = Math.min(bytes.length, end + 1 - position);
|
||||
if (to > from) { chunks.push(bytes.subarray(from, to)); received += to - from; }
|
||||
position += bytes.length;
|
||||
if (position > end) break;
|
||||
}
|
||||
if (received !== end - start + 1) return unavailable("file_changed_during_read");
|
||||
return binary(Buffer.concat(chunks), size, start);
|
||||
} finally { resolved.stream.destroy(); await resolved.cleanup(); }
|
||||
}
|
||||
const file = await fs.open(resolved.realPath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
const stat = await file.stat();
|
||||
|
||||
@@ -133,6 +133,7 @@ export const DELIVERY_QUEUES = {
|
||||
connectionEvents: "connection-events",
|
||||
connectionContinuity: "connection-continuity",
|
||||
gitRestorePoints: "git-restore-points",
|
||||
remoteWorkspaceSnapshots: "remote-workspace-snapshots",
|
||||
nativeWorkspaceSyncRetention: "native-workspace-sync-retention",
|
||||
question: "question-responses",
|
||||
pluginSchedules: "plugin-schedules",
|
||||
|
||||
@@ -294,6 +294,7 @@ async function resolveConfigSecretRefsForRuntime(input: {
|
||||
companyId: string;
|
||||
config: Record<string, unknown>;
|
||||
schema: Record<string, unknown> | null;
|
||||
readOnlyInspection?: boolean;
|
||||
context: {
|
||||
consumerId: string;
|
||||
issueId?: string | null;
|
||||
@@ -313,7 +314,10 @@ async function resolveConfigSecretRefsForRuntime(input: {
|
||||
nextConfig = writeConfigValueAtPath(
|
||||
nextConfig,
|
||||
path,
|
||||
await secrets.resolveSecretValue(input.companyId, trimmed, "latest", {
|
||||
input.readOnlyInspection ? await secrets.resolveSecretValueForReadOnlyEnvironmentInspection(input.companyId, trimmed, "latest", {
|
||||
environmentId: input.context.consumerId, configPath: path,
|
||||
issueId: input.context.issueId, heartbeatRunId: input.context.heartbeatRunId,
|
||||
}) : await secrets.resolveSecretValue(input.companyId, trimmed, "latest", {
|
||||
consumerType: "environment",
|
||||
consumerId: input.context.consumerId,
|
||||
actorType: "system",
|
||||
@@ -659,6 +663,7 @@ export async function resolveEnvironmentDriverConfigForRuntime(
|
||||
// configuration and tooling so the test reflects what real agent runs use.
|
||||
applyCustomImageTemplate?: boolean;
|
||||
readOnlyCustomImageTemplate?: boolean;
|
||||
readOnlyInspection?: boolean;
|
||||
onResolvedCustomImageTemplate?: (identity: { id: string; capturedAt: string | null; templateKind: string; bindingField: string }) => void;
|
||||
},
|
||||
): Promise<ParsedEnvironmentConfig> {
|
||||
@@ -704,6 +709,7 @@ export async function resolveEnvironmentDriverConfigForRuntime(
|
||||
companyId,
|
||||
config: parsed.config as Record<string, unknown>,
|
||||
schema,
|
||||
readOnlyInspection: context?.readOnlyInspection,
|
||||
context: {
|
||||
consumerId: environmentId!,
|
||||
issueId: context?.issueId ?? null,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { freshWorkspaceSchema } from "@paperclipai/plugin-sdk";
|
||||
import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRepositoryPath } from "./remote-workspaces.js";
|
||||
/**
|
||||
* Centralized environment run orchestrator.
|
||||
*
|
||||
@@ -412,6 +413,7 @@ export function environmentRunOrchestrator(
|
||||
effectiveExecutionWorkspaceMode: string | null;
|
||||
persistedExecutionWorkspace: ExecutionWorkspace | null;
|
||||
runnerTaskProjectIds?: readonly string[];
|
||||
assertOwned?: () => Promise<void>;
|
||||
/**
|
||||
* The host duplex observability recorder for this run. The orchestrator threads
|
||||
* it to `resolveEnvironmentExecutionTarget`, which stamps it on the sandbox
|
||||
@@ -429,6 +431,51 @@ export function environmentRunOrchestrator(
|
||||
effectiveExecutionWorkspaceMode,
|
||||
} = input;
|
||||
let { lease, persistedExecutionWorkspace } = input;
|
||||
const remoteAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata);
|
||||
if (remoteAuthority && persistedExecutionWorkspace) {
|
||||
if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") {
|
||||
throw new EnvironmentRunError("unsupported_adapter_environment", "Remote workspace storage requires a qualified native sandbox");
|
||||
}
|
||||
const target = await resolveEnvironmentExecutionTarget({
|
||||
db, companyId, adapterType, environment, leaseId: lease.id, leaseMetadata: lease.metadata,
|
||||
lease, environmentRuntime, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null,
|
||||
});
|
||||
if (!target || target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) {
|
||||
throw new EnvironmentRunError("transport_resolution_failed", "Remote workspace storage requires a sandbox file transport");
|
||||
}
|
||||
const prepared = await prepareRemoteWorkspace({
|
||||
db, workspace: persistedExecutionWorkspace, runId: heartbeatRunId, lease, target,
|
||||
source: remoteAuthority.source,
|
||||
assertOwned: input.assertOwned,
|
||||
provisionCommand: persistedExecutionWorkspace.config?.provisionCommand,
|
||||
runtimeProvisionCommand: persistedExecutionWorkspace.config?.runtimeProvisionCommand,
|
||||
});
|
||||
lease = prepared.lease;
|
||||
const workspaceRealization = {
|
||||
version: 1, mode: "in_place", authoritativeRoot: prepared.remoteRoot,
|
||||
pathAliases: [], outboundRestorePaths: [], provider: lease.provider,
|
||||
environmentId: environment.id, leaseId: lease.id, providerLeaseId: lease.providerLeaseId,
|
||||
additional: (remoteAuthority.source.repositories ?? []).filter(repository => repository.readOnly)
|
||||
.map(repository => ({ path: remoteWorkspaceRepositoryPath(prepared.remoteRoot, repository),
|
||||
projectId: repository.projectId ?? null, projectWorkspaceId: repository.projectWorkspaceId ?? null,
|
||||
repoUrl: repository.repoUrl, repoRef: repository.ref })),
|
||||
};
|
||||
const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, {
|
||||
...lease.metadata, remoteCwd: prepared.remoteRoot, workspaceRealization,
|
||||
});
|
||||
if (!updatedLease) throw new Error("Remote workspace lease disappeared during preparation");
|
||||
lease = updatedLease;
|
||||
persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(companyId, prepared.workspace.id);
|
||||
if (!persistedExecutionWorkspace) throw new Error("Remote workspace disappeared during preparation");
|
||||
input.executionWorkspace.cwd = prepared.remoteRoot;
|
||||
const executionTarget: AdapterExecutionTarget = {
|
||||
...target, remoteCwd: prepared.remoteRoot,
|
||||
workspaceRealization: { mode: "in_place", authoritativeRoot: prepared.remoteRoot,
|
||||
pathAliases: [], outboundRestorePaths: [] },
|
||||
};
|
||||
return { lease, workspaceRealization, executionTarget,
|
||||
remoteExecution: adapterExecutionTargetToRemoteSpec(executionTarget), persistedExecutionWorkspace };
|
||||
}
|
||||
|
||||
if (executionWorkspace.providerBinding) {
|
||||
const binding = executionWorkspace.providerBinding;
|
||||
|
||||
@@ -1268,6 +1268,7 @@ function createSandboxEnvironmentDriver(
|
||||
db: Db,
|
||||
options: {
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
readOnlyInspection?: boolean;
|
||||
pluginWorkerReadyTimeoutMs?: number;
|
||||
pluginWorkerReadyPollMs?: number;
|
||||
deferredOrphanCleanupBufferLimit?: number;
|
||||
@@ -1498,7 +1499,7 @@ function createSandboxEnvironmentDriver(
|
||||
id: input.environment.id,
|
||||
driver: "sandbox",
|
||||
config: sandboxConfigForLeaseMetadata(metadataConfig),
|
||||
});
|
||||
}, { readOnlyInspection: options.readOnlyInspection });
|
||||
if (parsed.driver === "sandbox") {
|
||||
return dropInternalPluginSandboxConfigKeys(parsed.config as unknown as Record<string, unknown>);
|
||||
}
|
||||
@@ -1510,16 +1511,19 @@ function createSandboxEnvironmentDriver(
|
||||
db,
|
||||
input.lease.companyId,
|
||||
input.environment,
|
||||
{ readOnlyInspection: options.readOnlyInspection },
|
||||
);
|
||||
if (parsed.driver === "sandbox" && parsed.config.provider === input.provider) {
|
||||
return dropInternalPluginSandboxConfigKeys(parsed.config as unknown as Record<string, unknown>);
|
||||
}
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (options.readOnlyInspection) throw error;
|
||||
// Lease metadata below is intentionally kept sufficient for cleanup
|
||||
// after the environment config changes or becomes invalid.
|
||||
}
|
||||
}
|
||||
|
||||
if (options.readOnlyInspection) throw new Error("Inspection requires a validated sandbox configuration.");
|
||||
return {
|
||||
provider: input.provider,
|
||||
...dropInternalPluginSandboxConfigKeys(input.lease.metadata),
|
||||
@@ -3586,6 +3590,7 @@ export function environmentRuntimeService(
|
||||
options: {
|
||||
drivers?: EnvironmentRuntimeDriver[];
|
||||
pluginWorkerManager?: PluginWorkerManager;
|
||||
readOnlyInspection?: boolean;
|
||||
pluginWorkerReadyTimeoutMs?: number;
|
||||
pluginWorkerReadyPollMs?: number;
|
||||
deferredOrphanCleanupBufferLimit?: number;
|
||||
@@ -3602,6 +3607,7 @@ export function environmentRuntimeService(
|
||||
createSshEnvironmentDriver(db),
|
||||
createSandboxEnvironmentDriver(db, {
|
||||
pluginWorkerManager: options.pluginWorkerManager,
|
||||
readOnlyInspection: options.readOnlyInspection,
|
||||
pluginWorkerReadyTimeoutMs: options.pluginWorkerReadyTimeoutMs,
|
||||
pluginWorkerReadyPollMs: options.pluginWorkerReadyPollMs,
|
||||
deferredOrphanCleanupBufferLimit: options.deferredOrphanCleanupBufferLimit,
|
||||
@@ -3647,6 +3653,7 @@ export function environmentRuntimeService(
|
||||
scopeCondition?: ReturnType<typeof and>;
|
||||
runId?: string;
|
||||
failureReason: string;
|
||||
anyLeasePolicy?: boolean;
|
||||
}): Promise<EnvironmentLease | null> {
|
||||
if (await nativeWorkspaceExportPendingForLease(db, input.leaseRow)) return null;
|
||||
if (!(await allowLegacyShutdownWorkspaceCleanup(db, input.leaseRow))) return null;
|
||||
@@ -3671,7 +3678,7 @@ export function environmentRuntimeService(
|
||||
eq(environmentLeases.companyId, input.leaseRow.companyId),
|
||||
eq(environmentLeases.environmentId, input.environment.id),
|
||||
input.runId ? eq(environmentLeases.heartbeatRunId, input.runId)
|
||||
: eq(environmentLeases.leasePolicy, "reuse_by_environment"),
|
||||
: input.anyLeasePolicy ? undefined : eq(environmentLeases.leasePolicy, "reuse_by_environment"),
|
||||
eq(environmentLeases.status, input.leaseRow.status),
|
||||
input.scopeCondition,
|
||||
sql`coalesce(${environmentLeases.metadata}->>'pendingCleanupInFlight', 'false') != 'true'`,
|
||||
@@ -3732,6 +3739,22 @@ export function environmentRuntimeService(
|
||||
return {
|
||||
getDriver,
|
||||
|
||||
async destroyRemoteWorkspaceLease(input: {
|
||||
companyId: string; workspaceId: string; leaseId: string; environmentId: string; providerLeaseId: string;
|
||||
}): Promise<EnvironmentLease | null> {
|
||||
const scope = and(eq(environmentLeases.executionWorkspaceId, input.workspaceId),
|
||||
eq(environmentLeases.providerLeaseId, input.providerLeaseId));
|
||||
const [leaseRow] = await db.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, input.leaseId),
|
||||
eq(environmentLeases.environmentId, input.environmentId), scope,
|
||||
));
|
||||
if (!leaseRow) throw new Error("remote_workspace_cleanup_lease_changed");
|
||||
const environment = await environmentsSvc.getById(input.environmentId);
|
||||
if (!environment || environment.driver !== "sandbox") throw new Error("remote_workspace_cleanup_environment_unavailable");
|
||||
return destroyClaimedSandboxLease({ environment, leaseRow, scopeCondition: scope, anyLeasePolicy: true,
|
||||
failureReason: "execution_workspace_closed" });
|
||||
},
|
||||
|
||||
/**
|
||||
* Read the sandbox duplex bridge kill switch for a new run. The host calls it
|
||||
* per run before it selects the callback bridge transport. It reads the
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createReusableLeaseHandoff, preserveReusableLeaseHandoff, withoutReusableLeaseHandoff } from "./environment-lease-handoff.js";
|
||||
import { clearRemoteWorkspaceLocationForReleasedLease } from "./remote-workspace-snapshot-retention.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
|
||||
import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
|
||||
@@ -7,7 +8,7 @@ import { withoutCleanupVerification } from "./cleanup-verification-proof.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import { SANDBOX_PLATFORM_PROBE_METADATA_KEY } from "./environment-custom-image-runtime.js";
|
||||
import { and, desc, eq, inArray, isNull, ne, sql, type SQL } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, isNull, ne, sql, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -1695,6 +1696,16 @@ export function environmentService(db: Db) {
|
||||
const cleanupSettled = options?.expectedPendingCleanupAttemptId && options.cleanupStatus === "success"
|
||||
&& status !== "pending_cleanup";
|
||||
const row = await db.transaction(async tx => {
|
||||
if (options?.remoteExecutionTermination?.state === "destroyed") {
|
||||
// Workspace preparation locks its authority before the lease. Keep
|
||||
// that order when confirmed destruction clears the same authority.
|
||||
const [identity] = await tx.select({ companyId: environmentLeases.companyId }).from(environmentLeases)
|
||||
.where(eq(environmentLeases.id, id));
|
||||
if (identity) await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, identity.companyId),
|
||||
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->'location'->>'leaseId' = ${id}`,
|
||||
)).orderBy(asc(executionWorkspaces.id)).for("update");
|
||||
}
|
||||
const condition = and(eq(environmentLeases.id, id), options?.expectedPendingCleanupAttemptId
|
||||
? and(eq(environmentLeases.status, "pending_cleanup"),
|
||||
sql`${environmentLeases.metadata}->>'pendingCleanupAttemptId' = ${options.expectedPendingCleanupAttemptId}`)
|
||||
@@ -1724,7 +1735,7 @@ export function environmentService(db: Db) {
|
||||
.where(and(eq(sandboxCreationOperations.id, previous.creationOperationId), eq(sandboxCreationOperations.companyId, previous.companyId),
|
||||
eq(sandboxCreationOperations.leaseId, id), eq(sandboxCreationOperations.state, "cleanup_pending")));
|
||||
}
|
||||
return tx.update(environmentLeases)
|
||||
const released = await tx.update(environmentLeases)
|
||||
.set({
|
||||
status,
|
||||
releasedAt: status === "retained" ? null : now,
|
||||
@@ -1741,6 +1752,8 @@ export function environmentService(db: Db) {
|
||||
.where(condition)
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (released) await clearRemoteWorkspaceLocationForReleasedLease({ db: tx, lease: toEnvironmentLease(released) });
|
||||
return released;
|
||||
});
|
||||
return row ? toEnvironmentLease(row) : null;
|
||||
},
|
||||
|
||||
@@ -20,7 +20,8 @@ export async function notifyExecutionRunWork(transaction: object, scope?: Execut
|
||||
await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
|
||||
}
|
||||
for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
|
||||
DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors,
|
||||
DELIVERY_QUEUES.remoteWorkspaceSnapshots]) {
|
||||
await notifyDeliveryWork(transaction, queue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { measureRunTransaction } from "./execution-transaction-timing.js";
|
||||
import { prepareRemoteWorkspaceRepository, readRemoteWorkspaceAuthority } from "./remote-workspaces.js";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { EnvironmentLease } from "@paperclipai/shared";
|
||||
import { beginRunPhase, measureRunPhase } from "./execution-preparation-timing.js";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
@@ -196,9 +199,12 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
return measureRunPhase("workspace.repository.binding", async () => {
|
||||
const [issue] = await measureRunPhase("workspace.repository.db.issue", () => reader.select().from(issues).where(and(eq(issues.id, input.issueId), eq(issues.companyId, input.companyId))));
|
||||
const [workspace] = await measureRunPhase("workspace.repository.db.workspace", () => reader.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, input.workspaceId), eq(executionWorkspaces.companyId, input.companyId))));
|
||||
const remoteAuthority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
if (!issue || !workspace || issue.executionWorkspaceId !== workspace.id || !workspace.cwd
|
||||
|| (verifyDirectory && await measureRunPhase("workspace.repository.paths", async () =>
|
||||
await canonicalRepositoryRoot(workspace.cwd!) !== await canonicalRepositoryRoot(input.cwd)))) {
|
||||
|| (verifyDirectory && (remoteAuthority
|
||||
? workspace.cwd !== input.cwd || remoteAuthority.writerRunId !== input.runId
|
||||
: await measureRunPhase("workspace.repository.paths", async () =>
|
||||
await canonicalRepositoryRoot(workspace.cwd!) !== await canonicalRepositoryRoot(input.cwd))))) {
|
||||
throw forbidden("Task repository publication binding is unavailable");
|
||||
}
|
||||
const acquisitionRestricted = await requiresAnonymousAcquisition(reader, { issue, workspace, agentId: input.agentId, runId: input.runId });
|
||||
@@ -318,7 +324,8 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
if (pin && pin !== candidatePin) throw conflict("Shared repository publication receipts disagree; repair is required before reuse");
|
||||
return candidatePin;
|
||||
}
|
||||
async function prepareForAdmission(input: { companyId: string; issueId: string; workspaceId: string; cwd: string; agentId: string; runId: string; responsibleUserId: string | null }) {
|
||||
async function prepareForAdmission(input: { companyId: string; issueId: string; workspaceId: string; cwd: string; agentId: string; runId: string; responsibleUserId: string | null;
|
||||
remoteTarget?: AdapterExecutionTarget | null; remoteLease?: EnvironmentLease; assertOwned?: () => Promise<void> }) {
|
||||
const { acquisitionRestricted, workspace } = await publicationBinding(db, input, false);
|
||||
const rows = await list(input.companyId, input.workspaceId);
|
||||
if (!rows.length) return [];
|
||||
@@ -326,6 +333,54 @@ export function executionWorkspaceRepositoryService(db: Db) {
|
||||
if (acquisitionRestricted && rows.some(row => row.state !== "ready" || !row.pinnedCommit || !/^[a-f0-9]{40,64}$/.test(row.pinnedCommit))) {
|
||||
throw forbidden(controllerAcquisitionMessage);
|
||||
}
|
||||
if (readRemoteWorkspaceAuthority(workspace.metadata)) {
|
||||
if (!input.remoteTarget || !input.remoteLease) throw conflict("Remote repository preparation requires its bound sandbox");
|
||||
const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board");
|
||||
const resolveAuth = createGitRemoteAuthProvider(db, input.companyId, { issueId: input.issueId, heartbeatRunId: input.runId,
|
||||
agentId: input.agentId, responsibleUserId: input.responsibleUserId });
|
||||
const prepared: Array<{ id: string; cwd: string; repoUrl: string; relativePath: string; pinnedCommit: string; branchName: string | null }> = [];
|
||||
for (const row of rows) {
|
||||
assertAcquisitionReceipt(row, acquisitionRestricted);
|
||||
const source = resolveAcquisitionSource(row.catalogRepositoryId ? { kind: "catalog", id: row.catalogRepositoryId }
|
||||
: { kind: "url", url: row.repoUrl }, available, acquisitionRestricted);
|
||||
if (source.repositoryIdentity !== row.repositoryIdentity || row.relativePath !== taskRepositoryRelativePath(row.repositoryIdentity)) {
|
||||
throw conflict("Repository preparation receipt does not match its source");
|
||||
}
|
||||
const verify = async () => {
|
||||
await input.assertOwned?.();
|
||||
await publicationBinding(db, input);
|
||||
if (!acquisitionRestricted) await assertCurrentControllerAcquisition(input);
|
||||
if (source.catalogRepositoryId) {
|
||||
const current = await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board");
|
||||
const authorized = resolveAcquisitionSource({ kind: "catalog", id: source.catalogRepositoryId }, current, false);
|
||||
if (authorized.repositoryIdentity !== row.repositoryIdentity) throw conflict("Repository catalog authorization changed");
|
||||
}
|
||||
};
|
||||
await verify();
|
||||
const anonymous = buildGitAuthInvocation({ token: "", source: "managed_connection", secretName: null });
|
||||
anonymous.env = { ...anonymous.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_SYSTEM: "/dev/null" };
|
||||
try {
|
||||
const result = await prepareRemoteWorkspaceRepository({
|
||||
db, workspace, runId: input.runId, lease: input.remoteLease, target: input.remoteTarget,
|
||||
repository: { id: row.id, relativePath: row.relativePath, repoUrl: row.repoUrl,
|
||||
ref: row.requestedRef === "HEAD" ? null : row.requestedRef,
|
||||
...(row.pinnedCommit ? { pinnedCommit: row.pinnedCommit } : {}) },
|
||||
assertOwned: verify, verifyOnly: acquisitionRestricted,
|
||||
resolveGitAuth: source.catalogRepositoryId ? resolveAuth : async () => anonymous,
|
||||
});
|
||||
await verify();
|
||||
await db.update(executionWorkspaceRepositories).set({ state: "ready", pinnedCommit: result.pinnedCommit,
|
||||
branchName: result.branchName, failureCode: null, updatedAt: new Date() })
|
||||
.where(and(eq(executionWorkspaceRepositories.id, row.id), eq(executionWorkspaceRepositories.companyId, input.companyId)));
|
||||
prepared.push({ ...result, id: row.id, repoUrl: row.repoUrl, relativePath: row.relativePath });
|
||||
} catch (error) {
|
||||
await db.update(executionWorkspaceRepositories).set({ state: "needs_repair", failureCode: "repository_preparation_failed", updatedAt: new Date() })
|
||||
.where(and(eq(executionWorkspaceRepositories.id, row.id), eq(executionWorkspaceRepositories.companyId, input.companyId)));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return prepared;
|
||||
}
|
||||
const root = await measureRunPhase("workspace.repository.paths", () => fs.realpath(input.cwd));
|
||||
const available = acquisitionRestricted ? [] : await catalog(input.companyId, input.responsibleUserId, input.responsibleUserId === "local-board");
|
||||
const resolveAuth = createGitRemoteAuthProvider(db, input.companyId, { issueId: input.issueId, heartbeatRunId: input.runId, agentId: input.agentId, responsibleUserId: input.responsibleUserId });
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { cleanupRemoteWorkspaceAllocation, inspectRemoteWorkspacePersistence, type DestroyRemoteWorkspaceLease } from "./remote-workspace-lifecycle.js";
|
||||
import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, pluginWorkspaceSupportsMergeAncestry, type WorkspaceWorkers } from "./plugin-workspaces.js";
|
||||
import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
import { measureRunTransaction } from "./execution-transaction-timing.js";
|
||||
@@ -264,12 +266,25 @@ function preserveWorkspaceMetadataForPatch(
|
||||
current: typeof executionWorkspaces.$inferSelect, patch: Partial<typeof executionWorkspaces.$inferInsert>,
|
||||
) {
|
||||
const nextPatch = { ...patch };
|
||||
if (readRemoteWorkspaceAuthority(current.metadata)) {
|
||||
for (const key of workspaceSourceKeys) {
|
||||
if (patch[key] !== undefined && patch[key] !== current[key]) throw conflict("Remote workspace coordinates belong to its sandbox storage");
|
||||
}
|
||||
}
|
||||
const remoteAuthorityKeys = ["remoteWorkspace", "remoteWorkspaceMigration", "remoteWorkspaceRecoverySourceRunIds", "remoteWorkspaceRecoveredFiles"] as const;
|
||||
if (patch.metadata !== undefined && remoteAuthorityKeys.some(key => current.metadata?.[key] !== undefined || patch.metadata?.[key] !== undefined)) {
|
||||
nextPatch.metadata = { ...patch.metadata };
|
||||
for (const key of remoteAuthorityKeys) {
|
||||
if (current.metadata?.[key] !== undefined) nextPatch.metadata[key] = current.metadata[key];
|
||||
else delete nextPatch.metadata[key];
|
||||
}
|
||||
}
|
||||
if (current.providerType === "plugin") {
|
||||
for (const key of workspaceSourceKeys) {
|
||||
if (patch[key] !== undefined && patch[key] !== current[key]) throw conflict("Managed workspace coordinates belong to its storage plugin");
|
||||
}
|
||||
if (patch.metadata !== undefined) {
|
||||
nextPatch.metadata = { ...patch.metadata };
|
||||
nextPatch.metadata = { ...nextPatch.metadata };
|
||||
for (const key of ["workspaceProvider", "workspaceProviderOwner", "workspaceProviderSource", "workspaceProviderBaseSha", "workspaceProviderRemoved", "workspaceProviderBranchMode", "workspaceProviderRemoval", "workspaceProviderRetired", "workspaceProviderRestoration", "lifecycleGeneration", "reopenPendingConsumption", "reopenPendingConsumptionSince"] as const) {
|
||||
if (current.metadata?.[key] !== undefined) nextPatch.metadata[key] = current.metadata[key];
|
||||
else delete nextPatch.metadata[key];
|
||||
@@ -313,6 +328,7 @@ export type ReopenClosedIsolatedExecutionWorkspaceResult =
|
||||
|
||||
export type ExecutionWorkspaceServiceOptions = {
|
||||
pluginWorkerManager?: WorkspaceWorkers;
|
||||
destroyRemoteWorkspaceLease?: DestroyRemoteWorkspaceLease;
|
||||
gitCacheExpirationEnabled?: boolean;
|
||||
gitBaseExpirationEnabled?: boolean;
|
||||
gitCacheExpirationDryRun?: boolean;
|
||||
@@ -1424,7 +1440,14 @@ const WORKSPACE_COMPACTION_TRASH_STALE_MS = 60 * 60 * 1000;
|
||||
const WORKSPACE_COMPACTION_TRASH_PURGE_INTERVAL_MS = 30 * 60 * 1000;
|
||||
|
||||
export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServiceOptions = {}) {
|
||||
const destroyRemoteLease: DestroyRemoteWorkspaceLease = input => opts.destroyRemoteWorkspaceLease
|
||||
? opts.destroyRemoteWorkspaceLease(input)
|
||||
: import("./environment-runtime.js").then(({ environmentRuntimeService }) => environmentRuntimeService(db).destroyRemoteWorkspaceLease(input));
|
||||
async function inspectWorkspace(workspace: ExecutionWorkspace) {
|
||||
if (readRemoteWorkspaceAuthority(workspace.metadata)) {
|
||||
const state = await inspectRemoteWorkspacePersistence(db, workspace);
|
||||
return { statusInspectionSucceeded: state.ready, warnings: state.reason ? [state.reason] : [], git: null };
|
||||
}
|
||||
if (workspace.providerType !== "plugin") return (opts.inspectGitCloseReadiness ?? inspectGitCloseReadiness)(workspace);
|
||||
const [row] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, workspace.id), eq(executionWorkspaces.companyId, workspace.companyId)));
|
||||
if (!row) throw notFound("Workspace is unavailable");
|
||||
@@ -2439,6 +2462,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
|
||||
async function runTerminalWorkspaceCleanup(workspace: ExecutionWorkspaceRow, expectedHeadSha: string | null) {
|
||||
if (readRemoteWorkspaceAuthority(workspace.metadata)) {
|
||||
if (await workspaceHasActiveRun(workspace)) throw conflict("Workspace still has active execution");
|
||||
return cleanupRemoteWorkspaceAllocation({ db, workspace, destroyLease: destroyRemoteLease });
|
||||
}
|
||||
if (workspace.providerType === "plugin") {
|
||||
if (await workspaceHasActiveRun(workspace)) throw conflict("Workspace still has active execution");
|
||||
await removePluginWorkspace(db, opts.pluginWorkerManager, workspace, expectedHeadSha);
|
||||
@@ -3020,6 +3047,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
return toExecutionWorkspace(row, (runtimeServices?.get(row.id) ?? []).map(toRuntimeService));
|
||||
},
|
||||
|
||||
cleanupRemoteStorage: async (workspace: ExecutionWorkspace) => cleanupRemoteWorkspaceAllocation({
|
||||
db, workspace, destroyLease: destroyRemoteLease,
|
||||
}),
|
||||
|
||||
getById: async (id: string) => {
|
||||
const row = await db
|
||||
.select()
|
||||
@@ -3116,8 +3147,10 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
const { deliveryState } = await assessDelivery(workspace, git);
|
||||
const warnings = [...gitWarnings];
|
||||
const blockingReasons: string[] = [];
|
||||
const remoteWorkspace = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!statusInspectionSucceeded) {
|
||||
blockingReasons.push("Paperclip could not verify the workspace git status. Retry before destructive cleanup.");
|
||||
blockingReasons.push(remoteWorkspace ? gitWarnings[0] ?? "The remote workspace save is incomplete."
|
||||
: "Paperclip could not verify the workspace git status. Retry before destructive cleanup.");
|
||||
}
|
||||
const isSharedWorkspace = executionWorkspace.mode === "shared_workspace";
|
||||
const workspacePath = readNullableString(executionWorkspace.providerRef) ?? readNullableString(executionWorkspace.cwd);
|
||||
@@ -3148,7 +3181,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
}
|
||||
|
||||
if (isSharedWorkspace) {
|
||||
if (isSharedWorkspace && !remoteWorkspace) {
|
||||
warnings.push("This shared workspace session points at project workspace infrastructure. Archiving it only removes the session record.");
|
||||
}
|
||||
|
||||
@@ -3230,12 +3263,12 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
},
|
||||
];
|
||||
for (const action of configuredCleanupCommands) {
|
||||
if (!action.command || executionWorkspace.providerType === "plugin") continue;
|
||||
if (!action.command || executionWorkspace.providerType === "plugin" || remoteWorkspace) continue;
|
||||
plannedActions.push(action);
|
||||
}
|
||||
|
||||
const teardownCommand = config?.teardownCommand ?? projectPolicy?.workspaceStrategy?.teardownCommand ?? null;
|
||||
if (teardownCommand && executionWorkspace.providerType !== "plugin") {
|
||||
if (teardownCommand && executionWorkspace.providerType !== "plugin" && !remoteWorkspace) {
|
||||
plannedActions.push({
|
||||
kind: "teardown_command",
|
||||
label: "Run teardown command",
|
||||
@@ -3248,6 +3281,13 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
plannedActions.push({ kind: "remove_provider_workspace", label: "Remove provider workspace",
|
||||
description: "The storage plugin will verify remote recovery and remove the workspace.", command: null });
|
||||
}
|
||||
if (remoteWorkspace) {
|
||||
plannedActions.push({ kind: "remove_provider_workspace", label: "Release remote workspace compute",
|
||||
description: "Release the sandbox and keep the saved workspace files.", command: null });
|
||||
if (configuredCleanupCommands.some(action => action.command) || teardownCommand) {
|
||||
blockingReasons.push("Remote workspace close does not support custom cleanup or teardown commands.");
|
||||
}
|
||||
}
|
||||
|
||||
if (executionWorkspace.providerType === "git_worktree" && workspacePath) {
|
||||
plannedActions.push({
|
||||
@@ -3258,7 +3298,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
});
|
||||
}
|
||||
|
||||
if (executionWorkspace.providerType !== "plugin" && git?.createdByRuntime && executionWorkspace.branchName) {
|
||||
if (executionWorkspace.providerType !== "plugin" && !remoteWorkspace && git?.createdByRuntime && executionWorkspace.branchName) {
|
||||
plannedActions.push({
|
||||
kind: "git_branch_delete",
|
||||
label: "Delete runtime-created branch",
|
||||
@@ -3347,6 +3387,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
or(and(inArray(executionWorkspaces.status, ["active", "idle", "in_review"]), isNull(executionWorkspaces.closedAt)),
|
||||
and(eq(executionWorkspaces.providerType, "plugin"), inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
|
||||
sql`(${executionWorkspaces.metadata}->>'workspaceProviderRemoved') IS DISTINCT FROM 'true'`),
|
||||
and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
|
||||
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->>'schema' = 'paperclip.remote-workspace.v1'`,
|
||||
sql`jsonb_typeof(${executionWorkspaces.metadata}->'remoteWorkspace'->'location') = 'object'`),
|
||||
opts.gitCacheExpirationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
|
||||
sql`${executionWorkspaces.metadata}->'gitCacheExpiration'->>'completed' = 'false'`) : undefined,
|
||||
opts.nestedCloneMigrationEnabled ? and(inArray(executionWorkspaces.status, ["archived", "cleanup_failed"]),
|
||||
@@ -3493,7 +3536,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
if (isClosedExecutionWorkspaceStatus(workspace.status) && !hasPendingGitExpiration(workspace)) continue;
|
||||
}
|
||||
if (opts.nestedCloneMigrationEnabled && workspace.providerType !== "plugin"
|
||||
if (opts.nestedCloneMigrationEnabled && workspace.providerType !== "plugin" && !readRemoteWorkspaceAuthority(workspace.metadata)
|
||||
&& (isNestedCloneMigrationCandidate(workspace) || hasPendingNestedCloneMigration(workspace))) {
|
||||
const nestedOnly = isClosedExecutionWorkspaceStatus(workspace.status) && !hasPendingGitExpiration(workspace)
|
||||
&& !hasPendingIsolatedRepositoryRehomeCleanup(workspace);
|
||||
@@ -3551,7 +3594,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
);
|
||||
if (reopenPending) terminalWorkspaceDeadlines.set(workspace.id, now().getTime());
|
||||
const treeOpen = issueTree.some(issue => !TERMINAL_ISSUE_STATUSES.has(issue.status));
|
||||
const gitExpirationCandidate = workspace.providerType !== "plugin" && Boolean(opts.gitCacheExpirationEnabled)
|
||||
const remoteWorkspace = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
const gitExpirationCandidate = workspace.providerType !== "plugin" && !remoteWorkspace && Boolean(opts.gitCacheExpirationEnabled)
|
||||
&& (hasPendingGitExpiration(workspace) || (workspace.mode === "isolated_workspace" && workspace.strategyType === "git_worktree"));
|
||||
const usingIssueIds = sourceIssue && gitExpirationCandidate && !reopenPending
|
||||
&& TERMINAL_ISSUE_STATUSES.has(sourceIssue.status)
|
||||
@@ -3566,8 +3610,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
});
|
||||
}
|
||||
const idleMs = opts.gitRestorePoints?.idleMs;
|
||||
const pendingExpiration = opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin" && hasPendingGitExpiration(workspace);
|
||||
if (pendingExpiration || (sourceIssue && idleMs && !reopenPending && opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin"
|
||||
const pendingExpiration = opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin" && !remoteWorkspace && hasPendingGitExpiration(workspace);
|
||||
if (pendingExpiration || (sourceIssue && idleMs && !reopenPending && opts.gitCacheExpirationEnabled && workspace.providerType !== "plugin" && !remoteWorkspace
|
||||
&& !isClosedExecutionWorkspaceStatus(workspace.status)
|
||||
&& workspace.mode === "isolated_workspace" && workspace.strategyType === "git_worktree")) {
|
||||
const lastActivityAt = pendingExpiration ? null : await gitWorkspaceLastActivityAt(db, workspace);
|
||||
@@ -3667,6 +3711,21 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
else result.skippedActiveRun += 1;
|
||||
continue;
|
||||
}
|
||||
if (remoteWorkspace && isClosedExecutionWorkspaceStatus(workspace.status)) {
|
||||
const generation = readExecutionWorkspaceLifecycleGeneration(workspace.metadata);
|
||||
try {
|
||||
const cleanup = await cleanupTerminalWorkspace(workspace, null, generation);
|
||||
if (cleanup.skippedReopened) result.skippedReopened += 1;
|
||||
else if (!cleanup.cleaned) result.cleanupFailed += 1;
|
||||
else { result.archived += 1; terminalWorkspaceDeadlines.delete(workspace.id); }
|
||||
} catch (error) {
|
||||
result.cleanupFailed += 1;
|
||||
terminalWorkspaceDeadlines.set(workspace.id, now().getTime() + TERMINAL_WORKSPACE_RETRY_MS);
|
||||
await markTerminalCleanupFailedFenced({ workspaceId: workspace.id, capturedGeneration: generation,
|
||||
cleanupReason: `${ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON} | ${error instanceof Error ? error.message : String(error)}` });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let deliveryReferences: GitHubPullRequestReference[] | null = null;
|
||||
if (workspace.providerType === "plugin" && !isClosedExecutionWorkspaceStatus(workspace.status)
|
||||
&& !reopenPending && !workspace.metadata?.workspaceProviderRemoval && !workspace.metadata?.workspaceProviderRestoration) {
|
||||
@@ -3727,6 +3786,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
const { git, statusInspectionSucceeded } = await inspection;
|
||||
if (!statusInspectionSucceeded) {
|
||||
if (remoteWorkspace) terminalWorkspaceDeadlines.delete(workspace.id);
|
||||
result.skippedUndelivered += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -3755,6 +3815,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
assessment.deliveryState !== "merged_via_pr"
|
||||
&& assessment.deliveryState !== "merged_by_ancestry"
|
||||
) {
|
||||
if (remoteWorkspace) terminalWorkspaceDeadlines.delete(workspace.id);
|
||||
result.skippedUndelivered += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -3978,7 +4039,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
else result.archived += 1;
|
||||
} catch (error) {
|
||||
result.cleanupFailed += 1;
|
||||
if (archived.providerType === "plugin") terminalWorkspaceDeadlines.set(archived.id, now().getTime() + TERMINAL_WORKSPACE_RETRY_MS);
|
||||
if (archived.providerType === "plugin" || remoteWorkspace) terminalWorkspaceDeadlines.set(archived.id, now().getTime() + TERMINAL_WORKSPACE_RETRY_MS);
|
||||
const failure = error instanceof Error ? error.message : String(error);
|
||||
// Mark cleanup_failed only while the row is still closed at the
|
||||
// generation this sweep captured. A reopen that raced after the failure
|
||||
@@ -4112,6 +4173,24 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
if (!freshSnapshot || freshSnapshot.sourceIssueId !== sourceIssueId) {
|
||||
return { ok: false, code: "not_reopenable", message: "Execution workspace source changed during reopen" };
|
||||
}
|
||||
if (readRemoteWorkspaceAuthority(freshSnapshot.metadata)) {
|
||||
return withExecutionWorkspaceLifecycleLock(db, freshSnapshot.id, async tx => {
|
||||
const [row] = await tx.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, issue.companyId), eq(executionWorkspaces.id, freshSnapshot.id)));
|
||||
if (!row || row.sourceIssueId !== sourceIssueId) return { ok: false, code: "not_reopenable", message: "Workspace source changed" };
|
||||
await assertTaskWorkspaceExecutionAccess(tx, input.authorizationActor, issue.companyId, { issueId: issue.id, workspaceId: row.id });
|
||||
const persistence = await inspectRemoteWorkspacePersistence(tx, row);
|
||||
if (!persistence.ready) return { ok: false, code: "rebuild_failed", message: persistence.reason ?? "Saved workspace is unavailable" };
|
||||
const generation = readExecutionWorkspaceLifecycleGeneration(row.metadata);
|
||||
if (!isClosedExecutionWorkspaceStatus(row.status)) return { ok: true, reopened: false, workspace: toExecutionWorkspace(row), generation };
|
||||
const [active] = await tx.update(executionWorkspaces).set({ status: "active", closedAt: null,
|
||||
cleanupReason: null, cleanupEligibleAt: null, lastUsedAt: now(), updatedAt: now(),
|
||||
metadata: setMetadataReopenPendingConsumption(bumpExecutionWorkspaceLifecycleGeneration(row.metadata), now()),
|
||||
}).where(eq(executionWorkspaces.id, row.id)).returning();
|
||||
await notifyTerminalWorkspaceWork(tx, `workspace:${row.id}`);
|
||||
return { ok: true, reopened: true, workspace: toExecutionWorkspace(active!), generation: generation + 1 };
|
||||
});
|
||||
}
|
||||
const snapshot = freshSnapshot.providerType === "plugin" ? freshSnapshot : await rehomeExpiredIsolatedRestoration(db, freshSnapshot);
|
||||
if (snapshot.providerType === "plugin") {
|
||||
return withExecutionWorkspaceLifecycleLock(db, snapshot.id, async tx => {
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { asc, and, eq } from "drizzle-orm";
|
||||
import { projectWorkspaces, type Db } from "@paperclipai/db";
|
||||
import type { TaskWorkspaceIntent } from "@paperclipai/shared";
|
||||
import { assertTaskWorkspaceExecutionAccess } from "../task-workspace-access.js";
|
||||
import { managedProjectCheckoutName, projectRepositoryIdentity } from "../managed-project-checkout-name.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { projectService } from "../projects.js";
|
||||
import { authorizationService } from "../authorization.js";
|
||||
import { readRemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "../remote-workspaces.js";
|
||||
import { isMultiProjectWorkspaceSyncEnabled, isMultiProjectWorkspaceSyncRemoteEnabled,
|
||||
resolveRunReferencedProjects, WorkspaceValidationFailure, type ResolvedWorkspaceForRun } from "./workspaces.js";
|
||||
import type { HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";
|
||||
|
||||
function remoteRepositorySource(repoUrl: string): boolean {
|
||||
const value = repoUrl.trim();
|
||||
if (!value || /[\x00-\x1f\x7f]/.test(value)) throw new WorkspaceValidationFailure(
|
||||
"The repository source is invalid.", { reason: "remote_workspace_source_invalid" });
|
||||
if (/^[a-z]:/i.test(value) || /^file:/i.test(value)) return false;
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
if (["https:", "http:", "ssh:", "git:", "git+ssh:"].includes(parsed.protocol) && parsed.hostname) return true;
|
||||
} catch { /* Git also accepts scp-style SSH sources and local paths. */ }
|
||||
if (!value.includes("://") && /^(?:[^@:/\s]+@)?(?:\[[a-f\d:]+\]|[a-z\d.-]+):[^:\s][^\r\n]*$/i.test(value)) return true;
|
||||
if (!/^[a-z][a-z\d+.-]*:/i.test(value)) return false;
|
||||
throw new WorkspaceValidationFailure("The repository source is invalid.", { reason: "remote_workspace_source_invalid" });
|
||||
}
|
||||
|
||||
type RemoteWorkspaceSourceResolution = { authority: "remote"; source: RemoteWorkspaceSource;
|
||||
projectId: string | null; projectWorkspaceId: string | null; warnings?: string[];
|
||||
referencedProjectFailures?: ResolvedWorkspaceForRun["referencedProjectFailures"] } | { authority: "host" };
|
||||
|
||||
/** Resolve an authorized source without materializing any controller directory. */
|
||||
export async function resolveRemoteWorkspaceSource(db: Db, input: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId: string;
|
||||
runId: string;
|
||||
responsibleUserId: string | null;
|
||||
projectId: string | null;
|
||||
projectWorkspaceId: string | null;
|
||||
selection?: TaskWorkspaceIntent | null;
|
||||
existingWorkspace?: HeartbeatWorkspacePreparationInput["reuse"]["existingExecutionWorkspace"];
|
||||
taskDirectory?: boolean;
|
||||
configuredCwd?: string | null;
|
||||
automaticEnrollment?: boolean;
|
||||
ref?: string | null;
|
||||
branchName?: string | null;
|
||||
branchMode?: "create" | "existing";
|
||||
}): Promise<RemoteWorkspaceSourceResolution> {
|
||||
const selected = await assertTaskWorkspaceExecutionAccess(db, {
|
||||
type: "agent", source: "agent_jwt", companyId: input.companyId,
|
||||
agentId: input.agentId, runId: input.runId,
|
||||
onBehalfOfUserId: input.responsibleUserId === "local-board" ? null : input.responsibleUserId,
|
||||
}, input.companyId, { issueId: input.issueId,
|
||||
...(!input.selection && input.existingWorkspace ? { workspaceId: input.existingWorkspace.id } : {}),
|
||||
...(input.selection ? { persistedSelection: input.selection.selection } : {}),
|
||||
});
|
||||
const hostSource = (): { authority: "host" } => {
|
||||
if (input.automaticEnrollment) return { authority: "host" };
|
||||
throw new WorkspaceValidationFailure("Import local repository folders before using them as remote workspace sources.",
|
||||
{ reason: "remote_workspace_local_source_requires_import" });
|
||||
};
|
||||
const withReferences = async (base: RemoteWorkspaceSource, anchorProjectId: string | null,
|
||||
sourceId: string | null, savedSource = false): Promise<RemoteWorkspaceSourceResolution> => {
|
||||
const extra: NonNullable<RemoteWorkspaceSource["repositories"]> =
|
||||
(base.repositories ?? []).filter(repository => !repository.readOnly);
|
||||
// A verified saved archive owns these files after import. Its original Git
|
||||
// source remains an identity and need not be reachable from the new sandbox.
|
||||
if (!savedSource) {
|
||||
if (base.kind === "git" && !remoteRepositorySource(base.repoUrl)) return hostSource();
|
||||
for (const repository of extra) if (!remoteRepositorySource(repository.repoUrl)) return hostSource();
|
||||
}
|
||||
const referenced = isMultiProjectWorkspaceSyncEnabled() ? await resolveRunReferencedProjects(input.issueId, anchorProjectId, {
|
||||
companyId: input.companyId,
|
||||
actor: { type: "agent", source: "agent_jwt", companyId: input.companyId,
|
||||
agentId: input.agentId, runId: input.runId,
|
||||
onBehalfOfUserId: input.responsibleUserId === "local-board" ? null : input.responsibleUserId },
|
||||
issues: issueService(db), projects: projectService(db), access: authorizationService(db),
|
||||
}) : { additional: [], warnings: [], failures: [] };
|
||||
if (referenced.additional.length > 0) {
|
||||
if (!isMultiProjectWorkspaceSyncRemoteEnabled()) throw new WorkspaceValidationFailure(
|
||||
"Referenced remote workspace access is disabled.", { reason: "remote_referenced_workspace_disabled" });
|
||||
// Native sandbox providers currently run with external sandbox authority.
|
||||
// They cannot enforce a separate read-only filesystem root for a project.
|
||||
throw new WorkspaceValidationFailure("This remote runtime cannot safely provide read-only project folders.",
|
||||
{ reason: "remote_workspace_readonly_sources_unsupported" });
|
||||
}
|
||||
return { authority: "remote", source: { ...base, ...(extra.length ? { repositories: extra } : { repositories: undefined }) },
|
||||
projectId: anchorProjectId, projectWorkspaceId: sourceId,
|
||||
warnings: referenced.warnings, referencedProjectFailures: referenced.failures };
|
||||
};
|
||||
const existing = readRemoteWorkspaceAuthority(input.existingWorkspace?.metadata);
|
||||
if (existing) return withReferences(existing.source, input.existingWorkspace!.projectId,
|
||||
input.existingWorkspace!.projectWorkspaceId, existing.snapshotId !== null);
|
||||
if (input.existingWorkspace && input.automaticEnrollment) return { authority: "host" };
|
||||
if (input.existingWorkspace) throw new WorkspaceValidationFailure(
|
||||
"The existing workspace must be imported before it can use remote workspace storage.",
|
||||
{ reason: "remote_workspace_migration_required" });
|
||||
if (input.taskDirectory || input.selection?.selection.kind === "task_directory") {
|
||||
return withReferences({ kind: "empty" }, null, null);
|
||||
}
|
||||
if (input.configuredCwd) return hostSource();
|
||||
const projectId = selected.task.projectId;
|
||||
const sourceId = input.selection?.selection.kind === "configured_source"
|
||||
? input.selection.selection.projectWorkspaceId : selected.source?.id ?? selected.task.projectWorkspaceId;
|
||||
const [source] = sourceId ? await db.select().from(projectWorkspaces).where(and(
|
||||
eq(projectWorkspaces.id, sourceId), eq(projectWorkspaces.companyId, input.companyId),
|
||||
)) : projectId ? await db.select().from(projectWorkspaces).where(and(
|
||||
eq(projectWorkspaces.projectId, projectId), eq(projectWorkspaces.companyId, input.companyId),
|
||||
)).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id)).limit(1) : [];
|
||||
if (!source) {
|
||||
if (sourceId || projectId) throw new WorkspaceValidationFailure(
|
||||
"The selected project has no repository source for a remote workspace.",
|
||||
{ reason: "remote_workspace_source_unavailable" });
|
||||
return withReferences({ kind: "empty" }, null, null);
|
||||
}
|
||||
// Recheck authorization against the concrete configured source; the earliest
|
||||
// project default may have been selected after the initial task read.
|
||||
await assertTaskWorkspaceExecutionAccess(db, {
|
||||
type: "agent", source: "agent_jwt", companyId: input.companyId,
|
||||
agentId: input.agentId, runId: input.runId,
|
||||
onBehalfOfUserId: input.responsibleUserId === "local-board" ? null : input.responsibleUserId,
|
||||
}, input.companyId, { issueId: input.issueId,
|
||||
...(selected.source ? { projectWorkspaceId: source.id } : {}),
|
||||
});
|
||||
if (source.sourceType === "remote_managed") return hostSource();
|
||||
if (source.cwd && source.cwd !== "/__paperclip_repo_only__") return hostSource();
|
||||
if (!source.repoUrl) throw new WorkspaceValidationFailure(
|
||||
"The selected workspace has no repository URL.", { reason: "remote_workspace_source_unavailable" });
|
||||
if (!remoteRepositorySource(source.repoUrl)) return hostSource();
|
||||
const sameProject = await db.select().from(projectWorkspaces).where(and(
|
||||
eq(projectWorkspaces.companyId, input.companyId), eq(projectWorkspaces.projectId, source.projectId),
|
||||
)).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id));
|
||||
const repositories: NonNullable<RemoteWorkspaceSource["repositories"]> = [];
|
||||
const identities = new Set([projectRepositoryIdentity(source.repoUrl)]);
|
||||
for (const repository of sameProject) {
|
||||
if (!repository.repoUrl || identities.has(projectRepositoryIdentity(repository.repoUrl))) continue;
|
||||
if (repository.sourceType === "remote_managed") return hostSource();
|
||||
if (repository.cwd && repository.cwd !== "/__paperclip_repo_only__") return hostSource();
|
||||
if (!remoteRepositorySource(repository.repoUrl)) return hostSource();
|
||||
identities.add(projectRepositoryIdentity(repository.repoUrl));
|
||||
repositories.push({ id: repository.id, relativePath: `.paperclip-repositories/${managedProjectCheckoutName({
|
||||
id: repository.id, repoUrl: repository.repoUrl, repoRef: repository.repoRef,
|
||||
})}`, repoUrl: repository.repoUrl, ref: repository.repoRef, projectId: source.projectId,
|
||||
projectWorkspaceId: repository.id });
|
||||
}
|
||||
return withReferences({ kind: "git", repoUrl: source.repoUrl,
|
||||
ref: input.ref ?? source.repoRef ?? source.defaultRef ?? null,
|
||||
...(input.branchName ? { branchName: input.branchName } : {}),
|
||||
...(input.branchMode ? { branchMode: input.branchMode } : {}),
|
||||
...(repositories.length ? { repositories } : {}),
|
||||
}, source.projectId, source.id);
|
||||
}
|
||||
@@ -2,6 +2,8 @@ import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-c
|
||||
|
||||
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
|
||||
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js";
|
||||
import { environmentService } from "../environments.js";
|
||||
import { writeExecutionControl } from "../execution-control-notifications.js";
|
||||
import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js";
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
@@ -235,6 +237,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
const { issueRef, issueContext, context, issueId, taskKey, executionContinuation } = input.task;
|
||||
const { nativeRuntimeResolution, nativeExecution, nativeRunnerInstanceId, adapter, runtimeForAdapter, getNativeFreshSessionHandoff, getFreshSessionHandoff, runOptions } = input.runtime;
|
||||
const { persistedExecutionWorkspace, executionWorkspace, executionTarget, workspaceOperationRecorder, nativeWorkspaceSync, workspaceRestoreSource, remoteExecution } = input.workspace;
|
||||
const remoteWorkspaceAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata);
|
||||
const { runtimeConfig, resolvedConfig, resolvedInstanceSettings, configuredTaskEnvironment, adapterEnv, agentIdentity, observability, managedAiRuntime, runtimeEnv, useHostGitHub, githubSelection } = input.config;
|
||||
const { runtimeSessionParamsForAdapter, configuredModel, sessionConfigMetadata, goalCheckpointSession } = input.session;
|
||||
const { nativeInstructionWorkingCopy, collectStoppedInstructions, nativeInstructionReservation, releaseInstructionCopy } = input.instructions;
|
||||
@@ -290,7 +293,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
pluginWorkspaceFinalization = workspaceRecord
|
||||
? await inspectPluginWorkspaceForFinalization(db, options.pluginWorkerManager, workspaceRecord, agent.id, run.id)
|
||||
: null;
|
||||
if (pluginWorkspaceFinalization || workspaceRecord?.strategyType !== "git_worktree") return null;
|
||||
if (pluginWorkspaceFinalization || readRemoteWorkspaceAuthority(workspaceRecord?.metadata)
|
||||
|| workspaceRecord?.strategyType !== "git_worktree") return null;
|
||||
|
||||
const worktreePath =
|
||||
readNonEmptyString(workspaceRecord.providerRef) ??
|
||||
@@ -960,6 +964,19 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
// finalize row.
|
||||
const completeWorkspace = async (ownership?: NativeWorkspaceFinalizationOwnership) => {
|
||||
try {
|
||||
if (remoteWorkspaceAuthority && persistedExecutionWorkspace) {
|
||||
if (!ownership || !executionTarget) throw new Error("remote_workspace_ownership_required");
|
||||
const current = await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id);
|
||||
const authority = readRemoteWorkspaceAuthority(current?.metadata);
|
||||
const lease = authority?.location
|
||||
? await environmentService(db).getLeaseById(authority.location.leaseId) : null;
|
||||
if (!current || !lease || lease.companyId !== run.companyId || lease.heartbeatRunId !== run.id) {
|
||||
throw new Error("workspace_sync_out_unrecoverable");
|
||||
}
|
||||
await recordWorkspaceFinalize("succeeded", { owningService: "native_workspace_finalizer", storage: "remote" },
|
||||
async () => { await saveRemoteWorkspace({ db, workspace: current, runId: run.id, lease,
|
||||
target: executionTarget, assertOwned: ownership.assertHeld }); });
|
||||
}
|
||||
if (nativeWorkspaceSync) {
|
||||
const exported = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, run.companyId),
|
||||
@@ -994,7 +1011,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
db,
|
||||
runId: run.id,
|
||||
workspaceFinalizeStatus,
|
||||
preserveProviderAttempt: Boolean(nativeWorkspaceSync),
|
||||
preserveProviderAttempt: Boolean(nativeWorkspaceSync || remoteWorkspaceAuthority),
|
||||
}));
|
||||
await dispatchPendingNativeStatusWakeups({
|
||||
companyId: run.companyId,
|
||||
@@ -1017,7 +1034,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
if (nativeWorkspaceSync) {
|
||||
if (nativeWorkspaceSync || remoteWorkspaceAuthority) {
|
||||
const owned = await measureRunPhase("execution.workspace_ownership", () => withNativeWorkspaceFinalizationOwnership({
|
||||
db, companyId: run.companyId, runId: run.id,
|
||||
}, completeWorkspace));
|
||||
@@ -1051,7 +1068,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
input.effects.onNativeOwnershipHeld();
|
||||
throw adapterErr;
|
||||
}
|
||||
if (nativeRuntimeResolution.kind === "native" && nativeWorkspaceSync?.reference.state === "prepared"
|
||||
if (nativeRuntimeResolution.kind === "native" && (nativeWorkspaceSync?.reference.state === "prepared" || remoteWorkspaceAuthority?.dirty)
|
||||
&& !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) {
|
||||
input.effects.onProviderResourceDisposition("stop_and_retain");
|
||||
}
|
||||
@@ -1144,7 +1161,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
.where(eq(nativeRunFinalizations.runId, run.id))
|
||||
.limit(1)
|
||||
.then((rows) => rows[0]?.resultId ?? null);
|
||||
if (proposedResult && nativeWorkspaceSync) {
|
||||
if (proposedResult && (nativeWorkspaceSync || remoteWorkspaceAuthority)) {
|
||||
const workspaceFailureMessage =
|
||||
adapterErr instanceof Error ? adapterErr.message : "";
|
||||
const unrecoverable =
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { museRunnerBroker } from "../muse-runner-broker.js";
|
||||
import { readRemoteWorkspaceAuthority } from "../remote-workspaces.js";
|
||||
import { measureRunTransaction } from "../execution-transaction-timing.js";
|
||||
import { measureRunPhase } from "../execution-preparation-timing.js";
|
||||
import {
|
||||
@@ -118,7 +119,7 @@ export interface HeartbeatRuntimeSelectionInput {
|
||||
getFreshSessionHandoff: (() => Promise<string | null>) | undefined;
|
||||
};
|
||||
workspace: {
|
||||
persistedExecutionWorkspace: { id: string } | null;
|
||||
persistedExecutionWorkspace: { id: string; metadata?: Record<string, unknown> | null } | null;
|
||||
executionWorkspace: Omit<Parameters<typeof buildNativeExecutionInput>[0]["workspace"], "id">;
|
||||
executionTarget: Parameters<typeof prepareNativeWorkspaceSync>[0]["target"];
|
||||
activeEnvironmentLease: Pick<Parameters<typeof prepareNativeWorkspaceSync>[0], "lease">;
|
||||
@@ -1047,7 +1048,8 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
});
|
||||
if (!nativeSelected) return { selected: false };
|
||||
stopControllerLease();
|
||||
nativeWorkspaceSync = await measureRunPhase("runtime_selection.workspace_sync", () => prepareNativeWorkspaceSync({
|
||||
nativeWorkspaceSync = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata) ? null
|
||||
: await measureRunPhase("runtime_selection.workspace_sync", () => prepareNativeWorkspaceSync({
|
||||
db,
|
||||
runId: run.id,
|
||||
companyId: agent.companyId,
|
||||
|
||||
@@ -120,10 +120,33 @@ postgresTests("upgrade readiness persisted execution evidence", () => {
|
||||
expect((await readUpgradeReadiness(db, input)).ready).toBe(false);
|
||||
await db.update(nativeRunFinalizations).set({ ...lease, failureCode: null }).where(eq(nativeRunFinalizations.runId, runId));
|
||||
}
|
||||
// Remote-authoritative runs retain their logical workspace identity without a host sync record.
|
||||
const remoteWorkspace = { schema: "paperclip.remote-workspace-run.v1", workspaceId: workspace.workspaceId,
|
||||
leaseId: workspace.leaseId, providerLeaseId: workspace.providerLeaseId, remoteCwd: workspace.remoteCwd,
|
||||
epoch: 1, state: "prepared", snapshotId: null };
|
||||
const remoteProfile = { remoteWorkspace };
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: remoteProfile }).where(eq(heartbeatRuns.id, runId));
|
||||
expect(await readUpgradeReadiness(db, input)).toMatchObject({ ready: true, recoverableRuns: 1, blockingRuns: 0 });
|
||||
for (const remote of [{ ...remoteWorkspace, state: "saved", snapshotId: randomUUID() },
|
||||
remoteWorkspace]) {
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { remoteWorkspace: remote,
|
||||
nativeWorkspaceSync: { ...workspace, workspaceId: randomUUID() } } }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await readUpgradeReadiness(db, input)).ready).toBe(true); // remote identity takes precedence
|
||||
}
|
||||
for (const remote of [null, {}, { ...remoteWorkspace, schema: "unknown" }, { ...remoteWorkspace, epoch: 0 },
|
||||
{ ...remoteWorkspace, workspaceId: randomUUID() }, { ...remoteWorkspace, leaseId: "invalid" },
|
||||
{ ...remoteWorkspace, remoteCwd: "" }, { ...remoteWorkspace, snapshotId: "invalid" }]) {
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceSync: workspace,
|
||||
remoteWorkspace: remote } }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await readUpgradeReadiness(db, input)).ready).toBe(false); // no fallback to the legacy identity
|
||||
}
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: execution.runnerProfileJson }).where(eq(heartbeatRuns.id, runId));
|
||||
// Plugin task admission owns an environment lease identity and deliberately has no workspace-sync record.
|
||||
const pluginLeaseId = randomUUID();
|
||||
await db.insert(environmentLeases).values({ id: pluginLeaseId, companyId, heartbeatRunId: runId, issueId,
|
||||
provider: "plugin", providerLeaseId: "synthetic-task", metadata: { agentId, runnerTaskLease: true, runnerTaskDispatched: true } });
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: remoteProfile }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await readUpgradeReadiness(db, input)).ready).toBe(false); // contradictory task authorities cannot be adopted
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: {} }).where(eq(heartbeatRuns.id, runId));
|
||||
expect((await readUpgradeReadiness(db, input)).ready).toBe(false); // workspace identity is not plugin task authority
|
||||
const pluginHeader = JSON.stringify({ ...JSON.parse(header), state: { ...JSON.parse(header).state,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import { inspectNativeRunnerUpgradeHeader } from "../native-runtime/native-runner-checkpoint.js";
|
||||
import { readNativeWorkspaceSyncReference } from "../native-runtime/native-workspace-sync.js";
|
||||
import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../native-runtime/native-runner-recovery-contract.js";
|
||||
@@ -23,10 +24,12 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
const owner = await currentNativeControllerIdentity();
|
||||
const rows = await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local statement_timeout = '2s'`);
|
||||
return tx.execute<Candidate & { companyId: string; issueId: string; runnerId: string; sessionId: string; workspace: unknown; pluginLeaseId: string | null; material: Record<string, unknown> | null }>(sql`
|
||||
return tx.execute<Candidate & { companyId: string; issueId: string; runnerId: string; sessionId: string; workspace: unknown; remoteWorkspace: unknown; hasRemoteWorkspace: boolean; pluginLeaseId: string | null; material: Record<string, unknown> | null }>(sql`
|
||||
select r.id, r.company_id as "companyId", r.native_issue_id as "issueId",
|
||||
r.runner_instance_id as "runnerId", r.native_session_id as "sessionId",
|
||||
r.runner_profile_json->'nativeWorkspaceSync' as workspace, p.id as "pluginLeaseId",
|
||||
r.runner_profile_json->'nativeWorkspaceSync' as workspace,
|
||||
r.runner_profile_json->'remoteWorkspace' as "remoteWorkspace",
|
||||
r.runner_profile_json ? 'remoteWorkspace' as "hasRemoteWorkspace", p.id as "pluginLeaseId",
|
||||
case when pg_column_size(c.material) <= 262144 and sum(pg_column_size(c.material)) over () <= 8388608 then c.material else null end as material, (
|
||||
r.runtime_mode = 'native' and a.adapter_type = 'paperclip_runner'
|
||||
and r.runner_instance_id is not null and r.native_session_id is not null
|
||||
@@ -54,12 +57,17 @@ export async function readUpgradeReadiness(db: Db, input: {
|
||||
limit ${MAX_UPGRADE_RUNS + 1}`);
|
||||
}, { accessMode: "read only" });
|
||||
const candidates = rows.map(row => {
|
||||
const workspace = readNativeWorkspaceSyncReference(row.workspace);
|
||||
const remote = remoteWorkspaceRunReferenceSchema.safeParse(row.remoteWorkspace);
|
||||
// A remote reference owns the logical workspace identity. Corruption or conflicting
|
||||
// plugin authority must not fall back to a different recoverable-looking record.
|
||||
const workspaceId = row.hasRemoteWorkspace
|
||||
? remote.success && !row.pluginLeaseId ? remote.data.workspaceId : null
|
||||
: row.pluginLeaseId ?? readNativeWorkspaceSyncReference(row.workspace)?.workspaceId;
|
||||
const identity = row.recoverable && row.material
|
||||
? inspectNativeRunnerUpgradeHeader({ companyId: row.companyId, issueId: row.issueId, runId: row.id }, row.material) : null;
|
||||
return { id: row.id, recoverable: Boolean((row.pluginLeaseId || workspace) && identity && identity.runId === row.id
|
||||
return { id: row.id, recoverable: Boolean(workspaceId && identity && identity.runId === row.id
|
||||
&& identity.runnerInstanceId === row.runnerId && identity.normalizedSessionId === row.sessionId
|
||||
&& identity.environmentLeaseId === (row.pluginLeaseId ?? workspace?.workspaceId)) };
|
||||
&& identity.environmentLeaseId === workspaceId) };
|
||||
});
|
||||
// Additive evidence does not strengthen or redefine the existing ready flag.
|
||||
const retained = await readNativeControllerRecoveryInventory(db).catch(() => ({
|
||||
|
||||
@@ -7,9 +7,9 @@ import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { captureDirectorySnapshot, directorySnapshotSha256, serializeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, plugins, projects, projectWorkspaces, type Db } from "@paperclipai/db";
|
||||
import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { heartbeatService } from "../heartbeat.js";
|
||||
@@ -25,12 +25,14 @@ import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } fr
|
||||
import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js";
|
||||
import * as managedRepositoryCheckout from "../managed-repository-checkout.js";
|
||||
import * as freshWorkspaceSeed from "../fresh-workspace-seed.js";
|
||||
import { requestTaskRetry } from "../task-retry.js";
|
||||
import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js";
|
||||
import { createSystemRecovery } from "../system-recovery-state.js";
|
||||
import { isolatedRemoteSourcePath } from "../isolated-remote-workspace-source.js";
|
||||
import { cleanupExecutionWorkspaceArtifacts } from "../workspace-runtime.js";
|
||||
import { buildProjectMentionHref } from "@paperclipai/shared";
|
||||
import { buildProjectMentionHref, readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { migrateRemoteWorkspace } from "../remote-workspace-migration.js";
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { effectiveProjectWorkspacePolicy } from "../project-workspace-provider.js";
|
||||
import { resolveExecutionWorkspaceMode, buildExecutionWorkspaceAdapterConfig } from "../execution-workspace-policy.js";
|
||||
@@ -38,6 +40,8 @@ import { createLocalDiskStorageProvider } from "../../storage/local-disk-provide
|
||||
import { createStorageService } from "../../storage/service.js";
|
||||
import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace";
|
||||
import { selectHeartbeatEnvironment } from "./environment-selection.js";
|
||||
import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js";
|
||||
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_PREPARATION_TEST_DATABASE_URL");
|
||||
@@ -47,6 +51,10 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let home: string;
|
||||
const persistenceKeys = ["DATABASE_URL", "PAPERCLIP_STORAGE_PROVIDER", "PAPERCLIP_SECRETS_MASTER_KEY", "PAPERCLIP_SECRETS_MASTER_KEY_FILE"];
|
||||
let persistenceBefore = new Map<string, string | undefined>();
|
||||
const configureRemotePersistence = () => Object.assign(process.env, { DATABASE_URL: database.connectionString,
|
||||
PAPERCLIP_STORAGE_PROVIDER: "s3", PAPERCLIP_SECRETS_MASTER_KEY: "remote-workspace-preparation-fixture" });
|
||||
|
||||
beforeAll(async () => {
|
||||
home = await mkdtemp(join(tmpdir(), "paperclip-workspace-preparation-"));
|
||||
@@ -57,6 +65,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
: await startEmbeddedPostgresTestDatabase("paperclip-workspace-preparation");
|
||||
db = createDb(database.connectionString);
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
if (external) await db.execute(sql`truncate table environments restart identity cascade`);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
@@ -65,11 +74,18 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
if (home) await rm(home, { recursive: true, force: true });
|
||||
});
|
||||
beforeEach(async () => {
|
||||
persistenceBefore = new Map(persistenceKeys.map(key => [key, process.env[key]]));
|
||||
vi.restoreAllMocks();
|
||||
await db.execute(sql`truncate table companies, plugins restart identity cascade`);
|
||||
await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const [key, value] of persistenceBefore) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
}
|
||||
});
|
||||
|
||||
async function fixture(isolated = false) {
|
||||
const [company] = await db.insert(companies).values({ name: "Workspace preparation", issuePrefix: "WSP" }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Workspace agent", status: "running", adapterType: "process" }).returning();
|
||||
@@ -108,6 +124,464 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
};
|
||||
return { input, source, issue, run, envOrchestrator };
|
||||
}
|
||||
it.each(["unconfigured", "operator-disabled"] as const)("auto-enrollment keeps %s native sandboxes on their existing host path", async mode => {
|
||||
const { input, source, envOrchestrator } = await fixture();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
if (mode === "operator-disabled") {
|
||||
configureRemotePersistence();
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false;
|
||||
} else {
|
||||
Object.assign(process.env, { DATABASE_URL: database.connectionString, PAPERCLIP_STORAGE_PROVIDER: "local_disk" });
|
||||
}
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("host authority retained"));
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("host authority retained");
|
||||
expect(resolve).toHaveBeenCalledOnce(); expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("auto-enrollment preserves an existing host workspace even when persistence is ready", async () => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: issue.companyId,
|
||||
sourceIssueId: issue.id, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
|
||||
name: "Existing host workspace", cwd: source, providerType: "local_fs", strategyType: "project_primary",
|
||||
mode: "shared_workspace", status: "active" }).returning();
|
||||
await writeFile(join(source, "unpublished.txt"), "keep existing work\n");
|
||||
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
|
||||
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(workspace);
|
||||
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(workspace);
|
||||
input.reuse.requestedShouldReuseExisting = true;
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
configureRemotePersistence();
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("host authority retained"));
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("host authority retained");
|
||||
expect(resolve).toHaveBeenCalledOnce(); expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await readFile(join(source, "unpublished.txt"), "utf8")).toBe("keep existing work\n");
|
||||
expect((await db.select().from(executionWorkspaces))[0]?.metadata?.remoteWorkspace).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ kind: "empty", repoUrl: null, branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "ssh://git@github.com/example/remote.git", branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "git@github.com:example/remote.git", branchMode: null, fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "existing", fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: false },
|
||||
{ kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true },
|
||||
] as const)("reserves a $kind remote workspace ($repoUrl, branch: $branchMode, fresh: $fresh) without materializing a controller checkout", async ({ kind, repoUrl, branchMode, fresh }) => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
const [savedEnvironment] = await db.insert(environments).values({ name: `Remote ${randomUUID()}`,
|
||||
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
|
||||
const environment = { ...input.environment.localEnvironment, id: savedEnvironment.id,
|
||||
driver: "sandbox" as const, config: { provider: "daytona" } };
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, input.agent.id));
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = environment;
|
||||
input.environment.selectedEnvironmentId = environment.id;
|
||||
expect(input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled).toBe(true);
|
||||
input.policy.requestedExecutionWorkspaceMode = "agent_default";
|
||||
input.policy.useIsolatedTaskDirectory = kind === "empty";
|
||||
if (branchMode) {
|
||||
input.policy.requestedExecutionWorkspaceMode = "isolated_workspace";
|
||||
input.policy.effectiveExecutionWorkspaceMode = "isolated_workspace";
|
||||
input.config.mergedConfig = { workspaceStrategy: { type: "git_worktree", baseRef: "main",
|
||||
...(branchMode === "existing" ? { existingBranch: "feature/diverged" } : { branchTemplate: "test-{{issue.identifier}}" }) } };
|
||||
}
|
||||
if (kind === "git") {
|
||||
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl, repoRef: "main" })
|
||||
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
|
||||
}
|
||||
if (fresh) {
|
||||
const sourceRunId = randomUUID(), retryIntentId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId: issue.companyId, agentId: input.agent.id,
|
||||
issueId: issue.id, status: "failed", finishedAt: new Date() });
|
||||
await db.insert(agentWakeupRequests).values({ id: retryIntentId, companyId: issue.companyId,
|
||||
agentId: input.agent.id, reason: "task_retry", source: "on_demand", status: "coalesced", runId: input.run.id,
|
||||
payload: { issueId: issue.id } });
|
||||
const readiness = { mode: "fresh_isolated_workspace" as const, retryIntentId, sourceRunId, retainedSourceIds: [] };
|
||||
await db.insert(issueRecoveryActions).values({ companyId: issue.companyId, sourceIssueId: issue.id,
|
||||
kind: "active_run_watchdog", status: "resolved", ownerType: "system", nextAction: "Synthetic committed fresh attempt",
|
||||
cause: "native_continuation_requires_reconciliation", fingerprint: randomUUID(), evidence: { runId: sourceRunId,
|
||||
explicitUserContinuation: { runId: input.run.id, readiness }, systemRecovery: {
|
||||
...createSystemRecovery(sourceRunId, retryIntentId), state: "exhausted", freshAttemptRunId: input.run.id,
|
||||
} } });
|
||||
input.task.freshWorkspace = await readFreshWorkspaceReadiness(db, input.run);
|
||||
expect(input.task.freshWorkspace).toEqual(readiness);
|
||||
}
|
||||
await rm(source, { recursive: true });
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("host source resolver called"));
|
||||
const hostSeed = vi.spyOn(freshWorkspaceSeed, "materializeFreshWorkspaceSeed");
|
||||
const repositoryPreparation = vi.spyOn(input.services.executionWorkspacesSvc, "prepareTaskRepositoriesForAdmission");
|
||||
vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
|
||||
expect(args.persistedExecutionWorkspace).toMatchObject({ providerType: "cloud_sandbox" });
|
||||
expect(args.persistedExecutionWorkspace).toMatchObject({ cwd: expect.stringMatching(/^\/workspaces\//) });
|
||||
throw new Error("reached remote acquisition");
|
||||
});
|
||||
const configured = { DATABASE_URL: database.connectionString, PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
PAPERCLIP_SECRETS_MASTER_KEY: "remote-workspace-preparation-fixture" };
|
||||
const previous = new Map(Object.keys(configured).map(key => [key, process.env[key]]));
|
||||
Object.assign(process.env, configured);
|
||||
try {
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached remote acquisition");
|
||||
expect(resolve).not.toHaveBeenCalled();
|
||||
expect(repositoryPreparation).not.toHaveBeenCalled();
|
||||
expect(hostSeed).not.toHaveBeenCalled();
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.sourceIssueId, issue.id));
|
||||
expect(workspace.mode).toBe(branchMode ? "isolated_workspace" : "shared_workspace");
|
||||
expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { kind }, writerRunId: input.run.id });
|
||||
if (branchMode) expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: {
|
||||
ref: "main", branchMode, branchName: branchMode === "existing" ? "feature/diverged" : "test-WSP-1",
|
||||
} });
|
||||
await expect(access(workspace.cwd!)).rejects.toThrow();
|
||||
} finally {
|
||||
for (const [key, value] of previous) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects automatic Core services before remote reservation or acquisition", async () => {
|
||||
const { input, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
input.policy.useIsolatedTaskDirectory = true;
|
||||
input.config.mergedConfig = { workspaceRuntime: { services: [{ name: "web", command: "node server.js" }] } };
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
|
||||
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_runtime_services_unsupported" },
|
||||
});
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
expect(resolve).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("auto-enrollment leaves a configured local folder on the host authority", async () => {
|
||||
const { input, envOrchestrator } = await fixture();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
configureRemotePersistence();
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("local folder retained"));
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("local folder retained");
|
||||
expect(resolve).toHaveBeenCalledOnce();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["configured-folder", "explicit-task-directory", "enrolled-remote"] as const)(
|
||||
"preserves effective configured cwd authority for %s", async scenario => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
await writeFile(join(source, "unpublished.txt"), "configured folder work\n");
|
||||
await db.update(issues).set({ projectId: null, projectWorkspaceId: null }).where(eq(issues.id, issue.id));
|
||||
input.task.issueRef = { ...issue, projectId: null, projectWorkspaceId: null };
|
||||
input.task.context = { issueId: issue.id };
|
||||
input.config.mergedConfig = { cwd: source };
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, input.agent.id));
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
const [savedEnvironment] = await db.insert(environments).values({ name: `Configured cwd ${randomUUID()}`,
|
||||
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
id: savedEnvironment.id, driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.selectedEnvironmentId = savedEnvironment.id;
|
||||
expect(input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled).toBe(true);
|
||||
if (scenario === "explicit-task-directory") {
|
||||
input.task.workspaceSelection = { version: 1, source: "explicit", selection: { kind: "task_directory" } };
|
||||
await db.update(issues).set({ workspaceSelection: input.task.workspaceSelection }).where(eq(issues.id, issue.id));
|
||||
input.policy.useIsolatedTaskDirectory = true;
|
||||
}
|
||||
let enrolledId: string | undefined;
|
||||
if (scenario === "enrolled-remote") {
|
||||
enrolledId = randomUUID();
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ id: enrolledId, companyId: issue.companyId,
|
||||
sourceIssueId: issue.id, name: "Existing remote workspace", cwd: `/workspaces/${enrolledId}`,
|
||||
providerType: "cloud_sandbox", strategyType: "project_primary", mode: "shared_workspace", status: "active",
|
||||
metadata: { remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null,
|
||||
source: { kind: "empty" }, location: null, writerRunId: null, dirty: false } } }).returning();
|
||||
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
|
||||
input.task.issueRef.executionWorkspaceId = workspace.id;
|
||||
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(workspace);
|
||||
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(workspace);
|
||||
input.reuse.requestedShouldReuseExisting = true;
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false;
|
||||
}
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun").mockRejectedValue(new Error("configured cwd acquisition boundary"));
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("configured cwd acquisition boundary");
|
||||
expect(acquire).toHaveBeenCalledOnce();
|
||||
const [workspace] = await db.select().from(executionWorkspaces);
|
||||
if (scenario === "configured-folder") {
|
||||
expect(resolve).toHaveBeenCalledOnce();
|
||||
expect(workspace).toMatchObject({ providerType: "local_fs", cwd: source });
|
||||
expect(workspace.metadata?.remoteWorkspace).toBeUndefined();
|
||||
} else {
|
||||
expect(resolve).not.toHaveBeenCalled();
|
||||
expect(workspace).toMatchObject({ providerType: "cloud_sandbox" });
|
||||
expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { kind: "empty" }, writerRunId: input.run.id });
|
||||
if (enrolledId) expect(workspace.id).toBe(enrolledId);
|
||||
}
|
||||
expect(await readFile(join(source, "unpublished.txt"), "utf8")).toBe("configured folder work\n");
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["path", "file-url"] as const)("resumes an imported %s Git source from saved authority after its host files disappear", async sourceKind => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
process.env.PAPERCLIP_SECRETS_MASTER_KEY = "72".repeat(32);
|
||||
const upstream = join(home, `import-upstream-${randomUUID()}`);
|
||||
await execFile("git", ["clone", "--bare", source, upstream]);
|
||||
const repoUrl = sourceKind === "path" ? upstream : pathToFileURL(upstream).href;
|
||||
await execFile("git", ["-C", source, "remote", "add", "origin", repoUrl]);
|
||||
await writeFile(join(source, "unsaved.txt"), "imported untracked work\n");
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.run.id));
|
||||
const [original] = await db.insert(executionWorkspaces).values({ companyId: issue.companyId,
|
||||
sourceIssueId: issue.id, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
|
||||
name: "Imported Git workspace", cwd: source, providerRef: source, repoUrl, baseRef: "main", branchName: "main",
|
||||
providerType: "local_fs", strategyType: "project_primary", mode: "shared_workspace", status: "active" }).returning();
|
||||
await db.update(issues).set({ executionWorkspaceId: original.id }).where(eq(issues.id, issue.id));
|
||||
const storage = createStorageService(createLocalDiskStorageProvider(join(home, `import-objects-${randomUUID()}`)));
|
||||
await expect(migrateRemoteWorkspace({ db, storage, companyId: issue.companyId, workspaceId: original.id, apply: true }))
|
||||
.resolves.toMatchObject({ applied: true });
|
||||
const [imported] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, original.id));
|
||||
const saved = readRemoteWorkspaceAuthority(imported.metadata)!;
|
||||
expect(saved).toMatchObject({ source: { kind: "git", repoUrl, ref: "main", branchName: "main" },
|
||||
snapshotId: expect.any(String), location: null, dirty: false, writerRunId: null });
|
||||
await rm(source, { recursive: true });
|
||||
await rm(upstream, { recursive: true });
|
||||
|
||||
const [next] = await db.insert(heartbeatRuns).values({ companyId: issue.companyId, agentId: input.agent.id,
|
||||
issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();
|
||||
input.run = next;
|
||||
await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, input.agent.id));
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
const [environment] = await db.insert(environments).values({ name: `Imported source ${randomUUID()}`,
|
||||
driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning();
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
id: environment.id, driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.selectedEnvironmentId = environment.id;
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = false;
|
||||
input.task.issueRef = { ...issue, executionWorkspaceId: imported.id };
|
||||
input.reuse.existingExecutionWorkspace = toExecutionWorkspace(imported);
|
||||
input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(imported);
|
||||
input.reuse.requestedShouldReuseExisting = true;
|
||||
input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: imported.id,
|
||||
issueExecutionWorkspacePreference: null, existingExecutionWorkspaceStatus: imported.status });
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const hostSeed = vi.spyOn(freshWorkspaceSeed, "materializeFreshWorkspaceSeed");
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
|
||||
expect(args.persistedExecutionWorkspace).toMatchObject({ id: imported.id, providerType: "cloud_sandbox" });
|
||||
const [reserved] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, imported.id));
|
||||
expect(readRemoteWorkspaceAuthority(reserved.metadata)).toMatchObject({
|
||||
snapshotId: saved.snapshotId, source: saved.source,
|
||||
});
|
||||
throw new Error("imported saved authority reached remote acquisition");
|
||||
});
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("imported saved authority reached remote acquisition");
|
||||
expect(acquire).toHaveBeenCalledOnce();
|
||||
expect(resolve).not.toHaveBeenCalled();
|
||||
expect(hostSeed).not.toHaveBeenCalled();
|
||||
const [after] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, imported.id));
|
||||
expect(readRemoteWorkspaceAuthority(after.metadata)).toMatchObject({ snapshotId: saved.snapshotId, source: saved.source });
|
||||
await expect(access(source)).rejects.toThrow();
|
||||
await expect(access(upstream)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.each(["file:///tmp/repository", "/tmp/repository", "../repository", "C:\\work\\repository", "C:repository"])(
|
||||
"auto-enrollment leaves local repository source %s on the host authority", async repoUrl => {
|
||||
const { input, issue, envOrchestrator } = await fixture();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl })
|
||||
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
configureRemotePersistence();
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("local repository retained"));
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("local repository retained");
|
||||
expect(resolve).toHaveBeenCalledOnce();
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["https://", "ftp://example.com/repo", "javascript://invalid", "https://example.com/repo\ninvalid"])(
|
||||
"rejects malformed or unsupported repository source %s without a host fallback", async repoUrl => {
|
||||
const { input, issue, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl })
|
||||
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
|
||||
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_source_invalid" },
|
||||
});
|
||||
expect(resolve).not.toHaveBeenCalled(); expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not turn an unauthorized configured-source selection into a host fallback", async () => {
|
||||
const { input, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.task.workspaceSelection = { version: 1, source: "explicit",
|
||||
selection: { kind: "configured_source", projectWorkspaceId: randomUUID(), mode: "shared" } };
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({ status: 403 });
|
||||
expect(resolve).not.toHaveBeenCalled(); expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["source", "effective-policy"] as const)("keeps plugin repository ownership via %s out of automatic Core enrollment", async ownership => {
|
||||
const { input, issue, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
if (ownership === "effective-policy") input.policy.projectExecutionWorkspacePolicy = {
|
||||
enabled: true, storageProviderPluginId: randomUUID(), defaultMode: "isolated_workspace",
|
||||
};
|
||||
await db.update(projectWorkspaces).set({ cwd: null, sourceType: ownership === "source" ? "remote_managed" : "git", repoUrl: "https://github.com/example/managed.git" })
|
||||
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun").mockRejectedValue(new Error("managed ownership retained"));
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("managed ownership retained");
|
||||
expect(resolve).toHaveBeenCalledOnce(); expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["shared_workspace", "agent_default"] as const)("rejects concurrent shared writers resolved from %s before resolving a remote source", async requestedMode => {
|
||||
const { input, envOrchestrator } = await fixture();
|
||||
configureRemotePersistence();
|
||||
input.policy.useIsolatedTaskDirectory = true;
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
input.policy.requestedExecutionWorkspaceMode = requestedMode;
|
||||
input.policy.issueExecutionWorkspaceSettings = { sharedWorkspaceConcurrency: "allow" };
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("one active writer");
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([false, true])("holds execution while the selected workspace is migrating (remote: %s)", async remote => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: issue.companyId,
|
||||
sourceIssueId: issue.id, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId,
|
||||
name: "Migrating workspace", cwd: source, providerType: "local_fs", strategyType: "project_primary",
|
||||
mode: "shared_workspace", status: "active", metadata: { remoteWorkspaceMigration: { state: "running" } } }).returning();
|
||||
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
|
||||
// The admission snapshot may predate the operator starting the import.
|
||||
input.reuse.existingExecutionWorkspace = toExecutionWorkspace({ ...workspace, metadata: {} });
|
||||
if (remote) {
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
}
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
|
||||
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_migration_in_progress" },
|
||||
});
|
||||
expect(resolve).not.toHaveBeenCalled();
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");
|
||||
});
|
||||
|
||||
it("rejects read-only project references before reserving or acquiring a remote workspace", async () => {
|
||||
const { input, source, issue, envOrchestrator } = await fixture();
|
||||
const previous = new Map(["PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE"]
|
||||
.map(key => [key, process.env[key]]));
|
||||
vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC", "true");
|
||||
vi.stubEnv("PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE", "true");
|
||||
configureRemotePersistence();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
await db.update(projectWorkspaces).set({ cwd: null, sourceType: "git", repoUrl: "https://github.com/example/remote.git" })
|
||||
.where(eq(projectWorkspaces.id, issue.projectWorkspaceId!));
|
||||
const [reference] = await db.insert(projects).values({ companyId: issue.companyId, name: "Referenced project" }).returning();
|
||||
await db.insert(projectWorkspaces).values({ companyId: issue.companyId, projectId: reference.id,
|
||||
name: "Referenced repository", sourceType: "git", repoUrl: "https://github.com/example/reference.git" });
|
||||
await db.update(issues).set({ description: `[project](${buildProjectMentionHref(reference.id)})` }).where(eq(issues.id, issue.id));
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
try {
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toMatchObject({
|
||||
code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_readonly_sources_unsupported" },
|
||||
});
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
expect(await readdir(source)).not.toContain(".paperclip-repositories");
|
||||
} finally {
|
||||
for (const [key, value] of previous) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("fences recovered native preparation when its controller claim changes or expires", async () => {
|
||||
const { input, issue, envOrchestrator } = await fixture();
|
||||
const controller = await currentNativeControllerIdentity();
|
||||
const leaseOwner = `recovery:${randomUUID()}`;
|
||||
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, input.run.id));
|
||||
input.run.runtimeMode = "native";
|
||||
configureRemotePersistence();
|
||||
input.policy.useIsolatedTaskDirectory = true;
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
input.environment.selectedEnvironmentForConfig = { ...input.environment.localEnvironment,
|
||||
driver: "sandbox", config: { provider: "daytona" } };
|
||||
input.environment.resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled = true;
|
||||
input.reuse.runOptions = { nativeLeaseOwner: leaseOwner };
|
||||
await db.insert(nativeRunFinalizations).values({ runId: input.run.id, companyId: issue.companyId,
|
||||
issueId: issue.id, phase: "preparing", attempt: 2, controllerGeneration: 3, leaseOwner,
|
||||
leaseExpiresAt: new Date(Date.now() + 60_000), controllerBootId: controller.bootId,
|
||||
controllerPid: controller.pid, controllerProcessStartedAt: controller.processStartedAt });
|
||||
const authority = { db, companyId: issue.companyId, runId: input.run.id, leaseOwner,
|
||||
controllerGeneration: 3, providerAttempt: 2 };
|
||||
await expect(assertNativeSessionPreparationAuthority(authority)).resolves.toBeUndefined();
|
||||
await expect(assertNativeSessionPreparationAuthority({ ...authority, controllerGeneration: 2 }))
|
||||
.rejects.toThrow("native_session_lease_lost");
|
||||
await db.update(nativeRunFinalizations).set({ leaseOwner: "successor-controller" })
|
||||
.where(eq(nativeRunFinalizations.runId, input.run.id));
|
||||
const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
|
||||
await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("native_session_lease_lost");
|
||||
expect(resolve).not.toHaveBeenCalled();
|
||||
expect(acquire).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
await db.update(nativeRunFinalizations).set({ leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, input.run.id));
|
||||
await expect(assertNativeSessionPreparationAuthority(authority)).rejects.toThrow("native_session_lease_lost");
|
||||
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + 60_000) })
|
||||
.where(eq(nativeRunFinalizations.runId, input.run.id));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, input.run.id));
|
||||
await expect(assertNativeSessionPreparationAuthority(authority)).rejects.toThrow("native_session_lease_lost");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ inherited: false, mode: "isolated_workspace" as const, fresh: false },
|
||||
{ inherited: false, mode: "isolated_workspace" as const, fresh: true },
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import { readRemoteWorkspaceAuthority, reserveRemoteWorkspace, shouldUseRemoteWorkspaces } from "../remote-workspaces.js";
|
||||
import { readFreshWorkspaceReadiness } from "../fresh-workspace-continuation.js";
|
||||
import { assertNativeSessionPreparationAuthority } from "../native-runtime/native-execution-lease.js";
|
||||
import { resolveRemoteWorkspaceSource } from "./remote-workspace-source.js";
|
||||
import { reserveFreshPluginTaskWorkspace, stageProviderFreshWorkspace } from "../provider-fresh-workspace.js";
|
||||
import { freshWorkspaceSchema, type PluginFreshWorkspace } from "@paperclipai/plugin-sdk";
|
||||
import { pluginRegistryService } from "../plugin-registry.js";
|
||||
@@ -55,6 +59,7 @@ import {
|
||||
} from "../workspace-instance-cleanup.js";
|
||||
import { isRuntimeOwnedGitBranch } from "../execution-workspace-branch-ownership.js";
|
||||
import {
|
||||
resolveSharedWorkspaceConcurrency,
|
||||
issueExecutionWorkspaceModeForPersistedWorkspace,
|
||||
resolveEffectiveWorkspaceStrategyType,
|
||||
} from "../execution-workspace-policy.js";
|
||||
@@ -139,7 +144,7 @@ export interface HeartbeatWorkspacePreparationInput {
|
||||
nativeRecoveryExecutionWorkspaceId: string | null;
|
||||
persistedNativeExecutionInput: NativeExecutionInput | null;
|
||||
isDotRun: boolean;
|
||||
runOptions: { nativeRestartRecovery?: NativeRestartRecoveryClaim };
|
||||
runOptions: { nativeLeaseOwner?: string; nativeRestartRecovery?: NativeRestartRecoveryClaim };
|
||||
};
|
||||
services: {
|
||||
workspaceWorkers?: WorkspaceWorkers;
|
||||
@@ -220,6 +225,34 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
appendRunEvent,
|
||||
} = input.services;
|
||||
const pluginTaskWorkspace = selectedEnvironmentForConfig?.driver === "plugin" && agent.adapterType === "paperclip_runner";
|
||||
for (const workspaceId of new Set([existingExecutionWorkspace?.id, reusableExistingExecutionWorkspace?.id].filter(Boolean))) {
|
||||
const [workspace] = await db.select({ metadata: executionWorkspacesTable.metadata }).from(executionWorkspacesTable)
|
||||
.where(and(eq(executionWorkspacesTable.companyId, agent.companyId), eq(executionWorkspacesTable.id, workspaceId!)));
|
||||
if (parseObject(workspace?.metadata?.remoteWorkspaceMigration).state === "running") {
|
||||
throw new WorkspaceValidationFailure("The workspace is being moved to remote storage. Try again after the move completes.",
|
||||
{ reason: "remote_workspace_migration_in_progress" });
|
||||
}
|
||||
}
|
||||
const enrolledRemoteAuthority = readRemoteWorkspaceAuthority((reusableExistingExecutionWorkspace ?? existingExecutionWorkspace)?.metadata) !== null;
|
||||
let remoteAuthority = !isDotRun && shouldUseRemoteWorkspaces({
|
||||
adapterType: agent.adapterType,
|
||||
environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
|
||||
experimentalEnabled: resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled && Boolean(issueId && issueRef)
|
||||
&& !projectExecutionWorkspacePolicy?.storageProviderPluginId,
|
||||
existingWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace,
|
||||
});
|
||||
const assertRemotePreparationOwned = run.runtimeMode === "native" ? () => assertNativeSessionPreparationAuthority({
|
||||
db, companyId: agent.companyId, runId: run.id,
|
||||
leaseOwner: runOptions.nativeLeaseOwner ?? runOptions.nativeRestartRecovery?.leaseOwner,
|
||||
...(runOptions.nativeRestartRecovery ? {
|
||||
controllerGeneration: runOptions.nativeRestartRecovery.controllerGeneration,
|
||||
providerAttempt: runOptions.nativeRestartRecovery.providerAttempt,
|
||||
} : {}),
|
||||
}) : () => controllerLease.assertOwned();
|
||||
if (enrolledRemoteAuthority) await assertRemotePreparationOwned();
|
||||
if (remoteAuthority && (!issueId || !issueRef)) throw new WorkspaceValidationFailure(
|
||||
"Remote workspace storage requires a task.", { reason: "remote_workspace_task_required" });
|
||||
const remoteSourceState: { value: Extract<Awaited<ReturnType<typeof resolveRemoteWorkspaceSource>>, { authority: "remote" }> | null } = { value: null };
|
||||
const pluginConfig = parseObject(selectedEnvironmentForConfig?.config);
|
||||
if (fresh && !isDotRun && pluginTaskWorkspace) {
|
||||
const plugin = typeof pluginConfig.pluginKey === "string" ? await pluginRegistryService(db).getByKey(pluginConfig.pluginKey) : null;
|
||||
@@ -269,6 +302,46 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
return preflightEnvironment.driver;
|
||||
},
|
||||
resolveWorkspace: async () => {
|
||||
if (remoteAuthority) {
|
||||
const strategy = parseObject(mergedConfig.workspaceStrategy);
|
||||
const sourceState = await resolveRemoteWorkspaceSource(db, {
|
||||
companyId: agent.companyId, issueId: issueId!, agentId: agent.id, runId: run.id,
|
||||
responsibleUserId, projectId: issueRef!.projectId,
|
||||
projectWorkspaceId: issueRef!.projectWorkspaceId, selection: input.task.workspaceSelection,
|
||||
existingWorkspace: fresh ? null : reusableExistingExecutionWorkspace ?? existingExecutionWorkspace,
|
||||
taskDirectory: useIsolatedTaskDirectory, automaticEnrollment: !enrolledRemoteAuthority,
|
||||
configuredCwd: readNonEmptyString(mergedConfig.cwd),
|
||||
ref: readNonEmptyString(strategy.baseRef),
|
||||
branchMode: readNonEmptyString(strategy.existingBranch) ? "existing" : "create",
|
||||
branchName: readNonEmptyString(strategy.existingBranch) ??
|
||||
(resolveEffectiveWorkspaceStrategyType(requestedExecutionWorkspaceMode, mergedConfig) === "git_worktree"
|
||||
? sanitizeBranchName(renderWorkspaceTemplate(String(strategy.branchTemplate ?? "{{issue.identifier}}-{{slug}}"), {
|
||||
issue: issueRef, agent, projectId: issueRef!.projectId, repoRef: null,
|
||||
})) : null),
|
||||
});
|
||||
if (sourceState.authority === "host") remoteAuthority = false;
|
||||
else {
|
||||
remoteSourceState.value = sourceState;
|
||||
if (!enrolledRemoteAuthority) await assertRemotePreparationOwned();
|
||||
if (effectiveExecutionWorkspaceMode === "shared_workspace" && resolveSharedWorkspaceConcurrency({
|
||||
projectPolicy: projectExecutionWorkspacePolicy, issueSettings: issueExecutionWorkspaceSettings,
|
||||
}) === "allow") throw new WorkspaceValidationFailure(
|
||||
"Remote workspace storage requires one active writer per workspace.", { reason: "remote_workspace_concurrent_writers_unsupported" });
|
||||
if (Array.isArray(workspaceServices) && workspaceServices.length > 0) throw new WorkspaceValidationFailure(
|
||||
"Remote workspace storage does not support Core workspace services.", { reason: "remote_workspace_runtime_services_unsupported" });
|
||||
const source = sourceState.source;
|
||||
return {
|
||||
cwd: reusableExistingExecutionWorkspace?.cwd ?? `/workspaces/${run.id}`,
|
||||
source: "task_session" as const,
|
||||
projectId: remoteSourceState.value!.projectId, workspaceId: remoteSourceState.value!.projectWorkspaceId,
|
||||
repoUrl: source.kind === "git" ? source.repoUrl : null,
|
||||
repoRef: source.kind === "git" ? source.ref : null,
|
||||
workspaceHints: [], warnings: remoteSourceState.value.warnings ?? [], baseCwdFallback: false,
|
||||
materializationFailures: [], additionalWorkspaces: [],
|
||||
referencedProjectFailures: remoteSourceState.value.referencedProjectFailures ?? [],
|
||||
};
|
||||
}
|
||||
}
|
||||
if (isDotRun) {
|
||||
// This is private controller storage, never a provider filesystem.
|
||||
// External providers receive workspace.access=none and cwd=null.
|
||||
@@ -396,6 +469,18 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
: workspace;
|
||||
},
|
||||
}));
|
||||
const remoteFresh = remoteAuthority && fresh ? await readFreshWorkspaceReadiness(db, run) : null;
|
||||
if (remoteAuthority && fresh && (remoteFresh?.mode !== "fresh_isolated_workspace"
|
||||
|| input.task.freshWorkspace?.mode !== "fresh_isolated_workspace"
|
||||
|| remoteFresh.retryIntentId !== input.task.freshWorkspace.retryIntentId
|
||||
|| remoteFresh.sourceRunId !== input.task.freshWorkspace.sourceRunId)) {
|
||||
throw new WorkspaceValidationFailure("The committed fresh workspace recovery authorization changed.",
|
||||
{ reason: "remote_workspace_fresh_authorization_changed" });
|
||||
}
|
||||
const remoteFreshBinding = remoteFresh?.mode === "fresh_isolated_workspace"
|
||||
? (await db.select({ workspaceId: issues.executionWorkspaceId, revision: issues.workspaceBindingRevision }).from(issues)
|
||||
.where(and(eq(issues.companyId, agent.companyId), eq(issues.id, issueId!))))[0] : null;
|
||||
if (remoteFresh && !remoteFreshBinding) throw new Error("remote_workspace_task_binding_missing");
|
||||
const hostProvisionDriver = selectedEnvironmentForConfig?.driver ?? lowTrustPreflightEnvironmentDriver;
|
||||
let providerFresh: PluginFreshWorkspace | undefined;
|
||||
let previousProviderWorkspaceId: string | null = null;
|
||||
@@ -416,7 +501,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
previousWorkspaceId: previousProviderWorkspaceId });
|
||||
}
|
||||
}
|
||||
const pluginPrepared = isDotRun ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
const pluginPrepared = isDotRun || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId,
|
||||
issueId, runId: run.id, agentId: agent.id, responsibleUserId,
|
||||
existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
|
||||
@@ -439,7 +524,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
source: { projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId,
|
||||
repoUrl: resolvedWorkspace.repoUrl, baseRef: resolvedWorkspace.repoRef } }) : null;
|
||||
const freshTaskReservation = freshTaskReservationId ? await executionWorkspacesSvc.getStoredById(agent.companyId, freshTaskReservationId) : null;
|
||||
const copyOnlyProvisioning = pluginPrepared ? true : await timeWorkspace("workspace_provision_policy", () => workspaceRequiresCopyOnlyProvisioning(db, {
|
||||
const copyOnlyProvisioning = pluginPrepared || remoteAuthority ? true : await timeWorkspace("workspace_provision_policy", () => workspaceRequiresCopyOnlyProvisioning(db, {
|
||||
companyId: agent.companyId,
|
||||
task: issueRef ? { ...issueRef, workspaceSelection: input.task.workspaceSelection } : null,
|
||||
sourceProjectId: resolvedWorkspace.projectId,
|
||||
@@ -466,7 +551,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
throw new WorkspaceValidationFailure("The selected plugin environment does not support Core Git worktree preparation.",
|
||||
{ reason: "plugin_workspace_strategy_unsupported" });
|
||||
}
|
||||
if (!pluginPrepared) await timeWorkspace("workspace_base_validation", () => assertGitWorktreeBaseWorkspaceReady({
|
||||
if (!pluginPrepared && !remoteAuthority) await timeWorkspace("workspace_base_validation", () => assertGitWorktreeBaseWorkspaceReady({
|
||||
requestedExecutionWorkspaceMode,
|
||||
config: hostExecutionWorkspaceConfig,
|
||||
issue: issueRef,
|
||||
@@ -598,7 +683,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
agentId: agent.id,
|
||||
},
|
||||
);
|
||||
if (freshNeedsSeed && issueId && !pluginPrepared && !pluginTaskWorkspace) {
|
||||
if (freshNeedsSeed && issueId && !pluginPrepared && !pluginTaskWorkspace && !remoteAuthority) {
|
||||
await controllerLease.assertOwned();
|
||||
const verifySourceAccess = async () => {
|
||||
await assertTaskWorkspaceExecutionAccess(db, {
|
||||
@@ -629,7 +714,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
warnings: [], created: false, branchCreatedByRuntime: false,
|
||||
baseRefSha: pluginPrepared.metadata?.workspaceProviderBaseSha as string | null,
|
||||
} as RealizedExecutionWorkspace, reusedExecutionWorkspace: Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy }
|
||||
: isDotRun || pluginTaskWorkspace ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: pluginTaskWorkspace && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
|
||||
: isDotRun || pluginTaskWorkspace || remoteAuthority ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace || remoteAuthority ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: remoteSourceState.value?.source.kind === "git" ? remoteSourceState.value!.source.branchName ?? null : null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: (pluginTaskWorkspace || remoteAuthority) && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
|
||||
{
|
||||
requestedShouldReuseExisting,
|
||||
existingExecutionWorkspaceId:
|
||||
@@ -733,7 +818,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
}),
|
||||
},
|
||||
));
|
||||
if (input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && issueId) {
|
||||
if (!remoteAuthority && input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && issueId) {
|
||||
const recoveredFiles = providerFresh ? providerFresh.recoverySnapshots.map(snapshot => ({
|
||||
operationId: snapshot.operationId, sourceRunId: snapshot.sourceRunId,
|
||||
path: `.paperclip-recovered/${snapshot.operationId}`,
|
||||
@@ -855,7 +940,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
]
|
||||
: null;
|
||||
if (
|
||||
!pluginPrepared && !persistedWorktreeInstanceRoot &&
|
||||
!pluginPrepared && !remoteAuthority && !persistedWorktreeInstanceRoot &&
|
||||
executionWorkspace.strategy === "git_worktree" &&
|
||||
executionWorkspace.worktreePath
|
||||
) {
|
||||
@@ -897,7 +982,33 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
pendingForwardBranchReconcile?.recordedBranchName ??
|
||||
executionWorkspace.branchName;
|
||||
try {
|
||||
persistedExecutionWorkspace =
|
||||
if (remoteAuthority) {
|
||||
if (!remoteSourceState.value || !issueId) throw new Error("remote_workspace_source_missing");
|
||||
await assertRemotePreparationOwned();
|
||||
const reserved = await reserveRemoteWorkspace({
|
||||
db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id,
|
||||
environmentId: selectedEnvironmentId,
|
||||
mode: effectiveExecutionWorkspaceMode === "shared_workspace" ? "shared_workspace"
|
||||
: effectiveExecutionWorkspaceMode === "operator_branch" ? "operator_branch" : "isolated_workspace",
|
||||
projectId: remoteSourceState.value!.projectId, projectWorkspaceId: remoteSourceState.value!.projectWorkspaceId,
|
||||
existingWorkspaceId: reusableExistingExecutionWorkspace?.id ?? existingExecutionWorkspace?.id ?? null,
|
||||
source: remoteSourceState.value!.source,
|
||||
...(remoteFresh?.mode === "fresh_isolated_workspace" ? {
|
||||
fresh: { attemptId: remoteFresh.retryIntentId, previousWorkspaceId: remoteFreshBinding!.workspaceId,
|
||||
expectedBindingRevision: remoteFreshBinding!.revision },
|
||||
recoverySourceRunIds: [remoteFresh.sourceRunId],
|
||||
} : {}),
|
||||
});
|
||||
persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(agent.companyId, reserved.id);
|
||||
if (!persistedExecutionWorkspace?.cwd) throw new Error("remote_workspace_reservation_missing");
|
||||
persistedExecutionWorkspace = await executionWorkspacesSvc.update(reserved.id, {
|
||||
metadata: { ...nextExecutionWorkspaceMetadata, ...reserved.metadata,
|
||||
remoteWorkspace: reserved.metadata?.remoteWorkspace },
|
||||
});
|
||||
if (!persistedExecutionWorkspace?.cwd) throw new Error("remote_workspace_reservation_missing");
|
||||
executionWorkspace.cwd = persistedExecutionWorkspace.cwd;
|
||||
resolvedWorkspace.cwd = executionWorkspace.cwd;
|
||||
} else persistedExecutionWorkspace =
|
||||
pluginPrepared ? await executionWorkspacesSvc.update(pluginPrepared.id, { metadata: nextExecutionWorkspaceMetadata }) :
|
||||
freshTaskReservation ? await executionWorkspacesSvc.update(freshTaskReservation.id, { metadata: nextExecutionWorkspaceMetadata }) :
|
||||
resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&
|
||||
@@ -1047,14 +1158,16 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
}
|
||||
await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
|
||||
const projectRepositoryPaths: string[] = [];
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
if (remoteAuthority) projectRepositoryPaths.push(...(remoteSourceState.value?.source.repositories ?? [])
|
||||
.filter(repository => !repository.readOnly).map(repository => repository.relativePath));
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
const taskRepositories = await timeWorkspace("workspace_repository_admission", () =>
|
||||
executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId,
|
||||
issueId, workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd,
|
||||
agentId: agent.id, runId: run.id, responsibleUserId }));
|
||||
projectRepositoryPaths.push(...taskRepositories.map(repository => repository.relativePath));
|
||||
}
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && executionWorkspace.projectId
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && executionWorkspace.projectId
|
||||
&& (resolvedWorkspace.source === "project_primary" || Boolean(reusableExistingExecutionWorkspace?.projectWorkspaceId))
|
||||
&& !resolvedWorkspace.baseCwdFallback && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
const repositoryRows = await db.select().from(projectWorkspaces).where(and(
|
||||
@@ -1112,9 +1225,13 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
const sandboxRecovery = remoteRecovery && remoteRecovery.remote.kind !== "plugin_task" ? remoteRecovery.remote : null;
|
||||
const recoveryWorkspace = sandboxRecovery
|
||||
? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null;
|
||||
if (remoteRecovery && (remoteRecovery.runId !== run.id || (sandboxRecovery && (!recoveryWorkspace ||
|
||||
recoveryWorkspace.providerLeaseId !== sandboxRecovery.providerLeaseId ||
|
||||
recoveryWorkspace.remoteCwd !== sandboxRecovery.remoteCwd)))) {
|
||||
const remoteRecoveryLocation = remoteAuthority
|
||||
? readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata)?.location ?? null : null;
|
||||
if (remoteRecovery && (remoteRecovery.runId !== run.id || (sandboxRecovery && (remoteAuthority
|
||||
? !remoteRecoveryLocation || remoteRecoveryLocation.providerLeaseId !== sandboxRecovery.providerLeaseId
|
||||
|| remoteRecoveryLocation.remoteCwd !== sandboxRecovery.remoteCwd
|
||||
: !recoveryWorkspace || recoveryWorkspace.providerLeaseId !== sandboxRecovery.providerLeaseId
|
||||
|| recoveryWorkspace.remoteCwd !== sandboxRecovery.remoteCwd)))) {
|
||||
throw new Error("native_remote_recovery_lease_mismatch");
|
||||
}
|
||||
acquiredEnvironment = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({
|
||||
@@ -1134,8 +1251,8 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
agentId: agent.id,
|
||||
persistedExecutionWorkspace,
|
||||
executionWorkspaceSettings: environmentExecutionWorkspaceSettings,
|
||||
...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : sandboxRecovery && recoveryWorkspace ? { reattachRemoteLease: {
|
||||
leaseId: recoveryWorkspace.leaseId,
|
||||
...(pluginRecovery ? { reattachRemoteLease: pluginRecovery } : sandboxRecovery && (recoveryWorkspace || remoteRecoveryLocation) ? { reattachRemoteLease: {
|
||||
leaseId: (remoteRecoveryLocation ?? recoveryWorkspace)!.leaseId,
|
||||
providerLeaseId: sandboxRecovery.providerLeaseId,
|
||||
remoteCwd: sandboxRecovery.remoteCwd,
|
||||
} } : {}),
|
||||
@@ -1226,6 +1343,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
effectiveExecutionWorkspaceMode,
|
||||
persistedExecutionWorkspace,
|
||||
runnerTaskProjectIds: resolvedWorkspace.runnerTaskProjectIds,
|
||||
assertOwned: remoteAuthority ? assertRemotePreparationOwned : controllerLease.assertOwned,
|
||||
duplexObservabilityRecorder,
|
||||
}));
|
||||
nativeRunnerPreparationSpans.push({
|
||||
@@ -1260,7 +1378,34 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
|
||||
const workspaceRealization = realizationResult.workspaceRealization;
|
||||
const executionTarget = realizationResult.executionTarget;
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId)
|
||||
if (remoteAuthority && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
const repositories = await timeWorkspace("workspace_repository_admission", () =>
|
||||
executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId, issueId,
|
||||
workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd, agentId: agent.id, runId: run.id,
|
||||
responsibleUserId, remoteTarget: executionTarget, remoteLease: activeEnvironmentLease.lease,
|
||||
assertOwned: assertRemotePreparationOwned }));
|
||||
projectRepositoryPaths.push(...repositories.map(repository => repository.relativePath));
|
||||
persistedExecutionWorkspace = await executionWorkspacesSvc.getStoredById(agent.companyId, persistedExecutionWorkspace.id);
|
||||
}
|
||||
if (remoteFresh?.mode === "fresh_isolated_workspace") {
|
||||
const recorded = persistedExecutionWorkspace?.metadata?.remoteWorkspaceRecoveredFiles;
|
||||
const recoveredFiles = (Array.isArray(recorded) ? recorded : []).map(value => {
|
||||
const file = parseObject(value);
|
||||
if (typeof file.operationId !== "string" || typeof file.sourceRunId !== "string"
|
||||
|| file.path !== `.paperclip-recovered/${file.operationId}` || !/^[a-f0-9-]{36}$/.test(file.operationId)) {
|
||||
throw new Error("remote_workspace_recovered_files_invalid");
|
||||
}
|
||||
return { operationId: file.operationId, sourceRunId: file.sourceRunId, path: file.path as string };
|
||||
});
|
||||
context.recoveredWorkspaceFiles = recoveredFiles;
|
||||
const continuation = context.executionContinuation as import("@paperclipai/shared").ExecutionContinuationEnvelope | null;
|
||||
if (continuation) {
|
||||
continuation.recoveredFiles = recoveredFiles;
|
||||
const wake = context.paperclipWake as Record<string, unknown> | undefined;
|
||||
if (wake) wake.executionContinuation = continuation;
|
||||
}
|
||||
}
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId)
|
||||
&& !(executionTarget?.kind === "remote" && (executionTarget.transport === "ssh"
|
||||
|| (executionTarget.transport === "sandbox" && executionTarget.runnerTask)))) {
|
||||
const workspaceId = persistedNativeExecutionInput?.binding.executionWorkspaceId ?? nativeRecoveryExecutionWorkspaceId!;
|
||||
|
||||
@@ -249,6 +249,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
enableAiConnectionRouters: parsed.data.enableAiConnectionRouters ?? false,
|
||||
enableManagedSandboxOnly: parsed.data.enableManagedSandboxOnly ?? false,
|
||||
enableIsolatedWorkspaces: parsed.data.enableIsolatedWorkspaces ?? false,
|
||||
remoteWorkspaceStorageEnabled: parsed.data.remoteWorkspaceStorageEnabled ?? true,
|
||||
enableIsolatedWorkspacesByDefault: parsed.data.enableIsolatedWorkspacesByDefault ?? false,
|
||||
enableStreamlinedLeftNavigation: parsed.data.enableStreamlinedLeftNavigation ?? true,
|
||||
enableStreamlinedUi: parsed.data.enableStreamlinedUi ?? true,
|
||||
@@ -303,6 +304,11 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
enableAiConnectionRouters: false,
|
||||
enableManagedSandboxOnly: false,
|
||||
enableIsolatedWorkspaces: false,
|
||||
// An unrelated malformed field must not erase an explicit enrollment opt-out.
|
||||
remoteWorkspaceStorageEnabled: !(
|
||||
raw !== null && typeof raw === "object" && !Array.isArray(raw)
|
||||
&& "remoteWorkspaceStorageEnabled" in raw && raw.remoteWorkspaceStorageEnabled === false
|
||||
),
|
||||
enableIsolatedWorkspacesByDefault: false,
|
||||
enableStreamlinedLeftNavigation: true,
|
||||
enableStreamlinedUi: true,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { livenessDb, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { and, eq, gt, sql } from "drizzle-orm";
|
||||
import { currentNativeControllerIdentity, type NativeControllerIdentity } from "./native-restart-recovery.js";
|
||||
import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js";
|
||||
@@ -9,6 +9,27 @@ export const NATIVE_SESSION_EXECUTION_LEASE_TTL_MS = 20 * 60_000;
|
||||
const NATIVE_SESSION_EXECUTION_LEASE_RENEW_INTERVAL_MS = 5 * 60_000;
|
||||
const RENEWAL_STATEMENT_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Preparation can perform provider I/O only under its current recovery claim. */
|
||||
export async function assertNativeSessionPreparationAuthority(input: {
|
||||
db: Db; companyId: string; runId: string; leaseOwner: string | undefined;
|
||||
controllerGeneration?: number; providerAttempt?: number;
|
||||
}) {
|
||||
if (!input.leaseOwner) throw new Error("native_session_lease_lost");
|
||||
const controller = await currentNativeControllerIdentity();
|
||||
const [owned] = await input.db.select({ runId: nativeRunFinalizations.runId })
|
||||
.from(nativeRunFinalizations).innerJoin(heartbeatRuns, eq(heartbeatRuns.id, nativeRunFinalizations.runId))
|
||||
.where(and(eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId),
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.status, "running"),
|
||||
eq(nativeRunFinalizations.leaseOwner, input.leaseOwner),
|
||||
eq(nativeRunFinalizations.controllerBootId, controller.bootId), eq(nativeRunFinalizations.controllerPid, controller.pid),
|
||||
eq(nativeRunFinalizations.controllerProcessStartedAt, controller.processStartedAt),
|
||||
gt(nativeRunFinalizations.leaseExpiresAt, sql`clock_timestamp()`),
|
||||
...(input.controllerGeneration === undefined ? [] : [eq(nativeRunFinalizations.controllerGeneration, input.controllerGeneration)]),
|
||||
...(input.providerAttempt === undefined ? [] : [eq(nativeRunFinalizations.attempt, input.providerAttempt)]),
|
||||
)).limit(1);
|
||||
if (!owned) throw new Error("native_session_lease_lost");
|
||||
}
|
||||
|
||||
export async function renewNativeSessionExecutionLease(input: {
|
||||
db: Db;
|
||||
runId: string;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import type { CancelledWorkspacePreservationScope } from "../cancelled-workspace-preservation-notifications.js";
|
||||
import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
@@ -13,11 +14,18 @@ import { NativeWorkspaceFinalizationBusyError } from "./native-workspace-finaliz
|
||||
import { hasAcknowledgedNativeStopIntent } from "../acknowledged-native-stop.js";
|
||||
import { logger } from "../../middleware/logger.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
|
||||
import { preserveRemoteWorkspacePreparationFailure } from "./native-workspace-finalization-state.js";
|
||||
|
||||
|
||||
function workspaceRecoveryReference(profile: Record<string, unknown> | null | undefined) {
|
||||
const remote = remoteWorkspaceRunReferenceSchema.safeParse(profile?.remoteWorkspace);
|
||||
return readNativeWorkspaceSyncReference(profile?.nativeWorkspaceSync) ?? (remote.success ? remote.data : null);
|
||||
}
|
||||
|
||||
export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "nativeIssueId" | "issueId" | "contextSnapshot" | "runnerProfileJson">,
|
||||
options: { allowPreterminal?: boolean; requireCancelledStop?: boolean; restartRecovery?: NativeRestartRecoveryClaim } = {}) {
|
||||
if (readNativeWorkspaceSyncReference(selected.runnerProfileJson?.nativeWorkspaceSync)?.state !== "prepared") return;
|
||||
if (workspaceRecoveryReference(selected.runnerProfileJson)?.state !== "prepared") return;
|
||||
const issueId = selected.nativeIssueId ?? selected.issueId ?? selected.contextSnapshot?.issueId;
|
||||
if (typeof issueId !== "string") return;
|
||||
return db.transaction(async tx => {
|
||||
@@ -32,7 +40,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
|
||||
)).for("update");
|
||||
if (options.requireCancelledStop && coordinator?.phase !== "terminal_failure") return;
|
||||
const claim = options.restartRecovery;
|
||||
const reference = readNativeWorkspaceSyncReference(selected.runnerProfileJson?.nativeWorkspaceSync)!;
|
||||
const reference = workspaceRecoveryReference(selected.runnerProfileJson)!;
|
||||
const ownsClaim = Boolean(options.allowPreterminal && coordinator?.phase === "observed" && claim
|
||||
&& claim.runId === selected.id && claim.leaseOwner === coordinator.leaseOwner
|
||||
&& claim.controllerGeneration === coordinator.controllerGeneration && claim.providerAttempt === coordinator.attempt
|
||||
@@ -47,7 +55,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
|
||||
)).for("update");
|
||||
if (!run || run.runtimeMode !== "native" || (!options.allowPreterminal && run.status !== "failed")) return;
|
||||
if (options.requireCancelledStop && (run.status !== "cancelled" || !hasAcknowledgedNativeStopIntent(run))) return;
|
||||
const currentReference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
|
||||
const currentReference = workspaceRecoveryReference(run.runnerProfileJson);
|
||||
if (!currentReference || currentReference.state !== "prepared"
|
||||
|| (claim?.kind === "reattach_remote_runner" && (claim.remote.kind === "plugin_task" || claim.remote.providerLeaseId !== currentReference.providerLeaseId
|
||||
|| claim.remote.remoteCwd !== currentReference.remoteCwd))) return;
|
||||
@@ -95,10 +103,19 @@ export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof
|
||||
if (!lease.heartbeatRunId) return false;
|
||||
const [run] = await db.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId),
|
||||
eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.status, "failed"),
|
||||
inArray(heartbeatRuns.status, ["failed", "cancelled", "timed_out", "interrupted"]),
|
||||
)).limit(1);
|
||||
if (!run || hasRequiredWorkspaceRecovery(run.resultJson)) return false;
|
||||
try {
|
||||
if (run.runtimeMode === "legacy") {
|
||||
const parsed = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
|
||||
if (!parsed.success) return false;
|
||||
const [source] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId),
|
||||
eq(environmentLeases.id, parsed.data.leaseId), eq(environmentLeases.heartbeatRunId, run.id))).limit(1);
|
||||
const retained = source && await preserveRemoteWorkspacePreparationFailure(db, source);
|
||||
return Boolean(retained && hasRequiredWorkspaceRecovery(retained.resultJson));
|
||||
}
|
||||
if (run.status !== "failed") return false;
|
||||
const retained = await preserveNativeFailedWorkspace(db, run);
|
||||
return Boolean(retained && hasRequiredWorkspaceRecovery(retained.resultJson));
|
||||
} catch (error) {
|
||||
@@ -137,7 +154,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) {
|
||||
)), eq(heartbeatRuns.runtimeMode, "native"),
|
||||
eq(heartbeatRuns.status, "cancelled"), eq(nativeRunFinalizations.phase, "terminal_failure"),
|
||||
isNull(nativeRunFinalizations.resultId), isNull(nativeRunFinalizations.leaseOwner),
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'`,
|
||||
sql`(${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')`,
|
||||
sql`${heartbeatRuns.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged'`,
|
||||
cursor ? gt(heartbeatRuns.id, cursor) : undefined,
|
||||
)).orderBy(asc(heartbeatRuns.id)).limit(5);
|
||||
@@ -163,7 +180,7 @@ export async function preserveCancelledNativeWorkspaces(db: Db) {
|
||||
|
||||
function pendingCancelledWorkspace() {
|
||||
return sql`${heartbeatRuns.runtimeMode} = 'native' AND ${heartbeatRuns.status} = 'cancelled'
|
||||
AND ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'
|
||||
AND (${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' OR ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared')
|
||||
AND ${heartbeatRuns.resultJson}->'nativeCancellation'->>'dispatchState' = 'acknowledged'
|
||||
AND (${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'
|
||||
AND ${heartbeatRuns.resultJson}->>'workspaceRestoreFailure' IN (${workspaceRestoreFailureCodesSql})) IS NOT TRUE`;
|
||||
@@ -176,7 +193,7 @@ export function registerCancelledWorkspacePreservationWork(db: Db,
|
||||
type Candidate = Pick<typeof heartbeatRuns.$inferSelect, "id" | "companyId" | "agentId" | "issueId" | "nativeIssueId"
|
||||
| "runtimeMode" | "driverKind" | "status" | "resultJson" | "runnerProfileJson">;
|
||||
const eligible = (run: Candidate) => run.nativeIssueId !== null && hasAcknowledgedNativeStopIntent(run)
|
||||
&& readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync)?.state === "prepared"
|
||||
&& workspaceRecoveryReference(run.runnerProfileJson)?.state === "prepared"
|
||||
&& !hasRequiredWorkspaceRecovery(run.resultJson) && !nativeRunHasNoWorkspaceAccess(run);
|
||||
const scopeCondition = (scope?: CancelledWorkspacePreservationScope) => scope ? and(
|
||||
eq(heartbeatRuns.companyId, scope.companyId), scope.runId ? eq(heartbeatRuns.id, scope.runId) : undefined) : undefined;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
|
||||
import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
|
||||
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
|
||||
import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRunChange } from "../unsafe-workspace-export-notifications.js";
|
||||
@@ -629,12 +630,14 @@ export async function claimNativeRestartRecoveries(input: {
|
||||
}
|
||||
|
||||
const profile = row.run.runnerProfileJson ?? {};
|
||||
const remoteWorkspace = readNativeWorkspaceSyncReference(profile.nativeWorkspaceSync);
|
||||
const storedRemoteWorkspace = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace);
|
||||
const remoteWorkspace = readNativeWorkspaceSyncReference(profile.nativeWorkspaceSync)
|
||||
?? (storedRemoteWorkspace.success ? storedRemoteWorkspace.data : null);
|
||||
const [pluginTask] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, row.run.companyId), eq(environmentLeases.heartbeatRunId, row.run.id),
|
||||
sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`,
|
||||
)).orderBy(desc(environmentLeases.createdAt)).limit(1);
|
||||
const remoteSandbox = profile.nativeWorkspaceSync != null;
|
||||
const remoteSandbox = profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null;
|
||||
const remoteProcess = remoteSandbox || !!pluginTask;
|
||||
const durableAuthority = (remoteWorkspace || pluginTask) && await hasNativeRunnerCheckpoint(tx, {
|
||||
companyId: row.run.companyId, issueId: row.coordinator.issueId, runId: row.run.id,
|
||||
|
||||
@@ -9,19 +9,24 @@ import { and, eq, inArray, isNull, sql } from "drizzle-orm";
|
||||
import {
|
||||
agentWakeupRequests,
|
||||
environmentLeases,
|
||||
executionWorkspaces,
|
||||
executionWorkspaceSnapshots,
|
||||
heartbeatRunEvents,
|
||||
heartbeatRuns,
|
||||
issues,
|
||||
issueRecoveryActions,
|
||||
nativeRunFinalizations,
|
||||
toolInvocations,
|
||||
workspaceOperations,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import { decideNativeReplacement } from "./native-replacement-evidence.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { issueRecoveryActionService } from "../issue-recovery-actions.js";
|
||||
import { buildExecutionContinuation } from "../execution-continuation.js";
|
||||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js";
|
||||
|
||||
export const NATIVE_SAFE_REPLACEMENT_REASON = "native_safe_replacement";
|
||||
const record = (v: unknown): Record<string, unknown> =>
|
||||
@@ -38,6 +43,72 @@ const processAlive = (pid: number | null) => {
|
||||
}
|
||||
};
|
||||
|
||||
type ReplacementDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
|
||||
/** A saved remote source is proved by its current ledger, never a controller path. */
|
||||
async function remoteWorkspaceReplacementEvidence(
|
||||
db: ReplacementDb,
|
||||
run: typeof heartbeatRuns.$inferSelect,
|
||||
issueId: string,
|
||||
leases: Array<typeof environmentLeases.$inferSelect>,
|
||||
lock = false,
|
||||
) {
|
||||
const [task] = await db.select({ workspaceId: issues.executionWorkspaceId }).from(issues).where(and(
|
||||
eq(issues.companyId, run.companyId), eq(issues.id, issueId),
|
||||
));
|
||||
const query = task?.workspaceId ? db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, run.companyId), eq(executionWorkspaces.id, task.workspaceId),
|
||||
)) : null;
|
||||
const [workspace] = query ? await (lock ? query.for("update") : query) : [];
|
||||
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
if (run.runnerProfileJson?.remoteWorkspace == null && !authority) return null;
|
||||
if (lock) {
|
||||
const [currentRun] = await db.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id),
|
||||
)).for("update");
|
||||
if (!currentRun || currentRun.status !== "failed") return { workspacePreserved: false, providerStopped: false };
|
||||
run = currentRun;
|
||||
leases = await db.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
)).for("update");
|
||||
}
|
||||
const providerStopped = leases.length > 0 && leases.every(hasRemoteTerminationReceipt);
|
||||
const denied = { workspacePreserved: false, providerStopped };
|
||||
const parsed = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
|
||||
if (!workspace || !authority || !parsed.success || parsed.data.state !== "saved" || !parsed.data.snapshotId
|
||||
|| run.runnerProfileJson?.nativeWorkspaceSync != null || ["archived", "cleanup_failed"].includes(workspace.status)) return denied;
|
||||
const reference = parsed.data;
|
||||
const executionWorkspace = record(record(run.runnerProfileJson?.nativeExecutionInput).workspace);
|
||||
if (workspace.id !== reference.workspaceId || executionWorkspace.cwd !== reference.remoteCwd
|
||||
|| authority.dirty || authority.writerRunId || authority.epoch !== reference.epoch
|
||||
|| authority.snapshotId !== reference.snapshotId) return denied;
|
||||
const lease = leases.find(row => row.id === reference.leaseId && row.providerLeaseId === reference.providerLeaseId
|
||||
&& (!row.executionWorkspaceId || row.executionWorkspaceId === workspace.id)
|
||||
&& row.metadata?.remoteWorkspaceId === workspace.id && row.metadata?.remoteCwd === reference.remoteCwd);
|
||||
if (!lease || !providerStopped) return denied;
|
||||
if (authority.location ? authority.location.leaseId !== lease.id || authority.location.environmentId !== lease.environmentId
|
||||
|| authority.location.providerLeaseId !== reference.providerLeaseId || authority.location.remoteCwd !== reference.remoteCwd
|
||||
: record(lease.metadata?.remoteExecutionTermination).state !== "destroyed") return denied;
|
||||
const snapshotQuery = db.select().from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, run.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
|
||||
eq(executionWorkspaceSnapshots.id, reference.snapshotId!), eq(executionWorkspaceSnapshots.status, "committed"),
|
||||
eq(executionWorkspaceSnapshots.epoch, reference.epoch), eq(executionWorkspaceSnapshots.sourceRunId, run.id),
|
||||
));
|
||||
const [snapshot] = await (lock ? snapshotQuery.for("update") : snapshotQuery);
|
||||
if (!snapshot?.sha256 || !snapshot.objectKey || !snapshot.key || !snapshot.manifestDigest || !snapshot.byteSize) return denied;
|
||||
const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.id, snapshot.sourceOperationId), eq(workspaceOperations.companyId, run.companyId),
|
||||
eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.heartbeatRunId, run.id),
|
||||
eq(workspaceOperations.issueId, issueId), eq(workspaceOperations.phase, "remote_workspace_snapshot"),
|
||||
eq(workspaceOperations.status, "succeeded"),
|
||||
sql`${workspaceOperations.metadata}->>'boundary' = 'final'`,
|
||||
sql`${workspaceOperations.metadata}->>'epoch' = ${String(reference.epoch)}`,
|
||||
sql`${workspaceOperations.metadata}->>'snapshotId' = ${snapshot.id}`,
|
||||
sql`${workspaceOperations.metadata}->>'sha256' = ${snapshot.sha256}`,
|
||||
));
|
||||
return { workspacePreserved: Boolean(operation), providerStopped };
|
||||
}
|
||||
|
||||
export function nativeReplacementPending() {
|
||||
return and(
|
||||
eq(heartbeatRuns.status, "failed"),
|
||||
@@ -208,6 +279,7 @@ export async function reconcileSafeNativeReplacements(
|
||||
} catch {
|
||||
/* The named operator outcome retains incomplete context. */
|
||||
}
|
||||
const remoteWorkspace = await remoteWorkspaceReplacementEvidence(db, run, coordinator.issueId, leases);
|
||||
const decision = decideNativeReplacement({
|
||||
failedSession: true,
|
||||
// A facade transport failure can conceal an authoritative protocol
|
||||
@@ -223,8 +295,9 @@ export async function reconcileSafeNativeReplacements(
|
||||
coordinator.leaseOwner === null && run.status === "failed",
|
||||
providerStopped:
|
||||
!processAlive(run.processPid) &&
|
||||
!processAlive(run.processGroupId ? -run.processGroupId : null),
|
||||
workspacePreserved:
|
||||
!processAlive(run.processGroupId ? -run.processGroupId : null) &&
|
||||
(remoteWorkspace?.providerStopped ?? true),
|
||||
workspacePreserved: remoteWorkspace ? remoteWorkspace.workspacePreserved :
|
||||
typeof workspace.cwd === "string" &&
|
||||
(await stat(workspace.cwd).then(
|
||||
(s) => s.isDirectory(),
|
||||
@@ -333,6 +406,7 @@ export async function reconcileSafeNativeReplacements(
|
||||
(task.checkoutRunId !== null && task.checkoutRunId !== run.id) ||
|
||||
!current ||
|
||||
current.phase !== "terminal_failure" ||
|
||||
current.leaseOwner !== null ||
|
||||
current.failureCode !== coordinator.failureCode ||
|
||||
current.failureDetail?.successorRunId ||
|
||||
current.failureDetail?.replacementDenied ||
|
||||
@@ -353,6 +427,10 @@ export async function reconcileSafeNativeReplacements(
|
||||
});
|
||||
if (!ownsBlock) return false;
|
||||
}
|
||||
if (remoteWorkspace) {
|
||||
const currentWorkspace = await remoteWorkspaceReplacementEvidence(tx, run, coordinator.issueId, leases, true);
|
||||
if (!currentWorkspace?.workspacePreserved || !currentWorkspace.providerStopped) return false;
|
||||
}
|
||||
if (stoppedSession) {
|
||||
const [currentRun] = await tx.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
|
||||
|
||||
@@ -1,13 +1,91 @@
|
||||
import { environmentLeases, heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { environmentLeases, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { and, eq, exists, sql } from "drizzle-orm";
|
||||
import { nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js";
|
||||
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
|
||||
import { hasRequiredWorkspaceRecovery, nativeRunHasNoWorkspaceAccess } from "../workspace-restore-recovery-state.js";
|
||||
import { recordRequiredWorkspaceRestoreFailure } from "../legacy-execution-recovery.js";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-workspace-sync.js";
|
||||
|
||||
type WorkspaceSourceLease = Pick<typeof environmentLeases.$inferSelect,
|
||||
"id" | "companyId" | "heartbeatRunId" | "provider" | "providerLeaseId">;
|
||||
|
||||
/** Preparation publishes file authority before native execution is selected.
|
||||
* Neither runtimeMode nor a later execution-input projection owns these files. */
|
||||
async function preparedRemoteWorkspaceSource(db: Db, lease: WorkspaceSourceLease) {
|
||||
if (!lease.heartbeatRunId || !lease.provider || lease.provider === "local" || !lease.providerLeaseId) return null;
|
||||
const [bound] = await db.select({ lease: environmentLeases, run: {
|
||||
id: heartbeatRuns.id, runtimeMode: heartbeatRuns.runtimeMode,
|
||||
controllerBootId: heartbeatRuns.controllerBootId,
|
||||
issueId: sql<string | null>`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text,
|
||||
${heartbeatRuns.contextSnapshot}->>'issueId')`,
|
||||
reference: sql<unknown>`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'`,
|
||||
legacyReference: sql<unknown>`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'`,
|
||||
} }).from(environmentLeases)
|
||||
.innerJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId),
|
||||
eq(heartbeatRuns.companyId, environmentLeases.companyId)))
|
||||
.where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId),
|
||||
eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId), eq(environmentLeases.provider, lease.provider),
|
||||
eq(environmentLeases.providerLeaseId, lease.providerLeaseId),
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared'`)).limit(1);
|
||||
if (!bound) return null;
|
||||
const parsed = remoteWorkspaceRunReferenceSchema.safeParse(bound.run.reference);
|
||||
if (!parsed.success || parsed.data.state !== "prepared" || bound.run.legacyReference != null) return null;
|
||||
const reference = parsed.data;
|
||||
const issueId = bound.run.issueId;
|
||||
if (typeof issueId !== "string" || bound.lease.issueId !== issueId || reference.leaseId !== bound.lease.id
|
||||
|| reference.providerLeaseId !== bound.lease.providerLeaseId || reference.remoteCwd !== bound.lease.metadata?.remoteCwd
|
||||
|| bound.lease.executionWorkspaceId && bound.lease.executionWorkspaceId !== reference.workspaceId) return null;
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, lease.companyId), eq(executionWorkspaces.id, reference.workspaceId))).limit(1);
|
||||
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
if (!workspace || workspace.sourceIssueId !== issueId && workspace.mode !== "shared_workspace"
|
||||
|| !authority || !authority.dirty || authority.writerRunId !== bound.run.id || authority.epoch !== reference.epoch
|
||||
|| authority.snapshotId !== reference.snapshotId || !authority.location
|
||||
|| authority.location.environmentId !== bound.lease.environmentId || authority.location.leaseId !== bound.lease.id
|
||||
|| authority.location.providerLeaseId !== bound.lease.providerLeaseId || authority.location.remoteCwd !== reference.remoteCwd) return null;
|
||||
return { ...bound, workspace: workspace!, reference, issueId };
|
||||
}
|
||||
|
||||
/** File-only recovery for setup that never reached native admission. The live
|
||||
* preparer must still own its controller; orphan repair accepts terminal runs
|
||||
* only. This records no provider result and never starts a provider turn. */
|
||||
export async function preserveRemoteWorkspacePreparationFailure(db: Db, lease: WorkspaceSourceLease,
|
||||
options: { assertOwned?: () => Promise<void> } = {}) {
|
||||
const candidate = await preparedRemoteWorkspaceSource(db, lease);
|
||||
if (!candidate || candidate.run.runtimeMode !== "legacy") return null;
|
||||
// This callback renews the run on the liveness connection. Finish it before
|
||||
// locking that row; the captured owner is rechecked under the lock below.
|
||||
await options.assertOwned?.();
|
||||
return db.transaction(async tx => {
|
||||
await tx.execute(sql`select set_config('statement_timeout', '15000', true), set_config('lock_timeout', '1000', true)`);
|
||||
const [task] = await tx.select({ id: issues.id }).from(issues).where(and(
|
||||
eq(issues.companyId, lease.companyId), eq(issues.id, candidate.issueId))).for("update");
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, candidate.run.id))).for("update");
|
||||
if (!task || !run || run.runtimeMode !== "legacy" || run.runnerProfileJson?.nativeExecutionInput != null
|
||||
|| options.assertOwned && (run.controllerBootId !== candidate.run.controllerBootId
|
||||
|| run.controllerBootId !== null && (!run.controllerLeaseExpiresAt || run.controllerLeaseExpiresAt.getTime() <= Date.now()))
|
||||
|| (!options.assertOwned && (!["failed", "cancelled", "timed_out", "interrupted"].includes(run.status)
|
||||
|| run.controllerLeaseExpiresAt && run.controllerLeaseExpiresAt.getTime() > Date.now()))) return null;
|
||||
const [coordinator] = await tx.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations)
|
||||
.where(and(eq(nativeRunFinalizations.companyId, lease.companyId), eq(nativeRunFinalizations.runId, run.id)));
|
||||
if (coordinator) return null;
|
||||
await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, lease.companyId), eq(executionWorkspaces.id, candidate.reference.workspaceId))).for("update");
|
||||
await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, lease.companyId), eq(environmentLeases.id, lease.id))).for("update");
|
||||
const current = await preparedRemoteWorkspaceSource(tx as unknown as Db, lease);
|
||||
if (!current || current.issueId !== task.id || current.run.id !== run.id) return null;
|
||||
if (options.assertOwned && run.controllerBootId !== null
|
||||
&& (!run.controllerLeaseExpiresAt || run.controllerLeaseExpiresAt.getTime() <= Date.now())) return null;
|
||||
if (hasRequiredWorkspaceRecovery(run.resultJson)) return run;
|
||||
return await recordRequiredWorkspaceRestoreFailure(tx as unknown as Db, run,
|
||||
{ workspaceRestoreFailure: "restore_failed" }, current.lease);
|
||||
});
|
||||
}
|
||||
|
||||
export async function nativeWorkspaceExportPendingForLease(db: Db, lease: WorkspaceSourceLease): Promise<boolean> {
|
||||
if (!lease.heartbeatRunId || !lease.provider || lease.provider === "local" || !lease.providerLeaseId) return false;
|
||||
if (await preparedRemoteWorkspaceSource(db, lease)) return true;
|
||||
const input = sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'`;
|
||||
const [run] = await db.select({
|
||||
id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { classifyNativeWorkspaceFailure } from "./native-workspace-failure.js";
|
||||
import { readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "../remote-workspaces.js";
|
||||
import { recordNativeFinalizationFailure } from "./native-run-finalizer.js";
|
||||
import { inspectPluginWorkspaceForFinalization } from "./native-plugin-workspace-finalization.js";
|
||||
import type { WorkspaceWorkers } from "../plugin-workspaces.js";
|
||||
@@ -157,6 +158,42 @@ export async function attemptNativeWorkspaceFinalization(input: {
|
||||
},
|
||||
run: async () => {
|
||||
await ownership.assertHeld();
|
||||
const remoteAuthority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
if (remoteAuthority && workspace) {
|
||||
if (!input.environmentRuntime || !remoteAuthority.location) {
|
||||
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
|
||||
}
|
||||
const location = remoteAuthority.location;
|
||||
const environmentsSvc = environmentService(input.db);
|
||||
const lease = await environmentsSvc.getLeaseById(location.leaseId);
|
||||
const environment = lease?.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
|
||||
if (!lease || !environment || lease.companyId !== bound.companyId
|
||||
|| lease.heartbeatRunId !== input.runId || lease.executionWorkspaceId !== workspace.id
|
||||
|| lease.providerLeaseId !== location.providerLeaseId || environment.id !== location.environmentId
|
||||
|| ["expired", "failed", "pending_cleanup"].includes(lease.status)) {
|
||||
return workspaceSyncFailure("workspace_sync_out_unrecoverable");
|
||||
}
|
||||
const target = await resolveEnvironmentExecutionTarget({
|
||||
db: input.db, companyId: bound.companyId, adapterType: "paperclip_runner",
|
||||
environment, leaseId: lease.id, leaseMetadata: lease.metadata, lease,
|
||||
environmentRuntime: input.environmentRuntime,
|
||||
});
|
||||
if (!target) return workspaceSyncFailure("workspace_sync_out_unrecoverable");
|
||||
try {
|
||||
const saved = await saveRemoteWorkspace({
|
||||
db: input.db, workspace, runId: input.runId, lease, target,
|
||||
assertOwned: ownership.assertHeld,
|
||||
});
|
||||
await ownership.assertHeld();
|
||||
return { status: "succeeded", exitCode: 0,
|
||||
system: "Native workspace files were saved to durable storage.\n",
|
||||
metadata: { remoteWorkspace: saved } };
|
||||
} catch (error) {
|
||||
await ownership.assertHeld();
|
||||
return workspaceSyncFailure(classifyNativeWorkspaceFailure(error).code === "workspace_sync_out_unrecoverable"
|
||||
? "workspace_sync_out_unrecoverable" : "workspace_sync_out_failed");
|
||||
}
|
||||
}
|
||||
if (nativeWorkspaceSync) {
|
||||
if (!input.environmentRuntime) {
|
||||
return {
|
||||
|
||||
@@ -1342,6 +1342,19 @@ export class PaperclipRunnerToolAuthority {
|
||||
return { bytes, filename: basename(file.path!), contentType: "application/octet-stream" };
|
||||
}
|
||||
const resolved = await workspaceFileResourceService(this.db).prepareDownload(this.binding.issueId, { path: file.path!, workspace: "auto" });
|
||||
if ("stream" in resolved) {
|
||||
try {
|
||||
const chunks: Buffer[] = [];
|
||||
let length = 0;
|
||||
for await (const chunk of resolved.stream) {
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
length += bytes.length;
|
||||
if (length > RUNNER_API_MAX_BYTES) throw badRequest("Workspace file exceeds API transfer limit");
|
||||
chunks.push(bytes);
|
||||
}
|
||||
return { bytes: Buffer.concat(chunks), filename: resolved.resource.title, contentType: resolved.resource.contentType ?? "application/octet-stream" };
|
||||
} finally { await resolved.cleanup(); }
|
||||
}
|
||||
const handle = await openRunnerApiWorkspaceFile(resolved.realPath);
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
|
||||
@@ -6,6 +6,7 @@ import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { executionWorkspaces, projectWorkspaces, type Db } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync";
|
||||
import { acquireGitExpirationLocks, recoverGitExpirationLocks, type GitExpirationLockLocation } from "./git-workspace-expiration-locks.js";
|
||||
import { allocatedBytes, withAnonymousGitFallback, type GitExpirationFailure } from "./git-workspace-expiration.js";
|
||||
@@ -75,17 +76,24 @@ async function canonicalPath(target: string) {
|
||||
return path.join(await fs.realpath(path.dirname(target)), path.basename(target));
|
||||
}
|
||||
|
||||
function hasRemoteStorageFence(workspace: Pick<Workspace, "metadata">) {
|
||||
const migration = workspace.metadata?.remoteWorkspaceMigration;
|
||||
return Boolean(readRemoteWorkspaceAuthority(workspace.metadata)) || Boolean(migration && typeof migration === "object"
|
||||
&& !Array.isArray(migration) && (migration as Record<string, unknown>).state === "running");
|
||||
}
|
||||
|
||||
export function isNestedCloneMigrationCandidate(workspace: Workspace) {
|
||||
return workspace.providerType !== "plugin" && OPEN_STATUSES.includes(workspace.status) && !workspace.closedAt
|
||||
return !hasRemoteStorageFence(workspace) && workspace.providerType !== "plugin" && OPEN_STATUSES.includes(workspace.status) && !workspace.closedAt
|
||||
&& Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId);
|
||||
}
|
||||
|
||||
function canRecover(workspace: Workspace, name: string) {
|
||||
return workspace.providerType !== "plugin" && Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId)
|
||||
return !hasRemoteStorageFence(workspace) && workspace.providerType !== "plugin" && Boolean(workspace.cwd && workspace.projectId && workspace.sourceIssueId)
|
||||
&& pendingRecords(workspace)[name] !== undefined;
|
||||
}
|
||||
|
||||
function pendingRecords(workspace: Pick<Workspace, "metadata">) {
|
||||
if (hasRemoteStorageFence(workspace)) return {};
|
||||
const value = workspace.metadata?.[NESTED_CLONE_MIGRATION_PENDING_KEY];
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import type { Readable } from "node:stream";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { environments, environmentLeases, executionWorkspaces, type Db } from "@paperclipai/db";
|
||||
import type { Environment, EnvironmentLease, WorkspaceFileListMode, WorkspaceFileSource } from "@paperclipai/shared";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const READ_TIMEOUT_MS = 30_000;
|
||||
const MAX_HELPER_OUTPUT_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
export interface RemoteWorkspaceFileEntry {
|
||||
kind: "file" | "directory";
|
||||
relativePath: string;
|
||||
byteSize: number | null;
|
||||
modifiedAt: string | null;
|
||||
text?: boolean;
|
||||
sha256?: string;
|
||||
}
|
||||
|
||||
export interface RemoteWorkspaceFileListInput {
|
||||
relativePath?: string;
|
||||
mode: WorkspaceFileListMode;
|
||||
query?: string | null;
|
||||
limit: number;
|
||||
offset: number;
|
||||
maxScannedEntries: number;
|
||||
maxDepth: number;
|
||||
}
|
||||
|
||||
export interface RemoteWorkspaceDownload {
|
||||
stream: Readable;
|
||||
byteSize: number;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface RemoteWorkspaceFileAccess {
|
||||
source: WorkspaceFileSource;
|
||||
stat(relativePath: string): Promise<RemoteWorkspaceFileEntry>;
|
||||
read(relativePath: string, options: { maxBytes: number }): Promise<Buffer>;
|
||||
list(input: RemoteWorkspaceFileListInput): Promise<{
|
||||
entries: RemoteWorkspaceFileEntry[];
|
||||
scannedCount: number;
|
||||
truncated: boolean;
|
||||
}>;
|
||||
download(relativePath: string): Promise<RemoteWorkspaceDownload>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface SavedWorkspaceFiles {
|
||||
root: string;
|
||||
savedAt: string;
|
||||
cleanup(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface RemoteWorkspaceFileOptions {
|
||||
environmentRuntime?: Pick<EnvironmentRuntimeService, "execute" | "syncOut">;
|
||||
loadSavedWorkspace?(input: {
|
||||
companyId: string;
|
||||
workspaceId: string;
|
||||
snapshotId: string;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<SavedWorkspaceFiles>;
|
||||
}
|
||||
|
||||
export interface RemoteWorkspaceFiles {
|
||||
open(input: { companyId: string; workspaceId: string; signal?: AbortSignal }): Promise<RemoteWorkspaceFileAccess | null>;
|
||||
}
|
||||
|
||||
// Run the same confined helper in the execution environment and in a verified,
|
||||
// disposable saved-version directory. No absolute path comes from a file caller.
|
||||
export const REMOTE_WORKSPACE_FILES_SCRIPT = String.raw`
|
||||
const fs = require('node:fs/promises');
|
||||
const { constants } = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { execFile } = require('node:child_process');
|
||||
const { promisify } = require('node:util');
|
||||
const input = JSON.parse(process.argv[1]);
|
||||
const denied = new Set(['.git','.paperclip','.paperclip-runtime','node_modules','.pnpm-store','.yarn','.cache','.turbo','.next','.vite','.vercel','dist','build','coverage','runtime-services','.runtime']);
|
||||
const fail = code => { const error = new Error(code); error.code = code; throw error; };
|
||||
const deniedPath = relative => {
|
||||
const parts = relative.toLowerCase().split('/');
|
||||
const file = parts.at(-1) || '';
|
||||
return parts.some(part => denied.has(part) || part === '.aws' || part === '.ssh') ||
|
||||
file === '.env' || file.startsWith('.env.') || /\.(pem|key|p12|pfx)$/.test(file) ||
|
||||
['id_rsa','id_ed25519','.npmrc','.pypirc','.netrc','kubeconfig'].includes(file) ||
|
||||
(parts.at(-2) === '.docker' && file === 'config.json') || (parts.at(-2) === '.kube' && file === 'config');
|
||||
};
|
||||
const normalize = (relative, allowRoot = false) => {
|
||||
if (allowRoot && !relative) return '';
|
||||
if (typeof relative !== 'string' || !relative || relative.length > 4096 || /[\\\u0000-\u001f\u007f]/u.test(relative) ||
|
||||
path.isAbsolute(relative) || /^[a-z][a-z0-9+.-]*:/iu.test(relative) || relative.startsWith('~') ||
|
||||
relative.split('/').some(part => part === '..' || part === '.' || part === '')) fail('invalid_path');
|
||||
if (deniedPath(relative)) fail('denied_path');
|
||||
return relative;
|
||||
};
|
||||
const same = (a,b) => ['dev','ino','size','mtimeNs','ctimeNs'].every(key => a[key] === b[key]);
|
||||
const within = (root,target) => { const relative = path.relative(root,target); return relative === '' || (relative !== '..' && !relative.startsWith('../') && !path.isAbsolute(relative)); };
|
||||
(async () => {
|
||||
const rootEntry = await fs.lstat(input.root);
|
||||
if (!rootEntry.isDirectory() || rootEntry.isSymbolicLink()) fail('workspace_unavailable');
|
||||
const root = await fs.realpath(input.root);
|
||||
const confined = async (relative, allowRoot = false) => {
|
||||
relative = normalize(relative, allowRoot);
|
||||
let current = root;
|
||||
for (const part of relative ? relative.split('/') : []) {
|
||||
current = path.join(current,part);
|
||||
const stat = await fs.lstat(current);
|
||||
if (stat.isSymbolicLink()) fail('symlink_denied');
|
||||
}
|
||||
const canonical = await fs.realpath(current);
|
||||
if (!within(root,canonical) || canonical !== current) fail('outside_workspace');
|
||||
return canonical;
|
||||
};
|
||||
const entry = async (relative, withHash = false) => {
|
||||
const target = await confined(relative);
|
||||
const stat = await fs.lstat(target);
|
||||
if (!stat.isDirectory() && !stat.isFile()) fail('not_regular_file');
|
||||
const result = {kind:stat.isDirectory()?'directory':'file',relativePath:relative,byteSize:stat.isFile()?stat.size:null,modifiedAt:stat.mtime.toISOString()};
|
||||
if (stat.isFile() && input.action === 'stat') {
|
||||
const handle = await fs.open(target,constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try {
|
||||
const before = await handle.stat({bigint:true});
|
||||
if (!before.isFile() || before.nlink !== 1n) fail('not_regular_file');
|
||||
const sample = Buffer.alloc(4096);
|
||||
const {bytesRead} = await handle.read(sample,0,sample.length,0);
|
||||
let controls = 0;
|
||||
let nul = false;
|
||||
for (const byte of sample.subarray(0,bytesRead)) { if (!byte) nul = true; if (byte < 32 && ![9,10,13].includes(byte)) controls++; }
|
||||
result.text = !nul && (!bytesRead || controls / bytesRead < .02);
|
||||
if (withHash) { const hash = createHash('sha256'); for await (const bytes of handle.createReadStream({start:0,autoClose:false})) hash.update(bytes); result.sha256=hash.digest('hex'); }
|
||||
const after = await handle.stat({bigint:true});
|
||||
const current = await fs.lstat(target,{bigint:true});
|
||||
if (!same(before,after) || !same(after,current) || await confined(relative) !== target) fail('file_changed');
|
||||
} finally { await handle.close(); }
|
||||
}
|
||||
return result;
|
||||
};
|
||||
let result;
|
||||
if (input.action === 'stat') result=await entry(input.relativePath,input.withHash === true);
|
||||
else if (input.action === 'list') {
|
||||
const requested = input.options;
|
||||
const start = normalize(requested.relativePath || '',true);
|
||||
const startPath = await confined(start,true);
|
||||
if (!(await fs.stat(startPath)).isDirectory()) fail('not_directory');
|
||||
const maxScanned = Math.min(5000,Math.max(1,requested.maxScannedEntries));
|
||||
const maxDepth = Math.min(20,Math.max(0,requested.maxDepth));
|
||||
const limit = Math.min(100,Math.max(1,requested.limit));
|
||||
const offset = Math.min(10000,Math.max(0,requested.offset));
|
||||
const query = (requested.query || '').toLowerCase();
|
||||
let scannedCount=0,truncated=false;
|
||||
const entries=[];
|
||||
const matches = relative => !query || relative.toLowerCase().includes(query);
|
||||
const add = async relative => {
|
||||
try { const value=await entry(relative); if(matches(relative)) entries.push(value); }
|
||||
catch(error) { if(!['ENOENT','ENOTDIR','denied_path','symlink_denied','not_regular_file'].includes(error.code)) throw error; }
|
||||
};
|
||||
if (requested.mode === 'changed') {
|
||||
let stdout;
|
||||
try { ({stdout}=await promisify(execFile)('git',['-c','core.fsmonitor=false','-c','core.hooksPath=/dev/null','-C',root,'status','--porcelain=v1','-z','--untracked-files=all'],{encoding:'utf8',timeout:10000,maxBuffer:1048576,env:{...process.env,GIT_OPTIONAL_LOCKS:'0',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null'}})); }
|
||||
catch { fail('changed_unavailable'); }
|
||||
const tokens=stdout.split('\0').filter(Boolean);
|
||||
for(let i=0;i<tokens.length;i++) {
|
||||
if(++scannedCount>maxScanned) {truncated=true;break;}
|
||||
const token=tokens[i];
|
||||
const status=token.slice(0,2),relative=token.slice(3);
|
||||
if(status.includes('R')||status.includes('C')) i++;
|
||||
if(relative && !deniedPath(relative)) await add(relative);
|
||||
}
|
||||
} else {
|
||||
const immediate=requested.mode==='all'&&!query;
|
||||
const queue=[{relative:start,depth:0}];
|
||||
scan: while(queue.length) {
|
||||
const current=queue.shift();
|
||||
const directory=await confined(current.relative,true);
|
||||
const children=await fs.readdir(directory,{withFileTypes:true});
|
||||
children.sort((a,b)=>immediate&&a.isDirectory()!==b.isDirectory()?(a.isDirectory()?-1:1):a.name.localeCompare(b.name));
|
||||
for(const child of children) {
|
||||
if(++scannedCount>maxScanned) {truncated=true;break scan;}
|
||||
const relative=current.relative?current.relative+'/'+child.name:child.name;
|
||||
if(deniedPath(relative)||child.isSymbolicLink()||/[\\\u0000-\u001f\u007f]/u.test(relative)) continue;
|
||||
if(child.isDirectory()) {
|
||||
if(immediate) await add(relative);
|
||||
else if(current.depth<maxDepth) queue.push({relative,depth:current.depth+1});
|
||||
else truncated=true;
|
||||
} else if(child.isFile()) await add(relative);
|
||||
if(immediate&&entries.length>offset+limit) {truncated=true;break scan;}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(requested.mode==='recent') entries.sort((a,b)=>(b.modifiedAt||'').localeCompare(a.modifiedAt||'')||a.relativePath.localeCompare(b.relativePath));
|
||||
else if(requested.mode==='changed') entries.sort((a,b)=>a.relativePath.localeCompare(b.relativePath));
|
||||
result={entries:entries.slice(offset,offset+limit),scannedCount:Math.min(scannedCount,maxScanned),truncated:truncated||entries.length>offset+limit};
|
||||
} else fail('invalid_operation');
|
||||
if(await fs.realpath(input.root)!==root) fail('workspace_unavailable');
|
||||
process.stdout.write(JSON.stringify(result));
|
||||
})().catch(error => {
|
||||
const allowed=new Set(['invalid_path','denied_path','symlink_denied','outside_workspace','workspace_unavailable','not_regular_file','not_directory','file_changed','changed_unavailable','ENOENT','ENOTDIR']);
|
||||
process.stderr.write(allowed.has(error.code)?error.code:'remote_read_failed');
|
||||
process.exitCode=1;
|
||||
});
|
||||
`;
|
||||
|
||||
function fileFailure(code: string): HttpError {
|
||||
if (["ENOENT", "ENOTDIR"].includes(code)) return notFound("Workspace file not found", { code: "not_found" });
|
||||
if (["denied_path", "symlink_denied", "outside_workspace", "invalid_path"].includes(code)) {
|
||||
return new HttpError(403, "Workspace file path is denied", { code });
|
||||
}
|
||||
const safe = new Set(["workspace_unavailable", "workspace_changed", "not_regular_file", "not_directory", "file_changed", "changed_unavailable", "too_large"]);
|
||||
return unprocessable("Workspace files are unavailable", { code: safe.has(code) ? code : "workspace_unavailable" });
|
||||
}
|
||||
|
||||
function parseFileEntry(value: unknown): RemoteWorkspaceFileEntry {
|
||||
const entry = value as Partial<RemoteWorkspaceFileEntry> | null;
|
||||
if (!entry || !["file", "directory"].includes(entry.kind ?? "") || typeof entry.relativePath !== "string" ||
|
||||
!entry.relativePath || entry.relativePath.length > 4096 || /[\\\u0000-\u001f\u007f]/u.test(entry.relativePath) ||
|
||||
path.posix.isAbsolute(entry.relativePath) || entry.relativePath.split("/").some(part => !part || part === "." || part === "..") ||
|
||||
(entry.kind === "file" ? !Number.isSafeInteger(entry.byteSize) || entry.byteSize! < 0 : entry.byteSize !== null) ||
|
||||
(entry.modifiedAt !== null && (typeof entry.modifiedAt !== "string" || !Number.isFinite(Date.parse(entry.modifiedAt)))) ||
|
||||
(entry.text !== undefined && typeof entry.text !== "boolean") ||
|
||||
(entry.sha256 !== undefined && !/^[a-f0-9]{64}$/.test(entry.sha256))) throw fileFailure("workspace_unavailable");
|
||||
return entry as RemoteWorkspaceFileEntry;
|
||||
}
|
||||
|
||||
function parseHelperResult(action: "stat" | "list", value: unknown, withHash: boolean): unknown {
|
||||
if (action === "stat") {
|
||||
const entry = parseFileEntry(value);
|
||||
if (withHash && entry.kind === "file" && !entry.sha256) throw fileFailure("workspace_unavailable");
|
||||
return entry;
|
||||
}
|
||||
const list = value as { entries?: unknown; scannedCount?: unknown; truncated?: unknown } | null;
|
||||
if (!list || !Array.isArray(list.entries) || list.entries.length > 100 ||
|
||||
!Number.isSafeInteger(list.scannedCount) || Number(list.scannedCount) < 0 || Number(list.scannedCount) > 5000 ||
|
||||
typeof list.truncated !== "boolean") throw fileFailure("workspace_unavailable");
|
||||
return { entries: list.entries.map(parseFileEntry), scannedCount: list.scannedCount, truncated: list.truncated };
|
||||
}
|
||||
|
||||
async function verifyDownload(file: string, expected: RemoteWorkspaceFileEntry): Promise<number> {
|
||||
const handle = await fs.open(file, "r");
|
||||
try {
|
||||
const stat = await handle.stat();
|
||||
if (!stat.isFile() || stat.size !== expected.byteSize) throw fileFailure("file_changed");
|
||||
const hash = createHash("sha256");
|
||||
for await (const bytes of handle.createReadStream({ autoClose: false })) hash.update(bytes);
|
||||
if (expected.sha256 && hash.digest("hex") !== expected.sha256) throw fileFailure("file_changed");
|
||||
return stat.size;
|
||||
} finally { await handle.close(); }
|
||||
}
|
||||
|
||||
export function createRemoteWorkspaceFiles(db: Db, options: RemoteWorkspaceFileOptions = {}): RemoteWorkspaceFiles {
|
||||
return {
|
||||
async open(input) {
|
||||
input.signal?.throwIfAborted();
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId),
|
||||
)).limit(1);
|
||||
if (!workspace) throw notFound("Workspace not found");
|
||||
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!authority) return null;
|
||||
const initialAuthority = JSON.stringify(authority);
|
||||
const checkCurrent = async () => {
|
||||
input.signal?.throwIfAborted();
|
||||
const [current] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId),
|
||||
)).limit(1);
|
||||
if (!current || JSON.stringify(readRemoteWorkspaceAuthority(current.metadata)) !== initialAuthority) {
|
||||
throw fileFailure("workspace_changed");
|
||||
}
|
||||
};
|
||||
let saved: SavedWorkspaceFiles | undefined;
|
||||
let environment: Environment | undefined;
|
||||
let lease: EnvironmentLease | undefined;
|
||||
let source: WorkspaceFileSource;
|
||||
let root: string;
|
||||
const loadSaved = async () => {
|
||||
if (!authority.snapshotId || authority.dirty || authority.writerRunId || !options.loadSavedWorkspace) {
|
||||
throw fileFailure("workspace_unavailable");
|
||||
}
|
||||
saved = await options.loadSavedWorkspace({ companyId: input.companyId, workspaceId: input.workspaceId, snapshotId: authority.snapshotId,
|
||||
...(input.signal ? { signal: input.signal } : {}) });
|
||||
return { root: saved.root, source: { kind: "saved", savedAt: saved.savedAt, snapshotId: authority.snapshotId } as WorkspaceFileSource };
|
||||
};
|
||||
if (authority.location) {
|
||||
const [leaseRow] = await db.select().from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, authority.location.leaseId),
|
||||
)).limit(1);
|
||||
const [environmentRow] = await db.select().from(environments).where(eq(environments.id, authority.location.environmentId)).limit(1);
|
||||
if (!leaseRow || !environmentRow || leaseRow.environmentId !== environmentRow.id ||
|
||||
leaseRow.providerLeaseId !== authority.location.providerLeaseId ||
|
||||
leaseRow.metadata?.remoteCwd !== authority.location.remoteCwd) throw fileFailure("workspace_unavailable");
|
||||
if (hasRemoteTerminationReceipt(leaseRow)
|
||||
&& (leaseRow.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "stopped") {
|
||||
({ root, source } = await loadSaved());
|
||||
} else {
|
||||
if (!options.environmentRuntime || !["active", "retained", "pending_cleanup"].includes(leaseRow.status)) {
|
||||
throw fileFailure("workspace_unavailable");
|
||||
}
|
||||
lease = leaseRow as EnvironmentLease;
|
||||
environment = environmentRow as Environment;
|
||||
root = authority.location.remoteCwd;
|
||||
source = { kind: "live", savedAt: null, snapshotId: null };
|
||||
}
|
||||
} else {
|
||||
({ root, source } = await loadSaved());
|
||||
}
|
||||
let closed = false;
|
||||
const close = async () => { if (closed) return; closed = true; await saved?.cleanup(); };
|
||||
const execute = async <T>(action: "stat" | "list", args: Record<string, unknown>): Promise<T> => {
|
||||
if (closed) throw fileFailure("workspace_unavailable");
|
||||
await checkCurrent();
|
||||
const payload = JSON.stringify({ root, action, ...args });
|
||||
let stdout: string;
|
||||
if (saved) {
|
||||
try {
|
||||
({ stdout } = await execFileAsync(process.execPath, ["--input-type=commonjs", "-e", REMOTE_WORKSPACE_FILES_SCRIPT, payload], {
|
||||
cwd: root, timeout: READ_TIMEOUT_MS, maxBuffer: MAX_HELPER_OUTPUT_BYTES,
|
||||
signal: input.signal, env: { ...process.env, NODE_OPTIONS: "", NODE_PATH: "" },
|
||||
}));
|
||||
} catch (error) { throw fileFailure(String((error as { stderr?: string }).stderr ?? "workspace_unavailable").trim()); }
|
||||
} else {
|
||||
const result = await options.environmentRuntime!.execute({
|
||||
environment: environment!, lease: lease!, command: "node",
|
||||
args: ["--input-type=commonjs", "-e", REMOTE_WORKSPACE_FILES_SCRIPT, payload],
|
||||
cwd: root, timeoutMs: READ_TIMEOUT_MS, bypassSession: true, env: { NODE_OPTIONS: "", NODE_PATH: "" },
|
||||
}).catch(() => { throw fileFailure("workspace_unavailable"); });
|
||||
if (result.timedOut || result.exitCode !== 0) throw fileFailure(result.timedOut ? "workspace_unavailable" : result.stderr.trim());
|
||||
stdout = result.stdout;
|
||||
}
|
||||
await checkCurrent();
|
||||
try { return parseHelperResult(action, JSON.parse(stdout), args.withHash === true) as T; }
|
||||
catch { throw fileFailure("workspace_unavailable"); }
|
||||
};
|
||||
const stat = (relativePath: string) => execute<RemoteWorkspaceFileEntry>("stat", { relativePath });
|
||||
const download = async (relativePath: string): Promise<RemoteWorkspaceDownload> => {
|
||||
const expected = await execute<RemoteWorkspaceFileEntry>("stat", { relativePath, withHash: true });
|
||||
if (expected.kind !== "file") throw fileFailure("not_regular_file");
|
||||
if (saved) {
|
||||
await checkCurrent();
|
||||
const file = path.join(root, ...relativePath.split("/"));
|
||||
return { stream: createReadStream(file), byteSize: expected.byteSize!, cleanup: async () => {} };
|
||||
}
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-file-"));
|
||||
const cleanup = async () => { await fs.rm(directory, { recursive: true, force: true }); };
|
||||
try {
|
||||
await options.environmentRuntime!.syncOut({ environment: environment!, lease: lease!, operations: [{
|
||||
operationId: `workspace-file:${randomUUID()}`,
|
||||
files: [{ sourcePath: path.posix.join(root, relativePath), targetPath: path.join(directory, "file"), kind: "file", mode: 0o600 }],
|
||||
}] }).catch(() => { throw fileFailure("workspace_unavailable"); });
|
||||
await verifyDownload(path.join(directory, "file"), expected);
|
||||
await checkCurrent();
|
||||
return { stream: createReadStream(path.join(directory, "file")), byteSize: expected.byteSize!, cleanup };
|
||||
} catch (error) { await cleanup(); throw error; }
|
||||
};
|
||||
return {
|
||||
source, stat, close,
|
||||
list: listInput => execute("list", { options: listInput }),
|
||||
async read(relativePath, readOptions) {
|
||||
const expected = await stat(relativePath);
|
||||
if (expected.kind !== "file") throw fileFailure("not_regular_file");
|
||||
if (expected.byteSize! > readOptions.maxBytes) throw fileFailure("too_large");
|
||||
const transfer = await download(relativePath);
|
||||
try {
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
for await (const chunk of transfer.stream) {
|
||||
input.signal?.throwIfAborted();
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
||||
size += bytes.length;
|
||||
if (size > readOptions.maxBytes) throw fileFailure("too_large");
|
||||
chunks.push(bytes);
|
||||
}
|
||||
await checkCurrent();
|
||||
return Buffer.concat(chunks);
|
||||
} finally { transfer.stream.destroy(); await transfer.cleanup(); }
|
||||
},
|
||||
download,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, environmentLeases, environments,
|
||||
executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, projects, workspaceOperations, workspaceRuntimeServices, type Db } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, type RemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
|
||||
import { environmentRuntimeService } from "./environment-runtime.js";
|
||||
import { environmentService } from "./environments.js";
|
||||
import { executionWorkspaceService, toExecutionWorkspace } from "./execution-workspaces.js";
|
||||
import { remoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, restartDesiredRuntimeServicesOnStartup, startRuntimeServicesForWorkspaceControl } from "./workspace-runtime.js";
|
||||
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
import { workspaceOperationService } from "./workspace-operations.js";
|
||||
import { inventoryRemoteWorkspaceMigration } from "./remote-workspace-migration.js";
|
||||
import { hasPendingNestedCloneMigration, isNestedCloneMigrationCandidate, listNestedCloneMigrationNames,
|
||||
migrateNestedRepositoryClone, pendingNestedCloneMigrationNames } from "./nested-clone-migration.js";
|
||||
|
||||
const url = resolveExternalTestDatabaseUrl("PAPERCLIP_REMOTE_WORKSPACE_LIFECYCLE_TEST_DATABASE_URL");
|
||||
describe.skipIf(!url)("remote workspace close and allocation cleanup", () => {
|
||||
let db: Db;
|
||||
const directories: string[] = [];
|
||||
beforeAll(async () => { await applyPendingMigrations(url!); db = createDb(url!); }, 120_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
await db.execute(sql`truncate table companies, execution_workspace_snapshots, environments restart identity cascade`);
|
||||
});
|
||||
afterEach(async () => { vi.restoreAllMocks(); await Promise.all(directories.splice(0).map(path => rm(path, { recursive: true, force: true }))); });
|
||||
afterAll(async () => { await closeRegisteredClients(url!, { scope: "database" }); });
|
||||
|
||||
async function fixture(receiptState: "destroyed" | "stopped" | "invalid" = "destroyed") {
|
||||
const [company] = await db.insert(companies).values({ name: "Remote close", issuePrefix: "RC" }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Native worker", adapterType: "paperclip_runner" }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Saved work", status: "done",
|
||||
completedAt: new Date(0), assigneeAgentId: agent.id }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id,
|
||||
nativeIssueId: issue.id, runtimeMode: "native", status: "succeeded", finishedAt: new Date() }).returning();
|
||||
const [environment] = await db.insert(environments).values({ name: `Remote close ${randomUUID()}`, driver: "sandbox", config: { provider: "daytona" } }).returning();
|
||||
const cwd = await mkdtemp(join(tmpdir(), "paperclip-remote-close-")); directories.push(cwd);
|
||||
await writeFile(join(cwd, "host-sentinel.txt"), "Host files must never be cleaned as remote files.");
|
||||
const workspaceId = randomUUID(), leaseId = randomUUID(), snapshotId = randomUUID(), providerLeaseId = `allocation-${randomUUID()}`;
|
||||
const authority: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1,
|
||||
snapshotId, source: { kind: "empty" }, writerRunId: null, dirty: false,
|
||||
location: { environmentId: environment.id, leaseId, providerLeaseId, remoteCwd: cwd } };
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: company.id, sourceIssueId: issue.id,
|
||||
name: "Remote files", cwd, providerRef: cwd, providerType: "cloud_sandbox", mode: "isolated_workspace",
|
||||
strategyType: "git_worktree", status: "active", metadata: { remoteWorkspace: authority } }).returning();
|
||||
await db.update(issues).set({ executionWorkspaceId: workspace.id }).where(eq(issues.id, issue.id));
|
||||
await db.insert(executionWorkspaceSnapshots).values({ id: snapshotId, companyId: company.id,
|
||||
executionWorkspaceId: workspace.id, sourceOperationId: randomUUID(), sourceRunId: run.id,
|
||||
generation: 1, epoch: 1, status: "committed", objectKey: "saved-workspace-object", sha256: "a".repeat(64), byteSize: 1 });
|
||||
await db.insert(environmentLeases).values({ id: leaseId, companyId: company.id, environmentId: environment.id,
|
||||
heartbeatRunId: run.id, issueId: issue.id, executionWorkspaceId: workspace.id, provider: "daytona", providerLeaseId,
|
||||
leasePolicy: "ephemeral", status: "active", metadata: { driver: "sandbox", remoteCwd: cwd } });
|
||||
const environmentsSvc = environmentService(db);
|
||||
const base = environmentRuntimeService(db).getDriver("sandbox")!;
|
||||
const destroyProvider = vi.fn(async (input: Parameters<NonNullable<typeof base.destroyRunLease>>[0]) => environmentsSvc.releaseLease(input.lease.id, "released", {
|
||||
cleanupStatus: "success", remoteExecutionTermination: remoteTerminationReceipt(input.lease, {
|
||||
state: receiptState === "stopped" ? "stopped" : "destroyed",
|
||||
providerLeaseId: receiptState === "invalid" ? "wrong-allocation" : input.lease.providerLeaseId,
|
||||
}),
|
||||
}));
|
||||
const runtime = environmentRuntimeService(db, { drivers: [{ ...base, destroyRunLease: destroyProvider }] });
|
||||
const inspectLocal = vi.fn(async () => { throw new Error("Controller Git inspection must not run"); });
|
||||
const service = executionWorkspaceService(db, { destroyRemoteWorkspaceLease: input => runtime.destroyRemoteWorkspaceLease(input),
|
||||
inspectGitCloseReadiness: inspectLocal, workspaceReaperCooldownDays: 0, gitCacheExpirationEnabled: true,
|
||||
gitRestorePoints: undefined });
|
||||
return { company, issue, run, workspace, snapshotId, authority, service, destroyProvider, inspectLocal, cwd, environmentsSvc, leaseId };
|
||||
}
|
||||
const current = async (id: string) => (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!;
|
||||
|
||||
it.each([true, false])("rejects automatic Core services before controller access (skip provision: %s)", async skipHostProvision => {
|
||||
const f = await fixture();
|
||||
const inspect = vi.spyOn(fs, "lstat");
|
||||
const log = vi.fn();
|
||||
await expect(ensureRuntimeServicesForRun({ db, runId: f.run.id,
|
||||
agent: { id: f.run.agentId, companyId: f.company.id, name: "Native worker" }, issue: null,
|
||||
workspace: { cwd: f.cwd, baseCwd: f.cwd, source: "task_session", projectId: null, workspaceId: null,
|
||||
repoUrl: null, repoRef: null, strategy: "git_worktree", branchName: null, worktreePath: f.cwd,
|
||||
warnings: [], created: false, branchCreatedByRuntime: false }, executionWorkspaceId: f.workspace.id,
|
||||
config: { workspaceRuntime: { services: [{ name: "web", command: "/__paperclip_missing_remote_service__", cwd: "." }] } },
|
||||
adapterEnv: {}, skipHostProvision, onLog: log,
|
||||
})).rejects.toMatchObject({ code: "workspace_validation_failed", resultJson: { reason: "remote_workspace_runtime_services_unsupported" } });
|
||||
expect(inspect).not.toHaveBeenCalled();
|
||||
expect(log).not.toHaveBeenCalled();
|
||||
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
|
||||
});
|
||||
|
||||
it.each(["remote", "import"])("does not inspect or recover nested controller clones under a %s storage fence", async mode => {
|
||||
const f = await fixture();
|
||||
const [project] = await db.insert(projects).values({ companyId: f.company.id, name: "Remote project" }).returning();
|
||||
await db.update(executionWorkspaces).set({ projectId: project.id,
|
||||
...(mode === "import" ? { providerType: "local_fs" } : {}),
|
||||
metadata: { ...(mode === "remote" ? { remoteWorkspace: f.authority } : {
|
||||
remoteWorkspaceMigration: { schema: "paperclip.remote-workspace-migration.v1", state: "running", operationId: randomUUID() },
|
||||
}),
|
||||
nestedCloneMigrationPending: { "repo-123456789abc": { phase: "cleanup" } } } }).where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
const workspace = await current(f.workspace.id);
|
||||
const inspect = vi.spyOn(fs, "lstat");
|
||||
expect(isNestedCloneMigrationCandidate(workspace)).toBe(false);
|
||||
expect(hasPendingNestedCloneMigration(workspace)).toBe(false);
|
||||
expect(pendingNestedCloneMigrationNames(workspace)).toEqual([]);
|
||||
expect(await listNestedCloneMigrationNames(db, workspace)).toEqual([]);
|
||||
expect(await migrateNestedRepositoryClone(db, workspace, "repo-123456789abc")).toMatchObject({ outcome: "skipped", reason: "not_candidate" });
|
||||
expect(inspect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("closes remote compute under the lifecycle fence and keeps saved files and host paths intact", async () => {
|
||||
const f = await fixture();
|
||||
await f.service.update(f.workspace.id, { metadata: { userNote: "Keep", remoteWorkspace: { ...f.authority, dirty: false, location: null } } });
|
||||
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)?.location).toEqual(f.authority.location);
|
||||
await expect(f.service.update(f.workspace.id, { providerType: "local_fs" })).rejects.toThrow("sandbox storage");
|
||||
const readiness = await f.service.getCloseReadiness(f.workspace.id);
|
||||
expect(readiness).toMatchObject({ state: "ready", git: null, isDestructiveCloseAllowed: true });
|
||||
expect(readiness!.plannedActions.map(action => action.kind)).toEqual(["archive_record", "remove_provider_workspace"]);
|
||||
const archived = await f.service.archiveWorkspaceUnderLifecycleLock({ id: f.workspace.id, patch: {}, closedAt: new Date() });
|
||||
if (archived?.outcome !== "archived") throw new Error("Expected a close receipt");
|
||||
await f.service.fenceClosedWorkspaceDestruction({ workspaceId: f.workspace.id, capturedGeneration: archived.capturedGeneration,
|
||||
destroy: () => f.service.cleanupRemoteStorage(archived.workspace) });
|
||||
expect(f.destroyProvider).toHaveBeenCalledTimes(1);
|
||||
expect(f.destroyProvider.mock.calls[0]![0].lease).toMatchObject({ status: "pending_cleanup", leasePolicy: "ephemeral" });
|
||||
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)).toMatchObject({ location: null, snapshotId: f.snapshotId });
|
||||
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.snapshotId)))[0]!.status).toBe("committed");
|
||||
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
|
||||
expect(f.inspectLocal).not.toHaveBeenCalled();
|
||||
await expect(cleanupExecutionWorkspaceArtifacts({ workspace: f.workspace })).rejects.toThrow("sandbox lifecycle");
|
||||
});
|
||||
|
||||
it.each(["stopped", "invalid"] as const)("retains location until destruction is proved (%s)", async state => {
|
||||
const f = await fixture(state);
|
||||
await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date() }).where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
await expect(f.service.cleanupRemoteStorage(toExecutionWorkspace({ ...f.workspace, status: "archived" }))).rejects.toThrow("remote_workspace_cleanup_unconfirmed");
|
||||
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)?.location).toEqual(f.authority.location);
|
||||
});
|
||||
|
||||
it("blocks close while a save is pending and leaves the allocation available for recovery", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, dirty: true, writerRunId: f.run.id } } })
|
||||
.where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
expect(await f.service.getCloseReadiness(f.workspace.id)).toMatchObject({ state: "blocked", isDestructiveCloseAllowed: false });
|
||||
expect(f.destroyProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves the automatic delivery gate and retries an explicitly closed remote allocation", async () => {
|
||||
const f = await fixture();
|
||||
const active = await f.service.sweepTerminalWorkspaces();
|
||||
expect(active).toMatchObject({ archived: 0, skippedUndelivered: 1 });
|
||||
expect(f.destroyProvider).not.toHaveBeenCalled();
|
||||
await db.update(executionWorkspaces).set({ status: "cleanup_failed", closedAt: new Date() }).where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
const closed = await f.service.sweepTerminalWorkspaces();
|
||||
expect(closed).toMatchObject({ archived: 1, cleanupFailed: 0 });
|
||||
expect(f.destroyProvider).toHaveBeenCalledTimes(1);
|
||||
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)?.location).toBeNull();
|
||||
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
|
||||
expect(f.inspectLocal).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reopens saved remote work by changing its lifecycle without rebuilding a controller folder", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date(),
|
||||
metadata: { remoteWorkspace: { ...f.authority, location: null } } }).where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
const reopened = await f.service.reopenClosedIsolatedExecutionWorkspaceForIssue({ workspaceId: f.workspace.id,
|
||||
issue: { id: f.issue.id, companyId: f.company.id, projectId: null }, actor: { actorType: "user", agentId: null },
|
||||
authorizationActor: { type: "board", source: "local_implicit", isInstanceAdmin: true } });
|
||||
expect(reopened).toMatchObject({ ok: true, reopened: true, workspace: { status: "active", closedAt: null } });
|
||||
expect(readRemoteWorkspaceAuthority((await current(f.workspace.id)).metadata)).toMatchObject({ snapshotId: f.snapshotId, location: null });
|
||||
expect(f.destroyProvider).not.toHaveBeenCalled();
|
||||
expect(f.inspectLocal).not.toHaveBeenCalled();
|
||||
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ caller: "direct", state: "remote" }, { caller: "startup", state: "remote" },
|
||||
{ caller: "direct", state: "migrating" }, { caller: "startup", state: "migrating" },
|
||||
] as const)("rechecks $caller control authority after waiting for $state migration publication", async ({ caller, state }) => {
|
||||
const f = await fixture();
|
||||
const escaped = join(f.cwd, "unexpected-start.txt");
|
||||
const command = `node -e 'require("fs").writeFileSync(${JSON.stringify(escaped)},"unexpected")'`;
|
||||
const config = { desiredState: "running", runtimeProvisionCommand: command,
|
||||
workspaceRuntime: { services: [{ name: "web", command, cwd: "." }] } };
|
||||
await db.update(executionWorkspaces).set({ providerType: "local_fs", strategyType: "project_primary", metadata: { config } })
|
||||
.where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
let release!: () => void;
|
||||
const released = new Promise<void>(resolve => { release = resolve; });
|
||||
let ready!: (pid: number) => void;
|
||||
const locked = new Promise<number>(resolve => { ready = resolve; });
|
||||
const migration = withExecutionWorkspaceLifecycleLock(db, f.workspace.id, async tx => {
|
||||
const [row] = await tx.execute(sql`select pg_backend_pid() as pid`);
|
||||
ready(Number(row.pid));
|
||||
await released;
|
||||
await db.update(executionWorkspaces).set({ providerType: state === "remote" ? "cloud_sandbox" : "local_fs",
|
||||
metadata: state === "remote" ? { config, remoteWorkspace: f.authority } : { config,
|
||||
remoteWorkspaceMigration: { schema: "paperclip.remote-workspace-migration.v1", state: "running" } } })
|
||||
.where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
});
|
||||
const blocker = await locked;
|
||||
const starting = caller === "startup" ? restartDesiredRuntimeServicesOnStartup(db) : startRuntimeServicesForWorkspaceControl({
|
||||
db, actor: { id: null, companyId: f.company.id, name: "Board" }, issue: null, executionWorkspaceId: f.workspace.id,
|
||||
workspace: { cwd: f.cwd, baseCwd: f.cwd, source: "task_session", projectId: null, workspaceId: null,
|
||||
repoUrl: null, repoRef: null, strategy: "project_primary", branchName: null, worktreePath: null,
|
||||
warnings: [], created: false, branchCreatedByRuntime: false }, config, adapterEnv: {},
|
||||
});
|
||||
const settled = starting.then(value => ({ value }), error => ({ error }));
|
||||
try {
|
||||
// Prove the stale request is waiting on the actual PostgreSQL lifecycle
|
||||
// fence, then switch authority before admitting it. No timing sleep.
|
||||
await vi.waitFor(async () => {
|
||||
const [row] = await db.execute(sql`select count(*)::int as waiting from pg_stat_activity
|
||||
where ${blocker} = any(pg_blocking_pids(pid))`);
|
||||
expect(Number(row.waiting)).toBeGreaterThan(0);
|
||||
});
|
||||
} finally { release(); }
|
||||
await migration;
|
||||
const result = await settled;
|
||||
if (caller === "startup") expect(result).toMatchObject({ value: { restarted: 0, failed: 1 } });
|
||||
else expect(result).toMatchObject({ error: { status: state === "remote" ? 422 : 409,
|
||||
details: { code: state === "remote" ? "remote_workspace_runtime_commands_unsupported" : "workspace_runtime_migration_in_progress" } } });
|
||||
await expect(fs.access(escaped)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await db.select().from(workspaceRuntimeServices)).toEqual([]);
|
||||
expect(await readFile(join(f.cwd, "host-sentinel.txt"), "utf8")).toContain("never be cleaned");
|
||||
});
|
||||
|
||||
it("publishes runtime admission before releasing the migration fence and keeps migration operations blocking", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(executionWorkspaces).set({ providerType: "local_fs", strategyType: "project_primary", metadata: {} })
|
||||
.where(eq(executionWorkspaces.id, f.workspace.id));
|
||||
const operations = workspaceOperationService(db);
|
||||
let finish!: () => void;
|
||||
const finished = new Promise<void>(resolve => { finish = resolve; });
|
||||
let admitted!: () => void;
|
||||
const admission = new Promise<void>(resolve => { admitted = resolve; });
|
||||
const running = operations.createRecorder({ companyId: f.company.id, executionWorkspaceId: f.workspace.id }).recordOperation({
|
||||
phase: "workspace_provision", metadata: { action: "run" },
|
||||
admit: publish => withExecutionWorkspaceLifecycleLock(db, f.workspace.id, async () => { await publish(); }),
|
||||
run: async () => { admitted(); await finished; return {}; },
|
||||
});
|
||||
try {
|
||||
await admission;
|
||||
const inventory = await withExecutionWorkspaceLifecycleLock(db, f.workspace.id,
|
||||
() => inventoryRemoteWorkspaceMigration(db, { companyId: f.company.id, workspaceId: f.workspace.id }));
|
||||
expect(inventory.reasons).toContain("workspace_operation_unresolved");
|
||||
} finally { finish(); }
|
||||
await running;
|
||||
const [migration] = await db.insert(workspaceOperations).values({ companyId: f.company.id, executionWorkspaceId: f.workspace.id,
|
||||
phase: "remote_workspace_migration", status: "running", metadata: {}, startedAt: new Date(0), updatedAt: new Date(0) }).returning();
|
||||
await expect(operations.assertRuntimeControlAvailable({ executionWorkspaceId: f.workspace.id, action: "start" }))
|
||||
.rejects.toMatchObject({ status: 409, details: { activeOperationId: migration.id } });
|
||||
expect((await operations.getById(migration.id))?.status).toBe("running");
|
||||
});
|
||||
|
||||
it("does not resolve a controller folder when a remote control has no selected services", async () => {
|
||||
const f = await fixture();
|
||||
await expect(startRuntimeServicesForWorkspaceControl({ db,
|
||||
actor: { id: null, companyId: f.company.id, name: "Board" }, issue: null, executionWorkspaceId: f.workspace.id,
|
||||
workspace: { get cwd(): string { throw new Error("Empty control read a controller folder"); },
|
||||
baseCwd: f.cwd, source: "task_session", projectId: null, workspaceId: null, repoUrl: null, repoRef: null,
|
||||
strategy: "git_worktree", branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false },
|
||||
config: { desiredState: "stopped", workspaceRuntime: { services: [{ name: "web", command: "node web.js" }] } },
|
||||
adapterEnv: {}, respectDesiredStates: true,
|
||||
})).resolves.toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { executionWorkspaces, executionWorkspaceSnapshots, type Db } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, type EnvironmentLease } from "@paperclipai/shared";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
|
||||
type Workspace = Pick<typeof executionWorkspaces.$inferSelect, "id" | "companyId" | "metadata">;
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
export type DestroyRemoteWorkspaceLease = (input: {
|
||||
companyId: string; workspaceId: string; leaseId: string; environmentId: string; providerLeaseId: string;
|
||||
}) => Promise<EnvironmentLease | null>;
|
||||
|
||||
export async function inspectRemoteWorkspacePersistence(db: Db | Transaction, workspace: Workspace) {
|
||||
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!authority || authority.dirty || authority.writerRunId || !authority.snapshotId) {
|
||||
return { ready: false, reason: "The workspace still has unsaved work or an unfinished writer.", authority };
|
||||
}
|
||||
const [snapshot] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
|
||||
eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.status, "committed"),
|
||||
));
|
||||
return { ready: Boolean(snapshot), reason: snapshot ? null : "The saved workspace is unavailable. Its remote files were retained.", authority };
|
||||
}
|
||||
|
||||
/** Called within the workspace lifecycle fence; the saved workspace remains durable. */
|
||||
export async function cleanupRemoteWorkspaceAllocation(input: {
|
||||
db: Db; workspace: Workspace; destroyLease: DestroyRemoteWorkspaceLease;
|
||||
}) {
|
||||
const [workspace] = await input.db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.workspace.companyId), eq(executionWorkspaces.id, input.workspace.id),
|
||||
));
|
||||
if (!workspace || !["archived", "cleanup_failed"].includes(workspace.status)) throw new Error("remote_workspace_cleanup_binding_changed");
|
||||
const persistence = await inspectRemoteWorkspacePersistence(input.db, workspace);
|
||||
if (!persistence.ready || !persistence.authority) throw new Error(persistence.reason ?? "remote_workspace_saved_state_required");
|
||||
const location = persistence.authority.location;
|
||||
if (location) {
|
||||
const released = await input.destroyLease({ companyId: workspace.companyId, workspaceId: workspace.id,
|
||||
leaseId: location.leaseId, environmentId: location.environmentId, providerLeaseId: location.providerLeaseId });
|
||||
if (!released || !hasRemoteTerminationReceipt(released)
|
||||
|| (released.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state !== "destroyed") {
|
||||
throw new Error("remote_workspace_cleanup_unconfirmed");
|
||||
}
|
||||
}
|
||||
return { cleaned: true, warnings: [] as string[] };
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { activityLog, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
|
||||
executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import { createStorageService } from "../storage/service.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js";
|
||||
import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js";
|
||||
import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
|
||||
import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C", cwd, ...args], { env: { ...process.env,
|
||||
GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_AUTHOR_NAME: "Migration", GIT_COMMITTER_NAME: "Migration",
|
||||
GIT_AUTHOR_EMAIL: "migration@example.test", GIT_COMMITTER_EMAIL: "migration@example.test" } })).stdout.trim();
|
||||
|
||||
const url = process.env.PAPERCLIP_TEST_DATABASE_URL;
|
||||
if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required");
|
||||
const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("remote workspace migration", () => {
|
||||
let db: ReturnType<typeof createDb>, root: string, storage: StorageService;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
beforeAll(async () => {
|
||||
if (url) { await applyPendingMigrations(url); db = createDb(url); }
|
||||
else { temporary = await startEmbeddedPostgresTestDatabase("workspace-migration-"); db = createDb(temporary.connectionString); }
|
||||
}, 90_000);
|
||||
afterAll(async () => { if (url) await closeRegisteredClients(url, { scope: "database" }); else await temporary?.cleanup(); });
|
||||
beforeEach(async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32));
|
||||
root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-migration-test-")); storage = createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))); });
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs(); await db.delete(activityLog); await db.delete(workspaceOperations); await db.delete(executionWorkspaceSnapshots);
|
||||
await db.delete(workspaceRuntimeServices); await db.delete(executionWorkspaceRuntimeLeases);
|
||||
await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects);
|
||||
await db.delete(companies); await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
async function seed() {
|
||||
const companyId = randomUUID(), workspaceId = randomUUID(), cwd = path.join(root, "original"); await fs.mkdir(cwd);
|
||||
await fs.writeFile(path.join(cwd, "saved.txt"), "original saved work\n");
|
||||
await db.insert(companies).values({ id: companyId, name: "Migration", issuePrefix: `M${companyId.slice(0, 6)}` });
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Existing task" }).returning();
|
||||
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issue!.id, mode: "isolated_workspace",
|
||||
strategyType: "project_primary", providerType: "local_fs", name: "Existing workspace", cwd, providerRef: cwd, metadata: { retainedNote: "keep" } });
|
||||
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issue!.id));
|
||||
return { companyId, workspaceId, issueId: issue!.id, cwd };
|
||||
}
|
||||
async function workspace(id: string) { return (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!; }
|
||||
it("defaults to dry run and preserves both database authority and local files", async () => {
|
||||
const f = await seed(); expect(await migrateRemoteWorkspace({ db, storage, ...f })).toMatchObject({ eligible: true, applied: false, alreadyMigrated: false });
|
||||
expect((await workspace(f.workspaceId)).providerType).toBe("local_fs");
|
||||
expect(await db.select().from(workspaceOperations)).toHaveLength(0);
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
|
||||
});
|
||||
it("verifies saved files, switches authority once, and retains the original folder", async () => {
|
||||
const f = await seed();
|
||||
expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ applied: true, alreadyMigrated: true });
|
||||
const row = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(row.metadata);
|
||||
expect(authority).toMatchObject({ epoch: 1, location: null, dirty: false, writerRunId: null });
|
||||
expect(authority?.snapshotId).toMatch(/^[a-f0-9-]{36}$/);
|
||||
expect(row.metadata?.retainedNote).toBe("keep"); expect(row.metadata?.remoteWorkspaceMigration).toBeUndefined();
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
|
||||
expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ replayed: true });
|
||||
expect(await db.select().from(executionWorkspaceSnapshots)).toHaveLength(1);
|
||||
expect(await db.select().from(activityLog)).toHaveLength(1);
|
||||
});
|
||||
it.each([false, true])("removes plaintext verification files with read-only folders after import (source changed: %s)", async sourceChanged => {
|
||||
const f = await seed(), protectedDir = path.join(f.cwd, "protected"), scratchRoot = path.join(root, "scratch");
|
||||
await fs.mkdir(protectedDir); await fs.mkdir(scratchRoot);
|
||||
await fs.writeFile(path.join(protectedDir, "saved.txt"), "read-only saved work\n", { mode: 0o444 });
|
||||
await fs.chmod(protectedDir, 0o555);
|
||||
vi.stubEnv("TMPDIR", scratchRoot);
|
||||
const get = storage.getObject.bind(storage); let reads = 0;
|
||||
if (sourceChanged) storage.getObject = async (...args) => {
|
||||
if (++reads === 2) await fs.writeFile(path.join(f.cwd, "saved.txt"), "new local work\n");
|
||||
return get(...args);
|
||||
};
|
||||
try {
|
||||
const migration = migrateRemoteWorkspace({ db, storage, ...f, apply: true });
|
||||
if (sourceChanged) {
|
||||
await expect(migration).rejects.toThrow("remote_workspace_migration_source_changed");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
} else {
|
||||
await expect(migration).resolves.toMatchObject({ applied: true, alreadyMigrated: true });
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)?.snapshotId).toBeTruthy();
|
||||
}
|
||||
expect(await fs.readdir(scratchRoot)).toEqual([]);
|
||||
expect((await fs.stat(protectedDir)).mode & 0o777).toBe(0o555);
|
||||
expect((await fs.stat(path.join(protectedDir, "saved.txt"))).mode & 0o777).toBe(0o444);
|
||||
expect(await fs.readFile(path.join(protectedDir, "saved.txt"), "utf8")).toBe("read-only saved work\n");
|
||||
} finally {
|
||||
await fs.chmod(protectedDir, 0o755);
|
||||
await removeTaskDirectoryTree(scratchRoot);
|
||||
}
|
||||
});
|
||||
it("resumes an interrupted upload with the original operation identity", async () => {
|
||||
const f = await seed(), put = storage.putFile.bind(storage); let fail = true;
|
||||
storage.putFile = async input => { const saved = await put(input); if (fail) { fail = false; throw new Error("upload reply lost"); } return saved; };
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("upload reply lost");
|
||||
const before = await inventoryRemoteWorkspaceMigration(db, f); expect(before.operationId).not.toBeNull();
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
expect(await migrateRemoteWorkspace({ db, storage, ...f, apply: true })).toMatchObject({ applied: true });
|
||||
const operations = await db.select().from(workspaceOperations); expect(operations).toHaveLength(1); expect(operations[0]?.id).toBe(before.operationId);
|
||||
});
|
||||
it("imports dirty unpushed Git work and restores it after the original folder disappears", async () => {
|
||||
const f = await seed(); await git(f.cwd, "init", "-q", "-b", "feature");
|
||||
await git(f.cwd, "add", "."); await git(f.cwd, "commit", "-qm", "unpushed work");
|
||||
await git(f.cwd, "remote", "add", "origin", "https://example.invalid/repository.git");
|
||||
const head = await git(f.cwd, "rev-parse", "HEAD");
|
||||
await fs.writeFile(path.join(f.cwd, "saved.txt"), "staged edit\n"); await git(f.cwd, "add", "saved.txt");
|
||||
await fs.writeFile(path.join(f.cwd, "saved.txt"), "unstaged edit\n");
|
||||
await db.update(executionWorkspaces).set({ repoUrl: "https://example.invalid/repository.git", branchName: "feature" }).where(eq(executionWorkspaces.id, f.workspaceId));
|
||||
await migrateRemoteWorkspace({ db, storage, ...f, apply: true });
|
||||
const authority = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!;
|
||||
const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! });
|
||||
await fs.rm(f.cwd, { recursive: true });
|
||||
const archivePath = path.join(root, "cold.pcws"), targetDir = path.join(root, "cold");
|
||||
await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath });
|
||||
await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest });
|
||||
expect(await git(targetDir, "rev-parse", "HEAD")).toBe(head);
|
||||
expect(await git(targetDir, "show", ":saved.txt")).toBe("staged edit");
|
||||
expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("unstaged edit\n");
|
||||
});
|
||||
it("blocks unresolved writers and explicitly parallel shared workspaces", async () => {
|
||||
const f = await seed();
|
||||
await db.update(executionWorkspaces).set({ mode: "shared_workspace" }).where(eq(executionWorkspaces.id, f.workspaceId));
|
||||
await db.update(issues).set({ executionWorkspaceSettings: { sharedWorkspaceConcurrency: "allow" } }).where(eq(issues.id, f.issueId));
|
||||
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["parallel_shared_writers_unsupported"] });
|
||||
await db.update(issues).set({ executionWorkspaceSettings: { sharedWorkspaceConcurrency: "serialize" } }).where(eq(issues.id, f.issueId));
|
||||
await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId, phase: "workspace_command", status: "running" });
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_operation_unresolved");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
});
|
||||
it.each(["running", "starting", "unknown", "failed"])("blocks a %s local service after its control lease expires", async status => {
|
||||
const f = await seed(), serviceId = randomUUID();
|
||||
await db.insert(executionWorkspaceRuntimeLeases).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId,
|
||||
ownerKey: `issue:${f.issueId}`, ownerIssueId: f.issueId, lastAction: "start", expiresAt: new Date(Date.now() - 60_000) });
|
||||
await db.insert(workspaceRuntimeServices).values({ id: serviceId, companyId: f.companyId, executionWorkspaceId: f.workspaceId,
|
||||
scopeType: "execution_workspace", scopeId: f.workspaceId, serviceName: "Synthetic preview", status,
|
||||
lifecycle: "shared", provider: "local_process", command: "node synthetic-preview.js", cwd: f.cwd });
|
||||
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["workspace_runtime_service_unsettled"] });
|
||||
expect(await migrateRemoteWorkspace({ db, storage, ...f })).toMatchObject({ applied: false, eligible: false });
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled");
|
||||
expect((await workspace(f.workspaceId)).providerType).toBe("local_fs");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
expect(await db.select().from(workspaceOperations)).toHaveLength(0);
|
||||
expect(await db.select().from(executionWorkspaceSnapshots)).toHaveLength(0);
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
|
||||
await db.$client`update workspace_runtime_services set status = 'stopped' where id = ${serviceId}::uuid`;
|
||||
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true, reasons: [] });
|
||||
});
|
||||
it.each([{ repository: { phase: "cleanup" } }, "unrecognized-state"])("blocks an unresolved or malformed nested clone migration marker: %j", async marker => {
|
||||
const f = await seed();
|
||||
await db.update(executionWorkspaces).set({ metadata: { retainedNote: "keep", nestedCloneMigrationPending: marker } }).where(eq(executionWorkspaces.id, f.workspaceId));
|
||||
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["nested_clone_migration_unresolved"] });
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("nested_clone_migration_unresolved");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
|
||||
});
|
||||
it("blocks a running project service inherited by a shared workspace", async () => {
|
||||
const f = await seed(), projectId = randomUUID(), projectWorkspaceId = randomUUID();
|
||||
await db.insert(projects).values({ id: projectId, companyId: f.companyId, name: "Shared synthetic project" });
|
||||
await db.insert(projectWorkspaces).values({ id: projectWorkspaceId, companyId: f.companyId, projectId,
|
||||
name: "Shared synthetic source", sourceType: "local", cwd: f.cwd });
|
||||
await db.update(executionWorkspaces).set({ mode: "shared_workspace", projectId, projectWorkspaceId }).where(eq(executionWorkspaces.id, f.workspaceId));
|
||||
await db.insert(workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId, projectWorkspaceId,
|
||||
scopeType: "project_workspace", scopeId: projectWorkspaceId, serviceName: "Inherited preview", status: "running",
|
||||
lifecycle: "shared", provider: "local_process", cwd: f.cwd });
|
||||
expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: false, reasons: ["workspace_runtime_service_unsettled"] });
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
|
||||
});
|
||||
it("rechecks a service that becomes unresolved during snapshot verification", async () => {
|
||||
const f = await seed(), get = storage.getObject.bind(storage); let reads = 0;
|
||||
storage.getObject = async (...args) => {
|
||||
if (++reads === 2) await db.insert(workspaceRuntimeServices).values({ id: randomUUID(), companyId: f.companyId,
|
||||
executionWorkspaceId: f.workspaceId, scopeType: "execution_workspace", scopeId: f.workspaceId,
|
||||
serviceName: "Late synthetic preview", status: "running", lifecycle: "shared", provider: "local_process", cwd: f.cwd });
|
||||
return get(...args);
|
||||
};
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_runtime_service_unsettled");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n");
|
||||
});
|
||||
it("rechecks a writer that appears while the restored copy is being verified", async () => {
|
||||
const f = await seed(), get = storage.getObject.bind(storage); let reads = 0;
|
||||
storage.getObject = async (...args) => {
|
||||
if (++reads === 2) await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId,
|
||||
phase: "workspace_command", status: "running" });
|
||||
return get(...args);
|
||||
};
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("workspace_operation_unresolved");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
});
|
||||
it("preserves a local edit made during verification and refuses activation", async () => {
|
||||
const f = await seed(), get = storage.getObject.bind(storage); let reads = 0;
|
||||
storage.getObject = async (...args) => { if (++reads === 2) await fs.writeFile(path.join(f.cwd, "saved.txt"), "new local work\n"); return get(...args); };
|
||||
await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("source_changed");
|
||||
expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull();
|
||||
expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("new local work\n");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,205 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
import {
|
||||
executionWorkspaceRepositories, executionWorkspaceRuntimeLeases, executionWorkspaceSnapshots, executionWorkspaces,
|
||||
heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, projects, workspaceOperations, workspaceRuntimeServices, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { persistActivity, publishActivity } from "./activity-log.js";
|
||||
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
|
||||
import { beginWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot } from "./remote-workspace-snapshot-storage.js";
|
||||
import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js";
|
||||
import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js";
|
||||
|
||||
const SCHEMA = "paperclip.remote-workspace-migration.v1";
|
||||
type Workspace = typeof executionWorkspaces.$inferSelect;
|
||||
type ReadDb = Pick<Db, "select">;
|
||||
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
function canonical(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
|
||||
if (value && typeof value === "object") return `{${Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`).join(",")}}`;
|
||||
return JSON.stringify(value) ?? "null";
|
||||
}
|
||||
function fingerprint(workspace: Workspace) {
|
||||
const metadata = { ...workspace.metadata }; delete metadata.remoteWorkspaceMigration;
|
||||
return createHash("sha256").update(canonical({ companyId: workspace.companyId, id: workspace.id, projectId: workspace.projectId,
|
||||
projectWorkspaceId: workspace.projectWorkspaceId, sourceIssueId: workspace.sourceIssueId, mode: workspace.mode, strategyType: workspace.strategyType,
|
||||
providerType: workspace.providerType, cwd: workspace.cwd, providerRef: workspace.providerRef, repoUrl: workspace.repoUrl,
|
||||
baseRef: workspace.baseRef, branchName: workspace.branchName, status: workspace.status, closedAt: workspace.closedAt?.toISOString(), metadata })).digest("hex");
|
||||
}
|
||||
|
||||
async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string }) {
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId)));
|
||||
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
|
||||
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
const marker = object(workspace.metadata?.remoteWorkspaceMigration);
|
||||
const allowedOperation = input.operationId ?? (marker.schema === SCHEMA && typeof marker.operationId === "string" ? marker.operationId : undefined);
|
||||
const reasons: string[] = [];
|
||||
if (authority) return { workspace, reasons, authority, marker, source: authority.source };
|
||||
if (!["local_fs", "git_worktree"].includes(workspace.providerType)) reasons.push("workspace_provider_unsupported");
|
||||
if (!["active", "idle", "in_review"].includes(workspace.status)) reasons.push("workspace_not_active");
|
||||
if (!workspace.sourceIssueId) reasons.push("task_binding_required");
|
||||
const nestedMigration = workspace.metadata?.nestedCloneMigrationPending;
|
||||
if (nestedMigration != null && (typeof nestedMigration !== "object" || Array.isArray(nestedMigration)
|
||||
|| Object.keys(nestedMigration).length > 0)) reasons.push("nested_clone_migration_unresolved");
|
||||
const cwd = workspace.cwd ?? workspace.providerRef;
|
||||
if (!cwd || !path.isAbsolute(cwd)) reasons.push("local_source_missing");
|
||||
else {
|
||||
const entry = await fs.lstat(cwd).catch(() => null);
|
||||
if (!entry?.isDirectory() || entry.isSymbolicLink()) reasons.push("local_source_unsafe_or_missing");
|
||||
}
|
||||
const [issue] = workspace.sourceIssueId ? await db.select().from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))) : [];
|
||||
if (workspace.sourceIssueId && (!issue || issue.executionWorkspaceId !== workspace.id)) reasons.push("task_binding_changed");
|
||||
const [project] = workspace.projectId ? await db.select({ policy: projects.executionWorkspacePolicy }).from(projects)
|
||||
.where(and(eq(projects.companyId, workspace.companyId), eq(projects.id, workspace.projectId))) : [];
|
||||
const issuePolicy = object(issue?.executionWorkspaceSettings), projectPolicy = object(project?.policy);
|
||||
if (workspace.mode === "shared_workspace" && (issuePolicy.sharedWorkspaceConcurrency ?? (projectPolicy.enabled === true ? projectPolicy.sharedWorkspaceConcurrency : undefined)) === "allow") reasons.push("parallel_shared_writers_unsupported");
|
||||
if (issue?.executionRunId || issue?.checkoutRunId) reasons.push("task_writer_bound");
|
||||
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, workspace.companyId),
|
||||
inArray(heartbeatRuns.status, ["queued", "running"]), sql`(${heartbeatRuns.nativeIssueId} = ${workspace.sourceIssueId}::uuid
|
||||
or ${heartbeatRuns.issueId} = ${workspace.sourceIssueId}::uuid
|
||||
or ${heartbeatRuns.contextSnapshot}->>'executionWorkspaceId' = ${workspace.id}
|
||||
or ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'workspaceId' = ${workspace.id})`,
|
||||
)).limit(1);
|
||||
if (run) reasons.push("active_execution");
|
||||
const [unexported] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, workspace.companyId),
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'workspaceId' = ${workspace.id}`,
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared'`,
|
||||
)).limit(1);
|
||||
if (unexported) reasons.push("remote_copyback_unresolved");
|
||||
const [finalization] = workspace.sourceIssueId ? await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, workspace.companyId), eq(nativeRunFinalizations.issueId, workspace.sourceIssueId),
|
||||
sql`${nativeRunFinalizations.phase} not in ('committed', 'terminal_failure')`,
|
||||
)).limit(1) : [];
|
||||
if (finalization) reasons.push("execution_finalization_unresolved");
|
||||
const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
|
||||
allowedOperation ? ne(workspaceOperations.id, allowedOperation) : undefined,
|
||||
sql`(${workspaceOperations.status} in ('running', 'pending') or ${workspaceOperations.metadata}->>'uncertain' = 'true')`,
|
||||
)).limit(1);
|
||||
if (operation) reasons.push("workspace_operation_unresolved");
|
||||
const [lease] = await db.select({ id: executionWorkspaceRuntimeLeases.id }).from(executionWorkspaceRuntimeLeases).where(and(
|
||||
eq(executionWorkspaceRuntimeLeases.companyId, workspace.companyId), eq(executionWorkspaceRuntimeLeases.executionWorkspaceId, workspace.id),
|
||||
sql`${executionWorkspaceRuntimeLeases.expiresAt} > clock_timestamp()`,
|
||||
)).limit(1);
|
||||
if (lease) reasons.push("workspace_runtime_owned");
|
||||
// A control lease can expire while the service process remains alive. Require
|
||||
// settled service state independently, including inherited shared services.
|
||||
const runtime = object(workspace.metadata?.config).workspaceRuntime;
|
||||
const inheritedSourceId = workspace.mode === "shared_workspace" && workspace.projectWorkspaceId
|
||||
&& (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) ? workspace.projectWorkspaceId : null;
|
||||
const [service] = await db.select({ id: workspaceRuntimeServices.id }).from(workspaceRuntimeServices).where(and(
|
||||
eq(workspaceRuntimeServices.companyId, workspace.companyId), ne(workspaceRuntimeServices.status, "stopped"),
|
||||
or(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id), inheritedSourceId ? and(
|
||||
eq(workspaceRuntimeServices.projectWorkspaceId, inheritedSourceId), eq(workspaceRuntimeServices.scopeType, "project_workspace"),
|
||||
) : undefined),
|
||||
)).limit(1);
|
||||
if (service) reasons.push("workspace_runtime_service_unsettled");
|
||||
const [recovery] = workspace.sourceIssueId ? await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, workspace.companyId), eq(issueRecoveryActions.sourceIssueId, workspace.sourceIssueId),
|
||||
inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
)).limit(1) : [];
|
||||
if (recovery) reasons.push("workspace_recovery_unresolved");
|
||||
const repositories = await db.select().from(executionWorkspaceRepositories).where(and(eq(executionWorkspaceRepositories.companyId, workspace.companyId),
|
||||
eq(executionWorkspaceRepositories.executionWorkspaceId, workspace.id)));
|
||||
if (repositories.some(repository => repository.state !== "ready")) reasons.push("repository_preparation_unresolved");
|
||||
const source: RemoteWorkspaceSource = workspace.repoUrl ? { kind: "git", repoUrl: workspace.repoUrl,
|
||||
ref: workspace.baseRef, branchName: workspace.branchName } : { kind: "empty" };
|
||||
if (repositories.length) source.repositories = repositories.map(repository => ({ id: repository.id, relativePath: repository.relativePath,
|
||||
repoUrl: repository.repoUrl, ref: repository.requestedRef, ...(repository.pinnedCommit ? { pinnedCommit: repository.pinnedCommit } : {}) }));
|
||||
return { workspace, reasons: [...new Set(reasons)], authority, marker, source };
|
||||
}
|
||||
|
||||
export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string }) {
|
||||
const result = await inspect(db, input);
|
||||
return { companyId: input.companyId, workspaceId: input.workspaceId, eligible: result.reasons.length === 0,
|
||||
alreadyMigrated: Boolean(result.authority), reasons: result.reasons, localPath: result.workspace.cwd ?? result.workspace.providerRef,
|
||||
source: result.source, operationId: typeof result.marker.operationId === "string" ? result.marker.operationId : null,
|
||||
snapshotId: result.authority?.snapshotId ?? null };
|
||||
}
|
||||
|
||||
/** Administrative import. The CLI enforces external deployment prerequisites. */
|
||||
export async function migrateRemoteWorkspace(input: {
|
||||
db: Db; storage: StorageService; companyId: string; workspaceId: string; apply?: boolean; source?: RemoteWorkspaceSource;
|
||||
}) {
|
||||
if (!input.apply) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: false };
|
||||
return withExecutionWorkspaceLifecycleLock(input.db, input.workspaceId, async lock => {
|
||||
const reserved = await input.db.transaction(async tx => {
|
||||
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
|
||||
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
|
||||
if (workspace.sourceIssueId) await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))).for("update");
|
||||
const current = await inspect(tx as unknown as Db, input);
|
||||
if (current.authority) return { workspace, authority: current.authority, operation: null, source: current.source, sourceFingerprint: "" };
|
||||
if (current.reasons.length) throw new Error(`remote_workspace_migration_blocked:${current.reasons.join(",")}`);
|
||||
const source = remoteWorkspaceSourceSchema.parse(input.source ?? current.source);
|
||||
const sourceFingerprint = fingerprint(workspace);
|
||||
const marker = current.marker;
|
||||
if (marker.schema === SCHEMA && typeof marker.operationId === "string") {
|
||||
if (marker.sourceFingerprint !== sourceFingerprint || canonical(marker.source) !== canonical(source)) throw new Error("remote_workspace_migration_source_changed");
|
||||
const [operation] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, workspace.companyId),
|
||||
eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration")));
|
||||
if (!operation) throw new Error("remote_workspace_migration_operation_missing");
|
||||
return { workspace, authority: null, operation, source, sourceFingerprint };
|
||||
}
|
||||
const [operation] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id,
|
||||
issueId: workspace.sourceIssueId, cwd: workspace.cwd ?? workspace.providerRef, phase: "remote_workspace_migration", status: "running",
|
||||
metadata: { schema: SCHEMA, sourceFingerprint, source } }).returning();
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
|
||||
await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspaceMigration: {
|
||||
schema: SCHEMA, operationId: operation!.id, state: "running", sourceFingerprint, source,
|
||||
} }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id));
|
||||
return { workspace, authority: null, operation: operation!, source, sourceFingerprint };
|
||||
});
|
||||
if (reserved.authority) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: true };
|
||||
const operation = reserved.operation!;
|
||||
const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.companyId, workspaceId: input.workspaceId,
|
||||
operationId: operation.id, generation: 1, epoch: 1 });
|
||||
const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-migration-"));
|
||||
try {
|
||||
const archivePath = path.join(scratch, "workspace.pcws");
|
||||
const captured = await captureRemoteWorkspaceDirectory({ cwd: reserved.workspace.cwd ?? reserved.workspace.providerRef!, archivePath });
|
||||
const ref = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage, snapshotId: snapshot.id,
|
||||
companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest });
|
||||
const restoredArchive = path.join(scratch, "verified.pcws"), restored = path.join(scratch, "restored");
|
||||
await loadWorkspaceSnapshotArchive({ storage: input.storage, reference: ref, targetPath: restoredArchive });
|
||||
await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest });
|
||||
const currentArchive = path.join(scratch, "current.pcws");
|
||||
const current = await captureRemoteWorkspaceDirectory({ cwd: reserved.workspace.cwd ?? reserved.workspace.providerRef!, archivePath: currentArchive });
|
||||
if (current.manifestDigest !== captured.manifestDigest) throw new Error("remote_workspace_migration_source_changed");
|
||||
await lock.execute(sql`select 1`);
|
||||
const publication = await input.db.transaction(async tx => {
|
||||
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
|
||||
if (!workspace) throw new Error("remote_workspace_migration_workspace_missing");
|
||||
const existing = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (existing?.snapshotId === ref.id) return null;
|
||||
if (existing || fingerprint(workspace) !== reserved.sourceFingerprint
|
||||
|| object(workspace.metadata?.remoteWorkspaceMigration).operationId !== operation.id) throw new Error("remote_workspace_migration_publication_changed");
|
||||
if (workspace.sourceIssueId) await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, workspace.sourceIssueId)).for("update");
|
||||
const validation = await inspect(tx as unknown as Db, { ...input, operationId: operation.id });
|
||||
if (validation.reasons.length) throw new Error(`remote_workspace_migration_blocked:${validation.reasons.join(",")}`);
|
||||
const [uploaded] = await tx.select().from(executionWorkspaceSnapshots).where(and(eq(executionWorkspaceSnapshots.companyId, workspace.companyId),
|
||||
eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), eq(executionWorkspaceSnapshots.id, ref.id))).for("update");
|
||||
if (!uploaded || uploaded.status !== "uploaded" || uploaded.sourceOperationId !== operation.id) throw new Error("remote_workspace_migration_snapshot_changed");
|
||||
const authority: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: ref.id,
|
||||
source: reserved.source, location: null, writerRunId: null, dirty: false };
|
||||
const metadata: Record<string, unknown> = { ...workspace.metadata, remoteWorkspace: authority }; delete metadata.remoteWorkspaceMigration;
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
|
||||
await tx.update(executionWorkspaceSnapshots).set({ status: "committed", deleteAfter: null, updatedAt: new Date() }).where(eq(executionWorkspaceSnapshots.id, ref.id));
|
||||
await tx.update(executionWorkspaces).set({ providerType: "cloud_sandbox", providerRef: null, cwd: `/workspaces/${workspace.id}`, metadata,
|
||||
updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id));
|
||||
await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(),
|
||||
metadata: { ...operation.metadata, snapshotId: ref.id, manifestDigest: ref.manifestDigest } }).where(eq(workspaceOperations.id, operation.id));
|
||||
return (await persistActivity(tx as unknown as Db, { companyId: workspace.companyId, actorType: "system", actorId: "workspace-migration",
|
||||
action: "workspace.remote_storage_migrated", entityType: "execution_workspace", entityId: workspace.id,
|
||||
details: { snapshotId: ref.id, operationId: operation.id, manifestDigest: ref.manifestDigest } })).publication;
|
||||
});
|
||||
if (publication) publishActivity(publication);
|
||||
return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: false };
|
||||
} finally { await removeTaskDirectoryTree(scratch); }
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { captureRemoteWorkspaceDirectory, remoteWorkspaceArchiveScript, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
|
||||
|
||||
const archiveIo = vi.hoisted(() => ({ afterChmod: undefined as ((target: string, mode: number) => void) | undefined }));
|
||||
vi.mock("node:fs/promises", async importOriginal => {
|
||||
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
||||
return { ...actual, async chmod(target: string, mode: number) {
|
||||
await actual.chmod(target, mode); archiveIo.afterChmod?.(target, mode);
|
||||
} };
|
||||
});
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const env = { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull,
|
||||
GIT_AUTHOR_NAME: "Snapshot", GIT_COMMITTER_NAME: "Snapshot", GIT_AUTHOR_EMAIL: "snapshot@example.test", GIT_COMMITTER_EMAIL: "snapshot@example.test" };
|
||||
const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C", cwd, ...args], { env })).stdout.trim();
|
||||
describe("portable remote workspace archives", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-archive-test-")); });
|
||||
afterEach(async () => { archiveIo.afterChmod = undefined; await fs.rm(root, { recursive: true, force: true }); });
|
||||
async function source() { const cwd = path.join(root, "source"); await fs.mkdir(cwd); return cwd; }
|
||||
async function capture(cwd: string) { const archivePath = path.join(root, "snapshot.pcws"); return { archivePath, ...await captureRemoteWorkspaceDirectory({ cwd, archivePath }) }; }
|
||||
async function restore(snapshot: Awaited<ReturnType<typeof capture>>) {
|
||||
const targetDir = path.join(root, "restored");
|
||||
await restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir, expectedManifestDigest: snapshot.manifestDigest });
|
||||
return targetDir;
|
||||
}
|
||||
async function sourceBytes(directory: string): Promise<string> {
|
||||
const entries: unknown[] = [];
|
||||
for (const name of (await fs.readdir(directory)).sort()) {
|
||||
const target = path.join(directory, name), stat = await fs.lstat(target);
|
||||
entries.push([name, stat.mode & 0o777, stat.isDirectory() ? await sourceBytes(target)
|
||||
: stat.isSymbolicLink() ? await fs.readlink(target) : createHash("sha256").update(await fs.readFile(target)).digest("hex")]);
|
||||
}
|
||||
return JSON.stringify(entries);
|
||||
}
|
||||
it("restores binary files, empty entries, executable modes and confined symlinks without its source", async () => {
|
||||
const cwd = await source();
|
||||
await fs.mkdir(path.join(cwd, "empty")); await fs.writeFile(path.join(cwd, "empty-file"), "");
|
||||
const bytes = Buffer.from([0, 255, 1, 128]); await fs.writeFile(path.join(cwd, "data.bin"), bytes);
|
||||
await fs.writeFile(path.join(cwd, "run"), "#!/bin/sh\n", { mode: 0o755 });
|
||||
await fs.symlink("data.bin", path.join(cwd, "link"));
|
||||
const snapshot = await capture(cwd); await fs.rm(cwd, { recursive: true });
|
||||
const restored = await restore(snapshot);
|
||||
expect(await fs.readFile(path.join(restored, "data.bin"))).toEqual(bytes);
|
||||
expect(await fs.readFile(path.join(restored, "empty-file"))).toHaveLength(0);
|
||||
expect(await fs.readdir(path.join(restored, "empty"))).toEqual([]);
|
||||
expect((await fs.stat(path.join(restored, "run"))).mode & 0o777).toBe(0o755);
|
||||
expect(await fs.readlink(path.join(restored, "link"))).toBe("data.bin");
|
||||
});
|
||||
it("round trips an empty workspace", async () => {
|
||||
const snapshot = await capture(await source()); const restored = await restore(snapshot);
|
||||
expect(await fs.readdir(restored)).toEqual([]);
|
||||
});
|
||||
it("rejects a Files entry limit before extraction while ordinary restores keep their full contents", async () => {
|
||||
const cwd = await source(); await fs.writeFile(path.join(cwd, "first.txt"), "first\n"); await fs.writeFile(path.join(cwd, "second.txt"), "second\n");
|
||||
const snapshot = await capture(cwd), targetDir = path.join(root, "limited");
|
||||
await expect(restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir,
|
||||
expectedManifestDigest: snapshot.manifestDigest, maxEntries: 1 })).rejects.toThrow("remote_workspace_archive_entry_limit");
|
||||
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
|
||||
const restored = await restore(snapshot);
|
||||
expect(await fs.readFile(path.join(restored, "first.txt"), "utf8")).toBe("first\n");
|
||||
expect(await fs.readFile(path.join(restored, "second.txt"), "utf8")).toBe("second\n");
|
||||
});
|
||||
it("checks a Files manifest byte cap before allocating or reading its body", async () => {
|
||||
const archivePath = path.join(root, "oversized-manifest.pcws"), targetDir = path.join(root, "limited-manifest");
|
||||
const length = Buffer.alloc(4); length.writeUInt32BE(1024);
|
||||
await fs.writeFile(archivePath, Buffer.concat([Buffer.from("PCWS1\n"), length]));
|
||||
await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: "a".repeat(64), maxManifestBytes: 16 }))
|
||||
.rejects.toThrow("remote_workspace_archive_manifest_too_large");
|
||||
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
|
||||
await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: "a".repeat(64) }))
|
||||
.rejects.toThrow("remote_workspace_archive_truncated");
|
||||
});
|
||||
it("does not publish an empty archive when restoration was already canceled", async () => {
|
||||
const snapshot = await capture(await source()), targetDir = path.join(root, "canceled");
|
||||
const controller = new AbortController(), reason = new Error("restore canceled"); controller.abort(reason);
|
||||
await expect(restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir,
|
||||
expectedManifestDigest: snapshot.manifestDigest, signal: controller.signal })).rejects.toBe(reason);
|
||||
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
|
||||
});
|
||||
it("removes private staging after cancellation during read-only directory mode restoration", async () => {
|
||||
const cwd = await source(), directory = path.join(cwd, "readonly"); await fs.mkdir(directory);
|
||||
await fs.writeFile(path.join(directory, "saved.txt"), "saved content"); await fs.chmod(directory, 0o555);
|
||||
const snapshot = await capture(cwd), targetDir = path.join(root, "interrupted");
|
||||
const controller = new AbortController(), reason = new Error("restore canceled while applying modes");
|
||||
let appliedReadOnlyModes = 0;
|
||||
archiveIo.afterChmod = (target, mode) => {
|
||||
if (target.includes(".workspace-restore-") && mode === 0o555) { appliedReadOnlyModes++; controller.abort(reason); }
|
||||
};
|
||||
try {
|
||||
await expect(restoreRemoteWorkspaceArchive({ archivePath: snapshot.archivePath, targetDir,
|
||||
expectedManifestDigest: snapshot.manifestDigest, signal: controller.signal })).rejects.toBe(reason);
|
||||
expect(appliedReadOnlyModes).toBe(1);
|
||||
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect((await fs.readdir(root)).filter(name => name.startsWith(".workspace-restore-"))).toEqual([]);
|
||||
} finally { archiveIo.afterChmod = undefined; await fs.chmod(directory, 0o700); }
|
||||
});
|
||||
it("preserves read-only directory modes without making capture scratch unremovable", async () => {
|
||||
const cwd = await source(), directory = path.join(cwd, "readonly"); await fs.mkdir(directory);
|
||||
await fs.writeFile(path.join(directory, "file"), "retained\n"); await fs.chmod(directory, 0o555);
|
||||
let restored: string | undefined;
|
||||
try {
|
||||
restored = await restore(await capture(cwd));
|
||||
expect((await fs.stat(path.join(restored, "readonly"))).mode & 0o777).toBe(0o555);
|
||||
expect(await fs.readFile(path.join(restored, "readonly", "file"), "utf8")).toBe("retained\n");
|
||||
} finally { await fs.chmod(directory, 0o700); if (restored) await fs.chmod(path.join(restored, "readonly"), 0o700); }
|
||||
});
|
||||
it.each(["ordinary", "linked", "borrowed"])("restores %s Git with staged/unstaged/unpushed data after all sources disappear", async kind => {
|
||||
const base = await source(); await git(base, "init", "-q", "-b", "main");
|
||||
await fs.writeFile(path.join(base, "work.txt"), "base\n"); await git(base, "add", "."); await git(base, "commit", "-qm", "base");
|
||||
let cwd = base;
|
||||
if (kind === "linked") { cwd = path.join(root, "linked"); await git(base, "worktree", "add", "-q", "-b", "feature", cwd); }
|
||||
if (kind === "borrowed") { cwd = path.join(root, "borrowed"); await git(root, "clone", "-q", "--shared", base, cwd); }
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), "unpushed\n"); await git(cwd, "commit", "-qam", "unpushed");
|
||||
const head = await git(cwd, "rev-parse", "HEAD");
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), "staged\n"); await git(cwd, "add", "work.txt");
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), "unstaged\n"); await fs.writeFile(path.join(cwd, "untracked"), "work\n");
|
||||
const snapshot = await capture(cwd); await fs.rm(base, { recursive: true, force: true }); if (cwd !== base) await fs.rm(cwd, { recursive: true });
|
||||
const restored = await restore(snapshot);
|
||||
expect(await git(restored, "rev-parse", "HEAD")).toBe(head);
|
||||
expect(await git(restored, "show", ":work.txt")).toBe("staged");
|
||||
expect(await fs.readFile(path.join(restored, "work.txt"), "utf8")).toBe("unstaged\n");
|
||||
expect(await fs.readFile(path.join(restored, "untracked"), "utf8")).toBe("work\n");
|
||||
expect((await fs.lstat(path.join(restored, ".git"))).isDirectory()).toBe(true);
|
||||
await expect(fs.access(path.join(restored, ".git", "objects", "info", "alternates"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await git(restored, "fsck", "--full", "--no-dangling");
|
||||
});
|
||||
it.each(["rebase-merge", "rebase-apply", "linked-rebase", "sequencer", "MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD"])(
|
||||
"rejects a real unfinished %s operation without changing its source or publishing an archive", async operation => {
|
||||
let cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), "base\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
|
||||
await git(cwd, "checkout", "-qb", "feature");
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), "feature\n"); await git(cwd, "commit", "-qam", "feature change");
|
||||
const first = await git(cwd, "rev-parse", "HEAD");
|
||||
await fs.writeFile(path.join(cwd, "later.txt"), "remaining sequence step\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "later change");
|
||||
const second = await git(cwd, "rev-parse", "HEAD");
|
||||
await git(cwd, "checkout", "-q", "main");
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), "main\n"); await git(cwd, "commit", "-qam", "main change");
|
||||
let command: string[];
|
||||
if (operation === "linked-rebase") {
|
||||
const linked = path.join(root, "linked"); await git(cwd, "worktree", "add", "-q", linked, "feature"); cwd = linked;
|
||||
}
|
||||
if (operation.includes("rebase")) {
|
||||
await git(cwd, "checkout", "-q", "feature");
|
||||
command = ["rebase", operation === "rebase-apply" ? "--apply" : "--merge", "main"];
|
||||
} else if (operation === "sequencer") command = ["cherry-pick", first, second];
|
||||
else if (operation === "CHERRY_PICK_HEAD") command = ["cherry-pick", first];
|
||||
else if (operation === "REVERT_HEAD") command = ["revert", "--no-edit", first];
|
||||
else command = ["merge", "--no-edit", "feature"];
|
||||
await expect(git(cwd, ...command)).rejects.toThrow();
|
||||
const gitDir = await git(cwd, "rev-parse", "--absolute-git-dir");
|
||||
const stateName = operation === "linked-rebase" ? "rebase-merge" : operation;
|
||||
await fs.access(path.join(gitDir, stateName));
|
||||
const before = await git(cwd, "status", "--porcelain=v2");
|
||||
const index = await fs.readFile(path.join(gitDir, "index"));
|
||||
const work = await fs.readFile(path.join(cwd, "work.txt"));
|
||||
const state = (await fs.readdir(gitDir)).sort();
|
||||
const bytes = await sourceBytes(cwd), administration = await sourceBytes(gitDir);
|
||||
await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_git_operation_in_progress");
|
||||
expect(await sourceBytes(cwd)).toBe(bytes);
|
||||
expect(await sourceBytes(gitDir)).toBe(administration);
|
||||
expect(await git(cwd, "status", "--porcelain=v2")).toBe(before);
|
||||
expect(await fs.readFile(path.join(gitDir, "index"))).toEqual(index);
|
||||
expect(await fs.readFile(path.join(cwd, "work.txt"))).toEqual(work);
|
||||
expect((await fs.readdir(gitDir)).sort()).toEqual(state);
|
||||
await fs.access(path.join(gitDir, stateName));
|
||||
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
if (operation === "sequencer") {
|
||||
const helper = path.join(root, "helper.cjs"); await fs.writeFile(helper, remoteWorkspaceArchiveScript());
|
||||
await expect(execute(process.execPath, [helper, "capture", cwd, path.join(root, "remote.pcws")], { env }))
|
||||
.rejects.toMatchObject({ stderr: expect.stringContaining("remote_workspace_archive_git_operation_in_progress") });
|
||||
await expect(fs.access(path.join(root, "remote.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await sourceBytes(cwd)).toBe(bytes);
|
||||
}
|
||||
const abort = operation.includes("rebase") ? "rebase" : operation === "MERGE_HEAD" ? "merge"
|
||||
: operation === "REVERT_HEAD" ? "revert" : "cherry-pick";
|
||||
await git(cwd, abort, "--abort");
|
||||
const head = await git(cwd, "rev-parse", "HEAD"), status = await git(cwd, "status", "--porcelain=v2");
|
||||
const restored = await restore(await capture(cwd));
|
||||
expect(await git(restored, "rev-parse", "HEAD")).toBe(head);
|
||||
expect(await git(restored, "status", "--porcelain=v2")).toBe(status);
|
||||
},
|
||||
);
|
||||
it("rejects an unfinished bisect without discarding its portable selection history", async () => {
|
||||
const cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
|
||||
for (const value of ["base", "middle", "bad"]) {
|
||||
await fs.writeFile(path.join(cwd, "work.txt"), `${value}\n`); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", value);
|
||||
}
|
||||
await git(cwd, "bisect", "start", "HEAD", "HEAD~2");
|
||||
const bytes = await sourceBytes(cwd);
|
||||
await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_git_operation_in_progress");
|
||||
expect(await sourceBytes(cwd)).toBe(bytes);
|
||||
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("rejects an external symlink before creating an archive", async () => {
|
||||
const cwd = await source(); await fs.symlink("../outside", path.join(cwd, "unsafe"));
|
||||
await expect(capture(cwd)).rejects.toThrow("unsafe_symlink");
|
||||
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("preserves shallow repository boundaries and a nested managed checkout", async () => {
|
||||
const base = await source(); await git(base, "init", "-q", "-b", "main");
|
||||
for (const value of ["one", "two"]) {
|
||||
await fs.writeFile(path.join(base, "file"), value); await git(base, "add", "."); await git(base, "commit", "-qm", value);
|
||||
}
|
||||
const cwd = path.join(root, "task"), checkouts = path.join(cwd, ".paperclip-repositories"); await fs.mkdir(checkouts, { recursive: true });
|
||||
const repository = path.join(checkouts, "repo");
|
||||
await git(root, "clone", "-q", "--depth=1", `file://${base}`, repository);
|
||||
const snapshot = await capture(cwd); await fs.rm(base, { recursive: true }); await fs.rm(cwd, { recursive: true });
|
||||
const restored = path.join(await restore(snapshot), ".paperclip-repositories", "repo");
|
||||
expect(await git(restored, "rev-parse", "--is-shallow-repository")).toBe("true");
|
||||
expect(await git(restored, "rev-list", "--count", "HEAD")).toBe("1");
|
||||
expect(await fs.readFile(path.join(restored, "file"), "utf8")).toBe("two");
|
||||
});
|
||||
it("produces the same saved contents for repeated captures of an unchanged Git clone", async () => {
|
||||
const base = await source(); await git(base, "init", "-q", "-b", "main");
|
||||
await fs.writeFile(path.join(base, "file"), "stable\n"); await git(base, "add", "."); await git(base, "commit", "-qm", "base");
|
||||
const cwd = path.join(root, "clone"); await git(root, "clone", "-q", base, cwd);
|
||||
const first = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "first.pcws") });
|
||||
await new Promise(resolve => setTimeout(resolve, 1100));
|
||||
const second = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "second.pcws") });
|
||||
expect(second.manifestDigest).toBe(first.manifestDigest); expect(second.sha256).toBe(first.sha256);
|
||||
});
|
||||
it("preserves portable secondary remotes and tracking settings without local paths or executable configuration", async () => {
|
||||
const cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
|
||||
await fs.writeFile(path.join(cwd, "file"), "portable tracking\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
|
||||
const head = await git(cwd, "rev-parse", "HEAD");
|
||||
await git(cwd, "remote", "add", "origin", "https://github.com/synthetic/work.git");
|
||||
await git(cwd, "remote", "add", "upstream", "ssh://git@example.invalid/team/base.git");
|
||||
await git(cwd, "config", "--add", "remote.upstream.url", "https://backup.example.invalid/team/base.git");
|
||||
await git(cwd, "config", "remote.upstream.pushurl", "git@example.invalid:team/publish.git");
|
||||
await git(cwd, "update-ref", "refs/remotes/upstream/main", head);
|
||||
await git(cwd, "config", "branch.main.remote", "upstream");
|
||||
await git(cwd, "config", "branch.main.merge", "refs/heads/main");
|
||||
await git(cwd, "config", "branch.main.rebase", "true");
|
||||
await git(cwd, "config", "user.name", "Synthetic repository author");
|
||||
await git(cwd, "config", "user.email", "repository-author@example.invalid");
|
||||
await git(cwd, "remote", "add", "local-cache", path.join(root, "old-host-cache.git"));
|
||||
await git(cwd, "config", "branch.cache.remote", "local-cache");
|
||||
await git(cwd, "config", "branch.cache.merge", "refs/heads/main");
|
||||
for (const [key, value] of [["core.hooksPath", path.join(root, "old-host-hooks")], ["core.worktree", cwd],
|
||||
["filter.synthetic.smudge", "echo SYNTHETIC_SENSITIVE"], ["credential.helper", "echo SYNTHETIC_SENSITIVE"],
|
||||
["include.path", path.join(root, "old-host-gitconfig")]]) await git(cwd, "config", key!, value!);
|
||||
const snapshot = await capture(cwd); await fs.rm(cwd, { recursive: true });
|
||||
const restored = await restore(snapshot);
|
||||
expect(await git(restored, "remote")).toBe("origin\nupstream");
|
||||
expect(await git(restored, "config", "--get-all", "remote.upstream.url")).toBe("ssh://git@example.invalid/team/base.git\nhttps://backup.example.invalid/team/base.git");
|
||||
expect(await git(restored, "remote", "get-url", "--push", "upstream")).toBe("git@example.invalid:team/publish.git");
|
||||
expect(await git(restored, "config", "remote.upstream.fetch")).toBe("+refs/heads/*:refs/remotes/upstream/*");
|
||||
expect(await git(restored, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}")).toBe("upstream/main");
|
||||
expect(await git(restored, "config", "branch.main.rebase")).toBe("true");
|
||||
const configuration = await fs.readFile(path.join(restored, ".git", "config"), "utf8");
|
||||
expect(configuration).not.toContain(root); expect(configuration).not.toContain("SYNTHETIC_SENSITIVE");
|
||||
expect(configuration).not.toContain("local-cache"); expect(configuration).not.toContain('branch "cache"');
|
||||
await fs.writeFile(path.join(restored, "next-file"), "later work\n"); await git(restored, "add", "next-file");
|
||||
await execute("git", ["-C", restored, "-c", `core.hooksPath=${os.devNull}`, "-c", "commit.gpgsign=false", "commit", "-qm", "restored workspace continues"], {
|
||||
env: { PATH: process.env.PATH, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_CONFIG_SYSTEM: os.devNull, GIT_CONFIG_COUNT: "0" },
|
||||
});
|
||||
expect(await git(restored, "log", "-1", "--format=%an <%ae>")).toBe("Synthetic repository author <repository-author@example.invalid>");
|
||||
await git(restored, "fsck", "--full", "--no-dangling");
|
||||
});
|
||||
it.each(["https://synthetic-token@github.com/team/repo.git", "https://github.com/team/repo.git?access_token=SYNTHETIC_ONLY"])(
|
||||
"rejects credentials in any saved remote URL: %s", async remoteUrl => {
|
||||
const cwd = await source(); await git(cwd, "init", "-q", "-b", "main");
|
||||
await fs.writeFile(path.join(cwd, "file"), "source\n"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
|
||||
await git(cwd, "remote", "add", "upstream", remoteUrl);
|
||||
await expect(capture(cwd)).rejects.toThrow("credential_in_git_remote");
|
||||
await expect(fs.access(path.join(root, "snapshot.pcws"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
},
|
||||
);
|
||||
it("omits reinstallable dependencies while preserving deliberately tracked files", async () => {
|
||||
const cwd = await source(); await fs.writeFile(path.join(cwd, "package.json"), "{}\n");
|
||||
await fs.mkdir(path.join(cwd, "node_modules")); await fs.writeFile(path.join(cwd, "node_modules", "dependency"), "reinstall\n");
|
||||
const first = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "dependencies.pcws") });
|
||||
const targetDir = path.join(root, "without-dependencies");
|
||||
await restoreRemoteWorkspaceArchive({ archivePath: path.join(root, "dependencies.pcws"), targetDir, expectedManifestDigest: first.manifestDigest });
|
||||
await expect(fs.access(path.join(targetDir, "node_modules"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await git(cwd, "init", "-q", "-b", "main"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "deliberately tracked dependency");
|
||||
const second = await captureRemoteWorkspaceDirectory({ cwd, archivePath: path.join(root, "tracked.pcws") });
|
||||
const trackedTarget = path.join(root, "with-tracked-dependencies");
|
||||
await restoreRemoteWorkspaceArchive({ archivePath: path.join(root, "tracked.pcws"), targetDir: trackedTarget, expectedManifestDigest: second.manifestDigest });
|
||||
expect(await fs.readFile(path.join(trackedTarget, "node_modules", "dependency"), "utf8")).toBe("reinstall\n");
|
||||
});
|
||||
it("rejects a symlink source root", async () => {
|
||||
const cwd = await source(), linked = path.join(root, "linked-source"); await fs.symlink(cwd, linked);
|
||||
await expect(capture(linked)).rejects.toThrow("source_invalid");
|
||||
});
|
||||
it("rejects path traversal before creating a restored directory", async () => {
|
||||
const manifest = { schema: "paperclip.remote-workspace-archive.v1", entries: [{ path: "../escape", type: "file", mode: 0o600,
|
||||
size: 0, sha256: createHash("sha256").digest("hex") }] };
|
||||
const bytes = Buffer.from(JSON.stringify(manifest)), length = Buffer.alloc(4); length.writeUInt32BE(bytes.length);
|
||||
const archivePath = path.join(root, "unsafe.pcws"), targetDir = path.join(root, "target");
|
||||
await fs.writeFile(archivePath, Buffer.concat([Buffer.from("PCWS1\n"), length, bytes]));
|
||||
await expect(restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: createHash("sha256").update(bytes).digest("hex") })).rejects.toThrow("invalid_entry");
|
||||
await expect(fs.access(targetDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(fs.access(path.join(root, "escape"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("rejects a corrupt file without publishing a partial restored directory", async () => {
|
||||
const cwd = await source(); await fs.writeFile(path.join(cwd, "file"), "saved"); const snapshot = await capture(cwd);
|
||||
const file = await fs.open(snapshot.archivePath, "r+"); try { const size = (await file.stat()).size; await file.write(Buffer.from("X"), 0, 1, size - 1); } finally { await file.close(); }
|
||||
await expect(restore(snapshot)).rejects.toThrow("file_hash_mismatch");
|
||||
await expect(fs.access(path.join(root, "restored"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("executes the generated helper in a separate Node process", async () => {
|
||||
const cwd = await source(); await fs.writeFile(path.join(cwd, "file"), "remote helper\n");
|
||||
await git(cwd, "init", "-q", "-b", "main"); await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base");
|
||||
await git(cwd, "remote", "add", "upstream", "https://example.invalid/team/base.git");
|
||||
await git(cwd, "config", "branch.main.remote", "upstream"); await git(cwd, "config", "branch.main.merge", "refs/heads/main");
|
||||
const script = path.join(root, "helper.cjs"), archivePath = path.join(root, "remote.pcws"), target = path.join(root, "remote-restored");
|
||||
await fs.writeFile(script, remoteWorkspaceArchiveScript());
|
||||
const receipt = JSON.parse((await execute(process.execPath, [script, "capture", cwd, archivePath])).stdout);
|
||||
await fs.rm(cwd, { recursive: true });
|
||||
await execute(process.execPath, [script, "restore", archivePath, target, receipt.manifestDigest]);
|
||||
expect(await fs.readFile(path.join(target, "file"), "utf8")).toBe("remote helper\n");
|
||||
expect(await git(target, "remote", "get-url", "upstream")).toBe("https://example.invalid/team/base.git");
|
||||
expect(await git(target, "config", "branch.main.remote")).toBe("upstream");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,352 @@
|
||||
/**
|
||||
* A portable, streaming workspace archive. This function is intentionally
|
||||
* self-contained: the same implementation runs locally and in a sandbox.
|
||||
*/
|
||||
async function workspaceArchive(input: {
|
||||
kind: "capture" | "restore";
|
||||
cwd?: string;
|
||||
archivePath: string;
|
||||
manifestPath?: string;
|
||||
targetDir?: string;
|
||||
expectedManifestDigest?: string;
|
||||
exclude?: string[];
|
||||
signal?: AbortSignal;
|
||||
maxEntries?: number;
|
||||
maxManifestBytes?: number;
|
||||
}): Promise<{ schema: "paperclip.remote-workspace-archive.v1"; manifestDigest: string; byteSize: number; sha256: string; entries: number }> {
|
||||
const fs = await import("node:fs/promises");
|
||||
const { createReadStream } = await import("node:fs");
|
||||
const path = await import("node:path");
|
||||
const os = await import("node:os");
|
||||
const crypto = await import("node:crypto");
|
||||
const child = await import("node:child_process");
|
||||
const schema = "paperclip.remote-workspace-archive.v1" as const;
|
||||
const magic = Buffer.from("PCWS1\n");
|
||||
type Entry = { path: string; type: "directory" | "file" | "symlink"; mode?: number; size?: number; sha256?: string; target?: string };
|
||||
const fail = (reason: string): never => { throw new Error(`remote_workspace_archive_${reason}`); };
|
||||
const check = () => input.signal?.throwIfAborted();
|
||||
check();
|
||||
const hashFile = async (file: string) => {
|
||||
check();
|
||||
const hash = crypto.createHash("sha256"); let size = 0;
|
||||
for await (const bytes of createReadStream(file, { signal: input.signal })) { check(); hash.update(bytes); size += bytes.length; }
|
||||
return { sha256: hash.digest("hex"), byteSize: size };
|
||||
};
|
||||
const safeRelative = (value: unknown): value is string => typeof value === "string" && value.length > 0
|
||||
&& !value.includes("\\") && !/[\u0000-\u001f\u007f]/u.test(value) && !path.posix.isAbsolute(value)
|
||||
&& value.split("/").every(part => part !== "" && part !== "." && part !== "..");
|
||||
const safeLink = (relative: string, target: string) => {
|
||||
if (typeof target !== "string" || !target || target.includes("\\") || /[\u0000-\u001f\u007f]/u.test(target) || path.posix.isAbsolute(target)) return false;
|
||||
const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(relative), target));
|
||||
return resolved !== ".." && !resolved.startsWith("../");
|
||||
};
|
||||
const git = (cwd: string, args: string[], stdin?: string) => new Promise<string>((resolve, reject) => {
|
||||
check();
|
||||
const command = child.execFile("git", ["-c", "core.fsmonitor=false", "-c", "pack.threads=1", "-c", `core.hooksPath=${os.devNull}`, "-C", cwd, ...args], {
|
||||
signal: input.signal, timeout: 300_000, maxBuffer: 64 * 1024 * 1024,
|
||||
env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0",
|
||||
GIT_AUTHOR_NAME: "Paperclip", GIT_COMMITTER_NAME: "Paperclip", GIT_AUTHOR_EMAIL: "workspace@paperclip.invalid", GIT_COMMITTER_EMAIL: "workspace@paperclip.invalid",
|
||||
GIT_AUTHOR_DATE: "@0 +0000", GIT_COMMITTER_DATE: "@0 +0000" },
|
||||
}, (error, stdout) => error ? reject(error) : resolve(stdout));
|
||||
if (stdin !== undefined) command.stdin?.end(stdin);
|
||||
});
|
||||
const portableRemoteUrl = (value: string) => {
|
||||
if (!value || /[\u0000-\u0020\u007f\\]/u.test(value)) return false;
|
||||
if (/^[a-z][a-z0-9+.-]*:\/\//iu.test(value)) {
|
||||
let url: URL; try { url = new URL(value); } catch { return false; }
|
||||
if (!["http:", "https:", "ssh:", "git:"].includes(url.protocol)) return false;
|
||||
if (url.password || (url.username && url.protocol !== "ssh:") || url.search || url.hash) fail("credential_in_git_remote");
|
||||
return Boolean(url.hostname);
|
||||
}
|
||||
if (/^[a-z]:[\/\\]/iu.test(value)) return false;
|
||||
return /^(?:[a-z0-9._-]+@)?(?:[a-z0-9._-]+|\[[a-f0-9:]+\]):[^\s]+$/iu.test(value);
|
||||
};
|
||||
const portableGitSettings = async (repo: string) => {
|
||||
// Read only the repository's own allowlisted values. Includes, execution
|
||||
// hooks, filters, credential helpers, and machine-specific paths stay out.
|
||||
const raw = await git(repo, ["config", "--null", "--local", "--no-includes", "--get-regexp",
|
||||
"^(remote\\..*\\.(url|pushurl|fetch)|branch\\..*\\.(remote|merge|rebase)|user\\.(name|email))$"])
|
||||
.catch((error: { code?: number }) => { if (error.code === 1) return ""; throw error; });
|
||||
const settings = raw.split("\0").filter(Boolean).map(row => {
|
||||
const split = row.indexOf("\n"), key = row.slice(0, split), value = row.slice(split + 1);
|
||||
if (split < 1 || /[\u0000-\u0020\u007f\\]/u.test(key) || /[\u0000-\u001f\u007f\\]/u.test(value)) fail("unsupported_git_setting");
|
||||
return { key, value, remote: /^remote\.(.+)\.(url|pushurl|fetch)$/.exec(key), branch: /^branch\.(.+)\.(remote|merge|rebase)$/.exec(key), identity: /^user\.(name|email)$/.test(key) };
|
||||
});
|
||||
const remotes = new Set(settings.flatMap(item => item.remote?.[2] === "url" ? [item.remote[1]!] : []));
|
||||
for (const setting of settings) if (setting.remote && ["url", "pushurl"].includes(setting.remote[2]!)) {
|
||||
if (!portableRemoteUrl(setting.value)) remotes.delete(setting.remote[1]!);
|
||||
}
|
||||
const branches = new Map<string, boolean>();
|
||||
for (const setting of settings) if (setting.branch?.[2] === "remote") {
|
||||
const name = setting.branch[1]!;
|
||||
branches.set(name, (branches.get(name) ?? true) && (setting.value === "." || remotes.has(setting.value)));
|
||||
}
|
||||
const retained: Array<{ key: string; value: string }> = [];
|
||||
for (const setting of settings) {
|
||||
if (setting.remote) {
|
||||
if (!remotes.has(setting.remote[1]!)) continue;
|
||||
if (setting.remote[2] === "fetch") {
|
||||
const refs = setting.value.replace(/^[+^]/u, "").split(":");
|
||||
if (refs.length > 2 || !refs.some(Boolean)) fail("unsupported_git_setting");
|
||||
for (const ref of refs.filter(Boolean)) {
|
||||
if (ref.startsWith("-")) fail("unsupported_git_setting");
|
||||
await git(repo, ["check-ref-format", "--allow-onelevel", "--refspec-pattern", ref]).catch(() => fail("unsupported_git_setting"));
|
||||
}
|
||||
}
|
||||
} else if (setting.branch) {
|
||||
if (!(branches.get(setting.branch[1]!) ?? remotes.has("origin"))) continue;
|
||||
if (setting.branch[2] === "merge") {
|
||||
if (setting.value.startsWith("-")) fail("unsupported_git_setting");
|
||||
await git(repo, ["check-ref-format", "--allow-onelevel", setting.value]).catch(() => fail("unsupported_git_setting"));
|
||||
}
|
||||
if (setting.branch[2] === "rebase" && !["true", "false", "merges", "interactive", "preserve", "yes", "no", "on", "off", "1", "0", ""].includes(setting.value)) fail("unsupported_git_setting");
|
||||
} else if (!setting.identity) fail("unsupported_git_setting");
|
||||
retained.push({ key: setting.key, value: setting.value });
|
||||
}
|
||||
return retained;
|
||||
};
|
||||
const assertGitOperationComplete = async (repo: string) => {
|
||||
const gitDir = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim();
|
||||
// A mirror plus the selected index is not a portable continuation of an
|
||||
// unfinished Git operation. Never acknowledge a copy that drops its todo,
|
||||
// scripts, autostash, or operation-only object references. Inspect the
|
||||
// selected worktree's administration directory without following links.
|
||||
for (const name of ["rebase-merge", "rebase-apply", "sequencer", "MERGE_HEAD", "CHERRY_PICK_HEAD",
|
||||
"REVERT_HEAD", "REBASE_HEAD", "MERGE_AUTOSTASH", "BISECT_START"]) {
|
||||
const state = await fs.lstat(path.join(gitDir, name)).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
return null;
|
||||
});
|
||||
if (state) fail("git_operation_in_progress: finish or abort the Git operation before saving the workspace");
|
||||
}
|
||||
};
|
||||
const directories: string[] = [];
|
||||
const excludedPaths = new Set<string>();
|
||||
const walk = async (root: string, copyTo?: string, skipGit = false) => {
|
||||
const entries: Entry[] = [];
|
||||
const visit = async (relative: string, parentIsRepository = false) => {
|
||||
const directory = relative ? path.join(root, relative) : root;
|
||||
const directoryStat = await fs.lstat(directory);
|
||||
if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink() || await fs.realpath(directory) !== directory) fail("changed");
|
||||
const names = (await fs.readdir(directory)).sort();
|
||||
const inRepository = parentIsRepository || names.includes(".git");
|
||||
for (const name of names) {
|
||||
check();
|
||||
const next = relative ? `${relative}/${name}` : name;
|
||||
if (!safeRelative(next)) fail("unsupported_filename");
|
||||
if (input.exclude?.some(excluded => next === excluded || next.startsWith(`${excluded}/`)) || next === ".paperclip-runtime" || next.startsWith(".paperclip-runtime/")) {
|
||||
excludedPaths.add(next); continue;
|
||||
}
|
||||
const source = path.join(root, next), target = copyTo ? path.join(copyTo, next) : null;
|
||||
const stat = await fs.lstat(source);
|
||||
if (skipGit && name === ".git") { if (stat.isSymbolicLink()) fail("unsafe_git_link"); directories.push(relative); continue; }
|
||||
if (stat.isDirectory() && name === "node_modules") {
|
||||
const packageManifest = await fs.lstat(path.join(directory, "package.json")).catch(() => null);
|
||||
if (packageManifest?.isFile() && !packageManifest.isSymbolicLink()) {
|
||||
// A failing Git read cannot establish that data is untracked.
|
||||
const tracked = inRepository ? await git(directory, ["ls-files", "-z", "--", name]) : "";
|
||||
if (!tracked) { excludedPaths.add(next); continue; }
|
||||
}
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
entries.push({ path: next, type: "directory", mode: stat.mode & 0o777 });
|
||||
if (target) await fs.mkdir(target, { mode: 0o700 });
|
||||
await visit(next, inRepository);
|
||||
} else if (stat.isSymbolicLink()) {
|
||||
const link = await fs.readlink(source);
|
||||
if (!safeLink(next, link)) fail("unsafe_symlink");
|
||||
entries.push({ path: next, type: "symlink", target: link });
|
||||
if (target) await fs.symlink(link, target);
|
||||
} else if (stat.isFile()) {
|
||||
const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK);
|
||||
let output: Awaited<ReturnType<typeof fs.open>> | undefined;
|
||||
try {
|
||||
const before = await handle.stat({ bigint: true });
|
||||
if (!before.isFile() || BigInt(stat.dev) !== before.dev || BigInt(stat.ino) !== before.ino || await fs.realpath(source) !== source) fail("changed");
|
||||
if (target) output = await fs.open(target, "wx", 0o600);
|
||||
const digest = crypto.createHash("sha256"); let bytes = 0;
|
||||
for await (const part of handle.createReadStream({ autoClose: false, signal: input.signal })) {
|
||||
check(); digest.update(part); bytes += part.length;
|
||||
await output?.writeFile(part);
|
||||
}
|
||||
const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true });
|
||||
if (bytes !== Number(before.size) || await fs.realpath(source) !== source || ["dev", "ino", "size", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed");
|
||||
entries.push({ path: next, type: "file", mode: stat.mode & 0o777, size: bytes, sha256: digest.digest("hex") });
|
||||
if (target) await fs.chmod(target, stat.mode & 0o777);
|
||||
} finally { await output?.close(); await handle.close(); }
|
||||
} else fail("unsupported_entry");
|
||||
}
|
||||
};
|
||||
await visit("");
|
||||
return entries;
|
||||
};
|
||||
const manifestHash = (manifest: { schema: string; entries: Entry[] }) => crypto.createHash("sha256").update(JSON.stringify(manifest)).digest("hex");
|
||||
if (input.kind === "capture") {
|
||||
if (!input.cwd) fail("source_missing");
|
||||
const sourceEntry = await fs.lstat(input.cwd!);
|
||||
if (!sourceEntry.isDirectory() || sourceEntry.isSymbolicLink()) fail("source_invalid");
|
||||
const root = await fs.realpath(input.cwd!);
|
||||
if (!(await fs.lstat(root)).isDirectory()) fail("source_invalid");
|
||||
const scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-capture-")));
|
||||
try {
|
||||
const tree = path.join(scratch, "tree"); await fs.mkdir(tree, { mode: 0o700 });
|
||||
const before = await walk(root, tree, true);
|
||||
const repositories = [...new Set(directories)];
|
||||
const gitSignatures = new Map<string, string>();
|
||||
const signature = async (repo: string, settings?: Awaited<ReturnType<typeof portableGitSettings>>) => {
|
||||
await assertGitOperationComplete(repo);
|
||||
const index = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-path", "index"])).trim();
|
||||
const indexHash = await hashFile(index).catch((error: NodeJS.ErrnoException) => error.code === "ENOENT" ? null : Promise.reject(error));
|
||||
return JSON.stringify([await git(repo, ["status", "--porcelain=v2", "-z", "--untracked-files=no"]), await git(repo, ["for-each-ref", "--format=%(objectname) %(refname)"]), await git(repo, ["symbolic-ref", "-q", "HEAD"]).catch(() => git(repo, ["rev-parse", "HEAD"])), indexHash, settings ?? await portableGitSettings(repo)]);
|
||||
};
|
||||
for (let index = 0; index < repositories.length; index++) {
|
||||
const relative = repositories[index]!, repo = path.join(root, relative), normalized = path.join(scratch, `repo-${index}.git`);
|
||||
const settings = await portableGitSettings(repo);
|
||||
gitSignatures.set(relative, await signature(repo, settings));
|
||||
// Mirror copies all refs; dissociation brings borrowed objects into this
|
||||
// private repository. The selected worktree's index and HEAD are separate.
|
||||
await git(root, ["clone", "--mirror", "--no-hardlinks", "--dissociate", "--", repo, normalized]);
|
||||
const destination = path.join(tree, relative, ".git");
|
||||
await fs.rename(normalized, destination);
|
||||
const sourceGit = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim();
|
||||
const sourceCommon = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim();
|
||||
for (const name of ["HEAD", "index", "ORIG_HEAD", "MERGE_HEAD", "MERGE_MSG", "CHERRY_PICK_HEAD", "REVERT_HEAD"]) {
|
||||
await fs.copyFile(path.join(sourceGit, name), path.join(destination, name)).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; });
|
||||
}
|
||||
for (const directory of new Set([sourceGit, sourceCommon])) {
|
||||
for (const name of await fs.readdir(directory)) if (/^sharedindex\.[a-f0-9]+$/.test(name)) await fs.copyFile(path.join(directory, name), path.join(destination, name));
|
||||
}
|
||||
for (const name of ["exclude", "attributes"]) {
|
||||
await fs.mkdir(path.join(destination, "info"), { recursive: true });
|
||||
await fs.copyFile(path.join(sourceCommon, "info", name), path.join(destination, "info", name)).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; });
|
||||
}
|
||||
const restoredRepo = path.join(tree, relative);
|
||||
await git(restoredRepo, ["config", "core.bare", "false"]);
|
||||
await git(restoredRepo, ["config", "--unset-all", "remote.origin.mirror"]).catch(() => undefined);
|
||||
await git(restoredRepo, ["remote", "remove", "origin"]).catch(() => undefined);
|
||||
for (const setting of settings) await git(restoredRepo, ["config", "--add", setting.key, setting.value]);
|
||||
// Removing the source-only remote also removes its refs. Restore every
|
||||
// source ref without using the network; objects are already self-contained.
|
||||
const refs = await git(repo, ["for-each-ref", "--format=%(objectname) %(refname)"]);
|
||||
const updates = refs.split("\n").filter(Boolean).map(row => {
|
||||
const [object, ref] = row.split(" "); return `update ${ref} ${object}\n`;
|
||||
}).join("");
|
||||
if (updates) await git(restoredRepo, ["update-ref", "--stdin"], updates);
|
||||
await git(restoredRepo, ["repack", "-a", "-d"]);
|
||||
await fs.rm(path.join(destination, "objects", "info", "alternates"), { force: true });
|
||||
await git(restoredRepo, ["fsck", "--connectivity-only", "--no-dangling"]);
|
||||
if (await signature(repo) !== gitSignatures.get(relative)) fail("changed");
|
||||
}
|
||||
directories.length = 0;
|
||||
const after = await walk(root, undefined, true);
|
||||
if (JSON.stringify(before) !== JSON.stringify(after)) fail("changed");
|
||||
for (const relative of repositories) await assertGitOperationComplete(path.join(root, relative));
|
||||
const entries = await walk(tree);
|
||||
// Keep private staging writable for cleanup; the manifest preserves the
|
||||
// source modes and restoration applies them only after writing children.
|
||||
const originals = new Map(before.map(entry => [entry.path, entry]));
|
||||
for (const entry of entries) if (entry.type === "directory") {
|
||||
const original = originals.get(entry.path); if (original) entry.mode = original.mode;
|
||||
}
|
||||
const manifest = { schema, exclusionPolicy: "paperclip.remote-workspace-exclusions.v1", excluded: [...excludedPaths].sort(), entries }, body = Buffer.from(JSON.stringify(manifest));
|
||||
if (body.length > 64 * 1024 * 1024) fail("manifest_too_large");
|
||||
const digest = manifestHash(manifest), length = Buffer.alloc(4); length.writeUInt32BE(body.length);
|
||||
const output = await fs.open(input.archivePath, "wx", 0o600);
|
||||
try {
|
||||
await output.writeFile(Buffer.concat([magic, length, body]));
|
||||
for (const entry of entries) if (entry.type === "file") {
|
||||
for await (const bytes of createReadStream(path.join(tree, entry.path), { signal: input.signal })) { check(); await output.writeFile(bytes); }
|
||||
}
|
||||
} finally { await output.close(); }
|
||||
if (input.manifestPath) await fs.writeFile(input.manifestPath, body, { flag: "wx", mode: 0o600 });
|
||||
return { schema, manifestDigest: digest, ...await hashFile(input.archivePath), entries: entries.length };
|
||||
} finally { await fs.rm(scratch, { recursive: true, force: true }); }
|
||||
}
|
||||
if (!input.targetDir || !/^[a-f0-9]{64}$/.test(input.expectedManifestDigest ?? "")) fail("restore_binding_invalid");
|
||||
if (input.maxEntries !== undefined && (!Number.isSafeInteger(input.maxEntries) || input.maxEntries < 0)) fail("restore_binding_invalid");
|
||||
if (input.maxManifestBytes !== undefined && (!Number.isSafeInteger(input.maxManifestBytes) || input.maxManifestBytes < 0)) fail("restore_binding_invalid");
|
||||
const archive = await fs.open(input.archivePath, "r");
|
||||
let staging: string | undefined;
|
||||
const stagedDirectories: string[] = [];
|
||||
try {
|
||||
const prefix = Buffer.alloc(magic.length + 4);
|
||||
if ((await archive.read(prefix, 0, prefix.length, 0)).bytesRead !== prefix.length || !prefix.subarray(0, magic.length).equals(magic)) fail("invalid_header");
|
||||
check();
|
||||
const manifestSize = prefix.readUInt32BE(magic.length);
|
||||
if (manifestSize > 64 * 1024 * 1024 || input.maxManifestBytes !== undefined && manifestSize > input.maxManifestBytes) fail("manifest_too_large");
|
||||
const bytes = Buffer.alloc(manifestSize);
|
||||
if ((await archive.read(bytes, 0, manifestSize, prefix.length)).bytesRead !== manifestSize) fail("truncated");
|
||||
check();
|
||||
const manifest = JSON.parse(bytes.toString("utf8")) as { schema: string; entries: Entry[] };
|
||||
if (manifest.schema !== schema || !Array.isArray(manifest.entries) || manifestHash(manifest) !== input.expectedManifestDigest) fail("manifest_mismatch");
|
||||
if (input.maxEntries !== undefined && manifest.entries.length > input.maxEntries) fail("entry_limit");
|
||||
const kinds = new Map<string, string>(); let bodySize = 0;
|
||||
for (const entry of manifest.entries) {
|
||||
check();
|
||||
if (!entry || !safeRelative(entry.path) || kinds.has(entry.path)) fail("invalid_entry");
|
||||
const parent = path.posix.dirname(entry.path);
|
||||
if (parent !== "." && kinds.get(parent) !== "directory") fail("invalid_parent");
|
||||
if (entry.type === "file") {
|
||||
if (!Number.isSafeInteger(entry.size) || entry.size! < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256 ?? "")) fail("invalid_file");
|
||||
bodySize += entry.size!;
|
||||
} else if (entry.type === "symlink") { if (!safeLink(entry.path, entry.target!)) fail("unsafe_symlink"); }
|
||||
else if (entry.type !== "directory") fail("invalid_type");
|
||||
if (entry.type !== "symlink" && (!Number.isInteger(entry.mode) || entry.mode! < 0 || entry.mode! > 0o777)) fail("invalid_mode");
|
||||
kinds.set(entry.path, entry.type);
|
||||
}
|
||||
if (!Number.isSafeInteger(bodySize) || (await archive.stat()).size !== prefix.length + manifestSize + bodySize) fail("size_mismatch");
|
||||
const parent = await fs.realpath(path.dirname(input.targetDir!));
|
||||
const existing = await fs.lstat(input.targetDir!).catch((error: NodeJS.ErrnoException) => { if (error.code !== "ENOENT") throw error; return null; });
|
||||
if (existing && (!existing.isDirectory() || existing.isSymbolicLink() || (await fs.readdir(input.targetDir!)).length)) fail("target_not_empty");
|
||||
check();
|
||||
staging = await fs.mkdtemp(path.join(parent, ".workspace-restore-"));
|
||||
let offset = prefix.length + manifestSize;
|
||||
const buffer = Buffer.alloc(1024 * 1024);
|
||||
for (const entry of manifest.entries) {
|
||||
check(); const target = path.join(staging, entry.path);
|
||||
if (entry.type === "directory") { await fs.mkdir(target, { mode: 0o700 }); stagedDirectories.push(target); }
|
||||
else if (entry.type === "symlink") await fs.symlink(entry.target!, target);
|
||||
else {
|
||||
const output = await fs.open(target, "wx", 0o600), digest = crypto.createHash("sha256");
|
||||
try {
|
||||
let remaining = entry.size!;
|
||||
while (remaining) {
|
||||
check(); const read = await archive.read(buffer, 0, Math.min(buffer.length, remaining), offset);
|
||||
if (!read.bytesRead) fail("truncated");
|
||||
const part = buffer.subarray(0, read.bytesRead); await output.writeFile(part); digest.update(part);
|
||||
offset += read.bytesRead; remaining -= read.bytesRead;
|
||||
}
|
||||
if (digest.digest("hex") !== entry.sha256) fail("file_hash_mismatch");
|
||||
} finally { await output.close(); }
|
||||
await fs.chmod(target, entry.mode!);
|
||||
}
|
||||
}
|
||||
for (const entry of manifest.entries.filter(entry => entry.type === "directory").reverse()) {
|
||||
check(); await fs.chmod(path.join(staging, entry.path), entry.mode!);
|
||||
}
|
||||
const content = await hashFile(input.archivePath);
|
||||
check();
|
||||
await fs.rename(staging, input.targetDir!); staging = undefined;
|
||||
return { schema, manifestDigest: input.expectedManifestDigest!, ...content, entries: manifest.entries.length };
|
||||
} finally {
|
||||
await archive.close();
|
||||
if (staging) {
|
||||
for (const directory of stagedDirectories) await fs.chmod(directory, 0o700).catch(() => undefined);
|
||||
await fs.rm(staging, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function captureRemoteWorkspaceDirectory(input: { cwd: string; archivePath: string; manifestPath?: string; exclude?: string[]; signal?: AbortSignal }) {
|
||||
return workspaceArchive({ ...input, kind: "capture" });
|
||||
}
|
||||
|
||||
export function restoreRemoteWorkspaceArchive(input: { archivePath: string; targetDir: string; expectedManifestDigest: string; signal?: AbortSignal; maxEntries?: number; maxManifestBytes?: number }) {
|
||||
return workspaceArchive({ ...input, kind: "restore" });
|
||||
}
|
||||
|
||||
export function remoteWorkspaceArchiveScript(): string {
|
||||
// tsx's function-name preservation helper may appear in Function#toString.
|
||||
const source = workspaceArchive.toString().replaceAll("__vite_ssr_dynamic_import__", "import");
|
||||
return `const __name = (value) => value;\nconst archive = ${source};\nconst [kind, first, second, digest] = process.argv.slice(2);\narchive(kind === 'capture' ? {kind,cwd:first,archivePath:second} : {kind,archivePath:first,targetDir:second,expectedManifestDigest:digest}).then(value=>process.stdout.write(JSON.stringify(value))).catch(error=>{process.stderr.write(error.message);process.exitCode=1;});\n`;
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots,
|
||||
executionWorkspaces, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import { createStorageService } from "../storage/service.js";
|
||||
import { createS3StorageProvider } from "../storage/s3-provider.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { issueRecoveryActionService } from "./issue-recovery-actions.js";
|
||||
import { notifyRemoteWorkspaceSnapshots, registerRemoteWorkspaceSnapshotRetentionWork, remoteWorkspaceSnapshotRetentionService } from "./remote-workspace-snapshot-retention.js";
|
||||
|
||||
const url = process.env.PAPERCLIP_TEST_DATABASE_URL;
|
||||
if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1))) throw new Error("Disposable remote_workspace_*test database required");
|
||||
const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
|
||||
async function versionedS3Fixture() {
|
||||
vi.stubEnv("AWS_ACCESS_KEY_ID", "synthetic-access-key");
|
||||
vi.stubEnv("AWS_SECRET_ACCESS_KEY", "synthetic-secret-key");
|
||||
vi.stubEnv("AWS_SESSION_TOKEN", "");
|
||||
const versions: { key: string; versionId: string; marker: boolean; size: number }[] = [];
|
||||
const requests: { method: string; key: string; versionId: string | null; keyMarker: string | null; versionMarker: string | null }[] = [];
|
||||
const faults = { denied: false, listDenied: false, missingCursor: false, repeatCursor: false, preserveVersions: false, hangList: false, abortedLists: 0 };
|
||||
let serial = 0;
|
||||
const add = (key: string, marker = false, versionId = `version-${++serial}`) => {
|
||||
versions.unshift({ key, versionId, marker, size: 14 });
|
||||
};
|
||||
const xml = (value: string) => value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
||||
const server = createServer((req, res) => {
|
||||
req.resume();
|
||||
req.on("end", () => {
|
||||
const parsed = new URL(req.url!, "http://fixture.invalid");
|
||||
const key = decodeURIComponent(parsed.pathname.replace(/^\/bucket\/?/, ""));
|
||||
const versionId = parsed.searchParams.get("versionId");
|
||||
const keyMarker = parsed.searchParams.get("key-marker"), versionMarker = parsed.searchParams.get("version-id-marker");
|
||||
requests.push({ method: req.method!, key, versionId, keyMarker, versionMarker });
|
||||
if (parsed.searchParams.has("versions")) {
|
||||
if (faults.listDenied) {
|
||||
res.writeHead(403, { "content-type": "application/xml" });
|
||||
res.end("<Error><Code>AccessDenied</Code><Message>Version listing permission is required</Message></Error>");
|
||||
return;
|
||||
}
|
||||
if (faults.hangList) { res.on("close", () => { faults.abortedLists++; }); return; }
|
||||
const prefix = parsed.searchParams.get("prefix")!;
|
||||
const matches = versions.filter(value => value.key.startsWith(prefix)).sort((a, b) => a.key.localeCompare(b.key));
|
||||
const start = keyMarker ? matches.findIndex(value => value.key === keyMarker && value.versionId === versionMarker) + 1 : 0;
|
||||
const page = matches.slice(start, start + 2), truncated = matches.length > start + page.length;
|
||||
const last = faults.repeatCursor && keyMarker ? { key: keyMarker, versionId: versionMarker! } : page.at(-1);
|
||||
res.setHeader("content-type", "application/xml");
|
||||
res.end(`<?xml version="1.0"?><ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">` +
|
||||
`<IsTruncated>${truncated}</IsTruncated>` +
|
||||
(truncated && !faults.missingCursor ? `<NextKeyMarker>${xml(last!.key)}</NextKeyMarker><NextVersionIdMarker>${xml(last!.versionId)}</NextVersionIdMarker>` : "") +
|
||||
page.map(value => `<${value.marker ? "DeleteMarker" : "Version"}><Key>${xml(value.key)}</Key><VersionId>${xml(value.versionId)}</VersionId>` +
|
||||
`<IsLatest>false</IsLatest><LastModified>2026-10-30T12:00:00Z</LastModified>${value.marker ? "" : `<Size>${value.size}</Size>`}</${value.marker ? "DeleteMarker" : "Version"}>`).join("") +
|
||||
"</ListVersionsResult>");
|
||||
} else if (req.method === "PUT") {
|
||||
add(key); res.setHeader("x-amz-version-id", versions[0]!.versionId); res.end();
|
||||
} else if (req.method === "DELETE") {
|
||||
if (versionId && faults.denied) {
|
||||
res.writeHead(403, { "content-type": "application/xml" });
|
||||
res.end("<Error><Code>AccessDenied</Code><Message>Object retention forbids deletion</Message></Error>");
|
||||
return;
|
||||
}
|
||||
if (versionId) {
|
||||
if (!faults.preserveVersions) {
|
||||
const index = versions.findIndex(value => value.key === key && value.versionId === versionId);
|
||||
if (index >= 0) versions.splice(index, 1);
|
||||
}
|
||||
} else add(key, true);
|
||||
res.writeHead(204); res.end();
|
||||
} else if (req.method === "HEAD") {
|
||||
const latest = versions.find(value => value.key === key);
|
||||
if (!latest || latest.marker) { res.writeHead(404); res.end(); }
|
||||
else { res.setHeader("content-length", latest.size); res.end(); }
|
||||
} else { res.writeHead(400); res.end(); }
|
||||
});
|
||||
});
|
||||
await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
const endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
|
||||
const provider = createS3StorageProvider({ bucket: "bucket", region: "us-east-1", prefix: "scoped-prefix", endpoint, forcePathStyle: true });
|
||||
return { provider, versions, requests, faults, add,
|
||||
close: async () => { server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve())); } };
|
||||
}
|
||||
|
||||
describe.skipIf(!support.supported)("remote workspace snapshot retention", () => {
|
||||
let db: ReturnType<typeof createDb>, root: string, storage: StorageService;
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
const now = new Date("2026-10-30T12:00:00Z");
|
||||
beforeAll(async () => {
|
||||
if (url) { await applyPendingMigrations(url); db = createDb(url); }
|
||||
else { temporary = await startEmbeddedPostgresTestDatabase("workspace-snapshot-retention-"); db = createDb(temporary.connectionString); }
|
||||
}, 90_000);
|
||||
afterAll(async () => { if (url) await closeRegisteredClients(url, { scope: "database" }); else await temporary?.cleanup(); });
|
||||
beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-retention-test-")); storage = createStorageService(createLocalDiskStorageProvider(root)); });
|
||||
afterEach(async () => {
|
||||
await db.delete(issueRecoveryActions); await db.delete(workspaceOperations); await db.delete(executionWorkspaceSnapshots);
|
||||
await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(companies);
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
async function seed(status: "superseded" | "pending" | "uploaded" | "committed" = "superseded", id = randomUUID()) {
|
||||
const companyId = randomUUID(), workspaceId = randomUUID(), operationId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `R${companyId.slice(0, 6)}` });
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Retention source" }).returning();
|
||||
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issue!.id,
|
||||
mode: "isolated_workspace", strategyType: "project_primary", name: "Retained" });
|
||||
const keys = ["winner", "ambiguous"].map(name => `${companyId}/workspace-snapshots/${workspaceId}/${id}/${name}.pcws`);
|
||||
for (const objectKey of keys) await storage.putFile({ objectKey, companyId, namespace: "workspace-snapshots", originalFilename: null,
|
||||
contentType: "application/octet-stream", body: Buffer.from("retained bytes") });
|
||||
const [row] = await db.insert(executionWorkspaceSnapshots).values({ id, companyId, executionWorkspaceId: workspaceId,
|
||||
sourceOperationId: operationId, generation: 1, epoch: 1, status, objectKey: keys[0],
|
||||
metadata: { uploads: keys.map(objectKey => ({ objectKey })) },
|
||||
deleteAfter: new Date(now.getTime() - 1000), createdAt: new Date(now.getTime() - 10 * 86400_000) }).returning();
|
||||
return { row: row!, companyId, workspaceId, issueId: issue!.id, operationId, keys };
|
||||
}
|
||||
const service = () => remoteWorkspaceSnapshotRetentionService(db, { storage, now: () => now });
|
||||
it("deletes every recorded attempt and winning object after grace", async () => {
|
||||
const f = await seed(); const owner = service();
|
||||
expect(await owner.tick()).toMatchObject({ deleted: 1, failed: 0 });
|
||||
for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false);
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
});
|
||||
it("purges exact-key S3 versions and delete markers before settling retention", async () => {
|
||||
const s3 = await versionedS3Fixture();
|
||||
try {
|
||||
storage = createStorageService(s3.provider);
|
||||
const f = await seed();
|
||||
await storage.putFile({ objectKey: f.keys[0]!, companyId: f.companyId, namespace: "workspace-snapshots", originalFilename: null,
|
||||
contentType: "application/octet-stream", body: Buffer.from("older retry bytes") });
|
||||
await storage.deleteObject(f.companyId, f.keys[0]!);
|
||||
expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false);
|
||||
expect(s3.requests.filter(value => value.method === "GET")).toHaveLength(0);
|
||||
const neighbor = `scoped-prefix/${f.keys[0]}-neighbor`;
|
||||
s3.add(neighbor);
|
||||
s3.add(`scoped-prefix/${f.keys[1]}`, false, "null");
|
||||
const owner = service();
|
||||
expect(await owner.tick()).toMatchObject({ deleted: 1, failed: 0 });
|
||||
expect(s3.versions).toEqual([expect.objectContaining({ key: neighbor })]);
|
||||
const versionDeletes = s3.requests.filter(value => value.method === "DELETE" && value.versionId !== null);
|
||||
expect(versionDeletes).toHaveLength(5);
|
||||
expect(versionDeletes.every(value => f.keys.some(key => value.key === `scoped-prefix/${key}`))).toBe(true);
|
||||
expect(s3.requests.some(value => value.keyMarker && value.versionMarker)).toBe(true);
|
||||
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]!.status).toBe("deleted");
|
||||
} finally { await s3.close(); }
|
||||
});
|
||||
it.each(["missingCursor", "repeatCursor"] as const)("does not settle S3 cleanup with an invalid %s listing", async fault => {
|
||||
const s3 = await versionedS3Fixture();
|
||||
try {
|
||||
storage = createStorageService(s3.provider);
|
||||
const f = await seed();
|
||||
for (let index = 0; index < 5; index++) s3.add(`scoped-prefix/${f.keys[0]}`);
|
||||
s3.faults[fault] = true;
|
||||
expect(await service().tick()).toMatchObject({ deleted: 0, failed: 1 });
|
||||
expect(s3.requests.filter(value => value.method === "DELETE")).toHaveLength(0);
|
||||
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0])
|
||||
.toMatchObject({ status: "delete_requested", lastError: "snapshot_cleanup_failed" });
|
||||
} finally { await s3.close(); }
|
||||
});
|
||||
it.each(["denied", "listDenied", "preserveVersions"] as const)("keeps retryable S3 cleanup for %s and settles after verified recovery", async fault => {
|
||||
const s3 = await versionedS3Fixture();
|
||||
try {
|
||||
storage = createStorageService(s3.provider);
|
||||
const f = await seed();
|
||||
s3.faults[fault] = true;
|
||||
let at = now;
|
||||
const owner = remoteWorkspaceSnapshotRetentionService(db, { storage, now: () => at });
|
||||
expect(await owner.tick()).toMatchObject({ deleted: 0, failed: 1 });
|
||||
expect(s3.versions.length).toBeGreaterThan(0);
|
||||
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0])
|
||||
.toMatchObject({ status: "delete_requested", lastError: "snapshot_cleanup_failed" });
|
||||
s3.faults[fault] = false;
|
||||
at = new Date(now.getTime() + 16 * 60_000);
|
||||
expect(await owner.tick()).toMatchObject({ deleted: 1, failed: 0 });
|
||||
expect(s3.versions).toEqual([]);
|
||||
} finally { await s3.close(); }
|
||||
});
|
||||
it("cancels an in-flight S3 version listing and leaves its retention obligation intact", async () => {
|
||||
const s3 = await versionedS3Fixture();
|
||||
try {
|
||||
storage = createStorageService(s3.provider);
|
||||
const f = await seed();
|
||||
const preAborted = new AbortController(); preAborted.abort();
|
||||
const before = s3.requests.length;
|
||||
await expect(storage.deleteObject(f.companyId, f.keys[0]!, { allVersions: true, signal: preAborted.signal }))
|
||||
.rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(s3.requests).toHaveLength(before);
|
||||
await expect(storage.deleteObject(randomUUID(), f.keys[0]!, { allVersions: true })).rejects.toMatchObject({ status: 403 });
|
||||
expect(s3.requests).toHaveLength(before);
|
||||
|
||||
s3.faults.hangList = true;
|
||||
const controller = new AbortController();
|
||||
const running = service().tick(controller.signal);
|
||||
const observed = running.catch(error => error);
|
||||
await vi.waitFor(() => expect(s3.requests.some(value => value.method === "GET")).toBe(true));
|
||||
controller.abort();
|
||||
expect(await observed).toMatchObject({ name: "AbortError" });
|
||||
await vi.waitFor(() => expect(s3.faults.abortedLists).toBe(1));
|
||||
expect(s3.requests.filter(value => value.method === "DELETE")).toHaveLength(0);
|
||||
expect((await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id)))[0]!.status)
|
||||
.toBe("delete_requested");
|
||||
} finally { await s3.close(); }
|
||||
});
|
||||
it("pins the current snapshot and leaves no idle deadline", async () => {
|
||||
const f = await seed("committed");
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { snapshotId: f.row.id } } }).where(eq(executionWorkspaces.id, f.workspaceId));
|
||||
const owner = service(); expect(await owner.tick()).toMatchObject({ pinned: 1, deleted: 0 });
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(true);
|
||||
});
|
||||
it("pins active restore operations and releases an already-due generation on settlement", async () => {
|
||||
const f = await seed();
|
||||
const [op] = await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId,
|
||||
phase: "remote_workspace_restore", status: "running", metadata: { snapshotId: f.row.id } }).returning();
|
||||
const owner = service(); expect(await owner.tick()).toMatchObject({ pinned: 1 }); expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
await db.update(workspaceOperations).set({ status: "succeeded" }).where(eq(workspaceOperations.id, op!.id));
|
||||
expect(await owner.tick()).toMatchObject({ deleted: 1 });
|
||||
});
|
||||
it("leaves healthy current versions outside later unrelated maintenance passes", async () => {
|
||||
const current = await seed("committed");
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { snapshotId: current.row.id } } })
|
||||
.where(eq(executionWorkspaces.id, current.workspaceId));
|
||||
const owner = service();
|
||||
expect(await owner.tick()).toMatchObject({ checked: 1, pinned: 1, deleted: 0 });
|
||||
const obsolete = await seed();
|
||||
expect(await owner.tick()).toMatchObject({ checked: 1, pinned: 0, deleted: 1 });
|
||||
expect((await storage.headObject(current.companyId, current.keys[0]!)).exists).toBe(true);
|
||||
expect((await storage.headObject(obsolete.companyId, obsolete.keys[0]!)).exists).toBe(false);
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
});
|
||||
it("revisits a parent when cleanup of an unfinished child releases its last pin", async () => {
|
||||
const parent = await seed("superseded", "10000000-0000-4000-8000-000000000001");
|
||||
await db.insert(executionWorkspaceSnapshots).values({ id: "20000000-0000-4000-8000-000000000001",
|
||||
companyId: parent.companyId, executionWorkspaceId: parent.workspaceId, sourceOperationId: randomUUID(),
|
||||
parentSnapshotId: parent.row.id, generation: 2, epoch: 1, status: "pending",
|
||||
deleteAfter: new Date(now.getTime() - 1000), createdAt: new Date(now.getTime() - 10 * 86400_000) });
|
||||
const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
|
||||
const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
|
||||
try {
|
||||
await vi.waitFor(async () => expect((await storage.headObject(parent.companyId, parent.keys[0]!)).exists).toBe(false),
|
||||
{ timeout: 4000, interval: 40 });
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
} finally { await worker.stop(); await coordinator.stop(); }
|
||||
});
|
||||
it("pins an unfinished upload and an active recovery hold", async () => {
|
||||
const f = await seed("pending");
|
||||
await db.insert(workspaceOperations).values({ id: f.operationId, companyId: f.companyId, executionWorkspaceId: f.workspaceId,
|
||||
phase: "remote_workspace_snapshot", status: "running" });
|
||||
const owner = service(); expect(await owner.tick()).toMatchObject({ pinned: 1 });
|
||||
await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.id, f.operationId));
|
||||
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog",
|
||||
ownerType: "board", cause: "workspace_recovery", fingerprint: randomUUID(), nextAction: "Recover saved files" });
|
||||
expect(await owner.tick()).toMatchObject({ pinned: 1, deleted: 0 }); expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
});
|
||||
it("rejects a foreign upload key without deleting another company's object", async () => {
|
||||
const f = await seed(); const foreignKey = `${randomUUID()}/workspace-snapshots/foreign`;
|
||||
await db.update(executionWorkspaceSnapshots).set({ metadata: { uploads: [{ objectKey: foreignKey }] } }).where(eq(executionWorkspaceSnapshots.id, f.row.id));
|
||||
expect(await service().tick()).toMatchObject({ failed: 1, deleted: 0 });
|
||||
for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(true);
|
||||
expect(await service().nextDeadlineAt()).toBeGreaterThan(now.getTime());
|
||||
});
|
||||
it("settles without recurring reads and wakes on the first real producer", async () => {
|
||||
const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
|
||||
const run = vi.spyOn(owner, "tick"); const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
|
||||
try {
|
||||
await vi.waitFor(() => expect(run).toHaveBeenCalledOnce());
|
||||
await new Promise(resolve => setTimeout(resolve, 30)); expect(run).toHaveBeenCalledOnce();
|
||||
const f = await seed();
|
||||
await db.transaction(async tx => { await notifyRemoteWorkspaceSnapshots(tx, `workspace:${f.workspaceId}`); });
|
||||
await vi.waitFor(async () => expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false));
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
} finally { await worker.stop(); await coordinator.stop(); }
|
||||
});
|
||||
it("finishes bounded pages and revisits a lower UUID produced during a pass", async () => {
|
||||
const later = await seed("superseded", "20000000-0000-4000-8000-000000000001");
|
||||
const last = await seed("superseded", "30000000-0000-4000-8000-000000000001");
|
||||
let earlier: Awaited<ReturnType<typeof seed>> | undefined;
|
||||
const remove = storage.deleteObject.bind(storage); let inserted = false;
|
||||
storage.deleteObject = async (...args) => {
|
||||
if (!inserted) {
|
||||
inserted = true; earlier = await seed("superseded", "10000000-0000-4000-8000-000000000001");
|
||||
await db.transaction(async tx => { await notifyRemoteWorkspaceSnapshots(tx, `workspace:${earlier!.workspaceId}`); });
|
||||
}
|
||||
await remove(...args);
|
||||
};
|
||||
const owner = remoteWorkspaceSnapshotRetentionService(db, { storage, now: () => now, batchSize: 1 });
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
|
||||
const run = vi.spyOn(owner, "tick"), worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
|
||||
try {
|
||||
await vi.waitFor(async () => {
|
||||
expect(earlier).toBeDefined();
|
||||
for (const fixture of [earlier!, later, last]) for (const key of fixture.keys) {
|
||||
expect((await storage.headObject(fixture.companyId, key)).exists).toBe(false);
|
||||
}
|
||||
}, { timeout: 8000, interval: 40 });
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
const completed = run.mock.calls.length; await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(run).toHaveBeenCalledTimes(completed);
|
||||
} finally { await worker.stop(); await coordinator.stop(); }
|
||||
});
|
||||
it("becomes quiet while recovery pins a snapshot and wakes on the real recovery resolution", async () => {
|
||||
const f = await seed();
|
||||
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "active_run_watchdog",
|
||||
ownerType: "board", cause: "workspace_recovery", fingerprint: randomUUID(), nextAction: "Recover saved files" });
|
||||
const owner = service(), coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
|
||||
const worker = registerRemoteWorkspaceSnapshotRetentionWork(coordinator, owner, () => {});
|
||||
try {
|
||||
await worker.ready;
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(true);
|
||||
const resolved = await issueRecoveryActionService(db).resolveActiveForIssue({ companyId: f.companyId, sourceIssueId: f.issueId,
|
||||
status: "resolved", outcome: "restored", resolutionNote: "Verified saved files" });
|
||||
expect(resolved?.status).toBe("resolved");
|
||||
await vi.waitFor(async () => expect((await storage.headObject(f.companyId, f.keys[0]!)).exists).toBe(false));
|
||||
expect(await owner.nextDeadlineAt()).toBeNull();
|
||||
} finally { await worker.stop(); await coordinator.stop(); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
import { and, asc, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from "drizzle-orm";
|
||||
import { executionWorkspaceSnapshots, executionWorkspaces, heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, type EnvironmentLease } from "@paperclipai/shared";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
|
||||
export const REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS = 7 * 86400_000;
|
||||
export const REMOTE_WORKSPACE_SNAPSHOT_RETRY_MS = 15 * 60_000;
|
||||
type Snapshot = typeof executionWorkspaceSnapshots.$inferSelect;
|
||||
type Tx = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
type ReadDb = Pick<Db, "select"> | Pick<Tx, "select">;
|
||||
const date = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? new Date(value) : null;
|
||||
|
||||
export function notifyRemoteWorkspaceSnapshots(transaction: object, hint?: string) {
|
||||
return notifyDeliveryWork(transaction, DELIVERY_QUEUES.remoteWorkspaceSnapshots, hint);
|
||||
}
|
||||
|
||||
async function pinned(db: ReadDb, row: Snapshot) {
|
||||
const [workspace] = await db.select({ id: executionWorkspaces.id, sourceIssueId: executionWorkspaces.sourceIssueId,
|
||||
metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, row.companyId), eq(executionWorkspaces.id, row.executionWorkspaceId),
|
||||
));
|
||||
if ((workspace?.metadata?.remoteWorkspace as { snapshotId?: unknown } | undefined)?.snapshotId === row.id) return true;
|
||||
const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, row.companyId), eq(workspaceOperations.executionWorkspaceId, row.executionWorkspaceId),
|
||||
or(inArray(workspaceOperations.status, ["running", "pending"]),
|
||||
sql`(${workspaceOperations.metadata}->'saveWork'->>'schema' = 'paperclip.remote-workspace-save-work.v1'
|
||||
and (${workspaceOperations.metadata}#>'{saveWork,unknown}' is distinct from 'false'::jsonb
|
||||
or case when jsonb_typeof(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 'array'
|
||||
then jsonb_array_length(${workspaceOperations.metadata}#>'{saveWork,activeIds}') <> 0 else true end))`),
|
||||
sql`(${workspaceOperations.id} = ${row.sourceOperationId}::uuid
|
||||
or ${workspaceOperations.metadata}->>'snapshotId' = ${row.id}
|
||||
or ${workspaceOperations.metadata}->>'parentSnapshotId' = ${row.id})`,
|
||||
)).limit(1);
|
||||
if (operation) return true;
|
||||
const [child] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, row.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, row.executionWorkspaceId),
|
||||
eq(executionWorkspaceSnapshots.parentSnapshotId, row.id), inArray(executionWorkspaceSnapshots.status, ["pending", "uploaded"]),
|
||||
)).limit(1);
|
||||
if (child) return true;
|
||||
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, row.companyId),
|
||||
inArray(heartbeatRuns.status, ["queued", "running"]),
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'workspaceId' = ${row.executionWorkspaceId}`,
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'snapshotId' = ${row.id}`,
|
||||
)).limit(1);
|
||||
if (run) return true;
|
||||
const [recovery] = await db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, row.companyId), inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
sql`(${workspace?.sourceIssueId ? sql`${issueRecoveryActions.sourceIssueId} = ${workspace.sourceIssueId}::uuid` : sql`false`}
|
||||
or ${issueRecoveryActions.evidence}->>'snapshotId' = ${row.id}
|
||||
or ${issueRecoveryActions.evidence}->>'workspaceId' = ${row.executionWorkspaceId}
|
||||
or ${issueRecoveryActions.evidence}->'remoteWorkspace'->>'snapshotId' = ${row.id})`,
|
||||
)).limit(1);
|
||||
return Boolean(recovery);
|
||||
}
|
||||
|
||||
/** Producers wake this owner; pinned history carries no speculative retry. */
|
||||
export function remoteWorkspaceSnapshotRetentionService(db: Db, options: {
|
||||
storage: StorageService; now?: () => Date; batchSize?: number;
|
||||
}) {
|
||||
const now = options.now ?? (() => new Date()), batch = options.batchSize ?? 50;
|
||||
let after: string | null = null;
|
||||
let scanning = false;
|
||||
let rescanRequested = false;
|
||||
async function tick(signal?: AbortSignal) {
|
||||
scanning = true;
|
||||
try {
|
||||
const at = now(), result = { checked: 0, pinned: 0, scheduled: 0, deleted: 0, failed: 0, truncated: false };
|
||||
const rows = await db.select().from(executionWorkspaceSnapshots).where(and(
|
||||
sql`${executionWorkspaceSnapshots.status} <> 'deleted'`,
|
||||
// Ordinary execution notifications must not turn each healthy current
|
||||
// version into a per-workspace maintenance read. A version with a stale
|
||||
// deadline still needs one pass to clear it; orphaned versions need grace.
|
||||
or(ne(executionWorkspaceSnapshots.status, "committed"), isNotNull(executionWorkspaceSnapshots.deleteAfter),
|
||||
sql`not exists (select 1 from ${executionWorkspaces} w
|
||||
where w.company_id = ${executionWorkspaceSnapshots.companyId}
|
||||
and w.id = ${executionWorkspaceSnapshots.executionWorkspaceId}
|
||||
and w.metadata->'remoteWorkspace'->>'snapshotId' = ${executionWorkspaceSnapshots.id}::text)`),
|
||||
or(isNull(executionWorkspaceSnapshots.deleteAfter),
|
||||
lte(executionWorkspaceSnapshots.deleteAfter, at),
|
||||
eq(executionWorkspaceSnapshots.status, "committed")),
|
||||
after ? sql`${executionWorkspaceSnapshots.id} > ${after}::uuid` : undefined,
|
||||
)).orderBy(asc(executionWorkspaceSnapshots.id)).limit(batch + 1);
|
||||
result.truncated = rows.length > batch;
|
||||
for (const candidate of rows.slice(0, batch)) {
|
||||
signal?.throwIfAborted(); result.checked++; after = candidate.id;
|
||||
try {
|
||||
const claimed = await db.transaction(async tx => {
|
||||
const [row] = await tx.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, candidate.id)).for("update");
|
||||
if (!row || row.status === "deleted") return null;
|
||||
if (await pinned(tx, row)) {
|
||||
result.pinned++;
|
||||
const metadata = { ...row.metadata };
|
||||
if (row.status === "superseded" && row.deleteAfter) metadata.retentionDueAt = row.deleteAfter.toISOString();
|
||||
if (row.status === "committed") delete metadata.retentionDueAt;
|
||||
if (row.deleteAfter) await tx.update(executionWorkspaceSnapshots).set({ deleteAfter: null, metadata }).where(eq(executionWorkspaceSnapshots.id, row.id));
|
||||
return null;
|
||||
}
|
||||
const deadline = row.deleteAfter ?? (row.status === "superseded" ? date(row.metadata?.retentionDueAt) : null)
|
||||
?? new Date((row.status === "pending" || row.status === "uploaded" ? row.createdAt.getTime() : at.getTime()) + REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS);
|
||||
if (row.status !== "delete_requested" && deadline > at) {
|
||||
result.scheduled++;
|
||||
await tx.update(executionWorkspaceSnapshots).set({ deleteAfter: deadline,
|
||||
...(row.status === "committed" ? { status: "superseded" as const } : {}) }).where(eq(executionWorkspaceSnapshots.id, row.id));
|
||||
return null;
|
||||
}
|
||||
if (row.status === "delete_requested" && row.deleteAfter && row.deleteAfter > at) return null;
|
||||
const [saved] = await tx.update(executionWorkspaceSnapshots).set({ status: "delete_requested", deleteAfter: at, lastError: null })
|
||||
.where(eq(executionWorkspaceSnapshots.id, row.id)).returning();
|
||||
return saved!;
|
||||
});
|
||||
if (!claimed) continue;
|
||||
// New readers cannot adopt a delete_requested record. Recheck all
|
||||
// durable references immediately before any irreversible object I/O.
|
||||
if (await pinned(db, claimed)) {
|
||||
await db.update(executionWorkspaceSnapshots).set({ deleteAfter: null }).where(eq(executionWorkspaceSnapshots.id, claimed.id));
|
||||
result.pinned++; continue;
|
||||
}
|
||||
const uploads = Array.isArray(claimed.metadata?.uploads) ? claimed.metadata.uploads : [];
|
||||
const keys = new Set<string>();
|
||||
for (const value of uploads) {
|
||||
if (!value || typeof value !== "object" || typeof (value as { objectKey?: unknown }).objectKey !== "string") throw new Error("snapshot_upload_reference_invalid");
|
||||
keys.add((value as { objectKey: string }).objectKey);
|
||||
}
|
||||
if (claimed.objectKey) keys.add(claimed.objectKey);
|
||||
const prefix = `${claimed.companyId}/workspace-snapshots/${claimed.executionWorkspaceId}/${claimed.id}/`;
|
||||
for (const key of keys) {
|
||||
if (!key.startsWith(prefix) || key.includes("..") || key.includes("\\")) throw new Error("snapshot_upload_scope_invalid");
|
||||
}
|
||||
for (const key of keys) {
|
||||
signal?.throwIfAborted();
|
||||
await options.storage.deleteObject(claimed.companyId, key, { signal, allVersions: true });
|
||||
if ((await options.storage.headObject(claimed.companyId, key, { signal })).exists) throw new Error("snapshot_object_still_present");
|
||||
}
|
||||
await db.update(executionWorkspaceSnapshots).set({ status: "deleted", deleteAfter: null, lastError: null, updatedAt: at })
|
||||
.where(and(eq(executionWorkspaceSnapshots.id, claimed.id), eq(executionWorkspaceSnapshots.status, "delete_requested")));
|
||||
// A parent inspected earlier in this page can now lose its last pin.
|
||||
// Revisit it once instead of waiting for an unrelated execution event.
|
||||
if (claimed.parentSnapshotId) rescanRequested = true;
|
||||
result.deleted++;
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
result.failed++;
|
||||
await db.update(executionWorkspaceSnapshots).set({ lastError: "snapshot_cleanup_failed",
|
||||
deleteAfter: new Date(at.getTime() + REMOTE_WORKSPACE_SNAPSHOT_RETRY_MS) }).where(eq(executionWorkspaceSnapshots.id, candidate.id));
|
||||
}
|
||||
}
|
||||
if (!result.truncated) {
|
||||
after = null;
|
||||
// Finish the current ordered scan before revisiting a producer that may
|
||||
// have inserted a lower UUID. Repeated hints cannot starve later pages.
|
||||
if (rescanRequested) { rescanRequested = false; result.truncated = true; }
|
||||
}
|
||||
return result;
|
||||
} finally { scanning = false; }
|
||||
}
|
||||
async function nextDeadlineAt() {
|
||||
const [row] = await db.select({ at: sql<Date | null>`min(${executionWorkspaceSnapshots.deleteAfter})` }).from(executionWorkspaceSnapshots)
|
||||
.where(sql`${executionWorkspaceSnapshots.status} <> 'deleted'`);
|
||||
return row?.at ? new Date(row.at).getTime() : null;
|
||||
}
|
||||
return { tick, nextDeadlineAt, resetCursor: () => { if (scanning || after !== null) rescanRequested = true; } };
|
||||
}
|
||||
|
||||
export function registerRemoteWorkspaceSnapshotRetentionWork(
|
||||
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, service: ReturnType<typeof remoteWorkspaceSnapshotRetentionService>,
|
||||
onResult: (result: Awaited<ReturnType<typeof service.tick>>) => void,
|
||||
) {
|
||||
let next: number | null = null;
|
||||
return coordinator.register(DELIVERY_QUEUES.remoteWorkspaceSnapshots, {
|
||||
retryMs: REMOTE_WORKSPACE_SNAPSHOT_RETRY_MS, pausePassiveMaintenance: true,
|
||||
onSettled: () => service.resetCursor(),
|
||||
run: async signal => { next = null; const result = await service.tick(signal); onResult(result);
|
||||
next = result.truncated ? Date.now() + 1000 : await service.nextDeadlineAt(); },
|
||||
hasPending: async () => false, nextRunAt: () => next,
|
||||
});
|
||||
}
|
||||
|
||||
/** Called in the lease-release transaction; an absent PID/timeout grants no authority. */
|
||||
export async function clearRemoteWorkspaceLocationForReleasedLease(input: { db: Db | Tx; lease: EnvironmentLease }) {
|
||||
if (!hasRemoteTerminationReceipt(input.lease)
|
||||
|| (input.lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined)?.state !== "destroyed") return;
|
||||
const workspaces = await input.db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.lease.companyId),
|
||||
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->'location'->>'leaseId' = ${input.lease.id}`,
|
||||
)).for("update");
|
||||
for (const workspace of workspaces) {
|
||||
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!authority?.location || authority.dirty || !authority.snapshotId || authority.writerRunId
|
||||
|| authority.location.providerLeaseId !== input.lease.providerLeaseId || authority.location.environmentId !== input.lease.environmentId) continue;
|
||||
const [saved] = await input.db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
|
||||
eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.status, "committed")));
|
||||
if (!saved) continue;
|
||||
await notifyRemoteWorkspaceSnapshots(input.db, `workspace:${workspace.id}`);
|
||||
await input.db.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspace: { ...authority, location: null } }, updatedAt: new Date() })
|
||||
.where(and(eq(executionWorkspaces.id, workspace.id),
|
||||
sql`${executionWorkspaces.metadata}->'remoteWorkspace' = ${JSON.stringify(authority)}::jsonb`));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { applyPendingMigrations, closeRegisteredClients, createDb, executionWorkspaceSnapshots } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js";
|
||||
import { createStorageService } from "../storage/service.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
|
||||
import { beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot } from "./remote-workspace-snapshot-storage.js";
|
||||
|
||||
const externalDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL;
|
||||
if (externalDatabaseUrl && !/^remote_workspace_[a-z_]*test$/.test(new URL(externalDatabaseUrl).pathname.slice(1))) {
|
||||
throw new Error("Workspace storage tests require a disposable remote_workspace_*test database");
|
||||
}
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const suite = support.supported ? describe : describe.skip;
|
||||
if (!support.supported) console.warn(`Remote workspace snapshot DB tests unavailable: ${support.reason}`);
|
||||
suite("remote workspace snapshot storage", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: ReturnType<typeof createDb>;
|
||||
let root: string, storage: StorageService;
|
||||
beforeAll(async () => {
|
||||
if (externalDatabaseUrl) { await applyPendingMigrations(externalDatabaseUrl); db = createDb(externalDatabaseUrl); }
|
||||
else { database = await startEmbeddedPostgresTestDatabase("workspace-snapshot-"); db = createDb(database.connectionString); }
|
||||
}, 90_000);
|
||||
afterAll(async () => { if (externalDatabaseUrl) await closeRegisteredClients(externalDatabaseUrl); else await database?.cleanup(); });
|
||||
beforeEach(async () => {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "41".repeat(32));
|
||||
root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-store-test-"));
|
||||
storage = createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")));
|
||||
});
|
||||
afterEach(async () => { vi.unstubAllEnvs(); await db.delete(executionWorkspaceSnapshots); await fs.rm(root, { recursive: true, force: true }); });
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), workspaceId = randomUUID(), operationId = randomUUID();
|
||||
const cwd = path.join(root, "source"), archivePath = path.join(root, "archive.pcws"); await fs.mkdir(cwd);
|
||||
await fs.writeFile(path.join(cwd, "saved.txt"), "acknowledged workspace content\n");
|
||||
const captured = await captureRemoteWorkspaceDirectory({ cwd, archivePath });
|
||||
const row = await beginWorkspaceSnapshot(db, { companyId, workspaceId, operationId, generation: 1, epoch: 3 });
|
||||
const upload = { db, storage, snapshotId: row.id, companyId, workspaceId, archivePath, manifestDigest: captured.manifestDigest };
|
||||
return { companyId, workspaceId, operationId, cwd, archivePath, captured, row, upload };
|
||||
}
|
||||
it("streams encrypted saved data through the real ledger and filesystem object backend", async () => {
|
||||
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
|
||||
expect(ref.byteSize).toBeGreaterThan(f.captured.byteSize);
|
||||
const object = await storage.getObject(f.companyId, ref.objectKey); const parts: Buffer[] = [];
|
||||
for await (const part of object.stream) parts.push(part);
|
||||
expect(Buffer.concat(parts).includes(Buffer.from("acknowledged workspace content"))).toBe(false);
|
||||
await fs.rm(f.cwd, { recursive: true }); await fs.rm(f.archivePath);
|
||||
const restoredArchive = path.join(root, "restored.pcws"), targetDir = path.join(root, "restored");
|
||||
await loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath: restoredArchive });
|
||||
await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir, expectedManifestDigest: ref.manifestDigest });
|
||||
expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("acknowledged workspace content\n");
|
||||
const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, ref.id));
|
||||
expect(row?.status).toBe("uploaded"); // The owning service still controls atomic publication.
|
||||
});
|
||||
it("reconciles an upload whose reply was lost after object persistence, without the old scratch file", async () => {
|
||||
const f = await fixture(); const put = storage.putFile.bind(storage); let loseReply = true;
|
||||
storage.putFile = async input => { const result = await put(input); if (loseReply) { loseReply = false; throw new Error("lost upload reply"); } return result; };
|
||||
await expect(uploadWorkspaceSnapshot(f.upload)).rejects.toThrow("lost upload reply");
|
||||
await fs.rm(f.archivePath);
|
||||
const ref = await uploadWorkspaceSnapshot(f.upload);
|
||||
expect(ref.manifestDigest).toBe(f.captured.manifestDigest);
|
||||
expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: ref.id })).toEqual(ref);
|
||||
const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, ref.id));
|
||||
expect((row?.metadata?.uploads as unknown[]).length).toBe(1);
|
||||
});
|
||||
it("retains an unsuccessful operation and rejects a content-changing retry", async () => {
|
||||
const f = await fixture(); storage.putFile = async () => { throw new Error("storage unavailable"); };
|
||||
await expect(uploadWorkspaceSnapshot(f.upload)).rejects.toThrow("storage unavailable");
|
||||
const [row] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, f.row.id));
|
||||
expect(row?.status).toBe("pending"); expect((row?.metadata?.uploads as unknown[]).length).toBe(1);
|
||||
await expect(uploadWorkspaceSnapshot({ ...f.upload, manifestDigest: "b".repeat(64) })).rejects.toThrow("operation_content_changed");
|
||||
});
|
||||
it("keeps operation identity and company/workspace scope exact", async () => {
|
||||
const f = await fixture();
|
||||
expect((await beginWorkspaceSnapshot(db, { ...f, generation: 1, epoch: 3 })).id).toBe(f.row.id);
|
||||
await expect(beginWorkspaceSnapshot(db, { ...f, generation: 1, epoch: 4 })).rejects.toThrow("operation_binding_changed");
|
||||
await uploadWorkspaceSnapshot(f.upload);
|
||||
expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: f.row.id, companyId: randomUUID() })).toBeNull();
|
||||
expect(await getWorkspaceSnapshot(db, { ...f, snapshotId: f.row.id, workspaceId: randomUUID() })).toBeNull();
|
||||
});
|
||||
it("rejects object corruption and leaves the requested plaintext path unpublished", async () => {
|
||||
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
|
||||
const objectPath = path.join(root, "objects", ref.objectKey), object = await fs.open(objectPath, "r+");
|
||||
try { await object.write(Buffer.from("X"), 0, 1, 15); } finally { await object.close(); }
|
||||
const targetPath = path.join(root, "corrupted.pcws");
|
||||
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toThrow("object_hash_mismatch");
|
||||
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("rejects a foreign wrapped key even when ciphertext metadata is unchanged", async () => {
|
||||
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
|
||||
const foreignId = randomUUID();
|
||||
await expect(loadWorkspaceSnapshotArchive({ storage, reference: { ...ref, id: foreignId, objectKey: ref.objectKey.replace(ref.id, foreignId) },
|
||||
targetPath: path.join(root, "foreign.pcws") })).rejects.toThrow("key_binding_invalid");
|
||||
});
|
||||
it("does not overwrite an existing caller file", async () => {
|
||||
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload), targetPath = path.join(root, "existing.pcws");
|
||||
await fs.writeFile(targetPath, "existing");
|
||||
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toMatchObject({ code: "EEXIST" });
|
||||
expect(await fs.readFile(targetPath, "utf8")).toBe("existing");
|
||||
});
|
||||
it("does not start object retrieval for an already canceled snapshot load", async () => {
|
||||
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
|
||||
const getObject = vi.spyOn(storage, "getObject");
|
||||
const controller = new AbortController(), reason = new Error("request already canceled");
|
||||
controller.abort(reason);
|
||||
const targetPath = path.join(root, "canceled.pcws");
|
||||
const error = await loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath, signal: controller.signal }).catch(error => error);
|
||||
expect(getObject).not.toHaveBeenCalled();
|
||||
expect(error).toBe(reason);
|
||||
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
it("cancels an active object body and removes private decryption scratch before rejecting", async () => {
|
||||
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
|
||||
const ciphertext = await fs.readFile(path.join(root, "objects", ref.objectKey));
|
||||
const controller = new AbortController();
|
||||
let produced = 0, closed = false;
|
||||
const body = new Readable({
|
||||
highWaterMark: 1,
|
||||
read() {
|
||||
if (produced === 2) { controller.abort(new Error("client disconnected")); return; }
|
||||
this.push(ciphertext.subarray(produced, ++produced));
|
||||
},
|
||||
destroy(error, callback) { closed = true; callback(error); },
|
||||
});
|
||||
storage.getObject = vi.fn(async (_companyId, _objectKey, options) => {
|
||||
expect(options?.signal).toBe(controller.signal);
|
||||
return { stream: body, contentLength: ciphertext.length };
|
||||
});
|
||||
const scratchRoot = path.join(root, "private-scratch"); await fs.mkdir(scratchRoot);
|
||||
const temp = vi.spyOn(os, "tmpdir").mockReturnValue(scratchRoot);
|
||||
const targetPath = path.join(root, "interrupted.pcws");
|
||||
try {
|
||||
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath, signal: controller.signal }))
|
||||
.rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(closed).toBe(true); expect(body.destroyed).toBe(true);
|
||||
expect(produced).toBeGreaterThan(0); expect(produced).toBeLessThan(ciphertext.length);
|
||||
const atRejection = produced; await new Promise(resolve => setImmediate(resolve)); expect(produced).toBe(atRejection);
|
||||
expect(await fs.readdir(scratchRoot)).toEqual([]);
|
||||
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally { temp.mockRestore(); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,219 @@
|
||||
import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { link, mkdtemp, open, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Transform } from "node:stream";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { executionWorkspaceSnapshots, type Db } from "@paperclipai/db";
|
||||
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
|
||||
import type { StoredSecretVersionMaterial } from "../secrets/types.js";
|
||||
import type { StorageService } from "../storage/types.js";
|
||||
import { notifyRemoteWorkspaceSnapshots, REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS } from "./remote-workspace-snapshot-retention.js";
|
||||
|
||||
const SCHEMA = "paperclip.workspace-snapshot.v1";
|
||||
const KEY_SCHEMA = "paperclip.workspace-snapshot-key.v1";
|
||||
const MAGIC = Buffer.from("PCWSE1\n");
|
||||
const FORMAT = "paperclip.remote-workspace-archive.v1";
|
||||
type SnapshotRow = typeof executionWorkspaceSnapshots.$inferSelect;
|
||||
export interface WorkspaceSnapshotRef {
|
||||
schema: typeof SCHEMA;
|
||||
id: string;
|
||||
companyId: string;
|
||||
workspaceId: string;
|
||||
generation: number;
|
||||
objectKey: string;
|
||||
sha256: string;
|
||||
byteSize: number;
|
||||
key: StoredSecretVersionMaterial;
|
||||
manifestDigest: string;
|
||||
}
|
||||
type UploadAttempt = Omit<WorkspaceSnapshotRef, "schema" | "companyId" | "workspaceId" | "generation" | "id">;
|
||||
function failure(reason: string): never { throw new Error(`remote_workspace_snapshot_${reason}`); }
|
||||
const aad = (ref: Pick<WorkspaceSnapshotRef, "companyId" | "workspaceId" | "id" | "generation">) =>
|
||||
Buffer.from(JSON.stringify([SCHEMA, ref.companyId, ref.workspaceId, ref.id, ref.generation]));
|
||||
const scope = (input: { companyId: string; workspaceId: string; snapshotId: string }) => and(
|
||||
eq(executionWorkspaceSnapshots.id, input.snapshotId), eq(executionWorkspaceSnapshots.companyId, input.companyId),
|
||||
eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspaceId),
|
||||
);
|
||||
function reference(row: SnapshotRow): WorkspaceSnapshotRef {
|
||||
if (!row.objectKey || !row.sha256 || !row.byteSize || !row.key || !row.manifestDigest) failure("not_uploaded");
|
||||
const value = { schema: SCHEMA, id: row.id, companyId: row.companyId, workspaceId: row.executionWorkspaceId,
|
||||
generation: row.generation, objectKey: row.objectKey!, sha256: row.sha256!, byteSize: row.byteSize!, key: row.key!, manifestDigest: row.manifestDigest! } as WorkspaceSnapshotRef;
|
||||
validateReference(value);
|
||||
return value;
|
||||
}
|
||||
function validateReference(ref: WorkspaceSnapshotRef) {
|
||||
if (ref.schema !== SCHEMA || !Number.isSafeInteger(ref.generation) || ref.generation < 1
|
||||
|| !ref.objectKey.startsWith(`${ref.companyId}/workspace-snapshots/${ref.workspaceId}/${ref.id}/`)
|
||||
|| !/^[a-f0-9]{64}$/.test(ref.sha256) || !/^[a-f0-9]{64}$/.test(ref.manifestDigest)
|
||||
|| !Number.isSafeInteger(ref.byteSize) || ref.byteSize < MAGIC.length + 12 + 16
|
||||
|| !ref.key || typeof ref.key !== "object") failure("reference_invalid");
|
||||
}
|
||||
async function digestFile(file: string, signal?: AbortSignal) {
|
||||
const digest = createHash("sha256"); let byteSize = 0;
|
||||
for await (const part of createReadStream(file, { signal })) { digest.update(part); byteSize += part.length; }
|
||||
return { sha256: digest.digest("hex"), byteSize };
|
||||
}
|
||||
async function verifyObject(storage: StorageService, ref: WorkspaceSnapshotRef, signal?: AbortSignal) {
|
||||
const object = await storage.getObject(ref.companyId, ref.objectKey, { signal });
|
||||
const digest = createHash("sha256"); let size = 0;
|
||||
try {
|
||||
for await (const chunk of object.stream) {
|
||||
signal?.throwIfAborted(); size += chunk.length;
|
||||
if (size > ref.byteSize) failure("object_size_mismatch");
|
||||
digest.update(chunk);
|
||||
}
|
||||
if (size !== ref.byteSize || digest.digest("hex") !== ref.sha256) failure("object_hash_mismatch");
|
||||
} finally { object.stream.destroy(); }
|
||||
}
|
||||
|
||||
export async function beginWorkspaceSnapshot(db: Db, input: {
|
||||
companyId: string; workspaceId: string; operationId: string; runId?: string | null;
|
||||
parentSnapshotId?: string | null; generation: number; epoch: number;
|
||||
}) {
|
||||
if (!Number.isSafeInteger(input.generation) || input.generation < 1 || !Number.isSafeInteger(input.epoch) || input.epoch < 0) failure("generation_invalid");
|
||||
await db.transaction(async tx => {
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${input.workspaceId}`);
|
||||
await tx.insert(executionWorkspaceSnapshots).values({ companyId: input.companyId, executionWorkspaceId: input.workspaceId,
|
||||
sourceOperationId: input.operationId, sourceRunId: input.runId ?? null, parentSnapshotId: input.parentSnapshotId ?? null,
|
||||
generation: input.generation, epoch: input.epoch, format: FORMAT,
|
||||
deleteAfter: new Date(Date.now() + REMOTE_WORKSPACE_SNAPSHOT_GRACE_MS) }).onConflictDoNothing();
|
||||
});
|
||||
const [row] = await db.select().from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, input.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspaceId),
|
||||
eq(executionWorkspaceSnapshots.sourceOperationId, input.operationId),
|
||||
));
|
||||
if (!row || row.generation !== input.generation || row.epoch !== input.epoch || row.parentSnapshotId !== (input.parentSnapshotId ?? null)
|
||||
|| row.sourceRunId !== (input.runId ?? null)) failure("operation_binding_changed");
|
||||
return row!;
|
||||
}
|
||||
|
||||
export async function getWorkspaceSnapshot(db: Db, input: { companyId: string; workspaceId: string; snapshotId: string }) {
|
||||
const [row] = await db.select().from(executionWorkspaceSnapshots).where(scope(input));
|
||||
return row && ["uploaded", "committed", "superseded"].includes(row.status) ? reference(row) : null;
|
||||
}
|
||||
|
||||
/** Upload attempts are journaled before remote I/O. The caller owns publication. */
|
||||
export async function uploadWorkspaceSnapshot(input: {
|
||||
db: Db; storage: StorageService; snapshotId: string; companyId: string; workspaceId: string;
|
||||
archivePath: string; manifestDigest: string; signal?: AbortSignal;
|
||||
}): Promise<WorkspaceSnapshotRef> {
|
||||
const [row] = await input.db.select().from(executionWorkspaceSnapshots).where(scope(input));
|
||||
if (!row) failure("missing");
|
||||
if (["uploaded", "committed", "superseded"].includes(row!.status)) {
|
||||
const saved = reference(row!);
|
||||
if (saved.manifestDigest !== input.manifestDigest) failure("operation_content_changed");
|
||||
await verifyObject(input.storage, saved, input.signal); return saved;
|
||||
}
|
||||
if (row!.status !== "pending" || !/^[a-f0-9]{64}$/.test(input.manifestDigest)) failure("not_writable");
|
||||
const identity = { id: row!.id, companyId: row!.companyId, workspaceId: row!.executionWorkspaceId, generation: row!.generation };
|
||||
const publish = async (attempt: UploadAttempt) => {
|
||||
const updated = await input.db.transaction(async tx => {
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${input.workspaceId}`);
|
||||
return (await tx.update(executionWorkspaceSnapshots).set({ ...attempt, status: "uploaded", updatedAt: new Date() })
|
||||
.where(and(scope(input), eq(executionWorkspaceSnapshots.status, "pending"))).returning())[0];
|
||||
});
|
||||
const winning = updated ?? (await input.db.select().from(executionWorkspaceSnapshots).where(scope(input)))[0];
|
||||
if (!winning || !["uploaded", "committed", "superseded"].includes(winning.status)) failure("publication_changed");
|
||||
const saved = reference(winning!);
|
||||
if (saved.manifestDigest !== input.manifestDigest) failure("operation_content_changed");
|
||||
return saved;
|
||||
};
|
||||
// Recover a completed upload even when the previous process lost its reply
|
||||
// and its scratch directory. Failed attempts remain referenced for cleanup.
|
||||
const attempts = Array.isArray(row!.metadata?.uploads) ? row!.metadata!.uploads as UploadAttempt[] : [];
|
||||
for (const attempt of attempts) {
|
||||
if (attempt.manifestDigest !== input.manifestDigest) failure("operation_content_changed");
|
||||
const ref = { schema: SCHEMA, ...identity, ...attempt } as WorkspaceSnapshotRef;
|
||||
validateReference(ref);
|
||||
if ((await input.storage.headObject(ref.companyId, ref.objectKey, { signal: input.signal })).exists) {
|
||||
await verifyObject(input.storage, ref, input.signal); return publish(attempt);
|
||||
}
|
||||
}
|
||||
const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-encrypt-"));
|
||||
try {
|
||||
const encrypted = path.join(scratch, "archive.encrypted"), dataKey = randomBytes(32), iv = randomBytes(12);
|
||||
const cipher = createCipheriv("aes-256-gcm", dataKey, iv); cipher.setAAD(aad(identity));
|
||||
const plaintext = await digestFile(input.archivePath, input.signal);
|
||||
const { material } = await localEncryptedProvider.createVersion({ value: JSON.stringify({ schema: KEY_SCHEMA,
|
||||
...identity, dataKey: dataKey.toString("base64"), plaintextSha256: plaintext.sha256, plaintextBytes: plaintext.byteSize }) });
|
||||
const header = await open(encrypted, "wx", 0o600);
|
||||
try { await header.writeFile(Buffer.concat([MAGIC, iv])); } finally { await header.close(); }
|
||||
const incomingDigest = createHash("sha256"); let incomingBytes = 0;
|
||||
const observed = new Transform({ transform(bytes, _encoding, callback) { incomingDigest.update(bytes); incomingBytes += bytes.length; callback(null, bytes); } });
|
||||
await pipeline(createReadStream(input.archivePath, { signal: input.signal }), observed, cipher, createWriteStream(encrypted, { flags: "a", mode: 0o600 }), { signal: input.signal });
|
||||
if (incomingBytes !== plaintext.byteSize || incomingDigest.digest("hex") !== plaintext.sha256) failure("archive_changed");
|
||||
const footer = await open(encrypted, "a"); try { await footer.writeFile(cipher.getAuthTag()); } finally { await footer.close(); }
|
||||
dataKey.fill(0);
|
||||
const content = await digestFile(encrypted, input.signal);
|
||||
const attempt: UploadAttempt = { objectKey: `${identity.companyId}/workspace-snapshots/${identity.workspaceId}/${identity.id}/${randomUUID()}.pcws`,
|
||||
...content, key: material, manifestDigest: input.manifestDigest };
|
||||
const recorded = await input.db.transaction(async tx => {
|
||||
const [current] = await tx.select().from(executionWorkspaceSnapshots).where(scope(input)).for("update");
|
||||
if (!current || current.status !== "pending") return false;
|
||||
const previous = Array.isArray(current.metadata?.uploads) ? current.metadata.uploads : [];
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${input.workspaceId}`);
|
||||
await tx.update(executionWorkspaceSnapshots).set({ metadata: { ...current.metadata, uploads: [...previous, attempt] }, updatedAt: new Date() }).where(scope(input));
|
||||
return true;
|
||||
});
|
||||
if (!recorded) {
|
||||
const saved = await getWorkspaceSnapshot(input.db, input);
|
||||
if (!saved || saved.manifestDigest !== input.manifestDigest) failure("publication_changed");
|
||||
await verifyObject(input.storage, saved, input.signal); return saved;
|
||||
}
|
||||
// The object key is unique per attempt, so a late losing uploader cannot
|
||||
// overwrite an object already used by a committed generation.
|
||||
await input.storage.putFile({ companyId: identity.companyId, namespace: "workspace-snapshots", objectKey: attempt.objectKey,
|
||||
originalFilename: null, contentType: "application/octet-stream", body: createReadStream(encrypted),
|
||||
byteSize: attempt.byteSize, sha256: attempt.sha256, signal: input.signal });
|
||||
await verifyObject(input.storage, { schema: SCHEMA, ...identity, ...attempt } as WorkspaceSnapshotRef, input.signal);
|
||||
return await publish(attempt);
|
||||
} finally { await rm(scratch, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
/** Authentication finishes in private scratch before any plaintext is published. */
|
||||
export async function loadWorkspaceSnapshotArchive(input: {
|
||||
storage: StorageService; reference: WorkspaceSnapshotRef; targetPath: string; signal?: AbortSignal; temporaryParent?: string;
|
||||
}): Promise<void> {
|
||||
input.signal?.throwIfAborted();
|
||||
const ref = input.reference; validateReference(ref);
|
||||
const wrapped = JSON.parse(await localEncryptedProvider.resolveVersion({ material: ref.key, externalRef: null }));
|
||||
if (wrapped.schema !== KEY_SCHEMA || wrapped.companyId !== ref.companyId || wrapped.workspaceId !== ref.workspaceId
|
||||
|| wrapped.id !== ref.id || wrapped.generation !== ref.generation || !/^[a-f0-9]{64}$/.test(wrapped.plaintextSha256 ?? "")
|
||||
|| !Number.isSafeInteger(wrapped.plaintextBytes) || wrapped.plaintextBytes < 0) failure("key_binding_invalid");
|
||||
const scratch = await mkdtemp(path.join(input.temporaryParent ?? os.tmpdir(), "pc-workspace-decrypt-"));
|
||||
try {
|
||||
input.signal?.throwIfAborted();
|
||||
const ciphertext = path.join(scratch, "ciphertext"), plaintext = path.join(scratch, "plaintext");
|
||||
const object = await input.storage.getObject(ref.companyId, ref.objectKey, { signal: input.signal });
|
||||
let received = 0; const digest = createHash("sha256");
|
||||
const bounded = new Transform({ transform(bytes, _encoding, callback) {
|
||||
received += bytes.length; digest.update(bytes);
|
||||
callback(received > ref.byteSize ? new Error("remote_workspace_snapshot_object_size_mismatch") : null, bytes);
|
||||
} });
|
||||
try { await pipeline(object.stream, bounded, createWriteStream(ciphertext, { flags: "wx", mode: 0o600 }), { signal: input.signal }); }
|
||||
finally { object.stream.destroy(); }
|
||||
if (received !== ref.byteSize || digest.digest("hex") !== ref.sha256) failure("object_hash_mismatch");
|
||||
const file = await open(ciphertext, "r"), header = Buffer.alloc(MAGIC.length + 12), tag = Buffer.alloc(16);
|
||||
try { await file.read(header, 0, header.length, 0); await file.read(tag, 0, tag.length, ref.byteSize - tag.length); } finally { await file.close(); }
|
||||
if (!header.subarray(0, MAGIC.length).equals(MAGIC)) failure("ciphertext_header_invalid");
|
||||
const key = Buffer.from(wrapped.dataKey, "base64"); if (key.length !== 32) failure("key_invalid");
|
||||
const decipher = createDecipheriv("aes-256-gcm", key, header.subarray(MAGIC.length));
|
||||
decipher.setAAD(aad(ref)); decipher.setAuthTag(tag);
|
||||
try {
|
||||
await pipeline(createReadStream(ciphertext, { start: header.length, end: ref.byteSize - tag.length - 1, signal: input.signal }),
|
||||
decipher, createWriteStream(plaintext, { flags: "wx", mode: 0o600 }), { signal: input.signal });
|
||||
} finally { key.fill(0); }
|
||||
const actual = await digestFile(plaintext, input.signal);
|
||||
if (actual.sha256 !== wrapped.plaintextSha256 || actual.byteSize !== wrapped.plaintextBytes) failure("plaintext_hash_mismatch");
|
||||
// Publish a complete file without replacing an existing destination. A
|
||||
// failed/cancelled relay cannot leave a plaintext cache that looks complete.
|
||||
const staged = path.join(path.dirname(input.targetPath), `.workspace-snapshot-${randomUUID()}.tmp`);
|
||||
try {
|
||||
await pipeline(createReadStream(plaintext, { signal: input.signal }), createWriteStream(staged, { flags: "wx", mode: 0o600 }), { signal: input.signal });
|
||||
input.signal?.throwIfAborted(); await link(staged, input.targetPath);
|
||||
} finally { await rm(staged, { force: true }); }
|
||||
} finally { await rm(scratch, { recursive: true, force: true }); }
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
import {
|
||||
captureRemoteWorkspaceArchive,
|
||||
cleanupRemoteWorkspaceArchive,
|
||||
prepareRemoteRepository,
|
||||
prepareRemoteWorkspaceDirectory,
|
||||
restoreRemoteWorkspaceDirectory,
|
||||
type RemoteWorkspaceTarget,
|
||||
} from "./remote-workspace-transport.js";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
|
||||
describe("remote workspace transport safety with real sandbox commands", () => {
|
||||
let scratch: string;
|
||||
let workspace: string;
|
||||
beforeEach(async () => {
|
||||
scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "remote-workspace-transport-")));
|
||||
workspace = path.join(scratch, "workspace");
|
||||
await fs.mkdir(workspace);
|
||||
});
|
||||
afterEach(async () => { await fs.rm(scratch, { recursive: true, force: true }); });
|
||||
|
||||
async function git(cwd: string, ...args: string[]) {
|
||||
return exec("git", ["-C", cwd, "-c", "user.name=Synthetic fixture", "-c", "user.email=fixture@example.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", ...args], {
|
||||
env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1", GIT_TERMINAL_PROMPT: "0" },
|
||||
});
|
||||
}
|
||||
|
||||
async function repository(options: { filter?: boolean; revisions?: boolean } = {}) {
|
||||
const root = path.join(scratch, "origin");
|
||||
await fs.mkdir(root);
|
||||
await git(root, "init", "--initial-branch=main");
|
||||
if (options.filter) await fs.writeFile(path.join(root, ".gitattributes"), "*.txt filter=synthetic-probe\n");
|
||||
await fs.writeFile(path.join(root, "source.txt"), "initial synthetic source\n");
|
||||
await git(root, "add", ".");
|
||||
await git(root, "commit", "-m", "Initial synthetic source");
|
||||
if (options.revisions) {
|
||||
await fs.writeFile(path.join(root, "source.txt"), "later synthetic source\n");
|
||||
await git(root, "commit", "-am", "Later synthetic source");
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function target(ambientEnv: Record<string, string> = {}) {
|
||||
const execute = vi.fn<CommandManagedRuntimeRunner["execute"]>(async input => {
|
||||
try {
|
||||
const result = await exec(input.command, input.args ?? [], {
|
||||
cwd: input.cwd ?? workspace, timeout: input.timeoutMs, maxBuffer: 4 * 1024 * 1024,
|
||||
// Carry only generated fixture values across this execution boundary.
|
||||
env: { PATH: process.env.PATH, ...ambientEnv, ...input.env },
|
||||
});
|
||||
return { ...result, exitCode: 0, timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() };
|
||||
} catch (error) {
|
||||
const failure = error as { code?: number; stdout?: string; stderr?: string; signal?: string };
|
||||
return { exitCode: typeof failure.code === "number" ? failure.code : 1, stdout: failure.stdout ?? "",
|
||||
stderr: failure.stderr ?? "", timedOut: false, signal: failure.signal ?? null,
|
||||
pid: process.pid, startedAt: new Date().toISOString() };
|
||||
}
|
||||
});
|
||||
const transfer: NonNullable<CommandManagedRuntimeRunner["syncOut"]> = async operations => {
|
||||
const results = [];
|
||||
for (const operation of operations) {
|
||||
let bytesTransferred = 0;
|
||||
for (const file of operation.files) {
|
||||
await fs.mkdir(path.dirname(file.targetPath), { recursive: true });
|
||||
await fs.cp(file.sourcePath, file.targetPath, { recursive: file.kind === "directory" });
|
||||
if (file.kind === "file") bytesTransferred += (await fs.stat(file.targetPath)).size;
|
||||
}
|
||||
results.push({ operationId: operation.operationId, filesTransferred: operation.files.length, bytesTransferred });
|
||||
}
|
||||
return { operations: results };
|
||||
};
|
||||
const runner: CommandManagedRuntimeRunner = { execute, syncOut: transfer, syncIn: transfer };
|
||||
return { remote: { kind: "remote", transport: "sandbox", remoteCwd: workspace, runner } as RemoteWorkspaceTarget, execute };
|
||||
}
|
||||
|
||||
it("rejects a repository ancestor symlink before writing outside the workspace", async () => {
|
||||
const origin = await repository();
|
||||
const outside = path.join(scratch, "outside");
|
||||
await fs.mkdir(outside);
|
||||
await fs.writeFile(path.join(outside, "sentinel"), "untouched\n");
|
||||
await fs.symlink(outside, path.join(workspace, "repos"));
|
||||
const { remote } = target();
|
||||
await expect(prepareRemoteRepository({ target: remote, remoteCwd: path.join(workspace, "repos", "clone"),
|
||||
repoUrl: origin, ref: "HEAD", resolveGitAuth: async () => null })).rejects.toThrow();
|
||||
expect(await fs.readdir(outside)).toEqual(["sentinel"]);
|
||||
expect(await fs.readFile(path.join(outside, "sentinel"), "utf8")).toBe("untouched\n");
|
||||
});
|
||||
|
||||
it.each(["existing", "create"] as const)("selects the %s remote branch without confusing its tip with the base ref", async branchMode => {
|
||||
const origin = await repository();
|
||||
await git(origin, "checkout", "-b", "feature/diverged");
|
||||
await fs.writeFile(path.join(origin, "source.txt"), "existing feature work\n");
|
||||
await git(origin, "commit", "-am", "Existing feature work");
|
||||
const featureTip = (await git(origin, "rev-parse", "HEAD")).stdout.trim();
|
||||
await git(origin, "checkout", "main");
|
||||
await fs.writeFile(path.join(origin, "source.txt"), "new main work\n");
|
||||
await git(origin, "commit", "-am", "New main work");
|
||||
const mainTip = (await git(origin, "rev-parse", "HEAD")).stdout.trim();
|
||||
expect(featureTip).not.toBe(mainTip);
|
||||
const { remote } = target(), root = path.join(workspace, "selected-branch");
|
||||
await prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root,
|
||||
preparationIdentity: createHash("sha256").update(`synthetic-${branchMode}-branch`).digest("hex"),
|
||||
source: { kind: "git", repoUrl: origin, ref: "main", branchName: "feature/diverged", branchMode },
|
||||
resolveGitAuth: async () => null });
|
||||
expect((await git(root, "rev-parse", "HEAD")).stdout.trim()).toBe(branchMode === "existing" ? featureTip : mainTip);
|
||||
expect((await git(root, "symbolic-ref", "--short", "HEAD")).stdout.trim()).toBe("feature/diverged");
|
||||
expect((await git(origin, "rev-parse", "refs/heads/feature/diverged")).stdout.trim()).toBe(featureTip);
|
||||
});
|
||||
|
||||
it("rejects a missing existing remote branch without publishing a task folder", async () => {
|
||||
const origin = await repository(), { remote } = target(), root = path.join(workspace, "missing-branch");
|
||||
await expect(prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root,
|
||||
preparationIdentity: createHash("sha256").update("synthetic-missing-existing-branch").digest("hex"),
|
||||
source: { kind: "git", repoUrl: origin, ref: "main", branchName: "feature/missing", branchMode: "existing" },
|
||||
resolveGitAuth: async () => null })).rejects.toThrow();
|
||||
expect(await fs.lstat(root).catch(() => null)).toBeNull();
|
||||
});
|
||||
|
||||
it("adopts an atomic restored folder after a lost reply without overwriting later edits", async () => {
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "saved synthetic files\n");
|
||||
const { remote, execute } = target();
|
||||
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() });
|
||||
try {
|
||||
const root = path.join(scratch, "restored");
|
||||
const original = execute.getMockImplementation()!;
|
||||
let lost = false;
|
||||
execute.mockImplementation(async input => {
|
||||
const result = await original(input);
|
||||
if (!lost && input.args?.[0] === "-e" && input.args[1]?.includes("workspace-restored.json") && result.exitCode === 0) {
|
||||
lost = true; throw new Error("synthetic lost restore reply");
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath,
|
||||
manifestDigest: captured.manifestDigest, restorationIdentity: "a".repeat(64) };
|
||||
await expect(restoreRemoteWorkspaceDirectory(input)).rejects.toThrow("synthetic lost restore reply");
|
||||
await fs.writeFile(path.join(root, "newer.txt"), "retain later edits\n");
|
||||
const restored = await restoreRemoteWorkspaceDirectory(input);
|
||||
expect(restored.cleanupPending).toBe(true);
|
||||
await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot, operationId: restored.operationId });
|
||||
expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("saved synthetic files\n");
|
||||
expect(await fs.readFile(path.join(root, "newer.txt"), "utf8")).toBe("retain later edits\n");
|
||||
expect(await fs.readdir(path.join(scratch, ".paperclip-runtime", "workspace-snapshots"))).toEqual([]);
|
||||
await expect(restoreRemoteWorkspaceDirectory({ ...input, restorationIdentity: "b".repeat(64) })).rejects.toThrow();
|
||||
expect(await fs.readFile(path.join(root, "newer.txt"), "utf8")).toBe("retain later edits\n");
|
||||
} finally { await captured.cleanup(); }
|
||||
});
|
||||
|
||||
it("keeps a completed restore available when the staging cleanup reply is lost", async () => {
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "durable synthetic files\n");
|
||||
const { remote, execute } = target();
|
||||
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() });
|
||||
try {
|
||||
const actual = execute.getMockImplementation()!;
|
||||
let loseCleanup = true;
|
||||
execute.mockImplementation(async input => {
|
||||
const result = await actual(input);
|
||||
if (loseCleanup && input.args?.[1]?.includes("remote_workspace_export_cleanup_failed") && result.exitCode === 0) {
|
||||
throw new Error("synthetic lost cleanup reply");
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const root = path.join(scratch, "restored-after-cleanup-failure");
|
||||
const input = { target: remote, remoteCwd: root, archivePath: captured.archivePath,
|
||||
manifestDigest: captured.manifestDigest, restorationIdentity: "c".repeat(64) };
|
||||
const restored = await restoreRemoteWorkspaceDirectory(input);
|
||||
expect(restored.cleanupPending).toBe(true);
|
||||
expect(await fs.readFile(path.join(root, "saved.txt"), "utf8")).toBe("durable synthetic files\n");
|
||||
await expect(cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot,
|
||||
operationId: restored.operationId })).rejects.toThrow("synthetic lost cleanup reply");
|
||||
await fs.writeFile(path.join(root, "later.txt"), "keep later work\n");
|
||||
loseCleanup = false;
|
||||
const retried = await restoreRemoteWorkspaceDirectory(input);
|
||||
await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: retried.stagingRoot, operationId: retried.operationId });
|
||||
expect(await fs.readFile(path.join(root, "later.txt"), "utf8")).toBe("keep later work\n");
|
||||
} finally { await captured.cleanup(); }
|
||||
});
|
||||
|
||||
it("publishes one complete capture pair when the same operation starts simultaneously", async () => {
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "simultaneously captured work\n");
|
||||
const barrier = path.join(scratch, "capture-start-barrier"); await fs.mkdir(barrier);
|
||||
const { remote, execute } = target(), run = execute.getMockImplementation()!;
|
||||
execute.mockImplementation(async input => {
|
||||
const args = input.args ?? [];
|
||||
if (args[4]?.endsWith("archive.cjs") && args[5]) {
|
||||
// Hold the actual codec subprocesses until both captures have started.
|
||||
// The codec and all archive writes still run unchanged on real files.
|
||||
const body = Buffer.from(args[5], "base64").toString("utf8");
|
||||
const controlled = `const barrierFs=require('node:fs/promises');(async()=>{
|
||||
const barrier=${JSON.stringify(barrier)};await barrierFs.writeFile(barrier+'/'+process.pid,'ready');
|
||||
const until=Date.now()+10000;while((await barrierFs.readdir(barrier)).length<2){
|
||||
if(Date.now()>until)throw Error('capture barrier timed out');await new Promise(resolve=>setTimeout(resolve,5));}
|
||||
${body}
|
||||
})().catch(error=>{process.stderr.write(error.message);process.exitCode=1});`;
|
||||
return run({ ...input, args: [...args.slice(0, 5), Buffer.from(controlled).toString("base64")] });
|
||||
}
|
||||
return run(input);
|
||||
});
|
||||
const operationId = randomUUID();
|
||||
const outcomes = await Promise.allSettled([0, 1].map(() => captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })));
|
||||
const completed = outcomes.flatMap(result => result.status === "fulfilled" ? [result.value] : []);
|
||||
try {
|
||||
expect(outcomes.map(result => result.status)).toEqual(["fulfilled", "fulfilled"]);
|
||||
expect(await fs.readdir(barrier)).toHaveLength(2);
|
||||
expect(completed[0]!.manifestDigest).toBe(completed[1]!.manifestDigest);
|
||||
expect(completed[0]!.sha256).toBe(completed[1]!.sha256);
|
||||
for (const saved of completed) expect(createHash("sha256").update(await fs.readFile(saved.archivePath)).digest("hex")).toBe(saved.sha256);
|
||||
const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
expect((await fs.readdir(directory)).filter(name => name.startsWith("capture-"))).toEqual([]);
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "later work must not replace a completed capture\n");
|
||||
const replay = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try { expect(replay.sha256).toBe(completed[0]!.sha256); }
|
||||
finally { await replay.cleanup(); }
|
||||
} finally { await Promise.all(completed.map(saved => saved.cleanup())); }
|
||||
}, 15_000);
|
||||
|
||||
it("recaptures after an abandoned partial attempt without unlinking unknown staging", async () => {
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "work after interrupted capture\n");
|
||||
const operationId = randomUUID(), directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
const abandoned = path.join(directory, "capture-abandoned"); await fs.mkdir(abandoned, { recursive: true });
|
||||
const legacy = path.join(directory, "workspace.pcws"), partial = path.join(abandoned, "workspace.pcws");
|
||||
await fs.writeFile(legacy, "unknown old in-progress archive\n"); await fs.writeFile(partial, "abandoned partial archive\n");
|
||||
const { remote } = target();
|
||||
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try {
|
||||
expect(await fs.readFile(legacy, "utf8")).toBe("unknown old in-progress archive\n");
|
||||
expect(await fs.readFile(partial, "utf8")).toBe("abandoned partial archive\n");
|
||||
expect((await fs.readdir(directory)).filter(name => name.startsWith("capture-"))).toEqual(["capture-abandoned"]);
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "newer work\n");
|
||||
const replay = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try { expect(replay.sha256).toBe(captured.sha256); }
|
||||
finally { await replay.cleanup(); }
|
||||
} finally { await captured.cleanup(); }
|
||||
});
|
||||
|
||||
it("prefers the canonical capture and reads a completed legacy pair without replacing it", async () => {
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "immutable completed capture\n");
|
||||
const operationId = randomUUID(), directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
const { remote } = target();
|
||||
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try {
|
||||
const canonical = path.join(directory, "captured"), archive = path.join(directory, "workspace.pcws"), receipt = path.join(directory, "receipt.json");
|
||||
const legacyReceipt = { ...JSON.parse(await fs.readFile(path.join(canonical, "receipt.json"), "utf8")),
|
||||
layout: "untrusted-layout", archivePath: path.join(scratch, "untrusted-source") };
|
||||
await fs.writeFile(receipt, JSON.stringify(legacyReceipt));
|
||||
await fs.writeFile(archive, "corrupt legacy bytes ignored when canonical exists\n");
|
||||
const canonicalReplay = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try { expect(canonicalReplay.sha256).toBe(captured.sha256); }
|
||||
finally { await canonicalReplay.cleanup(); }
|
||||
await fs.copyFile(path.join(canonical, "workspace.pcws"), archive);
|
||||
await fs.rm(canonical, { recursive: true });
|
||||
await fs.writeFile(path.join(workspace, "saved.txt"), "later uncaptured work\n");
|
||||
const fallback = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try { expect(fallback.sha256).toBe(captured.sha256); }
|
||||
finally { await fallback.cleanup(); }
|
||||
expect(await fs.readFile(receipt, "utf8")).toBe(JSON.stringify(legacyReceipt));
|
||||
expect(createHash("sha256").update(await fs.readFile(archive)).digest("hex")).toBe(captured.sha256);
|
||||
expect(await fs.lstat(canonical).catch(() => null)).toBeNull();
|
||||
await fs.writeFile(receipt, "invalid completed receipt\n");
|
||||
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId }))
|
||||
.rejects.toThrow("remote_workspace_operation_failed");
|
||||
expect(await fs.readFile(receipt, "utf8")).toBe("invalid completed receipt\n");
|
||||
expect(createHash("sha256").update(await fs.readFile(archive)).digest("hex")).toBe(captured.sha256);
|
||||
expect(await fs.lstat(canonical).catch(() => null)).toBeNull();
|
||||
} finally { await captured.cleanup(); }
|
||||
});
|
||||
|
||||
it("rejects a preexisting canonical capture symlink without touching its target", async () => {
|
||||
const operationId = randomUUID(), directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
const outside = path.join(scratch, "outside-capture"); await fs.mkdir(outside); await fs.mkdir(directory, { recursive: true });
|
||||
await fs.writeFile(path.join(outside, "workspace.pcws"), "outside archive sentinel\n");
|
||||
await fs.symlink(outside, path.join(directory, "captured"));
|
||||
const { remote } = target();
|
||||
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId }))
|
||||
.rejects.toThrow("remote_workspace_operation_failed");
|
||||
expect(await fs.readFile(path.join(outside, "workspace.pcws"), "utf8")).toBe("outside archive sentinel\n");
|
||||
expect(await fs.readdir(outside)).toEqual(["workspace.pcws"]);
|
||||
});
|
||||
|
||||
it("preserves only the fixed unfinished Git operation diagnostic across capture commands", async () => {
|
||||
await git(workspace, "init", "--initial-branch=main");
|
||||
await fs.writeFile(path.join(workspace, "work.txt"), "base\n"); await git(workspace, "add", "."); await git(workspace, "commit", "-m", "base");
|
||||
await git(workspace, "checkout", "-b", "feature");
|
||||
await fs.writeFile(path.join(workspace, "work.txt"), "feature\n"); await git(workspace, "commit", "-am", "feature");
|
||||
await git(workspace, "checkout", "main");
|
||||
await fs.writeFile(path.join(workspace, "work.txt"), "main\n"); await git(workspace, "commit", "-am", "main");
|
||||
await expect(git(workspace, "merge", "--no-edit", "feature")).rejects.toThrow();
|
||||
const { remote, execute } = target();
|
||||
const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace";
|
||||
const operationId = randomUUID();
|
||||
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId })).rejects.toThrow(gitOperation);
|
||||
const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
for (const name of ["workspace.pcws", "receipt.json"]) await expect(fs.access(path.join(directory, name))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const failure = { exitCode: 1, stdout: "", stderr: `${gitOperation}\nSYNTHETIC_PRIVATE_PATH_OR_CREDENTIAL`,
|
||||
timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() };
|
||||
execute.mockResolvedValueOnce(failure);
|
||||
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }))
|
||||
.rejects.toThrow(/^remote_workspace_operation_failed$/);
|
||||
execute.mockResolvedValueOnce({ ...failure, timedOut: true, stderr: gitOperation });
|
||||
await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() }))
|
||||
.rejects.toThrow(/^remote_workspace_operation_failed$/);
|
||||
});
|
||||
|
||||
it("never follows a preexisting snapshot receipt temporary symlink", async () => {
|
||||
await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n");
|
||||
const operationId = randomUUID();
|
||||
const directory = path.join(workspace, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
const outside = path.join(scratch, "outside-sentinel");
|
||||
await fs.mkdir(directory, { recursive: true });
|
||||
await fs.writeFile(outside, "untouched\n");
|
||||
const oldTemporaryPath = path.join(directory, "receipt.json.tmp");
|
||||
await fs.symlink(outside, oldTemporaryPath);
|
||||
const { remote } = target();
|
||||
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId });
|
||||
try {
|
||||
expect(captured.sha256).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(await fs.readFile(outside, "utf8")).toBe("untouched\n");
|
||||
expect((await fs.lstat(oldTemporaryPath)).isSymbolicLink()).toBe(true);
|
||||
} finally { await captured.cleanup(); }
|
||||
});
|
||||
|
||||
it("does not run ambient checkout filters with the Git transport credential", async () => {
|
||||
const origin = await repository({ filter: true, revisions: true });
|
||||
const globalConfig = path.join(scratch, "synthetic-gitconfig");
|
||||
const sentinel = path.join(scratch, "filter-received-credential");
|
||||
await git(origin, "config", "--file", globalConfig, "filter.synthetic-probe.smudge",
|
||||
`sh -c 'printf %s "$PAPERCLIP_GIT_TOKEN" > "${sentinel}"; cat'`);
|
||||
const calibration = path.join(scratch, "filter-calibration");
|
||||
await git(scratch, "clone", "--no-hardlinks", "--", origin, calibration);
|
||||
await exec("git", ["-C", calibration, "checkout", "--detach", "HEAD~1"], {
|
||||
env: { PATH: process.env.PATH, GIT_CONFIG_GLOBAL: globalConfig, GIT_CONFIG_NOSYSTEM: "1", PAPERCLIP_GIT_TOKEN: "SYNTHETIC_ONLY" },
|
||||
});
|
||||
expect(await fs.readFile(sentinel, "utf8")).toBe("SYNTHETIC_ONLY");
|
||||
await fs.rm(sentinel);
|
||||
const { remote } = target({ GIT_CONFIG_GLOBAL: globalConfig });
|
||||
// Inject a fake credential for a local synthetic source so the real Git
|
||||
// subprocess boundary is exercised without contacting a credentialed host.
|
||||
await prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: path.join(workspace, "clone"),
|
||||
preparationIdentity: createHash("sha256").update("synthetic-filter-preparation").digest("hex"),
|
||||
source: { kind: "git", repoUrl: origin, ref: "HEAD~1" },
|
||||
resolveGitAuth: async () => ({ configArgs: [], env: { PAPERCLIP_GIT_TOKEN: "SYNTHETIC_ONLY" },
|
||||
source: "company_secret", secretName: "SYNTHETIC_FIXTURE" }) });
|
||||
expect(await fs.readFile(path.join(workspace, "clone", "source.txt"), "utf8")).toBe("initial synthetic source\n");
|
||||
expect(await fs.lstat(sentinel).catch(() => null)).toBeNull();
|
||||
});
|
||||
|
||||
it("adopts only the bound completed preparation after its reply is lost", async () => {
|
||||
const origin = await repository();
|
||||
const { remote, execute } = target();
|
||||
const actualExecute = execute.getMockImplementation()!;
|
||||
let loseReply = true;
|
||||
execute.mockImplementation(async input => {
|
||||
const result = await actualExecute(input);
|
||||
if (loseReply && result.exitCode === 0 && !JSON.parse(input.args?.[3] ?? "{}").probeOnly) {
|
||||
loseReply = false; throw new Error("synthetic lost preparation reply");
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const preparedRoot = path.join(workspace, "clone");
|
||||
const input = { target: remote, remoteCwd: preparedRoot,
|
||||
preparationIdentity: createHash("sha256").update(JSON.stringify({ run: "synthetic", workspace: "synthetic", epoch: 1, lease: "synthetic" })).digest("hex"),
|
||||
source: { kind: "git" as const, repoUrl: origin, ref: "unused-base-ref", branchName: "main", branchMode: "existing" as const },
|
||||
resolveGitAuth: async () => null };
|
||||
await expect(prepareRemoteWorkspaceDirectory(input)).rejects.toThrow("synthetic lost preparation reply");
|
||||
await fs.writeFile(path.join(preparedRoot, "in-flight.txt"), "preserve this unsaved work\n");
|
||||
// A second clone is now impossible; successful retry must use its durable
|
||||
// completed-preparation receipt and retain the existing folder.
|
||||
await fs.rm(origin, { recursive: true, force: true });
|
||||
const unavailableAuth = vi.fn(async () => { throw new Error("synthetic Git credential service unavailable"); });
|
||||
await expect(prepareRemoteWorkspaceDirectory({ ...input, resolveGitAuth: unavailableAuth })).resolves.toBeUndefined();
|
||||
expect(unavailableAuth).not.toHaveBeenCalled();
|
||||
expect(await fs.readFile(path.join(preparedRoot, "source.txt"), "utf8")).toBe("initial synthetic source\n");
|
||||
expect(await fs.readFile(path.join(preparedRoot, "in-flight.txt"), "utf8")).toBe("preserve this unsaved work\n");
|
||||
await expect(prepareRemoteWorkspaceDirectory({ ...input, preparationIdentity: "f".repeat(64), resolveGitAuth: unavailableAuth })).rejects.toThrow("remote_workspace_operation_failed");
|
||||
expect(unavailableAuth).not.toHaveBeenCalled();
|
||||
expect(await fs.readFile(path.join(preparedRoot, "in-flight.txt"), "utf8")).toBe("preserve this unsaved work\n");
|
||||
});
|
||||
|
||||
it("verifies a saved repository without resolving unavailable transport credentials", async () => {
|
||||
const origin = await repository();
|
||||
const { remote } = target();
|
||||
const root = path.join(workspace, "saved-repository");
|
||||
const prepared = await prepareRemoteRepository({ target: remote, remoteCwd: root,
|
||||
repoUrl: origin, ref: "HEAD", resolveGitAuth: async () => null });
|
||||
await fs.writeFile(path.join(root, "unsaved.txt"), "preserve local work\n");
|
||||
await fs.rm(origin, { recursive: true, force: true });
|
||||
const resolveGitAuth = vi.fn(async () => { throw new Error("synthetic Git credential service unavailable"); });
|
||||
const input = { target: remote, remoteCwd: root, repoUrl: origin, ref: "HEAD",
|
||||
pinnedCommit: prepared.pinnedCommit, verifyOnly: true, resolveGitAuth };
|
||||
await expect(prepareRemoteRepository(input)).resolves.toEqual(prepared);
|
||||
await expect(prepareRemoteRepository({ ...input, pinnedCommit: "0".repeat(40) })).rejects.toThrow("remote_workspace_operation_failed");
|
||||
await expect(prepareRemoteRepository({ ...input, repoUrl: `${origin}-foreign` })).rejects.toThrow("remote_workspace_operation_failed");
|
||||
expect(resolveGitAuth).not.toHaveBeenCalled();
|
||||
expect(await fs.readFile(path.join(root, "unsaved.txt"), "utf8")).toBe("preserve local work\n");
|
||||
});
|
||||
|
||||
it.each(["path", "file-url"] as const)("verifies an archived secondary %s repository after its local origin disappears", async sourceKind => {
|
||||
const origin = await repository(), { remote } = target();
|
||||
const repoUrl = sourceKind === "path" ? origin : pathToFileURL(origin).href;
|
||||
const root = path.join(workspace, "secondary");
|
||||
const prepared = await prepareRemoteRepository({ target: remote, remoteCwd: root, repoUrl,
|
||||
ref: "HEAD", resolveGitAuth: async () => null });
|
||||
await fs.writeFile(path.join(root, "source.txt"), "staged imported work\n");
|
||||
await git(root, "add", "source.txt");
|
||||
await fs.writeFile(path.join(root, "source.txt"), "unstaged imported work\n");
|
||||
await fs.writeFile(path.join(root, "untracked.txt"), "untracked imported work\n");
|
||||
const status = (await git(root, "status", "--porcelain=v2")).stdout;
|
||||
const captured = await captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() });
|
||||
try {
|
||||
await fs.rm(root, { recursive: true });
|
||||
await fs.rm(origin, { recursive: true });
|
||||
const restoredRoot = path.join(scratch, "saved-root");
|
||||
const restored = await restoreRemoteWorkspaceDirectory({ target: remote, remoteCwd: restoredRoot,
|
||||
archivePath: captured.archivePath, manifestDigest: captured.manifestDigest, restorationIdentity: "c".repeat(64) });
|
||||
await cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: restored.stagingRoot, operationId: restored.operationId });
|
||||
const secondary = path.join(restoredRoot, "secondary");
|
||||
expect((await git(secondary, "remote")).stdout.trim()).toBe("");
|
||||
const resolveGitAuth = vi.fn(async () => { throw new Error("original Git source is unavailable"); });
|
||||
const input = { target: remote, remoteCwd: secondary, repoUrl, ref: "HEAD",
|
||||
pinnedCommit: prepared.pinnedCommit, savedArchive: true, resolveGitAuth };
|
||||
await expect(prepareRemoteRepository(input)).resolves.toEqual({ ...prepared, cwd: secondary });
|
||||
expect((await git(secondary, "status", "--porcelain=v2")).stdout).toBe(status);
|
||||
expect(await fs.readFile(path.join(secondary, "source.txt"), "utf8")).toBe("unstaged imported work\n");
|
||||
await expect(prepareRemoteRepository({ ...input, savedArchive: false })).rejects.toThrow("remote_workspace_operation_failed");
|
||||
await expect(prepareRemoteRepository({ ...input, pinnedCommit: "0".repeat(40) })).rejects.toThrow("remote_workspace_operation_failed");
|
||||
const missing = path.join(restoredRoot, "missing");
|
||||
await expect(prepareRemoteRepository({ ...input, remoteCwd: missing })).rejects.toThrow("remote_workspace_operation_failed");
|
||||
expect(await fs.lstat(missing).catch(() => null)).toBeNull();
|
||||
await git(secondary, "remote", "add", "origin", "https://example.invalid/foreign.git");
|
||||
await expect(prepareRemoteRepository(input)).rejects.toThrow("remote_workspace_operation_failed");
|
||||
expect(resolveGitAuth).not.toHaveBeenCalled();
|
||||
} finally { await captured.cleanup(); }
|
||||
});
|
||||
|
||||
it("keeps the preparation pin and later agent commits across the next admission", async () => {
|
||||
const origin = await repository(), { remote } = target(), root = path.join(workspace, "managed");
|
||||
const prepared = await prepareRemoteRepository({ target: remote, remoteCwd: root,
|
||||
repoUrl: origin, ref: "HEAD", resolveGitAuth: async () => null });
|
||||
await fs.writeFile(path.join(root, "source.txt"), "agent committed work\n");
|
||||
await git(root, "commit", "-am", "Agent progress");
|
||||
const head = (await git(root, "rev-parse", "HEAD")).stdout.trim();
|
||||
expect(head).not.toBe(prepared.pinnedCommit);
|
||||
await fs.writeFile(path.join(root, "untracked.txt"), "still in progress\n");
|
||||
await fs.rm(origin, { recursive: true, force: true });
|
||||
const resolveGitAuth = vi.fn(async () => { throw new Error("origin unavailable"); });
|
||||
expect(await prepareRemoteRepository({ target: remote, remoteCwd: root, repoUrl: origin,
|
||||
ref: "HEAD", pinnedCommit: prepared.pinnedCommit, resolveGitAuth })).toEqual(prepared);
|
||||
expect((await git(root, "rev-parse", "HEAD")).stdout.trim()).toBe(head);
|
||||
expect(await fs.readFile(path.join(root, "untracked.txt"), "utf8")).toBe("still in progress\n");
|
||||
expect(resolveGitAuth).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses trusted SSH rewrites only for clone and ignores ambient Git configuration", async () => {
|
||||
const origin = await repository({ revisions: true });
|
||||
const remoteUrl = "ssh://git@127.0.0.1:1/synthetic-repository";
|
||||
const rewrite = { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: `url.${pathToFileURL(origin).href}.insteadOf`,
|
||||
GIT_CONFIG_VALUE_0: remoteUrl };
|
||||
const resolveGitAuth = async () => ({ configArgs: [], env: rewrite,
|
||||
source: "company_secret" as const, secretName: "SYNTHETIC_FIXTURE" });
|
||||
const { remote } = target({ GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "remote.origin.url",
|
||||
GIT_CONFIG_VALUE_0: "synthetic-wrong-ambient-origin" });
|
||||
const root = path.join(workspace, "rewritten-root");
|
||||
await prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: root,
|
||||
preparationIdentity: "a".repeat(64), source: { kind: "git", repoUrl: remoteUrl, ref: "HEAD~1" }, resolveGitAuth });
|
||||
expect(await fs.readFile(path.join(root, "source.txt"), "utf8")).toBe("initial synthetic source\n");
|
||||
const nested = path.join(workspace, "rewritten-repository");
|
||||
const receipt = await prepareRemoteRepository({ target: remote, remoteCwd: nested, repoUrl: remoteUrl,
|
||||
ref: "HEAD~1", resolveGitAuth });
|
||||
expect(await fs.readFile(path.join(nested, "source.txt"), "utf8")).toBe("initial synthetic source\n");
|
||||
expect((await git(nested, "config", "--get", "remote.origin.url")).stdout.trim()).toBe(remoteUrl);
|
||||
const unavailableAuth = vi.fn(async () => { throw new Error("unexpected credential lookup"); });
|
||||
await expect(prepareRemoteRepository({ target: remote, remoteCwd: nested, repoUrl: remoteUrl,
|
||||
ref: "HEAD~1", pinnedCommit: receipt.pinnedCommit, resolveGitAuth: unavailableAuth })).resolves.toEqual(receipt);
|
||||
expect(unavailableAuth).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,270 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { RemoteWorkspaceSource } from "@paperclipai/shared";
|
||||
import { remoteWorkspaceArchiveScript } from "./remote-workspace-snapshot-archive.js";
|
||||
import type { GitRemoteAuthProvider } from "./git-credentials.js";
|
||||
|
||||
export type RemoteWorkspaceTarget = Extract<AdapterExecutionTarget, { transport: "sandbox" }>;
|
||||
|
||||
export function requireRemoteWorkspaceTarget(target: AdapterExecutionTarget | null): RemoteWorkspaceTarget {
|
||||
if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask
|
||||
|| !target.runner.syncIn || !target.runner.syncOut) throw new Error("remote_workspace_transport_unsupported");
|
||||
return target;
|
||||
}
|
||||
|
||||
async function command(target: RemoteWorkspaceTarget, args: string[], env?: Record<string, string>, timeoutMs = 300_000) {
|
||||
const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd,
|
||||
env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true });
|
||||
if (result.exitCode !== 0 || result.timedOut) {
|
||||
const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace";
|
||||
throw new Error(!result.timedOut && result.stderr.trim() === gitOperation ? gitOperation : "remote_workspace_operation_failed");
|
||||
}
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
const PREPARE = String.raw`
|
||||
const fs = require('node:fs/promises'), path = require('node:path'), cp = require('node:child_process');
|
||||
const {promisify}=require('node:util'); const exec=promisify(cp.execFile);
|
||||
(async()=>{
|
||||
const [root,raw]=process.argv.slice(1), input=JSON.parse(raw);
|
||||
if (!path.isAbsolute(root)||!/^[a-f0-9]{64}$/.test(input.preparationIdentity)) throw Error('invalid_root');
|
||||
const noConfig={GIT_TERMINAL_PROMPT:'0',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_CONFIG_SYSTEM:'/dev/null',GIT_CONFIG_COUNT:'0',GIT_CONFIG_PARAMETERS:'',GIT_TEMPLATE_DIR:'',GIT_ASKPASS:'',SSH_ASKPASS:''};
|
||||
const isolated={...process.env,...noConfig};for(const key of Object.keys(isolated))if(/^GIT_CONFIG_(KEY|VALUE)_\d+$/.test(key))delete isolated[key];
|
||||
const cloneEnv={...isolated};for(const key of input.authEnvKeys)if(/^GIT_CONFIG_(COUNT|(KEY|VALUE)_\d+)$/.test(key)&&process.env[key]!==undefined)cloneEnv[key]=process.env[key];
|
||||
const localEnv={...isolated};for(const key of input.authEnvKeys)delete localEnv[key];Object.assign(localEnv,noConfig);
|
||||
let current='/'; for(const part of path.dirname(root).split('/').filter(Boolean)) {
|
||||
current=path.join(current,part); await fs.mkdir(current,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});
|
||||
const s=await fs.lstat(current); if(!s.isDirectory()||s.isSymbolicLink())throw Error('unsafe_root');
|
||||
}
|
||||
const prior=await fs.lstat(root).catch(e=>{if(e.code==='ENOENT')return null;throw e});
|
||||
if(prior){
|
||||
if(!prior.isDirectory()||prior.isSymbolicLink()||await fs.realpath(root)!==root)throw Error('unsafe_root');
|
||||
const runtime=path.join(root,'.paperclip-runtime'),marker=path.join(runtime,'workspace-ready.json');
|
||||
const dir=await fs.lstat(runtime);if(!dir.isDirectory()||dir.isSymbolicLink())throw Error('unsafe_receipt');
|
||||
const file=await fs.lstat(marker);if(!file.isFile()||file.isSymbolicLink()||file.nlink!==1||file.size>16384)throw Error('unsafe_receipt');
|
||||
if(JSON.parse(await fs.readFile(marker,'utf8')).preparationIdentity!==input.preparationIdentity)throw Error('workspace_owner_changed');
|
||||
process.stdout.write(JSON.stringify({cwd:root}));return;
|
||||
}
|
||||
if(input.probeOnly){process.stdout.write(JSON.stringify({cloneRequired:true}));return;}
|
||||
const staged=await fs.mkdtemp(root+'.preparing-');
|
||||
if(input.kind==='git'){
|
||||
await exec('git',['-c','core.hooksPath=/dev/null','-c','init.templateDir=',...input.authArgs,'clone','--no-hardlinks','--',input.repoUrl,staged],{env:cloneEnv});
|
||||
const git=async args=>exec('git',['-c','core.hooksPath=/dev/null','-C',staged,...args],{env:localEnv});
|
||||
if(input.branchMode==='existing'){
|
||||
if(!input.branchName)throw Error('existing_branch_required');
|
||||
await git(['check-ref-format','--branch',input.branchName]);
|
||||
const remoteRef='refs/remotes/origin/'+input.branchName;
|
||||
await git(['rev-parse','--verify','--quiet',remoteRef+'^{commit}']);
|
||||
const local=await git(['show-ref','--verify','--quiet','refs/heads/'+input.branchName]).then(()=>true,()=>false);
|
||||
await git(local?['checkout',input.branchName]:['checkout','-b',input.branchName,remoteRef]);
|
||||
}else{
|
||||
if(input.ref)await git(['checkout','--detach',input.ref]);
|
||||
if(input.branchName)await git(['checkout','-B',input.branchName]);
|
||||
}
|
||||
await fs.mkdir(path.join(staged,'.git','info'),{recursive:true,mode:0o700});
|
||||
await fs.appendFile(path.join(staged,'.git','info','exclude'),'\n/.paperclip-runtime/\n/.paperclip-repositories/\n');
|
||||
}else{
|
||||
await fs.writeFile(path.join(staged,'.gitignore'),'/.paperclip-runtime/\n/.paperclip-repositories/\n',{flag:'wx',mode:0o600});
|
||||
}
|
||||
const runtime=path.join(staged,'.paperclip-runtime');await fs.mkdir(runtime,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});
|
||||
const runtimeStat=await fs.lstat(runtime);if(!runtimeStat.isDirectory()||runtimeStat.isSymbolicLink())throw Error('unsafe_receipt');
|
||||
await fs.writeFile(path.join(runtime,'workspace-ready.json'),JSON.stringify({preparationIdentity:input.preparationIdentity}),{flag:'wx',mode:0o600});
|
||||
if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root');
|
||||
await fs.rename(staged,root);
|
||||
process.stdout.write(JSON.stringify({cwd:await fs.realpath(root)}));
|
||||
})().catch(()=>{process.stderr.write('remote_workspace_prepare_failed');process.exitCode=1});`;
|
||||
|
||||
export async function prepareRemoteWorkspaceDirectory(input: {
|
||||
target: RemoteWorkspaceTarget; remoteCwd: string; source: RemoteWorkspaceSource; resolveGitAuth: GitRemoteAuthProvider;
|
||||
preparationIdentity: string;
|
||||
}) {
|
||||
if (input.source.kind === "git") {
|
||||
const prior = JSON.parse(await command(input.target, ["-e", PREPARE, input.remoteCwd,
|
||||
JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity,
|
||||
authArgs: [], authEnvKeys: [], probeOnly: true })])) as { cwd?: string; cloneRequired?: boolean };
|
||||
if (prior.cwd === input.remoteCwd) return;
|
||||
if (prior.cloneRequired !== true) throw new Error("remote_workspace_root_changed");
|
||||
}
|
||||
const auth = input.source.kind === "git" ? await input.resolveGitAuth(input.source.repoUrl) : null;
|
||||
const output = await command(input.target, ["-e", PREPARE, input.remoteCwd,
|
||||
JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity,
|
||||
authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env);
|
||||
const result = JSON.parse(output) as { cwd: string };
|
||||
if (result.cwd !== input.remoteCwd) throw new Error("remote_workspace_root_changed");
|
||||
}
|
||||
|
||||
export function remoteWorkspaceRepositoryPath(remoteRoot: string, repository: {
|
||||
id: string; relativePath: string; readOnly?: boolean;
|
||||
}) {
|
||||
if (!/^[a-zA-Z0-9_-]+$/.test(repository.id) || path.posix.isAbsolute(repository.relativePath)
|
||||
|| repository.relativePath.split("/").some(part => !part || part === "." || part === "..")
|
||||
|| /[\\\u0000-\u001f]/u.test(repository.relativePath)) throw new Error("remote_workspace_repository_path_invalid");
|
||||
return repository.readOnly
|
||||
? path.posix.join(path.posix.dirname(remoteRoot), ".paperclip-referenced", path.posix.basename(remoteRoot), repository.id)
|
||||
: path.posix.join(remoteRoot, repository.relativePath);
|
||||
}
|
||||
|
||||
export async function prepareRemoteRepository(input: {
|
||||
target: RemoteWorkspaceTarget; remoteCwd: string; repoUrl: string; ref: string | null;
|
||||
pinnedCommit?: string; verifyOnly?: boolean; savedArchive?: boolean; resolveGitAuth: GitRemoteAuthProvider;
|
||||
}) {
|
||||
const script = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');const {promisify}=require('node:util');const exec=promisify(cp.execFile);
|
||||
(async()=>{const [root,raw]=process.argv.slice(1),input=JSON.parse(raw);const noConfig={GIT_TERMINAL_PROMPT:'0',GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_CONFIG_SYSTEM:'/dev/null',GIT_CONFIG_COUNT:'0',GIT_CONFIG_PARAMETERS:'',GIT_TEMPLATE_DIR:'',GIT_ASKPASS:'',SSH_ASKPASS:''};
|
||||
const env={...process.env,...noConfig};for(const key of Object.keys(env))if(/^GIT_CONFIG_(KEY|VALUE)_\d+$/.test(key))delete env[key];
|
||||
const cloneEnv={...env};for(const key of input.authEnvKeys)if(/^GIT_CONFIG_(COUNT|(KEY|VALUE)_\d+)$/.test(key)&&process.env[key]!==undefined)cloneEnv[key]=process.env[key];
|
||||
const localEnv={...env};for(const key of input.authEnvKeys)delete localEnv[key];Object.assign(localEnv,noConfig);const opts={env:localEnv};
|
||||
let parent='/';for(const part of path.dirname(root).split('/').filter(Boolean)){parent=path.join(parent,part);
|
||||
await fs.mkdir(parent,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const entry=await fs.lstat(parent);
|
||||
if(!entry.isDirectory()||entry.isSymbolicLink()||await fs.realpath(parent)!==parent)throw Error('unsafe_repository_parent');}
|
||||
const existing=await fs.lstat(root).catch(e=>{if(e.code==='ENOENT')return null;throw e});
|
||||
if(existing){if(!existing.isDirectory()||existing.isSymbolicLink())throw Error('unsafe_repository')}
|
||||
else{if(input.verifyOnly||input.savedArchive)throw Error('repository_missing');if(input.probeOnly){process.stdout.write(JSON.stringify({cloneRequired:true}));return;}await exec('git',['-c','core.hooksPath=/dev/null','-c','init.templateDir=',...input.authArgs,'clone','--no-hardlinks','--',input.repoUrl,root],{env:cloneEnv});
|
||||
if(input.ref)await exec('git',['-c','core.hooksPath=/dev/null','-C',root,'checkout','--detach',input.ref],opts);}
|
||||
const git=async args=>(await exec('git',['-c','core.hooksPath=/dev/null','-C',root,...args],opts)).stdout.trim();
|
||||
if(await fs.realpath(root)!==root)throw Error('unsafe_repository');
|
||||
const hasOrigin=(await git(['remote'])).split('\n').includes('origin');
|
||||
const origin=hasOrigin?await git(['remote','get-url','origin']):null;
|
||||
if(origin!==input.repoUrl&&!(input.savedArchive&&origin===null))throw Error('repository_origin_changed');
|
||||
const head=await git(['rev-parse','HEAD']);if(input.pinnedCommit)await git(['cat-file','-e',input.pinnedCommit+'^{commit}']);
|
||||
const branch=await git(['symbolic-ref','--short','-q','HEAD']).catch(()=>null);process.stdout.write(JSON.stringify({cwd:root,pinnedCommit:input.pinnedCommit||head,branchName:branch}));
|
||||
})().catch(()=>{process.stderr.write('remote_workspace_repository_failed');process.exitCode=1});`;
|
||||
const payload = { ...input, target: undefined, resolveGitAuth: undefined };
|
||||
const prior = JSON.parse(await command(input.target, ["-e", script, input.remoteCwd,
|
||||
JSON.stringify({ ...payload, authArgs: [], authEnvKeys: [], probeOnly: true })])) as {
|
||||
cwd?: string; pinnedCommit?: string; branchName?: string | null; cloneRequired?: boolean;
|
||||
};
|
||||
if (prior.cloneRequired !== true) {
|
||||
if (prior.cwd !== input.remoteCwd || !/^[a-f0-9]{40,64}$/.test(prior.pinnedCommit ?? "")) throw new Error("remote_workspace_repository_receipt_invalid");
|
||||
return { cwd: prior.cwd, pinnedCommit: prior.pinnedCommit!, branchName: prior.branchName ?? null };
|
||||
}
|
||||
const auth = await input.resolveGitAuth(input.repoUrl);
|
||||
const output = await command(input.target, ["-e", script, input.remoteCwd,
|
||||
JSON.stringify({ ...payload, authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env);
|
||||
const value = JSON.parse(output) as { cwd: string; pinnedCommit: string; branchName: string | null };
|
||||
if (value.cwd !== input.remoteCwd || !/^[a-f0-9]{40,64}$/.test(value.pinnedCommit)) throw new Error("remote_workspace_repository_receipt_invalid");
|
||||
return value;
|
||||
}
|
||||
|
||||
async function installArchiveHelper(target: RemoteWorkspaceTarget, remoteCwd: string, operationId: string) {
|
||||
if (!/^[a-f0-9-]{36}$/.test(operationId)) throw new Error("remote_workspace_operation_identity_invalid");
|
||||
const directory = path.posix.join(remoteCwd, ".paperclip-runtime", "workspace-snapshots", operationId);
|
||||
const helper = path.posix.join(directory, "archive.cjs");
|
||||
const install = String.raw`const fs=require('node:fs/promises'),path=require('node:path');(async()=>{
|
||||
const [root,dir,file,body]=process.argv.slice(1);let p='/';
|
||||
for(const s of root.split('/').filter(Boolean)){p=path.join(p,s);await fs.mkdir(p,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const st=await fs.lstat(p);if(st.isSymbolicLink()||!st.isDirectory())throw Error('unsafe');}
|
||||
for(const s of path.relative(root,dir).split('/')){p=path.join(p,s);await fs.mkdir(p,{mode:0o700}).catch(e=>{if(e.code!=='EEXIST')throw e});const st=await fs.lstat(p);if(st.isSymbolicLink()||!st.isDirectory())throw Error('unsafe');}
|
||||
const existing=await fs.lstat(file).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(existing&&(!existing.isFile()||existing.isSymbolicLink()))throw Error('unsafe');
|
||||
const temp=file+'.'+require('node:crypto').randomUUID()+'.tmp';await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file);
|
||||
})().catch(()=>{process.exitCode=1});`;
|
||||
await command(target, ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]);
|
||||
return { directory, helper, archive: path.posix.join(directory, "workspace.pcws"), receipt: path.posix.join(directory, "receipt.json") };
|
||||
}
|
||||
|
||||
/** Only immutable operation staging is removed; working files are never a target. */
|
||||
export async function cleanupRemoteWorkspaceArchive(input: {
|
||||
target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string;
|
||||
}) {
|
||||
if (!/^[a-f0-9-]{36}$/.test(input.operationId)) throw new Error("remote_workspace_operation_identity_invalid");
|
||||
const script = String.raw`const fs=require('node:fs/promises'),path=require('node:path');(async()=>{
|
||||
const [root,id]=process.argv.slice(1);if(!path.isAbsolute(root)||!/^[a-f0-9-]{36}$/.test(id))throw Error('invalid');
|
||||
const dir=path.join(root,'.paperclip-runtime','workspace-snapshots',id);let current='/';
|
||||
for(const part of dir.split('/').filter(Boolean)){current=path.join(current,part);
|
||||
const entry=await fs.lstat(current).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(!entry)return;
|
||||
if(!entry.isDirectory()||entry.isSymbolicLink())throw Error('unsafe');}
|
||||
await fs.rm(dir,{recursive:true,force:true});
|
||||
})().catch(()=>{process.stderr.write('remote_workspace_export_cleanup_failed');process.exitCode=1});`;
|
||||
await command(input.target, ["-e", script, input.remoteCwd, input.operationId], undefined, 10_000);
|
||||
}
|
||||
|
||||
export async function captureRemoteWorkspaceArchive(input: {
|
||||
target: RemoteWorkspaceTarget; remoteCwd: string; operationId: string;
|
||||
}) {
|
||||
const staged = await installArchiveHelper(input.target, input.remoteCwd, input.operationId);
|
||||
// Publish the archive and receipt as one immutable directory. Concurrent
|
||||
// invocations own separate attempts; no invocation unlinks another's archive.
|
||||
const capture = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');const {promisify}=require('node:util');(async()=>{
|
||||
const [helper,root,directory]=process.argv.slice(1),canonical=path.join(directory,'captured');
|
||||
const readPair=async(dir,layout,required)=>{
|
||||
const st=await fs.lstat(dir).catch(e=>{if(e.code==='ENOENT')return null;throw e});if(!st)return null;
|
||||
if(!st.isDirectory()||st.isSymbolicLink()||await fs.realpath(dir)!==dir)throw Error('unsafe_capture');
|
||||
const receipt=path.join(dir,'receipt.json'),archive=path.join(dir,'workspace.pcws');
|
||||
const receiptStat=await fs.lstat(receipt).catch(e=>{if(!required&&e.code==='ENOENT')return null;throw e});if(!receiptStat)return null;
|
||||
const archiveStat=await fs.lstat(archive);
|
||||
if(!receiptStat.isFile()||receiptStat.isSymbolicLink()||receiptStat.nlink!==1||receiptStat.size>16384
|
||||
||!archiveStat.isFile()||archiveStat.isSymbolicLink()||archiveStat.nlink!==1)throw Error('unsafe_capture');
|
||||
const result=JSON.parse(await fs.readFile(receipt,'utf8'));if(archiveStat.size!==result.byteSize)throw Error('invalid_capture');
|
||||
return {...result,layout};
|
||||
};
|
||||
let result=await readPair(canonical,'captured',true)??await readPair(directory,'legacy',false),attempt;
|
||||
try{if(!result){
|
||||
attempt=await fs.mkdtemp(path.join(directory,'capture-'));
|
||||
const archive=path.join(attempt,'workspace.pcws'),receipt=path.join(attempt,'receipt.json');
|
||||
const captured=await promisify(cp.execFile)(process.execPath,[helper,'capture',root,archive],{maxBuffer:1048576});
|
||||
await fs.writeFile(receipt,JSON.stringify(JSON.parse(captured.stdout)),{mode:0o600,flag:'wx'});
|
||||
await readPair(attempt,'captured',true);
|
||||
try{await fs.rename(attempt,canonical);attempt=undefined}catch(e){if(!['EEXIST','ENOTEMPTY'].includes(e.code))throw e;}
|
||||
result=await readPair(canonical,'captured',true);if(!result)throw Error('capture_missing');
|
||||
}process.stdout.write(JSON.stringify(result));}finally{if(attempt)await fs.rm(attempt,{recursive:true,force:true});}
|
||||
})().catch(error=>{
|
||||
const gitOperation='remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace';
|
||||
process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation:'remote_workspace_capture_failed');process.exitCode=1;
|
||||
});`;
|
||||
const { layout, ...receipt } = JSON.parse(await command(input.target, ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as {
|
||||
manifestDigest: string; sha256: string; byteSize: number; entries: number; layout: "legacy" | "captured";
|
||||
};
|
||||
if (!["legacy", "captured"].includes(layout) || !/^[a-f0-9]{64}$/.test(receipt.manifestDigest) || !/^[a-f0-9]{64}$/.test(receipt.sha256)
|
||||
|| !Number.isSafeInteger(receipt.byteSize) || receipt.byteSize <= 0) throw new Error("remote_workspace_capture_receipt_invalid");
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-upload-"));
|
||||
const archivePath = path.join(root, "workspace.pcws");
|
||||
try {
|
||||
await input.target.runner!.syncOut!([{ operationId: input.operationId,
|
||||
files: [{ kind: "file", sourcePath: layout === "captured" ? path.posix.join(staged.directory, "captured", "workspace.pcws") : staged.archive, targetPath: archivePath }],
|
||||
}]);
|
||||
const hash = createHash("sha256"); let bytes = 0;
|
||||
for await (const chunk of createReadStream(archivePath)) { hash.update(chunk); bytes += chunk.length; }
|
||||
if (hash.digest("hex") !== receipt.sha256 || bytes !== receipt.byteSize) throw new Error("remote_workspace_capture_transfer_mismatch");
|
||||
return { ...receipt, archivePath, cleanup: () => rm(root, { recursive: true, force: true }) };
|
||||
} catch (error) { await rm(root, { recursive: true, force: true }); throw error; }
|
||||
}
|
||||
|
||||
export async function restoreRemoteWorkspaceDirectory(input: {
|
||||
target: RemoteWorkspaceTarget; remoteCwd: string; archivePath: string; manifestDigest: string; restorationIdentity: string;
|
||||
}) {
|
||||
if (!/^[a-f0-9]{64}$/.test(input.restorationIdentity)) throw new Error("remote_workspace_restore_identity_invalid");
|
||||
const id = input.restorationIdentity;
|
||||
const operationId = `${id.slice(0, 8)}-${id.slice(8, 12)}-${id.slice(12, 16)}-${id.slice(16, 20)}-${id.slice(20, 32)}`;
|
||||
// Stage outside the destination; the restore validator requires an empty target.
|
||||
const parent = path.posix.dirname(input.remoteCwd);
|
||||
const staged = await installArchiveHelper(input.target, parent, operationId);
|
||||
await input.target.runner!.syncIn!([{ operationId,
|
||||
files: [{ kind: "file", sourcePath: input.archivePath, targetPath: staged.archive }],
|
||||
}]);
|
||||
// Publish the restored folder and its identity together. A lost successful
|
||||
// reply can then adopt the same files without repeating a destructive restore.
|
||||
const restore = String.raw`const fs=require('node:fs/promises'),path=require('node:path'),cp=require('node:child_process');
|
||||
(async()=>{const [helper,archive,root,digest,identity]=process.argv.slice(1);
|
||||
if(!/^[a-f0-9]{64}$/.test(identity))throw Error('invalid_identity');
|
||||
const prior=await fs.lstat(root).catch(e=>{if(e.code==='ENOENT')return null;throw e});
|
||||
if(prior){
|
||||
if(!prior.isDirectory()||prior.isSymbolicLink()||await fs.realpath(root)!==root)throw Error('unsafe_root');
|
||||
const runtime=path.join(root,'.paperclip-runtime'),marker=path.join(runtime,'workspace-restored.json');
|
||||
const dir=await fs.lstat(runtime);if(!dir.isDirectory()||dir.isSymbolicLink())throw Error('unsafe_receipt');
|
||||
const file=await fs.lstat(marker);if(!file.isFile()||file.isSymbolicLink()||file.nlink!==1||file.size>16384)throw Error('unsafe_receipt');
|
||||
const receipt=JSON.parse(await fs.readFile(marker,'utf8'));
|
||||
if(receipt.identity!==identity||receipt.manifestDigest!==digest)throw Error('workspace_owner_changed');return;
|
||||
}
|
||||
const staged=await fs.mkdtemp(root+'.restoring-');
|
||||
await require('node:util').promisify(cp.execFile)(process.execPath,[helper,'restore',archive,staged,digest],{maxBuffer:1048576});
|
||||
const runtime=path.join(staged,'.paperclip-runtime');await fs.mkdir(runtime,{mode:0o700});
|
||||
await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify({identity,manifestDigest:digest}),{flag:'wx',mode:0o600});
|
||||
if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root');await fs.rename(staged,root);
|
||||
})().catch(()=>{process.stderr.write('remote_workspace_restore_failed');process.exitCode=1});`;
|
||||
await command(input.target, ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity]);
|
||||
// Publish the restored location before cleanup can fail, stall, or lose its
|
||||
// reply. The owner journals this receipt and then reclaims disposable staging.
|
||||
return { operationId, stagingRoot: parent, cleanupPending: true };
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,710 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtemp, rm, statfs } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import {
|
||||
environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issues, workspaceOperations, type Db,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
readRemoteWorkspaceAuthority, remoteWorkspaceSourceSchema,
|
||||
type RemoteWorkspaceAuthority, type RemoteWorkspaceRunReference, type RemoteWorkspaceSource,
|
||||
type EnvironmentLease,
|
||||
} from "@paperclipai/shared";
|
||||
import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { getStorageService, type StorageService } from "../storage/index.js";
|
||||
import { loadConfig } from "../config.js";
|
||||
import { conflict, forbidden, unprocessable } from "../errors.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { createGitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
|
||||
import { assertExecutionWorkspaceRecordAccess, assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { persistActivity } from "./activity-log.js";
|
||||
import {
|
||||
captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath,
|
||||
requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory,
|
||||
} from "./remote-workspace-transport.js";
|
||||
import {
|
||||
beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot,
|
||||
} from "./remote-workspace-snapshot-storage.js";
|
||||
import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js";
|
||||
import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js";
|
||||
import { attachPreservedWorkspaceSnapshots } from "./workspace-recovery-snapshot.js";
|
||||
import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js";
|
||||
import { notifyTerminalWorkspaceWork } from "./terminal-workspace-work.js";
|
||||
import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js";
|
||||
|
||||
export { readRemoteWorkspaceAuthority, type RemoteWorkspaceAuthority, type RemoteWorkspaceSource } from "@paperclipai/shared";
|
||||
export { remoteWorkspaceRepositoryPath } from "./remote-workspace-transport.js";
|
||||
export { clearRemoteWorkspaceLocationForReleasedLease } from "./remote-workspace-snapshot-retention.js";
|
||||
type Workspace = Pick<typeof executionWorkspaces.$inferSelect, "id" | "companyId" | "metadata" | "sourceIssueId" | "projectId" | "projectWorkspaceId">;
|
||||
type Transaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
const object = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
const SAVE_WORK_SCHEMA = "paperclip.remote-workspace-save-work.v1";
|
||||
const RETIRED_SAVE_SCHEMA = "paperclip.remote-workspace-retired-save.v1";
|
||||
type SaveWork = { schema: typeof SAVE_WORK_SCHEMA; activeIds: string[]; unknown: boolean; quiescedAt: string | null };
|
||||
function readSaveWork(metadata: Record<string, unknown> | null | undefined): SaveWork | null {
|
||||
const work = object(metadata?.saveWork);
|
||||
return work.schema === SAVE_WORK_SCHEMA && Array.isArray(work.activeIds)
|
||||
&& work.activeIds.every(id => typeof id === "string") && new Set(work.activeIds).size === work.activeIds.length
|
||||
&& typeof work.unknown === "boolean" && (work.quiescedAt === null
|
||||
|| typeof work.quiescedAt === "string" && Number.isFinite(Date.parse(work.quiescedAt)))
|
||||
? work as SaveWork : null;
|
||||
}
|
||||
function sameExportLocation(value: unknown, location: NonNullable<RemoteWorkspaceAuthority["location"]>) {
|
||||
const recorded = object(value);
|
||||
return (["environmentId", "leaseId", "providerLeaseId", "remoteCwd"] as const).every(key => recorded[key] === location[key]);
|
||||
}
|
||||
|
||||
export function shouldUseRemoteWorkspaces(input: {
|
||||
adapterType: string; environmentDriver: string | null; experimentalEnabled: boolean; existingWorkspace?: { metadata?: Record<string, unknown> | null } | null;
|
||||
}) {
|
||||
const enrolled = readRemoteWorkspaceAuthority(input.existingWorkspace?.metadata) !== null;
|
||||
if (enrolled && (input.adapterType !== "paperclip_runner" || input.environmentDriver !== "sandbox")) {
|
||||
throw conflict("This workspace requires its native sandbox execution environment.", { code: "remote_workspace_execution_required" });
|
||||
}
|
||||
if (enrolled) return true;
|
||||
// Existing host authority changes only through the explicit import workflow.
|
||||
if (input.existingWorkspace || !input.experimentalEnabled || input.adapterType !== "paperclip_runner"
|
||||
|| input.environmentDriver !== "sandbox") return false;
|
||||
return configuredRemoteWorkspacePersistence() !== null;
|
||||
}
|
||||
|
||||
function configuredRemoteWorkspacePersistence(storage?: StorageService) {
|
||||
const config = loadConfig();
|
||||
const databaseUrl = config.databaseUrl?.trim();
|
||||
const keyFilePath = process.env.PAPERCLIP_SECRETS_MASTER_KEY?.trim() ? null : config.secretsMasterKeyFilePath;
|
||||
if ((storage?.provider ?? config.storageProvider) !== "s3" || !databaseUrl || !/^postgres(?:ql)?:\/\//.test(databaseUrl)
|
||||
|| (keyFilePath !== null && !existsSync(keyFilePath))) {
|
||||
return null;
|
||||
}
|
||||
return { databaseUrl, keyFilePath };
|
||||
}
|
||||
|
||||
export function assertRemoteWorkspacePersistence(storage: StorageService = getStorageService()) {
|
||||
const persistence = configuredRemoteWorkspacePersistence(storage);
|
||||
if (!persistence) throw conflict("Remote workspace storage requires external Postgres, object storage, and a stable configured encryption key.",
|
||||
{ code: "remote_workspace_persistence_unavailable" });
|
||||
return persistence;
|
||||
}
|
||||
|
||||
function reference(workspaceId: string, authority: RemoteWorkspaceAuthority, state: "prepared" | "saved"): RemoteWorkspaceRunReference {
|
||||
if (!authority.location) throw new Error("remote_workspace_location_missing");
|
||||
return { schema: "paperclip.remote-workspace-run.v1", workspaceId, epoch: authority.epoch,
|
||||
leaseId: authority.location.leaseId, providerLeaseId: authority.location.providerLeaseId,
|
||||
remoteCwd: authority.location.remoteCwd, state, snapshotId: authority.snapshotId };
|
||||
}
|
||||
|
||||
async function currentWorkspace(tx: Transaction, companyId: string, workspaceId: string) {
|
||||
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, workspaceId),
|
||||
eq(executionWorkspaces.companyId, companyId))).for("update");
|
||||
if (!workspace) throw new Error("remote_workspace_missing");
|
||||
return workspace;
|
||||
}
|
||||
|
||||
export async function reserveRemoteWorkspace(input: {
|
||||
db: Db; companyId: string; issueId: string; runId: string; agentId: string; environmentId: string;
|
||||
projectId: string | null; projectWorkspaceId: string | null; existingWorkspaceId: string | null;
|
||||
source: RemoteWorkspaceSource; storage?: StorageService;
|
||||
mode?: string;
|
||||
fresh?: { attemptId: string; previousWorkspaceId: string | null; expectedBindingRevision: number };
|
||||
recoverySourceRunIds?: string[];
|
||||
}) {
|
||||
assertRemoteWorkspacePersistence(input.storage ?? getStorageService());
|
||||
const source = remoteWorkspaceSourceSchema.parse(input.source);
|
||||
return input.db.transaction(async tx => {
|
||||
const [issue] = await tx.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.issueId))).for("update");
|
||||
const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
|
||||
if (!issue || !run || run.agentId !== input.agentId || run.status !== "running" ||
|
||||
![run.nativeIssueId, run.issueId, object(run.contextSnapshot).issueId].includes(input.issueId)) throw forbidden("Workspace preparation requires its current task execution.");
|
||||
if (input.fresh && (issue.executionWorkspaceId !== input.fresh.previousWorkspaceId
|
||||
|| issue.workspaceBindingRevision !== input.fresh.expectedBindingRevision)) throw conflict("The task workspace selection changed.");
|
||||
let selected = input.fresh ? input.fresh.attemptId : input.existingWorkspaceId ?? issue.executionWorkspaceId;
|
||||
let selectedSharedSource = false;
|
||||
if (!selected && input.mode === "shared_workspace" && input.projectWorkspaceId) {
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`remote-shared-workspace:${input.companyId}:${input.projectWorkspaceId}`}, 0))`);
|
||||
const [shared] = await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.projectWorkspaceId, input.projectWorkspaceId),
|
||||
eq(executionWorkspaces.mode, "shared_workspace"), eq(executionWorkspaces.status, "active"),
|
||||
sql`${executionWorkspaces.metadata}->'remoteWorkspace'->>'schema' = 'paperclip.remote-workspace.v1'`,
|
||||
)).limit(1);
|
||||
selected = shared?.id ?? null;
|
||||
selectedSharedSource = selected !== null;
|
||||
}
|
||||
const [selectedRow] = selected ? await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, selected))).for("update") : [];
|
||||
const existing = selectedRow ?? null;
|
||||
if (selected && !existing && !input.fresh) throw new Error("remote_workspace_missing");
|
||||
const authority = readRemoteWorkspaceAuthority(existing?.metadata);
|
||||
if (existing && !authority) throw conflict("Import this existing workspace before enabling remote storage.", { code: "remote_workspace_migration_required" });
|
||||
if (existing && authority) {
|
||||
if ((existing.sourceIssueId !== input.issueId && existing.mode !== "shared_workspace") || ["archived", "cleanup_failed"].includes(existing.status)) throw conflict("Workspace ownership changed.");
|
||||
if (existing.sourceIssueId !== input.issueId) {
|
||||
const actor = { type: "agent", source: "agent_jwt", companyId: input.companyId,
|
||||
agentId: input.agentId, runId: input.runId,
|
||||
onBehalfOfUserId: run.responsibleUserId === "local-board" ? null : run.responsibleUserId } as const;
|
||||
await assertTaskWorkspaceExecutionAccess(tx, actor, input.companyId, selectedSharedSource
|
||||
? { issueId: input.issueId, projectWorkspaceId: input.projectWorkspaceId }
|
||||
: { issueId: input.issueId, workspaceId: existing.id });
|
||||
await assertExecutionWorkspaceRecordAccess(tx, actor, input.companyId, existing.id);
|
||||
}
|
||||
const [uncertain] = await tx.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, existing.id),
|
||||
eq(workspaceOperations.phase, "remote_workspace_provision"), sql`${workspaceOperations.metadata}->>'uncertain' = 'true'`,
|
||||
sql`${workspaceOperations.heartbeatRunId} is distinct from ${input.runId}::uuid`,
|
||||
)).limit(1);
|
||||
if (uncertain) throw conflict("The prior provisioning operation needs verification before this workspace can run again.", { code: "remote_workspace_provision_unverified" });
|
||||
if (authority.writerRunId && authority.writerRunId !== input.runId) {
|
||||
const [prior] = await tx.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, authority.writerRunId)));
|
||||
const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
|
||||
eq(environmentLeases.heartbeatRunId, authority.writerRunId)));
|
||||
const unusedReservation = !authority.dirty && prior && ["failed", "cancelled", "interrupted", "timed_out", "succeeded"].includes(prior.status)
|
||||
&& leases.every(hasRemoteTerminationReceipt);
|
||||
if (!unusedReservation) throw conflict("The workspace still has an unfinished writer or save operation.", { code: "remote_workspace_writer_pending" });
|
||||
}
|
||||
if (authority.writerRunId === input.runId) return existing;
|
||||
const next: RemoteWorkspaceAuthority = { ...authority, source, epoch: authority.epoch + 1, writerRunId: input.runId };
|
||||
const [saved] = await tx.update(executionWorkspaces).set({ metadata: { ...existing.metadata, remoteWorkspace: next }, updatedAt: new Date() })
|
||||
.where(eq(executionWorkspaces.id, existing.id)).returning();
|
||||
if (issue.executionWorkspaceId !== existing.id) await tx.update(issues).set({ executionWorkspaceId: existing.id,
|
||||
executionWorkspacePreference: "reuse_existing", workspaceBindingRevision: issue.workspaceBindingRevision + 1,
|
||||
updatedAt: new Date() }).where(eq(issues.id, issue.id));
|
||||
return saved!;
|
||||
}
|
||||
const id = input.fresh?.attemptId ?? randomUUID();
|
||||
const initial: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1,
|
||||
snapshotId: null, source, location: null, writerRunId: input.runId, dirty: false };
|
||||
const [workspace] = await tx.insert(executionWorkspaces).values({ id, companyId: input.companyId, sourceIssueId: input.issueId,
|
||||
projectId: input.projectId, projectWorkspaceId: input.projectWorkspaceId, mode: input.mode ?? "isolated_workspace",
|
||||
strategyType: source.kind === "git" ? "git_worktree" : "project_primary", name: "Task workspace", status: "active",
|
||||
providerType: "cloud_sandbox", cwd: `/workspaces/${id}`, repoUrl: source.kind === "git" ? source.repoUrl : null,
|
||||
baseRef: source.kind === "git" ? source.ref : null, branchName: source.kind === "git" ? source.branchName ?? null : null,
|
||||
metadata: { remoteWorkspace: initial, ...(input.recoverySourceRunIds?.length
|
||||
? { remoteWorkspaceRecoverySourceRunIds: input.recoverySourceRunIds } : {}) },
|
||||
}).returning();
|
||||
await tx.update(issues).set({ executionWorkspaceId: id, executionWorkspacePreference: "reuse_existing",
|
||||
workspaceBindingRevision: issue.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, issue.id));
|
||||
await persistActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", actorId: "remote-workspaces",
|
||||
action: "workspace.remote_created", entityType: "execution_workspace", entityId: id, runId: input.runId,
|
||||
issueId: input.issueId, details: { authority: "sandbox", epoch: initial.epoch } });
|
||||
return workspace!;
|
||||
});
|
||||
}
|
||||
|
||||
async function persistRunReference(tx: Transaction, companyId: string, runId: string, value: RemoteWorkspaceRunReference) {
|
||||
const [run] = await tx.select({ profile: heartbeatRuns.runnerProfileJson, nativeIssueId: heartbeatRuns.nativeIssueId,
|
||||
issueId: heartbeatRuns.issueId, contextIssueId: sql<unknown>`${heartbeatRuns.contextSnapshot}->'issueId'` }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId))).for("update");
|
||||
if (!run) throw new Error("remote_workspace_run_missing");
|
||||
await notifyCancelledWorkspacePreservation(tx, { companyId, runId });
|
||||
await tx.update(heartbeatRuns).set({ runnerProfileJson: { ...run.profile, remoteWorkspace: value }, updatedAt: new Date() })
|
||||
.where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
|
||||
return run.nativeIssueId ?? run.issueId ?? (typeof run.contextIssueId === "string" ? run.contextIssueId : null);
|
||||
}
|
||||
|
||||
/** Retry acknowledged or explicitly retired exports, never active captures. */
|
||||
async function cleanupSavedExports(input: { db: Db; companyId: string; workspaceId: string;
|
||||
location: NonNullable<RemoteWorkspaceAuthority["location"]>; target: ReturnType<typeof requireRemoteWorkspaceTarget>;
|
||||
assertOwned: () => Promise<void> }) {
|
||||
for (;;) {
|
||||
const rows = await input.db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata,
|
||||
phase: workspaceOperations.phase }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.executionWorkspaceId, input.workspaceId),
|
||||
inArray(workspaceOperations.phase, ["remote_workspace_snapshot", "remote_workspace_restore"]),
|
||||
sql`(${workspaceOperations.status} = 'succeeded' or (${workspaceOperations.status} = 'failed'
|
||||
and ${workspaceOperations.metadata}->'retiredSave'->>'schema' = ${RETIRED_SAVE_SCHEMA}))`,
|
||||
sql`(${workspaceOperations.metadata}->'saveWork'->>'schema' is distinct from ${SAVE_WORK_SCHEMA}
|
||||
or (${workspaceOperations.metadata}#>'{saveWork,unknown}' = 'false'::jsonb and case when jsonb_typeof(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 'array'
|
||||
then jsonb_array_length(${workspaceOperations.metadata}#>'{saveWork,activeIds}') = 0 else false end))`,
|
||||
sql`${workspaceOperations.metadata}->'exportCleanup'->>'state' = 'pending'`,
|
||||
sql`${workspaceOperations.metadata}->'exportCleanup'->>'environmentId' = ${input.location.environmentId}`,
|
||||
sql`${workspaceOperations.metadata}->'exportCleanup'->>'providerLeaseId' = ${input.location.providerLeaseId}`,
|
||||
sql`${workspaceOperations.metadata}->'exportCleanup'->>'remoteCwd' = ${input.location.remoteCwd}`,
|
||||
)).limit(20);
|
||||
if (!rows.length) return true;
|
||||
for (const row of rows) {
|
||||
await input.assertOwned();
|
||||
let cleaned = false;
|
||||
try {
|
||||
await cleanupRemoteWorkspaceArchive({ target: input.target,
|
||||
remoteCwd: row.phase === "remote_workspace_restore" ? path.posix.dirname(input.location.remoteCwd) : input.location.remoteCwd,
|
||||
operationId: row.id });
|
||||
cleaned = true;
|
||||
} catch { /* Keep a durable retry obligation; the acknowledged snapshot is safe. */ }
|
||||
await input.assertOwned();
|
||||
await input.db.update(workspaceOperations).set({ metadata: sql`jsonb_set(coalesce(${workspaceOperations.metadata}, '{}'::jsonb),
|
||||
'{exportCleanup}', coalesce(${workspaceOperations.metadata}->'exportCleanup', '{}'::jsonb) ||
|
||||
${JSON.stringify({ state: cleaned ? "cleaned" : "pending", lastError: cleaned ? null : "remote_workspace_export_cleanup_failed" })}::jsonb)`,
|
||||
updatedAt: new Date() })
|
||||
.where(and(eq(workspaceOperations.id, row.id), eq(workspaceOperations.companyId, input.companyId),
|
||||
inArray(workspaceOperations.status, ["succeeded", "failed"])));
|
||||
if (!cleaned) return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The caller retains execution admission and provider ownership; this owns file authority. */
|
||||
function isSavedWorkspaceRepository(authority: RemoteWorkspaceAuthority,
|
||||
repository: NonNullable<RemoteWorkspaceSource["repositories"]>[number]) {
|
||||
return authority.snapshotId !== null && (authority.source.repositories ?? []).some(saved =>
|
||||
saved.id === repository.id && saved.relativePath === repository.relativePath && saved.repoUrl === repository.repoUrl
|
||||
&& (saved.ref ?? "HEAD") === (repository.ref ?? "HEAD") && saved.pinnedCommit === repository.pinnedCommit
|
||||
&& (saved.readOnly ?? false) === (repository.readOnly ?? false));
|
||||
}
|
||||
|
||||
export async function prepareRemoteWorkspace(input: {
|
||||
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
|
||||
source?: RemoteWorkspaceSource; assertOwned?: () => Promise<void>;
|
||||
provisionCommand?: string | null; runtimeProvisionCommand?: string | null; storage?: StorageService;
|
||||
}) {
|
||||
const target = requireRemoteWorkspaceTarget(input.target);
|
||||
const initial = readRemoteWorkspaceAuthority(input.workspace.metadata);
|
||||
if ((input.source ?? initial?.source)?.repositories?.some(repository => repository.readOnly)) {
|
||||
throw conflict("The selected native sandbox cannot enforce read-only referenced project roots.",
|
||||
{ code: "remote_workspace_readonly_sources_unsupported" });
|
||||
}
|
||||
if (!initial || initial.writerRunId !== input.runId || !input.lease.environmentId || !input.lease.providerLeaseId
|
||||
|| input.lease.companyId !== input.workspace.companyId || input.lease.heartbeatRunId !== input.runId) throw new Error("remote_workspace_binding_mismatch");
|
||||
await input.assertOwned?.();
|
||||
const sameAllocation = initial.location?.providerLeaseId === input.lease.providerLeaseId
|
||||
&& initial.location?.environmentId === input.lease.environmentId;
|
||||
if (initial.location && !sameAllocation && initial.dirty) throw conflict("The prior workspace has unsaved work; recover it before replacing its allocation.");
|
||||
const remoteRoot = sameAllocation ? initial.location!.remoteCwd : path.posix.join(target.remoteCwd, ".paperclip-workspaces", input.workspace.id);
|
||||
const preparationIdentity = createHash("sha256").update(JSON.stringify([input.workspace.companyId, input.workspace.id,
|
||||
input.runId, initial.epoch, input.lease.id, input.lease.providerLeaseId, remoteRoot, initial.source])).digest("hex");
|
||||
let restoredExport: Awaited<ReturnType<typeof restoreRemoteWorkspaceDirectory>> | null = null;
|
||||
if (sameAllocation && !await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId,
|
||||
workspaceId: input.workspace.id, location: initial.location!, target,
|
||||
assertOwned: input.assertOwned ?? (async () => undefined) })) throw new Error("remote_workspace_export_cleanup_pending");
|
||||
if (!sameAllocation) {
|
||||
if (initial.snapshotId) {
|
||||
const snapshot = await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id, snapshotId: initial.snapshotId });
|
||||
if (!snapshot) throw new Error("remote_workspace_snapshot_missing");
|
||||
const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-restore-"));
|
||||
try {
|
||||
const archivePath = path.join(scratch, "workspace.pcws");
|
||||
await loadWorkspaceSnapshotArchive({ storage: input.storage ?? getStorageService(), reference: snapshot, targetPath: archivePath });
|
||||
restoredExport = await restoreRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, archivePath, manifestDigest: snapshot.manifestDigest,
|
||||
restorationIdentity: preparationIdentity });
|
||||
} finally { await rm(scratch, { recursive: true, force: true }); }
|
||||
} else {
|
||||
await prepareRemoteWorkspaceDirectory({ target, remoteCwd: remoteRoot, source: initial.source,
|
||||
preparationIdentity,
|
||||
resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) });
|
||||
}
|
||||
}
|
||||
for (const repository of (input.source ?? initial.source).repositories ?? []) {
|
||||
await input.assertOwned?.();
|
||||
// Only repositories already owned by the saved authority may rely on the
|
||||
// archive when their old local origin was intentionally omitted. New source
|
||||
// descriptors still follow normal acquisition and origin verification.
|
||||
await prepareRemoteRepository({ target, remoteCwd: remoteWorkspaceRepositoryPath(remoteRoot, repository),
|
||||
repoUrl: repository.repoUrl, ref: repository.ref, pinnedCommit: repository.pinnedCommit,
|
||||
savedArchive: isSavedWorkspaceRepository(initial, repository),
|
||||
resolveGitAuth: await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) });
|
||||
}
|
||||
const recoveredFiles: Array<{ operationId: string; sourceRunId: string; path: string }> = [];
|
||||
const recoverySourceRunIds = input.workspace.metadata?.remoteWorkspaceRecoverySourceRunIds;
|
||||
if (!sameAllocation && Array.isArray(recoverySourceRunIds) && recoverySourceRunIds.length && input.workspace.sourceIssueId) {
|
||||
const scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-recovery-import-"));
|
||||
try {
|
||||
const recovered = await attachPreservedWorkspaceSnapshots({ db: input.db, companyId: input.workspace.companyId,
|
||||
issueId: input.workspace.sourceIssueId, sourceRunIds: recoverySourceRunIds as string[], cwd: scratch });
|
||||
for (const item of recovered) {
|
||||
const relative = `.paperclip-recovered/${item.operationId}`;
|
||||
await target.runner!.syncIn!([{ operationId: randomUUID(), files: [{
|
||||
kind: "directory", sourcePath: item.path, targetPath: path.posix.join(remoteRoot, relative),
|
||||
}] }]);
|
||||
recoveredFiles.push({ operationId: item.operationId, sourceRunId: item.sourceRunId, path: relative });
|
||||
}
|
||||
} finally { await rm(scratch, { recursive: true, force: true }); }
|
||||
}
|
||||
await input.assertOwned?.();
|
||||
const prepared = await input.db.transaction(async tx => {
|
||||
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
|
||||
const current = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!current || current.epoch !== initial.epoch || current.writerRunId !== input.runId) throw new Error("remote_workspace_authority_changed");
|
||||
// The same run can retry with a different allocation and stale preparation
|
||||
// input. Its controller lease alone cannot replace already-published dirty files.
|
||||
if (current.dirty && current.location && (current.location.environmentId !== input.lease.environmentId
|
||||
|| current.location.providerLeaseId !== input.lease.providerLeaseId || current.location.leaseId !== input.lease.id
|
||||
|| current.location.remoteCwd !== remoteRoot)) throw new Error("remote_workspace_authority_changed");
|
||||
const authority: RemoteWorkspaceAuthority = { ...current, dirty: true, location: { environmentId: input.lease.environmentId!,
|
||||
leaseId: input.lease.id, providerLeaseId: input.lease.providerLeaseId!, remoteCwd: remoteRoot } };
|
||||
const [lease] = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.id, input.lease.id),
|
||||
eq(environmentLeases.companyId, workspace.companyId))).for("update");
|
||||
if (!lease || lease.status !== "active" || lease.environmentId !== input.lease.environmentId
|
||||
|| lease.providerLeaseId !== authority.location!.providerLeaseId || lease.heartbeatRunId !== input.runId
|
||||
|| (lease.executionWorkspaceId && lease.executionWorkspaceId !== workspace.id)) throw new Error("remote_workspace_lease_changed");
|
||||
const [savedLease] = await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, remoteCwd: remoteRoot, remoteWorkspaceId: workspace.id }, updatedAt: new Date() })
|
||||
.where(eq(environmentLeases.id, lease.id)).returning();
|
||||
const [savedWorkspace] = await tx.update(executionWorkspaces).set({ cwd: remoteRoot, providerRef: remoteRoot,
|
||||
metadata: { ...workspace.metadata, remoteWorkspace: authority,
|
||||
...(recoveredFiles.length ? { remoteWorkspaceRecoveredFiles: recoveredFiles } : {}) }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)).returning();
|
||||
const runReference = reference(workspace.id, authority, "prepared");
|
||||
const issueId = await persistRunReference(tx, workspace.companyId, input.runId, runReference);
|
||||
if (restoredExport) {
|
||||
const [existing] = await tx.select().from(workspaceOperations).where(eq(workspaceOperations.id, restoredExport.operationId)).for("update");
|
||||
if (existing && (existing.companyId !== workspace.companyId || existing.executionWorkspaceId !== workspace.id
|
||||
|| existing.heartbeatRunId !== input.runId || existing.phase !== "remote_workspace_restore")) throw new Error("remote_workspace_restore_receipt_changed");
|
||||
const metadata = { epoch: authority.epoch, snapshotId: initial.snapshotId,
|
||||
exportCleanup: { ...authority.location, stagingRoot: restoredExport.stagingRoot,
|
||||
state: restoredExport.cleanupPending ? "pending" : "cleaned" } };
|
||||
if (existing) await tx.update(workspaceOperations).set({ status: "succeeded", metadata, finishedAt: new Date(), updatedAt: new Date() })
|
||||
.where(eq(workspaceOperations.id, existing.id));
|
||||
else await tx.insert(workspaceOperations).values({ id: restoredExport.operationId, companyId: workspace.companyId,
|
||||
executionWorkspaceId: workspace.id, heartbeatRunId: input.runId, issueId,
|
||||
phase: "remote_workspace_restore", status: "succeeded", metadata, finishedAt: new Date() });
|
||||
}
|
||||
return { workspace: savedWorkspace!, lease: savedLease! as EnvironmentLease, remoteRoot, reference: runReference };
|
||||
});
|
||||
target.remoteCwd = remoteRoot;
|
||||
if (restoredExport?.cleanupPending) await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId,
|
||||
workspaceId: input.workspace.id, location: readRemoteWorkspaceAuthority(prepared.workspace.metadata)!.location!, target,
|
||||
assertOwned: input.assertOwned ?? (async () => undefined) });
|
||||
try {
|
||||
await provisionRemoteWorkspace({ db: input.db, companyId: input.workspace.companyId, workspaceId: input.workspace.id,
|
||||
runId: input.runId, epoch: initial.epoch, remoteCwd: remoteRoot, target, assertOwned: input.assertOwned,
|
||||
commands: [!initial.snapshotId ? input.provisionCommand : null, input.runtimeProvisionCommand] });
|
||||
if (!initial.snapshotId) {
|
||||
await saveRemoteWorkspace({ ...input, workspace: prepared.workspace, lease: prepared.lease, target,
|
||||
assertOwned: input.assertOwned ?? (async () => undefined), boundary: "initial" });
|
||||
const [workspace] = await input.db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.id, input.workspace.id), eq(executionWorkspaces.companyId, input.workspace.companyId)));
|
||||
prepared.workspace = workspace!;
|
||||
prepared.reference = reference(workspace!.id, readRemoteWorkspaceAuthority(workspace!.metadata)!, "prepared");
|
||||
}
|
||||
} catch (error) {
|
||||
// A first save can fail before runtime selection. Its already-published
|
||||
// authority is still a file obligation, independent of native admission.
|
||||
await preserveRemoteWorkspacePreparationFailure(input.db, prepared.lease, { assertOwned: input.assertOwned })
|
||||
.catch(preservationError => logger.error({ err: preservationError, runId: input.runId }, "Remote workspace preparation recovery was deferred; source remains protected"));
|
||||
throw error;
|
||||
}
|
||||
return prepared;
|
||||
}
|
||||
|
||||
/** The workspace lock fences new invocations while completed final authority
|
||||
* retires only attempts whose every registered invocation has settled. */
|
||||
async function retireQuiescedSaves(tx: Transaction, workspace: Workspace, runId: string, epoch: number,
|
||||
location: NonNullable<RemoteWorkspaceAuthority["location"]>) {
|
||||
const authority = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!authority || authority.epoch !== epoch || authority.dirty || authority.writerRunId !== null
|
||||
|| !authority.snapshotId || !sameExportLocation(authority.location, location)) return;
|
||||
const [final] = await tx.select({ id: executionWorkspaceSnapshots.id, generation: executionWorkspaceSnapshots.generation,
|
||||
operationId: workspaceOperations.id, metadata: workspaceOperations.metadata }).from(executionWorkspaceSnapshots)
|
||||
.innerJoin(workspaceOperations, and(eq(workspaceOperations.id, executionWorkspaceSnapshots.sourceOperationId),
|
||||
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
|
||||
eq(workspaceOperations.heartbeatRunId, runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"),
|
||||
eq(workspaceOperations.status, "succeeded"), sql`${workspaceOperations.metadata}->>'boundary' = 'final'`,
|
||||
sql`${workspaceOperations.metadata}->>'epoch' = ${String(epoch)}`))
|
||||
.where(and(eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
|
||||
eq(executionWorkspaceSnapshots.id, authority.snapshotId), eq(executionWorkspaceSnapshots.sourceRunId, runId),
|
||||
eq(executionWorkspaceSnapshots.epoch, epoch), eq(executionWorkspaceSnapshots.status, "committed"))).limit(1);
|
||||
if (!final || final.metadata?.snapshotId !== final.id || !sameExportLocation(final.metadata?.exportCleanup, location)) return;
|
||||
const candidates = await tx.select().from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
|
||||
eq(workspaceOperations.heartbeatRunId, runId), eq(workspaceOperations.phase, "remote_workspace_snapshot"),
|
||||
inArray(workspaceOperations.status, ["running", "pending"]), sql`${workspaceOperations.metadata}->>'epoch' = ${String(epoch)}`)).for("update");
|
||||
for (const candidate of candidates) {
|
||||
const work = readSaveWork(candidate.metadata), generation = Number(candidate.metadata?.generation);
|
||||
if (!work || work.unknown || work.activeIds.length || !work.quiescedAt || candidate.metadata?.boundary === "final"
|
||||
|| !Number.isSafeInteger(generation) || generation >= final.generation
|
||||
|| !sameExportLocation(candidate.metadata?.exportCleanup, location)) continue;
|
||||
const [snapshot] = await tx.select({ status: executionWorkspaceSnapshots.status, generation: executionWorkspaceSnapshots.generation,
|
||||
epoch: executionWorkspaceSnapshots.epoch, runId: executionWorkspaceSnapshots.sourceRunId }).from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id),
|
||||
eq(executionWorkspaceSnapshots.sourceOperationId, candidate.id))).for("update");
|
||||
if (snapshot && (!["pending", "uploaded"].includes(snapshot.status) || snapshot.generation !== generation
|
||||
|| snapshot.epoch !== epoch || snapshot.runId !== runId)) continue;
|
||||
const at = new Date();
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
|
||||
await tx.update(workspaceOperations).set({ status: "failed", finishedAt: at, updatedAt: at,
|
||||
metadata: { ...candidate.metadata, retiredSave: { schema: RETIRED_SAVE_SCHEMA, snapshotId: final.id,
|
||||
operationId: final.operationId, retiredAt: at.toISOString() } } }).where(eq(workspaceOperations.id, candidate.id));
|
||||
}
|
||||
}
|
||||
|
||||
async function settleSaveInvocation(input: { db: Db; workspace: Workspace; runId: string; epoch: number;
|
||||
location: NonNullable<RemoteWorkspaceAuthority["location"]>; operationId: string; invocationId: string }) {
|
||||
return input.db.transaction(async tx => {
|
||||
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
|
||||
const [operation] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.id, input.operationId),
|
||||
eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id),
|
||||
eq(workspaceOperations.heartbeatRunId, input.runId), sql`${workspaceOperations.metadata}->>'epoch' = ${String(input.epoch)}`)).for("update");
|
||||
const work = readSaveWork(operation?.metadata);
|
||||
if (!operation || !work || !work.activeIds.includes(input.invocationId)) return false;
|
||||
const activeIds = work.activeIds.filter(id => id !== input.invocationId);
|
||||
if (!activeIds.length && !work.unknown) await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
|
||||
await tx.update(workspaceOperations).set({ metadata: { ...operation.metadata,
|
||||
saveWork: { ...work, activeIds, quiescedAt: activeIds.length ? null : new Date().toISOString() } }, updatedAt: new Date() })
|
||||
.where(eq(workspaceOperations.id, operation.id));
|
||||
await retireQuiescedSaves(tx, workspace, input.runId, input.epoch, input.location);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export async function saveRemoteWorkspace(input: {
|
||||
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
|
||||
assertOwned: () => Promise<void>; storage?: StorageService; boundary?: string;
|
||||
}) {
|
||||
const target = requireRemoteWorkspaceTarget(input.target), boundary = input.boundary ?? "final";
|
||||
const expected = readRemoteWorkspaceAuthority(input.workspace.metadata);
|
||||
if (!expected?.location || expected.location.providerLeaseId !== input.lease.providerLeaseId || expected.location.leaseId !== input.lease.id) throw new Error("remote_workspace_save_binding_mismatch");
|
||||
await input.assertOwned();
|
||||
const invocationId = randomUUID();
|
||||
const operation = await input.db.transaction(async tx => {
|
||||
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
|
||||
const current = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!current || current.epoch !== expected.epoch || (current.writerRunId !== input.runId && current.dirty)) throw new Error("remote_workspace_authority_changed");
|
||||
const [previous] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, workspace.companyId),
|
||||
eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.heartbeatRunId, input.runId),
|
||||
eq(workspaceOperations.phase, "remote_workspace_snapshot"), sql`${workspaceOperations.metadata}->>'boundary' = ${boundary}`,
|
||||
sql`${workspaceOperations.metadata}->>'epoch' = ${String(current.epoch)}`)).limit(1).for("update");
|
||||
if (previous) {
|
||||
if (object(previous.metadata?.retiredSave).schema === RETIRED_SAVE_SCHEMA) throw new Error("remote_workspace_snapshot_retired");
|
||||
const work = readSaveWork(previous.metadata);
|
||||
const [registered] = await tx.update(workspaceOperations).set({ metadata: { ...previous.metadata,
|
||||
saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [...(work?.activeIds ?? []), invocationId],
|
||||
unknown: work?.unknown ?? true, quiescedAt: null } }, updatedAt: new Date() }).where(eq(workspaceOperations.id, previous.id)).returning();
|
||||
return registered!;
|
||||
}
|
||||
const [run] = await tx.select({ nativeIssueId: heartbeatRuns.nativeIssueId, issueId: heartbeatRuns.issueId,
|
||||
contextSnapshot: heartbeatRuns.contextSnapshot }).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, workspace.companyId), eq(heartbeatRuns.id, input.runId)));
|
||||
if (!run) throw new Error("remote_workspace_run_missing");
|
||||
const issueId = run.nativeIssueId ?? run.issueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null);
|
||||
// The workspace lock serializes generation reservations. Failed/deleted
|
||||
// snapshots still own their generation, and an operation can be committed
|
||||
// before beginWorkspaceSnapshot creates its ledger row. Count both journals
|
||||
// across epochs while retaining the last committed snapshot as the parent.
|
||||
const [reservation] = await tx.execute(sql`select greatest(
|
||||
coalesce((select max(${executionWorkspaceSnapshots.generation}) from ${executionWorkspaceSnapshots}
|
||||
where ${executionWorkspaceSnapshots.companyId} = ${workspace.companyId}
|
||||
and ${executionWorkspaceSnapshots.executionWorkspaceId} = ${workspace.id}), 0),
|
||||
coalesce((select max(case when ${workspaceOperations.metadata}->>'generation' ~ '^[0-9]+$'
|
||||
then (${workspaceOperations.metadata}->>'generation')::numeric else 0 end) from ${workspaceOperations}
|
||||
where ${workspaceOperations.companyId} = ${workspace.companyId}
|
||||
and ${workspaceOperations.executionWorkspaceId} = ${workspace.id}
|
||||
and ${workspaceOperations.phase} = 'remote_workspace_snapshot'), 0)
|
||||
) as generation`);
|
||||
const generation = Number(reservation?.generation) + 1;
|
||||
if (!Number.isSafeInteger(generation) || generation < 1) throw new Error("remote_workspace_snapshot_generation_invalid");
|
||||
const [created] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id,
|
||||
heartbeatRunId: input.runId, issueId, phase: "remote_workspace_snapshot", status: "running",
|
||||
metadata: { boundary, epoch: current.epoch, parentSnapshotId: current.snapshotId, generation,
|
||||
saveWork: { schema: SAVE_WORK_SCHEMA, activeIds: [invocationId], unknown: false, quiescedAt: null },
|
||||
exportCleanup: { ...current.location, state: "pending" } },
|
||||
}).returning();
|
||||
return created!;
|
||||
});
|
||||
let captured: Awaited<ReturnType<typeof captureRemoteWorkspaceArchive>> | null = null;
|
||||
try {
|
||||
const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId, workspaceId: input.workspace.id,
|
||||
operationId: operation.id, runId: input.runId, parentSnapshotId: operation.metadata?.parentSnapshotId as string | null,
|
||||
generation: operation.metadata?.generation as number, epoch: expected.epoch });
|
||||
if (operation.status === "succeeded" && ["committed", "superseded"].includes(snapshot.status)) {
|
||||
return { snapshotId: snapshot.id, generation: snapshot.generation, sha256: snapshot.sha256!, savedAt: snapshot.updatedAt.toISOString() };
|
||||
}
|
||||
const existingUpload = snapshot.status === "uploaded" ? await getWorkspaceSnapshot(input.db, { companyId: input.workspace.companyId,
|
||||
workspaceId: input.workspace.id, snapshotId: snapshot.id }) : null;
|
||||
captured = existingUpload ? null : await captureRemoteWorkspaceArchive({ target, remoteCwd: expected.location.remoteCwd, operationId: operation.id });
|
||||
await input.assertOwned();
|
||||
const saved = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage ?? getStorageService(), snapshotId: snapshot.id,
|
||||
companyId: input.workspace.companyId, workspaceId: input.workspace.id, archivePath: captured?.archivePath ?? "",
|
||||
manifestDigest: existingUpload?.manifestDigest ?? captured!.manifestDigest });
|
||||
await input.assertOwned();
|
||||
const at = new Date();
|
||||
await input.db.transaction(async tx => {
|
||||
const workspace = await currentWorkspace(tx, input.workspace.companyId, input.workspace.id);
|
||||
const current = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
if (!current || current.epoch !== expected.epoch || current.writerRunId !== input.runId || current.snapshotId !== snapshot.parentSnapshotId) {
|
||||
if (current?.epoch === expected.epoch && current.snapshotId === saved.id) return;
|
||||
throw new Error("remote_workspace_publication_fenced");
|
||||
}
|
||||
const authority: RemoteWorkspaceAuthority = { ...current, snapshotId: saved.id,
|
||||
dirty: boundary !== "final", writerRunId: boundary !== "final" ? input.runId : null };
|
||||
const [committed] = await tx.update(executionWorkspaceSnapshots).set({ status: "committed", updatedAt: at }).where(and(
|
||||
eq(executionWorkspaceSnapshots.id, saved.id), eq(executionWorkspaceSnapshots.companyId, workspace.companyId),
|
||||
eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), eq(executionWorkspaceSnapshots.status, "uploaded"))).returning({ id: executionWorkspaceSnapshots.id });
|
||||
if (!committed) throw new Error("remote_workspace_snapshot_publication_changed");
|
||||
await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`);
|
||||
if (boundary === "final") await notifyTerminalWorkspaceWork(tx, `workspace:${workspace.id}`);
|
||||
await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspace: authority }, updatedAt: at }).where(eq(executionWorkspaces.id, workspace.id));
|
||||
await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: at, updatedAt: at,
|
||||
metadata: sql`coalesce(${workspaceOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ snapshotId: saved.id, sha256: saved.sha256 })}::jsonb` })
|
||||
.where(eq(workspaceOperations.id, operation.id));
|
||||
await persistRunReference(tx, workspace.companyId, input.runId, reference(workspace.id, authority, boundary === "final" ? "saved" : "prepared"));
|
||||
await persistActivity(tx as unknown as Db, { companyId: workspace.companyId, actorType: "system", actorId: "remote-workspaces",
|
||||
action: "workspace.snapshot_saved", entityType: "execution_workspace", entityId: workspace.id, runId: input.runId,
|
||||
details: { snapshotId: saved.id, generation: saved.generation, epoch: current.epoch, boundary } });
|
||||
if (snapshot.parentSnapshotId) await tx.update(executionWorkspaceSnapshots).set({ status: "superseded",
|
||||
deleteAfter: new Date(at.getTime() + 7 * 86400_000), updatedAt: at }).where(and(eq(executionWorkspaceSnapshots.id, snapshot.parentSnapshotId),
|
||||
eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id)));
|
||||
});
|
||||
return { snapshotId: saved.id, generation: saved.generation, sha256: saved.sha256, savedAt: at.toISOString() };
|
||||
} finally {
|
||||
// Every upload/capture promise has settled before its durable token clears.
|
||||
// Missing completion evidence remains pinned; cleanup never masks the save error.
|
||||
try { await captured?.cleanup(); }
|
||||
catch { logger.warn({ runId: input.runId, operationId: operation.id }, "Remote workspace local archive cleanup failed"); }
|
||||
try {
|
||||
if (await settleSaveInvocation({ db: input.db, workspace: input.workspace, runId: input.runId, epoch: expected.epoch,
|
||||
location: expected.location, operationId: operation.id, invocationId })) {
|
||||
await cleanupSavedExports({ db: input.db, companyId: input.workspace.companyId, workspaceId: input.workspace.id,
|
||||
location: expected.location, target, assertOwned: input.assertOwned });
|
||||
}
|
||||
} catch { logger.warn({ runId: input.runId, operationId: operation.id }, "Remote workspace save settlement or export cleanup remains pending"); }
|
||||
}
|
||||
}
|
||||
|
||||
export async function prepareRemoteWorkspaceRepository(input: {
|
||||
db: Db; workspace: Workspace; runId: string; lease: EnvironmentLease; target: AdapterExecutionTarget | null;
|
||||
repository: { id: string; relativePath: string; repoUrl: string; ref: string | null; pinnedCommit?: string; readOnly?: boolean };
|
||||
assertOwned?: () => Promise<void>; resolveGitAuth?: ReturnType<typeof createGitRemoteAuthProvider>; verifyOnly?: boolean; storage?: StorageService;
|
||||
}) {
|
||||
const authority = readRemoteWorkspaceAuthority(input.workspace.metadata);
|
||||
if (!authority?.location || authority.writerRunId !== input.runId || authority.location.leaseId !== input.lease.id
|
||||
|| authority.location.providerLeaseId !== input.lease.providerLeaseId) throw new Error("remote_workspace_repository_binding_mismatch");
|
||||
await input.assertOwned?.();
|
||||
const target = requireRemoteWorkspaceTarget(input.target);
|
||||
const result = await prepareRemoteRepository({ target, remoteCwd: remoteWorkspaceRepositoryPath(authority.location.remoteCwd, input.repository),
|
||||
...input.repository, verifyOnly: input.verifyOnly, savedArchive: isSavedWorkspaceRepository(authority, input.repository),
|
||||
resolveGitAuth: input.resolveGitAuth ?? await createRemoteWorkspaceGitAuth(input.db, input.workspace.companyId, input.runId) });
|
||||
await input.assertOwned?.();
|
||||
await saveRemoteWorkspace({ ...input, target, boundary: `repository:${input.repository.id}:${result.pinnedCommit}`,
|
||||
assertOwned: input.assertOwned ?? (async () => undefined) });
|
||||
return result;
|
||||
}
|
||||
|
||||
type SavedWorkspaceFileLimits = {
|
||||
maxOpen: number; maxArchiveBytes: number; maxScratchBytes: number; maxEntries: number; maxManifestBytes: number; freeSpaceMarginBytes: number;
|
||||
};
|
||||
const SAVED_WORKSPACE_FILE_LIMITS: SavedWorkspaceFileLimits = {
|
||||
maxOpen: 2, maxArchiveBytes: 512 * 1024 * 1024, maxScratchBytes: 2 * 1024 * 1024 * 1024,
|
||||
maxEntries: 10_000, maxManifestBytes: 4 * 1024 * 1024, freeSpaceMarginBytes: 256 * 1024 * 1024,
|
||||
};
|
||||
let savedWorkspaceFileOpen = 0, savedWorkspaceFileReservedBytes = 0;
|
||||
|
||||
export function createSavedWorkspaceLoader(db: Db, storage: StorageService = getStorageService(), options: {
|
||||
limits?: Partial<SavedWorkspaceFileLimits>; availableBytes?: () => Promise<bigint>;
|
||||
} = {}) {
|
||||
const limits = { ...SAVED_WORKSPACE_FILE_LIMITS, ...options.limits };
|
||||
for (const key of Object.keys(limits) as Array<keyof SavedWorkspaceFileLimits>) {
|
||||
if (!Number.isSafeInteger(limits[key]) || limits[key] < (key === "freeSpaceMarginBytes" ? 0 : 1)
|
||||
|| limits[key] > SAVED_WORKSPACE_FILE_LIMITS[key]) throw new Error("invalid_saved_workspace_file_limits");
|
||||
}
|
||||
const availableBytes = options.availableBytes ?? (async () => {
|
||||
const disk = await statfs(os.tmpdir(), { bigint: true });
|
||||
return disk.bavail * disk.bsize;
|
||||
});
|
||||
return async (input: { companyId: string; workspaceId: string; snapshotId: string; signal?: AbortSignal }) => {
|
||||
input.signal?.throwIfAborted();
|
||||
const [row] = await db.select().from(executionWorkspaceSnapshots).where(and(eq(executionWorkspaceSnapshots.companyId, input.companyId),
|
||||
eq(executionWorkspaceSnapshots.executionWorkspaceId, input.workspaceId), eq(executionWorkspaceSnapshots.id, input.snapshotId),
|
||||
eq(executionWorkspaceSnapshots.status, "committed")));
|
||||
const saved = row ? await getWorkspaceSnapshot(db, input) : null;
|
||||
input.signal?.throwIfAborted();
|
||||
if (!saved) throw new Error("remote_workspace_snapshot_unavailable");
|
||||
if (saved.byteSize > limits.maxArchiveBytes) throw unprocessable("Saved workspace is too large for Files", { code: "too_large" });
|
||||
// Decryption can hold ciphertext, plaintext, and its atomic publication at
|
||||
// once. Bound filesystem allocation overhead even for many tiny entries.
|
||||
const reservation = 3 * saved.byteSize + limits.maxEntries * 16 * 1024;
|
||||
if (savedWorkspaceFileOpen >= limits.maxOpen || reservation > limits.maxScratchBytes - savedWorkspaceFileReservedBytes) {
|
||||
throw unprocessable("Saved workspace file extraction capacity is unavailable", { code: "workspace_unavailable" });
|
||||
}
|
||||
savedWorkspaceFileOpen++; savedWorkspaceFileReservedBytes += reservation;
|
||||
let scratch: string | undefined, released = false, cleaning: Promise<void> | undefined;
|
||||
// Source modes stay intact while reading. Cleanup changes only private directories and skips symlinks.
|
||||
const cleanup = async () => {
|
||||
if (released) return;
|
||||
if (!cleaning) cleaning = (async () => {
|
||||
if (scratch) await removeTaskDirectoryTree(scratch);
|
||||
released = true; savedWorkspaceFileOpen--; savedWorkspaceFileReservedBytes -= reservation;
|
||||
})();
|
||||
try { await cleaning; } catch (error) { cleaning = undefined; throw error; }
|
||||
};
|
||||
try {
|
||||
// Existing reservations may not have written their bytes yet. Keep them
|
||||
// deducted from free space until their owned directories are removed.
|
||||
if (await availableBytes() < BigInt(savedWorkspaceFileReservedBytes + limits.freeSpaceMarginBytes)) {
|
||||
throw unprocessable("Insufficient temporary disk space for saved workspace Files", { code: "workspace_unavailable" });
|
||||
}
|
||||
input.signal?.throwIfAborted();
|
||||
scratch = await mkdtemp(path.join(os.tmpdir(), "pc-workspace-files-"));
|
||||
const archive = path.join(scratch, "archive.pcws"), root = path.join(scratch, "files");
|
||||
await loadWorkspaceSnapshotArchive({ storage, reference: saved, targetPath: archive, signal: input.signal, temporaryParent: scratch });
|
||||
await restoreRemoteWorkspaceArchive({ archivePath: archive, targetDir: root, expectedManifestDigest: saved.manifestDigest,
|
||||
signal: input.signal, maxEntries: limits.maxEntries, maxManifestBytes: limits.maxManifestBytes });
|
||||
return { root, savedAt: row!.updatedAt.toISOString(), cleanup };
|
||||
} catch (error) {
|
||||
try { await cleanup(); } catch {
|
||||
logger.warn({ workspaceId: input.workspaceId }, "Failed to remove saved workspace extraction; capacity remains reserved");
|
||||
}
|
||||
if (error instanceof Error && ["remote_workspace_archive_entry_limit", "remote_workspace_archive_manifest_too_large"].includes(error.message)) {
|
||||
throw unprocessable("Saved workspace manifest exceeds Files extraction limits", { code: "too_large" });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Bind credentials to the current run's identity, including shared workspace users. */
|
||||
export async function createRemoteWorkspaceGitAuth(db: Db, companyId: string, runId: string,
|
||||
create: typeof createGitRemoteAuthProvider = createGitRemoteAuthProvider) {
|
||||
const [run] = await db.select({ agentId: heartbeatRuns.agentId, responsibleUserId: heartbeatRuns.responsibleUserId,
|
||||
nativeIssueId: heartbeatRuns.nativeIssueId, issueId: heartbeatRuns.issueId, status: heartbeatRuns.status,
|
||||
contextIssueId: sql<unknown>`${heartbeatRuns.contextSnapshot}->'issueId'`,
|
||||
}).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)));
|
||||
const issueId = run?.nativeIssueId ?? run?.issueId ?? run?.contextIssueId;
|
||||
if (!run || run.status !== "running" || typeof issueId !== "string"
|
||||
|| !/^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/i.test(issueId)) throw new Error("remote_workspace_git_run_unavailable");
|
||||
return create(db, companyId, { issueId, heartbeatRunId: runId, agentId: run.agentId,
|
||||
responsibleUserId: run.responsibleUserId === "local-board" ? null : run.responsibleUserId });
|
||||
}
|
||||
|
||||
/** Arbitrary provisioning commands cannot be replayed after an ambiguous reply. */
|
||||
export async function provisionRemoteWorkspace(input: {
|
||||
db: Db; companyId: string; workspaceId: string; runId: string; epoch: number;
|
||||
remoteCwd: string; target: ReturnType<typeof requireRemoteWorkspaceTarget>; commands: Array<string | null | undefined>;
|
||||
assertOwned?: () => Promise<void>;
|
||||
}) {
|
||||
for (const [index, command] of input.commands.entries()) {
|
||||
if (!command?.trim()) continue;
|
||||
await input.assertOwned?.();
|
||||
const digest = createHash("sha256").update(command).digest("hex");
|
||||
const operation = await input.db.transaction(async tx => {
|
||||
const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
|
||||
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
const [run] = await tx.select({ status: heartbeatRuns.status }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId)));
|
||||
if (!authority || authority.epoch !== input.epoch || authority.writerRunId !== input.runId || run?.status !== "running") throw new Error("remote_workspace_provision_fenced");
|
||||
const [existing] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, input.companyId),
|
||||
eq(workspaceOperations.executionWorkspaceId, input.workspaceId), eq(workspaceOperations.heartbeatRunId, input.runId),
|
||||
eq(workspaceOperations.phase, "remote_workspace_provision"), sql`${workspaceOperations.metadata}->>'index' = ${String(index)}`,
|
||||
sql`${workspaceOperations.metadata}->>'epoch' = ${String(input.epoch)}`)).limit(1);
|
||||
if (existing) {
|
||||
if (existing.metadata?.commandSha256 !== digest || existing.status !== "succeeded") throw new Error("remote_workspace_provision_unverified");
|
||||
return null;
|
||||
}
|
||||
const [created] = await tx.insert(workspaceOperations).values({ companyId: input.companyId, executionWorkspaceId: input.workspaceId,
|
||||
heartbeatRunId: input.runId, issueId: workspace!.sourceIssueId, phase: "remote_workspace_provision", status: "running",
|
||||
metadata: { epoch: input.epoch, index, commandSha256: digest, uncertain: true } }).returning();
|
||||
return created!;
|
||||
});
|
||||
if (!operation) continue;
|
||||
await input.assertOwned?.();
|
||||
const result = await input.target.runner!.execute({ command: "bash", args: ["-lc", command], cwd: input.remoteCwd,
|
||||
timeoutMs: 300_000, bypassSession: true });
|
||||
if (result.exitCode !== 0 || result.timedOut) throw new Error("remote_workspace_provision_unverified");
|
||||
await input.assertOwned?.();
|
||||
await input.db.transaction(async tx => {
|
||||
const [workspace] = await tx.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces)
|
||||
.where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update");
|
||||
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
if (authority?.epoch !== input.epoch || authority.writerRunId !== input.runId) throw new Error("remote_workspace_provision_fenced");
|
||||
await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(),
|
||||
metadata: { ...operation.metadata, uncertain: false } }).where(eq(workspaceOperations.id, operation.id));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1358,6 +1358,7 @@ export function secretService(db: Db | DbTransaction) {
|
||||
secretId: string,
|
||||
version: number | "latest",
|
||||
options?: SecretResolutionOptions,
|
||||
readOnlyEnvironmentInspection = false,
|
||||
): Promise<RuntimeSecretResolution> {
|
||||
const bindingContext = options?.bindingContext;
|
||||
const accessContext = options?.accessContext ?? bindingContext;
|
||||
@@ -1406,7 +1407,7 @@ export function secretService(db: Db | DbTransaction) {
|
||||
version: resolvedVersion,
|
||||
},
|
||||
});
|
||||
await Promise.all([
|
||||
if (!readOnlyEnvironmentInspection) await Promise.all([
|
||||
db
|
||||
.update(companySecrets)
|
||||
.set({ lastResolvedAt: new Date(), updatedAt: new Date() })
|
||||
@@ -1442,7 +1443,7 @@ export function secretService(db: Db | DbTransaction) {
|
||||
};
|
||||
} catch (err) {
|
||||
const errorCode = secretResolutionErrorCode(err);
|
||||
await recordAccessEvent({
|
||||
if (!readOnlyEnvironmentInspection) await recordAccessEvent({
|
||||
companyId,
|
||||
secretId: secret.id,
|
||||
userSecretDefinitionId: secret.userSecretDefinitionId ?? null,
|
||||
@@ -1478,6 +1479,23 @@ export function secretService(db: Db | DbTransaction) {
|
||||
return (await resolveSecretValueInternal(companyId, secretId, version, options)).value;
|
||||
}
|
||||
|
||||
// Cloud consumes and audits the signed inspection permit before the tenant
|
||||
// opens its read-only transaction. Keep the ordinary environment binding and
|
||||
// version checks, without attempting tenant audit writes inside that transaction.
|
||||
async function resolveSecretValueForReadOnlyEnvironmentInspection(
|
||||
companyId: string,
|
||||
secretId: string,
|
||||
version: number | "latest",
|
||||
context: { environmentId: string; configPath: string; issueId?: string | null; heartbeatRunId?: string | null },
|
||||
): Promise<string> {
|
||||
if (!context.environmentId || !context.configPath) throw unprocessable("Environment secret binding required");
|
||||
return (await resolveSecretValueInternal(companyId, secretId, version, {
|
||||
bindingContext: { consumerType: "environment", consumerId: context.environmentId,
|
||||
actorType: "system", actorId: null, configPath: context.configPath,
|
||||
issueId: context.issueId ?? null, heartbeatRunId: context.heartbeatRunId ?? null },
|
||||
}, true)).value;
|
||||
}
|
||||
|
||||
async function resolveSecretValueForEphemeralAccess(
|
||||
companyId: string,
|
||||
secretId: string,
|
||||
@@ -4618,6 +4636,7 @@ export function secretService(db: Db | DbTransaction) {
|
||||
resolveSecretValueForAgentAccess,
|
||||
listAgentSecretAccess,
|
||||
resolveSecretValueForEphemeralAccess,
|
||||
resolveSecretValueForReadOnlyEnvironmentInspection,
|
||||
resolveSecretValueForSandboxCleanup,
|
||||
resolveSecretValueForDeviceLoginCheck,
|
||||
|
||||
|
||||
@@ -154,7 +154,7 @@ describe.skipIf(!support.supported)("automatic skill package work", () => {
|
||||
let signalStarted: () => void = () => {};
|
||||
const started = new Promise<void>(resolve => { signalStarted = resolve; });
|
||||
let pendingSignal: AbortSignal | undefined;
|
||||
vi.spyOn(provider, method).mockImplementation(async input => {
|
||||
vi.spyOn(provider, method).mockImplementation(async (input: { signal?: AbortSignal }) => {
|
||||
pendingSignal = input.signal;
|
||||
signalStarted();
|
||||
return new Promise<never>(() => {});
|
||||
|
||||
@@ -5,6 +5,9 @@ export type UnsafeWorkspaceExportScope = { companyId: string; issueId?: string }
|
||||
export async function notifyUnsafeWorkspaceExportRecovery(transaction: object, scope: UnsafeWorkspaceExportScope) {
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.unsafeWorkspaceExportRecovery,
|
||||
JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }));
|
||||
// Recovery settlement can release a retained remote workspace snapshot.
|
||||
await notifyDeliveryWork(transaction, DELIVERY_QUEUES.remoteWorkspaceSnapshots,
|
||||
scope.issueId ? `issue:${scope.issueId}` : `company:${scope.companyId}`);
|
||||
}
|
||||
|
||||
export async function notifyUnsafeWorkspaceExportRunChange(transaction: object, run: {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { issueReadSqlCondition, projectReadSqlCondition, type AuthorizationActor } from "./authorization.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import { and, desc, eq, inArray, isNull } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { executionWorkspaces, issues, projects, projectWorkspaces } from "@paperclipai/db";
|
||||
@@ -17,7 +18,9 @@ import type {
|
||||
WorkspaceFilePreviewKind,
|
||||
WorkspaceFileSelector,
|
||||
WorkspaceFileWorkspaceKind,
|
||||
WorkspaceFileSource,
|
||||
} from "@paperclipai/shared";
|
||||
import type { RemoteWorkspaceFiles, RemoteWorkspaceFileAccess, RemoteWorkspaceFileEntry } from "./remote-workspace-files.js";
|
||||
import { HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import {
|
||||
isWorkspaceGitScanError,
|
||||
@@ -31,6 +34,7 @@ export const WORKSPACE_FILE_LIST_DEFAULT_LIMIT = 25;
|
||||
export const WORKSPACE_FILE_LIST_MAX_LIMIT = 100;
|
||||
export const WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES = 5_000;
|
||||
export const WORKSPACE_FILE_AVAILABILITY_CONCURRENCY = 8;
|
||||
const MAX_AVAILABILITY_REMOTE_WORKSPACES = 8;
|
||||
const MAX_RELATIVE_PATH_BYTES = 4096;
|
||||
const TEXT_SNIFF_BYTES = 4096;
|
||||
const MAX_LIST_DEPTH = 20;
|
||||
@@ -122,6 +126,8 @@ type WorkspaceCandidate = {
|
||||
label: string;
|
||||
rootPath: string | null;
|
||||
remote: boolean;
|
||||
remoteAuthority?: boolean;
|
||||
source?: WorkspaceFileSource;
|
||||
};
|
||||
|
||||
type NormalizedPath = {
|
||||
@@ -134,6 +140,12 @@ type LocalResolvedFile = {
|
||||
realPath: string;
|
||||
};
|
||||
|
||||
export type PreparedWorkspaceFileDownload = LocalResolvedFile | {
|
||||
resource: ResolvedWorkspaceResource;
|
||||
stream: Readable;
|
||||
cleanup(): Promise<void>;
|
||||
};
|
||||
|
||||
type LocalResolvedDirectory = {
|
||||
resource: ResolvedWorkspaceResource;
|
||||
realPath: string;
|
||||
@@ -266,14 +278,21 @@ function unavailableAvailabilityResult(
|
||||
async function mapWithConcurrency<T, R>(items: T[], concurrency: number, mapper: (item: T) => Promise<R>) {
|
||||
const results = new Array<R>(items.length);
|
||||
let nextIndex = 0;
|
||||
let failure: { error: unknown } | undefined;
|
||||
async function worker() {
|
||||
while (nextIndex < items.length) {
|
||||
while (nextIndex < items.length && !failure) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
results[index] = await mapper(items[index]!);
|
||||
try {
|
||||
results[index] = await mapper(items[index]!);
|
||||
} catch (error) {
|
||||
failure ??= { error };
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every started mapper must settle before its request-owned resources close.
|
||||
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
|
||||
if (failure) throw failure.error;
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -370,7 +389,7 @@ function listItemFromStat(input: {
|
||||
candidate: WorkspaceCandidate;
|
||||
relativePath: string;
|
||||
stat: { size: number; mtime: Date };
|
||||
}): WorkspaceFileListItem | null {
|
||||
}): WorkspaceFileListItem {
|
||||
const contentType = contentTypeForPath(input.relativePath);
|
||||
const previewKind = previewKindForKnownContentType(contentType) ?? "unsupported";
|
||||
const previewable = previewKind !== "unsupported" && input.stat.size <= previewCapForKind(previewKind);
|
||||
@@ -473,7 +492,7 @@ function remoteResource(candidate: WorkspaceCandidate, relativePath: string): Re
|
||||
function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceCandidate {
|
||||
const provider = row.providerType || row.strategyType || "local_fs";
|
||||
const rootPath = row.cwd || row.providerRef || null;
|
||||
const remote = !["local_fs", "git_worktree"].includes(provider) || !rootPath || row.status !== "active" || row.closedAt != null;
|
||||
const remote = Boolean(row.metadata?.remoteWorkspace) || !["local_fs", "git_worktree"].includes(provider) || !rootPath || row.status !== "active" || row.closedAt != null;
|
||||
return {
|
||||
companyId: row.companyId,
|
||||
workspaceKind: "execution_workspace",
|
||||
@@ -482,6 +501,7 @@ function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceC
|
||||
label: row.name,
|
||||
rootPath,
|
||||
remote,
|
||||
remoteAuthority: Boolean(row.metadata?.remoteWorkspace),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -680,6 +700,7 @@ function unavailableFileList(input: {
|
||||
unavailableReason: input.reason,
|
||||
workspace: input.candidate
|
||||
? {
|
||||
...(input.candidate.source ? { source: input.candidate.source } : {}),
|
||||
provider: input.candidate.provider,
|
||||
workspaceLabel: input.candidate.label,
|
||||
workspaceKind: input.candidate.workspaceKind,
|
||||
@@ -718,6 +739,7 @@ function availableFileList(input: {
|
||||
kind: "workspace_file_list",
|
||||
state: "available",
|
||||
workspace: {
|
||||
...(input.candidate.source ? { source: input.candidate.source } : {}),
|
||||
provider: input.candidate.provider,
|
||||
workspaceLabel: input.candidate.label,
|
||||
workspaceKind: input.candidate.workspaceKind,
|
||||
@@ -1041,7 +1063,42 @@ async function listChangedWorkspaceFiles(input: {
|
||||
};
|
||||
}
|
||||
|
||||
export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor) {
|
||||
export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor, options: { remoteFiles?: RemoteWorkspaceFiles } = {}) {
|
||||
async function remoteAccess(candidate: WorkspaceCandidate, signal?: AbortSignal) {
|
||||
if (!options.remoteFiles || candidate.workspaceKind !== "execution_workspace") return null;
|
||||
return options.remoteFiles.open({ companyId: candidate.companyId, workspaceId: candidate.workspaceId, signal });
|
||||
}
|
||||
|
||||
function remoteEntryResource(candidate: WorkspaceCandidate, entry: RemoteWorkspaceFileEntry, access: RemoteWorkspaceFileAccess): ResolvedWorkspaceResource {
|
||||
if (entry.kind === "directory") return { ...directoryResource({ candidate, relativePath: entry.relativePath }), source: access.source };
|
||||
const contentType = contentTypeForPath(entry.relativePath);
|
||||
const previewKind = previewKindForKnownContentType(contentType) ?? (entry.text ? "text" : "unsupported");
|
||||
const tooLarge = entry.byteSize! > previewCapForKind(previewKind);
|
||||
return {
|
||||
kind: "file", provider: candidate.provider, title: path.posix.basename(entry.relativePath),
|
||||
displayPath: candidate.projectName ? `${candidate.projectName} / ${entry.relativePath}` : entry.relativePath,
|
||||
workspaceLabel: candidate.label, workspaceKind: candidate.workspaceKind, workspaceId: candidate.workspaceId,
|
||||
projectId: candidate.projectId ?? null, projectName: candidate.projectName ?? null,
|
||||
contentType: contentType ?? (previewKind === "text" ? "text/plain; charset=utf-8" : "application/octet-stream"),
|
||||
byteSize: entry.byteSize, previewKind, source: access.source,
|
||||
denialReason: tooLarge ? "too_large" : previewKind === "unsupported" ? "unsupported_content" : null,
|
||||
capabilities: { preview: !tooLarge && previewKind !== "unsupported", download: true, listChildren: false },
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveRemote(candidate: WorkspaceCandidate, normalized: NormalizedPath, directory: boolean,
|
||||
requestAccess?: (candidate: WorkspaceCandidate) => Promise<RemoteWorkspaceFileAccess | null>) {
|
||||
throwIfDenied(normalized.segments);
|
||||
const access = await (requestAccess ?? remoteAccess)(candidate);
|
||||
if (!access) return null;
|
||||
try {
|
||||
const entry = await access.stat(normalized.relativePath);
|
||||
if (directory && entry.kind !== "directory") throw unprocessable("Workspace path is not a directory", { code: "not_directory" });
|
||||
if (!directory && entry.kind !== "file") throw unprocessable("Workspace file is not a regular file", { code: "not_regular_file" });
|
||||
return remoteEntryResource(candidate, entry, access);
|
||||
} finally { if (!requestAccess) await access.close(); }
|
||||
}
|
||||
|
||||
async function visibleCandidates(candidates: WorkspaceCandidate[], skipDeniedSources = false, task?: IssueRow) {
|
||||
if (!actor) return candidates;
|
||||
const visible: WorkspaceCandidate[] = [];
|
||||
@@ -1068,7 +1125,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
}
|
||||
candidate.label = "Task workspace";
|
||||
}
|
||||
if (candidate.rootPath && (!candidate.remote || candidate.provider === "local_fs" || LOCAL_PROJECT_WORKSPACE_SOURCE_TYPES.has(candidate.provider))) {
|
||||
if (candidate.rootPath && !candidate.remoteAuthority && (!candidate.remote || candidate.provider === "local_fs" || LOCAL_PROJECT_WORKSPACE_SOURCE_TYPES.has(candidate.provider))) {
|
||||
try {
|
||||
const canonicalCwd = await fs.realpath(candidate.rootPath);
|
||||
|
||||
@@ -1323,8 +1380,22 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
? await sameCompanyProjectWorkspaceCandidates(issue, new Set(initialCandidates.map((candidate) => candidate.workspaceId)))
|
||||
: [];
|
||||
const explicitTargets = await loadAvailabilityTargets(issue, queries);
|
||||
const remoteAccesses = new Map<string, Promise<RemoteWorkspaceFileAccess | null>>();
|
||||
const requestAccess = (candidate: WorkspaceCandidate) => {
|
||||
// UUIDs can be equal across workspace tables; project roots never borrow task access.
|
||||
if (!options.remoteFiles || candidate.workspaceKind !== "execution_workspace") return Promise.resolve(null);
|
||||
const key = `${candidate.companyId}:${candidate.workspaceId}`;
|
||||
const existing = remoteAccesses.get(key);
|
||||
if (existing) return existing;
|
||||
if (remoteAccesses.size >= MAX_AVAILABILITY_REMOTE_WORKSPACES) {
|
||||
throw unprocessable("Too many remote workspaces in one availability request", { code: "workspace_unavailable" });
|
||||
}
|
||||
const access = remoteAccess(candidate);
|
||||
remoteAccesses.set(key, access);
|
||||
return access;
|
||||
};
|
||||
|
||||
const results = await mapWithConcurrency(
|
||||
const outcome = await mapWithConcurrency(
|
||||
queries,
|
||||
WORKSPACE_FILE_AVAILABILITY_CONCURRENCY,
|
||||
async (item): Promise<WorkspaceFileAvailabilityResult> => {
|
||||
@@ -1349,6 +1420,16 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.remote) {
|
||||
try {
|
||||
const resource = await resolveRemote(candidate, item.normalizedPath, item.directory, requestAccess);
|
||||
if (resource) return availabilityResult(item.query, resource);
|
||||
} catch (error) {
|
||||
if (!hasExplicitTarget && item.query.workspace === "auto" && isHttpStatus(error, 404)) {
|
||||
lastNotFound = error;
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (hasExplicitTarget || item.query.workspace !== "auto") {
|
||||
return availabilityResult(item.query, remoteResource(candidate, item.normalizedPath.relativePath));
|
||||
}
|
||||
@@ -1393,9 +1474,16 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
return unavailableAvailabilityResult(item.query, reason);
|
||||
}
|
||||
},
|
||||
);
|
||||
).then(results => ({ results }), (error: unknown) => ({ error }));
|
||||
|
||||
return { kind: "workspace_file_availability", results };
|
||||
const opened = await Promise.allSettled(remoteAccesses.values());
|
||||
const closed = await Promise.allSettled(opened.map(async result => {
|
||||
if (result.status === "fulfilled") await result.value?.close();
|
||||
}));
|
||||
if ("error" in outcome) throw outcome.error;
|
||||
const cleanupFailure = closed.find(result => result.status === "rejected");
|
||||
if (cleanupFailure?.status === "rejected") throw cleanupFailure.reason;
|
||||
return { kind: "workspace_file_availability", results: outcome.results };
|
||||
}
|
||||
|
||||
async function resolve(issueId: string, input: {
|
||||
@@ -1417,6 +1505,16 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.remote) {
|
||||
try {
|
||||
const resource = await resolveRemote(candidate, normalized, isDirectoryRequest);
|
||||
if (resource) return resource;
|
||||
} catch (error) {
|
||||
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) {
|
||||
lastNotFound = error;
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (explicitTarget || selector !== "auto") return remoteResource(candidate, normalized.relativePath);
|
||||
continue;
|
||||
}
|
||||
@@ -1475,6 +1573,33 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.remote) {
|
||||
if (normalizedPath) throwIfDenied(normalizedPath.segments);
|
||||
const access = await remoteAccess(candidate, opts.scanContext?.signal);
|
||||
if (access) {
|
||||
try {
|
||||
const listed = await access.list({ relativePath: normalizedPath?.relativePath, mode, query: normalizedQuery,
|
||||
limit, offset, maxScannedEntries: WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES, maxDepth: MAX_LIST_DEPTH });
|
||||
const remoteCandidate = { ...candidate, source: access.source };
|
||||
const items = listed.entries.filter(entry => {
|
||||
try { return !shouldPruneSegments(normalizeWorkspaceRelativePath(entry.relativePath).segments); }
|
||||
catch { return false; }
|
||||
}).map(entry => ({
|
||||
...(entry.kind === "directory"
|
||||
? listItemFromDirectory({ candidate, relativePath: entry.relativePath, stat: entry.modifiedAt ? { mtime: new Date(entry.modifiedAt) } : undefined })
|
||||
: listItemFromStat({ candidate, relativePath: entry.relativePath, stat: { size: entry.byteSize!, mtime: new Date(entry.modifiedAt ?? 0) } })),
|
||||
source: access.source,
|
||||
}));
|
||||
return availableFileList({ selector, mode, path: normalizedPath?.relativePath ?? null, q, limit, offset,
|
||||
candidate: remoteCandidate, items, scannedCount: listed.scannedCount, truncated: listed.truncated });
|
||||
} catch (error) {
|
||||
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) { lastNotFound = error; continue; }
|
||||
if (error instanceof HttpError && error.status === 422 && !normalizedPath) {
|
||||
return unavailableFileList({ selector, mode, path: null, q, limit, offset,
|
||||
candidate: { ...candidate, source: access.source }, reason: expectedAvailabilityReason(error) ?? "workspace_unavailable" });
|
||||
}
|
||||
throw error;
|
||||
} finally { await access.close(); }
|
||||
}
|
||||
firstUnavailable ??= { candidate, reason: "remote_workspace" };
|
||||
if (explicitTarget || selector !== "auto") {
|
||||
return unavailableFileList({
|
||||
@@ -1681,6 +1806,25 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.remote) {
|
||||
throwIfDenied(normalized.segments);
|
||||
const access = await remoteAccess(candidate);
|
||||
if (access) {
|
||||
try {
|
||||
const resource = remoteEntryResource(candidate, await access.stat(normalized.relativePath), access);
|
||||
if (resource.kind !== "file" || !resource.capabilities.preview) {
|
||||
throw unprocessable("Workspace file cannot be previewed", { code: resource.denialReason ?? "not_regular_file" });
|
||||
}
|
||||
const data = await access.read(normalized.relativePath, { maxBytes: previewCapForKind(resource.previewKind) });
|
||||
if (resource.previewKind === "text" && !looksLikeText(data.subarray(0, Math.min(data.length, TEXT_SNIFF_BYTES)))) {
|
||||
throw unprocessable("Workspace file is not a text file", { code: "binary_content" });
|
||||
}
|
||||
return { resource, content: { encoding: resource.previewKind === "text" ? "utf8" : "base64",
|
||||
data: resource.previewKind === "text" ? data.toString("utf8") : data.toString("base64") } };
|
||||
} catch (error) {
|
||||
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) { lastNotFound = error; continue; }
|
||||
throw error;
|
||||
} finally { await access.close(); }
|
||||
}
|
||||
if (explicitTarget || selector !== "auto") {
|
||||
throw unprocessable("Remote workspaces cannot be previewed by the server", { code: "remote_workspace" });
|
||||
}
|
||||
@@ -1752,7 +1896,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
workspace?: WorkspaceFileSelector | null;
|
||||
projectId?: string | null;
|
||||
workspaceId?: string | null;
|
||||
}, opts: { issue?: IssueRow } = {}): Promise<LocalResolvedFile> {
|
||||
}, opts: { issue?: IssueRow } = {}): Promise<PreparedWorkspaceFileDownload> {
|
||||
const issue = opts.issue ?? await getIssue(issueId);
|
||||
const selector = input.workspace ?? "auto";
|
||||
const explicitTarget = Boolean(input.projectId || input.workspaceId);
|
||||
@@ -1765,6 +1909,21 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.remote) {
|
||||
throwIfDenied(normalized.segments);
|
||||
const access = await remoteAccess(candidate);
|
||||
if (access) {
|
||||
try {
|
||||
const resource = remoteEntryResource(candidate, await access.stat(normalized.relativePath), access);
|
||||
if (resource.kind !== "file") throw unprocessable("Workspace file is not a regular file", { code: "not_regular_file" });
|
||||
const transfer = await access.download(normalized.relativePath);
|
||||
return { resource: { ...resource, byteSize: transfer.byteSize }, stream: transfer.stream,
|
||||
cleanup: async () => { try { transfer.stream.destroy(); await transfer.cleanup(); } finally { await access.close(); } } };
|
||||
} catch (error) {
|
||||
await access.close();
|
||||
if (!explicitTarget && selector === "auto" && isHttpStatus(error, 404)) { lastNotFound = error; continue; }
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (explicitTarget || selector !== "auto") {
|
||||
throw unprocessable("Remote workspaces cannot be downloaded by the server", { code: "remote_workspace" });
|
||||
}
|
||||
|
||||
@@ -254,6 +254,8 @@ export interface WorkspaceOperationRecorder {
|
||||
* timeout error is rethrown, so a hung provider can never leave an active operation.
|
||||
*/
|
||||
timeoutMs?: number | null;
|
||||
/** Publish the running row under a caller-owned admission fence before run starts. */
|
||||
admit?: (publish: () => Promise<void>) => Promise<void>;
|
||||
run: (reportProgress: (input: {
|
||||
metadata?: Record<string, unknown> | null;
|
||||
system?: string | null;
|
||||
@@ -313,6 +315,12 @@ export function workspaceOperationService(db: Db) {
|
||||
const live: WorkspaceOperation[] = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
// Migration owns the workspace until its explicit publication/repair. It
|
||||
// has no runtime-control action stamp and must never be swept as abandoned.
|
||||
if (candidate.phase === "remote_workspace_migration") {
|
||||
live.push(toWorkspaceOperation(candidate));
|
||||
continue;
|
||||
}
|
||||
const action = readRuntimeControlAction(candidate.metadata);
|
||||
if (!action) continue;
|
||||
|
||||
@@ -543,7 +551,8 @@ export function workspaceOperationService(db: Db) {
|
||||
if (runtimeControlAction) liveRuntimeControlOperationIds.add(id);
|
||||
|
||||
try {
|
||||
await writeState(database => database.insert(workspaceOperations).values({
|
||||
let published = false;
|
||||
const publish = async () => { await writeState(database => database.insert(workspaceOperations).values({
|
||||
id,
|
||||
companyId: input.companyId,
|
||||
executionWorkspaceId,
|
||||
@@ -561,7 +570,10 @@ export function workspaceOperationService(db: Db) {
|
||||
) as Record<string, unknown> | null,
|
||||
startedAt,
|
||||
updatedAt: startedAt,
|
||||
}));
|
||||
})); published = true; };
|
||||
if (recordInput.admit) await recordInput.admit(publish);
|
||||
else await publish();
|
||||
if (!published) throw new Error("Workspace operation admission did not publish its running row");
|
||||
} catch (insertError) {
|
||||
liveRuntimeControlOperationIds.delete(id);
|
||||
throw insertError;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import type { ExecutionBlocker } from "@paperclipai/shared";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueRecoveryActions, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type ExecutionBlocker } from "@paperclipai/shared";
|
||||
import { z } from "zod";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
|
||||
|
||||
@@ -40,16 +40,34 @@ export async function verifiedWorkspaceRepairReceipt(db: Db, run: Pick<typeof he
|
||||
"id" | "companyId" | "nativeIssueId" | "issueId" | "contextSnapshot" | "runnerProfileJson">) {
|
||||
const issueId = run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId;
|
||||
const reference = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
|
||||
if (typeof issueId !== "string" || !reference || reference.state !== "finalized" || !reference.finalHostSha256) return null;
|
||||
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
|
||||
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
|
||||
if ((run.runnerProfileJson?.remoteWorkspace != null && !parsedRemote.success) || reference && parsedRemote.success) return null;
|
||||
if (typeof issueId !== "string" || (!remote && (!reference || reference.state !== "finalized" || !reference.finalHostSha256))) return null;
|
||||
const [receipt] = await db.select({ id: workspaceOperations.id, metadata: workspaceOperations.metadata }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, run.companyId), eq(workspaceOperations.issueId, issueId),
|
||||
eq(workspaceOperations.heartbeatRunId, run.id), eq(workspaceOperations.phase, "workspace_recovery"),
|
||||
eq(workspaceOperations.status, "succeeded"),
|
||||
remote ? sql`${workspaceOperations.metadata}->>'snapshotId' = ${remote.snapshotId}` : undefined,
|
||||
)).limit(1);
|
||||
const proof = receipt?.metadata;
|
||||
if (!proof || proof.leaseId !== reference.leaseId || proof.providerLeaseId !== reference.providerLeaseId
|
||||
|| proof.descriptorSha256 !== reference.descriptorSha256 || proof.finalHostSha256 !== reference.finalHostSha256
|
||||
|| typeof proof.recoveryActionId !== "string") return null;
|
||||
if (!proof || typeof proof.recoveryActionId !== "string") return null;
|
||||
if (remote) {
|
||||
const [workspace] = await db.select({ metadata: executionWorkspaces.metadata }).from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, run.companyId), eq(executionWorkspaces.id, remote.workspaceId),
|
||||
));
|
||||
const authority = readRemoteWorkspaceAuthority(workspace?.metadata);
|
||||
if (!authority || authority.dirty || authority.writerRunId || authority.epoch !== remote.epoch
|
||||
|| authority.snapshotId !== remote.snapshotId || proof.storage !== "remote_workspace"
|
||||
|| proof.leaseId !== remote.leaseId || proof.providerLeaseId !== remote.providerLeaseId || proof.epoch !== remote.epoch) return null;
|
||||
const [snapshot] = await db.select().from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, run.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, remote.workspaceId),
|
||||
eq(executionWorkspaceSnapshots.id, remote.snapshotId!), eq(executionWorkspaceSnapshots.status, "committed"),
|
||||
eq(executionWorkspaceSnapshots.epoch, remote.epoch), eq(executionWorkspaceSnapshots.sourceRunId, run.id),
|
||||
));
|
||||
if (!snapshot || snapshot.sha256 !== proof.sha256 || snapshot.generation !== proof.generation) return null;
|
||||
} else if (!reference || proof.leaseId !== reference.leaseId || proof.providerLeaseId !== reference.providerLeaseId
|
||||
|| proof.descriptorSha256 !== reference.descriptorSha256 || proof.finalHostSha256 !== reference.finalHostSha256) return null;
|
||||
const [action] = await db.select({ evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, run.companyId), eq(issueRecoveryActions.sourceIssueId, issueId),
|
||||
eq(issueRecoveryActions.id, proof.recoveryActionId),
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { retiredReusableLeasePredecessors } from "./environment-lease-handoff.js";
|
||||
import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
|
||||
import { environmentLeases, environments, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { environmentLeases, environments, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db";
|
||||
import { adapterExecutionControls } from "./adapter-execution-control.js";
|
||||
import { readNativeWorkspaceSyncReference } from "./native-runtime/native-workspace-sync.js";
|
||||
import { SystemRecoveryFailure } from "./system-recovery-state.js";
|
||||
import type { NativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js";
|
||||
import { readStopOnlyCleanup } from "./sandbox-stop-and-retain.js";
|
||||
import { ENVIRONMENT_LEASE_POLICIES, type EnvironmentLeasePolicy,
|
||||
import { ENVIRONMENT_LEASE_POLICIES, readRemoteWorkspaceAuthority, remoteWorkspaceRunReferenceSchema, type EnvironmentLeasePolicy,
|
||||
type WorkspaceSourceVerificationFailure } from "@paperclipai/shared";
|
||||
|
||||
export const workspaceExportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"];
|
||||
@@ -81,19 +81,42 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
|
||||
const rawReference = run.runnerProfileJson?.nativeWorkspaceSync;
|
||||
const reference = readNativeWorkspaceSyncReference(rawReference);
|
||||
if (rawReference != null && !reference) throw invalid("native_reference_invalid");
|
||||
const rawRemoteReference = run.runnerProfileJson?.remoteWorkspace;
|
||||
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(rawRemoteReference);
|
||||
const remoteReference = parsedRemote.success ? parsedRemote.data : null;
|
||||
if ((rawRemoteReference != null && !remoteReference) || (reference && remoteReference)) throw invalid("native_reference_invalid");
|
||||
const recoveryReference = reference ?? remoteReference;
|
||||
const [remoteWorkspace] = remoteReference ? await db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, remoteReference.workspaceId),
|
||||
)) : [];
|
||||
const remoteAuthority = readRemoteWorkspaceAuthority(remoteWorkspace?.metadata);
|
||||
if (remoteReference && (!remoteWorkspace || !remoteAuthority?.location || remoteAuthority.epoch !== remoteReference.epoch
|
||||
|| remoteAuthority.location.leaseId !== remoteReference.leaseId || remoteAuthority.location.providerLeaseId !== remoteReference.providerLeaseId
|
||||
|| remoteAuthority.location.remoteCwd !== remoteReference.remoteCwd
|
||||
|| (remoteReference.state === "prepared" ? !remoteAuthority.dirty || remoteAuthority.writerRunId !== run.id
|
||||
: remoteAuthority.dirty || remoteAuthority.writerRunId !== null || remoteAuthority.snapshotId !== remoteReference.snapshotId))) {
|
||||
throw invalid("workspace_reference_mismatch");
|
||||
}
|
||||
if (remoteReference?.state === "saved") {
|
||||
const [snapshot] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and(
|
||||
eq(executionWorkspaceSnapshots.companyId, input.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, remoteReference.workspaceId),
|
||||
eq(executionWorkspaceSnapshots.id, remoteReference.snapshotId!), eq(executionWorkspaceSnapshots.status, "committed"),
|
||||
));
|
||||
if (!snapshot) throw invalid("workspace_reference_mismatch");
|
||||
}
|
||||
const recorded = run.resultJson?.workspaceRestoreRecovery as { schema?: unknown; leaseIds?: unknown } | undefined;
|
||||
if (recorded != null && recorded.schema !== "paperclip.workspace-restore-recovery.v1") throw invalid("restore_reference_invalid");
|
||||
const historical = action.evidence.historicalWorkspaceSource as { schema?: unknown; runId?: unknown; leaseIds?: unknown } | undefined;
|
||||
if (historical != null && (historical.schema !== "paperclip.historical-workspace-source.v1" || historical.runId !== run.id)) throw invalid("historical_reference_invalid");
|
||||
const leaseIds = recorded ? recorded.leaseIds
|
||||
: reference && workspaceExportCauses.includes(action.cause) ? [reference.leaseId]
|
||||
: recoveryReference && workspaceExportCauses.includes(action.cause) ? [recoveryReference.leaseId]
|
||||
: historical?.leaseIds;
|
||||
if (!Array.isArray(leaseIds) || leaseIds.length === 0)
|
||||
throw new SystemRecoveryFailure("historical_source_missing", "use_alternative_strategy");
|
||||
if (options.maximumSources !== undefined && leaseIds.length > options.maximumSources)
|
||||
throw new WorkspaceSourceInspectionLimit();
|
||||
if (leaseIds.some(id => typeof id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) ||
|
||||
new Set(leaseIds).size !== leaseIds.length || (reference && !leaseIds.includes(reference.leaseId))) throw invalid("lease_reference_invalid");
|
||||
new Set(leaseIds).size !== leaseIds.length || (recoveryReference && !leaseIds.includes(recoveryReference.leaseId))) throw invalid("lease_reference_invalid");
|
||||
const [coordinator] = await db.select().from(nativeRunFinalizations).where(and(
|
||||
eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, run.id)));
|
||||
if (coordinator?.leaseOwner || adapterExecutionControls.has(run.id) ||
|
||||
@@ -102,7 +125,7 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
|
||||
// Reusable allocations can still owe copyback for a completed native result.
|
||||
// This authorizes only preservation of the exact descriptor, never reuse for
|
||||
// agent execution. Stop verification and physical ownership checks remain below.
|
||||
const acceptedExportResult = reference && workspaceExportCauses.includes(action.cause)
|
||||
const acceptedExportResult = recoveryReference && workspaceExportCauses.includes(action.cause)
|
||||
&& run.runtimeMode === "native" && coordinator?.issueId === input.issueId
|
||||
&& coordinator.resultId && run.completionContractId
|
||||
? (await db.select({ id: nativeRunResults.id }).from(nativeRunResults).where(and(
|
||||
@@ -116,14 +139,16 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
|
||||
for (const leaseId of leaseIds as string[]) {
|
||||
const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId),
|
||||
eq(environmentLeases.id, leaseId), eq(environmentLeases.heartbeatRunId, run.id)));
|
||||
const descriptor = reference?.leaseId === leaseId ? reference : null;
|
||||
const legacyDescriptor = reference?.leaseId === leaseId ? reference : null;
|
||||
const remoteDescriptor = remoteReference?.leaseId === leaseId ? remoteReference : null;
|
||||
const descriptor = legacyDescriptor ?? remoteDescriptor;
|
||||
const remoteCwd = lease?.metadata?.remoteCwd;
|
||||
if (!lease) throw invalid("source_lease_not_bound");
|
||||
if (lease.issueId !== input.issueId) throw invalid("lease_issue_mismatch");
|
||||
if (!lease.environmentId) throw invalid("lease_environment_missing");
|
||||
if (!lease.providerLeaseId) throw invalid("lease_allocation_missing");
|
||||
if (!lease.provider || lease.provider === "local") throw invalid("lease_provider_invalid");
|
||||
const reusableExport = descriptor && acceptedExportResult
|
||||
const reusableExport = descriptor && (acceptedExportResult || remoteDescriptor && remoteAuthority)
|
||||
&& ["reuse_by_environment", "reuse_by_execution_workspace"].includes(lease.leasePolicy);
|
||||
if (!["retain_on_failure", "ephemeral"].includes(lease.leasePolicy) && !reusableExport)
|
||||
throw invalid("lease_policy_not_retained", lease.leasePolicy);
|
||||
@@ -132,11 +157,13 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
|
||||
if (descriptor && remoteCwd !== descriptor.remoteCwd) throw invalid("cwd_reference_mismatch");
|
||||
if (descriptor && lease.executionWorkspaceId && lease.executionWorkspaceId !== descriptor.workspaceId)
|
||||
throw invalid("workspace_reference_mismatch");
|
||||
if (remoteDescriptor && lease.environmentId !== remoteAuthority!.location!.environmentId)
|
||||
throw invalid("workspace_reference_mismatch");
|
||||
const history = await db.select().from(environmentLeases).where(and(
|
||||
ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider),
|
||||
eq(environmentLeases.providerLeaseId, lease.providerLeaseId)));
|
||||
const predecessors = retiredReusableLeasePredecessors(lease, history);
|
||||
if (!predecessors || (predecessors.length && (run.runtimeMode !== "native" || !descriptor || lease.metadata?.agentId !== run.agentId)))
|
||||
if (!predecessors || (predecessors.length && ((run.runtimeMode !== "native" && !remoteDescriptor) || !descriptor || lease.metadata?.agentId !== run.agentId)))
|
||||
throw new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required");
|
||||
const predecessorRuns = predecessors.map(item => item.heartbeatRunId!);
|
||||
const priors = predecessorRuns.length ? await db.select().from(heartbeatRuns).where(and(
|
||||
@@ -148,14 +175,19 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
|
||||
for (const predecessor of predecessors) {
|
||||
const prior = priors.find(item => item.id === predecessor.heartbeatRunId);
|
||||
const priorReference = readNativeWorkspaceSyncReference(prior?.runnerProfileJson?.nativeWorkspaceSync);
|
||||
const parsedPriorRemote = remoteWorkspaceRunReferenceSchema.safeParse(prior?.runnerProfileJson?.remoteWorkspace);
|
||||
const priorRemote = parsedPriorRemote.success ? parsedPriorRemote.data : null;
|
||||
const priorDescriptor = priorReference ?? priorRemote;
|
||||
const priorSaved = priorReference ? priorReference.state === "finalized" && Boolean(priorReference.finalHostSha256)
|
||||
: priorRemote?.state === "saved" && Boolean(priorRemote.snapshotId) && Boolean(remoteDescriptor)
|
||||
&& priorRemote.epoch < remoteDescriptor!.epoch;
|
||||
const priorOwner = priorOwners.find(item => item.runId === predecessor.heartbeatRunId);
|
||||
if (!prior || prior.runtimeMode !== "native" || prior.agentId !== run.agentId || prior.nativeIssueId !== input.issueId ||
|
||||
!terminalWorkspaceRunStates.includes(prior.status) || adapterExecutionControls.has(prior.id) ||
|
||||
(prior.controllerLeaseExpiresAt && prior.controllerLeaseExpiresAt > new Date()) ||
|
||||
!priorOwner || priorOwner.leaseOwner || !["committed", "terminal_failure", "applied"].includes(priorOwner.phase) ||
|
||||
!priorReference || priorReference.state !== "finalized" || !priorReference.finalHostSha256 ||
|
||||
priorReference.leaseId !== predecessor.id || priorReference.providerLeaseId !== lease.providerLeaseId ||
|
||||
priorReference.workspaceId !== descriptor!.workspaceId || priorReference.remoteCwd !== remoteCwd)
|
||||
!priorDescriptor || !priorSaved || priorDescriptor.leaseId !== predecessor.id || priorDescriptor.providerLeaseId !== lease.providerLeaseId ||
|
||||
priorDescriptor.workspaceId !== descriptor!.workspaceId || priorDescriptor.remoteCwd !== remoteCwd)
|
||||
throw new SystemRecoveryFailure("allocation_ownership_ambiguous", "service_required");
|
||||
}
|
||||
// Snapshot export writes only a new staging directory. A native baseline
|
||||
@@ -175,7 +207,8 @@ export async function inspectWorkspaceRecoverySource(db: Db, input: WorkspaceRec
|
||||
}
|
||||
const [environment] = await db.select().from(environments).where(eq(environments.id, lease.environmentId));
|
||||
if (!environment) throw new SystemRecoveryFailure("source_unavailable", "service_required");
|
||||
sources.push({ lease, environment, reference: descriptor, remoteCwd });
|
||||
sources.push({ lease, environment, reference: legacyDescriptor, remoteReference: remoteDescriptor,
|
||||
remoteWorkspace: remoteDescriptor ? remoteWorkspace : null, remoteCwd });
|
||||
}
|
||||
return { run, action, sources };
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wai
|
||||
import { heartbeatRunEvents } from "@paperclipai/db";
|
||||
import { createHeartbeatQueue, shouldDeferFollowupWakeForSameIssue, shouldQueueFollowupForRunningIssueWake } from "./heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "../__tests__/helpers/heartbeat-queue.js";
|
||||
import { recordRequiredWorkspaceRestoreFailure } from "./legacy-execution-recovery.js";
|
||||
import { recordRequiredWorkspaceRestoreFailure, terminalizeLegacyExecution } from "./legacy-execution-recovery.js";
|
||||
import { admitExplicitNativeContinuation } from "./explicit-native-continuation.js";
|
||||
import { withQueuedCommentIdsInWakePayload } from "./issue-queued-comment-queue.js";
|
||||
import * as recoverySources from "./workspace-recovery-source.js";
|
||||
@@ -40,13 +40,13 @@ import { requestTaskRetry } from "./task-retry.js";
|
||||
import { readFreshWorkspaceReadiness } from "./fresh-workspace-continuation.js";
|
||||
import { execFile } from "node:child_process";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { cp, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { and, eq, inArray, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, approvals, budgetPolicies, costEvents, completionContracts, issueApprovals, issueTreeHolds, issueTreeHoldMembers, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, agentWakeupRequests, workspaceOperations } from "@paperclipai/db";
|
||||
import { resumePreservedResultFinalization } from "./preserved-result-finalization.js";
|
||||
import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
|
||||
@@ -64,6 +64,10 @@ import { heartbeatService } from "./heartbeat.js";
|
||||
import { hasVerifiedWorkspaceRepair, readWorkspaceRecovery } from "./workspace-recovery-receipt.js";
|
||||
import { claimNativeSessionResumptions } from "./native-runtime/native-finalization-reconciler.js";
|
||||
import { executionWorkspaces } from "@paperclipai/db";
|
||||
import { executionWorkspaceSnapshots } from "@paperclipai/db";
|
||||
import { createSavedWorkspaceLoader, readRemoteWorkspaceAuthority, saveRemoteWorkspace } from "./remote-workspaces.js";
|
||||
import { createReusableLeaseHandoff } from "./environment-lease-handoff.js";
|
||||
import { preserveRemoteWorkspacePreparationFailure } from "./native-runtime/native-workspace-finalization-state.js";
|
||||
import { getExecutionBlocker } from "./execution-blocker.js";
|
||||
import { issueRoutes } from "../routes/issues.js";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
@@ -80,6 +84,7 @@ describe("automatic retained workspace recovery", () => {
|
||||
let connectionString: string;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let root: string;
|
||||
const fixtureCompanyIds = new Set<string>();
|
||||
const priorHome = process.env.PAPERCLIP_HOME, priorInstance = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
beforeAll(async () => {
|
||||
root = await mkdtemp(path.join(os.tmpdir(), "workspace-recovery-"));
|
||||
@@ -98,9 +103,26 @@ describe("automatic retained workspace recovery", () => {
|
||||
if (priorHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = priorHome;
|
||||
if (priorInstance === undefined) delete process.env.PAPERCLIP_INSTANCE_ID; else process.env.PAPERCLIP_INSTANCE_ID = priorInstance;
|
||||
});
|
||||
afterEach(async () => {
|
||||
// Discovery is intentionally global and bounded. An unresolved source from
|
||||
// an earlier test must not consume the current fixture's recovery page.
|
||||
// Keep all history within each test until its assertions have completed.
|
||||
const companyIds = [...fixtureCompanyIds]; fixtureCompanyIds.clear();
|
||||
if (!companyIds.length) return;
|
||||
const leases = await db.select({ id: environmentLeases.id }).from(environmentLeases)
|
||||
.where(inArray(environmentLeases.companyId, companyIds));
|
||||
for (const lease of leases) targets.delete(lease.id);
|
||||
await db.delete(environmentLeases).where(inArray(environmentLeases.companyId, companyIds));
|
||||
// Historical adoption can reconstruct an action from either the lease or
|
||||
// the run descriptor, so retire both fixture inputs before deleting it.
|
||||
await db.update(heartbeatRuns).set({ resultJson: sql`${heartbeatRuns.resultJson} - 'workspaceRestoreRecovery' - 'workspaceRestoreFailure'` })
|
||||
.where(inArray(heartbeatRuns.companyId, companyIds));
|
||||
await db.delete(issueRecoveryActions).where(inArray(issueRecoveryActions.companyId, companyIds));
|
||||
});
|
||||
async function seed(mode = "native", nativeDescriptor = true, resourceDisposition: "destroy" | "keep_running" = "destroy", reusable = false, ephemeral = false) {
|
||||
const companyId = randomUUID(), issueId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), actionId: string = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Workspace recovery", defaultResponsibleUserId: "board", issuePrefix: `R${companyId.slice(0, 6)}` });
|
||||
fixtureCompanyIds.add(companyId);
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "idle", runtimeConfig: { heartbeat: { maxConcurrentRuns: 1 } } });
|
||||
await db.insert(issues).values({ id: issueId, companyId, assigneeAgentId: agentId, status: "blocked", title: "Recover existing work" });
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, issueId, nativeIssueId: mode === "native" ? issueId : null,
|
||||
@@ -166,6 +188,199 @@ describe("automatic retained workspace recovery", () => {
|
||||
const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!;
|
||||
const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!;
|
||||
|
||||
it.each([
|
||||
{ status: "failed", runtimeMode: "native", uploadFailure: "none" },
|
||||
{ status: "cancelled", runtimeMode: "native", uploadFailure: "none" },
|
||||
{ status: "failed", runtimeMode: "legacy", uploadFailure: "final" },
|
||||
{ status: "failed", runtimeMode: "legacy", uploadFailure: "initial" },
|
||||
] as const)("saves the authoritative remote workspace after $runtimeMode $status without replaying a turn or certifying external effects ($uploadFailure upload failure)", async ({ status, runtimeMode, uploadFailure }) => {
|
||||
const f = await seed(runtimeMode, false);
|
||||
const remoteCwd = await realpath(f.remote);
|
||||
targets.get(f.lease.id)!.remoteCwd = remoteCwd;
|
||||
f.runtime.resumeRunLease.mockImplementation(async () => ({ providerLeaseId: f.providerLeaseId, metadata: { remoteCwd } }));
|
||||
const workspaceId = randomUUID();
|
||||
let authority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null as string | null,
|
||||
source: { kind: "empty" }, writerRunId: f.runId, dirty: true,
|
||||
location: { environmentId: f.lease.environmentId!, leaseId: f.lease.id,
|
||||
providerLeaseId: f.providerLeaseId, remoteCwd } };
|
||||
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId, sourceIssueId: f.issueId,
|
||||
name: "Authoritative remote files", cwd: remoteCwd, providerRef: remoteCwd, providerType: "cloud_sandbox",
|
||||
mode: "isolated_workspace", strategyType: "project_primary", status: "active", metadata: { remoteWorkspace: authority } });
|
||||
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.issueId));
|
||||
const [boundLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
||||
await db.update(environmentLeases).set({ executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_execution_workspace",
|
||||
metadata: { ...boundLease.metadata, remoteCwd, agentId: (await run(f.runId)).agentId } })
|
||||
.where(eq(environmentLeases.id, f.lease.id));
|
||||
const before = await run(f.runId);
|
||||
if (uploadFailure === "final") {
|
||||
// A reused allocation can fail during preparation, before native input or
|
||||
// its finalization coordinator exists. Keep a real saved predecessor and
|
||||
// the ordinary exact lease handoff; neither grants a new provider turn.
|
||||
const [priorRun] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: before.agentId,
|
||||
issueId: f.issueId, nativeIssueId: f.issueId, runtimeMode: "native", status: "succeeded",
|
||||
createdAt: new Date(before.createdAt.getTime() - 2_000), finishedAt: new Date(before.createdAt.getTime() - 1_000),
|
||||
contextSnapshot: { issueId: f.issueId } }).returning();
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issueId,
|
||||
runId: priorRun.id, phase: "committed" });
|
||||
const [currentLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
||||
const [priorLease] = await db.insert(environmentLeases).values({ companyId: f.companyId, issueId: f.issueId,
|
||||
heartbeatRunId: priorRun.id, environmentId: currentLease.environmentId, executionWorkspaceId: workspaceId,
|
||||
provider: currentLease.provider, providerLeaseId: f.providerLeaseId, leasePolicy: "reuse_by_environment", status: "active",
|
||||
acquiredAt: new Date(currentLease.acquiredAt.getTime() - 1_000), metadata: currentLease.metadata }).returning();
|
||||
const priorAuthority = { ...authority, writerRunId: priorRun.id,
|
||||
location: { ...authority.location, leaseId: priorLease.id } };
|
||||
const [priorWorkspace] = await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: priorAuthority } })
|
||||
.where(eq(executionWorkspaces.id, workspaceId)).returning();
|
||||
const priorSaved = await saveRemoteWorkspace({ db, workspace: priorWorkspace, runId: priorRun.id,
|
||||
lease: priorLease as EnvironmentLease, target: { ...targets.get(f.lease.id)!, leaseId: priorLease.id },
|
||||
assertOwned: async () => undefined });
|
||||
const handoff = createReusableLeaseHandoff(priorLease, currentLease);
|
||||
await db.update(environmentLeases).set({ status: "expired", cleanupStatus: "success", releasedAt: currentLease.acquiredAt,
|
||||
metadata: { ...priorLease.metadata, reusableLeaseSuccessor: handoff } }).where(eq(environmentLeases.id, priorLease.id));
|
||||
await db.update(environmentLeases).set({ metadata: { ...currentLease.metadata, reusableLeasePredecessor: handoff } })
|
||||
.where(eq(environmentLeases.id, currentLease.id));
|
||||
authority = { ...authority, epoch: 2, snapshotId: priorSaved.snapshotId };
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: authority } })
|
||||
.where(eq(executionWorkspaces.id, workspaceId));
|
||||
expect(before.runtimeMode).toBe("legacy");
|
||||
expect(before.runnerProfileJson?.nativeExecutionInput).toBeUndefined();
|
||||
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))).toEqual([]);
|
||||
}
|
||||
if (runtimeMode === "legacy") await writeFile(path.join(f.remote, "pre-native-change.txt"), "current preparation wrote these files\n");
|
||||
await db.update(heartbeatRuns).set({ status, runnerProfileJson: { ...before.runnerProfileJson,
|
||||
...(status === "failed" && runtimeMode === "native" ? { nativeResourceDisposition: "destroy" } : {}),
|
||||
remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId: f.lease.id,
|
||||
providerLeaseId: f.providerLeaseId, remoteCwd, epoch: authority.epoch, state: "prepared", snapshotId: authority.snapshotId } },
|
||||
resultJson: { ...before.resultJson, ...(status === "cancelled" ? {
|
||||
nativeCancellation: { dispatchState: "acknowledged", requestedAt: new Date().toISOString() },
|
||||
} : {}) },
|
||||
}).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...authority, epoch: authority.epoch + 1 } } })
|
||||
.where(eq(executionWorkspaces.id, workspaceId));
|
||||
await expect(recoverySources.inspectWorkspaceRecoverySource(db, f.input)).rejects.toMatchObject({ code: "source_binding_invalid" });
|
||||
expect(f.runtime.resumeRunLease).not.toHaveBeenCalled();
|
||||
await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: authority } })
|
||||
.where(eq(executionWorkspaces.id, workspaceId));
|
||||
if (runtimeMode === "legacy") {
|
||||
// Remote file ownership exists before native runtime selection.
|
||||
expect((await recoverySources.inspectWorkspaceRecoverySource(db, f.input)).sources).toHaveLength(1);
|
||||
const failedUpload = vi.spyOn(getStorageService(), "putFile").mockRejectedValueOnce(new Error("Synthetic saved-copy outage"));
|
||||
try {
|
||||
if (uploadFailure === "initial") {
|
||||
// Exercise the original setup boundary, not a failed final save that
|
||||
// retries the same operation. The initial ledger reservation survives.
|
||||
await db.delete(issueRecoveryActions).where(eq(issueRecoveryActions.id, f.actionId));
|
||||
await db.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null, resultJson: null })
|
||||
.where(eq(heartbeatRuns.id, f.runId));
|
||||
const [originalLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
||||
const activeMetadata = { ...originalLease.metadata };
|
||||
delete activeMetadata.remoteExecutionTermination;
|
||||
const [activeLease] = await db.update(environmentLeases).set({ status: "active", releasedAt: null, cleanupStatus: null,
|
||||
metadata: activeMetadata }).where(eq(environmentLeases.id, f.lease.id)).returning();
|
||||
const [preparing] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
|
||||
await expect(saveRemoteWorkspace({ db, workspace: preparing, runId: f.runId, lease: activeLease as EnvironmentLease,
|
||||
target: targets.get(f.lease.id)!, assertOwned: async () => undefined, boundary: "initial" }))
|
||||
.rejects.toThrow("Synthetic saved-copy outage");
|
||||
const preserved = await preserveRemoteWorkspacePreparationFailure(db, activeLease, { assertOwned: async () => undefined });
|
||||
expect(preserved?.resultJson).toMatchObject({ workspaceRestoreFailure: "restore_failed",
|
||||
workspaceRestoreRecovery: { leaseIds: [f.lease.id] } });
|
||||
await terminalizeLegacyExecution({ db, run: preserved!, status: "failed", patch: {
|
||||
errorCode: "setup_failed", error: "Synthetic saved-copy outage", finishedAt: new Date(),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } },
|
||||
} });
|
||||
const repairs = await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId),
|
||||
sql`${issueRecoveryActions.evidence}->>'runId' = ${f.runId}`));
|
||||
expect(repairs).toHaveLength(1);
|
||||
f.actionId = repairs[0].id;
|
||||
f.input.actionId = repairs[0].id;
|
||||
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))).toEqual([]);
|
||||
expect((await run(f.runId)).runnerProfileJson).not.toHaveProperty("nativeExecutionInput");
|
||||
} else await recoverRetainedWorkspace(f.input);
|
||||
expect(failedUpload).toHaveBeenCalledTimes(1);
|
||||
} finally { failedUpload.mockRestore(); }
|
||||
expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).not.toBe("complete");
|
||||
const [unsaved] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
|
||||
expect(readRemoteWorkspaceAuthority(unsaved.metadata)).toMatchObject({ dirty: true, writerRunId: f.runId,
|
||||
epoch: authority.epoch, snapshotId: authority.snapshotId });
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(false);
|
||||
expect(await readFile(path.join(f.remote, "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
if (uploadFailure === "final") await makeRetryDue(f.actionId);
|
||||
}
|
||||
await recoverRetainedWorkspace(f.input);
|
||||
const recoveredAction = await action(f.actionId);
|
||||
expect(readSystemRecovery(recoveredAction.evidence)?.state).toBe("complete");
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
|
||||
const saved = readRemoteWorkspaceAuthority(workspace.metadata);
|
||||
expect(saved).toMatchObject({ dirty: false, writerRunId: null });
|
||||
expect(saved?.snapshotId).toBeTruthy();
|
||||
const snapshots = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.executionWorkspaceId, workspaceId));
|
||||
expect(snapshots).toHaveLength(runtimeMode === "legacy" ? 2 : 1);
|
||||
expect(snapshots.find(snapshot => snapshot.id === saved!.snapshotId))
|
||||
.toMatchObject({ status: "committed", sourceRunId: f.runId, generation: runtimeMode === "legacy" ? 2 : 1 });
|
||||
const extracted = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId, snapshotId: saved!.snapshotId! });
|
||||
try {
|
||||
expect(await readFile(path.join(extracted.root, "saved.txt"), "utf8")).toBe("finished work\n");
|
||||
if (runtimeMode === "legacy") expect(await readFile(path.join(extracted.root, "pre-native-change.txt"), "utf8"))
|
||||
.toBe("current preparation wrote these files\n");
|
||||
}
|
||||
finally { await extracted.cleanup(); }
|
||||
expect(await readFile(path.join(f.host, "newer.txt"), "utf8")).toBe("newer local work\n");
|
||||
expect(await run(f.runId)).toMatchObject({ status, errorCode: uploadFailure === "initial" ? "setup_failed" : "execution_interrupted",
|
||||
runnerProfileJson: { remoteWorkspace: { state: "saved", snapshotId: saved!.snapshotId } } });
|
||||
expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toHaveLength(0);
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, f.runId))).toHaveLength(0);
|
||||
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.companyId))).toHaveLength(0);
|
||||
expect(await db.select().from(workspaceOperations).where(and(eq(workspaceOperations.heartbeatRunId, f.runId),
|
||||
eq(workspaceOperations.phase, "workspace_recovery_snapshot")))).toHaveLength(0);
|
||||
expect(f.runtime.resumeRunLease).toHaveBeenCalledTimes(uploadFailure === "final" ? 2 : 1);
|
||||
expect(f.runtime.retryPendingSandboxTeardown).toHaveBeenCalledTimes(runtimeMode === "legacy" ? 2 : 1);
|
||||
expect(await hasVerifiedWorkspaceRepair(db, await run(f.runId))).toBe(true);
|
||||
await scheduleRecoveredWorkspaceCleanup(db, await run(f.runId));
|
||||
const [settledLease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id));
|
||||
if (runtimeMode === "legacy") {
|
||||
expect(settledLease).toMatchObject({ status: "released",
|
||||
leasePolicy: uploadFailure === "initial" ? "retain_on_failure" : "reuse_by_execution_workspace" });
|
||||
expect((await run(f.runId)).runtimeMode).toBe("legacy");
|
||||
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId))).toEqual([]);
|
||||
if (uploadFailure === "final") expect(snapshots.find(snapshot => snapshot.id === authority.snapshotId)).toMatchObject({ status: "superseded" });
|
||||
else expect(snapshots.find(snapshot => snapshot.generation === 1)).toMatchObject({ status: "pending" });
|
||||
// Explicit Retry may admit the preserved files. Saturate agent capacity
|
||||
// so this tests admission without dispatching a new agent turn.
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: before.agentId, status: "running" });
|
||||
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board",
|
||||
actorSource: "local_implicit", observedRunId: f.runId, idempotencyKey: randomUUID() });
|
||||
await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!);
|
||||
const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));
|
||||
expect(successor).toBeDefined();
|
||||
expect(await readFreshWorkspaceReadiness(db, successor))
|
||||
.toMatchObject({ mode: "resume_preserved_workspace", preservationReceiptId: expect.any(String) });
|
||||
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.runId, successor.id))).toEqual([]);
|
||||
expect(f.runtime.resumeRunLease).toHaveBeenCalledTimes(uploadFailure === "final" ? 2 : 1);
|
||||
return;
|
||||
}
|
||||
if (status === "failed") {
|
||||
expect(settledLease).toMatchObject({ status: "pending_cleanup", leasePolicy: "ephemeral",
|
||||
metadata: { workspaceRecoveryCleanup: { runId: f.runId, snapshotId: saved!.snapshotId, epoch: authority.epoch } } });
|
||||
const source = await run(f.runId);
|
||||
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" });
|
||||
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issueId, actorId: "local-board",
|
||||
actorSource: "local_implicit", observedRunId: source.id, idempotencyKey: randomUUID() });
|
||||
await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!);
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0])
|
||||
.toMatchObject({ status: "deferred_issue_execution", payload: { executionWait: { reason: "remote_cleanup" } } });
|
||||
await environmentService(db).releaseLease(settledLease.id, "released", { cleanupStatus: "success",
|
||||
remoteExecutionTermination: remoteTerminationReceipt(settledLease, { providerLeaseId: settledLease.providerLeaseId, state: "destroyed" }) });
|
||||
await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!);
|
||||
const [successor] = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.status, "queued")));
|
||||
expect(successor).toBeDefined();
|
||||
const readiness = await readFreshWorkspaceReadiness(db, successor);
|
||||
expect(readiness).toMatchObject({ mode: "resume_preserved_workspace", preservationReceiptId: expect.any(String) });
|
||||
} else {
|
||||
expect(settledLease).toMatchObject({ status: "released", leasePolicy: "reuse_by_execution_workspace" });
|
||||
expect(settledLease.metadata?.workspaceRecoveryCleanup).toBeUndefined();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
async function warmHandoffSource(hops = 1, beforeStop?: (lease: typeof environmentLeases.$inferSelect) => Promise<void>) {
|
||||
const prior = await seed("native", true, "keep_running", true), source = await run(prior.runId);
|
||||
let previousTarget = targets.get(prior.lease.id)!, previousLeaseId = prior.lease.id;
|
||||
@@ -216,8 +431,13 @@ describe("automatic retained workspace recovery", () => {
|
||||
let isolatedRoot: string | undefined;
|
||||
beforeAll(async () => {
|
||||
original = { db, root, connectionString, home: process.env.PAPERCLIP_HOME };
|
||||
isolated = await startEmbeddedPostgresTestDatabase("workspace-warm-handoff-");
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WORKSPACE_RECOVERY_HANDOFF_TEST_DATABASE_URL");
|
||||
if (external) {
|
||||
await applyPendingMigrations(external);
|
||||
isolated = { connectionString: external, cleanup: async () => {} };
|
||||
} else isolated = await startEmbeddedPostgresTestDatabase("workspace-warm-handoff-");
|
||||
isolatedDb = createDb(isolated.connectionString);
|
||||
if (external) await isolatedDb.execute(sql`TRUNCATE TABLE companies CASCADE`);
|
||||
isolatedRoot = await mkdtemp(path.join(os.tmpdir(), "workspace-warm-handoff-files-"));
|
||||
db = isolatedDb; root = isolatedRoot; connectionString = isolated.connectionString;
|
||||
process.env.PAPERCLIP_HOME = path.join(root, "home");
|
||||
|
||||
@@ -10,8 +10,8 @@ import { beginIdleTrackedWork } from "./task-admission.js";
|
||||
import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js";
|
||||
import { resumePreservedResultFinalization } from "./preserved-result-finalization.js";
|
||||
import { and, asc, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, environmentLeases, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
|
||||
import { agentWakeupRequests, environmentLeases, executionWorkspaces, heartbeatRuns, issues, issueRecoveryActions, nativeRunFinalizations, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { remoteWorkspaceRunReferenceSchema, type Environment, type EnvironmentLease } from "@paperclipai/shared";
|
||||
import { conflict } from "../errors.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
|
||||
@@ -31,6 +31,7 @@ import { SYSTEM_RECOVERY_MAX_RETRIES, createSystemRecovery, failSystemRecovery,
|
||||
import { inspectWorkspaceRecoverySource, workspaceRecoverySourceEligible, workspaceSourceBindingFailure } from "./workspace-recovery-source.js";
|
||||
import { preserveWorkspaceSnapshot } from "./workspace-recovery-snapshot.js";
|
||||
import { adoptHistoricalWorkspaceRecoveries } from "./workspace-recovery-backfill.js";
|
||||
import { saveRemoteWorkspace } from "./remote-workspaces.js";
|
||||
|
||||
const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"];
|
||||
const terminal = ["failed", "interrupted", "timed_out", "cancelled", "succeeded"];
|
||||
@@ -278,7 +279,29 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ
|
||||
await inspectWorkspaceRecoverySource(db, input);
|
||||
await progress({ phase: "preserve_files" });
|
||||
let operationId: string;
|
||||
if (reference) {
|
||||
if (source.remoteReference && source.remoteWorkspace) {
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(and(
|
||||
eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, source.remoteWorkspace.id),
|
||||
));
|
||||
if (!workspace) throw workspaceSourceBindingFailure("workspace_reference_mismatch");
|
||||
const saved = await saveRemoteWorkspace({ db, workspace, runId: input.runId,
|
||||
lease: cleanupLease as EnvironmentLease, target, assertOwned: ownership.assertHeld });
|
||||
await ownership.assertHeld();
|
||||
const [existing] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and(
|
||||
eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.heartbeatRunId, input.runId),
|
||||
eq(workspaceOperations.phase, "workspace_recovery"), eq(workspaceOperations.status, "succeeded"),
|
||||
sql`${workspaceOperations.metadata}->>'leaseId' = ${lease.id}`,
|
||||
sql`${workspaceOperations.metadata}->>'snapshotId' = ${saved.snapshotId}`,
|
||||
)).limit(1);
|
||||
const operation = existing ?? (await db.insert(workspaceOperations).values({ companyId: input.companyId,
|
||||
issueId: input.issueId, heartbeatRunId: input.runId, executionWorkspaceId: workspace.id,
|
||||
phase: "workspace_recovery", status: "succeeded", finishedAt: new Date(), metadata: {
|
||||
recoveryActionId: input.actionId, leaseId: lease.id, providerLeaseId: lease.providerLeaseId,
|
||||
snapshotId: saved.snapshotId, generation: saved.generation, sha256: saved.sha256,
|
||||
storage: "remote_workspace", epoch: source.remoteReference.epoch,
|
||||
} }).returning({ id: workspaceOperations.id }))[0]!;
|
||||
operationId = operation.id;
|
||||
} else if (reference) {
|
||||
const copied = await resumeNativeWorkspaceSync({ db, runId: input.runId, target, assertOwnership: ownership.assertHeld });
|
||||
if (!copied) throw new SystemRecoveryFailure("snapshot_verification_failed", "service_required");
|
||||
const after = await inspectWorkspaceRecoverySource(db, input);
|
||||
@@ -384,8 +407,12 @@ const unfinishedCursors = new WeakMap<Db, string>();
|
||||
* saved-file receipt and the original recorded disposition. */
|
||||
export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof heartbeatRuns.$inferSelect) {
|
||||
if (!await hasVerifiedWorkspaceRepair(db, run)) return;
|
||||
const ref = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
|
||||
if (ref?.resourceDisposition !== "destroy") return;
|
||||
const legacy = readNativeWorkspaceSyncReference(run.runnerProfileJson?.nativeWorkspaceSync);
|
||||
const parsedRemote = remoteWorkspaceRunReferenceSchema.safeParse(run.runnerProfileJson?.remoteWorkspace);
|
||||
const remote = parsedRemote.success && parsedRemote.data.state === "saved" ? parsedRemote.data : null;
|
||||
const ref = legacy ?? remote;
|
||||
const disposition = legacy?.resourceDisposition ?? (remote ? run.runnerProfileJson?.nativeResourceDisposition : undefined);
|
||||
if (!ref || disposition !== "destroy") return;
|
||||
const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, ref.leaseId)));
|
||||
if (!lease || !hasRemoteTerminationReceipt(lease)) return;
|
||||
const [other] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
|
||||
@@ -394,7 +421,8 @@ export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof hear
|
||||
if (other) return;
|
||||
await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed", leasePolicy: "ephemeral",
|
||||
updatedAt: new Date(), metadata: sql`(${environmentLeases.metadata} - 'workspaceRestoreRecovery' - 'remoteExecutionTermination') ||
|
||||
${JSON.stringify({ workspaceRecoveryCleanup: { runId: run.id, descriptorSha256: ref.descriptorSha256 },
|
||||
${JSON.stringify({ workspaceRecoveryCleanup: { runId: run.id, ...(legacy ? { descriptorSha256: legacy.descriptorSha256 }
|
||||
: { snapshotId: remote!.snapshotId, epoch: remote!.epoch }) },
|
||||
pendingCleanupInFlight: false, pendingCleanupRetryAfterMs: 0, pendingCleanupRetryAttempts: 0, pendingCleanupLeaseExpiresAtMs: 0 })}::jsonb`,
|
||||
}).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, ref.leaseId),
|
||||
eq(environmentLeases.heartbeatRunId, run.id), eq(environmentLeases.providerLeaseId, ref.providerLeaseId),
|
||||
@@ -403,10 +431,15 @@ export async function scheduleRecoveredWorkspaceCleanup(db: Db, run: typeof hear
|
||||
sql`${environmentLeases.metadata}->>'workspaceRecoveryCleanup' is null`)), { companyId: run.companyId, runId: run.id });
|
||||
}
|
||||
export function retainedWorkspaceCandidates(db: Db, cursor?: string, unfinishedOnly = false, obligationsOnly = false, inFlightRunIds: string[] = []) {
|
||||
const cleanup = sql`(${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,resourceDisposition}' = 'destroy' and exists (
|
||||
const cleanup = sql`(((${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,resourceDisposition}' = 'destroy'
|
||||
and ${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,leaseId}' is not null)
|
||||
or (${heartbeatRuns.runnerProfileJson}->>'nativeResourceDisposition' = 'destroy'
|
||||
and ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,schema}' = 'paperclip.remote-workspace-run.v1'
|
||||
and ${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,state}' = 'saved')) and exists (
|
||||
select 1 from ${environmentLeases} where ${environmentLeases.companyId} = ${heartbeatRuns.companyId}
|
||||
and ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id}
|
||||
and ${environmentLeases.id}::text = ${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,leaseId}'
|
||||
and ${environmentLeases.id}::text = coalesce(${heartbeatRuns.runnerProfileJson}#>>'{nativeWorkspaceSync,leaseId}',
|
||||
${heartbeatRuns.runnerProfileJson}#>>'{remoteWorkspace,leaseId}')
|
||||
and ${environmentLeases.status} = 'released' and ${environmentLeases.cleanupStatus} = 'success'
|
||||
and ${environmentLeases.metadata}->'workspaceRecoveryCleanup' is null))`;
|
||||
const waitingInput = sql`exists (select 1 from ${agentWakeupRequests}
|
||||
|
||||
@@ -21,6 +21,7 @@ import type { Db } from "@paperclipai/db";
|
||||
import { executionWorkspaces, issueComments, issues, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db";
|
||||
import {
|
||||
DEFAULT_TAILSCALE_HTTPS_EXPOSURE,
|
||||
readRemoteWorkspaceAuthority,
|
||||
deriveViteHmrPort,
|
||||
forceLoopbackBindInCommand,
|
||||
isRuntimeExposureAppPort,
|
||||
@@ -45,6 +46,7 @@ import {
|
||||
import { and, desc, eq, gte, inArray, isNull, lte, ne, or, sql } from "drizzle-orm";
|
||||
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
|
||||
import { conflict, unprocessable } from "../errors.js";
|
||||
import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js";
|
||||
import { resolveHomeAwarePath } from "../home-paths.js";
|
||||
import { hasVerifiedWorktreeSeedManifest, isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
|
||||
import {
|
||||
@@ -4279,6 +4281,7 @@ export async function cleanupExecutionWorkspaceArtifacts(input: {
|
||||
forceWorktreeRemoval?: boolean;
|
||||
}) {
|
||||
if (input.workspace.providerType === "plugin") throw new Error("Managed workspace cleanup requires its storage plugin");
|
||||
if (readRemoteWorkspaceAuthority(input.workspace.metadata)) throw new Error("Remote workspace cleanup requires its sandbox lifecycle");
|
||||
const warnings: string[] = [];
|
||||
const workspacePath = input.workspace.providerRef ?? input.workspace.cwd;
|
||||
const isolatedSource = await verifiedRecordedIsolatedRemoteSource(input.workspace);
|
||||
@@ -7538,14 +7541,14 @@ function selectRuntimeServiceEntries(input: {
|
||||
});
|
||||
}
|
||||
|
||||
async function isPersistedIsolatedExecutionWorkspace(input: {
|
||||
async function inspectPersistedRuntimeWorkspace(input: {
|
||||
db?: Db;
|
||||
companyId: string;
|
||||
executionWorkspaceId?: string | null;
|
||||
}) {
|
||||
if (!input.db || !input.executionWorkspaceId) return false;
|
||||
if (!input.db || !input.executionWorkspaceId) return { isolated: false, remote: false };
|
||||
const row = await input.db
|
||||
.select({ mode: executionWorkspaces.mode })
|
||||
.select({ mode: executionWorkspaces.mode, metadata: executionWorkspaces.metadata })
|
||||
.from(executionWorkspaces)
|
||||
.where(
|
||||
and(
|
||||
@@ -7555,7 +7558,7 @@ async function isPersistedIsolatedExecutionWorkspace(input: {
|
||||
)
|
||||
.limit(1)
|
||||
.then((rows) => rows[0] ?? null);
|
||||
return row?.mode === "isolated_workspace";
|
||||
return { isolated: row?.mode === "isolated_workspace", remote: Boolean(readRemoteWorkspaceAuthority(row?.metadata)) };
|
||||
}
|
||||
|
||||
type EnsureRuntimeServicesForRunInput = {
|
||||
@@ -7583,16 +7586,24 @@ async function ensureRuntimeServicesForRunInvocation(
|
||||
});
|
||||
const acquiredServiceIds: string[] = [];
|
||||
const refs: RuntimeServiceRef[] = [];
|
||||
const persistedWorkspace = await inspectPersistedRuntimeWorkspace({
|
||||
db: input.db,
|
||||
companyId: input.agent.companyId,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
});
|
||||
if (persistedWorkspace.remote) {
|
||||
if (rawServices.length > 0) {
|
||||
throw new WorkspaceRuntimeValidationFailure("Remote workspace storage does not support Core workspace services.",
|
||||
{ reason: "remote_workspace_runtime_services_unsupported" });
|
||||
}
|
||||
return [];
|
||||
}
|
||||
const allowFixedPortFallback = persistedWorkspace.isolated;
|
||||
const runtimeProvision = input.skipHostProvision
|
||||
? { command: "", kind: null }
|
||||
: resolveRuntimeProvision(input);
|
||||
const runtimeProvisionCommand = runtimeProvision.command;
|
||||
const provisionCoordinator = createRuntimeProvisionCoordinator();
|
||||
const allowFixedPortFallback = await isPersistedIsolatedExecutionWorkspace({
|
||||
db: input.db,
|
||||
companyId: input.agent.companyId,
|
||||
executionWorkspaceId: input.executionWorkspaceId,
|
||||
});
|
||||
runtimeServiceLeasesByRun.set(input.runId, acquiredServiceIds);
|
||||
|
||||
try {
|
||||
@@ -8212,6 +8223,7 @@ export async function startRuntimeServicesForWorkspaceControl(
|
||||
defaultDesiredState: readDesiredRuntimeState(input.config.desiredState) ?? "stopped",
|
||||
serviceStates: readConfiguredServiceStates(input.config),
|
||||
});
|
||||
if (services.length === 0) return [];
|
||||
const invocationId = input.invocationId ?? "workspace_control";
|
||||
const mutexPlan = resolveRuntimeStartMutexPlan({
|
||||
services,
|
||||
@@ -8225,7 +8237,32 @@ export async function startRuntimeServicesForWorkspaceControl(
|
||||
try {
|
||||
return await withRuntimeStartMutexes(
|
||||
mutexPlan.ownerKeys,
|
||||
() => startRuntimeServicesForWorkspaceControlInvocation(input),
|
||||
async () => {
|
||||
if (!input.db || !input.executionWorkspaceId) {
|
||||
return startRuntimeServicesForWorkspaceControlInvocation(input);
|
||||
}
|
||||
// Startup and direct callers can bypass the HTTP operation recorder.
|
||||
// Keep migration fenced through provision/bind/persistence, including the
|
||||
// HTTPS path which intentionally commits its reservation before binding.
|
||||
return withExecutionWorkspaceLifecycleLock(input.db, input.executionWorkspaceId, async (tx) => {
|
||||
const [fresh] = await tx.select({ cwd: executionWorkspaces.cwd, metadata: executionWorkspaces.metadata })
|
||||
.from(executionWorkspaces).where(and(eq(executionWorkspaces.id, input.executionWorkspaceId!),
|
||||
eq(executionWorkspaces.companyId, input.actor.companyId)));
|
||||
if (!fresh) throw new Error("Execution workspace not found before starting runtime services");
|
||||
if (readRemoteWorkspaceAuthority(fresh.metadata)) {
|
||||
throw unprocessable("Workspace commands are not supported for remote workspaces yet. Run the command through an agent task.",
|
||||
{ code: "remote_workspace_runtime_commands_unsupported" });
|
||||
}
|
||||
if (fresh.metadata?.remoteWorkspaceMigration) {
|
||||
throw conflict("The workspace is being moved to remote storage. Retry after migration completes.",
|
||||
{ code: "workspace_runtime_migration_in_progress" });
|
||||
}
|
||||
if (fresh.cwd !== input.workspace.cwd) {
|
||||
throw conflict("Execution workspace changed before command admission.", { code: "workspace_runtime_authority_changed" });
|
||||
}
|
||||
return startRuntimeServicesForWorkspaceControlInvocation(input);
|
||||
});
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
releaseRuntimeReplacementClaims(mutexPlan.replacementReuseKeys);
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
ListObjectsV2Command,
|
||||
ListObjectVersionsCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { putS3Multipart } from "./s3-multipart.js";
|
||||
import { withS3FailureDiagnostics } from "./s3-failure.js";
|
||||
@@ -79,6 +80,32 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid
|
||||
forcePathStyle: Boolean(config.forcePathStyle),
|
||||
});
|
||||
|
||||
async function listExactVersions(key: string, signal?: AbortSignal) {
|
||||
const versions = new Set<string>(), cursors = new Set<string>();
|
||||
let keyMarker: string | undefined, versionMarker: string | undefined;
|
||||
while (true) {
|
||||
signal?.throwIfAborted();
|
||||
const output = await client.send(new ListObjectVersionsCommand({ Bucket: bucket, Prefix: key,
|
||||
KeyMarker: keyMarker, VersionIdMarker: versionMarker, MaxKeys: 1000 }), { abortSignal: signal });
|
||||
if (typeof output.IsTruncated !== "boolean") throw new Error("s3_version_listing_invalid");
|
||||
for (const entry of [...(output.Versions ?? []), ...(output.DeleteMarkers ?? [])]) {
|
||||
if (typeof entry.Key !== "string" || !entry.Key.startsWith(key) ||
|
||||
typeof entry.VersionId !== "string" || !entry.VersionId) throw new Error("s3_version_listing_invalid");
|
||||
// Prefix is the narrowest S3 listing filter, but neighboring keys are never ours to delete.
|
||||
if (entry.Key === key) versions.add(entry.VersionId);
|
||||
}
|
||||
if (!output.IsTruncated) return versions;
|
||||
const nextKey = output.NextKeyMarker, nextVersion = output.NextVersionIdMarker;
|
||||
if (!nextKey || !nextKey.startsWith(key) || !nextVersion ||
|
||||
(keyMarker && Buffer.compare(Buffer.from(nextKey), Buffer.from(keyMarker)) < 0)) {
|
||||
throw new Error("s3_version_listing_cursor_invalid");
|
||||
}
|
||||
const cursor = JSON.stringify([nextKey, nextVersion]);
|
||||
if (cursors.has(cursor)) throw new Error("s3_version_listing_cursor_invalid");
|
||||
cursors.add(cursor); keyMarker = nextKey; versionMarker = nextVersion;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id: "s3",
|
||||
|
||||
@@ -182,6 +209,19 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid
|
||||
|
||||
async deleteObject(input): Promise<void> {
|
||||
const key = buildKey(prefix, input.objectKey);
|
||||
if (input.allVersions) {
|
||||
await withS3FailureDiagnostics("delete_object", async () => {
|
||||
// Finish listing before deleting: removing a cursor version must not skip later pages.
|
||||
const versions = await listExactVersions(key, input.signal);
|
||||
for (const versionId of versions) {
|
||||
input.signal?.throwIfAborted();
|
||||
await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: versionId }),
|
||||
{ abortSignal: input.signal });
|
||||
}
|
||||
if ((await listExactVersions(key, input.signal)).size) throw new Error("s3_object_versions_still_present");
|
||||
});
|
||||
return;
|
||||
}
|
||||
await withS3FailureDiagnostics("delete_object", () => client.send(
|
||||
new DeleteObjectCommand({
|
||||
Bucket: bucket,
|
||||
|
||||
@@ -151,7 +151,8 @@ export function createStorageService(provider: StorageProvider): StorageService
|
||||
|
||||
async deleteObject(companyId: string, objectKey: string, options) {
|
||||
ensureCompanyPrefix(companyId, objectKey);
|
||||
await cancellable(options?.signal, () => provider.deleteObject({ objectKey, signal: options?.signal }));
|
||||
await cancellable(options?.signal, () => provider.deleteObject({ objectKey, signal: options?.signal,
|
||||
...(options?.allVersions ? { allVersions: true } : {}) }));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -21,6 +21,13 @@ export interface GetObjectInput {
|
||||
};
|
||||
}
|
||||
|
||||
export interface DeleteObjectInput {
|
||||
objectKey: string;
|
||||
signal?: AbortSignal;
|
||||
/** Remove every version and delete marker of this exact key before returning. */
|
||||
allVersions?: boolean;
|
||||
}
|
||||
|
||||
export interface GetObjectResult {
|
||||
stream: Readable;
|
||||
contentType?: string;
|
||||
@@ -42,7 +49,7 @@ export interface StorageProvider {
|
||||
putObject(input: PutObjectInput): Promise<void>;
|
||||
getObject(input: GetObjectInput): Promise<GetObjectResult>;
|
||||
headObject(input: GetObjectInput): Promise<HeadObjectResult>;
|
||||
deleteObject(input: GetObjectInput): Promise<void>;
|
||||
deleteObject(input: DeleteObjectInput): Promise<void>;
|
||||
|
||||
putObjectConditional?(input: PutObjectInput & (
|
||||
| { ifNoneMatch: "*"; ifMatch?: never }
|
||||
@@ -79,5 +86,5 @@ export interface StorageService {
|
||||
putFile(input: PutFileInput): Promise<PutFileResult>;
|
||||
getObject(companyId: string, objectKey: string, options?: Pick<GetObjectInput, "range" | "signal">): Promise<GetObjectResult>;
|
||||
headObject(companyId: string, objectKey: string, options?: Pick<GetObjectInput, "signal">): Promise<HeadObjectResult>;
|
||||
deleteObject(companyId: string, objectKey: string, options?: Pick<GetObjectInput, "signal">): Promise<void>;
|
||||
deleteObject(companyId: string, objectKey: string, options?: Pick<DeleteObjectInput, "signal" | "allVersions">): Promise<void>;
|
||||
}
|
||||
Reference in new issue
Block a user