Keep computer personal files and workspaces remotely authoritative

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-10 08:31:36 -05:00
1 parent 1dcf713e21
commit 0cf2ea86e8
15 files changed
+491 -27

No files matched your search

@@ -16,6 +16,7 @@ function cacheKeyForTarget(command: string, target: AdapterExecutionTarget | nul
if (target.kind === "local") {
return `local:${target.environmentId ?? ""}:${target.leaseId ?? ""}:${command}`;
}
if (target.transport === "computer") return `computer:${target.environmentId}:${target.resourceAuthority.computerId}:${command}`;
if (target.transport === "sandbox") {
return [
"sandbox",
@@ -231,6 +231,31 @@ describe("claude remote execution", () => {
}));
});
it("keeps an authoritative remote workspace in place while delivering managed Claude assets", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-claude-in-place-"));
cleanupDirs.push(rootDir);
await execute({
runId: "run-in-place",
agent: { id: "agent-1", companyId: "company-1", name: "Claude", adapterType: "claude_local", adapterConfig: {} },
runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null },
config: { engine: "cli", command: "claude" },
context: { paperclipWorkspace: { cwd: rootDir, source: "task_session" } },
executionTarget: {
kind: "remote", transport: "ssh", remoteCwd: "/old-copied-workspace",
workspaceRealization: { mode: "in_place", authoritativeRoot: "/persistent-project", pathAliases: [], outboundRestorePaths: [] },
spec: { host: "127.0.0.1", port: 2222, username: "fixture", remoteWorkspacePath: "/persistent-project", remoteCwd: "/persistent-project",
privateKey: "PRIVATE KEY", knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA", strictHostKeyChecking: true },
},
onLog: async () => {},
});
expect(prepareWorkspaceForSshExecution).not.toHaveBeenCalled();
expect(restoreWorkspaceFromSshExecution).not.toHaveBeenCalled();
expect(syncDirectoryToSsh).toHaveBeenCalledTimes(2);
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({ remoteDir: "/persistent-project/.paperclip-runtime/claude/skills" }));
const call = runChildProcess.mock.calls[0] as unknown as [string, string, string[], { remoteExecution: { remoteCwd: string } }];
expect(call[3].remoteExecution.remoteCwd).toBe("/persistent-project");
});
it("does not resume saved Claude sessions for remote SSH execution without a matching remote identity", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-claude-remote-resume-"));
cleanupDirs.push(rootDir);
@@ -1,3 +1,4 @@
import { adapterExecutionTargetIsCommandBacked } from "@paperclipai/adapter-utils/execution-target";
import { createProviderStoppedBoundary } from "@paperclipai/adapter-utils/provider-stopped-boundary";
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
import fs from "node:fs/promises";
@@ -426,7 +427,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
legacyRemoteExecution: ctx.executionTransport?.remoteExecution,
});
const executionTargetIsRemote = adapterExecutionTargetIsRemote(executionTarget);
const executionTargetIsSandbox = executionTarget?.kind === "remote" && executionTarget.transport === "sandbox";
const executionTargetIsCommandBacked = adapterExecutionTargetIsCommandBacked(executionTarget);
const promptTemplate = asString(
config.promptTemplate,
@@ -482,7 +483,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
graceSec,
extraArgs,
} = runtimeConfig;
Object.assign(env, claudeSandboxPermissionEnv({ dangerouslySkipPermissions, targetIsSandbox: executionTargetIsSandbox }));
Object.assign(env, claudeSandboxPermissionEnv({ dangerouslySkipPermissions, targetIsSandbox: executionTargetIsCommandBacked }));
let loggedEnv = initialLoggedEnv;
let effectiveExecutionCwd = adapterExecutionTargetRemoteCwd(executionTarget, cwd);
const terminalResultCleanupGraceMs = Math.max(
@@ -612,11 +613,12 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
const claudeConfigSeedDir = useManagedRemoteClaudeConfig
? config.managedAiConnection ? sharedClaudeConfigDir : await prepareClaudeConfigSeed(process.env, onLog, agent.companyId)
: null;
const targetWorkspaceRealization = executionTarget?.workspaceRealization ?? null;
const preparedExecutionTargetRuntime = executionTargetIsRemote
? await (async () => {
await onLog(
"stdout",
`[paperclip] Syncing workspace and Claude runtime assets to ${describeAdapterExecutionTarget(executionTarget)}.\n`,
`[paperclip] Syncing ${targetWorkspaceRealization?.mode === "in_place" ? "Claude runtime assets" : "workspace and Claude runtime assets"} to ${describeAdapterExecutionTarget(executionTarget)}.\n`,
);
return await prepareAdapterExecutionTargetRuntime({
runId,
@@ -624,6 +626,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
adapterKey: "claude",
timeoutSec,
workspaceLocalDir: cwd,
workspaceRemoteDir: targetWorkspaceRealization?.mode === "in_place"
? targetWorkspaceRealization.authoritativeRoot : undefined,
syncWorkspace: targetWorkspaceRealization?.mode !== "in_place",
installCommand: SANDBOX_INSTALL_COMMAND,
detectCommand: command,
onProgress: (line) => onLog("stdout", line),
@@ -748,7 +753,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
}
}
let effectiveEffort = effort;
if (executionTargetIsSandbox && effort) {
if (executionTargetIsCommandBacked && effort) {
const supportsEffort = await claudeCommandSupportsEffortFlag({
runId,
command,
@@ -1,3 +1,4 @@
import { adapterExecutionTargetIsCommandBacked } from "@paperclipai/adapter-utils/execution-target";
import { createProviderStoppedBoundary } from "@paperclipai/adapter-utils/provider-stopped-boundary";
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
import fs from "node:fs/promises";
@@ -874,8 +875,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
effectiveExecutionCwd = preparedExecutionTargetRuntime.workspaceRemoteDir;
}
const runtimeExecutionTarget = overrideAdapterExecutionTargetRemoteCwd(executionTarget, effectiveExecutionCwd);
const executionTargetIsSandbox =
runtimeExecutionTarget?.kind === "remote" && runtimeExecutionTarget.transport === "sandbox";
const executionTargetIsCommandBacked =
adapterExecutionTargetIsCommandBacked(runtimeExecutionTarget);
const restoreRemoteWorkspace = preparedExecutionTargetRuntime
? () => preparedExecutionTargetRuntime.restoreWorkspace((line) => onLog("stdout", line))
: null;
@@ -1191,7 +1192,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
}
return notes;
})();
if (executionTargetIsSandbox) {
if (executionTargetIsCommandBacked) {
commandNotes.push(
"Added --skip-git-repo-check for sandbox execution because Codex requires an explicit trust bypass in headless remote workspaces.",
);
@@ -1226,7 +1227,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
forceSaferInvocation ? { ...config, fastMode: false } : config,
{
resumeSessionId,
skipGitRepoCheck: executionTargetIsSandbox,
skipGitRepoCheck: executionTargetIsCommandBacked,
networkAccess: env.PAPERCLIP_RUNNER_NETWORK_ACCESS !== "disabled",
},
);
@@ -1,3 +1,4 @@
import * as persistentFiles from "../services/persistent-agent-files.js";
import fs from "node:fs/promises";
import { execFile as execFileCallback } from "node:child_process";
import { promisify } from "node:util";
@@ -74,6 +75,55 @@ describe("persistent agent directories", () => {
await fs.writeFile(path.join(root, entryFile), initial);
});
it("keeps Boat personal bytes authoritative through warm checkpoints, collection, and editor conflicts", async () => {
const remoteRoot = "/home/user/paperclip/test/agents/target";
const remoteFiles = new Map([[entryFile, Buffer.from("remote instructions")], ["memory.txt", Buffer.from("remote memory")]]);
const readBytes = vi.fn(async (relative: string) => {
const bytes = remoteFiles.get(relative);
if (!bytes) throw { code: "not_found" };
return { bytes, sha256: fileHash(bytes) };
});
const writeBytes = vi.fn(async (relative: string, bytes: Buffer, base: string | null) => {
const current = remoteFiles.get(relative);
if ((current ? fileHash(current) : null) !== base) throw new Error("remote CAS rejected");
remoteFiles.set(relative, bytes); return { sha256: fileHash(bytes) };
});
const seedBytes = vi.fn();
const remote = { root: remoteRoot, list: async () => [], readBytes, writeBytes, seedBytes };
const lookup = vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue(remote as never);
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand");
try {
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
const target = { kind: "remote", transport: "computer", environmentId: randomUUID(), remoteCwd: "/workspace",
fileAuthority: { kind: "remote-persistent", placementId: randomUUID(), root: "/workspace", agentHome: remoteRoot } } as never;
const copy = (await copies.prepare({ companyId, agentId, runId, cwd: home, target, warm: true }))!;
expect(copy.executionRoot).toBe(remoteRoot);
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe("remote instructions");
expect(seedBytes).not.toHaveBeenCalled();
remoteFiles.set("memory.txt", Buffer.from("edited while warm"));
expect(await copies.hasChanges({ companyId, runId, target })).toBe(false);
await copies.checkpointWarm({ companyId, runId, target });
await copies.collectStopped({ companyId, runId, target });
await copies.release(companyId, runId);
expect(shell).not.toHaveBeenCalled();
expect(remoteFiles.get("memory.txt")?.toString()).toBe("edited while warm");
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
await expect(fs.stat(path.join(root, "memory.txt"))).rejects.toMatchObject({ code: "ENOENT" });
const store = agentFileStore(db);
await expect(store.write({ companyId, agentId, path: "memory.txt", bytes: Buffer.from("stale"), baseHash: fileHash(Buffer.from("remote memory")) }, board())).rejects.toMatchObject({ status: 409 });
expect(writeBytes).not.toHaveBeenCalled();
await store.write({ companyId, agentId, path: "memory.txt", bytes: Buffer.from("editor"), baseHash: fileHash(Buffer.from("edited while warm")) }, board());
expect(remoteFiles.get("memory.txt")?.toString()).toBe("editor");
const current = await revisions.readCurrent({ companyId, agentId }, board());
expect(current?.content).toBe("remote instructions");
await revisions.commit({ companyId, agentId, entryFile, content: "editor changed instructions", baseRevisionId: current!.revision.id, source: "api" }, board());
expect(remoteFiles.get(entryFile)?.toString()).toBe("editor changed instructions");
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { lookup.mockRestore(); shell.mockRestore(); }
});
describe("warm directory ownership", () => {
async function nextRun() {
const runId = randomUUID();
@@ -1,3 +1,4 @@
import * as computerModule from "../modules/computers/index.js";
import express from "express";
import type { Request } from "express";
import { execFile } from "node:child_process";
@@ -208,6 +209,35 @@ describeEmbeddedPostgres("workspace file resources", () => {
await tempDb?.cleanup();
});
it("browses a recorded computer placement and confines previews/downloads to safe relative files", async () => {
const workspace = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot: workspace.projectRoot, executionRoot: workspace.executionRoot });
const environmentId = crypto.randomUUID(), placementId = crypto.randomUUID();
await db.update(executionWorkspaces).set({ providerType: "computer", cwd: "/home/user/paperclip/remote/project",
metadata: { fileAuthority: { kind: "remote-persistent", environmentId, placementId } } })
.where(eq(executionWorkspaces.sourceIssueId, graph.issueId));
const bytes = Buffer.from("only on Boat");
const readBytes = vi.fn(async () => ({ bytes, sha256: "unused-by-preview" }));
const list = vi.fn(async () => [{ name: "README.md", kind: "file", size: bytes.length, mtimeMs: 1 },
{ name: ".env", kind: "file", size: 1, mtimeMs: 1 }, { name: ".paperclip-runtime", kind: "directory", size: 0, mtimeMs: 1 }]);
const workspaceFiles = vi.fn(async () => ({ root: "/home/user/paperclip/remote/project", list, readBytes }));
const factory = vi.spyOn(computerModule, "computerService").mockReturnValue({ workspaceFiles } as never);
try {
const service = workspaceFileResourceService(db);
const listed = await service.list(graph.issueId, { workspace: "execution" });
expect(listed.items.map(item => item.title)).toEqual(["README.md"]);
expect(workspaceFiles).toHaveBeenCalledWith({ companyId: graph.companyId, environmentId, placementId });
const content = await service.readContent(graph.issueId, { path: "README.md", workspace: "execution" });
expect(content.content).toEqual({ encoding: "utf8", data: "only on Boat" });
expect(readBytes).toHaveBeenCalledWith("README.md", WORKSPACE_FILE_TEXT_MAX_BYTES);
expect(await service.prepareDownload(graph.issueId, { path: "README.md", workspace: "execution" })).toMatchObject({ bytes });
readBytes.mockClear();
await expect(service.readContent(graph.issueId, { path: ".env", workspace: "execution" })).rejects.toMatchObject({ status: 403 });
await expect(service.readContent(graph.issueId, { path: "../other-company/secret", workspace: "execution" })).rejects.toMatchObject({ status: 403 });
expect(readBytes).not.toHaveBeenCalled();
} finally { factory.mockRestore(); await fs.rm(workspace.root, { recursive: true, force: true }); }
});
it("withholds private task files and private cross-project targets", async () => {
const { projectRoot, executionRoot, targetProjectRoot } = await makeWorkspace();
const graph = await seedGraph(db, { projectRoot, executionRoot, targetProjectRoot });
+3 -2
View File
@@ -56,7 +56,7 @@ export type WorkspaceFileResourceService = {
issueId: string,
input: { path: string; workspace?: "auto" | "execution" | "project" | null; projectId?: string | null; workspaceId?: string | null },
opts?: { issue?: Awaited<ReturnType<WorkspaceFileResourceService["getIssue"]>> },
): Promise<{ resource: ResolvedWorkspaceResource; realPath: string }>;
): Promise<{ resource: ResolvedWorkspaceResource; realPath: string } | { resource: ResolvedWorkspaceResource; bytes: Buffer }>;
};
type FileResourceLimiter = {
@@ -886,7 +886,8 @@ export function fileResourceRoutes(db: Db, opts: {
res.setHeader("Cache-Control", "private, max-age=60");
res.setHeader("X-Content-Type-Options", "nosniff");
res.setHeader("Content-Disposition", `attachment; filename="${safeAttachmentFilename(result.resource.title)}"`);
await pipeline(createReadStream(result.realPath), res);
if ("bytes" in result) res.end(result.bytes);
else await pipeline(createReadStream(result.realPath), res);
return;
}
@@ -1,3 +1,5 @@
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
import { materializeInstructionBytes, readInstructionBytes } from "./agent-instruction-files.js";
import fs from "node:fs/promises";
import path from "node:path";
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
@@ -20,6 +22,7 @@ import { logger } from "../middleware/logger.js";
type Copy = typeof copies.$inferSelect;
export class AgentDirectoryReuseInvalidatedError extends Error {}
const persistent = (row: Pick<Copy, "receipt">) => row.receipt?.fileAuthority === "remote-persistent";
const completed = new Set(["saved", "unchanged", "resolved", "unavailable"]);
const transports = new Map<string, PreparedAdapterExecutionTargetRuntime>();
const key = (row: Pick<Copy, "companyId" | "runId">) => `${row.companyId}:${row.runId}`;
@@ -42,12 +45,17 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
const store = agentFileStore(db);
const ownerFile = (row: Copy) => path.join(path.dirname(row.localRoot), "owner.json");
async function owns(row: Copy) {
if (row.receipt?.warm !== true) return true;
if (persistent(row) || row.receipt?.warm !== true) return true;
return await fs.readFile(ownerFile(row), "utf8").then(s => JSON.parse(s).runId === row.runId).catch(error => error.code === "ENOENT" && row.processStoppedAt !== null);
}
async function canReuse(companyId: string, agentId: string, runId: string) {
const row = await get(companyId, runId);
if (!row || row.agentId !== agentId || row.state !== "warm_saved" || row.errorCode || !await owns(row)) return false;
if (persistent(row)) {
const remote = await persistentAgentFiles(db, companyId, agentId, String(row.receipt?.environmentId));
if (!remote || remote.root !== row.executionRoot) return false;
return (await readPersistentAgentFile(remote, row.entryFile))?.sha256 === row.receipt?.sessionEntryHash;
}
const entry = baseline(row).entries.get(row.entryFile);
if (entry?.kind !== "file" || entry.hash !== row.receipt?.sessionEntryHash) return false;
return store.locked(companyId, agentId, actor(row), false, async (_tx, _agent, root) =>
@@ -96,6 +104,41 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
if (agentInstructionsBundleMode(agent) !== "managed") return null;
const bound = await resolveInstructionActor(db, { type: "agent", companyId: input.companyId, agentId: input.agentId, runId: input.runId });
const root = resolveManagedInstructionsRoot(agent);
if (input.target?.kind === "remote" && input.target.transport === "computer" && input.target.fileAuthority?.kind === "remote-persistent") {
const authority = input.target.fileAuthority;
const remote = await persistentAgentFiles(db, input.companyId, input.agentId, input.target.environmentId ?? undefined);
if (!remote || remote.root !== authority.agentHome) throw conflict("Persistent agent placement changed");
await store.locked(input.companyId, input.agentId, bound, false, async (_tx, _agent, canonical) => seedPersistentAgentHome(remote, canonical));
const entryFile = deriveBundleState(agent).entryFile;
let entry = await readPersistentAgentFile(remote, entryFile);
if (!entry) {
// An adopted directory may predate Paperclip. Deliver the missing managed
// entry with create-only CAS, without replacing any personal bytes.
const seed = await readInstructionBytes(root, entryFile);
if (!seed) throw notFound("The persistent agent instruction entry is missing");
await remote.writeBytes(entryFile, seed, null);
entry = await readPersistentAgentFile(remote, entryFile);
if (!entry) throw notFound("The persistent agent instruction entry is missing");
}
// Only prompt bytes are staged locally. This scratch is never an authority
// for personal files, nor is it uploaded into the persistent directory.
const localRoot = path.join(path.dirname(root), "file-sync", "runs", input.runId, "live");
await materializeInstructionBytes(localRoot, entryFile, entry.bytes);
const values = { entryFile, baseRevisionId: null, baseHash: entry.sha256, localRoot,
executionRoot: authority.agentHome, location: `remote:${input.target.environmentId}`, state: "prepared",
candidateBase64: null, candidateHash: null, processStoppedAt: null, attempts: 0,
nextAttemptAt: null, errorCode: null, errorMessage: null,
receipt: { schema: AGENT_FILES_CONTRACT, fileAuthority: "remote-persistent", placementId: authority.placementId,
environmentId: input.target.environmentId, workspaceCwd: input.cwd, warm: input.warm === true,
sessionEntryHash: entry.sha256, cleanupPending: false } };
const existing = await get(input.companyId, input.runId);
if (existing && (!persistent(existing) || existing.executionRoot !== authority.agentHome)) throw conflict("Agent file authority changed");
const result = existing ? await patch(existing, values)
: (await db.insert(copies).values({ companyId: input.companyId, agentId: input.agentId, runId: input.runId,
responsibleUserId: bound.onBehalfOfUserId!, ...values }).returning())[0]!;
input.onWarmHandoff?.(result);
return result;
}
if (input.reuseRunId) {
const prior = await get(input.companyId, input.reuseRunId);
if (!prior || prior.agentId !== input.agentId) throw conflict("Managed warm directory owner changed");
@@ -241,6 +284,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
return inspectAgentDirectory(row.localRoot);
}
async function hasChanges(row: Copy, target?: AdapterExecutionTarget | null) {
if (persistent(row)) return false; // Bytes already live on the authoritative computer.
try {
if (row.receipt?.retainedByRunId || row.receipt?.retirementRequired || typeof row.receipt?.materializationIdentity !== "string") return true;
const observed = await observe(row, target);
@@ -251,6 +295,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
}
/** Rebind one physical materialization; earlier runs lose collection authority. */
async function adopt(input: { companyId: string; agentId: string; runId: string; previousRunId: string; target?: AdapterExecutionTarget | null; cwd: string; allowRetirementHandoff?: boolean }) {
if (input.target?.kind === "remote" && input.target.transport === "computer" && input.target.fileAuthority?.kind === "remote-persistent") return prepareCopy({ ...input, warm: true });
let prior = await get(input.companyId, input.previousRunId);
if (!prior || prior.agentId !== input.agentId || prior.state !== "unchanged_turn" || prior.receipt?.retainedByRunId || prior.processStoppedAt
|| prior.receipt?.workspaceCwd !== input.cwd || prior.location !== (input.target?.kind === "remote" ? `remote:${input.target.environmentId ?? ""}` : "local")) return null;
@@ -317,6 +362,8 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
return adopted;
}
async function checkpoint(row: Copy, target?: AdapterExecutionTarget | null, stopped = false): Promise<Copy> {
if (persistent(row)) return patch(row, { state: stopped ? "unchanged" : "warm_saved", nextAttemptAt: null,
...(stopped ? { processStoppedAt: new Date() } : {}) });
if (!await owns(row) || row.state === "superseded") return row;
let failure: unknown;
for (let attempt = 0; attempt < 3; attempt++) {
@@ -356,6 +403,11 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
receipt: { ...row.receipt, storageWarning: storageLimit ? agentStorageWarning(failure instanceof AgentFileLimitError ? failure.message : "Agent folder exceeds a storage limit") : null } });
}
async function collectStopped(row: Copy, target?: AdapterExecutionTarget | null) {
if (persistent(row)) {
const settled = await patch(row, { state: "unchanged", processStoppedAt: new Date(), nextAttemptAt: null });
await release(settled);
return settled;
}
if (row.receipt?.retainedByRunId) return row;
if (!await owns(row) || row.state === "superseded") return row;
if (row.receipt?.warm === true && !completed.has(row.state)) {
@@ -412,6 +464,11 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
completed.has(row.state) && row.receipt?.cleanupPending === false;
}
async function release(row: Copy, target?: AdapterExecutionTarget | null) {
if (persistent(row)) {
// No remote command, snapshot, or directory deletion is legal here.
if (row.processStoppedAt) await fs.rm(path.dirname(row.localRoot), { recursive: true, force: true });
return;
}
if (releaseIsNoop(row) || !await owns(row)) return;
const destroyedOnly = row.receipt?.cleanupDestroyedOnly === true;
if (row.receipt?.retainedByRunId || row.state === "superseded" || !await owns(row) || !row.processStoppedAt) return;
@@ -503,6 +560,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
return lease;
}
async function recoverStoppedRemote(row: Copy) {
if (persistent(row)) return patch(row, { state: "unchanged", processStoppedAt: row.processStoppedAt ?? new Date(), nextAttemptAt: null });
const lease = await ownedRemoteLease(row);
const destroyed = lease && hasRemoteTerminationReceipt(lease)
&& (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed";
@@ -553,7 +611,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
const current = await get(row.companyId, row.runId);
// This check changes no receipt or bytes. A concurrent stop merely leaves
// cleanup to its owner or the recovery sweep; it grants no new authority.
if (!current || releaseIsNoop(current)) return;
if (!current || (!persistent(current) && releaseIsNoop(current))) return;
await serial(current, release, "agent_directory_release");
},
};
+23 -7
View File
@@ -1,3 +1,4 @@
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import { cachedAgentFileManifest, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js";
@@ -163,13 +164,15 @@ export async function snapshotAgentFiles(root: string, enforceLimits = true): Pr
export function agentFileStore(db: Db) {
async function locked<T>(companyId: string, agentId: string, actor: AuthorizationActor, write: boolean,
fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor) => Promise<T>) {
fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor, remote: Awaited<ReturnType<typeof persistentAgentFiles>>) => Promise<T>) {
return db.transaction(async tx => {
const [agent] = await tx.select().from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))).for("update");
if (!agent) throw notFound("Agent not found");
const bound = write ? await authorizeInstructionCommit(tx, actor, agent) : await authorizeInstructionRead(tx, actor, agent);
const root = await adoptAgentFiles(tx, agent);
return fn(tx, agent, root, bound);
const remote = await persistentAgentFiles(db, companyId, agentId);
if (remote) await seedPersistentAgentHome(remote, root);
return fn(tx, agent, root, bound, remote);
});
}
async function audit(tx: Tx, agent: Agent, actor: AuthorizationActor, details: Record<string, unknown>) {
@@ -183,29 +186,41 @@ export function agentFileStore(db: Db) {
download: async (companyId: string, agentId: string, relative: string, actor: AuthorizationActor) => {
const opened: { file: Awaited<ReturnType<typeof openAgentFile>> } = { file: null };
try {
const file = await locked(companyId, agentId, actor, false, async (_tx, _agent, root) => {
const file = await locked(companyId, agentId, actor, false, async (_tx, _agent, root, _bound, remote) => {
if (remote) return { remote: await readPersistentAgentFile(remote, relative) };
opened.file = await openAgentFile(root, instructionPath(relative));
return opened.file;
});
// Transfer outside the database lock. The open descriptor pins the file
// across concurrent atomic replacements; end bounds concurrent growth.
if (!file) return null;
if ("remote" in file) return file.remote ? { size: file.remote.bytes.length, stream: Readable.from([file.remote.bytes]) } : null;
if (file.size === 0) { await file.handle.close(); return { size: 0, stream: Readable.from([]) }; }
return { size: file.size, stream: file.handle.createReadStream({ end: file.size - 1 }) };
} catch (error) { await opened.file?.handle.close(); throw error; }
},
read: (companyId: string, agentId: string, relative: string, actor: AuthorizationActor) =>
locked(companyId, agentId, actor, false, (_tx, _agent, root) => readAgentFile(root, instructionPath(relative))),
locked(companyId, agentId, actor, false, async (_tx, _agent, root, _bound, remote) => remote ? (await readPersistentAgentFile(remote, relative))?.bytes ?? null : readAgentFile(root, instructionPath(relative))),
write: (input: { companyId: string; agentId: string; path: string; bytes: Buffer | null; baseHash: string | null }, actor: AuthorizationActor) =>
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound) => {
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound, remote) => {
const relative = agentFilePath(input.path);
if (input.bytes) assertFileSize(input.bytes.length, relative);
const previous = await inspectAgentFile(root, relative);
const remotePrevious = remote ? await readPersistentAgentFile(remote, relative) : null;
const previous = remote ? (remotePrevious ? { hash: remotePrevious.sha256, size: remotePrevious.bytes.length } : null) : await inspectAgentFile(root, relative);
const currentHash = previous?.hash ?? null;
const incomingHash = input.bytes === null ? null : fileHash(input.bytes);
if (currentHash === incomingHash) return { contentHash: currentHash, changed: false };
if (currentHash !== input.baseHash) throw conflict("This file changed since it was read. Reload before saving.", { code: "AGENT_FILE_CONFLICT", path: relative, currentHash });
if (input.bytes === null && relative === deriveBundleState(agent).entryFile) throw unprocessable("The configured instruction entry cannot be deleted");
if (remote) {
if (input.bytes !== null && relative === deriveBundleState(agent).entryFile) instructionBytes(input.bytes);
// The remote operation checks the hash again immediately before atomic
// replacement. Arbitrary shell writers do not participate in this lock.
if (input.bytes === null) await remote.remove(relative, currentHash!);
else await remote.writeBytes(relative, input.bytes, input.baseHash);
await audit(tx, agent, bound, { path: relative, contentHash: incomingHash });
return { contentHash: incomingHash, changed: true };
}
if (input.bytes !== null) {
if (relative === deriveBundleState(agent).entryFile) instructionBytes(input.bytes);
// A quota check needs metadata only. Do not hash unrelated large
@@ -221,7 +236,8 @@ export function agentFileStore(db: Db) {
return { contentHash: incomingHash, changed: true };
}),
apply: (input: { companyId: string; agentId: string; sourceDir: string; baseline: DirectorySnapshot; checkpoint?: AgentFileManifest }, actor: AuthorizationActor) =>
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound) => {
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound, remote) => {
if (remote) throw unprocessable("Persistent remote agent files cannot accept controller copy-back");
const incoming = input.checkpoint ? checkpointSnapshot(input.checkpoint) : await snapshotAgentFiles(input.sourceDir);
const entryPath = deriveBundleState(agent).entryFile;
if (incoming.entries.get(entryPath)?.kind !== "file") throw unprocessable("The configured instruction entry cannot be deleted");
@@ -1,3 +1,4 @@
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js";
import { createHash } from "node:crypto";
import { adoptAgentFiles, agentFileToken } from "./agent-file-store.js";
@@ -107,7 +108,7 @@ export function agentInstructionRevisionService(db: Db) {
}
const state = deriveBundleState(agent);
const entryFile = instructionPath(state.entryFile);
return { agent, entryFile, root: resolveManagedInstructionsRoot(agent) };
return { agent, entryFile, root: resolveManagedInstructionsRoot(agent), remote: await persistentAgentFiles(db, agent.companyId, agent.id) };
}
async function authorizeRead(
tx: Tx,
@@ -156,7 +157,9 @@ export function agentInstructionRevisionService(db: Db) {
// live in the agent directory; old UUID clients receive a content ETag.
async function currentFile(tx: Tx, target: InstructionTarget, state: Awaited<ReturnType<typeof lockTarget>>, actor?: RevisionActor) {
await adoptAgentFiles(tx, state.agent);
const bytes = await readInstructionBytes(state.root, state.entryFile);
if (state.remote) await seedPersistentAgentHome(state.remote, state.root);
const bytes = state.remote ? (await readPersistentAgentFile(state.remote, state.entryFile))?.bytes ?? null
: await readInstructionBytes(state.root, state.entryFile);
if (bytes === null) return null;
return currentRow(target, state.entryFile, bytes, actor);
}
@@ -235,7 +238,7 @@ export function agentInstructionRevisionService(db: Db) {
"Configured instruction entry changed; read the current entry and retry",
{ code: "INSTRUCTION_ENTRY_CHANGED", entryFile: state.entryFile },
);
await assertInstructionPathSafe(state.root, state.entryFile);
if (!state.remote) await assertInstructionPathSafe(state.root, state.entryFile);
const current = await currentFile(tx, input, state, bound);
const restored =
"restoreRevisionId" in input
@@ -285,7 +288,8 @@ export function agentInstructionRevisionService(db: Db) {
},
);
}
await materializeInstructionBytes(state.root, state.entryFile, candidate);
if (state.remote) await state.remote.writeBytes(state.entryFile, candidate, current?.contentHash ?? null);
else await materializeInstructionBytes(state.root, state.entryFile, candidate);
const row = currentRow(input, input.entryFile, candidate, bound);
await tx.insert(activityLog).values({ companyId: input.companyId,
actorType: bound.type === "plugin" ? "plugin" : bound.type === "board" ? "user" : "agent",
@@ -47,6 +47,7 @@ export async function collectStoppedInstructionCopyWithRetries<T extends Pick<Co
}
export function instructionWorkingCopyGuidance(copy: Pick<Copy, "executionRoot" | "entryFile" | "receipt">) {
if (copy.receipt?.fileAuthority === "remote-persistent") return `Your persistent agent directory is ${copy.executionRoot} (AGENT_HOME). Your instruction entry is ${copy.executionRoot}/${copy.entryFile}. Changes are saved directly on this computer and remain across turns. Provider HOME and credentials are separate runtime directories.`;
if (isAgentDirectoryCopy(copy)) return `Your persistent agent directory is ${copy.executionRoot} (AGENT_HOME). Your instruction entry is ${copy.executionRoot}/${copy.entryFile}. Read and write your own files and subfolders there. This directory belongs to this agent across tasks and sessions; task files belong in the task working directory. Paperclip restores this directory before execution and saves validated changes at turn boundaries. A warm native Codex session keeps the same writable directory between turns. Other sessions collect after the provider stops. Regular files, including binary files, persist; symlinks and special files are unsupported. Check the agent-files save receipt before claiming persistence; Only files you change or delete are synchronized. If another run changes the same file, the last completed synchronization wins. Temporary copies are removed when the owning session stops; there is no per-run file history. Storage allows 256 MiB per file, 2 GiB total, and 100,000 entries; the instruction entry must remain UTF-8 and at most 1 MiB. Reaching a storage limit never prevents this or future tasks from running. Remove or shrink files to free space; changes that exceed the limits will not be saved.${typeof copy.receipt?.storageWarning === "string" ? `\n\n${copy.receipt.storageWarning}` : ""}`;
return `Your editable agent instruction file is ${copy.executionRoot}/${copy.entryFile}. Edit this registered private copy normally. After this run stops, Paperclip saves changed content as a persistent revision if your responsible user still has permission and the baseline has not changed. Check the run's instruction-save receipt before claiming persistence. Use read_agent_instructions, update_agent_instructions, get_agent_instruction_history, and restore_agent_instructions for immediate saves and history. Read first and pin the returned revision. Preserve conflicts; never silently retry against a newer head. Repository instructions, skills, and the loaded prompt are separate and are not collected.`;
}
@@ -103,7 +104,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
}
async function prepare(input: { companyId: string; agentId: string; runId: string; target?: AdapterExecutionTarget | null; cwd: string; legacy?: boolean; warm?: boolean; reuseRunId?: string; onWarmHandoff?: (copy: Copy) => void }) {
const existing = await get(input.companyId, input.runId);
if (isAgentDirectoryCopy(existing) || (!existing && !input.legacy)) return directories.prepare(input);
if ((input.target?.kind === "remote" && input.target.transport === "computer" && input.target.fileAuthority?.kind === "remote-persistent") || isAgentDirectoryCopy(existing) || (!existing && !input.legacy)) return directories.prepare(input);
let refreshStoppedCopy = false;
const workspace = await fs.realpath(input.cwd);
const expectedLocalRoot = path.join(workspace, ".paperclip-runtime", `instruction-edits-${input.runId}`, "instructions");
+64 -2
View File
@@ -1,9 +1,10 @@
import { inspectAgentFile, fileHash, agentFilePath, MAX_AGENT_FILE_BYTES } from "./agent-file-store.js";
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
import { adoptAgentFiles, inspectAgentFile, fileHash, agentFilePath, MAX_AGENT_FILE_BYTES } from "./agent-file-store.js";
import fs from "node:fs/promises";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { agentInstructionHeads, agents, type Db } from "@paperclipai/db";
import { instructionPath, assertInstructionPathSafe, instructionBytes, readInstructionBytes } from "./agent-instruction-files.js";
import { instructionPath, assertInstructionPathSafe, instructionBytes, readInstructionBytes, MAX_INSTRUCTION_BYTES } from "./agent-instruction-files.js";
import { notFound, unprocessable } from "../errors.js";
import { resolveHomeAwarePath, resolvePaperclipInstanceRoot } from "../home-paths.js";
@@ -489,6 +490,30 @@ export function syncInstructionsBundleConfigFromFilePath(
}
export function agentInstructionsService(db?: Db) {
async function remoteFiles(agent: AgentLike) {
const remote = db ? await persistentAgentFiles(db, agent.companyId, agent.id) : null;
if (remote && db) {
const root = await db.transaction(async tx => {
const [current] = await tx.select().from(agents).where(and(eq(agents.id, agent.id), eq(agents.companyId, agent.companyId)));
if (!current) throw notFound("Agent not found");
return adoptAgentFiles(tx, current);
});
await seedPersistentAgentHome(remote, root);
}
return remote;
}
async function remotePaths(remote: NonNullable<Awaited<ReturnType<typeof remoteFiles>>>, relative = ""): Promise<Array<{ path: string; size: number }>> {
const result: Array<{ path: string; size: number }> = [];
for (const entry of await remote.list(relative || undefined)) {
const name = relative ? `${relative}/${entry.name}` : entry.name;
instructionPath(name);
if (entry.kind === "directory") {
if (!IGNORED_INSTRUCTIONS_DIRECTORY_NAMES.has(entry.name)) result.push(...await remotePaths(remote, name));
} else if (!IGNORED_INSTRUCTIONS_FILE_NAMES.has(entry.name)) result.push({ path: name, size: entry.size });
if (result.length > 100_000) throw unprocessable("Agent folder exceeds its entry limit");
}
return result;
}
async function assertUnversionedEntry(agent: AgentLike, entryFile: string, connection: Db | Parameters<Parameters<Db["transaction"]>[0]>[0] | undefined = db) {
if (!connection) throw unprocessable("Bundle initialization requires the database-backed instructions service");
const [head] = await connection.select().from(agentInstructionHeads).where(and(eq(agentInstructionHeads.companyId, agent.companyId),
@@ -497,6 +522,15 @@ export function agentInstructionsService(db?: Db) {
}
async function getBundle(agent: AgentLike): Promise<AgentInstructionsBundle> {
const state = await recoverManagedBundleState(agent, deriveBundleState(agent));
const remote = await remoteFiles(agent);
if (remote) {
const summaries: AgentInstructionsFileSummary[] = [];
for (const { path: relative, size } of await remotePaths(remote)) {
const file = size <= MAX_INSTRUCTION_BYTES ? await readPersistentAgentFile(remote, relative) : null;
summaries.push(summarizeFile(relative, state.entryFile, size, file?.bytes ?? null, file?.sha256));
}
return toBundle(agent, { ...state, rootPath: remote.root, resolvedEntryPath: `${remote.root}/${state.entryFile}` }, summaries);
}
if (!state.rootPath) return toBundle(agent, state, []);
await assertInstructionPathSafe(state.rootPath, state.entryFile);
const stat = await statIfExists(state.rootPath);
@@ -532,6 +566,13 @@ export function agentInstructionsService(db?: Db) {
content,
};
}
const remote = await remoteFiles(agent);
if (remote) {
const file = await readPersistentAgentFile(remote, relativePath);
if (!file) throw notFound("Instructions file not found");
const summary = summarizeFile(relativePath, state.entryFile, file.bytes.length, file.bytes, file.sha256);
return { ...summary, content: summary.binary ? "" : file.bytes.toString("utf8") };
}
if (!state.rootPath) throw notFound("Agent instructions bundle is not configured");
await assertInstructionPathSafe(state.rootPath, relativePath);
const file = await inspectAgentFile(state.rootPath, relativePath);
@@ -593,6 +634,14 @@ export function agentInstructionsService(db?: Db) {
const state = await recoverManagedBundleState(agent, deriveBundleState(agent));
const nextMode = input.mode ?? state.mode ?? "managed";
const nextEntryFile = input.entryFile ? normalizeRelativeFilePath(input.entryFile) : state.entryFile;
const remote = await remoteFiles(agent);
if (remote) {
if (nextMode !== "managed") throw unprocessable("Persistent computer files require managed instructions");
if (!await readPersistentAgentFile(remote, nextEntryFile)) throw notFound("The selected instruction entry does not exist on the computer");
const adapterConfig = applyBundleConfig(state.config, { mode: "managed", rootPath: resolveManagedInstructionsRoot(agent),
entryFile: nextEntryFile, clearLegacyPromptTemplate: input.clearLegacyPromptTemplate });
return { adapterConfig, bundle: await getBundle({ ...agent, adapterConfig }) };
}
let nextRootPath: string;
if (nextMode === "managed") {
@@ -665,6 +714,7 @@ export function agentInstructionsService(db?: Db) {
throw unprocessable("Entry edits require the canonical instruction commit service and baseRevisionId", { code: "INSTRUCTION_REVISION_REQUIRED" });
}
if (configured.mode !== "external") agentFilePath(relativePath);
if (await remoteFiles(agent)) throw unprocessable("Persistent remote file edits require the content API and expected content hash");
const prepared = await ensureWritableBundle(agent, options);
instructionBytes(content);
await assertInstructionPathSafe(prepared.state.rootPath!, relativePath);
@@ -693,6 +743,7 @@ export function agentInstructionsService(db?: Db) {
if (normalizedPath === state.entryFile) {
throw unprocessable("Cannot delete the bundle entry file");
}
if (await remoteFiles(agent)) throw unprocessable("Persistent remote file deletion requires the content API and expected content hash");
const absolutePath = await assertInstructionPathSafe(state.rootPath, normalizedPath);
await fs.rm(absolutePath, { force: true });
const adapterConfig = buildPersistedBundleConfig(derived, state);
@@ -728,6 +779,16 @@ export function agentInstructionsService(db?: Db) {
warnings: string[];
}> {
const state = await recoverManagedBundleState(agent, deriveBundleState(agent));
const remote = await remoteFiles(agent);
if (remote) {
const files: Record<string, string> = {};
for (const { path: relative, size } of await remotePaths(remote)) {
if (size > MAX_INSTRUCTION_BYTES) continue;
const file = await readPersistentAgentFile(remote, relative);
if (file) files[relative] = file.bytes.toString("utf8");
}
return { files, entryFile: state.entryFile, warnings: state.warnings };
}
if (state.rootPath) {
const stat = await statIfExists(state.rootPath);
if (stat?.isDirectory()) {
@@ -760,6 +821,7 @@ export function agentInstructionsService(db?: Db) {
entryFile?: string;
},
): Promise<{ bundle: AgentInstructionsBundle; adapterConfig: Record<string, unknown> }> {
if (await remoteFiles(agent)) throw unprocessable("Replace individual persistent files using their expected content hash");
const rootPath = resolveManagedInstructionsRoot(agent);
const entryFile = options?.entryFile ? normalizeRelativeFilePath(options.entryFile) : ENTRY_FILE_DEFAULT;
@@ -0,0 +1,39 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { seedPersistentAgentHome } from "./persistent-agent-files.js";
describe("persistent home initialization", () => {
const roots: string[] = [];
afterEach(async () => { for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); });
async function local() {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "boat-agent-seed-")); roots.push(root);
await fs.mkdir(path.join(root, "memory"));
await fs.writeFile(path.join(root, "AGENTS.md"), "instructions");
await fs.writeFile(path.join(root, "memory", "binary.bin"), Buffer.from([0, 255, 1]));
return root;
}
it("adopts an existing empty remote folder without reading or uploading the controller tree", async () => {
const seedBytes = vi.fn();
await seedPersistentAgentHome({ list: async () => [], seedBytes } as never, "/missing-controller-folder");
expect(seedBytes).not.toHaveBeenCalled();
});
it("seeds absent personal storage once with exact binary bytes", async () => {
const seedBytes = vi.fn(async () => ({ seeded: true }));
await seedPersistentAgentHome({ list: async () => { throw { code: "not_found" }; }, seedBytes } as never, await local());
expect(seedBytes).toHaveBeenCalledWith({ "AGENTS.md": Buffer.from("instructions"), "memory/binary.bin": Buffer.from([0, 255, 1]) });
});
it("does not interpret provider outages as an absent directory", async () => {
const seedBytes = vi.fn();
await expect(seedPersistentAgentHome({ list: async () => { throw new Error("offline"); }, seedBytes } as never, await local())).rejects.toThrow("offline");
expect(seedBytes).not.toHaveBeenCalled();
});
it("rejects symlinked initial files before any upload", async () => {
const root = await local();
await fs.symlink(path.join(root, "AGENTS.md"), path.join(root, "alias.md"));
const seedBytes = vi.fn();
await expect(seedPersistentAgentHome({ list: async () => { throw { code: "not_found" }; }, seedBytes } as never, root)).rejects.toThrow("links or special files");
expect(seedBytes).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,92 @@
import fs from "node:fs/promises";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { agents, environments, instanceSettings, type Db } from "@paperclipai/db";
import { computerService } from "../modules/computers/index.js";
import { HttpError } from "../errors.js";
import { instructionPath } from "./agent-instruction-files.js";
/** An assignment selects authority, never an arbitrary path supplied by a caller.
* The computer service separately checks the company attachment and placement. */
export async function persistentAgentFiles(db: Db, companyId: string, agentId: string, environmentId?: string) {
let selected = environmentId;
if (!selected) {
const [agent] = await db.select({ environmentId: agents.defaultEnvironmentId }).from(agents)
.where(and(eq(agents.companyId, companyId), eq(agents.id, agentId)));
if (!agent) return null;
selected = agent.environmentId ?? undefined;
}
if (!selected) {
const [settings] = await db.select({ environmentId: instanceSettings.defaultEnvironmentId }).from(instanceSettings)
.where(eq(instanceSettings.singletonKey, "default"));
selected = settings?.environmentId ?? undefined;
}
if (!selected) return null;
const [environment] = await db.select({ driver: environments.driver, config: environments.config }).from(environments)
.where(eq(environments.id, selected));
if (environment?.driver !== "computer" || environment.config.provider !== "boat") return null;
return computerFileAccess(() => computerService(db).files({ companyId, agentId, environmentId: selected! }));
}
export function isMissingRemoteFile(error: unknown) {
const value = error as { status?: number; code?: string };
return value?.status === 404 || value?.code === "not_found";
}
export async function readPersistentAgentFile(files: NonNullable<Awaited<ReturnType<typeof persistentAgentFiles>>>, relative: string) {
try { return await files.readBytes(instructionPath(relative)); }
catch (error) { if (isMissingRemoteFile(error)) return null; throw error; }
}
/** First attachment only. Existing directories are adopted, including empty ones.
* Later turns never read/upload the controller copy of personal files. */
export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnType<typeof persistentAgentFiles>>>, localRoot: string) {
try { await files.list(); return; }
catch (error) { if (!isMissingRemoteFile(error)) throw error; }
const seed: Record<string, Buffer> = {};
let total = 0, count = 0;
async function walk(relative: string) {
const entries = await fs.readdir(path.join(localRoot, relative), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT" && relative === "") return [];
throw error;
});
for (const entry of entries) {
const name = relative ? `${relative}/${entry.name}` : entry.name;
instructionPath(name);
if (++count > 100_000) throw new Error("Initial agent folder exceeds its entry limit");
const stat = await fs.lstat(path.join(localRoot, name));
if (stat.isDirectory()) await walk(name);
else if (stat.isFile() && stat.nlink === 1) {
total += stat.size;
if (stat.size > 256 * 1024 * 1024 || total > 2 * 1024 * 1024 * 1024) throw new Error("Initial agent folder exceeds its storage limit");
seed[name] = await fs.readFile(path.join(localRoot, name));
} else throw new Error("Agent folders cannot contain links or special files");
}
}
await walk("");
await files.seedBytes(seed);
}
/** Translate the domain's transport-neutral errors at the HTTP service edge. */
export async function computerFileAccess<T extends object>(open: () => Promise<T>): Promise<T> {
function translate(error: unknown): never {
const code = (error as { code?: string })?.code;
const status = code === "not_found" ? 404 : code === "forbidden" ? 403 : code === "conflict" ? 409
: code === "invalid" ? 422 : code === "provider_error" ? 502 : null;
if (status) throw new HttpError(status, error instanceof Error ? error.message : "Computer file operation failed", {
code: code === "conflict" ? "AGENT_FILE_CONFLICT" : code,
});
throw error;
}
let access: T;
try { access = await open(); } catch (error) { return translate(error); }
return new Proxy(access, {
get(target, key, receiver) {
const value = Reflect.get(target, key, receiver);
if (typeof value !== "function") return value;
return async (...args: unknown[]) => {
try { return await value.apply(target, args); } catch (error) { return translate(error); }
};
},
});
}
@@ -1,3 +1,5 @@
import { computerFileAccess } from "./persistent-agent-files.js";
import { computerService } from "../modules/computers/index.js";
import { issueReadSqlCondition, executionWorkspaceReadSqlCondition, projectReadSqlCondition, type AuthorizationActor } from "./authorization.js";
import fs from "node:fs/promises";
import path from "node:path";
@@ -45,6 +47,7 @@ export interface WorkspaceFileScanContext {
const DENIED_SEGMENTS = new Set([
".git",
".paperclip",
".paperclip-runtime",
"node_modules",
".pnpm-store",
".yarn",
@@ -121,6 +124,7 @@ type WorkspaceCandidate = {
label: string;
rootPath: string | null;
remote: boolean;
placement?: { companyId: string; environmentId: string; placementId: string };
};
type NormalizedPath = {
@@ -480,6 +484,13 @@ function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceC
label: row.name,
rootPath,
remote,
...(row.metadata?.fileAuthority && typeof row.metadata.fileAuthority === "object" &&
(row.metadata.fileAuthority as Record<string, unknown>).kind === "remote-persistent" &&
typeof (row.metadata.fileAuthority as Record<string, unknown>).environmentId === "string" &&
typeof (row.metadata.fileAuthority as Record<string, unknown>).placementId === "string"
? { placement: { companyId: row.companyId,
environmentId: (row.metadata.fileAuthority as Record<string, string>).environmentId!,
placementId: (row.metadata.fileAuthority as Record<string, string>).placementId! } } : {}),
};
}
@@ -1039,6 +1050,65 @@ async function listChangedWorkspaceFiles(input: {
}
export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor) {
async function placementFiles(candidate: WorkspaceCandidate) {
if (!candidate.placement) throw unprocessable("Workspace placement is unavailable", { code: "remote_workspace" });
return computerFileAccess(() => computerService(db).workspaceFiles(candidate.placement!));
}
async function statPersistentCandidate(candidate: WorkspaceCandidate, normalized: NormalizedPath, directory = false) {
throwIfDenied(normalized.segments);
const files = await placementFiles(candidate);
if (directory) {
await files.list(normalized.relativePath);
return directoryResource({ candidate, relativePath: normalized.relativePath });
}
const parent = path.posix.dirname(normalized.relativePath);
const entry = (await files.list(parent === "." ? undefined : parent)).find(item => item.name === path.posix.basename(normalized.relativePath));
if (!entry || entry.kind !== "file") throw notFound("Workspace file not found");
const item = listItemFromStat({ candidate, relativePath: normalized.relativePath,
stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!;
const { relativePath: _relative, modifiedAt: _modified, ...resource } = item;
return { ...resource, denialReason: item.capabilities.preview ? null : entry.size > previewCapForKind(item.previewKind) ? "too_large" : "unsupported_content" } as ResolvedWorkspaceResource;
}
async function persistentContent(candidate: WorkspaceCandidate, normalized: NormalizedPath): Promise<WorkspaceFileContent> {
const resource = await statPersistentCandidate(candidate, normalized);
if (!resource.capabilities.preview) throw unprocessable("Workspace file cannot be previewed", { code: resource.denialReason ?? "unsupported_content" });
const { bytes } = await (await placementFiles(candidate)).readBytes(normalized.relativePath, previewCapForKind(resource.previewKind));
if (resource.previewKind === "text" && !looksLikeText(bytes.subarray(0, TEXT_SNIFF_BYTES))) throw unprocessable("Workspace file is not text", { code: "binary_content" });
return { resource, content: { encoding: resource.previewKind === "text" ? "utf8" : "base64", data: bytes.toString(resource.previewKind === "text" ? "utf8" : "base64") } };
}
async function persistentList(candidate: WorkspaceCandidate, options: {
selector: WorkspaceFileSelector; mode: WorkspaceFileListMode; normalizedPath: NormalizedPath | null;
q: string | null; normalizedQuery: string | null; limit: number; offset: number;
}): Promise<WorkspaceFileListResponse> {
const { selector, mode, normalizedPath, q, normalizedQuery, limit, offset } = options;
if (mode === "changed") return unavailableFileList({ selector, mode, path: normalizedPath?.relativePath ?? null, q, limit, offset, candidate, reason: "changed_unavailable" });
if (normalizedPath) throwIfDenied(normalizedPath.segments);
const files = await placementFiles(candidate);
const items: WorkspaceFileListItem[] = [];
let scannedCount = 0, truncated = false;
const recursive = mode === "recent" || Boolean(normalizedQuery);
async function walk(relative: string, depth: number) {
if (depth > MAX_LIST_DEPTH) { truncated = true; return; }
for (const entry of await files.list(relative || undefined)) {
if (++scannedCount > WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES) { truncated = true; return; }
const name = relative ? `${relative}/${entry.name}` : entry.name;
const normalized = normalizeWorkspaceRelativePath(name);
if (shouldPruneSegments(normalized.segments)) continue;
if (entry.kind === "directory") {
if (recursive) await walk(name, depth + 1);
else items.push(listItemFromDirectory({ candidate, relativePath: name, stat: { mtime: new Date(entry.mtimeMs) } }));
} else if (matchesSearch(name, normalizedQuery)) {
items.push(listItemFromStat({ candidate, relativePath: name, stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!);
}
if (truncated) return;
}
}
await walk(normalizedPath?.relativePath ?? "", 0);
items.sort((a, b) => mode === "recent" ? String(b.modifiedAt).localeCompare(String(a.modifiedAt)) :
(a.kind === b.kind ? a.relativePath.localeCompare(b.relativePath) : a.kind === "directory" ? -1 : 1));
return availableFileList({ selector, mode, path: normalizedPath?.relativePath ?? null, q, limit, offset, candidate,
items: items.slice(offset, offset + limit), scannedCount, truncated: truncated || items.length > offset + limit });
}
async function visibleCandidates(candidates: WorkspaceCandidate[]) {
if (!actor) return candidates;
const visible: WorkspaceCandidate[] = [];
@@ -1305,6 +1375,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
const hasExplicitTarget = Boolean(explicitTarget?.candidate);
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.placement) return availabilityResult(item.query, await statPersistentCandidate(candidate, item.normalizedPath, item.directory));
if (candidate.remote) {
if (hasExplicitTarget || item.query.workspace !== "auto") {
return availabilityResult(item.query, remoteResource(candidate, item.normalizedPath.relativePath));
@@ -1373,6 +1444,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.placement) return statPersistentCandidate(candidate, normalized, isDirectoryRequest);
if (candidate.remote) {
if (explicitTarget || selector !== "auto") return remoteResource(candidate, normalized.relativePath);
continue;
@@ -1431,6 +1503,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let firstUnavailable: { candidate: WorkspaceCandidate; reason: string } | null = null;
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.placement) return persistentList(candidate, { selector, mode, normalizedPath, q, normalizedQuery, limit, offset });
if (candidate.remote) {
firstUnavailable ??= { candidate, reason: "remote_workspace" };
if (explicitTarget || selector !== "auto") {
@@ -1637,6 +1710,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.placement) return persistentContent(candidate, normalized);
if (candidate.remote) {
if (explicitTarget || selector !== "auto") {
throw unprocessable("Remote workspaces cannot be previewed by the server", { code: "remote_workspace" });
@@ -1709,7 +1783,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
workspace?: WorkspaceFileSelector | null;
projectId?: string | null;
workspaceId?: string | null;
}, opts: { issue?: IssueRow } = {}): Promise<LocalResolvedFile> {
}, opts: { issue?: IssueRow } = {}): Promise<LocalResolvedFile | { resource: ResolvedWorkspaceResource; bytes: Buffer }> {
const issue = opts.issue ?? await getIssue(issueId);
const selector = input.workspace ?? "auto";
const explicitTarget = Boolean(input.projectId || input.workspaceId);
@@ -1721,6 +1795,11 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
let lastNotFound: unknown = null;
for (const candidate of candidates) {
if (candidate.placement) {
const resource = await statPersistentCandidate(candidate, normalized);
const { bytes } = await (await placementFiles(candidate)).readBytes(normalized.relativePath);
return { resource: { ...resource, byteSize: bytes.length }, bytes };
}
if (candidate.remote) {
if (explicitTarget || selector !== "auto") {
throw unprocessable("Remote workspaces cannot be downloaded by the server", { code: "remote_workspace" });