mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 08:22:20 +02:00
Keep computer personal files and workspaces remotely authoritative
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
1dcf713e21
commit
0cf2ea86e8
15 files changed
+491
-27
No files matched your search
@@ -16,6 +16,7 @@ function cacheKeyForTarget(command: string, target: AdapterExecutionTarget | nul
|
||||
if (target.kind === "local") {
|
||||
return `local:${target.environmentId ?? ""}:${target.leaseId ?? ""}:${command}`;
|
||||
}
|
||||
if (target.transport === "computer") return `computer:${target.environmentId}:${target.resourceAuthority.computerId}:${command}`;
|
||||
if (target.transport === "sandbox") {
|
||||
return [
|
||||
"sandbox",
|
||||
|
||||
@@ -231,6 +231,31 @@ describe("claude remote execution", () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it("keeps an authoritative remote workspace in place while delivering managed Claude assets", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-claude-in-place-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
await execute({
|
||||
runId: "run-in-place",
|
||||
agent: { id: "agent-1", companyId: "company-1", name: "Claude", adapterType: "claude_local", adapterConfig: {} },
|
||||
runtime: { sessionId: null, sessionParams: null, sessionDisplayId: null, taskKey: null },
|
||||
config: { engine: "cli", command: "claude" },
|
||||
context: { paperclipWorkspace: { cwd: rootDir, source: "task_session" } },
|
||||
executionTarget: {
|
||||
kind: "remote", transport: "ssh", remoteCwd: "/old-copied-workspace",
|
||||
workspaceRealization: { mode: "in_place", authoritativeRoot: "/persistent-project", pathAliases: [], outboundRestorePaths: [] },
|
||||
spec: { host: "127.0.0.1", port: 2222, username: "fixture", remoteWorkspacePath: "/persistent-project", remoteCwd: "/persistent-project",
|
||||
privateKey: "PRIVATE KEY", knownHosts: "[127.0.0.1]:2222 ssh-ed25519 AAAA", strictHostKeyChecking: true },
|
||||
},
|
||||
onLog: async () => {},
|
||||
});
|
||||
expect(prepareWorkspaceForSshExecution).not.toHaveBeenCalled();
|
||||
expect(restoreWorkspaceFromSshExecution).not.toHaveBeenCalled();
|
||||
expect(syncDirectoryToSsh).toHaveBeenCalledTimes(2);
|
||||
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({ remoteDir: "/persistent-project/.paperclip-runtime/claude/skills" }));
|
||||
const call = runChildProcess.mock.calls[0] as unknown as [string, string, string[], { remoteExecution: { remoteCwd: string } }];
|
||||
expect(call[3].remoteExecution.remoteCwd).toBe("/persistent-project");
|
||||
});
|
||||
|
||||
it("does not resume saved Claude sessions for remote SSH execution without a matching remote identity", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-claude-remote-resume-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { adapterExecutionTargetIsCommandBacked } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { createProviderStoppedBoundary } from "@paperclipai/adapter-utils/provider-stopped-boundary";
|
||||
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
|
||||
import fs from "node:fs/promises";
|
||||
@@ -426,7 +427,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
legacyRemoteExecution: ctx.executionTransport?.remoteExecution,
|
||||
});
|
||||
const executionTargetIsRemote = adapterExecutionTargetIsRemote(executionTarget);
|
||||
const executionTargetIsSandbox = executionTarget?.kind === "remote" && executionTarget.transport === "sandbox";
|
||||
const executionTargetIsCommandBacked = adapterExecutionTargetIsCommandBacked(executionTarget);
|
||||
|
||||
const promptTemplate = asString(
|
||||
config.promptTemplate,
|
||||
@@ -482,7 +483,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
graceSec,
|
||||
extraArgs,
|
||||
} = runtimeConfig;
|
||||
Object.assign(env, claudeSandboxPermissionEnv({ dangerouslySkipPermissions, targetIsSandbox: executionTargetIsSandbox }));
|
||||
Object.assign(env, claudeSandboxPermissionEnv({ dangerouslySkipPermissions, targetIsSandbox: executionTargetIsCommandBacked }));
|
||||
let loggedEnv = initialLoggedEnv;
|
||||
let effectiveExecutionCwd = adapterExecutionTargetRemoteCwd(executionTarget, cwd);
|
||||
const terminalResultCleanupGraceMs = Math.max(
|
||||
@@ -612,11 +613,12 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
const claudeConfigSeedDir = useManagedRemoteClaudeConfig
|
||||
? config.managedAiConnection ? sharedClaudeConfigDir : await prepareClaudeConfigSeed(process.env, onLog, agent.companyId)
|
||||
: null;
|
||||
const targetWorkspaceRealization = executionTarget?.workspaceRealization ?? null;
|
||||
const preparedExecutionTargetRuntime = executionTargetIsRemote
|
||||
? await (async () => {
|
||||
await onLog(
|
||||
"stdout",
|
||||
`[paperclip] Syncing workspace and Claude runtime assets to ${describeAdapterExecutionTarget(executionTarget)}.\n`,
|
||||
`[paperclip] Syncing ${targetWorkspaceRealization?.mode === "in_place" ? "Claude runtime assets" : "workspace and Claude runtime assets"} to ${describeAdapterExecutionTarget(executionTarget)}.\n`,
|
||||
);
|
||||
return await prepareAdapterExecutionTargetRuntime({
|
||||
runId,
|
||||
@@ -624,6 +626,9 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
adapterKey: "claude",
|
||||
timeoutSec,
|
||||
workspaceLocalDir: cwd,
|
||||
workspaceRemoteDir: targetWorkspaceRealization?.mode === "in_place"
|
||||
? targetWorkspaceRealization.authoritativeRoot : undefined,
|
||||
syncWorkspace: targetWorkspaceRealization?.mode !== "in_place",
|
||||
installCommand: SANDBOX_INSTALL_COMMAND,
|
||||
detectCommand: command,
|
||||
onProgress: (line) => onLog("stdout", line),
|
||||
@@ -748,7 +753,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
}
|
||||
}
|
||||
let effectiveEffort = effort;
|
||||
if (executionTargetIsSandbox && effort) {
|
||||
if (executionTargetIsCommandBacked && effort) {
|
||||
const supportsEffort = await claudeCommandSupportsEffortFlag({
|
||||
runId,
|
||||
command,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { adapterExecutionTargetIsCommandBacked } from "@paperclipai/adapter-utils/execution-target";
|
||||
import { createProviderStoppedBoundary } from "@paperclipai/adapter-utils/provider-stopped-boundary";
|
||||
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
|
||||
import fs from "node:fs/promises";
|
||||
@@ -874,8 +875,8 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
effectiveExecutionCwd = preparedExecutionTargetRuntime.workspaceRemoteDir;
|
||||
}
|
||||
const runtimeExecutionTarget = overrideAdapterExecutionTargetRemoteCwd(executionTarget, effectiveExecutionCwd);
|
||||
const executionTargetIsSandbox =
|
||||
runtimeExecutionTarget?.kind === "remote" && runtimeExecutionTarget.transport === "sandbox";
|
||||
const executionTargetIsCommandBacked =
|
||||
adapterExecutionTargetIsCommandBacked(runtimeExecutionTarget);
|
||||
const restoreRemoteWorkspace = preparedExecutionTargetRuntime
|
||||
? () => preparedExecutionTargetRuntime.restoreWorkspace((line) => onLog("stdout", line))
|
||||
: null;
|
||||
@@ -1191,7 +1192,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
}
|
||||
return notes;
|
||||
})();
|
||||
if (executionTargetIsSandbox) {
|
||||
if (executionTargetIsCommandBacked) {
|
||||
commandNotes.push(
|
||||
"Added --skip-git-repo-check for sandbox execution because Codex requires an explicit trust bypass in headless remote workspaces.",
|
||||
);
|
||||
@@ -1226,7 +1227,7 @@ export async function execute(ctx: AdapterExecutionContext): Promise<AdapterExec
|
||||
forceSaferInvocation ? { ...config, fastMode: false } : config,
|
||||
{
|
||||
resumeSessionId,
|
||||
skipGitRepoCheck: executionTargetIsSandbox,
|
||||
skipGitRepoCheck: executionTargetIsCommandBacked,
|
||||
networkAccess: env.PAPERCLIP_RUNNER_NETWORK_ACCESS !== "disabled",
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import * as persistentFiles from "../services/persistent-agent-files.js";
|
||||
import fs from "node:fs/promises";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
@@ -74,6 +75,55 @@ describe("persistent agent directories", () => {
|
||||
await fs.writeFile(path.join(root, entryFile), initial);
|
||||
});
|
||||
|
||||
it("keeps Boat personal bytes authoritative through warm checkpoints, collection, and editor conflicts", async () => {
|
||||
const remoteRoot = "/home/user/paperclip/test/agents/target";
|
||||
const remoteFiles = new Map([[entryFile, Buffer.from("remote instructions")], ["memory.txt", Buffer.from("remote memory")]]);
|
||||
const readBytes = vi.fn(async (relative: string) => {
|
||||
const bytes = remoteFiles.get(relative);
|
||||
if (!bytes) throw { code: "not_found" };
|
||||
return { bytes, sha256: fileHash(bytes) };
|
||||
});
|
||||
const writeBytes = vi.fn(async (relative: string, bytes: Buffer, base: string | null) => {
|
||||
const current = remoteFiles.get(relative);
|
||||
if ((current ? fileHash(current) : null) !== base) throw new Error("remote CAS rejected");
|
||||
remoteFiles.set(relative, bytes); return { sha256: fileHash(bytes) };
|
||||
});
|
||||
const seedBytes = vi.fn();
|
||||
const remote = { root: remoteRoot, list: async () => [], readBytes, writeBytes, seedBytes };
|
||||
const lookup = vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue(remote as never);
|
||||
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand");
|
||||
try {
|
||||
const runId = randomUUID();
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId });
|
||||
const target = { kind: "remote", transport: "computer", environmentId: randomUUID(), remoteCwd: "/workspace",
|
||||
fileAuthority: { kind: "remote-persistent", placementId: randomUUID(), root: "/workspace", agentHome: remoteRoot } } as never;
|
||||
const copy = (await copies.prepare({ companyId, agentId, runId, cwd: home, target, warm: true }))!;
|
||||
expect(copy.executionRoot).toBe(remoteRoot);
|
||||
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe("remote instructions");
|
||||
expect(seedBytes).not.toHaveBeenCalled();
|
||||
remoteFiles.set("memory.txt", Buffer.from("edited while warm"));
|
||||
expect(await copies.hasChanges({ companyId, runId, target })).toBe(false);
|
||||
await copies.checkpointWarm({ companyId, runId, target });
|
||||
await copies.collectStopped({ companyId, runId, target });
|
||||
await copies.release(companyId, runId);
|
||||
expect(shell).not.toHaveBeenCalled();
|
||||
expect(remoteFiles.get("memory.txt")?.toString()).toBe("edited while warm");
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
await expect(fs.stat(path.join(root, "memory.txt"))).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const store = agentFileStore(db);
|
||||
await expect(store.write({ companyId, agentId, path: "memory.txt", bytes: Buffer.from("stale"), baseHash: fileHash(Buffer.from("remote memory")) }, board())).rejects.toMatchObject({ status: 409 });
|
||||
expect(writeBytes).not.toHaveBeenCalled();
|
||||
await store.write({ companyId, agentId, path: "memory.txt", bytes: Buffer.from("editor"), baseHash: fileHash(Buffer.from("edited while warm")) }, board());
|
||||
expect(remoteFiles.get("memory.txt")?.toString()).toBe("editor");
|
||||
const current = await revisions.readCurrent({ companyId, agentId }, board());
|
||||
expect(current?.content).toBe("remote instructions");
|
||||
await revisions.commit({ companyId, agentId, entryFile, content: "editor changed instructions", baseRevisionId: current!.revision.id, source: "api" }, board());
|
||||
expect(remoteFiles.get(entryFile)?.toString()).toBe("editor changed instructions");
|
||||
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
|
||||
|
||||
} finally { lookup.mockRestore(); shell.mockRestore(); }
|
||||
});
|
||||
|
||||
describe("warm directory ownership", () => {
|
||||
async function nextRun() {
|
||||
const runId = randomUUID();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import * as computerModule from "../modules/computers/index.js";
|
||||
import express from "express";
|
||||
import type { Request } from "express";
|
||||
import { execFile } from "node:child_process";
|
||||
@@ -208,6 +209,35 @@ describeEmbeddedPostgres("workspace file resources", () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("browses a recorded computer placement and confines previews/downloads to safe relative files", async () => {
|
||||
const workspace = await makeWorkspace();
|
||||
const graph = await seedGraph(db, { projectRoot: workspace.projectRoot, executionRoot: workspace.executionRoot });
|
||||
const environmentId = crypto.randomUUID(), placementId = crypto.randomUUID();
|
||||
await db.update(executionWorkspaces).set({ providerType: "computer", cwd: "/home/user/paperclip/remote/project",
|
||||
metadata: { fileAuthority: { kind: "remote-persistent", environmentId, placementId } } })
|
||||
.where(eq(executionWorkspaces.sourceIssueId, graph.issueId));
|
||||
const bytes = Buffer.from("only on Boat");
|
||||
const readBytes = vi.fn(async () => ({ bytes, sha256: "unused-by-preview" }));
|
||||
const list = vi.fn(async () => [{ name: "README.md", kind: "file", size: bytes.length, mtimeMs: 1 },
|
||||
{ name: ".env", kind: "file", size: 1, mtimeMs: 1 }, { name: ".paperclip-runtime", kind: "directory", size: 0, mtimeMs: 1 }]);
|
||||
const workspaceFiles = vi.fn(async () => ({ root: "/home/user/paperclip/remote/project", list, readBytes }));
|
||||
const factory = vi.spyOn(computerModule, "computerService").mockReturnValue({ workspaceFiles } as never);
|
||||
try {
|
||||
const service = workspaceFileResourceService(db);
|
||||
const listed = await service.list(graph.issueId, { workspace: "execution" });
|
||||
expect(listed.items.map(item => item.title)).toEqual(["README.md"]);
|
||||
expect(workspaceFiles).toHaveBeenCalledWith({ companyId: graph.companyId, environmentId, placementId });
|
||||
const content = await service.readContent(graph.issueId, { path: "README.md", workspace: "execution" });
|
||||
expect(content.content).toEqual({ encoding: "utf8", data: "only on Boat" });
|
||||
expect(readBytes).toHaveBeenCalledWith("README.md", WORKSPACE_FILE_TEXT_MAX_BYTES);
|
||||
expect(await service.prepareDownload(graph.issueId, { path: "README.md", workspace: "execution" })).toMatchObject({ bytes });
|
||||
readBytes.mockClear();
|
||||
await expect(service.readContent(graph.issueId, { path: ".env", workspace: "execution" })).rejects.toMatchObject({ status: 403 });
|
||||
await expect(service.readContent(graph.issueId, { path: "../other-company/secret", workspace: "execution" })).rejects.toMatchObject({ status: 403 });
|
||||
expect(readBytes).not.toHaveBeenCalled();
|
||||
} finally { factory.mockRestore(); await fs.rm(workspace.root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
it("withholds private task files and private cross-project targets", async () => {
|
||||
const { projectRoot, executionRoot, targetProjectRoot } = await makeWorkspace();
|
||||
const graph = await seedGraph(db, { projectRoot, executionRoot, targetProjectRoot });
|
||||
|
||||
@@ -56,7 +56,7 @@ export type WorkspaceFileResourceService = {
|
||||
issueId: string,
|
||||
input: { path: string; workspace?: "auto" | "execution" | "project" | null; projectId?: string | null; workspaceId?: string | null },
|
||||
opts?: { issue?: Awaited<ReturnType<WorkspaceFileResourceService["getIssue"]>> },
|
||||
): Promise<{ resource: ResolvedWorkspaceResource; realPath: string }>;
|
||||
): Promise<{ resource: ResolvedWorkspaceResource; realPath: string } | { resource: ResolvedWorkspaceResource; bytes: Buffer }>;
|
||||
};
|
||||
|
||||
type FileResourceLimiter = {
|
||||
@@ -886,7 +886,8 @@ export function fileResourceRoutes(db: Db, opts: {
|
||||
res.setHeader("Cache-Control", "private, max-age=60");
|
||||
res.setHeader("X-Content-Type-Options", "nosniff");
|
||||
res.setHeader("Content-Disposition", `attachment; filename="${safeAttachmentFilename(result.resource.title)}"`);
|
||||
await pipeline(createReadStream(result.realPath), res);
|
||||
if ("bytes" in result) res.end(result.bytes);
|
||||
else await pipeline(createReadStream(result.realPath), res);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
|
||||
import { materializeInstructionBytes, readInstructionBytes } from "./agent-instruction-files.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
|
||||
@@ -20,6 +22,7 @@ import { logger } from "../middleware/logger.js";
|
||||
|
||||
type Copy = typeof copies.$inferSelect;
|
||||
export class AgentDirectoryReuseInvalidatedError extends Error {}
|
||||
const persistent = (row: Pick<Copy, "receipt">) => row.receipt?.fileAuthority === "remote-persistent";
|
||||
const completed = new Set(["saved", "unchanged", "resolved", "unavailable"]);
|
||||
const transports = new Map<string, PreparedAdapterExecutionTargetRuntime>();
|
||||
const key = (row: Pick<Copy, "companyId" | "runId">) => `${row.companyId}:${row.runId}`;
|
||||
@@ -42,12 +45,17 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
const store = agentFileStore(db);
|
||||
const ownerFile = (row: Copy) => path.join(path.dirname(row.localRoot), "owner.json");
|
||||
async function owns(row: Copy) {
|
||||
if (row.receipt?.warm !== true) return true;
|
||||
if (persistent(row) || row.receipt?.warm !== true) return true;
|
||||
return await fs.readFile(ownerFile(row), "utf8").then(s => JSON.parse(s).runId === row.runId).catch(error => error.code === "ENOENT" && row.processStoppedAt !== null);
|
||||
}
|
||||
async function canReuse(companyId: string, agentId: string, runId: string) {
|
||||
const row = await get(companyId, runId);
|
||||
if (!row || row.agentId !== agentId || row.state !== "warm_saved" || row.errorCode || !await owns(row)) return false;
|
||||
if (persistent(row)) {
|
||||
const remote = await persistentAgentFiles(db, companyId, agentId, String(row.receipt?.environmentId));
|
||||
if (!remote || remote.root !== row.executionRoot) return false;
|
||||
return (await readPersistentAgentFile(remote, row.entryFile))?.sha256 === row.receipt?.sessionEntryHash;
|
||||
}
|
||||
const entry = baseline(row).entries.get(row.entryFile);
|
||||
if (entry?.kind !== "file" || entry.hash !== row.receipt?.sessionEntryHash) return false;
|
||||
return store.locked(companyId, agentId, actor(row), false, async (_tx, _agent, root) =>
|
||||
@@ -96,6 +104,41 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
if (agentInstructionsBundleMode(agent) !== "managed") return null;
|
||||
const bound = await resolveInstructionActor(db, { type: "agent", companyId: input.companyId, agentId: input.agentId, runId: input.runId });
|
||||
const root = resolveManagedInstructionsRoot(agent);
|
||||
if (input.target?.kind === "remote" && input.target.transport === "computer" && input.target.fileAuthority?.kind === "remote-persistent") {
|
||||
const authority = input.target.fileAuthority;
|
||||
const remote = await persistentAgentFiles(db, input.companyId, input.agentId, input.target.environmentId ?? undefined);
|
||||
if (!remote || remote.root !== authority.agentHome) throw conflict("Persistent agent placement changed");
|
||||
await store.locked(input.companyId, input.agentId, bound, false, async (_tx, _agent, canonical) => seedPersistentAgentHome(remote, canonical));
|
||||
const entryFile = deriveBundleState(agent).entryFile;
|
||||
let entry = await readPersistentAgentFile(remote, entryFile);
|
||||
if (!entry) {
|
||||
// An adopted directory may predate Paperclip. Deliver the missing managed
|
||||
// entry with create-only CAS, without replacing any personal bytes.
|
||||
const seed = await readInstructionBytes(root, entryFile);
|
||||
if (!seed) throw notFound("The persistent agent instruction entry is missing");
|
||||
await remote.writeBytes(entryFile, seed, null);
|
||||
entry = await readPersistentAgentFile(remote, entryFile);
|
||||
if (!entry) throw notFound("The persistent agent instruction entry is missing");
|
||||
}
|
||||
// Only prompt bytes are staged locally. This scratch is never an authority
|
||||
// for personal files, nor is it uploaded into the persistent directory.
|
||||
const localRoot = path.join(path.dirname(root), "file-sync", "runs", input.runId, "live");
|
||||
await materializeInstructionBytes(localRoot, entryFile, entry.bytes);
|
||||
const values = { entryFile, baseRevisionId: null, baseHash: entry.sha256, localRoot,
|
||||
executionRoot: authority.agentHome, location: `remote:${input.target.environmentId}`, state: "prepared",
|
||||
candidateBase64: null, candidateHash: null, processStoppedAt: null, attempts: 0,
|
||||
nextAttemptAt: null, errorCode: null, errorMessage: null,
|
||||
receipt: { schema: AGENT_FILES_CONTRACT, fileAuthority: "remote-persistent", placementId: authority.placementId,
|
||||
environmentId: input.target.environmentId, workspaceCwd: input.cwd, warm: input.warm === true,
|
||||
sessionEntryHash: entry.sha256, cleanupPending: false } };
|
||||
const existing = await get(input.companyId, input.runId);
|
||||
if (existing && (!persistent(existing) || existing.executionRoot !== authority.agentHome)) throw conflict("Agent file authority changed");
|
||||
const result = existing ? await patch(existing, values)
|
||||
: (await db.insert(copies).values({ companyId: input.companyId, agentId: input.agentId, runId: input.runId,
|
||||
responsibleUserId: bound.onBehalfOfUserId!, ...values }).returning())[0]!;
|
||||
input.onWarmHandoff?.(result);
|
||||
return result;
|
||||
}
|
||||
if (input.reuseRunId) {
|
||||
const prior = await get(input.companyId, input.reuseRunId);
|
||||
if (!prior || prior.agentId !== input.agentId) throw conflict("Managed warm directory owner changed");
|
||||
@@ -241,6 +284,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
return inspectAgentDirectory(row.localRoot);
|
||||
}
|
||||
async function hasChanges(row: Copy, target?: AdapterExecutionTarget | null) {
|
||||
if (persistent(row)) return false; // Bytes already live on the authoritative computer.
|
||||
try {
|
||||
if (row.receipt?.retainedByRunId || row.receipt?.retirementRequired || typeof row.receipt?.materializationIdentity !== "string") return true;
|
||||
const observed = await observe(row, target);
|
||||
@@ -251,6 +295,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
}
|
||||
/** Rebind one physical materialization; earlier runs lose collection authority. */
|
||||
async function adopt(input: { companyId: string; agentId: string; runId: string; previousRunId: string; target?: AdapterExecutionTarget | null; cwd: string; allowRetirementHandoff?: boolean }) {
|
||||
if (input.target?.kind === "remote" && input.target.transport === "computer" && input.target.fileAuthority?.kind === "remote-persistent") return prepareCopy({ ...input, warm: true });
|
||||
let prior = await get(input.companyId, input.previousRunId);
|
||||
if (!prior || prior.agentId !== input.agentId || prior.state !== "unchanged_turn" || prior.receipt?.retainedByRunId || prior.processStoppedAt
|
||||
|| prior.receipt?.workspaceCwd !== input.cwd || prior.location !== (input.target?.kind === "remote" ? `remote:${input.target.environmentId ?? ""}` : "local")) return null;
|
||||
@@ -317,6 +362,8 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
return adopted;
|
||||
}
|
||||
async function checkpoint(row: Copy, target?: AdapterExecutionTarget | null, stopped = false): Promise<Copy> {
|
||||
if (persistent(row)) return patch(row, { state: stopped ? "unchanged" : "warm_saved", nextAttemptAt: null,
|
||||
...(stopped ? { processStoppedAt: new Date() } : {}) });
|
||||
if (!await owns(row) || row.state === "superseded") return row;
|
||||
let failure: unknown;
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
@@ -356,6 +403,11 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
receipt: { ...row.receipt, storageWarning: storageLimit ? agentStorageWarning(failure instanceof AgentFileLimitError ? failure.message : "Agent folder exceeds a storage limit") : null } });
|
||||
}
|
||||
async function collectStopped(row: Copy, target?: AdapterExecutionTarget | null) {
|
||||
if (persistent(row)) {
|
||||
const settled = await patch(row, { state: "unchanged", processStoppedAt: new Date(), nextAttemptAt: null });
|
||||
await release(settled);
|
||||
return settled;
|
||||
}
|
||||
if (row.receipt?.retainedByRunId) return row;
|
||||
if (!await owns(row) || row.state === "superseded") return row;
|
||||
if (row.receipt?.warm === true && !completed.has(row.state)) {
|
||||
@@ -412,6 +464,11 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
completed.has(row.state) && row.receipt?.cleanupPending === false;
|
||||
}
|
||||
async function release(row: Copy, target?: AdapterExecutionTarget | null) {
|
||||
if (persistent(row)) {
|
||||
// No remote command, snapshot, or directory deletion is legal here.
|
||||
if (row.processStoppedAt) await fs.rm(path.dirname(row.localRoot), { recursive: true, force: true });
|
||||
return;
|
||||
}
|
||||
if (releaseIsNoop(row) || !await owns(row)) return;
|
||||
const destroyedOnly = row.receipt?.cleanupDestroyedOnly === true;
|
||||
if (row.receipt?.retainedByRunId || row.state === "superseded" || !await owns(row) || !row.processStoppedAt) return;
|
||||
@@ -503,6 +560,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
return lease;
|
||||
}
|
||||
async function recoverStoppedRemote(row: Copy) {
|
||||
if (persistent(row)) return patch(row, { state: "unchanged", processStoppedAt: row.processStoppedAt ?? new Date(), nextAttemptAt: null });
|
||||
const lease = await ownedRemoteLease(row);
|
||||
const destroyed = lease && hasRemoteTerminationReceipt(lease)
|
||||
&& (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed";
|
||||
@@ -553,7 +611,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
|
||||
const current = await get(row.companyId, row.runId);
|
||||
// This check changes no receipt or bytes. A concurrent stop merely leaves
|
||||
// cleanup to its owner or the recovery sweep; it grants no new authority.
|
||||
if (!current || releaseIsNoop(current)) return;
|
||||
if (!current || (!persistent(current) && releaseIsNoop(current))) return;
|
||||
await serial(current, release, "agent_directory_release");
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import { cachedAgentFileManifest, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js";
|
||||
@@ -163,13 +164,15 @@ export async function snapshotAgentFiles(root: string, enforceLimits = true): Pr
|
||||
|
||||
export function agentFileStore(db: Db) {
|
||||
async function locked<T>(companyId: string, agentId: string, actor: AuthorizationActor, write: boolean,
|
||||
fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor) => Promise<T>) {
|
||||
fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor, remote: Awaited<ReturnType<typeof persistentAgentFiles>>) => Promise<T>) {
|
||||
return db.transaction(async tx => {
|
||||
const [agent] = await tx.select().from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))).for("update");
|
||||
if (!agent) throw notFound("Agent not found");
|
||||
const bound = write ? await authorizeInstructionCommit(tx, actor, agent) : await authorizeInstructionRead(tx, actor, agent);
|
||||
const root = await adoptAgentFiles(tx, agent);
|
||||
return fn(tx, agent, root, bound);
|
||||
const remote = await persistentAgentFiles(db, companyId, agentId);
|
||||
if (remote) await seedPersistentAgentHome(remote, root);
|
||||
return fn(tx, agent, root, bound, remote);
|
||||
});
|
||||
}
|
||||
async function audit(tx: Tx, agent: Agent, actor: AuthorizationActor, details: Record<string, unknown>) {
|
||||
@@ -183,29 +186,41 @@ export function agentFileStore(db: Db) {
|
||||
download: async (companyId: string, agentId: string, relative: string, actor: AuthorizationActor) => {
|
||||
const opened: { file: Awaited<ReturnType<typeof openAgentFile>> } = { file: null };
|
||||
try {
|
||||
const file = await locked(companyId, agentId, actor, false, async (_tx, _agent, root) => {
|
||||
const file = await locked(companyId, agentId, actor, false, async (_tx, _agent, root, _bound, remote) => {
|
||||
if (remote) return { remote: await readPersistentAgentFile(remote, relative) };
|
||||
opened.file = await openAgentFile(root, instructionPath(relative));
|
||||
return opened.file;
|
||||
});
|
||||
// Transfer outside the database lock. The open descriptor pins the file
|
||||
// across concurrent atomic replacements; end bounds concurrent growth.
|
||||
if (!file) return null;
|
||||
if ("remote" in file) return file.remote ? { size: file.remote.bytes.length, stream: Readable.from([file.remote.bytes]) } : null;
|
||||
if (file.size === 0) { await file.handle.close(); return { size: 0, stream: Readable.from([]) }; }
|
||||
return { size: file.size, stream: file.handle.createReadStream({ end: file.size - 1 }) };
|
||||
} catch (error) { await opened.file?.handle.close(); throw error; }
|
||||
},
|
||||
read: (companyId: string, agentId: string, relative: string, actor: AuthorizationActor) =>
|
||||
locked(companyId, agentId, actor, false, (_tx, _agent, root) => readAgentFile(root, instructionPath(relative))),
|
||||
locked(companyId, agentId, actor, false, async (_tx, _agent, root, _bound, remote) => remote ? (await readPersistentAgentFile(remote, relative))?.bytes ?? null : readAgentFile(root, instructionPath(relative))),
|
||||
write: (input: { companyId: string; agentId: string; path: string; bytes: Buffer | null; baseHash: string | null }, actor: AuthorizationActor) =>
|
||||
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound) => {
|
||||
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound, remote) => {
|
||||
const relative = agentFilePath(input.path);
|
||||
if (input.bytes) assertFileSize(input.bytes.length, relative);
|
||||
const previous = await inspectAgentFile(root, relative);
|
||||
const remotePrevious = remote ? await readPersistentAgentFile(remote, relative) : null;
|
||||
const previous = remote ? (remotePrevious ? { hash: remotePrevious.sha256, size: remotePrevious.bytes.length } : null) : await inspectAgentFile(root, relative);
|
||||
const currentHash = previous?.hash ?? null;
|
||||
const incomingHash = input.bytes === null ? null : fileHash(input.bytes);
|
||||
if (currentHash === incomingHash) return { contentHash: currentHash, changed: false };
|
||||
if (currentHash !== input.baseHash) throw conflict("This file changed since it was read. Reload before saving.", { code: "AGENT_FILE_CONFLICT", path: relative, currentHash });
|
||||
if (input.bytes === null && relative === deriveBundleState(agent).entryFile) throw unprocessable("The configured instruction entry cannot be deleted");
|
||||
if (remote) {
|
||||
if (input.bytes !== null && relative === deriveBundleState(agent).entryFile) instructionBytes(input.bytes);
|
||||
// The remote operation checks the hash again immediately before atomic
|
||||
// replacement. Arbitrary shell writers do not participate in this lock.
|
||||
if (input.bytes === null) await remote.remove(relative, currentHash!);
|
||||
else await remote.writeBytes(relative, input.bytes, input.baseHash);
|
||||
await audit(tx, agent, bound, { path: relative, contentHash: incomingHash });
|
||||
return { contentHash: incomingHash, changed: true };
|
||||
}
|
||||
if (input.bytes !== null) {
|
||||
if (relative === deriveBundleState(agent).entryFile) instructionBytes(input.bytes);
|
||||
// A quota check needs metadata only. Do not hash unrelated large
|
||||
@@ -221,7 +236,8 @@ export function agentFileStore(db: Db) {
|
||||
return { contentHash: incomingHash, changed: true };
|
||||
}),
|
||||
apply: (input: { companyId: string; agentId: string; sourceDir: string; baseline: DirectorySnapshot; checkpoint?: AgentFileManifest }, actor: AuthorizationActor) =>
|
||||
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound) => {
|
||||
locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound, remote) => {
|
||||
if (remote) throw unprocessable("Persistent remote agent files cannot accept controller copy-back");
|
||||
const incoming = input.checkpoint ? checkpointSnapshot(input.checkpoint) : await snapshotAgentFiles(input.sourceDir);
|
||||
const entryPath = deriveBundleState(agent).entryFile;
|
||||
if (incoming.entries.get(entryPath)?.kind !== "file") throw unprocessable("The configured instruction entry cannot be deleted");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
|
||||
import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { adoptAgentFiles, agentFileToken } from "./agent-file-store.js";
|
||||
@@ -107,7 +108,7 @@ export function agentInstructionRevisionService(db: Db) {
|
||||
}
|
||||
const state = deriveBundleState(agent);
|
||||
const entryFile = instructionPath(state.entryFile);
|
||||
return { agent, entryFile, root: resolveManagedInstructionsRoot(agent) };
|
||||
return { agent, entryFile, root: resolveManagedInstructionsRoot(agent), remote: await persistentAgentFiles(db, agent.companyId, agent.id) };
|
||||
}
|
||||
async function authorizeRead(
|
||||
tx: Tx,
|
||||
@@ -156,7 +157,9 @@ export function agentInstructionRevisionService(db: Db) {
|
||||
// live in the agent directory; old UUID clients receive a content ETag.
|
||||
async function currentFile(tx: Tx, target: InstructionTarget, state: Awaited<ReturnType<typeof lockTarget>>, actor?: RevisionActor) {
|
||||
await adoptAgentFiles(tx, state.agent);
|
||||
const bytes = await readInstructionBytes(state.root, state.entryFile);
|
||||
if (state.remote) await seedPersistentAgentHome(state.remote, state.root);
|
||||
const bytes = state.remote ? (await readPersistentAgentFile(state.remote, state.entryFile))?.bytes ?? null
|
||||
: await readInstructionBytes(state.root, state.entryFile);
|
||||
if (bytes === null) return null;
|
||||
return currentRow(target, state.entryFile, bytes, actor);
|
||||
}
|
||||
@@ -235,7 +238,7 @@ export function agentInstructionRevisionService(db: Db) {
|
||||
"Configured instruction entry changed; read the current entry and retry",
|
||||
{ code: "INSTRUCTION_ENTRY_CHANGED", entryFile: state.entryFile },
|
||||
);
|
||||
await assertInstructionPathSafe(state.root, state.entryFile);
|
||||
if (!state.remote) await assertInstructionPathSafe(state.root, state.entryFile);
|
||||
const current = await currentFile(tx, input, state, bound);
|
||||
const restored =
|
||||
"restoreRevisionId" in input
|
||||
@@ -285,7 +288,8 @@ export function agentInstructionRevisionService(db: Db) {
|
||||
},
|
||||
);
|
||||
}
|
||||
await materializeInstructionBytes(state.root, state.entryFile, candidate);
|
||||
if (state.remote) await state.remote.writeBytes(state.entryFile, candidate, current?.contentHash ?? null);
|
||||
else await materializeInstructionBytes(state.root, state.entryFile, candidate);
|
||||
const row = currentRow(input, input.entryFile, candidate, bound);
|
||||
await tx.insert(activityLog).values({ companyId: input.companyId,
|
||||
actorType: bound.type === "plugin" ? "plugin" : bound.type === "board" ? "user" : "agent",
|
||||
|
||||
@@ -47,6 +47,7 @@ export async function collectStoppedInstructionCopyWithRetries<T extends Pick<Co
|
||||
}
|
||||
|
||||
export function instructionWorkingCopyGuidance(copy: Pick<Copy, "executionRoot" | "entryFile" | "receipt">) {
|
||||
if (copy.receipt?.fileAuthority === "remote-persistent") return `Your persistent agent directory is ${copy.executionRoot} (AGENT_HOME). Your instruction entry is ${copy.executionRoot}/${copy.entryFile}. Changes are saved directly on this computer and remain across turns. Provider HOME and credentials are separate runtime directories.`;
|
||||
if (isAgentDirectoryCopy(copy)) return `Your persistent agent directory is ${copy.executionRoot} (AGENT_HOME). Your instruction entry is ${copy.executionRoot}/${copy.entryFile}. Read and write your own files and subfolders there. This directory belongs to this agent across tasks and sessions; task files belong in the task working directory. Paperclip restores this directory before execution and saves validated changes at turn boundaries. A warm native Codex session keeps the same writable directory between turns. Other sessions collect after the provider stops. Regular files, including binary files, persist; symlinks and special files are unsupported. Check the agent-files save receipt before claiming persistence; Only files you change or delete are synchronized. If another run changes the same file, the last completed synchronization wins. Temporary copies are removed when the owning session stops; there is no per-run file history. Storage allows 256 MiB per file, 2 GiB total, and 100,000 entries; the instruction entry must remain UTF-8 and at most 1 MiB. Reaching a storage limit never prevents this or future tasks from running. Remove or shrink files to free space; changes that exceed the limits will not be saved.${typeof copy.receipt?.storageWarning === "string" ? `\n\n${copy.receipt.storageWarning}` : ""}`;
|
||||
return `Your editable agent instruction file is ${copy.executionRoot}/${copy.entryFile}. Edit this registered private copy normally. After this run stops, Paperclip saves changed content as a persistent revision if your responsible user still has permission and the baseline has not changed. Check the run's instruction-save receipt before claiming persistence. Use read_agent_instructions, update_agent_instructions, get_agent_instruction_history, and restore_agent_instructions for immediate saves and history. Read first and pin the returned revision. Preserve conflicts; never silently retry against a newer head. Repository instructions, skills, and the loaded prompt are separate and are not collected.`;
|
||||
}
|
||||
@@ -103,7 +104,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
|
||||
}
|
||||
async function prepare(input: { companyId: string; agentId: string; runId: string; target?: AdapterExecutionTarget | null; cwd: string; legacy?: boolean; warm?: boolean; reuseRunId?: string; onWarmHandoff?: (copy: Copy) => void }) {
|
||||
const existing = await get(input.companyId, input.runId);
|
||||
if (isAgentDirectoryCopy(existing) || (!existing && !input.legacy)) return directories.prepare(input);
|
||||
if ((input.target?.kind === "remote" && input.target.transport === "computer" && input.target.fileAuthority?.kind === "remote-persistent") || isAgentDirectoryCopy(existing) || (!existing && !input.legacy)) return directories.prepare(input);
|
||||
let refreshStoppedCopy = false;
|
||||
const workspace = await fs.realpath(input.cwd);
|
||||
const expectedLocalRoot = path.join(workspace, ".paperclip-runtime", `instruction-edits-${input.runId}`, "instructions");
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { inspectAgentFile, fileHash, agentFilePath, MAX_AGENT_FILE_BYTES } from "./agent-file-store.js";
|
||||
import { persistentAgentFiles, readPersistentAgentFile, seedPersistentAgentHome } from "./persistent-agent-files.js";
|
||||
import { adoptAgentFiles, inspectAgentFile, fileHash, agentFilePath, MAX_AGENT_FILE_BYTES } from "./agent-file-store.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agentInstructionHeads, agents, type Db } from "@paperclipai/db";
|
||||
import { instructionPath, assertInstructionPathSafe, instructionBytes, readInstructionBytes } from "./agent-instruction-files.js";
|
||||
import { instructionPath, assertInstructionPathSafe, instructionBytes, readInstructionBytes, MAX_INSTRUCTION_BYTES } from "./agent-instruction-files.js";
|
||||
import { notFound, unprocessable } from "../errors.js";
|
||||
import { resolveHomeAwarePath, resolvePaperclipInstanceRoot } from "../home-paths.js";
|
||||
|
||||
@@ -489,6 +490,30 @@ export function syncInstructionsBundleConfigFromFilePath(
|
||||
}
|
||||
|
||||
export function agentInstructionsService(db?: Db) {
|
||||
async function remoteFiles(agent: AgentLike) {
|
||||
const remote = db ? await persistentAgentFiles(db, agent.companyId, agent.id) : null;
|
||||
if (remote && db) {
|
||||
const root = await db.transaction(async tx => {
|
||||
const [current] = await tx.select().from(agents).where(and(eq(agents.id, agent.id), eq(agents.companyId, agent.companyId)));
|
||||
if (!current) throw notFound("Agent not found");
|
||||
return adoptAgentFiles(tx, current);
|
||||
});
|
||||
await seedPersistentAgentHome(remote, root);
|
||||
}
|
||||
return remote;
|
||||
}
|
||||
async function remotePaths(remote: NonNullable<Awaited<ReturnType<typeof remoteFiles>>>, relative = ""): Promise<Array<{ path: string; size: number }>> {
|
||||
const result: Array<{ path: string; size: number }> = [];
|
||||
for (const entry of await remote.list(relative || undefined)) {
|
||||
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
instructionPath(name);
|
||||
if (entry.kind === "directory") {
|
||||
if (!IGNORED_INSTRUCTIONS_DIRECTORY_NAMES.has(entry.name)) result.push(...await remotePaths(remote, name));
|
||||
} else if (!IGNORED_INSTRUCTIONS_FILE_NAMES.has(entry.name)) result.push({ path: name, size: entry.size });
|
||||
if (result.length > 100_000) throw unprocessable("Agent folder exceeds its entry limit");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
async function assertUnversionedEntry(agent: AgentLike, entryFile: string, connection: Db | Parameters<Parameters<Db["transaction"]>[0]>[0] | undefined = db) {
|
||||
if (!connection) throw unprocessable("Bundle initialization requires the database-backed instructions service");
|
||||
const [head] = await connection.select().from(agentInstructionHeads).where(and(eq(agentInstructionHeads.companyId, agent.companyId),
|
||||
@@ -497,6 +522,15 @@ export function agentInstructionsService(db?: Db) {
|
||||
}
|
||||
async function getBundle(agent: AgentLike): Promise<AgentInstructionsBundle> {
|
||||
const state = await recoverManagedBundleState(agent, deriveBundleState(agent));
|
||||
const remote = await remoteFiles(agent);
|
||||
if (remote) {
|
||||
const summaries: AgentInstructionsFileSummary[] = [];
|
||||
for (const { path: relative, size } of await remotePaths(remote)) {
|
||||
const file = size <= MAX_INSTRUCTION_BYTES ? await readPersistentAgentFile(remote, relative) : null;
|
||||
summaries.push(summarizeFile(relative, state.entryFile, size, file?.bytes ?? null, file?.sha256));
|
||||
}
|
||||
return toBundle(agent, { ...state, rootPath: remote.root, resolvedEntryPath: `${remote.root}/${state.entryFile}` }, summaries);
|
||||
}
|
||||
if (!state.rootPath) return toBundle(agent, state, []);
|
||||
await assertInstructionPathSafe(state.rootPath, state.entryFile);
|
||||
const stat = await statIfExists(state.rootPath);
|
||||
@@ -532,6 +566,13 @@ export function agentInstructionsService(db?: Db) {
|
||||
content,
|
||||
};
|
||||
}
|
||||
const remote = await remoteFiles(agent);
|
||||
if (remote) {
|
||||
const file = await readPersistentAgentFile(remote, relativePath);
|
||||
if (!file) throw notFound("Instructions file not found");
|
||||
const summary = summarizeFile(relativePath, state.entryFile, file.bytes.length, file.bytes, file.sha256);
|
||||
return { ...summary, content: summary.binary ? "" : file.bytes.toString("utf8") };
|
||||
}
|
||||
if (!state.rootPath) throw notFound("Agent instructions bundle is not configured");
|
||||
await assertInstructionPathSafe(state.rootPath, relativePath);
|
||||
const file = await inspectAgentFile(state.rootPath, relativePath);
|
||||
@@ -593,6 +634,14 @@ export function agentInstructionsService(db?: Db) {
|
||||
const state = await recoverManagedBundleState(agent, deriveBundleState(agent));
|
||||
const nextMode = input.mode ?? state.mode ?? "managed";
|
||||
const nextEntryFile = input.entryFile ? normalizeRelativeFilePath(input.entryFile) : state.entryFile;
|
||||
const remote = await remoteFiles(agent);
|
||||
if (remote) {
|
||||
if (nextMode !== "managed") throw unprocessable("Persistent computer files require managed instructions");
|
||||
if (!await readPersistentAgentFile(remote, nextEntryFile)) throw notFound("The selected instruction entry does not exist on the computer");
|
||||
const adapterConfig = applyBundleConfig(state.config, { mode: "managed", rootPath: resolveManagedInstructionsRoot(agent),
|
||||
entryFile: nextEntryFile, clearLegacyPromptTemplate: input.clearLegacyPromptTemplate });
|
||||
return { adapterConfig, bundle: await getBundle({ ...agent, adapterConfig }) };
|
||||
}
|
||||
let nextRootPath: string;
|
||||
|
||||
if (nextMode === "managed") {
|
||||
@@ -665,6 +714,7 @@ export function agentInstructionsService(db?: Db) {
|
||||
throw unprocessable("Entry edits require the canonical instruction commit service and baseRevisionId", { code: "INSTRUCTION_REVISION_REQUIRED" });
|
||||
}
|
||||
if (configured.mode !== "external") agentFilePath(relativePath);
|
||||
if (await remoteFiles(agent)) throw unprocessable("Persistent remote file edits require the content API and expected content hash");
|
||||
const prepared = await ensureWritableBundle(agent, options);
|
||||
instructionBytes(content);
|
||||
await assertInstructionPathSafe(prepared.state.rootPath!, relativePath);
|
||||
@@ -693,6 +743,7 @@ export function agentInstructionsService(db?: Db) {
|
||||
if (normalizedPath === state.entryFile) {
|
||||
throw unprocessable("Cannot delete the bundle entry file");
|
||||
}
|
||||
if (await remoteFiles(agent)) throw unprocessable("Persistent remote file deletion requires the content API and expected content hash");
|
||||
const absolutePath = await assertInstructionPathSafe(state.rootPath, normalizedPath);
|
||||
await fs.rm(absolutePath, { force: true });
|
||||
const adapterConfig = buildPersistedBundleConfig(derived, state);
|
||||
@@ -728,6 +779,16 @@ export function agentInstructionsService(db?: Db) {
|
||||
warnings: string[];
|
||||
}> {
|
||||
const state = await recoverManagedBundleState(agent, deriveBundleState(agent));
|
||||
const remote = await remoteFiles(agent);
|
||||
if (remote) {
|
||||
const files: Record<string, string> = {};
|
||||
for (const { path: relative, size } of await remotePaths(remote)) {
|
||||
if (size > MAX_INSTRUCTION_BYTES) continue;
|
||||
const file = await readPersistentAgentFile(remote, relative);
|
||||
if (file) files[relative] = file.bytes.toString("utf8");
|
||||
}
|
||||
return { files, entryFile: state.entryFile, warnings: state.warnings };
|
||||
}
|
||||
if (state.rootPath) {
|
||||
const stat = await statIfExists(state.rootPath);
|
||||
if (stat?.isDirectory()) {
|
||||
@@ -760,6 +821,7 @@ export function agentInstructionsService(db?: Db) {
|
||||
entryFile?: string;
|
||||
},
|
||||
): Promise<{ bundle: AgentInstructionsBundle; adapterConfig: Record<string, unknown> }> {
|
||||
if (await remoteFiles(agent)) throw unprocessable("Replace individual persistent files using their expected content hash");
|
||||
const rootPath = resolveManagedInstructionsRoot(agent);
|
||||
const entryFile = options?.entryFile ? normalizeRelativeFilePath(options.entryFile) : ENTRY_FILE_DEFAULT;
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { seedPersistentAgentHome } from "./persistent-agent-files.js";
|
||||
|
||||
describe("persistent home initialization", () => {
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => { for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); });
|
||||
async function local() {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "boat-agent-seed-")); roots.push(root);
|
||||
await fs.mkdir(path.join(root, "memory"));
|
||||
await fs.writeFile(path.join(root, "AGENTS.md"), "instructions");
|
||||
await fs.writeFile(path.join(root, "memory", "binary.bin"), Buffer.from([0, 255, 1]));
|
||||
return root;
|
||||
}
|
||||
it("adopts an existing empty remote folder without reading or uploading the controller tree", async () => {
|
||||
const seedBytes = vi.fn();
|
||||
await seedPersistentAgentHome({ list: async () => [], seedBytes } as never, "/missing-controller-folder");
|
||||
expect(seedBytes).not.toHaveBeenCalled();
|
||||
});
|
||||
it("seeds absent personal storage once with exact binary bytes", async () => {
|
||||
const seedBytes = vi.fn(async () => ({ seeded: true }));
|
||||
await seedPersistentAgentHome({ list: async () => { throw { code: "not_found" }; }, seedBytes } as never, await local());
|
||||
expect(seedBytes).toHaveBeenCalledWith({ "AGENTS.md": Buffer.from("instructions"), "memory/binary.bin": Buffer.from([0, 255, 1]) });
|
||||
});
|
||||
it("does not interpret provider outages as an absent directory", async () => {
|
||||
const seedBytes = vi.fn();
|
||||
await expect(seedPersistentAgentHome({ list: async () => { throw new Error("offline"); }, seedBytes } as never, await local())).rejects.toThrow("offline");
|
||||
expect(seedBytes).not.toHaveBeenCalled();
|
||||
});
|
||||
it("rejects symlinked initial files before any upload", async () => {
|
||||
const root = await local();
|
||||
await fs.symlink(path.join(root, "AGENTS.md"), path.join(root, "alias.md"));
|
||||
const seedBytes = vi.fn();
|
||||
await expect(seedPersistentAgentHome({ list: async () => { throw { code: "not_found" }; }, seedBytes } as never, root)).rejects.toThrow("links or special files");
|
||||
expect(seedBytes).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agents, environments, instanceSettings, type Db } from "@paperclipai/db";
|
||||
import { computerService } from "../modules/computers/index.js";
|
||||
import { HttpError } from "../errors.js";
|
||||
import { instructionPath } from "./agent-instruction-files.js";
|
||||
|
||||
/** An assignment selects authority, never an arbitrary path supplied by a caller.
|
||||
* The computer service separately checks the company attachment and placement. */
|
||||
export async function persistentAgentFiles(db: Db, companyId: string, agentId: string, environmentId?: string) {
|
||||
let selected = environmentId;
|
||||
if (!selected) {
|
||||
const [agent] = await db.select({ environmentId: agents.defaultEnvironmentId }).from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.id, agentId)));
|
||||
if (!agent) return null;
|
||||
selected = agent.environmentId ?? undefined;
|
||||
}
|
||||
if (!selected) {
|
||||
const [settings] = await db.select({ environmentId: instanceSettings.defaultEnvironmentId }).from(instanceSettings)
|
||||
.where(eq(instanceSettings.singletonKey, "default"));
|
||||
selected = settings?.environmentId ?? undefined;
|
||||
}
|
||||
if (!selected) return null;
|
||||
const [environment] = await db.select({ driver: environments.driver, config: environments.config }).from(environments)
|
||||
.where(eq(environments.id, selected));
|
||||
if (environment?.driver !== "computer" || environment.config.provider !== "boat") return null;
|
||||
return computerFileAccess(() => computerService(db).files({ companyId, agentId, environmentId: selected! }));
|
||||
}
|
||||
|
||||
export function isMissingRemoteFile(error: unknown) {
|
||||
const value = error as { status?: number; code?: string };
|
||||
return value?.status === 404 || value?.code === "not_found";
|
||||
}
|
||||
|
||||
export async function readPersistentAgentFile(files: NonNullable<Awaited<ReturnType<typeof persistentAgentFiles>>>, relative: string) {
|
||||
try { return await files.readBytes(instructionPath(relative)); }
|
||||
catch (error) { if (isMissingRemoteFile(error)) return null; throw error; }
|
||||
}
|
||||
|
||||
/** First attachment only. Existing directories are adopted, including empty ones.
|
||||
* Later turns never read/upload the controller copy of personal files. */
|
||||
export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnType<typeof persistentAgentFiles>>>, localRoot: string) {
|
||||
try { await files.list(); return; }
|
||||
catch (error) { if (!isMissingRemoteFile(error)) throw error; }
|
||||
const seed: Record<string, Buffer> = {};
|
||||
let total = 0, count = 0;
|
||||
async function walk(relative: string) {
|
||||
const entries = await fs.readdir(path.join(localRoot, relative), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
|
||||
if (error.code === "ENOENT" && relative === "") return [];
|
||||
throw error;
|
||||
});
|
||||
for (const entry of entries) {
|
||||
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
instructionPath(name);
|
||||
if (++count > 100_000) throw new Error("Initial agent folder exceeds its entry limit");
|
||||
const stat = await fs.lstat(path.join(localRoot, name));
|
||||
if (stat.isDirectory()) await walk(name);
|
||||
else if (stat.isFile() && stat.nlink === 1) {
|
||||
total += stat.size;
|
||||
if (stat.size > 256 * 1024 * 1024 || total > 2 * 1024 * 1024 * 1024) throw new Error("Initial agent folder exceeds its storage limit");
|
||||
seed[name] = await fs.readFile(path.join(localRoot, name));
|
||||
} else throw new Error("Agent folders cannot contain links or special files");
|
||||
}
|
||||
}
|
||||
await walk("");
|
||||
await files.seedBytes(seed);
|
||||
}
|
||||
|
||||
/** Translate the domain's transport-neutral errors at the HTTP service edge. */
|
||||
export async function computerFileAccess<T extends object>(open: () => Promise<T>): Promise<T> {
|
||||
function translate(error: unknown): never {
|
||||
const code = (error as { code?: string })?.code;
|
||||
const status = code === "not_found" ? 404 : code === "forbidden" ? 403 : code === "conflict" ? 409
|
||||
: code === "invalid" ? 422 : code === "provider_error" ? 502 : null;
|
||||
if (status) throw new HttpError(status, error instanceof Error ? error.message : "Computer file operation failed", {
|
||||
code: code === "conflict" ? "AGENT_FILE_CONFLICT" : code,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
let access: T;
|
||||
try { access = await open(); } catch (error) { return translate(error); }
|
||||
return new Proxy(access, {
|
||||
get(target, key, receiver) {
|
||||
const value = Reflect.get(target, key, receiver);
|
||||
if (typeof value !== "function") return value;
|
||||
return async (...args: unknown[]) => {
|
||||
try { return await value.apply(target, args); } catch (error) { return translate(error); }
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { computerFileAccess } from "./persistent-agent-files.js";
|
||||
import { computerService } from "../modules/computers/index.js";
|
||||
import { issueReadSqlCondition, executionWorkspaceReadSqlCondition, projectReadSqlCondition, type AuthorizationActor } from "./authorization.js";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
@@ -45,6 +47,7 @@ export interface WorkspaceFileScanContext {
|
||||
const DENIED_SEGMENTS = new Set([
|
||||
".git",
|
||||
".paperclip",
|
||||
".paperclip-runtime",
|
||||
"node_modules",
|
||||
".pnpm-store",
|
||||
".yarn",
|
||||
@@ -121,6 +124,7 @@ type WorkspaceCandidate = {
|
||||
label: string;
|
||||
rootPath: string | null;
|
||||
remote: boolean;
|
||||
placement?: { companyId: string; environmentId: string; placementId: string };
|
||||
};
|
||||
|
||||
type NormalizedPath = {
|
||||
@@ -480,6 +484,13 @@ function candidateFromExecutionWorkspace(row: ExecutionWorkspaceRow): WorkspaceC
|
||||
label: row.name,
|
||||
rootPath,
|
||||
remote,
|
||||
...(row.metadata?.fileAuthority && typeof row.metadata.fileAuthority === "object" &&
|
||||
(row.metadata.fileAuthority as Record<string, unknown>).kind === "remote-persistent" &&
|
||||
typeof (row.metadata.fileAuthority as Record<string, unknown>).environmentId === "string" &&
|
||||
typeof (row.metadata.fileAuthority as Record<string, unknown>).placementId === "string"
|
||||
? { placement: { companyId: row.companyId,
|
||||
environmentId: (row.metadata.fileAuthority as Record<string, string>).environmentId!,
|
||||
placementId: (row.metadata.fileAuthority as Record<string, string>).placementId! } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1039,6 +1050,65 @@ async function listChangedWorkspaceFiles(input: {
|
||||
}
|
||||
|
||||
export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor) {
|
||||
async function placementFiles(candidate: WorkspaceCandidate) {
|
||||
if (!candidate.placement) throw unprocessable("Workspace placement is unavailable", { code: "remote_workspace" });
|
||||
return computerFileAccess(() => computerService(db).workspaceFiles(candidate.placement!));
|
||||
}
|
||||
async function statPersistentCandidate(candidate: WorkspaceCandidate, normalized: NormalizedPath, directory = false) {
|
||||
throwIfDenied(normalized.segments);
|
||||
const files = await placementFiles(candidate);
|
||||
if (directory) {
|
||||
await files.list(normalized.relativePath);
|
||||
return directoryResource({ candidate, relativePath: normalized.relativePath });
|
||||
}
|
||||
const parent = path.posix.dirname(normalized.relativePath);
|
||||
const entry = (await files.list(parent === "." ? undefined : parent)).find(item => item.name === path.posix.basename(normalized.relativePath));
|
||||
if (!entry || entry.kind !== "file") throw notFound("Workspace file not found");
|
||||
const item = listItemFromStat({ candidate, relativePath: normalized.relativePath,
|
||||
stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!;
|
||||
const { relativePath: _relative, modifiedAt: _modified, ...resource } = item;
|
||||
return { ...resource, denialReason: item.capabilities.preview ? null : entry.size > previewCapForKind(item.previewKind) ? "too_large" : "unsupported_content" } as ResolvedWorkspaceResource;
|
||||
}
|
||||
async function persistentContent(candidate: WorkspaceCandidate, normalized: NormalizedPath): Promise<WorkspaceFileContent> {
|
||||
const resource = await statPersistentCandidate(candidate, normalized);
|
||||
if (!resource.capabilities.preview) throw unprocessable("Workspace file cannot be previewed", { code: resource.denialReason ?? "unsupported_content" });
|
||||
const { bytes } = await (await placementFiles(candidate)).readBytes(normalized.relativePath, previewCapForKind(resource.previewKind));
|
||||
if (resource.previewKind === "text" && !looksLikeText(bytes.subarray(0, TEXT_SNIFF_BYTES))) throw unprocessable("Workspace file is not text", { code: "binary_content" });
|
||||
return { resource, content: { encoding: resource.previewKind === "text" ? "utf8" : "base64", data: bytes.toString(resource.previewKind === "text" ? "utf8" : "base64") } };
|
||||
}
|
||||
async function persistentList(candidate: WorkspaceCandidate, options: {
|
||||
selector: WorkspaceFileSelector; mode: WorkspaceFileListMode; normalizedPath: NormalizedPath | null;
|
||||
q: string | null; normalizedQuery: string | null; limit: number; offset: number;
|
||||
}): Promise<WorkspaceFileListResponse> {
|
||||
const { selector, mode, normalizedPath, q, normalizedQuery, limit, offset } = options;
|
||||
if (mode === "changed") return unavailableFileList({ selector, mode, path: normalizedPath?.relativePath ?? null, q, limit, offset, candidate, reason: "changed_unavailable" });
|
||||
if (normalizedPath) throwIfDenied(normalizedPath.segments);
|
||||
const files = await placementFiles(candidate);
|
||||
const items: WorkspaceFileListItem[] = [];
|
||||
let scannedCount = 0, truncated = false;
|
||||
const recursive = mode === "recent" || Boolean(normalizedQuery);
|
||||
async function walk(relative: string, depth: number) {
|
||||
if (depth > MAX_LIST_DEPTH) { truncated = true; return; }
|
||||
for (const entry of await files.list(relative || undefined)) {
|
||||
if (++scannedCount > WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES) { truncated = true; return; }
|
||||
const name = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
const normalized = normalizeWorkspaceRelativePath(name);
|
||||
if (shouldPruneSegments(normalized.segments)) continue;
|
||||
if (entry.kind === "directory") {
|
||||
if (recursive) await walk(name, depth + 1);
|
||||
else items.push(listItemFromDirectory({ candidate, relativePath: name, stat: { mtime: new Date(entry.mtimeMs) } }));
|
||||
} else if (matchesSearch(name, normalizedQuery)) {
|
||||
items.push(listItemFromStat({ candidate, relativePath: name, stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!);
|
||||
}
|
||||
if (truncated) return;
|
||||
}
|
||||
}
|
||||
await walk(normalizedPath?.relativePath ?? "", 0);
|
||||
items.sort((a, b) => mode === "recent" ? String(b.modifiedAt).localeCompare(String(a.modifiedAt)) :
|
||||
(a.kind === b.kind ? a.relativePath.localeCompare(b.relativePath) : a.kind === "directory" ? -1 : 1));
|
||||
return availableFileList({ selector, mode, path: normalizedPath?.relativePath ?? null, q, limit, offset, candidate,
|
||||
items: items.slice(offset, offset + limit), scannedCount, truncated: truncated || items.length > offset + limit });
|
||||
}
|
||||
async function visibleCandidates(candidates: WorkspaceCandidate[]) {
|
||||
if (!actor) return candidates;
|
||||
const visible: WorkspaceCandidate[] = [];
|
||||
@@ -1305,6 +1375,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
const hasExplicitTarget = Boolean(explicitTarget?.candidate);
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.placement) return availabilityResult(item.query, await statPersistentCandidate(candidate, item.normalizedPath, item.directory));
|
||||
if (candidate.remote) {
|
||||
if (hasExplicitTarget || item.query.workspace !== "auto") {
|
||||
return availabilityResult(item.query, remoteResource(candidate, item.normalizedPath.relativePath));
|
||||
@@ -1373,6 +1444,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.placement) return statPersistentCandidate(candidate, normalized, isDirectoryRequest);
|
||||
if (candidate.remote) {
|
||||
if (explicitTarget || selector !== "auto") return remoteResource(candidate, normalized.relativePath);
|
||||
continue;
|
||||
@@ -1431,6 +1503,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
let firstUnavailable: { candidate: WorkspaceCandidate; reason: string } | null = null;
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.placement) return persistentList(candidate, { selector, mode, normalizedPath, q, normalizedQuery, limit, offset });
|
||||
if (candidate.remote) {
|
||||
firstUnavailable ??= { candidate, reason: "remote_workspace" };
|
||||
if (explicitTarget || selector !== "auto") {
|
||||
@@ -1637,6 +1710,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.placement) return persistentContent(candidate, normalized);
|
||||
if (candidate.remote) {
|
||||
if (explicitTarget || selector !== "auto") {
|
||||
throw unprocessable("Remote workspaces cannot be previewed by the server", { code: "remote_workspace" });
|
||||
@@ -1709,7 +1783,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
workspace?: WorkspaceFileSelector | null;
|
||||
projectId?: string | null;
|
||||
workspaceId?: string | null;
|
||||
}, opts: { issue?: IssueRow } = {}): Promise<LocalResolvedFile> {
|
||||
}, opts: { issue?: IssueRow } = {}): Promise<LocalResolvedFile | { resource: ResolvedWorkspaceResource; bytes: Buffer }> {
|
||||
const issue = opts.issue ?? await getIssue(issueId);
|
||||
const selector = input.workspace ?? "auto";
|
||||
const explicitTarget = Boolean(input.projectId || input.workspaceId);
|
||||
@@ -1721,6 +1795,11 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor)
|
||||
|
||||
let lastNotFound: unknown = null;
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.placement) {
|
||||
const resource = await statPersistentCandidate(candidate, normalized);
|
||||
const { bytes } = await (await placementFiles(candidate)).readBytes(normalized.relativePath);
|
||||
return { resource: { ...resource, byteSize: bytes.length }, bytes };
|
||||
}
|
||||
if (candidate.remote) {
|
||||
if (explicitTarget || selector !== "auto") {
|
||||
throw unprocessable("Remote workspaces cannot be downloaded by the server", { code: "remote_workspace" });
|
||||
|
||||
Reference in new issue
Block a user