mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-10 03:08:10 +02:00
## Thinking Path
> - Paperclip is the open source app people use to manage AI agents for
work
> - Agent adapters stage referenced projects into controlled sandboxes
> - The ignore scan must preserve exact Git path bytes and fail closed
on unsafe input
> - Unbounded ignored-path data and raw diagnostics can harm resource
use or expose host details
> - This pull request adds exact path parsing, input bounds, fixed
failure categories, and saturation-only retry
> - The benefit is safer and more predictable referenced-project staging
## Linked Issues or Issue Description
**What happened?**
The referenced-project ignore scan trimmed NUL-delimited Git paths. It
also accepted a large ignored-path set and exposed raw failure details
through staging errors and warnings.
**Expected behavior**
The scan must preserve leading and trailing whitespace in Git paths. It
must reject oversized ignored-path data and expose only fixed failure
categories.
**Steps to reproduce**
1. Run the referenced-project ignore scan with paths that start or end
with whitespace.
2. Provide more than 10,000 ignored entries or more than 2 MiB of path
bytes.
3. Trigger a scan failure and inspect the reported reason.
**Paperclip version or commit**
d560bc2ae2
**Deployment mode**
Built from source.
**Installation method**
Built from source.
**Agent adapter(s) involved**
Not adapter-specific.
**Database mode**
Not database-related.
**Additional context**
This change covers the overlay diff, untracked, deleted, and ignored Git
paths. It also retries only typed scheduler saturation failures.
## What Changed
- Preserve all bytes in NUL-delimited Git path records.
- Bound ignored-entry count and total UTF-8 path bytes during parsing.
- Redact scan failure details to three fixed reason categories.
- Retry only the typed scheduler saturation error, with three total
attempts and 1 second then 2 second waits.
- Add tests for path whitespace, limits, diagnostics, retry behavior,
and scheduler code parity.
## Verification
- `npx tsc --noEmit` in `packages/adapter-utils` passed.
- `npx vitest run packages/adapter-utils` passed with 977 tests and 4
skipped.
- Continuous integration must run the server suite and the full
repository gates.
## Risks
The scan now rejects ignored-path data above fixed limits. Saturation
retries add up to 3 seconds before a final failure. The resolver still
fails closed for all other errors.
## Model Used
OpenAI GPT-5. The model used tool calls, code inspection, and command
execution. The exact context window and reasoning mode are not exposed
by the runtime.
## Checklist
- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with `Fixes: #` / `Closes
#` / `Refs #` OR (b) described the issue in-PR following the relevant
issue template
- [x] I have not referenced internal/instance-local Paperclip issues or
links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip`
URLs)
- [x] My branch name describes the change (e.g. `docs/...`, `fix/...`)
and contains no internal Paperclip ticket id or instance-derived details
- [x] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented any risks above
- [x] All Paperclip CI gates are green
- [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge
---------
Co-authored-by: Paperclip <noreply@paperclip.ing>
326 lines
12 KiB
TypeScript
326 lines
12 KiB
TypeScript
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
|
|
const {
|
|
prepareWorkspaceForSshExecution,
|
|
restoreWorkspaceFromSshExecution,
|
|
runSshCommand,
|
|
syncDirectoryToSsh,
|
|
} = vi.hoisted(() => ({
|
|
prepareWorkspaceForSshExecution: vi.fn(async () => ({ gitBacked: false })),
|
|
restoreWorkspaceFromSshExecution: vi.fn(async () => undefined),
|
|
runSshCommand: vi.fn(async () => ({
|
|
stdout: Buffer.from('{"token":"remote"}\n').toString("base64"),
|
|
stderr: "",
|
|
})),
|
|
syncDirectoryToSsh: vi.fn(async (_input: { localDir: string }) => undefined),
|
|
}));
|
|
|
|
vi.mock("./ssh.js", () => ({
|
|
prepareWorkspaceForSshExecution,
|
|
restoreWorkspaceFromSshExecution,
|
|
runSshCommand,
|
|
syncDirectoryToSsh,
|
|
}));
|
|
|
|
import { prepareRemoteManagedRuntime } from "./remote-managed-runtime.js";
|
|
import { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js";
|
|
import { setExpensiveWorkspaceGitExecutor } from "./git-workspace-sync.js";
|
|
|
|
describe("remote managed runtime", () => {
|
|
const cleanupDirs: string[] = [];
|
|
|
|
afterEach(async () => {
|
|
vi.clearAllMocks();
|
|
while (cleanupDirs.length > 0) {
|
|
const dir = cleanupDirs.pop();
|
|
if (!dir) continue;
|
|
await rm(dir, { recursive: true, force: true }).catch(() => undefined);
|
|
}
|
|
});
|
|
|
|
it("restores runtime assets without restoring an in-place SSH workspace", async () => {
|
|
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-assets-only-"));
|
|
cleanupDirs.push(rootDir);
|
|
const workspaceDir = path.join(rootDir, "workspace");
|
|
const homeDir = path.join(rootDir, "home");
|
|
await mkdir(workspaceDir, { recursive: true });
|
|
await mkdir(homeDir, { recursive: true });
|
|
await writeFile(path.join(homeDir, "auth.json"), '{"token":"host"}\n', "utf8");
|
|
|
|
let restoredAuth = "";
|
|
const prepared = await prepareRemoteManagedRuntime({
|
|
spec: {
|
|
host: "127.0.0.1",
|
|
port: 2222,
|
|
username: "fixture",
|
|
remoteWorkspacePath: "/app",
|
|
remoteCwd: "/app",
|
|
privateKey: "PRIVATE KEY",
|
|
knownHosts: "KNOWN HOSTS",
|
|
strictHostKeyChecking: true,
|
|
},
|
|
runId: "run-in-place",
|
|
adapterKey: "codex",
|
|
workspaceLocalDir: workspaceDir,
|
|
workspaceRemoteDir: "/app",
|
|
syncWorkspace: false,
|
|
assets: [
|
|
{
|
|
key: "home",
|
|
localDir: homeDir,
|
|
restore: async ({ assetDir, readFile }) => {
|
|
restoredAuth = (await readFile(path.posix.join(assetDir, "auth.json"))).toString("utf8");
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
expect(prepareWorkspaceForSshExecution).not.toHaveBeenCalled();
|
|
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({
|
|
localDir: homeDir,
|
|
remoteDir: "/app/.paperclip-runtime/codex/home",
|
|
}));
|
|
|
|
await prepared.restoreWorkspace();
|
|
|
|
expect(restoreWorkspaceFromSshExecution).not.toHaveBeenCalled();
|
|
expect(runSshCommand).toHaveBeenCalledWith(
|
|
expect.anything(),
|
|
"base64 < '/app/.paperclip-runtime/codex/home/auth.json'",
|
|
{ maxBuffer: 1024 * 1024 },
|
|
);
|
|
expect(restoredAuth).toBe('{"token":"remote"}\n');
|
|
});
|
|
|
|
it("stages each additional project into its own isolated SSH dir, isolating one failure", async () => {
|
|
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-additional-"));
|
|
cleanupDirs.push(rootDir);
|
|
const workspaceDir = path.join(rootDir, "workspace");
|
|
const firstDir = path.join(rootDir, "referenced-first");
|
|
const secondDir = path.join(rootDir, "referenced-second");
|
|
const brokenDir = path.join(rootDir, "referenced-broken");
|
|
await mkdir(workspaceDir, { recursive: true });
|
|
|
|
// The transfer rejects only for the broken project's directory.
|
|
syncDirectoryToSsh.mockImplementation(async (input: { localDir: string }) => {
|
|
if (input.localDir === brokenDir) throw new Error("ssh transfer failed");
|
|
return undefined;
|
|
});
|
|
|
|
const prepared = await prepareRemoteManagedRuntime({
|
|
spec: {
|
|
host: "127.0.0.1",
|
|
port: 2222,
|
|
username: "fixture",
|
|
remoteWorkspacePath: "/app",
|
|
remoteCwd: "/app",
|
|
privateKey: "PRIVATE KEY",
|
|
knownHosts: "KNOWN HOSTS",
|
|
strictHostKeyChecking: true,
|
|
},
|
|
runId: "run-additional",
|
|
adapterKey: "codex",
|
|
workspaceLocalDir: workspaceDir,
|
|
workspaceRemoteDir: "/app",
|
|
syncWorkspace: false,
|
|
additionalSources: [
|
|
{ localPath: firstDir, projectId: "first", ignoreResolution: { kind: "other" } },
|
|
{ localPath: brokenDir, projectId: "broken", ignoreResolution: { kind: "other" } },
|
|
{ localPath: secondDir, projectId: "second", ignoreResolution: { kind: "other" } },
|
|
],
|
|
});
|
|
|
|
// Each healthy project staged into its OWN isolated dir under the runtime
|
|
// root; the broken one is skipped, not fatal.
|
|
expect(Object.keys(prepared.additionalSourceDirs).sort()).toEqual(["first", "second"]);
|
|
expect(prepared.additionalSourceDirs.first).toBe("/app/.paperclip-runtime/codex/project-first");
|
|
expect(prepared.additionalSourceDirs.second).toBe("/app/.paperclip-runtime/codex/project-second");
|
|
expect(prepared.additionalSourceDirs.broken).toBeUndefined();
|
|
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({
|
|
localDir: firstDir,
|
|
remoteDir: "/app/.paperclip-runtime/codex/project-first",
|
|
}));
|
|
expect(syncDirectoryToSsh).toHaveBeenCalledWith(expect.objectContaining({
|
|
localDir: secondDir,
|
|
remoteDir: "/app/.paperclip-runtime/codex/project-second",
|
|
}));
|
|
});
|
|
|
|
it("skips an additional project whose localPath is not absolute", async () => {
|
|
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-relative-"));
|
|
cleanupDirs.push(rootDir);
|
|
const workspaceDir = path.join(rootDir, "workspace");
|
|
const healthyDir = path.join(rootDir, "referenced-healthy");
|
|
await mkdir(workspaceDir, { recursive: true });
|
|
|
|
const prepared = await prepareRemoteManagedRuntime({
|
|
spec: {
|
|
host: "127.0.0.1",
|
|
port: 2222,
|
|
username: "fixture",
|
|
remoteWorkspacePath: "/app",
|
|
remoteCwd: "/app",
|
|
privateKey: "PRIVATE KEY",
|
|
knownHosts: "KNOWN HOSTS",
|
|
strictHostKeyChecking: true,
|
|
},
|
|
runId: "run-relative",
|
|
adapterKey: "codex",
|
|
workspaceLocalDir: workspaceDir,
|
|
workspaceRemoteDir: "/app",
|
|
syncWorkspace: false,
|
|
additionalSources: [
|
|
{ localPath: "relative/referenced", projectId: "relative", ignoreResolution: { kind: "other" } },
|
|
{ localPath: healthyDir, projectId: "healthy", ignoreResolution: { kind: "other" } },
|
|
],
|
|
});
|
|
|
|
// The relative-path project never reaches the transfer and is skipped; the
|
|
// absolute-path project still stages.
|
|
expect(Object.keys(prepared.additionalSourceDirs)).toEqual(["healthy"]);
|
|
expect(syncDirectoryToSsh).not.toHaveBeenCalledWith(expect.objectContaining({
|
|
localDir: "relative/referenced",
|
|
}));
|
|
});
|
|
|
|
it("passes a project's resolved Git-ignored paths to the SSH exclude list, escaped", async () => {
|
|
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-ignore-"));
|
|
cleanupDirs.push(rootDir);
|
|
const workspaceDir = path.join(rootDir, "workspace");
|
|
const projectDir = path.join(rootDir, "referenced-project");
|
|
await mkdir(workspaceDir, { recursive: true });
|
|
|
|
await prepareRemoteManagedRuntime({
|
|
spec: {
|
|
host: "127.0.0.1",
|
|
port: 2222,
|
|
username: "fixture",
|
|
remoteWorkspacePath: "/app",
|
|
remoteCwd: "/app",
|
|
privateKey: "PRIVATE KEY",
|
|
knownHosts: "KNOWN HOSTS",
|
|
strictHostKeyChecking: true,
|
|
},
|
|
runId: "run-ignore",
|
|
adapterKey: "codex",
|
|
workspaceLocalDir: workspaceDir,
|
|
workspaceRemoteDir: "/app",
|
|
syncWorkspace: false,
|
|
additionalSources: [
|
|
{
|
|
localPath: projectDir,
|
|
projectId: "proj",
|
|
ignoreResolution: { kind: "git", ignoredPaths: ["secret.env", "build", "weird[1].txt"] },
|
|
},
|
|
],
|
|
});
|
|
|
|
const call = syncDirectoryToSsh.mock.calls.find((entry) => entry[0].localDir === projectDir);
|
|
expect(call).toBeDefined();
|
|
const exclude = (call![0] as { exclude?: string[] }).exclude ?? [];
|
|
// The resolved ignored paths ride the exclude list, glob-escaped, on top of
|
|
// the fixed heavy-directory excludes the SSH lane already applies.
|
|
expect(exclude).toContain("secret.env");
|
|
expect(exclude).toContain("build");
|
|
expect(exclude).toContain("weird\\[1].txt");
|
|
expect(exclude).toContain("node_modules");
|
|
});
|
|
|
|
it("skips a project whose ignore resolution failed without ever calling syncDirectoryToSsh for it", async () => {
|
|
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-failed-"));
|
|
cleanupDirs.push(rootDir);
|
|
const workspaceDir = path.join(rootDir, "workspace");
|
|
const healthyDir = path.join(rootDir, "referenced-healthy");
|
|
const failedDir = path.join(rootDir, "referenced-failed");
|
|
await mkdir(workspaceDir, { recursive: true });
|
|
|
|
const prepared = await prepareRemoteManagedRuntime({
|
|
spec: {
|
|
host: "127.0.0.1",
|
|
port: 2222,
|
|
username: "fixture",
|
|
remoteWorkspacePath: "/app",
|
|
remoteCwd: "/app",
|
|
privateKey: "PRIVATE KEY",
|
|
knownHosts: "KNOWN HOSTS",
|
|
strictHostKeyChecking: true,
|
|
},
|
|
runId: "run-failed",
|
|
adapterKey: "codex",
|
|
workspaceLocalDir: workspaceDir,
|
|
workspaceRemoteDir: "/app",
|
|
syncWorkspace: false,
|
|
additionalSources: [
|
|
{ localPath: healthyDir, projectId: "healthy", ignoreResolution: { kind: "other" } },
|
|
{ localPath: failedDir, projectId: "failed", ignoreResolution: { kind: "failed", reason: "git status timed out" } },
|
|
],
|
|
});
|
|
|
|
// Fail closed: the failed project never reaches the transfer at all — no
|
|
// bytes are sent for it — while the healthy project still stages.
|
|
expect(Object.keys(prepared.additionalSourceDirs)).toEqual(["healthy"]);
|
|
expect(syncDirectoryToSsh).not.toHaveBeenCalledWith(expect.objectContaining({ localDir: failedDir }));
|
|
});
|
|
|
|
it("never leaks a raw absolute path into the remote per-project staging warning", async () => {
|
|
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-remote-runtime-redact-"));
|
|
cleanupDirs.push(rootDir);
|
|
const workspaceDir = path.join(rootDir, "workspace");
|
|
const failedDir = path.join(rootDir, "referenced-failed");
|
|
await mkdir(workspaceDir, { recursive: true });
|
|
await mkdir(failedDir, { recursive: true });
|
|
|
|
// A raw toplevel string that makes `failedDir` a non-descendant, carrying
|
|
// a sensitive absolute path — exactly the shape a caught Git diagnostic
|
|
// could embed. `resolveReferencedSourceIgnore` is the single choke point
|
|
// that must reduce it to the fixed category before anything downstream
|
|
// (here, the remote lane's warning) ever sees it.
|
|
const sensitivePath = "/srv/alice/project";
|
|
let ignoreResolution;
|
|
try {
|
|
setExpensiveWorkspaceGitExecutor(async (input) => {
|
|
if (input.operation === "referenced_source.toplevel") {
|
|
return { stdout: `${sensitivePath}\n`, stderr: "" };
|
|
}
|
|
return { stdout: "", stderr: "" };
|
|
});
|
|
ignoreResolution = await resolveReferencedSourceIgnore(failedDir);
|
|
} finally {
|
|
setExpensiveWorkspaceGitExecutor(null);
|
|
}
|
|
expect(ignoreResolution.kind).toBe("failed");
|
|
|
|
const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
|
try {
|
|
await prepareRemoteManagedRuntime({
|
|
spec: {
|
|
host: "127.0.0.1",
|
|
port: 2222,
|
|
username: "fixture",
|
|
remoteWorkspacePath: "/app",
|
|
remoteCwd: "/app",
|
|
privateKey: "PRIVATE KEY",
|
|
knownHosts: "KNOWN HOSTS",
|
|
strictHostKeyChecking: true,
|
|
},
|
|
runId: "run-redact",
|
|
adapterKey: "codex",
|
|
workspaceLocalDir: workspaceDir,
|
|
workspaceRemoteDir: "/app",
|
|
syncWorkspace: false,
|
|
additionalSources: [{ localPath: failedDir, projectId: "failed", ignoreResolution }],
|
|
});
|
|
|
|
const warnedText = warnSpy.mock.calls.map((call) => call.join(" ")).join("\n");
|
|
expect(warnedText).toContain("failed");
|
|
expect(warnedText).not.toContain(sensitivePath);
|
|
expect(warnedText).not.toContain(failedDir);
|
|
} finally {
|
|
warnSpy.mockRestore();
|
|
}
|
|
});
|
|
});
|