mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 00:05:49 +02:00
refactor: remove code comments added by recent PRs (#16681)
CONTRIBUTING.md says PRs add no code comments, but #16485, #16586, #16621 and #16672 added some. This removes the 215 comment-only lines git blame on master attributes to those merge commits, across 12 code files, plus one blank line the removal left doubled. Rewritten (not added) comments in scripts/assert-cloud-image-sentry.mjs and comments in .github workflows are left alone. No code changes. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
bebeb389f1
commit
e30550ef39
12 files changed
-216
No files matched your search
@@ -144,9 +144,6 @@ RUN pnpm --filter @paperclipai/plugin-sdk build
|
||||
ENV NODE_OPTIONS=--max-old-space-size=4096
|
||||
RUN pnpm --filter @paperclipai/server build
|
||||
RUN test -f server/dist/index.js || (echo "ERROR: server build output missing" && exit 1)
|
||||
# Bundle the server with its workspace packages compiled in, so production
|
||||
# starts with plain node instead of transpiling ~1,200 TypeScript sources
|
||||
# through the tsx loader on every boot (server/scripts/bundle-runtime.mjs).
|
||||
RUN pnpm --filter @paperclipai/server build:runtime \
|
||||
&& test -f server/dist-runtime/index.mjs || (echo "ERROR: server runtime bundle missing" && exit 1)
|
||||
RUN rm -rf packages/paperclip-runner/runner/target
|
||||
@@ -235,11 +232,7 @@ EXPOSE 3100
|
||||
# cgroup pid limit is exhausted and *every* fork() in the container fails.
|
||||
# tini reaps adopted orphans and forwards signals, so the exec chain below and
|
||||
# graceful shutdown are unchanged. Mirrors docker/agent-runtime/Dockerfile.base.
|
||||
# `-s` registers tini as a child subreaper, so it still reaps orphans when a
|
||||
# platform runs its own init as PID 1 (tini then warns and reaps nothing).
|
||||
ENTRYPOINT ["/usr/bin/tini", "-s", "--", "docker-entrypoint.sh"]
|
||||
# The runtime bundle needs no loader. tsx stays installed for the few
|
||||
# development-only paths that load workspace TypeScript on demand.
|
||||
CMD ["node", "server/dist-runtime/index.mjs"]
|
||||
|
||||
# Cloud image variant (build with `--target cloud`): the production image
|
||||
|
||||
@@ -1,20 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
# Boot a built server command and wait for it to become healthy.
|
||||
#
|
||||
# scripts/server-boot-smoke.sh node server/dist-runtime/index.mjs
|
||||
#
|
||||
# Starts the command against a throwaway instance (embedded Postgres,
|
||||
# local_trusted mode, loopback only) and waits until GET /api/health reports
|
||||
# status "ok", which means migrations applied and startup recovery finished.
|
||||
# Prints the time to listen and the time to ready, stops the server, and exits
|
||||
# non-zero if it never became ready, with the server log on stderr.
|
||||
#
|
||||
# The server runs with a cleared environment (only PATH and a scratch HOME),
|
||||
# from a scratch working directory, with its config path pointed into the
|
||||
# scratch directory, so no DATABASE_URL, config file or .env from the caller
|
||||
# can point it at a real instance. It listens on a free port chosen at start,
|
||||
# and a healthy response counts only after the child logs that it bound that
|
||||
# port. Relative command arguments are resolved against the repository root.
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -eq 0 ]; then
|
||||
@@ -71,7 +55,6 @@ while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
cat "$log" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Only trust the port once this child has bound it.
|
||||
if ! grep -q "listener bound on 127.0.0.1:$port" "$log" 2>/dev/null; then sleep 0.5; continue; fi
|
||||
body="$(curl -fsS --max-time 2 "http://127.0.0.1:$port/api/health" 2>/dev/null || true)"
|
||||
if [ -n "$body" ] && [ -z "$listen_ms" ]; then
|
||||
|
||||
@@ -1,36 +1,3 @@
|
||||
// Bundle the built server into `dist-runtime/` for production startup.
|
||||
//
|
||||
// The server's tsc output imports the workspace packages as TypeScript source
|
||||
// (their package exports point at `src/*.ts`), so production used to start
|
||||
// Node with the tsx loader. Every boot then resolved and transpiled thousands
|
||||
// of modules through tsx's hooks: about 8-10 s warm and over 20 s on a cold
|
||||
// container, all before the server could listen. This script runs after the
|
||||
// server `build` and bundles `dist/index.js`, with the workspace packages
|
||||
// compiled in, so production starts with plain `node` in a few seconds.
|
||||
//
|
||||
// The bundle keeps runtime behavior the same as running `dist/index.js`:
|
||||
//
|
||||
// - Third-party packages stay external. A package the server package itself
|
||||
// resolves to the same file stays a bare specifier, so Node resolves it at
|
||||
// runtime from server/node_modules exactly as before (the cloud image adds
|
||||
// optional peers such as @sentry/node there after the build). Only a
|
||||
// dependency that pnpm's strict layout hides from the server package, one
|
||||
// that only a workspace package can see, is emitted as an absolute path.
|
||||
// Resolution follows Node's rules (the `main` field and `node` condition),
|
||||
// not bundler-only `module` builds.
|
||||
// - Every bundled module keeps its own location: `import.meta.url`,
|
||||
// `import.meta.dirname` and `import.meta.filename` are replaced with the
|
||||
// module's original path, so code that finds files next to its source
|
||||
// (scripts, assets, worker entry files) keeps finding them.
|
||||
// - Code splitting keeps dynamic imports as separate chunks. Bundled into a
|
||||
// single file, the server's top-level await never settled (observed on the
|
||||
// production build); split chunks keep the module evaluation order Node
|
||||
// uses for the unbundled output.
|
||||
// - A `require` built from the chunk's location is provided for the few
|
||||
// modules that call `require` from ESM.
|
||||
// - The bundle's entry sets `process.argv[1]` to the original entry file
|
||||
// before loading it, so the server's main-module check (and anything else
|
||||
// that reads argv[1]) sees exactly what it saw when started unbundled.
|
||||
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
@@ -42,14 +9,6 @@ const serverDir = join(scriptDir, "..");
|
||||
|
||||
const REQUIRE_SHIM = "import { createRequire as __paperclipCreateRequire } from 'node:module'; const require = __paperclipCreateRequire(import.meta.url);";
|
||||
|
||||
/**
|
||||
* Replace a module's `import.meta` location properties with its original
|
||||
* path, so bundling does not move where the module looks for nearby files.
|
||||
*
|
||||
* @param {string} source Module source.
|
||||
* @param {string} file Absolute path of the module's original file.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function preserveModuleLocation(source, file) {
|
||||
return source
|
||||
.replace(/\bimport\.meta\.url\b/g, JSON.stringify(pathToFileURL(file).href))
|
||||
@@ -57,13 +16,6 @@ export function preserveModuleLocation(source, file) {
|
||||
.replace(/\bimport\.meta\.filename\b/g, JSON.stringify(file));
|
||||
}
|
||||
|
||||
/**
|
||||
* The bundle's entry: present the original entry file as the main module,
|
||||
* then load it.
|
||||
*
|
||||
* @param {string} entry Absolute path of the original entry file.
|
||||
* @returns {import("esbuild").Plugin}
|
||||
*/
|
||||
function runtimeEntry(entry) {
|
||||
return {
|
||||
name: "runtime-entry",
|
||||
@@ -78,7 +30,6 @@ function runtimeEntry(entry) {
|
||||
};
|
||||
}
|
||||
|
||||
/** @type {import("esbuild").Plugin} */
|
||||
const keepModuleLocations = {
|
||||
name: "keep-module-locations",
|
||||
setup(builder) {
|
||||
@@ -95,13 +46,6 @@ const keepModuleLocations = {
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {(specifier: string) => boolean} isBundled Whether a bare specifier is
|
||||
* bundled (workspace packages) rather than kept external.
|
||||
* @param {string} runtimeDir A directory inside the server package, from which
|
||||
* bare specifiers resolve at runtime.
|
||||
* @returns {import("esbuild").Plugin}
|
||||
*/
|
||||
function externalizeThirdParty(isBundled, runtimeDir) {
|
||||
return {
|
||||
name: "externalize-third-party",
|
||||
@@ -114,8 +58,6 @@ function externalizeThirdParty(isBundled, runtimeDir) {
|
||||
importer: args.importer,
|
||||
pluginData: { externalizeThirdParty: true },
|
||||
});
|
||||
// Builtins and anything esbuild cannot resolve stay as written;
|
||||
// Node resolves them at runtime exactly as before.
|
||||
if (resolved.errors.length > 0 || resolved.external) return { path: args.path, external: true };
|
||||
const fromServer = await builder.resolve(args.path, {
|
||||
kind: args.kind,
|
||||
@@ -129,16 +71,6 @@ function externalizeThirdParty(isBundled, runtimeDir) {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Bundle a built entry point and everything it imports from the workspace.
|
||||
*
|
||||
* @param {{
|
||||
* entry: string,
|
||||
* outdir: string,
|
||||
* isBundled?: (specifier: string) => boolean,
|
||||
* }} options
|
||||
* @returns {Promise<{ outfile: string, inputs: number, chunks: number }>}
|
||||
*/
|
||||
export async function bundleRuntime({ entry, outdir, isBundled = (specifier) => specifier.startsWith("@paperclipai/") }) {
|
||||
await rm(outdir, { recursive: true, force: true });
|
||||
const result = await build({
|
||||
|
||||
@@ -20,7 +20,6 @@ describe("bundleRuntime", () => {
|
||||
|
||||
beforeEach(() => {
|
||||
root = realpathSync(mkdtempSync(path.join(tmpdir(), "paperclip-bundle-runtime-")));
|
||||
// A workspace package that exports TypeScript source, like packages/*.
|
||||
const pkg = path.join(root, "packages", "fixture");
|
||||
write(path.join(pkg, "package.json"), JSON.stringify({ name: "@paperclipai/fixture", type: "module", exports: { ".": "./src/index.ts" } }));
|
||||
write(path.join(pkg, "src", "asset.txt"), "asset beside its source\n");
|
||||
@@ -35,31 +34,24 @@ describe("bundleRuntime", () => {
|
||||
"export const values = { doubled: double(21), onlyForPackage, nodeBuild };",
|
||||
"",
|
||||
].join("\n"));
|
||||
// A dependency only the workspace package can see (pnpm strict layout).
|
||||
write(path.join(pkg, "node_modules", "package-private-dep", "package.json"), JSON.stringify({ name: "package-private-dep", type: "module", main: "index.js" }));
|
||||
write(path.join(pkg, "node_modules", "package-private-dep", "index.js"), "export const onlyForPackage = \"resolved from the package\";\n");
|
||||
// A dependency whose bundler-only "module" build is not valid Node ESM;
|
||||
// Node ignores the "module" condition, bundlers prefer it.
|
||||
write(path.join(pkg, "node_modules", "dual-build-dep", "package.json"), JSON.stringify({
|
||||
name: "dual-build-dep", exports: { ".": { module: "./bundler/index.js", default: "./node.cjs" } },
|
||||
}));
|
||||
write(path.join(pkg, "node_modules", "dual-build-dep", "node.cjs"), "exports.nodeBuild = \"node build\";\n");
|
||||
write(path.join(pkg, "node_modules", "dual-build-dep", "bundler", "index.js"), "export * from \"./extensionless\";\n");
|
||||
|
||||
// The server package sees the workspace package through its node_modules.
|
||||
const server = path.join(root, "server");
|
||||
mkdirSync(path.join(server, "node_modules", "@paperclipai"), { recursive: true });
|
||||
symlinkSync(pkg, path.join(server, "node_modules", "@paperclipai", "fixture"), "dir");
|
||||
write(path.join(server, "package.json"), JSON.stringify({ name: "server", type: "module" }));
|
||||
// A dependency the server package resolves itself, linked from a store
|
||||
// directory the way pnpm installs packages.
|
||||
for (const [version, flavor] of [["1", "installed at build"], ["2", "replaced after build"]]) {
|
||||
const store = path.join(root, "node_modules", ".pnpm", `server-dep@${version}`);
|
||||
write(path.join(store, "package.json"), JSON.stringify({ name: "server-dep", type: "module", main: "index.js" }));
|
||||
write(path.join(store, "index.js"), `export const flavor = ${JSON.stringify(flavor)};\n`);
|
||||
}
|
||||
symlinkSync(path.join(root, "node_modules", ".pnpm", "server-dep@1"), path.join(server, "node_modules", "server-dep"), "dir");
|
||||
// The entry awaits a dynamic import of another built module.
|
||||
write(path.join(server, "dist", "late.js"), "import { values } from \"@paperclipai/fixture\";\nexport const late = () => values.doubled + 1;\n");
|
||||
write(path.join(server, "dist", "index.js"), [
|
||||
"import { asset, location, values } from \"@paperclipai/fixture\";",
|
||||
@@ -87,7 +79,6 @@ describe("bundleRuntime", () => {
|
||||
const output = JSON.parse(stdout);
|
||||
expect(output.values).toEqual({ doubled: 42, onlyForPackage: "resolved from the package", nodeBuild: "node build" });
|
||||
expect(output.late).toBe(43);
|
||||
// Modules keep their original locations, so nearby files still resolve.
|
||||
expect(output.asset).toBe("asset beside its source");
|
||||
expect(output.location).toBe(path.join(root, "packages", "fixture", "src"));
|
||||
expect(output.entryDir).toBe(path.join(root, "server", "dist"));
|
||||
@@ -96,9 +87,6 @@ describe("bundleRuntime", () => {
|
||||
it("leaves the server's own dependencies to runtime resolution", async () => {
|
||||
const outdir = path.join(root, "server", "dist-runtime");
|
||||
const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir });
|
||||
// Re-point the package after the build, as the cloud image does when it
|
||||
// installs optional peers into server/node_modules; the bundle must pick
|
||||
// up the replacement instead of the store path seen at build time.
|
||||
rmSync(path.join(root, "server", "node_modules", "server-dep"));
|
||||
symlinkSync(path.join(root, "node_modules", ".pnpm", "server-dep@2"), path.join(root, "server", "node_modules", "server-dep"), "dir");
|
||||
const { stdout } = await execFileAsync(process.execPath, [summary.outfile], { cwd: path.join(root, "server"), timeout: 30_000 });
|
||||
|
||||
@@ -74,8 +74,6 @@ describe("server image init", () => {
|
||||
const lines = entrypoints(dockerfile);
|
||||
expect(lines.length, "Dockerfile must declare an ENTRYPOINT").toBeGreaterThan(0);
|
||||
for (const line of lines) {
|
||||
// `-s` registers tini as a child subreaper, so it still reaps orphans
|
||||
// when a platform runs its own init as PID 1.
|
||||
expect(
|
||||
line,
|
||||
"node must not inherit PID 1: wrap the entrypoint in tini (as a subreaper) so adopted orphans are reaped",
|
||||
|
||||
@@ -27,8 +27,6 @@ async function commit(cwd: string, file: string, message: string) {
|
||||
await write(path.join(cwd, file), `${message}\n`);
|
||||
await git(cwd, "add", file);
|
||||
await git(cwd, "commit", "-q", "-m", message);
|
||||
// Committing leaves COMMIT_EDITMSG, which the storage check rejects on its
|
||||
// own; remove it so these tests reach the history checks.
|
||||
await fs.rm(path.join(cwd, ".git", "COMMIT_EDITMSG"), { force: true });
|
||||
return (await git(cwd, "rev-parse", "HEAD")).trim();
|
||||
}
|
||||
@@ -40,7 +38,6 @@ describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
let projectRemote: string;
|
||||
let prove: ManagedCheckoutProver;
|
||||
|
||||
// A bare upstream with one commit, plus the seed it was pushed from.
|
||||
async function upstream(name: string, files: Record<string, string>) {
|
||||
const seed = path.join(root, `${name}-seed`);
|
||||
await fs.mkdir(seed, { recursive: true });
|
||||
@@ -63,7 +60,6 @@ describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
({ seed: projectSeed, bare: projectRemote } = await upstream("project", { "README.md": "project\n", ".gitignore": "node_modules/\n" }));
|
||||
work = path.join(root, "work");
|
||||
await execFile("git", ["clone", "-q", main.bare, work]);
|
||||
// Workspace preparation excludes the managed namespace through info/exclude.
|
||||
await fs.appendFile(path.join(work, ".git", "info", "exclude"), "\n/.paperclip-repositories/\n");
|
||||
prove = (checkout, _name, signal) => assertReproducibleManagedCheckout(checkout, {
|
||||
repoUrl: projectRemote, resolveGitAuth: async () => null, signal,
|
||||
@@ -115,7 +111,6 @@ describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
await write(path.join(checkout, "notes.txt"));
|
||||
await expectRejected("dirty_or_untracked_files");
|
||||
await fs.rm(path.join(checkout, "notes.txt"));
|
||||
// skip-worktree hides this edit from `git status`.
|
||||
await git(checkout, "update-index", "--skip-worktree", "README.md");
|
||||
await write(path.join(checkout, "README.md"), "hidden edit\n");
|
||||
await expectRejected("unsupported_index_flags");
|
||||
@@ -126,17 +121,14 @@ describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
await commit(checkout, "branch.txt", "branch work");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
await git(checkout, "tag", "kept");
|
||||
// Loose tag refs fail the storage check on their own; packed ones reach the history check.
|
||||
await git(checkout, "pack-refs", "--all");
|
||||
await git(checkout, "reset", "-q", "--hard", "origin/main");
|
||||
// ORIG_HEAD alone would already reject; drop it to reach the history check.
|
||||
await fs.rm(path.join(checkout, ".git", "ORIG_HEAD"), { force: true });
|
||||
await git(checkout, "reflog", "expire", "--expire=now", "--all");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
await git(checkout, "tag", "-d", "kept");
|
||||
await commit(checkout, "amended.txt", "amended work");
|
||||
await git(checkout, "reset", "-q", "--hard", "origin/main");
|
||||
// ORIG_HEAD alone would already reject; drop it to reach the history checks.
|
||||
await fs.rm(path.join(checkout, ".git", "ORIG_HEAD"), { force: true });
|
||||
await expectRejected(/unique_reflog_commits|unpushed_managed_checkout/);
|
||||
});
|
||||
@@ -151,7 +143,6 @@ describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
|
||||
it("accepts pushed history after upstream advances", async () => {
|
||||
await managedCheckout();
|
||||
// Upstream moves on after the clone; the checkout never fetches it.
|
||||
await commit(projectSeed, "later.txt", "later upstream work");
|
||||
await git(projectSeed, "push", "-q", "origin", "main");
|
||||
await expectAccepted();
|
||||
@@ -188,7 +179,6 @@ describe("assertReproducibleGitFiles with Paperclip-managed files", () => {
|
||||
await commit(projectSeed, "feature.txt", "feature work");
|
||||
await git(projectSeed, "push", "-q", "origin", "feature");
|
||||
await managedCheckout();
|
||||
// The branch disappears upstream; the checkout's origin/feature is now the last copy.
|
||||
await git(projectRemote, "branch", "-q", "-D", "feature");
|
||||
await expectRejected("unpushed_managed_checkout");
|
||||
});
|
||||
|
||||
@@ -15,9 +15,6 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
// The pass records its candidate count on its timing object, then starts the
|
||||
// per-row loop with phase("candidates"). The mock captures the count and stops
|
||||
// the pass there, so these tests check candidate selection only.
|
||||
const captured = vi.hoisted(() => ({ counts: [] as number[] }));
|
||||
class StopAfterCandidateScan extends Error {}
|
||||
vi.mock("../services/native-runtime/native-reconciliation-timing.js", () => ({
|
||||
@@ -96,9 +93,6 @@ describeEmbeddedPostgres("native finalization candidate selection", () => {
|
||||
finishedAt: finishedAgoMs === null ? null : new Date(Date.now() - finishedAgoMs),
|
||||
contextSnapshot: { issueId },
|
||||
});
|
||||
// Result, assessment and decision rows need full contract chains. These
|
||||
// tests check candidate selection only, so foreign-key triggers are off
|
||||
// for this insert; check constraints still apply.
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`set local session_replication_role = replica`);
|
||||
await tx.insert(nativeRunFinalizations).values({
|
||||
@@ -117,7 +111,6 @@ describeEmbeddedPostgres("native finalization candidate selection", () => {
|
||||
const counts = [];
|
||||
for (let pass = 0; pass < 4; pass++) counts.push(await candidateCount(reconcileNativeFinalizations(db)));
|
||||
|
||||
// Two full upkeep pages, the remainder, then the rotation wraps.
|
||||
expect(counts).toEqual([
|
||||
NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS,
|
||||
NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS,
|
||||
@@ -126,8 +119,6 @@ describeEmbeddedPostgres("native finalization candidate selection", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
// More settled rows than one upkeep page, so the page is full and a
|
||||
// protected row can only be a candidate through its own protection.
|
||||
async function fillUpkeepPage() {
|
||||
for (let index = 0; index < NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS + 2; index++) await committedRun();
|
||||
}
|
||||
@@ -152,7 +143,6 @@ describeEmbeddedPostgres("native finalization candidate selection", () => {
|
||||
for (let index = 0; index < protectedRows; index++) {
|
||||
const decisionId = randomUUID();
|
||||
await committedRun({ decisionId });
|
||||
// As above, the decision row is not created; foreign-key triggers are off.
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`set local session_replication_role = replica`);
|
||||
await tx.insert(statusDecisionEffects).values({
|
||||
|
||||
@@ -13,10 +13,6 @@ export function createAgentAvatarPool(concurrency = 2, maxQueue = 64) {
|
||||
function spawn() {
|
||||
const source = new URL(import.meta.url).pathname.endsWith(".ts");
|
||||
const url = new URL(source ? "./agent-avatar-worker.ts" : "./agent-avatar-worker.js", import.meta.url);
|
||||
// The worker imports workspace packages as TypeScript source. A worker
|
||||
// inherits the tsx loader only when the main thread was started with it;
|
||||
// the production runtime bundle runs without a loader, so load the
|
||||
// worker through tsx's API instead.
|
||||
const loaderInherited = process.execArgv.some((arg) => arg.includes("tsx"));
|
||||
const worker = source || !loaderInherited
|
||||
? new Worker(`import(${JSON.stringify(import.meta.resolve('tsx/esm/api'))}).then(({tsImport}) => tsImport(${JSON.stringify(url.href)}, ${JSON.stringify(import.meta.url)}));`, { eval: true })
|
||||
|
||||
@@ -10,7 +10,6 @@ import type { GitRemoteAuthProvider } from "./git-credentials.js";
|
||||
import { assertReproducibleGitFiles, assertReproducibleBaseGitStorage, assertReproducibleManagedCheckout, inspectGitCacheRecipe, readGitCacheRestoration,
|
||||
verifyRemoteGitCommit, GitCacheRestorationFailure, type ManagedCheckoutProver, gitCacheCommand, readGitCacheExpiration, finishGitCacheTrash, type GitCacheExpiration, GIT_CACHE_RESTORATION_KEY, GIT_CACHE_EXPIRATION_REASON } from "./git-workspace-restoration.js";
|
||||
|
||||
|
||||
async function allocatedBytes(target: string, shouldContinue: () => boolean): Promise<number> {
|
||||
if (!shouldContinue()) throw new Error("scan_stopped");
|
||||
const entry = await fs.lstat(target);
|
||||
@@ -25,13 +24,6 @@ export function gitExpirationSkipReason(error: unknown) {
|
||||
}
|
||||
const skipReason = gitExpirationSkipReason;
|
||||
|
||||
/**
|
||||
* Expiration runs outside any agent run, so a company's managed GitHub
|
||||
* connection has no run identity to resolve, and its provider throws instead
|
||||
* of returning credentials. Upstream proofs then fall back to anonymous
|
||||
* access: a public repository still proves its commits, and a private one
|
||||
* fails to verify, so its workspace is kept, as before.
|
||||
*/
|
||||
export function withAnonymousGitFallback(provider: GitRemoteAuthProvider): GitRemoteAuthProvider {
|
||||
return async (remoteUrl) => provider(remoteUrl).catch(() => null);
|
||||
}
|
||||
@@ -143,12 +135,6 @@ async function inspectBase(db: Db, workspace: Workspace, signal?: AbortSignal, a
|
||||
return allocatedBytes(recipe.sourcePath, () => { signal?.throwIfAborted(); return true; });
|
||||
}
|
||||
|
||||
/**
|
||||
* Accepts a managed project checkout only when its name maps back to one of
|
||||
* the workspace's project repositories (workspace preparation re-clones it
|
||||
* from that row on the next run) and the checkout is proven reproducible
|
||||
* against that row's repository URL.
|
||||
*/
|
||||
function managedCheckoutProver(db: Db, workspace: Workspace, auth: GitRemoteAuthProvider): ManagedCheckoutProver {
|
||||
let rows: Promise<Array<{ id: string; repoUrl: string | null; repoRef: string | null; cwd: string | null }>> | undefined;
|
||||
return async (checkoutPath, name, signal) => {
|
||||
|
||||
@@ -93,9 +93,6 @@ export function readGitCacheRestoration(workspace: Workspace): GitCacheRestorati
|
||||
}
|
||||
|
||||
export async function gitCacheCommand(cwd: string, args: string[], signal?: AbortSignal) {
|
||||
// The inspected repositories are agent-writable. core.fsmonitor in their
|
||||
// local config would make `git status` run an arbitrary command as the
|
||||
// server; the command-line setting takes precedence over it.
|
||||
const result = await execFile("git", ["-c", "core.fsmonitor=false", "-C", cwd, ...args], {
|
||||
timeout: 60_000, signal, maxBuffer: 16 * 1024 * 1024,
|
||||
env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" },
|
||||
@@ -174,28 +171,10 @@ export async function assertReproducibleBaseGitStorage(cwd: string, signal?: Abo
|
||||
for (const directory of ["objects", "refs", "logs"]) await walk(directory);
|
||||
}
|
||||
|
||||
/**
|
||||
* Paperclip writes this file into the worktree on every Claude run (it merges
|
||||
* its permission rules into whatever is there), so a terminal worktree's copy
|
||||
* is regenerated when the task runs again.
|
||||
*/
|
||||
const PAPERCLIP_CLAUDE_SETTINGS_FILE = ".claude/settings.local.json";
|
||||
|
||||
/**
|
||||
* Proves that a managed project checkout (`.paperclip-repositories/<name>`)
|
||||
* holds nothing that re-cloning would not restore; throws otherwise. Only the
|
||||
* expiration worker supplies one, because it can map the checkout back to its
|
||||
* project repository and reach upstream.
|
||||
*/
|
||||
export type ManagedCheckoutProver = (checkoutPath: string, name: string, signal?: AbortSignal) => Promise<void>;
|
||||
|
||||
/**
|
||||
* The proof for one managed checkout, against the repository URL recorded on
|
||||
* its project workspace row (never the checkout's own, agent-writable config):
|
||||
* the checkout's tree, index, config and Git storage hold nothing unique, it
|
||||
* has no linked worktrees, and every commit reachable from its refs or
|
||||
* reflogs is reachable from a current upstream branch or tag.
|
||||
*/
|
||||
export async function assertReproducibleManagedCheckout(checkout: string, input: {
|
||||
repoUrl: string; resolveGitAuth: GitRemoteAuthProvider; signal?: AbortSignal;
|
||||
}) {
|
||||
@@ -208,18 +187,10 @@ export async function assertReproducibleManagedCheckout(checkout: string, input:
|
||||
const worktrees = (await gitCacheCommand(checkout, ["worktree", "list", "--porcelain"], signal))
|
||||
.split("\n").filter(line => line.startsWith("worktree "));
|
||||
if (worktrees.length !== 1) throw new Error("managed_checkout_has_worktrees");
|
||||
// Tree, index flags, local config, ignored files and in-progress state, as
|
||||
// for any worktree; nested managed checkouts are never accepted here.
|
||||
await assertReproducibleGitFiles(checkout, signal);
|
||||
// Hooks, info files and object/ref storage must be stock.
|
||||
await assertReproducibleBaseGitStorage(checkout, signal);
|
||||
const refs = (await gitCacheCommand(checkout, ["for-each-ref", "--format=%(refname)"], signal)).trim().split("\n").filter(Boolean);
|
||||
if (refs.some(ref => !/^refs\/(heads|tags|remotes\/origin)\//.test(ref))) throw new Error("unsupported_managed_checkout_refs");
|
||||
// Every commit a ref or reflog keeps must be on a current upstream branch or
|
||||
// tag. Local remote-tracking refs prove nothing on their own (they may be
|
||||
// stale), so the proof fetches upstream's refs fresh.
|
||||
// HEAD is listed explicitly: a detached HEAD is not a named ref, and once
|
||||
// its reflog expires nothing else would name its commits.
|
||||
const localTips = new Set([
|
||||
...(await gitCacheCommand(checkout, ["rev-parse", "--verify", "HEAD^{commit}"], signal)).split("\n"),
|
||||
...(await gitCacheCommand(checkout, ["for-each-ref", "--format=%(objectname)"], signal)).split("\n"),
|
||||
@@ -229,31 +200,18 @@ export async function assertReproducibleManagedCheckout(checkout: string, input:
|
||||
if (localTips.size && await hasCommitsMissingUpstream(checkout, [...localTips], input.repoUrl, input.resolveGitAuth, signal)) {
|
||||
throw new Error("unpushed_managed_checkout");
|
||||
}
|
||||
// Objects no ref or reflog reaches (a reset `git add`, a dropped stash) may
|
||||
// be the only copy of some work.
|
||||
const unreachable = await gitCacheCommand(checkout, ["fsck", "--connectivity-only", "--unreachable", "--no-progress"], signal);
|
||||
if (unreachable.trim()) throw new Error("unreachable_managed_checkout_objects");
|
||||
}
|
||||
|
||||
/** Upper bound on distinct ref and reflog commits a proof will consider. */
|
||||
const MANAGED_CHECKOUT_LOCAL_TIP_LIMIT = 5_000;
|
||||
|
||||
/**
|
||||
* Fetches upstream's current branches and tags into a throwaway proof store
|
||||
* that borrows the checkout's objects (so only commits the checkout lacks are
|
||||
* downloaded), then asks whether any local commit is unreachable from them.
|
||||
* It is one fetch and one rev-list however many refs either side has. The
|
||||
* fetch runs config-free with the company's Git identity, never the
|
||||
* checkout's own config.
|
||||
*/
|
||||
async function hasCommitsMissingUpstream(checkout: string, localTips: string[], repoUrl: string,
|
||||
resolveGitAuth: GitRemoteAuthProvider, signal?: AbortSignal) {
|
||||
const store = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-git-checkout-proof-"));
|
||||
try {
|
||||
const options = { timeout: 120_000, signal, maxBuffer: 16 * 1024 * 1024, cwd: os.tmpdir(),
|
||||
env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull } };
|
||||
// Local setup stays outside the catch below: a local failure (a full
|
||||
// disk, say) must not be reported as an upstream problem.
|
||||
await execFile("git", ["init", "--bare", "--quiet", store], options);
|
||||
await fs.writeFile(path.join(store, "objects", "info", "alternates"), `${path.join(await fs.realpath(checkout), ".git", "objects")}\n`);
|
||||
try {
|
||||
@@ -263,8 +221,6 @@ async function hasCommitsMissingUpstream(checkout: string, localTips: string[],
|
||||
{ ...options, env: { ...options.env, ...auth?.env } });
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
// An unreachable upstream or refused credentials proves nothing, so the
|
||||
// checkout is kept.
|
||||
throw new Error("managed_checkout_upstream_unavailable");
|
||||
}
|
||||
const missing = await execFile("git", ["-C", store, "rev-list", "--max-count=1", ...localTips, "--not", "--glob=refs/upstream/*"], options);
|
||||
@@ -274,11 +230,6 @@ async function hasCommitsMissingUpstream(checkout: string, localTips: string[],
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect every untracked/ignored file. Only repository-ignored regenerable
|
||||
* directories, Paperclip's Claude settings file, and managed project checkouts
|
||||
* that `proveManagedCheckout` accepts qualify.
|
||||
*/
|
||||
export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSignal, options: {
|
||||
proveManagedCheckout?: ManagedCheckoutProver;
|
||||
} = {}) {
|
||||
@@ -289,10 +240,6 @@ export async function assertReproducibleGitFiles(cwd: string, signal?: AbortSign
|
||||
const regenerable = await findRegenerableWorkspaceDirs(cwd, { signal });
|
||||
const ignored = (await gitCacheCommand(cwd, ["ls-files", "--others", "--ignored", "--exclude-standard", "-z"], signal)).split("\0").filter(Boolean);
|
||||
const unexplained = ignored.filter(file => file !== PAPERCLIP_CLAUDE_SETTINGS_FILE && !regenerable.some(dir => file.startsWith(`${dir}/`)));
|
||||
// Anything else unexplained must sit inside a managed project checkout that
|
||||
// the caller can prove reproducible. Task repositories (`task-repo-*`) carry
|
||||
// access receipts and staging clones (`*.clone-*`) are mid-publication, so
|
||||
// neither ever qualifies.
|
||||
const checkouts = new Set<string>();
|
||||
for (const file of unexplained) {
|
||||
const [namespace, name, rest] = file.split("/", 3);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/** Normalizes a repository URL so `.git` and trailing-slash variants match. */
|
||||
export function projectRepositoryIdentity(url: string) {
|
||||
return url.trim().replace(/\.git\/?$/, "").replace(/\/$/, "");
|
||||
}
|
||||
@@ -23,12 +22,6 @@ export function deriveRepoNameFromRepoUrl(repoUrl: string | null): string | null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Directory name of a project repository's managed checkout under
|
||||
* `.paperclip-repositories/`. It is derived from the project workspace row,
|
||||
* so the same row always maps to the same checkout, and a checkout name maps
|
||||
* back to the row it was cloned from.
|
||||
*/
|
||||
export function managedProjectCheckoutName(workspace: {
|
||||
id: string; repoUrl: string; repoRef: string | null; cwd?: string | null;
|
||||
}) {
|
||||
|
||||
@@ -634,20 +634,11 @@ type NativeReconciliationOptions = {
|
||||
|
||||
type NativeReconciliationPassTiming = ReturnType<typeof createNativeReconciliationPassTiming>;
|
||||
|
||||
// A committed finalization whose run ended this long ago, with no undelivered
|
||||
// decision effect, is settled: its result is projected and its effects are
|
||||
// delivered, so an unscoped pass has nothing to finish. Re-reading every such
|
||||
// row on every pass cost dozens of sequential queries per row, and startup
|
||||
// runs the pass twice before readiness. Recent rows stay in every pass, so the
|
||||
// crash-window repairs still run promptly.
|
||||
export const NATIVE_SETTLED_COMMITTED_GRACE_MS = 60 * 60 * 1000;
|
||||
// Settled rows still get the presentation and evidence upkeep, a few per pass
|
||||
// in a rotating order, so a late repair is delayed, never skipped.
|
||||
export const NATIVE_SETTLED_COMMITTED_UPKEEP_PER_PASS = 10;
|
||||
const NATIVE_SETTLED_RUN_STATUSES = ["succeeded", "failed", "cancelled", "timed_out"];
|
||||
let settledCommittedUpkeepCursor: string | null = null;
|
||||
|
||||
/** Test hook: restart the settled-row upkeep rotation from the beginning. */
|
||||
export function resetSettledCommittedUpkeepForTests() {
|
||||
settledCommittedUpkeepCursor = null;
|
||||
}
|
||||
@@ -668,7 +659,6 @@ function settledCommittedFinalization(now: Date): SQL {
|
||||
)!;
|
||||
}
|
||||
|
||||
/** The next settled committed runs in the upkeep rotation, wrapping at the end. */
|
||||
async function nextSettledCommittedUpkeep(db: Db, now: Date): Promise<string[]> {
|
||||
const page = (after: string | null) => db
|
||||
.select({ runId: heartbeatRuns.id })
|
||||
@@ -732,8 +722,6 @@ async function reconcileNativeFinalizationsPass(
|
||||
}
|
||||
timing.phase("candidate_scan");
|
||||
const now = new Date();
|
||||
// An unscoped pass skips settled committed rows, except the few in this
|
||||
// pass's upkeep rotation. A scoped pass reconciles exactly the given runs.
|
||||
const unscoped = !(runIds && runIds.length > 0);
|
||||
const upkeepRunIds = unscoped ? await nextSettledCommittedUpkeep(db, now) : [];
|
||||
const rows = await db
|
||||
|
||||
Reference in new issue
Block a user