diff --git a/doc/observability.md b/doc/observability.md index 1f64d27e159..ae89537d6a6 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -336,6 +336,9 @@ attempt or owning phase: | `workspace_sync.source_root`, `.repositories`, `.ignore_rules`, `.snapshot_git`, `.snapshot_baseline` | Physical source capture, managed repositories, ignore rules, Git metadata and baseline content snapshot. | | `workspace_sync.pack`, `.git_clone`, `.git_archive`, `.overlay_copy`, `.overlay_archive`, `.deleted_manifest` | Temporary shallow Git clone, its archive callback, selected working-tree overlay copy/archive, and the deleted-file manifest. The archive is inside the clone's lifetime; do not sum parent and child. | | `workspace_sync.seed_generation`, `.seed_lookup`, `.seed_replay`, `.seed_persist`, `.seed_publish`, `.seed_verify`, `.seed_extract`, `.seed_snapshot`, `.seed_cleanup` | Seed identity, cache lookup, replay, durable copies, publication, extraction/re-snapshot verification and temporary cleanup. Cache hits omit cold clone and verification work. | +| `workspace_sync.seed_cache_copy`, `.seed_cache_sync`, `.seed_cache_hash` | Archive linking/copying, filesystem durability, and checksum work inside cache publication. These are children of seed publication, not additional elapsed time to add to that parent. | +| `workspace_sync.seed_cache_capacity`, `.seed_cache_evict`, `.seed_cache_locked_publish` | Quota inventory, eviction planning/application, and the publication body after the company lock is acquired. Capacity may run before and inside the lock. A gap after seed verification and before locked publication can include lock acquisition; locked publication itself excludes that wait. Copy/sync/hash and the locked quota/eviction work are nested children. | +| `workspace_sync.git_index_refresh` | Fully awaited stat-metadata refresh after restoring a fresh OpenCode sandbox checkout. It shares the inbound barrier and command timeout; dirty working-tree content stays unstaged. Recovery and retained sandboxes skip it. | | `workspace_sync.inbound_barrier`, `.stage_workspace`, `.transfer` | The inbound scheduler barrier, whole workspace task, and provider `syncIn` call (including provider guards, upload and post-upload extraction). Parallel tasks overlap; provider transfer/exec children give the finer transport split when available. | | `workspace_sync.progress` | Awaited progress/status reporting, including the existing lifecycle run-log writes. These are not file transfer work. | | `workspace_sync.run_read`, `.descriptor_directory`, `.descriptor_read`, `.archive_hash`, `.warm_stamp`, `.durable_seed_verify`, `.manifests_persist`, `.descriptor_write`, `.reference_write` | Saved run state, recovery descriptors, archive hashing, remote warm-stamp check, manifests and final descriptor/reference persistence. | diff --git a/packages/adapter-utils/src/command-managed-runtime.ts b/packages/adapter-utils/src/command-managed-runtime.ts index 68321689715..ebe858153e6 100644 --- a/packages/adapter-utils/src/command-managed-runtime.ts +++ b/packages/adapter-utils/src/command-managed-runtime.ts @@ -415,6 +415,7 @@ export function createCommandManagedRuntimeClient(input: { args: shellCommandArgs(`rm -rf ${shellQuote(remotePath)}`), cwd: input.commandCwd, timeoutMs: input.timeoutMs, + bypassSession: true, }); requireSuccessfulResult(result, `remove ${remotePath}`); }, @@ -424,6 +425,9 @@ export function createCommandManagedRuntimeClient(input: { args: shellCommandArgs(command), cwd: input.commandCwd, timeoutMs: options.timeoutMs, + // These helpers consume only exit status. They need neither a + // persistent shell nor its separate, streamed output observations. + bypassSession: true, }); requireSuccessfulResult(result, command); }, @@ -546,6 +550,7 @@ export async function prepareCommandManagedRuntime(input: { syncWorkspace?: boolean; workspaceInboundMode?: WorkspaceInboundMode; workspaceSnapshotIsolation?: "worker"; + workspaceGitIndexRefresh?: boolean; workspaceDurableSeed?: WorkspaceDurableSeedPaths; workspaceCheckpoint?: boolean; onWorkspaceCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; @@ -622,6 +627,7 @@ export async function prepareCommandManagedRuntime(input: { syncWorkspace: input.syncWorkspace, workspaceInboundMode: input.workspaceInboundMode, workspaceSnapshotIsolation: input.workspaceSnapshotIsolation, + workspaceGitIndexRefresh: input.workspaceGitIndexRefresh, workspaceDurableSeed: input.workspaceDurableSeed, workspaceCheckpoint: input.workspaceCheckpoint, onWorkspaceCheckpoint: input.onWorkspaceCheckpoint, @@ -675,6 +681,7 @@ export async function prepareCommandManagedRuntime(input: { syncWorkspace: input.syncWorkspace, workspaceInboundMode: input.workspaceInboundMode, workspaceSnapshotIsolation: input.workspaceSnapshotIsolation, + workspaceGitIndexRefresh: input.workspaceGitIndexRefresh, workspaceDurableSeed: input.workspaceDurableSeed, workspaceCheckpoint: input.workspaceCheckpoint, onWorkspaceCheckpoint: input.onWorkspaceCheckpoint, diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index d166ed95ce7..da99bc82917 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -1446,6 +1446,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { syncWorkspace?: boolean; workspaceInboundMode?: WorkspaceInboundMode; workspaceSnapshotIsolation?: "worker"; + workspaceGitIndexRefresh?: boolean; workspaceDurableSeed?: WorkspaceDurableSeedPaths; workspaceCheckpoint?: boolean; onWorkspaceCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; @@ -1542,6 +1543,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: { syncWorkspace: input.syncWorkspace, workspaceInboundMode: input.workspaceInboundMode, workspaceSnapshotIsolation: input.workspaceSnapshotIsolation, + workspaceGitIndexRefresh: input.workspaceGitIndexRefresh, workspaceDurableSeed: input.workspaceDurableSeed, workspaceCheckpoint: input.workspaceCheckpoint, onWorkspaceCheckpoint: input.onWorkspaceCheckpoint, diff --git a/packages/adapter-utils/src/git-workspace-sync.ts b/packages/adapter-utils/src/git-workspace-sync.ts index d49486aaebc..278a9642131 100644 --- a/packages/adapter-utils/src/git-workspace-sync.ts +++ b/packages/adapter-utils/src/git-workspace-sync.ts @@ -95,6 +95,12 @@ function shellQuote(value: string) { return `'${value.replace(/'/g, `'\"'\"'`)}'`; } +/** Refresh restored stat metadata without staging dirty working-tree content. */ +export function buildRefreshWorkspaceGitIndexCommand(directory: string): string { + return `if git -C ${shellQuote(directory)} -c core.fsmonitor=false update-index -q --refresh >/dev/null; then :; ` + + 'else index_status=$?; if [ "$index_status" -ne 1 ]; then exit "$index_status"; fi; fi'; +} + export async function runLocalGit( localDir: string, args: string[], diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.ts index 50a74d1baed..5f9bc27f50b 100644 --- a/packages/adapter-utils/src/sandbox-managed-runtime.ts +++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts @@ -20,6 +20,7 @@ import { deleteLocalGitRef, fetchGitBundleIntoLocalRef, GIT_ARCHIVE_EXCLUDES, + buildRefreshWorkspaceGitIndexCommand, type GitWorkspaceSnapshot, integrateImportedGitHead, readGitWorkspaceSnapshot, @@ -1256,6 +1257,8 @@ export async function prepareSandboxManagedRuntime(input: { /** Selects authoritative host staging, exact durable-seed replay, or no-overwrite adoption. */ workspaceInboundMode?: WorkspaceInboundMode; workspaceSnapshotIsolation?: "worker"; + /** Fully await a restored Git index refresh inside the existing inbound barrier. */ + workspaceGitIndexRefresh?: boolean; workspaceDurableSeed?: WorkspaceDurableSeedPaths; workspaceCheckpoint?: boolean; onWorkspaceCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; @@ -1453,10 +1456,6 @@ export async function prepareSandboxManagedRuntime(input: { const seedGeneration = input.workspaceSeedCacheDirectory && baselineSnapshot && workspaceInboundMode !== "durable_seed" ? await runStepSpan("workspace.seed_generation", async () => workspaceSeedGeneration(baselineSnapshot!, gitSnapshot, repositories, input.workspaceSeedCacheScope)) : null; - const cachedSeed = seedGeneration - ? await runStepSpan("workspace.seed_lookup", () => readWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, - { companyDirectory: input.workspaceSeedCacheCompanyDirectory })) : null; - const replaySeed = workspaceInboundMode === "durable_seed" ? input.workspaceDurableSeed : cachedSeed; const sharedSeed = input.workspaceSeedCacheScope === "company" && seedGeneration !== null; // Every inbound staging step delegates to the provider through `client.syncIn`: @@ -1512,6 +1511,10 @@ export async function prepareSandboxManagedRuntime(input: { })); await withTempDir("paperclip-sandbox-sync-", async (tempDir) => { + const cachedSeed = seedGeneration + ? await runStepSpan("workspace.seed_lookup", () => readWorkspaceSeedGeneration(input.workspaceSeedCacheDirectory!, seedGeneration, + { companyDirectory: input.workspaceSeedCacheCompanyDirectory, privateCopyDirectory: tempDir })) : null; + const replaySeed = workspaceInboundMode === "durable_seed" ? input.workspaceDurableSeed : cachedSeed; const preservedNames = new Set([ ".paperclip-runtime", ...(gitSnapshot ? [".git"] : []), @@ -1594,7 +1597,7 @@ export async function prepareSandboxManagedRuntime(input: { const workspaceFiles: SandboxSyncFileMapping[] = []; const workspacePostUploadCommands: SandboxPostUploadCommand[] = []; let workspaceUploadBytes = 0; - const gitSeedTarPath = path.join(tempDir, sharedSeed ? "git-seed.tar" : "git-workspace.tar"); + const gitSeedTarPath = cachedSeed?.gitArchivePath ?? path.join(tempDir, sharedSeed ? "git-seed.tar" : "git-workspace.tar"); // Build both host tarballs (the git-history tar and the workspace-overlay // tar) inside one host span named `pack`. This span makes the host pack @@ -1610,7 +1613,7 @@ export async function prepareSandboxManagedRuntime(input: { // which sits under `.paperclip-runtime`, survives to run its own extract. if (gitSnapshot || repositories.length) { await reportWorkspaceStatus("git_sync", "Syncing git history to environment"); - const gitTarPath = path.join(tempDir, "git-workspace.tar"); + const gitTarPath = sharedSeed ? path.join(tempDir, "git-workspace.tar") : gitSeedTarPath; const remoteGitTar = path.posix.join( runtimeRootDir, "git-workspace-upload.tar", @@ -1620,7 +1623,7 @@ export async function prepareSandboxManagedRuntime(input: { if (!durableGitArchive) { throw new Error("workspace_durable_seed_git_missing"); } - await runStepSpan("workspace.seed_replay", () => copyDurableSeedArchive({ + if (!cachedSeed) await runStepSpan("workspace.seed_replay", () => copyDurableSeedArchive({ sourcePath: durableGitArchive, targetPath: gitSeedTarPath, expectedSha256: replaySeed.gitArchiveSha256, @@ -1664,9 +1667,9 @@ export async function prepareSandboxManagedRuntime(input: { // extracted git tree (no wipe); a plain workspace wipes every child except // the preserved names first. The extract runs AFTER the git extract. await reportWorkspaceStatus("config_sync", "Syncing workspace to environment"); - const workspaceTarPath = path.join(tempDir, "workspace.tar"); + const workspaceTarPath = cachedSeed?.workspaceArchivePath ?? path.join(tempDir, "workspace.tar"); if (replaySeed) { - await runStepSpan("workspace.seed_replay", () => copyDurableSeedArchive({ + if (!cachedSeed) await runStepSpan("workspace.seed_replay", () => copyDurableSeedArchive({ sourcePath: replaySeed.workspaceArchivePath, targetPath: workspaceTarPath, expectedSha256: @@ -1737,6 +1740,7 @@ export async function prepareSandboxManagedRuntime(input: { { ...seed, gitArchivePath }, { companyDirectory: input.workspaceSeedCacheCompanyDirectory, scope: input.workspaceSeedCacheScope, + runtimeSpan: runStepSpan, verify: () => runStepSpan("seed.verify", async () => { const verifiedRoot = path.join(tempDir, "verify-seed"); try { @@ -1775,6 +1779,13 @@ export async function prepareSandboxManagedRuntime(input: { progressLabel: "workspace", statusPhase: "config_sync", progressBytes: workspaceUploadBytes, + }).then(async () => { + if (input.workspaceGitIndexRefresh && gitSnapshot && workspaceInboundMode === "host_current") { + await runStepSpan("workspace.git_index_refresh", () => input.client.run( + buildRefreshWorkspaceGitIndexCommand(workspaceRemoteDir), + { timeoutMs: input.spec.timeoutMs }, + )); + } }); const [staged] = await Promise.allSettled([staging, publishSeed()]); if (staged.status === "rejected") throw staged.reason; diff --git a/packages/adapter-utils/src/workspace-seed-cache.test.ts b/packages/adapter-utils/src/workspace-seed-cache.test.ts index 4830605d9f3..6f85a587829 100644 --- a/packages/adapter-utils/src/workspace-seed-cache.test.ts +++ b/packages/adapter-utils/src/workspace-seed-cache.test.ts @@ -26,6 +26,45 @@ async function fixture() { } const generation = (index: number) => index.toString(16).padStart(64, "0"); +it("returns verified private copies that survive subsequent cache changes", async () => { + const f = await fixture(); + const git = path.join(f.root, "git.tar"); await fs.writeFile(git, "git recovery bytes"); + expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), { ...f.seed, gitArchivePath: git })).toBe(true); + const destination = path.join(f.root, "staging"); await fs.mkdir(destination); + const copy = await readWorkspaceSeedGeneration(f.workspace, generation(1), { privateCopyDirectory: destination }); + expect(copy).not.toBeNull(); + expect(path.dirname(copy!.workspaceArchivePath)).toBe(path.dirname(copy!.gitArchivePath!)); + expect(path.dirname(path.dirname(copy!.workspaceArchivePath))).toBe(destination); + await fs.writeFile(path.join(f.workspace, generation(1), "workspace.tar"), "changed shared cache"); + await fs.rm(path.join(f.workspace, generation(1), "git.tar")); + expect(await fs.readFile(copy!.workspaceArchivePath, "utf8")).toBe("run recovery bytes"); + expect(await fs.readFile(copy!.gitArchivePath!, "utf8")).toBe("git recovery bytes"); +}); + +it("rejects corrupted private bytes and removes incomplete materialization", async () => { + const f = await fixture(); + expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), f.seed)).toBe(true); + const destination = path.join(f.root, "staging"); await fs.mkdir(destination); + const copyFile = fs.copyFile.bind(fs); + vi.spyOn(fs, "copyFile").mockImplementationOnce(async (...args) => { + await copyFile(...args); + await fs.writeFile(args[1], "corrupted during copy"); + }); + expect(await readWorkspaceSeedGeneration(f.workspace, generation(1), { privateCopyDirectory: destination })).toBeNull(); + expect(await fs.readdir(destination)).toEqual([]); + expect(await fs.readFile(path.join(f.workspace, generation(1), "workspace.tar"), "utf8")).toBe("run recovery bytes"); +}); + +it("cleans a verified workspace copy if the cached Git archive is corrupt", async () => { + const f = await fixture(); + const git = path.join(f.root, "git.tar"); await fs.writeFile(git, "git recovery bytes"); + expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), { ...f.seed, gitArchivePath: git })).toBe(true); + await fs.writeFile(path.join(f.workspace, generation(1), "git.tar"), "corrupt"); + const destination = path.join(f.root, "staging"); await fs.mkdir(destination); + expect(await readWorkspaceSeedGeneration(f.workspace, generation(1), { privateCopyDirectory: destination })).toBeNull(); + expect(await fs.readdir(destination)).toEqual([]); +}); + it("stops new workspace generations at capacity while retaining reusable and per-run seeds", async () => { const f = await fixture(); const options = { companyDirectory: f.company, limits: { workspaceGenerations: 1 } }; expect(await publishWorkspaceSeedGeneration(f.workspace, generation(1), f.seed, options)).toBe(true); @@ -56,6 +95,50 @@ it.each(["companyGenerations", "companyWorkspaces"] as const)("serializes distin expect(await fs.readFile(f.archive, "utf8")).toBe("run recovery bytes"); }); +it("keeps company-lock contention outside the locked-publication span", async () => { + const f = await fixture(); + const entered = deferred(), release = deferred(), verified = deferred(); + const spans: string[][] = [[], []]; + const runtimeSpan = (index: number) => async (name: string, work: () => Promise): Promise => { + spans[index].push(name); + if (index === 0 && name === "workspace.seed_cache_locked_publish") { + entered.resolve(); + await release.promise; + } + return work(); + }; + const first = publishWorkspaceSeedGeneration(path.join(f.company, "first"), generation(1), f.seed, { + companyDirectory: f.company, runtimeSpan: runtimeSpan(0), + }); + let second: Promise | undefined; + try { + await entered.promise; + second = publishWorkspaceSeedGeneration(path.join(f.company, "second"), generation(2), f.seed, { + companyDirectory: f.company, runtimeSpan: runtimeSpan(1), + verify: async () => { verified.resolve(); return true; }, + }); + await verified.promise; + await new Promise(resolve => setImmediate(resolve)); + expect(spans[1]).toContain("workspace.seed_cache_capacity"); + expect(spans[1]).not.toContain("workspace.seed_cache_locked_publish"); + } finally { + release.resolve(); + await Promise.all([first, second]); + } + expect(await first).toBe(true); + expect(await second).toBe(true); + for (const names of spans) { + expect(names).toContain("workspace.seed_cache_locked_publish"); + expect(names).toContain("workspace.seed_cache_evict"); + expect(names).toContain("workspace.seed_cache_sync"); + expect(names).toContain("workspace.seed_cache_hash"); + } + for (const [name, index] of [["first", 1], ["second", 2]] as const) { + const cached = await readWorkspaceSeedGeneration(path.join(f.company, name), generation(index), { companyDirectory: f.company }); + expect(await fs.readFile(cached!.workspaceArchivePath, "utf8")).toBe("run recovery bytes"); + } +}); + it("applies the company byte cap across workspace roots", async () => { const f = await fixture(); const options = { companyDirectory: f.company, limits: { companyBytes: 4500 } }; diff --git a/packages/adapter-utils/src/workspace-seed-cache.ts b/packages/adapter-utils/src/workspace-seed-cache.ts index c5c9ce02b3d..880ee8ca39e 100644 --- a/packages/adapter-utils/src/workspace-seed-cache.ts +++ b/packages/adapter-utils/src/workspace-seed-cache.ts @@ -1,11 +1,12 @@ import fs from "node:fs/promises"; import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; -import { createReadStream } from "node:fs"; +import { constants, createReadStream } from "node:fs"; import { directorySnapshotSha256, withDirectoryMergeLock, type DirectorySnapshot } from "./workspace-restore-merge.js"; import { assertWorkspaceManifestDiskSpace } from "./workspace-manifest.js"; import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js"; import type { WorkspaceDurableSeedPaths } from "./sandbox-managed-runtime.js"; +import type { RuntimeSpanRunner } from "./acpx-engine/startup-timing.js"; export function workspaceSeedGeneration(baseline: DirectorySnapshot, git: GitWorkspaceSnapshot | null, repositories: NonNullable = [], scope: "workspace" | "company" = "workspace"): string { const identity = (snapshot: GitWorkspaceSnapshot | null): unknown => snapshot && ({ @@ -26,9 +27,12 @@ async function digest(file: string): Promise { return hash.digest("hex"); } +/** A privateCopyDirectory must be caller-owned staging; its caller owns cleanup. */ export async function readWorkspaceSeedGeneration(root: string, generation: string, - options: { companyDirectory?: string } = {}): Promise { + options: { companyDirectory?: string; privateCopyDirectory?: string } = {}): Promise { const directory = path.join(root, generation); + let privateCopy: string | null = null; + let accepted = false; try { for (const item of [root, directory]) { const stat = await fs.lstat(item); if (!stat.isDirectory() || stat.isSymbolicLink()) return null; } const receipt = path.join(directory, "receipt.json"); @@ -37,12 +41,23 @@ export async function readWorkspaceSeedGeneration(root: string, generation: stri const metadata = JSON.parse(await fs.readFile(receipt, "utf8")); if (!/^[a-f0-9]{64}$/.test(metadata.workspaceSha256) || (metadata.gitSha256 !== null && !/^[a-f0-9]{64}$/.test(metadata.gitSha256))) return null; if (metadata.version !== 1 || metadata.generation !== generation) return null; - const workspaceArchivePath = path.join(directory, "workspace.tar"); - const gitArchivePath = metadata.gitSha256 ? path.join(directory, "git.tar") : null; + if (options.privateCopyDirectory) { + const stat = await fs.lstat(options.privateCopyDirectory); + if (!stat.isDirectory() || stat.isSymbolicLink()) return null; + privateCopy = await fs.mkdtemp(path.join(options.privateCopyDirectory, "verified-seed-")); + } + const archiveDirectory = privateCopy ?? directory; + const workspaceArchivePath = path.join(archiveDirectory, "workspace.tar"); + const gitArchivePath = metadata.gitSha256 ? path.join(archiveDirectory, "git.tar") : null; for (const [file, expected] of [[workspaceArchivePath, metadata.workspaceSha256], [gitArchivePath, metadata.gitSha256]]) { if (!file) continue; - const stat = await fs.lstat(file); - if (!stat.isFile() || stat.isSymbolicLink() || await digest(file) !== expected) return null; + const source = path.join(directory, path.basename(file)); + const stat = await fs.lstat(source); + if (!stat.isFile() || stat.isSymbolicLink()) return null; + // Hash the exact private bytes staging will consume. This avoids a + // second source hash and remains valid if the shared cache changes later. + if (privateCopy) await fs.copyFile(source, file, constants.COPYFILE_FICLONE | constants.COPYFILE_EXCL); + if (await digest(file) !== expected) return null; } const used = await withDirectoryMergeLock(path.resolve(options.companyDirectory ?? root), async () => { const current = await fs.lstat(receipt).catch(() => null); @@ -52,9 +67,11 @@ export async function readWorkspaceSeedGeneration(root: string, generation: stri return true; }); if (!used) return null; + accepted = true; return { workspaceArchivePath, workspaceArchiveSha256: metadata.workspaceSha256, gitArchivePath, gitArchiveSha256: metadata.gitSha256 }; } catch { return null; } + finally { if (!accepted && privateCopy) await fs.rm(privateCopy, { recursive: true, force: true }).catch(() => undefined); } } export const WORKSPACE_SEED_CACHE_LIMITS = Object.freeze({ workspaceGenerations: 16, @@ -167,8 +184,11 @@ async function cacheUsage(root: string, limits: SeedCacheLimits): Promise<{ gene export async function publishWorkspaceSeedGeneration( root: string, generation: string, seed: WorkspaceDurableSeedPaths, options: { companyDirectory?: string; limits?: Partial; eviction?: Partial; - scope?: "workspace" | "company"; now?: () => number; verify?: () => Promise } = {}, + scope?: "workspace" | "company"; now?: () => number; verify?: () => Promise; + runtimeSpan?: RuntimeSpanRunner } = {}, ): Promise { + const measure = (name: string, work: () => Promise): Promise => + options.runtimeSpan ? options.runtimeSpan(name, work) : work(); const configuredLimits = { ...WORKSPACE_SEED_CACHE_LIMITS, ...options.limits }; const limits = options.scope === "company" ? { ...configuredLimits, workspaceGenerations: configuredLimits.companyGenerations, workspaceBytes: configuredLimits.companyBytes, @@ -217,20 +237,20 @@ export async function publishWorkspaceSeedGeneration( assertWorkspaceManifestDiskSpace(company, incomingBytes); return "available"; }; - const initialCapacity = await capacity(); + const initialCapacity = await measure("workspace.seed_cache_capacity", capacity); if (initialCapacity === "existing") return true; let incomingBytes = 4096; for (const archive of [seed.workspaceArchivePath, seed.gitArchivePath]) if (archive) incomingBytes += (await fs.lstat(archive)).size; if (incomingBytes > configuredLimits.workspaceBytes) return false; const plan = { generation, incomingBytes, limits, eviction, now: options.now?.() ?? Date.now() }; - if (initialCapacity === false && !await evictSeedGenerations(company, root, { ...plan, apply: false })) return false; + if (initialCapacity === false && !await measure("workspace.seed_cache_evict", () => evictSeedGenerations(company, root, { ...plan, apply: false }))) return false; if (options.verify && !await options.verify()) return false; - return await withDirectoryMergeLock(company, async () => { + return await withDirectoryMergeLock(company, () => measure("workspace.seed_cache_locked_publish", async () => { const lockedCompany = await fs.lstat(company); if (!lockedCompany.isDirectory() || lockedCompany.isSymbolicLink() || lockedCompany.dev !== companyStat.dev || lockedCompany.ino !== companyStat.ino) return false; - await evictSeedGenerations(company, root, { ...plan, apply: true }); - const currentCapacity = await capacity(); + await measure("workspace.seed_cache_evict", () => evictSeedGenerations(company, root, { ...plan, apply: true })); + const currentCapacity = await measure("workspace.seed_cache_capacity", capacity); if (currentCapacity !== "available") return currentCapacity === "existing"; await fs.mkdir(root, { recursive: true, mode: 0o700 }); const temporary = path.join(root, `.pending-${randomUUID()}`); @@ -238,10 +258,12 @@ export async function publishWorkspaceSeedGeneration( try { const copy = async (source: string, name: string) => { const target = path.join(temporary, name); - await fs.link(source, target).catch(() => fs.copyFile(source, target)); - const file = await fs.open(target, "r"); - try { await file.sync(); } finally { await file.close(); } - return digest(target); + await measure("workspace.seed_cache_copy", () => fs.link(source, target).catch(() => fs.copyFile(source, target))); + await measure("workspace.seed_cache_sync", async () => { + const file = await fs.open(target, "r"); + try { await file.sync(); } finally { await file.close(); } + }); + return measure("workspace.seed_cache_hash", () => digest(target)); }; const workspaceSha256 = await copy(seed.workspaceArchivePath, "workspace.tar"); const gitSha256 = seed.gitArchivePath ? await copy(seed.gitArchivePath, "git.tar") : null; @@ -249,6 +271,6 @@ export async function publishWorkspaceSeedGeneration( await fs.rename(temporary, path.join(root, generation)); return true; } finally { await fs.rm(temporary, { recursive: true, force: true }); } - }); + })); } catch { return false; } } diff --git a/packages/adapter-utils/src/workspace-seed-reuse.test.ts b/packages/adapter-utils/src/workspace-seed-reuse.test.ts index ad9def2e1ca..1d75b0fb822 100644 --- a/packages/adapter-utils/src/workspace-seed-reuse.test.ts +++ b/packages/adapter-utils/src/workspace-seed-reuse.test.ts @@ -52,7 +52,7 @@ it.each([ { nested: false, secondBranch: null }, { nested: true, secondBranch: null }, ])("reuses company seed content and restores branch identity ($nested, $secondBranch)", async ({ nested, secondBranch }) => { - const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-shared-seed-")); + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-shared seed-'")); roots.push(root); vi.stubEnv("PAPERCLIP_HOME", path.join(root, "home")); const source = path.join(root, "source"); @@ -99,6 +99,7 @@ it.each([ const input: Parameters[0] = { spec: { transport: "sandbox", provider: "filesystem", sandboxId: name, remoteCwd: remote, timeoutMs: 30_000, apiKey: null }, adapterKey: "test", client: client(), workspaceLocalDir: host, workspaceCheckpoint: true, + workspaceGitIndexRefresh: true, workspaceSeedCacheScope: "company", workspaceSeedCacheDirectory: path.join(companyDirectory, "shared-v1"), workspaceSeedCacheCompanyDirectory: companyDirectory, workspaceDurableSeed: { workspaceArchivePath: path.join(durable, "workspace.tar"), gitArchivePath: path.join(durable, "git.tar") }, @@ -128,12 +129,15 @@ it.each([ expect(await git(cwd, "rev-parse", "--abbrev-ref", "HEAD")).toBe(secondBranch ?? "HEAD"); expect(await git(cwd, "branch", "--list", "issue/first")).toBe(""); expect(await fs.readFile(path.join(cwd, "tracked.txt"), "utf8")).toBe("working-tree bytes\n"); + expect(await git(cwd, "diff", "--cached", "--exit-code")).toBe(""); expect(await fs.readFile(path.join(cwd, "new.txt"), "utf8")).toBe("untracked overlay\n"); expect(await fs.readlink(path.join(cwd, "link.txt"))).toBe("tracked.txt"); await expect(fs.stat(path.join(cwd, "deleted.txt"))).rejects.toMatchObject({ code: "ENOENT" }); await expect(fs.stat(path.join(cwd, "secret.txt"))).rejects.toMatchObject({ code: "ENOENT" }); } expect(first.phases).toContain("workspace.git_clone"); + expect(first.phases).toContain("workspace.git_index_refresh"); + expect(second.phases).toContain("workspace.git_index_refresh"); expect(second.phases).not.toContain("workspace.git_clone"); expect(second.phases).not.toContain("workspace.seed_publish"); const generations = await fs.readdir(path.join(company, "shared-v1")); @@ -142,6 +146,7 @@ it.each([ expect(current).toHaveLength(1); expect(await readWorkspaceSeedGeneration(path.join(company, "shared-v1"), current[0]!, { companyDirectory: company })).not.toBeNull(); const recovery = path.join(root, "replacement"); + const refreshCount = second.phases.filter(phase => phase === "workspace.git_index_refresh").length; const recovered = await prepareSandboxManagedRuntime({ ...second.input, spec: { ...second.input.spec, remoteCwd: recovery }, @@ -150,6 +155,7 @@ it.each([ workspaceBaseline: second.prepared.workspaceSyncSnapshot!.baseline, workspaceGitSnapshot: second.prepared.workspaceSyncSnapshot!.gitSnapshot, }); + expect(second.phases.filter(phase => phase === "workspace.git_index_refresh")).toHaveLength(refreshCount); second.remote = recovery; for (const relative of repositories) { const cwd = path.join(second.remote, relative); @@ -170,6 +176,17 @@ it.each([ const isolated = await prepare("other-company", secondBranch, path.join(root, "other-company-cache")); expect(isolated.phases).toContain("workspace.git_clone"); await isolated.prepared.cleanupWorkspaceSnapshot?.(); + await fs.writeFile(path.join(company, "shared-v1", current[0]!, "git.tar"), "corrupted cache"); + const fallback = await prepare("corrupted-cache", secondBranch); + expect(fallback.phases).toContain("workspace.git_clone"); + for (const relative of repositories) { + const cwd = path.join(fallback.remote, relative); + expect(await git(cwd, "rev-parse", "--abbrev-ref", "HEAD")).toBe(secondBranch ?? "HEAD"); + expect(await fs.readFile(path.join(cwd, "tracked.txt"), "utf8")).toBe("working-tree bytes\n"); + expect(await fs.readFile(path.join(cwd, "new.txt"), "utf8")).toBe("untracked overlay\n"); + await expect(fs.stat(path.join(cwd, "secret.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + } + await fallback.prepared.cleanupWorkspaceSnapshot?.(); }, 30_000); it("separates shared seeds when file contents, Git head, origin or repository layout differ", async () => { @@ -191,3 +208,60 @@ it("separates shared seeds when file contents, Git head, origin or repository la expect(workspaceSeedGeneration(second, git, [], "company")).not.toBe(generation); await disposeDirectorySnapshot(first); await disposeDirectorySnapshot(second); }); + +it.each([false, true])("awaits restored-index work before inbound preparation completes (failure=%s)", async failure => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-index-barrier-")); + roots.push(root); + vi.stubEnv("PAPERCLIP_HOME", path.join(root, "home")); + const host = path.join(root, "host"), remote = path.join(root, "remote"); + await fs.mkdir(host); + await git(host, "init", "-q", "-b", "main"); + await fs.writeFile(path.join(host, "tracked.txt"), "committed\n"); + await git(host, "add", "."); + await git(host, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "Seed"); + await fs.writeFile(path.join(host, "tracked.txt"), "dirty working tree\n"); + let release!: () => void, enter!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { enter = resolve; }); + let refreshing = false, completed = false; + const refreshTimeouts: Array = []; + const transport = client(); + const preparing = prepareAdapterExecutionTargetRuntime({ + runId: "index-barrier", adapterKey: "test", workspaceLocalDir: host, workspaceGitIndexRefresh: true, + target: { kind: "remote", transport: "sandbox", providerKey: "filesystem", remoteCwd: remote, timeoutMs: 45_000, + runner: { syncIn: operations => transport.syncIn!(operations), syncOut: operations => transport.syncOut!(operations), + execute: command => { + if (refreshing) refreshTimeouts.push(command.timeoutMs); + return runChildProcess("index-barrier", refreshing && failure ? "sh" : command.command, + refreshing && failure ? ["-c", "exit 7"] : command.args ?? [], { + cwd: command.cwd ?? root, env: command.env ?? {}, timeoutSec: (command.timeoutMs ?? 45_000) / 1000, + graceSec: 1, onLog: async () => {}, + }); + }, + }, + }, + runtimeSpan: async (name, work) => { + if (name !== "workspace.git_index_refresh") return work(); + enter(); await blocked; refreshing = true; + try { return await work(); } finally { refreshing = false; } + }, + }); + void preparing.then(() => { completed = true; }, () => {}); + try { + await Promise.race([entered, preparing.then(() => { throw new Error("Preparation bypassed the index barrier"); })]); + expect(await fs.readFile(path.join(remote, "tracked.txt"), "utf8")).toBe("dirty working tree\n"); + expect(completed).toBe(false); + release(); + if (failure) await expect(preparing).rejects.toThrow(); + else { + const prepared = await preparing; + expect(await git(remote, "diff", "--cached", "--exit-code")).toBe(""); + expect(await git(remote, "diff", "--name-only")).toBe("tracked.txt"); + await prepared.cleanupWorkspaceSnapshot?.(); + } + expect(refreshTimeouts).toEqual([45_000]); + } finally { + release(); + await preparing.catch(() => {}); + } +}); diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts index 166d62fad5e..86c257dd68e 100644 --- a/server/src/services/execution-preparation-timing.ts +++ b/server/src/services/execution-preparation-timing.ts @@ -82,7 +82,10 @@ export type ExecutionPhase = | "workspace_sync.manifests_persist" | "workspace_sync.descriptor_write" | "workspace_sync.reference_write" | "workspace_sync.source_root" | "workspace_sync.repositories" | "workspace_sync.ignore_rules" | "workspace_sync.seed_generation" | "workspace_sync.seed_lookup" | "workspace_sync.seed_publish" + | "workspace_sync.seed_cache_copy" | "workspace_sync.seed_cache_sync" | "workspace_sync.seed_cache_hash" + | "workspace_sync.seed_cache_capacity" | "workspace_sync.seed_cache_evict" | "workspace_sync.seed_cache_locked_publish" | "workspace_sync.git_clone" | "workspace_sync.git_archive" | "workspace_sync.seed_replay" | "workspace_sync.seed_persist" + | "workspace_sync.git_index_refresh" | "workspace_sync.overlay_copy" | "workspace_sync.overlay_archive" | "workspace_sync.deleted_manifest" | "workspace_sync.seed_extract" | "workspace_sync.seed_snapshot" | "workspace_sync.seed_cleanup" | "workspace_sync.inbound_barrier" | "workspace_sync.transfer" | "workspace_sync.progress" @@ -247,8 +250,15 @@ const workspaceSyncDetailPhases = { "workspace.seed_generation": "workspace_sync.seed_generation", "workspace.seed_lookup": "workspace_sync.seed_lookup", "workspace.seed_publish": "workspace_sync.seed_publish", + "workspace.seed_cache_copy": "workspace_sync.seed_cache_copy", + "workspace.seed_cache_sync": "workspace_sync.seed_cache_sync", + "workspace.seed_cache_hash": "workspace_sync.seed_cache_hash", + "workspace.seed_cache_capacity": "workspace_sync.seed_cache_capacity", + "workspace.seed_cache_evict": "workspace_sync.seed_cache_evict", + "workspace.seed_cache_locked_publish": "workspace_sync.seed_cache_locked_publish", "workspace.git_clone": "workspace_sync.git_clone", "workspace.git_archive": "workspace_sync.git_archive", + "workspace.git_index_refresh": "workspace_sync.git_index_refresh", "workspace.seed_replay": "workspace_sync.seed_replay", "workspace.seed_persist": "workspace_sync.seed_persist", "workspace.overlay_copy": "workspace_sync.overlay_copy", diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 667aa26d4df..f5909a970c5 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -1055,6 +1055,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele restartRecovery: runOptions.nativeRestartRecovery, sameRunRecovery: Boolean(runOptions.nativeLeaseOwner), resourceDisposition: providerResourceDispositionForRun, + workspaceGitIndexRefresh: nativeExecution?.provider.kind === "opencode", onCheckpoint: input.workspace.onCheckpoint, onPhase: input.workspace.onPhase, onEvent: onAdapterEvent, diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts index 4b5080b51cf..aa5a0c76e2a 100644 --- a/server/src/services/native-runtime/native-workspace-sync.ts +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -813,6 +813,7 @@ async function prepareRuntime(input: { durableSeed?: WorkspaceDurableSeedPaths; seedCacheDirectory?: string; seedCacheCompanyDirectory?: string; + workspaceGitIndexRefresh?: boolean; onPhase?: RuntimeSpanRunner; onCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; }): Promise { @@ -829,6 +830,7 @@ async function prepareRuntime(input: { workspaceRemoteDir: input.target.remoteCwd, workspaceInboundMode: input.mode, workspaceSnapshotIsolation: "worker", + workspaceGitIndexRefresh: input.workspaceGitIndexRefresh, workspaceDurableSeed: input.durableSeed, workspaceBaseline: input.baseline, workspaceGitSnapshot: input.gitSnapshot, @@ -901,6 +903,7 @@ export async function prepareNativeWorkspaceSync(input: { restartRecovery?: NativeRestartRecoveryClaim; sameRunRecovery?: boolean; resourceDisposition?: NativeWorkspaceResourceDisposition; + workspaceGitIndexRefresh?: boolean; onPhase?: RuntimeSpanRunner; onCheckpoint?: (metrics: WorkspaceCheckpointMetrics) => Promise; onEvent?: (event: AdapterRuntimeEvent) => Promise; @@ -1017,6 +1020,9 @@ export async function prepareNativeWorkspaceSync(input: { mode, seedCacheCompanyDirectory, seedCacheDirectory, + workspaceGitIndexRefresh: input.workspaceGitIndexRefresh === true + && acquisition === "created" && mode === "host_current" + && !input.restartRecovery && !input.sameRunRecovery, durableSeed: { workspaceArchivePath: seedPaths.workspaceArchivePath, gitArchivePath: seedPaths.gitArchivePath,