PaperClipAI/server/src/services/workspace-git-operation-scheduler.ts

import { spawn, type ChildProcess } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import { setExpensiveWorkspaceGitExecutor } from "@paperclipai/adapter-utils/git-workspace-sync";
import { HttpError } from "../errors.js";
import { logger } from "../middleware/logger.js";

export const WORKSPACE_GIT_SCAN_ERROR_CODES = {
  saturated: "workspace_git_scan_saturated",
  timeout: "workspace_git_scan_timeout",
  cancelled: "workspace_git_scan_cancelled",
  outputLimit: "workspace_git_scan_output_limit",
  failed: "workspace_git_scan_failed",
} as const;

export type WorkspaceGitScanErrorCode =
  (typeof WORKSPACE_GIT_SCAN_ERROR_CODES)[keyof typeof WORKSPACE_GIT_SCAN_ERROR_CODES];

export class WorkspaceGitScanError extends HttpError {
  readonly code: WorkspaceGitScanErrorCode;

  constructor(
    code: WorkspaceGitScanErrorCode,
    message: string,
    details: Record<string, unknown> = {},
  ) {
    const status = code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout
      ? 504
      : code === WORKSPACE_GIT_SCAN_ERROR_CODES.cancelled
        ? 499
        : 503;
    super(status, message, {
      code,
      retryable: code !== WORKSPACE_GIT_SCAN_ERROR_CODES.cancelled,
      ...details,
    });
    this.name = "WorkspaceGitScanError";
    this.code = code;
  }
}

export function isWorkspaceGitScanError(error: unknown): error is WorkspaceGitScanError {
  return error instanceof WorkspaceGitScanError;
}

export interface WorkspaceGitScanResult {
  stdout: string;
  stderr: string;
  canonicalWorkspacePath: string;
  workspaceHash: string;
  cacheHit: boolean;
  singleFlightJoined: boolean;
}

export interface WorkspaceGitScanInput {
  workspacePath: string;
  args: readonly string[];
  operation: string;
  /**
   * Stable, non-secret scheduling dimensions. Jobs sharing any recently-served
   * dimension are deprioritized, so changing issue ids cannot bypass fairness
   * for the same company, actor, or repository.
   */
  fairnessKeys?: readonly string[];
  signal?: AbortSignal;
  /** Successful-result cache duration. Use zero for correctness-sensitive guards. */
  cacheTtlMs?: number;
  /** Per-operation wall-clock deadline. Defaults to the process-wide setting. */
  timeoutMs?: number;
  env?: NodeJS.ProcessEnv;
  maxStdoutBytes?: number;
  maxStderrBytes?: number;
}

export interface WorkspaceGitSchedulerSnapshot {
  activeCount: number;
  queuedCount: number;
  inFlightCount: number;
  cacheEntryCount: number;
  cacheBytes: number;
  totals: {
    started: number;
    succeeded: number;
    failed: number;
    timedOut: number;
    cancelled: number;
    saturated: number;
    cacheHits: number;
    singleFlightJoins: number;
  };
}

export interface WorkspaceGitRunnerInput {
  canonicalWorkspacePath: string;
  args: readonly string[];
  env?: NodeJS.ProcessEnv;
  signal: AbortSignal;
  timeoutMs: number;
  killGraceMs: number;
  maxStdoutBytes: number;
  maxStderrBytes: number;
}

export interface WorkspaceGitRunnerResult {
  stdout: string;
  stderr: string;
}

export type WorkspaceGitRunner = (
  input: WorkspaceGitRunnerInput,
) => Promise<WorkspaceGitRunnerResult>;

export interface WorkspaceGitOperationSchedulerOptions {
  concurrency?: number;
  queueCapacity?: number;
  timeoutMs?: number;
  killGraceMs?: number;
  defaultCacheTtlMs?: number;
  maxCacheEntries?: number;
  maxCacheBytes?: number;
  maxStdoutBytes?: number;
  maxStderrBytes?: number;
  warningIntervalMs?: number;
  gitBinary?: string;
  gitArgsPrefix?: readonly string[];
  runner?: WorkspaceGitRunner;
  now?: () => number;
}

interface Waiter {
  id: symbol;
  signal?: AbortSignal;
  onAbort?: () => void;
  resolve: (result: WorkspaceGitScanResult) => void;
  reject: (error: unknown) => void;
  joined: boolean;
}

interface PendingScan {
  key: string;
  operation: string;
  canonicalWorkspacePath: string;
  workspaceHash: string;
  args: readonly string[];
  fairnessKeys: readonly string[];
  env?: NodeJS.ProcessEnv;
  timeoutMs: number;
  maxStdoutBytes: number;
  maxStderrBytes: number;
  cacheTtlMs: number;
  enqueuedAt: number;
  state: "queued" | "running";
  controller: AbortController;
  waiters: Map<symbol, Waiter>;
  joinCount: number;
}

interface CacheEntry {
  stdout: string;
  stderr: string;
  canonicalWorkspacePath: string;
  workspaceHash: string;
  expiresAt: number;
  bytes: number;
}

interface WarningBucket {
  lastLoggedAt: number;
  suppressed: number;
}

const DEFAULT_CONCURRENCY = 2;
const DEFAULT_QUEUE_CAPACITY = 32;
const DEFAULT_TIMEOUT_MS = 8_000;
const DEFAULT_KILL_GRACE_MS = 250;
const DEFAULT_CACHE_TTL_MS = 10_000;
const DEFAULT_CACHE_ENTRIES = 64;
const DEFAULT_CACHE_BYTES = 8 * 1024 * 1024;
const DEFAULT_OUTPUT_BYTES = 1024 * 1024;
const DEFAULT_WARNING_INTERVAL_MS = 10_000;

function clampInteger(value: number | undefined, fallback: number, min: number, max: number): number {
  if (!Number.isFinite(value)) return fallback;
  return Math.min(max, Math.max(min, Math.floor(value!)));
}

function envInteger(
  env: NodeJS.ProcessEnv,
  key: string,
  fallback: number,
  min: number,
  max: number,
): number {
  const raw = env[key]?.trim();
  if (!raw) return fallback;
  const parsed = Number(raw);
  return clampInteger(parsed, fallback, min, max);
}

export function workspaceGitSchedulerOptionsFromEnv(
  env: NodeJS.ProcessEnv = process.env,
): Required<Pick<
  WorkspaceGitOperationSchedulerOptions,
  "concurrency" | "queueCapacity" | "timeoutMs" | "defaultCacheTtlMs"
>> {
  return {
    concurrency: envInteger(env, "PAPERCLIP_WORKSPACE_GIT_SCAN_CONCURRENCY", DEFAULT_CONCURRENCY, 1, 16),
    queueCapacity: envInteger(env, "PAPERCLIP_WORKSPACE_GIT_SCAN_QUEUE_CAPACITY", DEFAULT_QUEUE_CAPACITY, 0, 1_024),
    timeoutMs: envInteger(env, "PAPERCLIP_WORKSPACE_GIT_SCAN_TIMEOUT_MS", DEFAULT_TIMEOUT_MS, 100, 120_000),
    defaultCacheTtlMs: envInteger(env, "PAPERCLIP_WORKSPACE_GIT_SCAN_CACHE_TTL_MS", DEFAULT_CACHE_TTL_MS, 0, 60_000),
  };
}

function workspaceIdentity(canonicalWorkspacePath: string): string {
  return createHash("sha256").update(canonicalWorkspacePath).digest("hex").slice(0, 16);
}

function scanKey(input: {
  canonicalWorkspacePath: string;
  args: readonly string[];
  env?: NodeJS.ProcessEnv;
  timeoutMs: number;
  maxStdoutBytes: number;
  maxStderrBytes: number;
}): string {
  // These variables can change status semantics. Hash values so neither keys nor
  // telemetry expose credentials or private config contents.
  const effectiveEnv = input.env ?? process.env;
  const semanticEnv = [
    "GIT_CONFIG_COUNT",
    "GIT_CONFIG_PARAMETERS",
    "GIT_DIR",
    "GIT_INDEX_FILE",
    "GIT_OPTIONAL_LOCKS",
    "GIT_WORK_TREE",
  ].map((key) => [key, effectiveEnv[key] ?? null]);
  return createHash("sha256")
    .update(JSON.stringify({
      workspacePath: input.canonicalWorkspacePath,
      args: input.args,
      semanticEnv,
      timeoutMs: input.timeoutMs,
      maxStdoutBytes: input.maxStdoutBytes,
      maxStderrBytes: input.maxStderrBytes,
    }))
    .digest("hex");
}

function abortError(workspaceHash: string): WorkspaceGitScanError {
  return new WorkspaceGitScanError(
    WORKSPACE_GIT_SCAN_ERROR_CODES.cancelled,
    "Workspace Git scan was cancelled",
    { workspaceHash },
  );
}

function signalChild(child: ChildProcess, signal: NodeJS.Signals): void {
  if (child.exitCode !== null || child.signalCode !== null) return;
  if (process.platform !== "win32" && child.pid) {
    try {
      process.kill(-child.pid, signal);
      return;
    } catch {
      // Fall back to the direct child if the process group already disappeared.
    }
  }
  try {
    child.kill(signal);
  } catch {
    // The close/error handler owns settlement; an already-dead child is benign.
  }
}

function createSpawnRunner(input: {
  gitBinary: string;
  gitArgsPrefix: readonly string[];
}): WorkspaceGitRunner {
  return (runInput) => new Promise<WorkspaceGitRunnerResult>((resolve, reject) => {
    if (runInput.signal.aborted) {
      reject(abortError(workspaceIdentity(runInput.canonicalWorkspacePath)));
      return;
    }

    const child = spawn(
      input.gitBinary,
      [...input.gitArgsPrefix, "-C", runInput.canonicalWorkspacePath, ...runInput.args],
      {
        cwd: runInput.canonicalWorkspacePath,
        env: runInput.env ?? process.env,
        stdio: ["ignore", "pipe", "pipe"],
        detached: process.platform !== "win32",
        windowsHide: true,
      },
    );
    const stdoutChunks: Buffer[] = [];
    const stderrChunks: Buffer[] = [];
    let stdoutBytes = 0;
    let stderrBytes = 0;
    let termination: "timeout" | "cancelled" | "output_limit" | null = null;
    let spawnError: Error | null = null;
    let settled = false;

    const terminate = (reason: NonNullable<typeof termination>) => {
      if (termination) return;
      termination = reason;
      signalChild(child, "SIGTERM");
      killTimer = setTimeout(() => signalChild(child, "SIGKILL"), runInput.killGraceMs);
      killTimer.unref?.();
    };

    const append = (
      chunk: Buffer | string,
      chunks: Buffer[],
      currentBytes: number,
      maxBytes: number,
    ): number => {
      const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
      const remaining = Math.max(0, maxBytes - currentBytes);
      if (remaining > 0) chunks.push(buffer.subarray(0, remaining));
      const nextBytes = currentBytes + buffer.length;
      if (nextBytes > maxBytes) terminate("output_limit");
      return nextBytes;
    };

    const onAbort = () => terminate("cancelled");
    runInput.signal.addEventListener("abort", onAbort, { once: true });
    child.stdout?.on("data", (chunk) => {
      stdoutBytes = append(chunk, stdoutChunks, stdoutBytes, runInput.maxStdoutBytes);
    });
    child.stderr?.on("data", (chunk) => {
      stderrBytes = append(chunk, stderrChunks, stderrBytes, runInput.maxStderrBytes);
    });
    child.once("error", (error) => {
      spawnError = error;
    });

    const timeoutTimer = setTimeout(() => terminate("timeout"), runInput.timeoutMs);
    timeoutTimer.unref?.();
    let killTimer: NodeJS.Timeout | null = null;

    child.once("close", (code, childSignal) => {
      if (settled) return;
      settled = true;
      clearTimeout(timeoutTimer);
      if (killTimer) clearTimeout(killTimer);
      runInput.signal.removeEventListener("abort", onAbort);
      const stdout = Buffer.concat(stdoutChunks).toString("utf8");
      const stderr = Buffer.concat(stderrChunks).toString("utf8");
      const workspaceHash = workspaceIdentity(runInput.canonicalWorkspacePath);

      if (termination === "timeout") {
        reject(new WorkspaceGitScanError(
          WORKSPACE_GIT_SCAN_ERROR_CODES.timeout,
          `Workspace Git scan timed out after ${runInput.timeoutMs}ms`,
          { workspaceHash, timeoutMs: runInput.timeoutMs },
        ));
        return;
      }
      if (termination === "cancelled") {
        reject(abortError(workspaceHash));
        return;
      }
      if (termination === "output_limit") {
        reject(new WorkspaceGitScanError(
          WORKSPACE_GIT_SCAN_ERROR_CODES.outputLimit,
          "Workspace Git scan exceeded its output limit",
          {
            workspaceHash,
            stdoutBytes,
            stderrBytes,
            maxStdoutBytes: runInput.maxStdoutBytes,
            maxStderrBytes: runInput.maxStderrBytes,
          },
        ));
        return;
      }
      if (spawnError) {
        reject(new WorkspaceGitScanError(
          WORKSPACE_GIT_SCAN_ERROR_CODES.failed,
          "Workspace Git scan could not start",
          { workspaceHash, cause: spawnError.message },
        ));
        return;
      }
      if (code !== 0) {
        reject(new WorkspaceGitScanError(
          WORKSPACE_GIT_SCAN_ERROR_CODES.failed,
          "Workspace Git scan failed",
          {
            workspaceHash,
            exitCode: code,
            signal: childSignal,
            // Keep the diagnostic bounded; callers never receive raw paths in telemetry.
            stderr: stderr.trim().slice(0, 1_000),
          },
        ));
        return;
      }
      resolve({ stdout, stderr });
    });
  });
}

export class WorkspaceGitOperationScheduler {
  private readonly concurrency: number;
  private readonly queueCapacity: number;
  private readonly timeoutMs: number;
  private readonly killGraceMs: number;
  private readonly defaultCacheTtlMs: number;
  private readonly maxCacheEntries: number;
  private readonly maxCacheBytes: number;
  private readonly maxStdoutBytes: number;
  private readonly maxStderrBytes: number;
  private readonly warningIntervalMs: number;
  private readonly runner: WorkspaceGitRunner;
  private readonly now: () => number;
  private readonly inFlight = new Map<string, PendingScan>();
  private readonly queue: PendingScan[] = [];
  private readonly cache = new Map<string, CacheEntry>();
  private readonly lastServedByFairnessKey = new Map<string, number>();
  private readonly warningBuckets = new Map<string, WarningBucket>();
  private cacheBytes = 0;
  private activeCount = 0;
  private serviceSequence = 0;
  private readonly totals = {
    started: 0,
    succeeded: 0,
    failed: 0,
    timedOut: 0,
    cancelled: 0,
    saturated: 0,
    cacheHits: 0,
    singleFlightJoins: 0,
  };

  constructor(options: WorkspaceGitOperationSchedulerOptions = {}) {
    this.concurrency = clampInteger(options.concurrency, DEFAULT_CONCURRENCY, 1, 16);
    this.queueCapacity = clampInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY, 0, 1_024);
    this.timeoutMs = clampInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS, 1, 120_000);
    this.killGraceMs = clampInteger(options.killGraceMs, DEFAULT_KILL_GRACE_MS, 1, 10_000);
    this.defaultCacheTtlMs = clampInteger(options.defaultCacheTtlMs, DEFAULT_CACHE_TTL_MS, 0, 60_000);
    this.maxCacheEntries = clampInteger(options.maxCacheEntries, DEFAULT_CACHE_ENTRIES, 0, 10_000);
    this.maxCacheBytes = clampInteger(options.maxCacheBytes, DEFAULT_CACHE_BYTES, 0, 1024 * 1024 * 1024);
    this.maxStdoutBytes = clampInteger(options.maxStdoutBytes, DEFAULT_OUTPUT_BYTES, 1, 128 * 1024 * 1024);
    this.maxStderrBytes = clampInteger(options.maxStderrBytes, DEFAULT_OUTPUT_BYTES, 1, 128 * 1024 * 1024);
    this.warningIntervalMs = clampInteger(options.warningIntervalMs, DEFAULT_WARNING_INTERVAL_MS, 1, 60 * 60_000);
    this.now = options.now ?? Date.now;
    this.runner = options.runner ?? createSpawnRunner({
      gitBinary: options.gitBinary ?? "git",
      gitArgsPrefix: options.gitArgsPrefix ?? [],
    });
  }

  snapshot(): WorkspaceGitSchedulerSnapshot {
    this.pruneExpiredCache();
    return {
      activeCount: this.activeCount,
      queuedCount: this.queue.length,
      inFlightCount: this.inFlight.size,
      cacheEntryCount: this.cache.size,
      cacheBytes: this.cacheBytes,
      totals: { ...this.totals },
    };
  }

  async run(input: WorkspaceGitScanInput): Promise<WorkspaceGitScanResult> {
    if (input.signal?.aborted) throw abortError("unresolved");
    let canonicalWorkspacePath: string;
    try {
      canonicalWorkspacePath = await fs.realpath(input.workspacePath);
    } catch (error) {
      throw new WorkspaceGitScanError(
        WORKSPACE_GIT_SCAN_ERROR_CODES.failed,
        "Workspace Git scan path is unavailable",
        { cause: error instanceof Error ? error.message : String(error) },
      );
    }
    if (input.signal?.aborted) throw abortError(workspaceIdentity(canonicalWorkspacePath));

    const workspaceHash = workspaceIdentity(canonicalWorkspacePath);
    const timeoutMs = clampInteger(input.timeoutMs, this.timeoutMs, 1, 120_000);
    const maxStdoutBytes = clampInteger(input.maxStdoutBytes, this.maxStdoutBytes, 1, 128 * 1024 * 1024);
    const maxStderrBytes = clampInteger(input.maxStderrBytes, this.maxStderrBytes, 1, 128 * 1024 * 1024);
    const key = scanKey({
      canonicalWorkspacePath,
      args: input.args,
      env: input.env,
      timeoutMs,
      maxStdoutBytes,
      maxStderrBytes,
    });
    const cacheTtlMs = clampInteger(input.cacheTtlMs, this.defaultCacheTtlMs, 0, 60_000);
    // A correctness-sensitive caller that explicitly disables caching must not
    // consume a result populated earlier by the file browser.
    const cached = cacheTtlMs > 0 ? this.readCache(key) : null;
    if (cached) {
      this.totals.cacheHits += 1;
      logger.debug({
        event: "workspace_git_scan",
        operation: input.operation,
        workspaceHash,
        cacheHit: true,
        singleFlightJoined: false,
        activeCount: this.activeCount,
        queuedCount: this.queue.length,
        cacheHitCount: this.totals.cacheHits,
      }, "workspace Git scan cache hit");
      return {
        stdout: cached.stdout,
        stderr: cached.stderr,
        canonicalWorkspacePath,
        workspaceHash,
        cacheHit: true,
        singleFlightJoined: false,
      };
    }

    const existing = this.inFlight.get(key);
    if (existing) {
      existing.joinCount += 1;
      this.totals.singleFlightJoins += 1;
      return this.addWaiter(existing, input.signal, true);
    }

    if (this.activeCount >= this.concurrency && this.queue.length >= this.queueCapacity) {
      this.totals.saturated += 1;
      this.warnRateLimited("saturated", {
        event: "workspace_git_scan",
        operation: input.operation,
        workspaceHash,
        outcome: "saturated",
        activeCount: this.activeCount,
        queuedCount: this.queue.length,
        saturationCount: this.totals.saturated,
      }, "workspace Git scan queue saturated");
      throw new WorkspaceGitScanError(
        WORKSPACE_GIT_SCAN_ERROR_CODES.saturated,
        "Changed files are temporarily unavailable because the Git scan queue is full",
        {
          workspaceHash,
          activeCount: this.activeCount,
          queuedCount: this.queue.length,
          retryAfterSeconds: 1,
        },
      );
    }

    const fairnessKeys = Array.from(new Set([
      `repository:${workspaceHash}`,
      ...(input.fairnessKeys ?? []).filter(Boolean),
    ])).sort();
    const scan: PendingScan = {
      key,
      operation: input.operation,
      canonicalWorkspacePath,
      workspaceHash,
      args: [...input.args],
      fairnessKeys,
      env: input.env,
      timeoutMs,
      maxStdoutBytes,
      maxStderrBytes,
      cacheTtlMs,
      enqueuedAt: this.now(),
      state: "queued",
      controller: new AbortController(),
      waiters: new Map(),
      joinCount: 0,
    };
    this.inFlight.set(key, scan);
    this.queue.push(scan);
    const promise = this.addWaiter(scan, input.signal, false);
    this.drain();
    return promise;
  }

  private addWaiter(
    scan: PendingScan,
    signal: AbortSignal | undefined,
    joined: boolean,
  ): Promise<WorkspaceGitScanResult> {
    if (signal?.aborted) return Promise.reject(abortError(scan.workspaceHash));
    return new Promise((resolve, reject) => {
      const waiter: Waiter = {
        id: Symbol("workspace-git-waiter"),
        signal,
        resolve,
        reject,
        joined,
      };
      if (signal) {
        waiter.onAbort = () => {
          this.removeWaiter(scan, waiter);
          reject(abortError(scan.workspaceHash));
        };
        signal.addEventListener("abort", waiter.onAbort, { once: true });
      }
      scan.waiters.set(waiter.id, waiter);
    });
  }

  private removeWaiter(scan: PendingScan, waiter: Waiter): void {
    if (!scan.waiters.delete(waiter.id)) return;
    if (waiter.signal && waiter.onAbort) {
      waiter.signal.removeEventListener("abort", waiter.onAbort);
    }
    if (scan.waiters.size > 0) return;

    if (scan.state === "queued") {
      const index = this.queue.indexOf(scan);
      if (index >= 0) this.queue.splice(index, 1);
      this.inFlight.delete(scan.key);
      this.totals.cancelled += 1;
      logger.debug({
        event: "workspace_git_scan",
        operation: scan.operation,
        workspaceHash: scan.workspaceHash,
        outcome: "cancelled_before_start",
        activeCount: this.activeCount,
        queuedCount: this.queue.length,
      }, "queued workspace Git scan cancelled");
      this.drain();
      this.pruneFairnessState();
      return;
    }
    // Detach the doomed single-flight entry immediately. A new caller arriving
    // while the child is terminating may enqueue a fresh scan instead of
    // joining an already-aborted promise; the active slot remains occupied
    // until the child actually closes.
    if (this.inFlight.get(scan.key) === scan) this.inFlight.delete(scan.key);
    scan.controller.abort();
  }

  private drain(): void {
    while (this.activeCount < this.concurrency && this.queue.length > 0) {
      const index = this.nextFairQueueIndex();
      const scan = this.queue.splice(index, 1)[0]!;
      if (scan.waiters.size === 0) {
        this.inFlight.delete(scan.key);
        continue;
      }
      this.start(scan);
    }
  }

  private nextFairQueueIndex(): number {
    let selectedIndex = 0;
    let selectedScore = Number.POSITIVE_INFINITY;
    let selectedEnqueuedAt = Number.POSITIVE_INFINITY;
    for (let index = 0; index < this.queue.length; index += 1) {
      const scan = this.queue[index]!;
      // Average recency across every dimension so a common repository/company
      // key cannot mask a fresh actor or issue. Repeatedly changing only one
      // dimension therefore cannot jump ahead of a wholly unserved group.
      const score = scan.fairnessKeys.reduce(
        (total, key) => total + (this.lastServedByFairnessKey.get(key) ?? 0),
        0,
      ) / scan.fairnessKeys.length;
      if (score < selectedScore || (score === selectedScore && scan.enqueuedAt < selectedEnqueuedAt)) {
        selectedIndex = index;
        selectedScore = score;
        selectedEnqueuedAt = scan.enqueuedAt;
      }
    }
    return selectedIndex;
  }

  private start(scan: PendingScan): void {
    scan.state = "running";
    this.activeCount += 1;
    this.totals.started += 1;
    this.serviceSequence += 1;
    for (const key of scan.fairnessKeys) this.lastServedByFairnessKey.set(key, this.serviceSequence);
    const startedAt = this.now();
    const queueWaitMs = Math.max(0, startedAt - scan.enqueuedAt);

    void this.runner({
      canonicalWorkspacePath: scan.canonicalWorkspacePath,
      args: scan.args,
      env: scan.env,
      signal: scan.controller.signal,
      timeoutMs: scan.timeoutMs,
      killGraceMs: this.killGraceMs,
      maxStdoutBytes: scan.maxStdoutBytes,
      maxStderrBytes: scan.maxStderrBytes,
    }).then(
      (result) => this.finishSuccess(scan, result, queueWaitMs, startedAt),
      (error) => this.finishFailure(scan, error, queueWaitMs, startedAt),
    );
  }

  private finishSuccess(
    scan: PendingScan,
    result: WorkspaceGitRunnerResult,
    queueWaitMs: number,
    startedAt: number,
  ): void {
    this.totals.succeeded += 1;
    if (scan.cacheTtlMs > 0) this.writeCache(scan.key, scan, result);
    const responseBase = {
      stdout: result.stdout,
      stderr: result.stderr,
      canonicalWorkspacePath: scan.canonicalWorkspacePath,
      workspaceHash: scan.workspaceHash,
      cacheHit: false,
    };
    for (const waiter of scan.waiters.values()) {
      this.detachWaiter(waiter);
      waiter.resolve({ ...responseBase, singleFlightJoined: waiter.joined });
    }
    logger.info({
      event: "workspace_git_scan",
      operation: scan.operation,
      workspaceHash: scan.workspaceHash,
      outcome: "success",
      queueWaitMs,
      executionMs: Math.max(0, this.now() - startedAt),
      activeCount: this.activeCount,
      queuedCount: this.queue.length,
      cacheHit: false,
      singleFlightJoinCount: scan.joinCount,
      exitOutcome: "zero",
    }, "workspace Git scan completed");
    this.release(scan);
  }

  private finishFailure(
    scan: PendingScan,
    error: unknown,
    queueWaitMs: number,
    startedAt: number,
  ): void {
    const normalized = isWorkspaceGitScanError(error)
      ? error
      : new WorkspaceGitScanError(
          WORKSPACE_GIT_SCAN_ERROR_CODES.failed,
          "Workspace Git scan failed",
          { cause: error instanceof Error ? error.message : String(error), workspaceHash: scan.workspaceHash },
        );
    if (normalized.code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout) {
      this.totals.timedOut += 1;
      this.warnRateLimited("timeout", {
        event: "workspace_git_scan",
        operation: scan.operation,
        workspaceHash: scan.workspaceHash,
        outcome: "timeout",
        queueWaitMs,
        executionMs: Math.max(0, this.now() - startedAt),
        activeCount: this.activeCount,
        queuedCount: this.queue.length,
        timeoutCount: this.totals.timedOut,
      }, "workspace Git scan timed out");
    } else if (normalized.code === WORKSPACE_GIT_SCAN_ERROR_CODES.cancelled) {
      this.totals.cancelled += 1;
    } else {
      this.totals.failed += 1;
    }
    for (const waiter of scan.waiters.values()) {
      this.detachWaiter(waiter);
      waiter.reject(normalized);
    }
    logger.info({
      event: "workspace_git_scan",
      operation: scan.operation,
      workspaceHash: scan.workspaceHash,
      outcome: normalized.code,
      queueWaitMs,
      executionMs: Math.max(0, this.now() - startedAt),
      activeCount: this.activeCount,
      queuedCount: this.queue.length,
      cacheHit: false,
      singleFlightJoinCount: scan.joinCount,
      exitOutcome: normalized.code,
    }, "workspace Git scan finished without a result");
    this.release(scan);
  }

  private detachWaiter(waiter: Waiter): void {
    if (waiter.signal && waiter.onAbort) {
      waiter.signal.removeEventListener("abort", waiter.onAbort);
    }
  }

  private release(scan: PendingScan): void {
    scan.waiters.clear();
    if (this.inFlight.get(scan.key) === scan) this.inFlight.delete(scan.key);
    this.activeCount = Math.max(0, this.activeCount - 1);
    this.drain();
    this.pruneFairnessState();
  }

  private pruneFairnessState(): void {
    if (this.lastServedByFairnessKey.size === 0) return;
    const liveKeys = new Set<string>();
    for (const scan of this.inFlight.values()) {
      for (const key of scan.fairnessKeys) liveKeys.add(key);
    }
    for (const key of this.lastServedByFairnessKey.keys()) {
      if (!liveKeys.has(key)) this.lastServedByFairnessKey.delete(key);
    }
  }

  private readCache(key: string): CacheEntry | null {
    const entry = this.cache.get(key);
    if (!entry) return null;
    if (entry.expiresAt <= this.now()) {
      this.deleteCacheEntry(key, entry);
      return null;
    }
    // LRU touch.
    this.cache.delete(key);
    this.cache.set(key, entry);
    return entry;
  }

  private writeCache(key: string, scan: PendingScan, result: WorkspaceGitRunnerResult): void {
    if (this.maxCacheEntries === 0 || this.maxCacheBytes === 0) return;
    const bytes = Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr);
    if (bytes > this.maxCacheBytes) return;
    const existing = this.cache.get(key);
    if (existing) this.deleteCacheEntry(key, existing);
    const entry: CacheEntry = {
      ...result,
      canonicalWorkspacePath: scan.canonicalWorkspacePath,
      workspaceHash: scan.workspaceHash,
      expiresAt: this.now() + scan.cacheTtlMs,
      bytes,
    };
    this.cache.set(key, entry);
    this.cacheBytes += bytes;
    while (this.cache.size > this.maxCacheEntries || this.cacheBytes > this.maxCacheBytes) {
      const oldest = this.cache.entries().next().value as [string, CacheEntry] | undefined;
      if (!oldest) break;
      this.deleteCacheEntry(oldest[0], oldest[1]);
    }
  }

  private deleteCacheEntry(key: string, entry: CacheEntry): void {
    if (!this.cache.delete(key)) return;
    this.cacheBytes = Math.max(0, this.cacheBytes - entry.bytes);
  }

  private pruneExpiredCache(): void {
    const now = this.now();
    for (const [key, entry] of this.cache) {
      if (entry.expiresAt <= now) this.deleteCacheEntry(key, entry);
    }
  }

  private warnRateLimited(
    key: string,
    fields: Record<string, unknown>,
    message: string,
  ): void {
    const now = this.now();
    const bucket = this.warningBuckets.get(key);
    if (bucket && now - bucket.lastLoggedAt < this.warningIntervalMs) {
      bucket.suppressed += 1;
      return;
    }
    const suppressedSinceLastWarning = bucket?.suppressed ?? 0;
    this.warningBuckets.set(key, { lastLoggedAt: now, suppressed: 0 });
    logger.warn({ ...fields, suppressedSinceLastWarning }, message);
  }
}

export function createWorkspaceGitOperationScheduler(
  options: WorkspaceGitOperationSchedulerOptions = {},
): WorkspaceGitOperationScheduler {
  return new WorkspaceGitOperationScheduler(options);
}

const envOptions = workspaceGitSchedulerOptionsFromEnv();
export const workspaceGitOperationScheduler = createWorkspaceGitOperationScheduler(envOptions);

setExpensiveWorkspaceGitExecutor(async (input) => {
  const result = await workspaceGitOperationScheduler.run({
    workspacePath: input.localDir,
    args: input.args,
    operation: input.operation,
    cacheTtlMs: 0,
    timeoutMs: input.timeout,
    maxStdoutBytes: input.maxBuffer,
    maxStderrBytes: input.maxBuffer,
  });
  return { stdout: result.stdout, stderr: result.stderr };
});

export function getWorkspaceGitOperationSchedulerSnapshot(): WorkspaceGitSchedulerSnapshot {
  return workspaceGitOperationScheduler.snapshot();
}