PaperClipAI/server/src/services/workspace-operations.ts

import { notifyNativeResultCommit } from "./native-result-commit-notifications.js";
import { notifyNativeWorkspaceFinalization } from "./native-workspace-finalization-notifications.js";
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
import { randomUUID } from "node:crypto";
import type { Db } from "@paperclipai/db";
import { workspaceOperations } from "@paperclipai/db";
import type { WorkspaceOperation, WorkspaceOperationPhase, WorkspaceOperationStatus } from "@paperclipai/shared";
import { asc, desc, eq, gte, inArray, isNull, lt, or, and } from "drizzle-orm";
import { conflict, notFound } from "../errors.js";
import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
import { instanceSettingsService } from "./instance-settings.js";
import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js";

type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;

/**
 * Managed runtime control actions. Every one of these mutates the runtime rows and
 * local listeners of a single execution workspace, so only one may be live at a time.
 */
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "repair", "run"]);

/**
 * Identity of this server process. A `running` runtime-control operation stamped with a
 * different owner id can only have been left behind by another (now gone) process, which
 * is what makes bounded recovery safe: we never guess about our own live operations.
 */
const RUNTIME_CONTROL_OWNER_ID = randomUUID();

/** How often a live operation refreshes its ownership stamp. */
export const RUNTIME_CONTROL_HEARTBEAT_MS = 10_000;

/**
 * How long an operation may go without a heartbeat before another owner may terminalize
 * it. Deliberately several heartbeat intervals so a slow-but-live start is never stolen.
 */
export const RUNTIME_CONTROL_STALE_AFTER_MS = 60_000;

/**
 * Last-resort ceiling for a managed runtime lifecycle control. Past this the operation is
 * failed by its own owner so it always reaches a terminal state, even when the underlying
 * start hangs on an unresponsive process or provider that keeps the owner alive. Generous
 * enough to cover a cold runtime provision (a full dependency install) on a large repo.
 */
export const RUNTIME_CONTROL_MAX_DURATION_MS = 30 * 60_000;

/**
 * Workspace jobs are operator-authored commands (builds, migrations, test suites) that can
 * legitimately run far longer than a lifecycle control, so they get a much wider ceiling —
 * still finite, so a wedged job cannot hold the workspace forever.
 */
export const WORKSPACE_JOB_MAX_DURATION_MS = 4 * 60 * 60_000;

function defaultRuntimeControlTimeoutMs(action: string | null) {
  if (!action) return null;
  return action === "run" ? WORKSPACE_JOB_MAX_DURATION_MS : RUNTIME_CONTROL_MAX_DURATION_MS;
}

export class WorkspaceOperationTimeoutError extends Error {
  constructor(public readonly timeoutMs: number, action: string | null) {
    super(
      `Managed workspace ${action ?? "runtime"} operation exceeded its ${Math.round(timeoutMs / 1000)}s time budget and was failed so it can be retried or stopped.`,
    );
    this.name = "WorkspaceOperationTimeoutError";
  }
}

type RuntimeControlOwnerStamp = {
  ownerId: string;
  pid: number;
  action: string;
  heartbeatAt: string;
};

function readRuntimeControlAction(metadata: unknown): string | null {
  if (!metadata || typeof metadata !== "object") return null;
  const action = (metadata as Record<string, unknown>).action;
  if (typeof action !== "string" || !RUNTIME_CONTROL_ACTIONS.has(action)) return null;
  return action;
}

function readRuntimeControlOwner(metadata: unknown): RuntimeControlOwnerStamp | null {
  if (!metadata || typeof metadata !== "object") return null;
  const owner = (metadata as Record<string, unknown>).runtimeControlOwner;
  if (!owner || typeof owner !== "object") return null;
  const record = owner as Record<string, unknown>;
  if (typeof record.ownerId !== "string") return null;
  return {
    ownerId: record.ownerId,
    pid: typeof record.pid === "number" ? record.pid : 0,
    action: typeof record.action === "string" ? record.action : "start",
    heartbeatAt: typeof record.heartbeatAt === "string" ? record.heartbeatAt : new Date(0).toISOString(),
  };
}

/**
 * Operation ids this process is actively driving. An operation owned by this process but
 * missing from this set can only be a leftover from a request that died without unwinding,
 * so it is safe to terminalize immediately instead of waiting out the staleness window.
 */
const liveRuntimeControlOperationIds = new Set<string>();

type NativeFinalizationActivityScope = {
  operationId: string;
  companyId: string;
  runId: string;
  issueId: string;
};
// Presentation-only positive proof. Persisted running rows can outlive a crash;
// this registry only exists while this process awaits the actual export callback.
const liveNativeFinalizationOperations = new Map<string, NativeFinalizationActivityScope>();

export function isNativeWorkspaceFinalizationOperationActive(scope: NativeFinalizationActivityScope): boolean {
  const active = liveNativeFinalizationOperations.get(scope.operationId);
  return Boolean(active && active.companyId === scope.companyId
    && active.runId === scope.runId && active.issueId === scope.issueId);
}

/**
 * Whether the process that stamped an operation is still alive on this host. A dead owner can
 * never heartbeat again, so its operation is recoverable immediately instead of after the
 * staleness window — which is what makes recovery after a crash or restart prompt. Signal 0
 * only probes; it never touches the process.
 */
function ownerProcessIsAlive(pid: number): boolean {
  if (!Number.isInteger(pid) || pid <= 0) return false;
  try {
    process.kill(pid, 0);
    return true;
  } catch (error) {
    // EPERM means the pid exists but belongs to another user: treat it as alive.
    return (error as NodeJS.ErrnoException)?.code === "EPERM";
  }
}

/**
 * Compare-and-swap predicate for an operation's `updated_at`.
 *
 * Postgres stores `timestamptz` at microsecond precision but the driver hands JS a
 * millisecond-precision `Date`, so an `=` against the value we just read never matches a row
 * whose timestamp came from the column's `defaultNow()` — which is exactly the shape of every
 * row a pre-recovery build left behind, i.e. the stranded operations this sweep exists to
 * clear. Matching the millisecond bucket instead is still a true CAS: a concurrent heartbeat
 * writes a `Date` in a different millisecond, so it still wins the row.
 */
function updatedAtUnchanged(updatedAt: Date) {
  return and(
    gte(workspaceOperations.updatedAt, updatedAt),
    lt(workspaceOperations.updatedAt, new Date(updatedAt.getTime() + 1)),
  );
}

export function resetWorkspaceRuntimeControlStateForTests() {
  liveRuntimeControlOperationIds.clear();
}

export function workspaceRuntimeControlOwnerIdForTests() {
  return RUNTIME_CONTROL_OWNER_ID;
}

/**
 * Same-process serialization of managed runtime controls. The durable sweep below is what
 * recovers stranded operations across processes; this in-memory claim is the cheaper first
 * line of defense that rejects an overlapping control in this process before any row is
 * written. Kept from the runtime-lease chain (PAP-17205) and composed with the ownership
 * machinery above: the claim is taken before the operation row exists, so a recovery sweep
 * can never see a live control it is not yet tracking.
 */
const activeRuntimeControls = new Map<string, { action: string; startedAt: Date }>();

export async function runExclusiveWorkspaceRuntimeControl<T>(input: {
  executionWorkspaceId: string;
  action: string;
  run: () => Promise<T>;
}): Promise<T> {
  const active = activeRuntimeControls.get(input.executionWorkspaceId);
  if (active) {
    throw conflict("A managed runtime control operation is already in progress for this execution workspace.", {
      code: "workspace_runtime_control_in_progress",
      executionWorkspaceId: input.executionWorkspaceId,
      activeAction: active.action,
      requestedAction: input.action,
      startedAt: active.startedAt.toISOString(),
      remediation: "Wait for the active operation to reach a terminal state before retrying.",
    });
  }

  const claim = { action: input.action, startedAt: new Date() };
  activeRuntimeControls.set(input.executionWorkspaceId, claim);
  try {
    return await input.run();
  } finally {
    if (activeRuntimeControls.get(input.executionWorkspaceId) === claim) {
      activeRuntimeControls.delete(input.executionWorkspaceId);
    }
  }
}

export function resetWorkspaceRuntimeControlLocksForTests() {
  activeRuntimeControls.clear();
}

function toWorkspaceOperation(row: WorkspaceOperationRow): WorkspaceOperation {
  return {
    id: row.id,
    companyId: row.companyId,
    executionWorkspaceId: row.executionWorkspaceId ?? null,
    heartbeatRunId: row.heartbeatRunId ?? null,
    issueId: row.issueId ?? null,
    phase: row.phase as WorkspaceOperationPhase,
    command: row.command ?? null,
    cwd: row.cwd ?? null,
    status: row.status as WorkspaceOperationStatus,
    exitCode: row.exitCode ?? null,
    logStore: row.logStore ?? null,
    logRef: row.logRef ?? null,
    logBytes: row.logBytes ?? null,
    logSha256: row.logSha256 ?? null,
    logCompressed: row.logCompressed,
    stdoutExcerpt: row.stdoutExcerpt ?? null,
    stderrExcerpt: row.stderrExcerpt ?? null,
    metadata: (row.metadata as Record<string, unknown> | null) ?? null,
    startedAt: row.startedAt,
    finishedAt: row.finishedAt ?? null,
    createdAt: row.createdAt,
    updatedAt: row.updatedAt,
  };
}

function appendExcerpt(current: string, chunk: string) {
  return `${current}${chunk}`.slice(-4096);
}

function combineMetadata(
  base: Record<string, unknown> | null | undefined,
  patch: Record<string, unknown> | null | undefined,
) {
  if (!base && !patch) return null;
  return {
    ...(base ?? {}),
    ...(patch ?? {}),
  };
}

export interface WorkspaceOperationRecorder {
  attachExecutionWorkspaceId(executionWorkspaceId: string | null): Promise<void>;
  recordOperation(input: {
    phase: WorkspaceOperationPhase;
    command?: string | null;
    cwd?: string | null;
    metadata?: Record<string, unknown> | null;
    /**
     * Wall-clock ceiling for `run`. On expiry the operation is failed (terminal) and the
     * timeout error is rethrown, so a hung provider can never leave an active operation.
     */
    timeoutMs?: number | null;
    run: (reportProgress: (input: {
      metadata?: Record<string, unknown> | null;
      system?: string | null;
    }) => Promise<void>) => Promise<{
      status?: WorkspaceOperationStatus;
      exitCode?: number | null;
      stdout?: string | null;
      stderr?: string | null;
      system?: string | null;
      metadata?: Record<string, unknown> | null;
    }>;
  }): Promise<WorkspaceOperation>;
}

export function workspaceOperationService(db: Db) {
  const instanceSettings = instanceSettingsService(db);
  const logStore = getWorkspaceOperationLogStore();

  async function getById(id: string) {
    const row = await db
      .select()
      .from(workspaceOperations)
      .where(eq(workspaceOperations.id, id))
      .then((rows) => rows[0] ?? null);
    return row ? toWorkspaceOperation(row) : null;
  }

  /**
   * Terminalize `running` managed runtime-control operations that no live owner can still
   * be driving, and return the ones that are genuinely still live.
   *
   * Recovery is bounded on both axes: an operation owned by another server process is only
   * reclaimed after {@link RUNTIME_CONTROL_STALE_AFTER_MS} without a heartbeat, and the
   * terminalizing UPDATE is a compare-and-swap on `updated_at`, so an owner that heartbeats
   * concurrently keeps its operation. Only this process's own abandoned operations (owned
   * by us but no longer tracked in-process) are reclaimed without waiting.
   */
  async function sweepRuntimeControlOperations(
    executionWorkspaceId: string | null | undefined,
    options?: { now?: Date; staleAfterMs?: number },
  ) {
    const now = options?.now ?? new Date();
    const staleAfterMs = options?.staleAfterMs ?? RUNTIME_CONTROL_STALE_AFTER_MS;
    const candidates = await db
      .select()
      .from(workspaceOperations)
      .where(
        executionWorkspaceId
          ? and(
              eq(workspaceOperations.executionWorkspaceId, executionWorkspaceId),
              eq(workspaceOperations.status, "running"),
            )
          : eq(workspaceOperations.status, "running"),
      );

    const reconciledIds: string[] = [];
    const live: WorkspaceOperation[] = [];

    for (const candidate of candidates) {
      const action = readRuntimeControlAction(candidate.metadata);
      if (!action) continue;

      const owner = readRuntimeControlOwner(candidate.metadata);
      const lastSignalAt = owner
        ? new Date(owner.heartbeatAt)
        : (candidate.updatedAt ?? candidate.startedAt ?? new Date(0));
      const silentForMs = now.getTime() - lastSignalAt.getTime();

      const ownedByThisProcess = owner?.ownerId === RUNTIME_CONTROL_OWNER_ID;
      const abandonedByThisProcess = ownedByThisProcess && !liveRuntimeControlOperationIds.has(candidate.id);
      const ownedHereAndLive = ownedByThisProcess && liveRuntimeControlOperationIds.has(candidate.id);
      // A previous server process that no longer exists cannot heartbeat again, so its
      // operations are recoverable at once instead of after the staleness window.
      const ownerProcessGone = owner !== null && !ownedByThisProcess && !ownerProcessIsAlive(owner.pid);

      if (ownedHereAndLive) {
        // This process is driving it right now; its own time budget is what terminalizes it.
        live.push(toWorkspaceOperation(candidate));
        continue;
      }
      if (!abandonedByThisProcess && !ownerProcessGone && silentForMs < staleAfterMs) {
        live.push(toWorkspaceOperation(candidate));
        continue;
      }

      const finishedAt = new Date();
      const reconciliationMessage =
        `Reconciled stale managed runtime ${action} operation: ${
          ownerProcessGone
            ? `the owning server process (pid ${owner?.pid}) is gone`
            : abandonedByThisProcess
              ? "the owning request no longer exists"
              : `no live owner has reported progress for ${Math.round(silentForMs / 1000)}s`
        }, so the operation was failed to release the execution workspace for retry or stop.\n`;
      // Compare-and-swap on `updated_at`: a live owner that heartbeats between the read and
      // this write moves the column and keeps its operation.
      const claimed = await db
        .update(workspaceOperations)
        .set({
          status: "failed",
          stderrExcerpt: appendExcerpt(candidate.stderrExcerpt ?? "", reconciliationMessage),
          metadata: combineMetadata(candidate.metadata as Record<string, unknown> | null, {
            reconciled: true,
            reconciliationReason: abandonedByThisProcess
              ? "abandoned_runtime_control"
              : "orphaned_runtime_control",
            reconciledAt: finishedAt.toISOString(),
          }),
          finishedAt,
          updatedAt: finishedAt,
        })
        .where(
          and(
            eq(workspaceOperations.id, candidate.id),
            eq(workspaceOperations.status, "running"),
            candidate.updatedAt
              ? updatedAtUnchanged(candidate.updatedAt)
              : isNull(workspaceOperations.updatedAt),
          ),
        )
        .returning({ id: workspaceOperations.id })
        .then((rows) => rows[0] ?? null);
      if (!claimed) {
        // Someone else moved the row; re-read so callers see the current truth.
        const refreshed = await getById(candidate.id);
        if (refreshed?.status === "running") live.push(refreshed);
        continue;
      }

      liveRuntimeControlOperationIds.delete(candidate.id);
      reconciledIds.push(candidate.id);

      if (candidate.logStore && candidate.logRef) {
        const handle = {
          store: candidate.logStore as "local_file",
          logRef: candidate.logRef,
        };
        try {
          await logStore.append(handle, {
            stream: "stderr",
            chunk: reconciliationMessage,
            ts: finishedAt.toISOString(),
          });
          const finalized = await logStore.finalize(handle);
          await db
            .update(workspaceOperations)
            .set({
              logBytes: finalized.bytes,
              logSha256: finalized.sha256,
              logCompressed: finalized.compressed,
              updatedAt: new Date(),
            })
            .where(eq(workspaceOperations.id, candidate.id));
        } catch {
          // The terminal row and stderr excerpt stay inspectable even when the external log
          // file from the previous process is gone.
        }
      }
    }

    return { reconciled: reconciledIds.length, operationIds: reconciledIds, live };
  }

  return {
    getById,

    /**
     * Bounded recovery entrypoint. Safe to call on startup and before every managed control.
     */
    async reconcileStaleRuntimeControlOperations(
      executionWorkspaceId?: string | null,
      options?: { now?: Date; staleAfterMs?: number },
    ) {
      const result = await sweepRuntimeControlOperations(executionWorkspaceId, options);
      return { reconciled: result.reconciled, operationIds: result.operationIds };
    },

    /**
     * Refuse a managed runtime control while another one is genuinely live for the same
     * execution workspace. Stale operations are recovered first, so a workspace stranded by
     * a dead request or a previous server process becomes controllable again on its own.
     */
    async assertRuntimeControlAvailable(input: {
      executionWorkspaceId: string;
      action: string;
      options?: { now?: Date; staleAfterMs?: number };
    }) {
      const { live } = await sweepRuntimeControlOperations(input.executionWorkspaceId, input.options);
      const blocking = live[0];
      if (!blocking) return;
      const owner = readRuntimeControlOwner(blocking.metadata);
      throw conflict("A managed runtime control operation is already in progress for this execution workspace.", {
        code: "workspace_runtime_control_in_progress",
        executionWorkspaceId: input.executionWorkspaceId,
        activeAction: readRuntimeControlAction(blocking.metadata) ?? owner?.action ?? null,
        requestedAction: input.action,
        activeOperationId: blocking.id,
        startedAt: blocking.startedAt instanceof Date ? blocking.startedAt.toISOString() : blocking.startedAt,
        remediation:
          "Wait for the active operation to reach a terminal state before retrying; abandoned operations are recovered automatically.",
      });
    },

    createRecorder(input: {
      companyId: string;
      heartbeatRunId?: string | null;
      executionWorkspaceId?: string | null;
      issueId?: string | null;
    }): WorkspaceOperationRecorder {
      let executionWorkspaceId = input.executionWorkspaceId ?? null;
      const createdIds: string[] = [];

      return {
        async attachExecutionWorkspaceId(nextExecutionWorkspaceId) {
          executionWorkspaceId = nextExecutionWorkspaceId ?? null;
          if (!executionWorkspaceId || createdIds.length === 0) return;
          await db
            .update(workspaceOperations)
            .set({
              executionWorkspaceId,
              updatedAt: new Date(),
            })
            .where(inArray(workspaceOperations.id, createdIds));
        },

        async recordOperation(recordInput) {
          async function writeState<T>(write: (database: Db) => Promise<T>): Promise<T> {
            if (recordInput.phase !== "workspace_finalize" || !input.heartbeatRunId) return write(db);
            return db.transaction(async tx => {
              await notifyNativeResultCommit(tx, { companyId: input.companyId,
                runId: input.heartbeatRunId!, issueId: input.issueId ?? undefined });
              await notifyNativeWorkspaceFinalization(tx, { companyId: input.companyId,
                runId: input.heartbeatRunId!, issueId: input.issueId ?? undefined });
              await notifyNativeStatusReassessment(tx, { companyId: input.companyId,
                runId: input.heartbeatRunId!, issueId: input.issueId ?? undefined });
              return write(tx as unknown as Db);
            });
          }
          const currentUserRedactionOptions = {
            enabled: (await instanceSettings.getGeneral()).censorUsernameInLogs,
          };
          const startedAt = new Date();
          const id = randomUUID();
          const handle = await logStore.begin({
            companyId: input.companyId,
            operationId: id,
          });

          let stdoutExcerpt = "";
          let stderrExcerpt = "";
          const append = async (stream: "stdout" | "stderr" | "system", chunk: string | null | undefined) => {
            if (!chunk) return;
            const sanitizedChunk = redactCurrentUserText(chunk, currentUserRedactionOptions);
            if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
            if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
            await logStore.append(handle, {
              stream,
              chunk: sanitizedChunk,
              ts: new Date().toISOString(),
            });
          };

          // Managed runtime controls get an ownership stamp so bounded recovery can tell a
          // slow-but-live operation from one abandoned by a dead request or server process.
          const runtimeControlAction = readRuntimeControlAction(recordInput.metadata);
          let currentMetadata = runtimeControlAction
            ? {
                ...(recordInput.metadata ?? {}),
                runtimeControlOwner: {
                  ownerId: RUNTIME_CONTROL_OWNER_ID,
                  pid: process.pid,
                  action: runtimeControlAction,
                  heartbeatAt: startedAt.toISOString(),
                } satisfies RuntimeControlOwnerStamp,
              }
            : recordInput.metadata ?? null;

          // Claim in-process before the row exists, so there is no ordering in which a recovery
          // sweep can observe an operation stamped by this process but not yet tracked by it —
          // which is the one state that would let recovery terminalize a genuinely live start.
          if (runtimeControlAction) liveRuntimeControlOperationIds.add(id);

          try {
            await writeState(database => database.insert(workspaceOperations).values({
              id,
              companyId: input.companyId,
              executionWorkspaceId,
              heartbeatRunId: input.heartbeatRunId ?? null,
              issueId: input.issueId ?? null,
              phase: recordInput.phase,
              command: recordInput.command ?? null,
              cwd: recordInput.cwd ?? null,
              status: "running",
              logStore: handle.store,
              logRef: handle.logRef,
              metadata: redactCurrentUserValue(
                currentMetadata,
                currentUserRedactionOptions,
              ) as Record<string, unknown> | null,
              startedAt,
              updatedAt: startedAt,
            }));
          } catch (insertError) {
            liveRuntimeControlOperationIds.delete(id);
            throw insertError;
          }
          createdIds.push(id);

          let heartbeatTimer: NodeJS.Timeout | null = null;
          let timeoutTimer: NodeJS.Timeout | null = null;
          if (runtimeControlAction) {
            heartbeatTimer = setInterval(() => {
              const heartbeatAt = new Date();
              currentMetadata = combineMetadata(currentMetadata, {
                runtimeControlOwner: {
                  ownerId: RUNTIME_CONTROL_OWNER_ID,
                  pid: process.pid,
                  action: runtimeControlAction,
                  heartbeatAt: heartbeatAt.toISOString(),
                } satisfies RuntimeControlOwnerStamp,
              });
              void db
                .update(workspaceOperations)
                .set({
                  metadata: redactCurrentUserValue(
                    currentMetadata,
                    currentUserRedactionOptions,
                  ) as Record<string, unknown> | null,
                  updatedAt: heartbeatAt,
                })
                .where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")))
                .catch(() => undefined);
            }, RUNTIME_CONTROL_HEARTBEAT_MS);
            heartbeatTimer.unref?.();
          }

          const reportProgress = async (progress: {
            metadata?: Record<string, unknown> | null;
            system?: string | null;
          }) => {
            await append("system", progress.system ?? null);
            currentMetadata = combineMetadata(currentMetadata, progress.metadata);
            await db
              .update(workspaceOperations)
              .set({
                metadata: redactCurrentUserValue(
                  currentMetadata,
                  currentUserRedactionOptions,
                ) as Record<string, unknown> | null,
                stdoutExcerpt: stdoutExcerpt || null,
                stderrExcerpt: stderrExcerpt || null,
                updatedAt: new Date(),
              })
              .where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")));
          };

          const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction);
          const runWithActivity = async () => {
            const nativeFinalization = recordInput.phase === "workspace_finalize"
              && recordInput.metadata?.owningService === "native_workspace_finalizer"
              && input.heartbeatRunId && input.issueId;
            if (nativeFinalization) liveNativeFinalizationOperations.set(id, {
              operationId: id, companyId: input.companyId,
              runId: input.heartbeatRunId!, issueId: input.issueId!,
            });
            try {
              return await recordInput.run(reportProgress);
            } finally {
              // Join the underlying callback, even if the recorder's timeout
              // already fired. A deadline cannot prove physical export stopped.
              liveNativeFinalizationOperations.delete(id);
            }
          };
          const settle = async () => {
            if (!timeoutMs || timeoutMs <= 0) return await runWithActivity();
            const timeout = new Promise<never>((_resolve, reject) => {
              timeoutTimer = setTimeout(
                () => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)),
                timeoutMs,
              );
              timeoutTimer.unref?.();
            });
            return await Promise.race([runWithActivity(), timeout]);
          };

          try {
            const result = await settle();
            await append("system", result.system ?? null);
            await append("stdout", result.stdout ?? null);
            await append("stderr", result.stderr ?? null);
            const finalized = await logStore.finalize(handle);
            const finishedAt = new Date();
            const row = await writeState(database => database
              .update(workspaceOperations)
              .set({
                executionWorkspaceId,
                status: result.status ?? "succeeded",
                exitCode: result.exitCode ?? null,
                stdoutExcerpt: stdoutExcerpt || null,
                stderrExcerpt: stderrExcerpt || null,
                logBytes: finalized.bytes,
                logSha256: finalized.sha256,
                logCompressed: finalized.compressed,
                metadata: redactCurrentUserValue(
                  combineMetadata(currentMetadata, result.metadata),
                  currentUserRedactionOptions,
                ) as Record<string, unknown> | null,
                finishedAt,
                updatedAt: finishedAt,
              })
              .where(eq(workspaceOperations.id, id))
              .returning()
              .then((rows) => rows[0] ?? null));
            if (!row) throw notFound("Workspace operation not found");
            return toWorkspaceOperation(row);
          } catch (error) {
            // The log file is best-effort once the operation has failed: a log write error (for
            // example ENOSPC on a full disk) must neither strand this row in `running` nor replace
            // the operation's own error. The excerpt still records the message.
            await append("stderr", error instanceof Error ? error.message : String(error)).catch(() => undefined);
            const finalized = await logStore.finalize(handle).catch(() => null);
            const finishedAt = new Date();
            await writeState(database => database
              .update(workspaceOperations)
              .set({
                executionWorkspaceId,
                status: "failed",
                stdoutExcerpt: stdoutExcerpt || null,
                stderrExcerpt: stderrExcerpt || null,
                logBytes: finalized?.bytes ?? null,
                logSha256: finalized?.sha256 ?? null,
                logCompressed: finalized?.compressed ?? false,
                // Only managed controls carry a failure reason; other phases keep the metadata
                // they were recorded with.
                ...(runtimeControlAction
                  ? {
                      metadata: redactCurrentUserValue(
                        combineMetadata(currentMetadata, {
                          failureReason: error instanceof WorkspaceOperationTimeoutError
                            ? "runtime_control_timeout"
                            : "runtime_control_error",
                        }),
                        currentUserRedactionOptions,
                      ) as Record<string, unknown> | null,
                    }
                  : {}),
                finishedAt,
                updatedAt: finishedAt,
              })
              .where(eq(workspaceOperations.id, id)));
            throw error;
          } finally {
            // Releasing the in-process claim before the request unwinds is what lets the very
            // next control call proceed, and lets recovery treat a leftover row as abandoned.
            if (heartbeatTimer) clearInterval(heartbeatTimer);
            if (timeoutTimer) clearTimeout(timeoutTimer);
            liveRuntimeControlOperationIds.delete(id);
          }
        },
      };
    },

    listForRun: async (runId: string, executionWorkspaceId?: string | null) => {
      const conditions = [eq(workspaceOperations.heartbeatRunId, runId)];
      if (executionWorkspaceId) {
        const cleanupCondition = and(
          eq(workspaceOperations.executionWorkspaceId, executionWorkspaceId)!,
          isNull(workspaceOperations.heartbeatRunId),
        )!;
        if (cleanupCondition) conditions.push(cleanupCondition);
      }

      const rows = await db
        .select()
        .from(workspaceOperations)
        .where(conditions.length === 1 ? conditions[0]! : or(...conditions)!)
        .orderBy(asc(workspaceOperations.startedAt), asc(workspaceOperations.createdAt), asc(workspaceOperations.id));

      return rows.map(toWorkspaceOperation);
    },

    listForExecutionWorkspace: async (executionWorkspaceId: string) => {
      const rows = await db
        .select()
        .from(workspaceOperations)
        .where(eq(workspaceOperations.executionWorkspaceId, executionWorkspaceId))
        .orderBy(desc(workspaceOperations.startedAt), desc(workspaceOperations.createdAt));
      return rows.map(toWorkspaceOperation);
    },

    readLog: async (operationId: string, opts?: { offset?: number; limitBytes?: number }) => {
      const operation = await getById(operationId);
      if (!operation) throw notFound("Workspace operation not found");
      if (!operation.logStore || !operation.logRef) throw notFound("Workspace operation log not found");

      const result = await logStore.read(
        {
          store: operation.logStore as "local_file",
          logRef: operation.logRef,
        },
        opts,
      );

      return {
        operationId,
        store: operation.logStore,
        logRef: operation.logRef,
        ...result,
        // Workspace-operation log chunks are sanitized before append-time storage.
        // Returning the stored chunk avoids another whole-string rewrite per poll.
        content: result.content,
      };
    },
  };
}

export { toWorkspaceOperation };