PaperClipAI/tests/runner-e2e/launch.ts

import { appendPrivateAttemptDiagnostic, readPrivateAttemptObservation, type PrivateAttemptDiagnosticInput } from "./private-attempt-diagnostic.js";
import { withholdPrivateScreenshots } from "./screenshot-privacy.js";
import { copyHealthHelloDetail } from "./chat-readiness-detail.js";
import { copyRetainedSourceSdk } from "./source-sdk-evidence.js";
import { prepareChatRemoteHost } from "./chat-remote-host.js";
import { POSTGRES_ARTIFACT_ENV, POSTGRES_IMAGE_ID_ENV } from "./docker-postgres.js";
import { prepareOwnedPublicOrigin, PublicOriginCleanupError, PUBLIC_ORIGIN_CONTROL_ENV, PUBLIC_ORIGIN_RECEIPT_ENV } from "./chat-public-origin.js";
import { DATABASE_LANE_ENV, DOCKER_DATABASE_ENV, dockerDatabaseLaunchState, publicDockerDatabaseIdentity, selectedDatabaseLane, startOwnedDockerDatabase } from "./docker-postgres.js";
import { ARCHIVE_CONTROL_ENV, ARCHIVE_SCENARIO, startArchiveFaultControl } from "./chat-archive-fault-control.js";
import { abnormalFinalizationResult } from "./abnormal-finalization.js";
import { CHAT_ORACLE_KEY_ENV } from "./chat-private-oracle.js";
import { stagePublishedChatInstall, ChatInstalledCleanupError } from "./chat-installed-lane.js";
import { stageChatPrivateInputs, chatLaunchCredentialNames, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js";
import { createPrivateDiagnosticStream, PRIVATE_DIAGNOSTICS_ENV, readPrivateDiagnosticSecrets, stagePrivateDiagnostics } from "./private-diagnostics.js";
import { sanitizePrivateReporterArtifacts } from "./private-reporter-artifacts.js";
import { findChatCatalogCase } from "./chat-catalog-registry.js";
import { readUpgradeArtifacts, verifyUpgradeArtifact, upgradeArtifactKey } from "./upgrade-artifacts.js";
import { recoverWorkerTiming } from "./worker-timing.js";
import { openEvalOperations, type EvalOperations } from "./operations.js";
import { readRecoveredFeedback } from "./feedback-recovery.js";
import { readInterruptedAttemptTrace } from "./trace-recovery.js";
import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js";
import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";
import { runnerE2EPlaywrightCaseTimeoutMs, runnerE2EPlaywrightInvocation, runnerE2EPlaywrightShutdownSignal, runnerE2EPlaywrightWatchdogMs } from "./web-server-command.js";
import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js";
import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
import { createRunnerE2ETemporaryRoot } from "./server-config.js";
import { createExternalControllerDatabase } from "./external-controller-database.js";
import { createHash, randomBytes } from "node:crypto";
import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
import { spawn } from "node:child_process";
import { createWriteStream } from "node:fs";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import {
  access,
  chmod,
  cp,
  lstat,
  mkdir,
  readFile,
  realpath,
  readdir,
  rm,
  symlink,
  writeFile,
} from "node:fs/promises";
import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js";
import { renderRunnerE2EDashboard } from "./dashboard.js";
import { packageEvidence, publishRunnerE2EAttemptEvidence } from "./evidence.js";
import { prepareRunnerE2ECampaignDirectory, resolveRunnerE2EResultsRoot } from "./results-root.js";
import { privateCellEvidenceOwner } from "./manual-private-report-assets.js";
import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js";
import { requiresPrivateManualEvidence } from "./manual-private-retention.js";
import { copySharedEvidence } from "./shared-evidence.js";
import { classifyFailure } from "./failure-classifier.js";
import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js";
import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js";
import { buildRunnerCampaign } from "./history.js";
import { readEvalOperationsReport } from "./operations-report.js";
import {
  applyEvalGatewayEnvironment,
  buildRunnerE2EProcessEnvironment,
  resolvePaperclipRemoteRunnerBinaryForHarness,
  resolvePaperclipRunnerBinaryForHarness,
} from "./harness-env.js";
import { assertRunnerDatabaseIsolation } from "./instance-isolation.js";
import {
  assertSecretFree,
  findSecretLeakInDirectory,
  isEphemeralCodexRuntimeAuthFile,
  isEphemeralPostgresScanFile,
  normalizedSecrets,
  secretScanFailureEvidence,
  sanitizeJson,
} from "./redaction.js";
import {
  buildMatrixJobs,
  parseRunnerSelectors,
  RunnerSelectorError,
  selectRunnerExecutions,
} from "./selectors.js";
import { resolveRunnerE2ESource } from "./source.js";
import {
  CREDENTIAL_NAMES,
  type MatrixExecution,
  type RunnerE2EResult,
} from "./types.js";
import { assertRemoteNativeEvidencePrerequisites, assertRunnerE2EPrerequisites } from "./prerequisites.js";
import { assertNativeCompletionSelection, prepareNativeCompletionPreflight, NATIVE_COMPLETION_PREFLIGHT_ENV } from "./native-completion-admission.js";
import { assertNativeInstructionSelection, prepareNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE } from "./native-instruction-consolidation.js";
import { prepareStockHarnessPreflight, STOCK_PREFLIGHT_ENV } from "./stock-harness-admission.js";

import {
  reapNewDetachedDarwinSharedMemory,
  snapshotDarwinSharedMemory,
} from "./shared-memory.js";
import { reserveRunnerE2EServerPort } from "./ports.js";
import {
  createResultExitGuard,
  enforceResultProcessIntegrity,
} from "./result-exit-guard.js";
import {
  observeDescendantProcessTree,
  type ObservedProcessGroup,
  readProcessTable,
} from "./process-tree.js";

const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local");
let resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results");
const manualEvidenceSecrets = new Set<string>();
let manualPrivateHandoff: ManualPrivateHandoff | undefined;
let manualDiagnostic: PrivateAttemptDiagnosticInput | undefined;
const activeProcessGroups = new Set<number>();
const activeProcessCleanup = new Map<number, Promise<string | null>>();
const activeProcessTerminators = new Map<number, () => void>();
const completedResultExitGraceMs = 120_000;
let cancelled = false;

function cleanId(value: string) {
  const result = value
    .replace(/[^A-Za-z0-9_.-]+/g, "-")
    .replace(/^-+|-+$/g, "");
  if (!result)
    throw new Error(
      `Invalid empty identifier derived from ${JSON.stringify(value)}`,
    );
  return result;
}

function secret(bytes = 32) {
  return randomBytes(bytes).toString("base64url");
}

async function makeDirectoryTreeRemovable(directory: string): Promise<void> {
  await chmod(directory, 0o700);
  const entries = await readdir(directory, { withFileTypes: true });
  await Promise.all(
    entries
      .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink())
      .map((entry) =>
        makeDirectoryTreeRemovable(path.join(directory, entry.name)),
      ),
  );
}

function stopProcessGroup(pid: number, signal: NodeJS.Signals = "SIGTERM") {
  try {
    if (process.platform === "win32") process.kill(pid, signal);
    else process.kill(-pid, signal);
  } catch {
    // The process tree already exited.
  }
}

function processGroupIsAlive(pid: number) {
  try {
    process.kill(process.platform === "win32" ? pid : -pid, 0);
    return true;
  } catch {
    return false;
  }
}

async function terminateProcessGroup(pid: number) {
  stopProcessGroup(pid, "SIGTERM");
  const gracefulDeadline = Date.now() + 5_000;
  while (processGroupIsAlive(pid) && Date.now() < gracefulDeadline) {
    await new Promise((resolve) => setTimeout(resolve, 50));
  }
  if (!processGroupIsAlive(pid)) return null;
  stopProcessGroup(pid, "SIGKILL");
  const forcedDeadline = Date.now() + 5_000;
  while (processGroupIsAlive(pid) && Date.now() < forcedDeadline) {
    await new Promise((resolve) => setTimeout(resolve, 50));
  }
  return processGroupIsAlive(pid)
    ? `Paperclip/Playwright process group ${pid} survived SIGKILL`
    : null;
}

function wait(milliseconds: number) {
  return new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
}

interface ProcessTreeDiagnostic {
  summary: string;
  groups: ObservedProcessGroup[];
}

async function processTreeDiagnostic(
  rootPid: number,
): Promise<ProcessTreeDiagnostic> {
  const table = await readProcessTable();
  if (!table) {
    return {
      summary: `process tree ${rootPid} (member inspection unavailable)`,
      groups: [],
    };
  }
  const observed = observeDescendantProcessTree(table, rootPid);
  const members = observed.members
    .slice(0, 64)
    .map(
      ({ process: candidate, depth }) =>
        `pid=${candidate.pid} ppid=${candidate.parentPid} pgid=${candidate.processGroupId} depth=${depth} kind=${candidate.kind}`,
    );
  const descendantGroupIds = observed.groups
    .filter((group) => group.processGroupId !== rootPid)
    .map((group) => group.processGroupId);
  return {
    summary:
      members.length > 0
        ? `process tree ${rootPid}: ${members.join("; ")}; descendant pgids=${descendantGroupIds.join(",") || "none"}`
        : `process tree ${rootPid}: no members reported`,
    groups: observed.groups,
  };
}

for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
  process.on(signal, () => {
    cancelled = true;
    for (const pid of activeProcessGroups) {
      const terminate = activeProcessTerminators.get(pid);
      if (terminate) {
        terminate();
        continue;
      }
      activeProcessCleanup.set(pid, terminateProcessGroup(pid));
    }
  });
}

async function loadLocalEnvironment(target: NodeJS.ProcessEnv) {
  const contents = await readFile(localEnvPath, "utf8").catch(
    (error: NodeJS.ErrnoException) => {
      if (error.code === "ENOENT") return null;
      throw error;
    },
  );
  if (contents === null) return;
  for (const [index, rawLine] of contents.split(/\r?\n/).entries()) {
    const line = rawLine.trim();
    if (!line || line.startsWith("#")) continue;
    const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
    if (!match)
      throw new Error(
        `Invalid ${path.basename(localEnvPath)} line ${index + 1}`,
      );
    if (target[match[1]] !== undefined) continue;
    let value = match[2].trim();
    if (
      (value.startsWith('"') && value.endsWith('"')) ||
      (value.startsWith("'") && value.endsWith("'"))
    ) {
      value = value.slice(1, -1);
    }
    target[match[1]] = value;
  }
}

async function prepareProviderPath(
  temporaryRoot: string,
  inheritedPath: string | undefined,
) {
  const toolBin = path.join(temporaryRoot, "provider-bin");
  await mkdir(toolBin, { recursive: true });
  const runnerRequire = createRequire(
    path.join(repositoryRoot, "packages/paperclip-runner/package.json"),
  );
  const codexAcpPackage = runnerRequire.resolve(
    "@agentclientprotocol/codex-acp/package.json",
  );
  const codexRequire = createRequire(codexAcpPackage);
  const codexPackage = codexRequire.resolve("@openai/codex/package.json");
  const codexManifest = JSON.parse(await readFile(codexPackage, "utf8")) as {
    bin?: string | Record<string, string>;
  };
  const codexBin =
    typeof codexManifest.bin === "string"
      ? codexManifest.bin
      : codexManifest.bin?.codex;
  if (!codexBin)
    throw new Error(
      "Production Codex ACP dependency does not expose its pinned Codex executable",
    );
  await symlink(
    path.resolve(path.dirname(codexPackage), codexBin),
    path.join(toolBin, "codex"),
  );
  const packageBin = path.join(
    repositoryRoot,
    "packages/paperclip-runner/node_modules/.bin",
  );
  return [toolBin, packageBin, inheritedPath]
    .filter(Boolean)
    .join(path.delimiter);
}

async function runProcess(
  invocation: { command: string; args: string[] },
  env: NodeJS.ProcessEnv,
  timeoutMs: number | null,
  logPath: string,
  completionPaths: readonly string[],
  interactive: boolean,
  secrets: () => readonly string[],
) {
  const log = createWriteStream(logPath, { flags: "a", mode: 0o600 });
  const child = spawn(invocation.command, invocation.args, {
    cwd: repositoryRoot,
    env,
    stdio: ["inherit", "pipe", "pipe"],
    detached: process.platform !== "win32",
  });
  if (!child.pid) throw new Error("Failed to start Playwright");
  activeProcessGroups.add(child.pid);
  const processOwner = createProcessTreeOwner(child);
  let outputTail = "";
  const recordOutput = (safe: string, destination: NodeJS.WriteStream) => {
    destination.write(safe);
    log.write(safe);
    outputTail += safe;
    if (outputTail.length > 1024 * 1024)
      outputTail = outputTail.slice(-1024 * 1024);
  };
  const stdout = createPrivateDiagnosticStream(secrets, safe => recordOutput(safe, process.stdout));
  const stderr = createPrivateDiagnosticStream(secrets, safe => recordOutput(safe, process.stderr));
  child.stdout?.on("data", (chunk: Buffer) => stdout.write(chunk));
  child.stderr?.on("data", (chunk: Buffer) => stderr.write(chunk));
  let childSettled = false;
  let postResultStallError: string | null = null;
  let boundedCleanup: Promise<string | null> | undefined;
  let cleanupSettled!: () => void;
  const cleanupFinished = new Promise<number>(resolve => { cleanupSettled = () => resolve(1); });
  const stopChildTree = (_diagnostic?: ProcessTreeDiagnostic) => {
    if (boundedCleanup) return;
    boundedCleanup = stopOwnedProcessTree(child, processOwner, 45_000, 5_000, runnerE2EPlaywrightShutdownSignal)
      .then(() => null, error => error instanceof Error ? error.message : String(error))
      .finally(cleanupSettled);
    activeProcessCleanup.set(child.pid!, boundedCleanup);
  };
  activeProcessTerminators.set(child.pid, stopChildTree);
  const resultExitGuard = createResultExitGuard({
    resultPaths: completionPaths,
    interactive,
    graceMs: completedResultExitGraceMs,
    now: Date.now,
    pathExists: (candidate) =>
      access(candidate).then(
        () => true,
        () => false,
      ),
    onExpired: async () => {
      const diagnostic = await processTreeDiagnostic(child.pid!);
      if (childSettled) return;
      postResultStallError = `Playwright remained alive for ${completedResultExitGraceMs}ms after every result was written`;
      stderr.write(Buffer.from(`\n${postResultStallError}; forcing bounded cleanup. ${diagnostic.summary}\n`));
      stopChildTree(diagnostic);
    },
  });
  const completionPoll = resultExitGuard.enabled
    ? setInterval(() => {
        void resultExitGuard.poll().catch(() => {
          if (childSettled || postResultStallError) return;
          postResultStallError =
            "Completed-result exit guard failed during bounded inspection";
          stderr.write(Buffer.from(`\n${postResultStallError}; forcing cleanup.\n`));
          stopChildTree();
        });
      }, 500)
    : undefined;
  completionPoll?.unref();
  let timedOut = false;
  const timer =
    timeoutMs === null
      ? undefined
      : setTimeout(() => {
          timedOut = true;
          stopChildTree();
        }, timeoutMs);
  timer?.unref();
  let spawnError: string | null = null;
  const childExit = new Promise<number>((resolve, reject) => {
    child.once("error", reject);
    child.once("exit", (code) => {
      childSettled = true;
      resolve(code ?? 1);
    });
  }).catch((error) => {
    childSettled = true;
    spawnError = error instanceof Error ? error.message : String(error);
    return 1;
  });
  const exitCode = await Promise.race([childExit, cleanupFinished]);
  if (timer) clearTimeout(timer);
  if (completionPoll) clearInterval(completionPoll);
  // Even a successful launcher exit can leave an already-observed detached
  // server behind. Retire that retained tree, never only the root group.
  stopChildTree();
  const processCleanupError = await boundedCleanup!;
  processOwner.stopObserving();
  // Even a failed direct-child kill must not hold cancellation forever or let
  // inherited pipes write into an ended log. The cleanup error remains fatal.
  if (processCleanupError) {
    child.stdout?.destroy();
    child.stderr?.destroy();
    if (child.exitCode === null && child.signalCode === null) child.unref();
  }
  if (child.pid) {
    activeProcessGroups.delete(child.pid);
    activeProcessCleanup.delete(child.pid);
    activeProcessTerminators.delete(child.pid);
  }
  stdout.end(); stderr.end();
  await new Promise<void>((resolve) => log.end(resolve));
  return {
    exitCode,
    timedOut,
    postResultStallError,
    processCleanupError,
    spawnError,
    outputTail,
    privateDiagnosticFailure: stdout.failed() || stderr.failed(),
    privateDiagnosticLeak: stdout.redactedSecret() || stderr.redactedSecret(),
  };
}

function syntheticResult(
  execution: MatrixExecution,
  attempt: number,
  startedAtMs: number,
  error: string,
  failureClass: RunnerE2EResult["failureClass"] = "transient_infrastructure",
): RunnerE2EResult {
  const finishedAtMs = Date.now();
  return {
    schema: "paperclip.runner-e2e.result/v2",
    executionId: execution.id,
    suiteId: execution.suite.id,
    suiteDefinitionHash: execution.suiteDefinitionHash,
    source: resolveRunnerE2ESource(),
    ...(execution.profile.ranking
      ? { rankingSnapshot: execution.profile.ranking }
      : {}),
    attempt,
    status: "failed",
    failureClass,
    error,
    profileId: execution.profile.id,
    environmentId: execution.environment.id,
    caseId: execution.task.id,
    provider: execution.profile.provider,
    model: execution.profile.model,
    runtimeMode: execution.profile.expectedRuntimeMode,
    startedAt: new Date(startedAtMs).toISOString(),
    finishedAt: new Date(finishedAtMs).toISOString(),
    durationMs: finishedAtMs - startedAtMs,
    cleanup: "not_started",
  };
}

async function readResult(resultPath: string, fallback: RunnerE2EResult) {
  try {
    return JSON.parse(await readFile(resultPath, "utf8")) as RunnerE2EResult;
  } catch {
    return fallback;
  }
}


async function runAttempt(input: {
  executions: readonly MatrixExecution[];
  attempt: number;
  campaignId: string;
  options: ReturnType<typeof parseRunnerSelectors>;
  operations?: EvalOperations;
}): Promise<RunnerE2EResult[]> {
  const { executions, attempt, campaignId, options } = input;
  const accountingCampaignId = input.operations?.manifest.limits.campaignId ?? process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? campaignId;
  const accountingAttemptId = randomBytes(16).toString("hex");
  const traceConfig = evalBraintrustConfig({ ...process.env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined });
  const execution = executions[0];
  if (!execution) throw new Error("A runner E2E cell must contain a task case");
  if (
    executions.some(
      (candidate) =>
        candidate.profile.id !== execution.profile.id ||
        candidate.environment.id !== execution.environment.id,
    )
  ) {
    throw new Error(
      "One isolated runner E2E harness cannot mix profiles or environments",
    );
  }
  if (process.env.GITHUB_ACTIONS === "true" && !input.operations && executions.some(candidate => requiresPrivateManualEvidence(candidate.id)) && !manualPrivateHandoff) throw new Error("Sensitive manual evidence requires verified restricted retention before setup");
  await prepareRunnerE2ECampaignDirectory(resultsRoot, campaignId);
  const upgrades = await readUpgradeArtifacts(process.env[upgradeArtifactKey]);
  if (upgrades) {
    // All attempts and the browser use A's installed fixture admission. The
    // supervisor changes only the server artifact; the original sandbox lives on.
    Object.assign(process.env, upgrades.artifacts.A.pins);
    for (const artifact of [upgrades.artifacts.A, upgrades.artifacts.B]) {
      await verifyUpgradeArtifact(artifact, process.env, executions);
    }
  }
  assertInstalledCliSelection(process.env, executions);
  let installedCli = await verifyInstalledCli(process.env, executions);
  const selectedDescriptor = execution.suite.id === "chat-features" ? findChatCatalogCase(execution.task.id) : undefined;
  let sourceEnvironment = { ...process.env };
  delete sourceEnvironment[CHAT_ORACLE_KEY_ENV];
  const databaseLane = selectedDatabaseLane(sourceEnvironment);
  const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions);
  const startedAtMs = Date.now();
  const sharedMemoryBaseline = snapshotDarwinSharedMemory();
  const temporaryParent = await realpath(os.tmpdir());
  const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent);
  const selectedPrivateInputsDirectory = path.join(temporaryRoot, "selected-private-inputs");
  const privateDir = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "capture") : path.join(temporaryRoot, "artifacts-private");
  const publishedResults: RunnerE2EResult[] = [];
  const publishedResultPaths = new Map<string, string>();
  let attemptSecrets: string[] = [];
  let processCleanupFailed = false;
  let startupReceiptPath: string | undefined;
  let dockerDatabase: Awaited<ReturnType<typeof startOwnedDockerDatabase>> | undefined;
  let ownedPublicOrigin: Awaited<ReturnType<typeof prepareOwnedPublicOrigin>>;
  let archiveControl: Awaited<ReturnType<typeof startArchiveFaultControl>> | undefined;
  let externalDatabase: Awaited<ReturnType<typeof createExternalControllerDatabase>> | undefined;
  let gateway: Awaited<ReturnType<EvalOperations["start"]>> | undefined;
  const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = [];
  try {
    const paperclipHome = path.join(temporaryRoot, "paperclip-home");
    const workspace = path.join(temporaryRoot, "workspace");
    const instanceId = `runner-e2e-${randomBytes(8).toString("hex")}`;
    const configPath = path.join(
      paperclipHome,
      "instances",
      instanceId,
      "config.json",
    );
    const port = await reserveRunnerE2EServerPort();
    if (process.env.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL) {
      externalDatabase = await createExternalControllerDatabase({
        adminUrl: process.env.PAPERCLIP_RUNNER_E2E_POSTGRES_ADMIN_URL, temporaryRoot, configPath, serverPort: port,
      });
    }
    await Promise.all([
      mkdir(paperclipHome, { recursive: true, mode: 0o700 }),
      mkdir(workspace, { recursive: true }),
      mkdir(privateDir, { recursive: true }),
    ]);
    if (selectedDescriptor?.installedSource === "published-global-npm") {
      if (executions.length !== 1) throw new Error("Published chat source requires one explicit operation");
      try { sourceEnvironment = { ...sourceEnvironment, ...await stagePublishedChatInstall({ execution, root: temporaryRoot, env: sourceEnvironment }) }; }
      catch (error) { if (error instanceof ChatInstalledCleanupError) processCleanupFailed = true; throw error; }
      installedCli = await verifyInstalledCli(sourceEnvironment, executions);
    }
    const providerPath = await prepareProviderPath(
      temporaryRoot,
      process.env.PATH,
    );
    if (requiresCodexCiSandbox(execution)) {
      await prepareCodexCiSandbox(repositoryRoot, temporaryRoot);
    }
    const agentJwtSecret = secret(48);
    const decisionSigningSecret = secret(48);
    const toolActionSigningSecret = secret(48);
    const betterAuthSecret = secret(48);
    gateway = await input.operations?.start(execution, accountingAttemptId, attempt);
    const credentialSources = {
      PAPERCLIP_E2E_GATEWAY_TOKEN: gateway?.token,
      PAPERCLIP_E2E_OTLP_RELAY_TOKEN: gateway?.observability?.token,
      ...Object.fromEntries([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => [name, process.env[name]])),
      PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,
      PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,
      PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
      BETTER_AUTH_SECRET: betterAuthSecret,
    };
    const credentials = normalizedSecrets(Object.values(credentialSources));
    attemptSecrets = credentials;
    const needsPrivateInputs = selectedDescriptor && ((selectedDescriptor.requiredCredentials?.length ?? 0) > 0 || selectedDescriptor.externalInput);
    if (needsPrivateInputs && executions.length !== 1) throw new Error("Private case inputs require one explicitly selected execution");
    await mkdir(selectedPrivateInputsDirectory, { mode: 0o700 });
    const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir: selectedPrivateInputsDirectory, env: process.env, secrets: credentials }) : undefined;
    ownedPublicOrigin = await prepareOwnedPublicOrigin({ controlDirectory: process.env[PUBLIC_ORIGIN_CONTROL_ENV], executionId: execution.id, scenarioId: selectedDescriptor?.scenarioId ?? "", attemptId: accountingAttemptId, backendOrigin: `http://127.0.0.1:${port}`, temporaryRoot, privateSecrets: credentials });
    delete sourceEnvironment[PUBLIC_ORIGIN_CONTROL_ENV]; delete sourceEnvironment[PUBLIC_ORIGIN_RECEIPT_ENV];
    if (selectedDescriptor?.scenarioId === ARCHIVE_SCENARIO) archiveControl = await startArchiveFaultControl({ selected: executions.map(row => row.id), temporaryRoot, backendOrigin: `http://127.0.0.1:${port}`, privateSecrets: credentials });
    if (databaseLane === "owned-docker-postgres") dockerDatabase = await startOwnedDockerDatabase({ temporaryRoot, privateSecrets: credentials });
    const diagnosticRegistry = stagePrivateDiagnostics(temporaryRoot, executions.map(row => row.id), credentials);
    if (manualPrivateHandoff) for (const credential of credentials) manualEvidenceSecrets.add(credential);
    if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 });
    if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 });
    const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness(
      executions,
      repositoryRoot,
    );
    const childEnv: NodeJS.ProcessEnv = {
      ...buildRunnerE2EProcessEnvironment(sourceEnvironment, executions),
      [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename,
      [PUBLIC_ORIGIN_RECEIPT_ENV]: ownedPublicOrigin?.filename,
      [ARCHIVE_CONTROL_ENV]: archiveControl?.filename,
      [DATABASE_LANE_ENV]: dockerDatabase ? "owned-docker-postgres" : undefined,
      [DOCKER_DATABASE_ENV]: dockerDatabase?.filename,
      [PRIVATE_DIAGNOSTICS_ENV]: diagnosticRegistry,
      PATH: providerPath,
      PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify(
        executions.map((candidate) => candidate.id),
      ),
      PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt),
      PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: accountingCampaignId,
      PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: accountingAttemptId,
      PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined,
      PAPERCLIP_RUNNER_E2E_PUBLIC_MCP: executions.some(candidate => candidate.task.flow === "public_mcp" || candidate.suite.id === "chat-features" && findChatCatalogCase(candidate.task.id)?.authentication === "human-session") ? "1" : "0",
      PAPERCLIP_RUNNER_E2E_PORT: String(port),
      PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot,
      PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir,
      PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace,
      PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(temporaryRoot, "artifacts-private", "server.log"),
      PAPERCLIP_RUNNER_BINARY: runnerBinary,
      PAPERCLIP_RUNNER_REMOTE_BINARY_PATH:
        installedCli ? undefined : resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary),
      // Vite's optimized dependency cache embeds revision query strings. A
      // private per-attempt cache prevents an earlier cell or local rebuild
      // from producing `504 Outdated Optimize Dep` during browser bootstrap.
      PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"),
      PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS: String(
        Math.max(
          ...executions.map(
            (candidate) =>
              runnerE2EPlaywrightCaseTimeoutMs(candidate.task.flow, candidate.task.attemptTimeoutMs[candidate.environment.id]),
          ),
        ),
      ),
      PAPERCLIP_HOME: paperclipHome,
      PAPERCLIP_INSTANCE_ID: instanceId,
      PAPERCLIP_CONFIG: configPath,
      PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,
      PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,
      PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
      BETTER_AUTH_SECRET: betterAuthSecret,
    };
    if (gateway) {
      applyEvalGatewayEnvironment(childEnv, execution.profile.credential, gateway);
      // The fixture/server controller needs Daytona provisioning access. The
      // remote provider still receives only the scoped model and OTLP tokens.
      if (execution.environment.id === "daytona") childEnv.DAYTONA_API_KEY = process.env.DAYTONA_API_KEY;
    }
    // The database URLs are stripped here and again at the Playwright web-server
    // boundary so a developer's shell can never redirect this paid test.
    delete childEnv.DATABASE_URL;
    delete childEnv.DATABASE_MIGRATION_URL;

    const playwrightArgs = [
      "test",
      "--config",
      "tests/runner-e2e/playwright.config.ts",
      ...(options.headed ? ["--headed"] : []),
      ...(options.ui ? ["--ui"] : []),
      ...(options.debug ? ["--debug"] : []),
    ];
    const watchdog =
      options.ui || options.debug
        ? null
        : runnerE2EPlaywrightWatchdogMs(executions);
    const processResult = await runProcess(
      runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs),
      childEnv,
      watchdog,
      path.join(privateDir, "playwright.log"),
      executions.map((candidate) =>
        path.join(privateDir, "cases", candidate.task.id, "result.json"),
      ),
      options.ui || options.debug,
      () => readPrivateDiagnosticSecrets(diagnosticRegistry, temporaryRoot),
    );
    credentials.splice(0, credentials.length, ...readPrivateDiagnosticSecrets(diagnosticRegistry, temporaryRoot));
    processCleanupFailed = processResult.processCleanupError !== null;
    if (options.installedStartupOnly) {
      const proofDir = path.join(resultsRoot, campaignId, "installed-startup-only");
      await mkdir(proofDir, { recursive: true });
      // This is private setup evidence, never a successful provider case/result.
      await cp(privateDir, path.join(proofDir, "private"), { recursive: true });
      const probe = JSON.parse(await readFile(path.join(privateDir, "installed-startup-only.json"), "utf8"));
      if (processResult.exitCode !== 0 || processResult.timedOut || processResult.spawnError || processCleanupFailed || probe.status !== "health_ui_zero_company_passed" || probe.providerCalls !== 0 || probe.qualified !== false) throw new Error("Installed startup-only actual launch chain failed");
      await assertRunnerDatabaseIsolation(configPath, temporaryRoot, dockerDatabase?.filename);
      startupReceiptPath = path.join(proofDir, "receipt.json");
      await writeFile(startupReceiptPath, `${JSON.stringify({ status: "installed_launch_chain_pending_scratch_cleanup", qualified: false, providerCalls: 0, credentialsLoaded: false, installedCli, processResult, probe }, null, 2)}\n`, { mode: 0o600 });
      return [];
    }
    const processFailure = processResult.spawnError
      ? `Playwright failed to start: ${processResult.spawnError}`
      : processResult.timedOut
        ? `Harness process exceeded ${Math.round((watchdog ?? 0) / 1000)} seconds`
        : `Playwright exited ${processResult.exitCode} before writing a result`;
    const processFailureClass =
      processResult.spawnError || processResult.timedOut
        ? "transient_infrastructure"
        : classifyFailure(
            new Error(
              processResult.outputTail
                ? `${processFailure}\n${processResult.outputTail}`
                : processFailure,
            ),
          );
    const results = await Promise.all(
      executions.map(async (candidate) => {
        const resultPath = path.join(
          privateDir,
          "cases",
          candidate.task.id,
          "result.json",
        );
        const fallback = syntheticResult(
          candidate,
          attempt,
          startedAtMs,
          processFailure,
          processFailureClass,
        );
        const result = await readResult(resultPath, fallback);
        if (databaseLane === "owned-docker-postgres") result.databaseIdentity = dockerDatabase ? publicDockerDatabaseIdentity(dockerDatabase.receipt) : (await dockerDatabaseLaunchState(databaseLane, temporaryRoot)).identity;
        if (!result.workerTiming) result.workerTiming = await recoverWorkerTiming(path.join(path.dirname(resultPath), "snapshots", "worker-timing.json"),
          { executionId: candidate.id, attempt, startedAt: result.startedAt, finishedAt: result.finishedAt });
        if (result === fallback) {
          result.feedback = await readRecoveredFeedback(path.dirname(resultPath), {
            campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,
          });
          if (traceConfig) result.tracing = await readInterruptedAttemptTrace(path.dirname(resultPath), {
            projectId: traceConfig.projectId, executionId: candidate.id,
          });
        }
        // Keep cleanup authority before evidence copying/publication can fail.
        rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status });
        const checked = enforceResultProcessIntegrity(result, processResult);
        return processResult.privateDiagnosticFailure ? { ...checked, status: "failed" as const, failureClass: checked.status === "passed" ? "secret_leak" as const : checked.failureClass,
          error: `${checked.error ? `${checked.error}; ` : ""}Private diagnostic redaction was unavailable; raw output was withheld.` } : checked;
      }),
    );
    let isolationError: unknown;
    try {
      await assertRunnerDatabaseIsolation(configPath, temporaryRoot, dockerDatabase?.filename);
    } catch (error) {
      isolationError = error;
    }
    let persistedStateError: unknown;
    let persistedStateFinding: { file: string; stateArea: string; reason: string; matchCategory?: string } | undefined;
    // Onboarding evaluates behavior; credential persistence belongs to a separate layer.
    // Keep artifact redaction/publication checks independent of this filesystem scan.
    const persistenceCheckedExecutions = executions.filter(
      (candidate) => candidate.suite.id !== "first-task",
    );
    if (persistenceCheckedExecutions.length > 0) {
      try {
        const expectedEphemeralCredentials = new Set<string>();
        for (const [label, directory] of [
          ["Paperclip home", paperclipHome],
          ["workspace", workspace],
        ] as const) {
          while (true) {
            // The managed Codex home may legitimately contain upstream source-code
            // fixtures with fake `sk-*` strings. Reject exact campaign credentials.
            const leak = await findSecretLeakInDirectory(directory, credentials, {
              includeShapes: false,
              sources: credentialSources,
              ...(label === "Paperclip home" ? { privateCodexRuntimeIdentityRoot: paperclipHome,
                ...(dockerDatabase ? { privateDockerDatabaseConfig: { temporaryRoot, configPath, receiptPath: dockerDatabase.filename } } : {}),
              } : {}),
              ignoreFile: (file) => expectedEphemeralCredentials.has(file),
              allowDisappearedFile: (file) =>
                label === "Paperclip home" &&
                isEphemeralPostgresScanFile(paperclipHome, file),
            });
            if (!leak) break;
            const isManagedCodexRuntimeAuth =
              label === "Paperclip home" &&
              isEphemeralCodexRuntimeAuthFile(paperclipHome, leak.file);
            if (isManagedCodexRuntimeAuth) {
              const metadata = await lstat(leak.file);
              if (metadata.isFile() && (metadata.mode & 0o777) === 0o600) {
                // Codex CLI API-key mode requires this one runtime auth file. It
                // lives only in the disposable cell root, is never published,
                // must be owner-only, and is removed with the root below.
                expectedEphemeralCredentials.add(leak.file);
                continue;
              }
            }
            persistedStateFinding = { ...leak, stateArea: label, file: path.relative(temporaryRoot, leak.file) };
            throw new Error(`Secret leak in persisted ${label} state at ${persistedStateFinding.file}: ${leak.reason}${leak.matchCategory ? ` (${leak.matchCategory})` : ""}`);
          }
        }
      } catch (error) {
        persistedStateError = error;
      }
    }
    const ownedServerLog = path.join(temporaryRoot, "artifacts-private", "server.log");
    if (manualPrivateHandoff && await access(ownedServerLog).then(() => true, () => false)) {
      await cp(ownedServerLog, path.join(privateDir, "server.log"), { errorOnExist: true, force: false });
    }
    const reporterPrivacy = await sanitizePrivateReporterArtifacts(privateDir, credentials, manualPrivateHandoff
      ? { owner: await privateCellEvidenceOwner(manualPrivateHandoff, execution.id) } : undefined);
    if (processResult.privateDiagnosticLeak) reporterPrivacy.push({ file: "playwright process stdout/stderr", disposition: "redacted", reason: "credential detected before diagnostic emission" });
    for (const [index, candidate] of executions.entries()) {
      let result: RunnerE2EResult = {
        ...results[index],
        accountingIdentity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId },
      };
      if (
        isolationError &&
        result.failureClass !== "cleanup_failure" &&
        result.failureClass !== "secret_leak"
      ) {
        const isolationMessage =
          isolationError instanceof Error
            ? isolationError.message
            : String(isolationError);
        const configWasUnavailableDuringTransientBootstrap =
          result.failureClass === "transient_infrastructure" &&
          typeof isolationError === "object" &&
          isolationError !== null &&
          "code" in isolationError &&
          isolationError.code === "ENOENT";
        result = {
          ...result,
          status: "failed",
          failureClass: configWasUnavailableDuringTransientBootstrap
            ? result.failureClass
            : "permanent_infrastructure",
          error: configWasUnavailableDuringTransientBootstrap
            ? `${result.error}; isolated config could not be inspected after bootstrap failure: ${isolationMessage}`
            : isolationMessage,
        };
      }
      if (persistedStateError && persistenceCheckedExecutions.includes(candidate)) {
        const persistedStateMessage =
          persistedStateError instanceof Error
            ? persistedStateError.message
            : String(persistedStateError);
        const persistedStateClass = classifyFailure(persistedStateError);
        if (
          persistedStateClass === "secret_leak" ||
          (result.failureClass !== "secret_leak" &&
            result.failureClass !== "cleanup_failure")
        ) {
          result = {
            ...result,
            status: "failed",
            failureClass:
              persistedStateClass === "secret_leak"
                ? "secret_leak"
                : "permanent_infrastructure",
            error: persistedStateMessage,
          };
        } else {
          result = {
            ...result,
            error: `${result.error}; persisted-state scan also failed: ${persistedStateMessage}`,
          };
        }
      }
      const casePrivateDir = path.join(privateDir, "cases", candidate.task.id);
      const uploadDir = path.join(
        resultsRoot,
        campaignId,
        candidate.suite.id,
        candidate.profile.id,
        candidate.environment.id,
        candidate.task.id,
        `attempt-${attempt}`,
      );
      await mkdir(casePrivateDir, { recursive: true });
      await copySharedEvidence(privateDir, casePrivateDir, temporaryRoot, credentials);
      if (reporterPrivacy.length) {
        const original = { status: result.status, failureClass: result.failureClass, error: result.error };
        const snapshots = path.join(casePrivateDir, "snapshots"); await mkdir(snapshots, { recursive: true });
        await writeFile(path.join(snapshots, "diagnostic-privacy.json"), `${JSON.stringify(sanitizeJson({ schemaVersion: 1, originalResult: original, issues: reporterPrivacy }, credentials), null, 2)}\n`, { mode: 0o600 });
        result = { ...result, status: "failed", failureClass: result.status === "passed" ? "secret_leak" : result.failureClass,
          error: `${result.error ? `${result.error}; ` : ""}Reporter privacy violation: ${reporterPrivacy.map(issue => `${issue.disposition} ${issue.file}`).join(", ")}` };
      }
      if (persistedStateError && persistenceCheckedExecutions.includes(candidate)) {
        const snapshots = path.join(casePrivateDir, "snapshots");
        await mkdir(snapshots, { recursive: true });
        const original = results[index];
        await writeFile(path.join(snapshots, "persisted-state-scan.json"), `${JSON.stringify(secretScanFailureEvidence({
          scanError: persistedStateError instanceof Error ? persistedStateError.message : String(persistedStateError),
          finding: persistedStateFinding,
          originalResult: { status: original.status, failureClass: original.failureClass, error: original.error },
        }, credentials), null, 2)}\n`, { mode: 0o600 });
      }
      result = withholdPrivateScreenshots(result, {
        persistedStateSecretFinding: !!persistedStateFinding && persistenceCheckedExecutions.includes(candidate),
        reporterEvidenceWithheld: reporterPrivacy.length > 0,
      });
      const resultPath = path.join(casePrivateDir, "result.json");
      if (manualPrivateHandoff) {
        manualDiagnostic = { stage: "package-original", result, observation: await readPrivateAttemptObservation(casePrivateDir) };
        await preserveOriginalFrameworkResult(casePrivateDir);
        await writeFile(
          resultPath,
          `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
          "utf8",
        );
        let evidence = await packageEvidence({
          privateDir: casePrivateDir,
          uploadDir,
          secrets: credentials,
          expectPassScreenshot: result.status === "passed",
          privateCell: { owner: await privateCellEvidenceOwner(manualPrivateHandoff, candidate.id), layout: "original" as const },
        });
        const originalEvidence = evidence, originalFeaturePassed = result.status === "passed";
        manualDiagnostic!.evidence = originalEvidence;
        if (evidence.leaks.length > 0 || evidence.missing.length > 0) {
          result = withholdPrivateScreenshots(result, { packagedEvidenceSecretFinding: evidence.leaks.length > 0 });
          result = {
            ...result,
            status: "failed",
            failureClass:
              evidence.leaks.length > 0
                ? "secret_leak"
                : "permanent_infrastructure",
            error:
              evidence.leaks.length > 0
                ? `Secret leak rejected from evidence: ${evidence.leaks.map((leak) => leak.file).join(", ")}`
                : `Required evidence missing: ${evidence.missing.join(", ")}`,
          };
          await writeFile(
            resultPath,
            `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
            "utf8",
          );
          evidence = await packageEvidence({
            privateDir: casePrivateDir,
            uploadDir,
            secrets: credentials,
            expectPassScreenshot: false,
            privateCell: { owner: await privateCellEvidenceOwner(manualPrivateHandoff, candidate.id), layout: "original" as const },
          });
        }
        manualDiagnostic!.stage = "seal-original";
        await sealStandardOriginalEvidence({ context: manualPrivateHandoff,
          executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed });
      } else {
        result = await publishRunnerE2EAttemptEvidence({
          result,
          resultsRoot,
          privateDir: casePrivateDir,
          uploadDir,
          secrets: credentials,
        });
        if (result.failureClass === "secret_leak") {
          result = withholdPrivateScreenshots(result, { packagedEvidenceSecretFinding: true });
          const safeResult = `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`;
          await writeFile(resultPath, safeResult, "utf8");
          await writeFile(path.join(uploadDir, "result.json"), safeResult, "utf8");
        }
      }
      await copyHealthHelloDetail(temporaryRoot, candidate.id, uploadDir, credentials);
      await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, credentials, repositoryRoot);
      console.log(
        `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`,
      );
      publishedResults.push(result);
      publishedResultPaths.set(
        candidate.id,
        path.join(uploadDir, "result.json"),
      );
    }
    return [...publishedResults];
  } catch (error) {
    if (manualDiagnostic) manualDiagnostic.error = error;
    if (error instanceof PublicOriginCleanupError) processCleanupFailed = true;
    // Packaging/diagnostic failures must not erase an actually launched attempt
    // or its primary startup failure. Retain only closed sanitized metadata;
    // raw reports stay private when their normal packaging path failed.
    for (const candidate of executions) {
      if (publishedResultPaths.has(candidate.id)) continue;
      const casePrivateDir = path.join(privateDir, "cases", candidate.task.id);
      const fallback = syntheticResult(candidate, attempt, startedAtMs, "Attempt terminated before a worker result was retained");
      const original = await readResult(path.join(casePrivateDir, "result.json"), fallback);
      if (manualPrivateHandoff) {
        manualDiagnostic ??= { stage: "execution", result: original, observation: await readPrivateAttemptObservation(casePrivateDir) };
        manualDiagnostic.error ??= error;
      }
      const result = abnormalFinalizationResult(original, error);
      if (databaseLane === "owned-docker-postgres") result.databaseIdentity = dockerDatabase ? publicDockerDatabaseIdentity(dockerDatabase.receipt) : (await dockerDatabaseLaunchState(databaseLane, temporaryRoot)).identity;
      const uploadDir = path.join(resultsRoot, campaignId, candidate.suite.id, candidate.profile.id, candidate.environment.id, candidate.task.id, `attempt-${attempt}`);
      await mkdir(uploadDir, { recursive: true });
      const resultPath = path.join(uploadDir, "result.json");
      const safe = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`;
      assertSecretFree(safe, attemptSecrets, resultPath);
      await writeFile(resultPath, safe, "utf8");
      await writeFile(path.join(uploadDir, "abnormal-finalization.json"), `${JSON.stringify(sanitizeJson({
        schemaVersion: 1, executionId: candidate.id, originalResult: original,
        packaging: "failed; raw diagnostics retained privately", error: error instanceof Error ? error.message : String(error),
      }, attemptSecrets), null, 2)}\n`, { mode: 0o600 });
      await copyRetainedSourceSdk(temporaryRoot, candidate.id, result.source?.sha ?? null, uploadDir, attemptSecrets, repositoryRoot).catch(() => {});
      if (rawCleanupResults.length < executions.length) rawCleanupResults.push({ cleanup: original.cleanup, synthetic: original === fallback, status: original.status });
      publishedResults.push(result); publishedResultPaths.set(candidate.id, resultPath);
    }
    return [...publishedResults];
  } finally {
    // runProcess has completed its owned Playwright/server process shutdown.
    let publicOriginCleanupError: unknown;
    try { await ownedPublicOrigin?.close(); } catch (error) { publicOriginCleanupError = error; processCleanupFailed = true; }
    let archiveCleanupError: unknown;
    try { await archiveControl?.close(); } catch (error) { archiveCleanupError = error; processCleanupFailed = true; }
    let dockerCleanupError: unknown;
    try { await dockerDatabase?.close(); } catch (error) { dockerCleanupError = error; processCleanupFailed = true; }
    const databaseFinalization = await dockerDatabaseLaunchState(databaseLane, temporaryRoot);
    if (databaseFinalization.cleanupUnconfirmed) processCleanupFailed = true;
    if (databaseFinalization.identity) for (const result of publishedResults) result.databaseIdentity = databaseFinalization.identity;
    let gatewayCleanupError: unknown;
    try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; }
    if (!processCleanupFailed && databaseFinalization.allowDarwinReap) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline);
    let cleanupError: unknown;
    // Raw selected keys are never diagnostic/archive artifacts, even when recovery state is retained.
    try { await rm(selectedPrivateInputsDirectory, { recursive: true, force: true }); }
    catch { cleanupError = new Error("Selected private credential cleanup failed; contents withheld"); }
    const resourceAdmissionStarted = await access(path.join(privateDir, "resource-admission-started"))
      .then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT");
    const preserveRecovery = mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults });
    if (!preserveRecovery) {
      try { await externalDatabase?.cleanup(); } catch (error) { cleanupError = error; }
    }
    if (preserveRecovery || cleanupError) {
      // Remote allocation cleanup is journaled in this instance's database.
      // Deleting it after the controller exits would strand uncertain creates.
      await chmod(temporaryRoot, 0o700);
      cleanupError = new Error(`Preserving private recovery state after unconfirmed cleanup: ${temporaryRoot}`);
    } else if (shouldKeepFailedDiagnostics({
      enabled: process.env.PAPERCLIP_RUNNER_E2E_KEEP_FAILED_PRIVATE === "1",
      expectedResults: executions.length,
      results: publishedResults,
    })) {
      // Explicit diagnosis only. Confirmed resource cleanup stays confirmed;
      // keep private traces for investigation without publishing provider data.
      await chmod(temporaryRoot, 0o700);
      console.warn(`Retained private failed-case diagnostics: ${temporaryRoot}`);
    } else if (
      temporaryRoot.startsWith(`${temporaryParent}${path.sep}paperclip-runner-e2e-`)
    ) {
      for (let cleanupAttempt = 1; cleanupAttempt <= 3; cleanupAttempt += 1) {
        try {
          await rm(temporaryRoot, { recursive: true, force: true });
          cleanupError = undefined;
          break;
        } catch (error) {
          cleanupError = error;
          if (cleanupAttempt < 3) {
            await makeDirectoryTreeRemovable(temporaryRoot).catch(() => {});
            await new Promise((resolve) =>
              setTimeout(resolve, cleanupAttempt * 250),
            );
          }
        }
      }
    } else {
      cleanupError = new Error(
        `Refusing to remove unexpected temporary path ${temporaryRoot}`,
      );
    }
    if (gatewayCleanupError || archiveCleanupError || dockerCleanupError || publicOriginCleanupError) cleanupError = new AggregateError([cleanupError, gatewayCleanupError, archiveCleanupError, dockerCleanupError, publicOriginCleanupError].filter(Boolean), "Eval owned gateway/transport cleanup failed");
    if (startupReceiptPath) {
      const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8"));
      receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed";
      receipt.temporaryRootRemoved = !cleanupError;
      await writeFile(startupReceiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
    }
    if (cleanupError) {
      const message = `Temporary runner E2E state cleanup failed at ${temporaryRoot}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
      for (const publishedResult of publishedResults) {
        const publishedResultPath = publishedResultPaths.get(
          publishedResult.executionId,
        );
        if (!publishedResultPath) continue;
        Object.assign(publishedResult, {
          status: "failed",
          failureClass: "cleanup_failure",
          error: publishedResult.error ? `${publishedResult.error}; ${message}` : message,
          cleanup: "failed",
        } satisfies Partial<RunnerE2EResult>);
        const safeResult = `${JSON.stringify(
          sanitizeJson(publishedResult, attemptSecrets),
          null,
          2,
        )}\n`;
        assertSecretFree(safeResult, attemptSecrets, publishedResultPath);
        await writeFile(publishedResultPath, safeResult, "utf8");
      }
      // Cleanup failure is a failed cell, but must not suppress later cells in
      // the same campaign. The result above carries the terminal failure.
      console.error(message);
      if (options.installedStartupOnly) throw new Error(message);
    }
    if (databaseLane === "owned-docker-postgres") for (const result of publishedResults) {
      const filename = publishedResultPaths.get(result.executionId); if (!filename) continue;
      const safe = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`;
      assertSecretFree(safe, attemptSecrets, filename); await writeFile(filename, safe, "utf8");
    }
    for (const result of publishedResults) {
      const resultPath = publishedResultPaths.get(result.executionId);
      const candidate = executions.find(candidate => candidate.id === result.executionId);
      if (!traceConfig || !result.tracing || !resultPath || !candidate) continue;
      const finalizationInput = {
        receipt: result.tracing,
        identity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,
          provider: candidate.profile.provider, model: candidate.profile.model, environment: candidate.environment.id },
        provenance: { suiteId: candidate.suite.id, suiteDefinitionHash: candidate.suiteDefinitionHash, profileId: candidate.profile.id,
          caseId: candidate.task.id, sourceRevision: resolveRunnerE2ESource().sha },
        result,
      };
      result.tracing.finalization = input.operations
        ? await finalizeEvalAttemptThroughRelay(traceConfig, finalizationInput,
          path.join(`${input.operations.manifest.ledgerPath}.otlp`, `${createHash("sha256").update(accountingAttemptId).digest("hex")}-finalization.jsonl`))
        : await finalizeEvalAttemptTrace(traceConfig, finalizationInput);
      const safeResult = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`;
      assertSecretFree(safeResult, attemptSecrets, resultPath);
      await writeFile(resultPath, safeResult, "utf8");
    }
  }
}

function printList(executions: readonly MatrixExecution[]) {
  console.log("ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS");
  for (const execution of executions) {
    console.log(
      [
        execution.id,
        execution.suite.id,
        execution.profile.generation,
        execution.profile.provider,
        execution.profile.model,
        execution.requiredCredentials.join(","),
      ].join("\t"),
    );
  }
}

async function runExecutionWithRetry(input: {
  execution: MatrixExecution;
  campaignId: string;
  options: ReturnType<typeof parseRunnerSelectors>;
  onAttempt: (result: RunnerE2EResult) => void;
  operations?: EvalOperations;
}): Promise<RunnerE2EResult> {
  const { execution, campaignId, options } = input;
  return executeWithAutomaticRetry({
    task: execution.task, options,
    qualificationCandidate: execution.profile.qualificationCandidate !== undefined,
    cancelled: () => cancelled,
    onRetry: result => console.warn(
      `Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`,
    ),
    runAttempt: async attempt => {
      const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options, operations: input.operations });
      if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`);
      input.onAttempt(result);
      return result;
    },
  });
}

async function runWithConcurrency<T, R>(
  values: readonly T[],
  concurrency: number,
  worker: (value: T) => Promise<R>,
): Promise<R[]> {
  const results = new Array<R>(values.length);
  let cursor = 0;
  const workers = Array.from(
    { length: Math.min(concurrency, values.length) },
    async () => {
      while (true) {
        const index = cursor;
        cursor += 1;
        if (index >= values.length) return;
        if (cancelled) throw new Error("Runner E2E campaign cancelled");
        results[index] = await worker(values[index]!);
      }
    },
  );
  const settled = await Promise.allSettled(workers);
  const failures = settled.filter((result): result is PromiseRejectedResult => result.status === "rejected");
  if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Eval campaign workers failed");
  return results;
}

async function main() {
  const args = process.argv.slice(2);
  if (args.includes("--target-config") || args.includes("product-journeys")) {
    const { launchProductJourneys } = await import("./product-journeys/launch.js");
    return launchProductJourneys(args, repositoryRoot);
  }
  let options: ReturnType<typeof parseRunnerSelectors>;
  try {
    options = parseRunnerSelectors(process.argv.slice(2));
  } catch (error) {
    if (error instanceof RunnerSelectorError)
      throw new Error(`${error.message}\nUse --list to inspect valid cells.`);
    throw error;
  }
  const executions = selectRunnerExecutions(options, runnerMatrix);
  manualPrivateHandoff = await readManualPrivateHandoff(process.env, executions.map(execution => execution.id));
  if (manualPrivateHandoff) await appendPrivateAttemptDiagnostic({ stage: "execution" });
  resultsRoot = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "results") : resolveRunnerE2EResultsRoot(repositoryRoot);
  if (options.installedStartupOnly) {
    assertInstalledStartupOnly(process.env, executions, options);
    assertRunnerE2EPrerequisites(executions);
    const campaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `installed-startup-${Date.now()}`);
    await runAttempt({ executions, attempt: 1, campaignId, options });
    console.log(`PASS installed launch chain (no provider qualification) -> ${path.join(resultsRoot, campaignId, "installed-startup-only")}`);
    return;
  }
  if (options.list) {
    printList(executions);
    return;
  }
  if (options.matrixJson) {
    const jobs = buildMatrixJobs(executions);
    console.log(
      JSON.stringify({
        include: jobs,
        needsDaytona: jobs.some((job) => job.needsDaytona),
        executionIds: executions.map((execution) => execution.id),
      }),
    );
    return;
  }

  if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST && executions.some(execution => execution.task.flow === "provider_connection")) throw new Error("Provider connection campaign has no budgeted routing qualification");
  if (executions.some(execution => execution.task.flow === "antigravity_agent_journey")) {
    if (options.connectionConfig || options.githubReviewConfig || options.provisionGitHubRepository) throw new Error("Use --antigravity-journey-config for the Antigravity journey.");
    const { runAntigravityJourneyCampaign } = await import("./antigravity-journey-launch.js");
    await runAntigravityJourneyCampaign({ executions, catalog: runnerMatrix, configFile: options.antigravityJourneyConfig, repositoryRoot,
      preflight: options.antigravityJourneyPreflight, recoverFile: options.antigravityJourneyRecover });
    return;
  }
  if (options.antigravityJourneyConfig || options.antigravityJourneyPreflight || options.antigravityJourneyRecover) throw new Error("Antigravity journey options require its explicit registered cell.");
  if (executions.some(execution => execution.task.flow === "github_review_journey")) {
    if (options.connectionConfig) throw new Error("Use --github-review-config for the GitHub journey.");
    const { runGitHubReviewCampaign } = await import("./github-review-launch.js");
    await runGitHubReviewCampaign({ executions, catalog: runnerMatrix, configFile: options.githubReviewConfig, repositoryRoot, provisionRepository: options.provisionGitHubRepository });
    return;
  }
  if (options.githubReviewConfig || options.provisionGitHubRepository) throw new Error("GitHub journey options require an explicit GitHub review journey selection.");
  if (executions.some(execution => execution.task.flow === "provider_connection")) {
    const { runConnectionCampaign } = await import("./connection-launch.js");
    await runConnectionCampaign({ executions, catalog: runnerMatrix, configFile: options.connectionConfig, repositoryRoot });
    return;
  }
  if (options.connectionConfig) throw new Error("--connection-config is only supported by the provider-connections suite");

  // Keep admission before local-env loading and credential checks. Pending
  // profiles remain discoverable, but cannot reach a provider.
  assertRunnerE2EPrerequisites(executions);
  assertNativeCompletionSelection(executions);
  assertNativeInstructionSelection(executions);
  const campaignId = cleanId(
    process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
      `local-${new Date().toISOString().replace(/[:.]/g, "-")}`,
  );
  await loadLocalEnvironment(process.env);
  resultsRoot = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "results") : resolveRunnerE2EResultsRoot(repositoryRoot);
  const summaryDir = await prepareRunnerE2ECampaignDirectory(resultsRoot, campaignId);
  const remoteChat = await prepareChatRemoteHost({ executions, env: process.env, repositoryRoot, outputDirectory: summaryDir });
  if (remoteChat) {
    process.env[DATABASE_LANE_ENV] = remoteChat.databaseLane;
    delete process.env[POSTGRES_ARTIFACT_ENV];
    delete process.env[POSTGRES_IMAGE_ID_ENV];
    options.maxAutomaticRetries = remoteChat.maxAutomaticRetries;
    options.maxParallel = Math.min(options.maxParallel, remoteChat.maxParallel);
  }
  if (executions.some(execution => execution.suite.id === "native-completion")) {
    process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(summaryDir);
  }
  if (executions.some(execution => execution.suite.id === NATIVE_INSTRUCTION_SUITE)) {
    process.env[NATIVE_INSTRUCTION_PREFLIGHT_ENV] = prepareNativeInstructionPreflight(summaryDir);
  }
  if (executions.some(execution => execution.suite.id === "stock-harness")) {
    process.env[STOCK_PREFLIGHT_ENV] = prepareStockHarnessPreflight(summaryDir);
  }

  assertRemoteNativeEvidencePrerequisites(executions, process.env);
  const missingCredentials = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST ? [] : [
    ...new Set(
      executions.flatMap((execution) => chatLaunchCredentialNames(execution, execution.suite.id === "chat-features" ? findChatCatalogCase(execution.task.id) : undefined)),
    ),
  ].filter((name) => !process.env[name]?.trim());
  if (missingCredentials.length > 0) {
    throw new Error(
      `Missing runner E2E credentials: ${missingCredentials.join(", ")}`,
    );
  }
  if (
    executions.some((execution) => execution.environment.id === "daytona") &&
    !isImmutableDaytonaImage(process.env.PAPERCLIP_E2E_DAYTONA_IMAGE)
  ) {
    throw new Error(
      "PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image@sha256 digest for Daytona cells",
    );
  }


  await writeFile(
    path.join(summaryDir, "invocation-policy.json"),
    `${JSON.stringify(
      {
        version: 1,
        maxAutomaticRetries: options.maxAutomaticRetries,
        retryClasses: ["transient_infrastructure", "provider_variance"],
        executions: executions.map(execution => ({
          executionId: execution.id,
          automaticRetryPolicy: execution.task.automaticRetryPolicy ?? "default",
          maxAutomaticRetries: effectiveAutomaticRetryLimit(execution.task, options.maxAutomaticRetries),
        })),
      },
      null,
      2,
    )}\n`,
    "utf8",
  );
  const requestedParallelism =
    options.headed || options.ui || options.debug ? 1 : options.maxParallel;
  console.log(
    `Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`,
  );
  const attempts: RunnerE2EResult[] = [];
  const operations = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST
    ? await openEvalOperations(process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, executions, options.maxAutomaticRetries, process.env) : undefined;
  let finalResults: RunnerE2EResult[] = [];
  let campaignFailed = false;
  try {
    if (operations) await writeFile(path.join(summaryDir, "operations-preflight.json"), `${JSON.stringify(operations.manifest, null, 2)}\n`, { mode: 0o600 });
    finalResults = await runWithConcurrency(executions, operations ? Math.min(requestedParallelism, operations.manifest.limits.maxConcurrent) : requestedParallelism,
      execution => runExecutionWithRetry({ execution, campaignId, options, operations, onAttempt: result => attempts.push(result) }));
  } catch {
    campaignFailed = true;
    finalResults = [...new Map(attempts.map(result => [result.executionId, result])).values()];
  } finally {
    try { await operations?.close(); } catch { campaignFailed = true; }
  }

  const generatedAt = new Date().toISOString();
  const operationsReport = operations ? await readEvalOperationsReport(operations.manifest.ledgerPath) : undefined;
  if (operationsReport?.status === "unavailable") campaignFailed = true;
  const campaign = buildRunnerCampaign({
    campaignId,
    generatedAt,
    expected: executions.map((execution) => execution.id),
    results: finalResults,
    attempts,
    ...(operationsReport ? { operations: operationsReport } : {}),
  });
  const campaignSecrets = normalizedSecrets(
    [...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => process.env[name]),
  );
  const safeCampaign = sanitizeJson(
    campaign,
    campaignSecrets,
  ) as typeof campaign;
  const campaignText = `${JSON.stringify(safeCampaign, null, 2)}\n`;
  assertSecretFree(campaignText, campaignSecrets, "campaign.json");
  await writeFile(path.join(summaryDir, "campaign.json"), campaignText, "utf8");
  const dashboard = renderRunnerE2EDashboard({
    title: `Runner E2E · ${campaignId}`,
    generatedAt,
    expected: executions.map((execution) => execution.id),
    catalog: runnerMatrix,
    campaign: safeCampaign,
    entries: safeCampaign.results.map((result) => ({
      result,
      valid: result.status === "passed" && result.cleanup === "passed",
      errors:
        result.status === "passed" && result.cleanup === "passed"
          ? []
          : [
              result.error ??
                result.failureClass ??
                `cleanup=${result.cleanup}`,
            ],
      evidenceBaseHref: [
        result.suiteId ?? "core-compatibility",
        result.profileId,
        result.environmentId,
        result.caseId,
        `attempt-${result.attempt}`,
      ].join("/"),
    })),
  });
  assertSecretFree(dashboard, campaignSecrets, "dashboard.html");
  await writeFile(path.join(summaryDir, "dashboard.html"), dashboard, "utf8");
  console.log(
    `Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`,
  );
  if (campaignFailed) console.error("Campaign finalization incomplete; inspect retained attempt artifacts and the operations ledger.");
  if (campaignFailed || safeCampaign.failed > 0) process.exitCode = 1;
}

await main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  if (manualPrivateHandoff) manualDiagnostic ??= { stage: "execution", error };
  process.exitCode = 1;
}).finally(async () => {
  if (manualPrivateHandoff) await sealManualResultEvidence(manualPrivateHandoff, [...manualEvidenceSecrets]);
}).catch((error) => {
  if (manualDiagnostic && !manualDiagnostic.error) manualDiagnostic.error = error;
  process.stderr.write("Private evidence finalization failed; no public fallback\n"); process.exitCode = 1;
}).finally(async () => {
  if (manualPrivateHandoff && manualDiagnostic) await appendPrivateAttemptDiagnostic(manualDiagnostic).catch(() => {});
});