mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 06:42:06 +01:00
DottaandPaperclip
8c9dd1e444
Add the Kimi lifecycle and recovery qualification harness. Preserve original grades and cleanup evidence, strict native ownership audits, bounded startup and cleanup, and early rejection of unsupported local fault execution before credentials or resources. Co-Authored-By: Paperclip <noreply@paperclip.ing>
1260 lines
54 KiB
TypeScript
1260 lines
54 KiB
TypeScript
import { readUpgradeArtifacts, verifyUpgradeArtifact, upgradeArtifactKey } from "./upgrade-artifacts.js";
|
|
import { recoverWorkerTiming } from "./worker-timing.js";
|
|
import { openEvalOperations, type EvalOperations } from "./operations.js";
|
|
import { readRecoveredFeedback } from "./feedback-recovery.js";
|
|
import { readInterruptedAttemptTrace } from "./trace-recovery.js";
|
|
import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js";
|
|
import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";
|
|
import { runnerE2EPlaywrightCaseTimeoutMs, runnerE2EPlaywrightInvocation, runnerE2EPlaywrightShutdownSignal, runnerE2EPlaywrightWatchdogMs } from "./web-server-command.js";
|
|
import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js";
|
|
import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js";
|
|
import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js";
|
|
import { createRunnerE2ETemporaryRoot } from "./server-config.js";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js";
|
|
import { spawn } from "node:child_process";
|
|
import { createWriteStream } from "node:fs";
|
|
import { createRequire } from "node:module";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import {
|
|
access,
|
|
chmod,
|
|
cp,
|
|
lstat,
|
|
mkdir,
|
|
readFile,
|
|
realpath,
|
|
readdir,
|
|
rm,
|
|
symlink,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js";
|
|
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
|
import { packageEvidence, publishRunnerE2EAttemptEvidence } from "./evidence.js";
|
|
import { prepareRunnerE2ECampaignDirectory, resolveRunnerE2EResultsRoot } from "./results-root.js";
|
|
import { privateCellEvidenceOwner } from "./manual-private-report-assets.js";
|
|
import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js";
|
|
import { requiresPrivateManualEvidence } from "./manual-private-retention.js";
|
|
import { copySharedEvidence } from "./shared-evidence.js";
|
|
import { classifyFailure } from "./failure-classifier.js";
|
|
import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js";
|
|
import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js";
|
|
import { buildRunnerCampaign } from "./history.js";
|
|
import { readEvalOperationsReport } from "./operations-report.js";
|
|
import {
|
|
applyEvalGatewayEnvironment,
|
|
buildRunnerE2EProcessEnvironment,
|
|
resolvePaperclipRemoteRunnerBinaryForHarness,
|
|
resolvePaperclipRunnerBinaryForHarness,
|
|
} from "./harness-env.js";
|
|
import { assertEmbeddedDatabaseIsolation } from "./instance-isolation.js";
|
|
import {
|
|
assertSecretFree,
|
|
findSecretLeakInDirectory,
|
|
isEphemeralCodexRuntimeAuthFile,
|
|
isEphemeralPostgresScanFile,
|
|
normalizedSecrets,
|
|
secretScanFailureEvidence,
|
|
sanitizeJson,
|
|
} from "./redaction.js";
|
|
import {
|
|
buildMatrixJobs,
|
|
parseRunnerSelectors,
|
|
RunnerSelectorError,
|
|
selectRunnerExecutions,
|
|
} from "./selectors.js";
|
|
import { resolveRunnerE2ESource } from "./source.js";
|
|
import {
|
|
CREDENTIAL_NAMES,
|
|
type MatrixExecution,
|
|
type RunnerE2EResult,
|
|
} from "./types.js";
|
|
import { assertRemoteNativeEvidencePrerequisites, assertRunnerE2EPrerequisites } from "./prerequisites.js";
|
|
import { assertNativeCompletionSelection, prepareNativeCompletionPreflight, NATIVE_COMPLETION_PREFLIGHT_ENV } from "./native-completion-admission.js";
|
|
import { assertNativeInstructionSelection, prepareNativeInstructionPreflight, NATIVE_INSTRUCTION_PREFLIGHT_ENV, NATIVE_INSTRUCTION_SUITE } from "./native-instruction-consolidation.js";
|
|
import { prepareStockHarnessPreflight, STOCK_PREFLIGHT_ENV } from "./stock-harness-admission.js";
|
|
|
|
import {
|
|
reapNewDetachedDarwinSharedMemory,
|
|
snapshotDarwinSharedMemory,
|
|
} from "./shared-memory.js";
|
|
import { reserveRunnerE2EServerPort } from "./ports.js";
|
|
import {
|
|
createResultExitGuard,
|
|
enforceResultProcessIntegrity,
|
|
} from "./result-exit-guard.js";
|
|
import {
|
|
observeDescendantProcessTree,
|
|
type ObservedProcessGroup,
|
|
readProcessTable,
|
|
} from "./process-tree.js";
|
|
|
|
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
|
const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local");
|
|
let resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results");
|
|
const manualEvidenceSecrets = new Set<string>();
|
|
let manualPrivateHandoff: ManualPrivateHandoff | undefined;
|
|
const activeProcessGroups = new Set<number>();
|
|
const activeProcessCleanup = new Map<number, Promise<string | null>>();
|
|
const activeProcessTerminators = new Map<number, () => void>();
|
|
const completedResultExitGraceMs = 120_000;
|
|
let cancelled = false;
|
|
|
|
function cleanId(value: string) {
|
|
const result = value
|
|
.replace(/[^A-Za-z0-9_.-]+/g, "-")
|
|
.replace(/^-+|-+$/g, "");
|
|
if (!result)
|
|
throw new Error(
|
|
`Invalid empty identifier derived from ${JSON.stringify(value)}`,
|
|
);
|
|
return result;
|
|
}
|
|
|
|
function secret(bytes = 32) {
|
|
return randomBytes(bytes).toString("base64url");
|
|
}
|
|
|
|
async function makeDirectoryTreeRemovable(directory: string): Promise<void> {
|
|
await chmod(directory, 0o700);
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
await Promise.all(
|
|
entries
|
|
.filter((entry) => entry.isDirectory() && !entry.isSymbolicLink())
|
|
.map((entry) =>
|
|
makeDirectoryTreeRemovable(path.join(directory, entry.name)),
|
|
),
|
|
);
|
|
}
|
|
|
|
function stopProcessGroup(pid: number, signal: NodeJS.Signals = "SIGTERM") {
|
|
try {
|
|
if (process.platform === "win32") process.kill(pid, signal);
|
|
else process.kill(-pid, signal);
|
|
} catch {
|
|
// The process tree already exited.
|
|
}
|
|
}
|
|
|
|
function processGroupIsAlive(pid: number) {
|
|
try {
|
|
process.kill(process.platform === "win32" ? pid : -pid, 0);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function terminateProcessGroup(pid: number) {
|
|
stopProcessGroup(pid, "SIGTERM");
|
|
const gracefulDeadline = Date.now() + 5_000;
|
|
while (processGroupIsAlive(pid) && Date.now() < gracefulDeadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
if (!processGroupIsAlive(pid)) return null;
|
|
stopProcessGroup(pid, "SIGKILL");
|
|
const forcedDeadline = Date.now() + 5_000;
|
|
while (processGroupIsAlive(pid) && Date.now() < forcedDeadline) {
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
return processGroupIsAlive(pid)
|
|
? `Paperclip/Playwright process group ${pid} survived SIGKILL`
|
|
: null;
|
|
}
|
|
|
|
function wait(milliseconds: number) {
|
|
return new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
|
}
|
|
|
|
interface ProcessTreeDiagnostic {
|
|
summary: string;
|
|
groups: ObservedProcessGroup[];
|
|
}
|
|
|
|
async function processTreeDiagnostic(
|
|
rootPid: number,
|
|
): Promise<ProcessTreeDiagnostic> {
|
|
const table = await readProcessTable();
|
|
if (!table) {
|
|
return {
|
|
summary: `process tree ${rootPid} (member inspection unavailable)`,
|
|
groups: [],
|
|
};
|
|
}
|
|
const observed = observeDescendantProcessTree(table, rootPid);
|
|
const members = observed.members
|
|
.slice(0, 64)
|
|
.map(
|
|
({ process: candidate, depth }) =>
|
|
`pid=${candidate.pid} ppid=${candidate.parentPid} pgid=${candidate.processGroupId} depth=${depth} kind=${candidate.kind}`,
|
|
);
|
|
const descendantGroupIds = observed.groups
|
|
.filter((group) => group.processGroupId !== rootPid)
|
|
.map((group) => group.processGroupId);
|
|
return {
|
|
summary:
|
|
members.length > 0
|
|
? `process tree ${rootPid}: ${members.join("; ")}; descendant pgids=${descendantGroupIds.join(",") || "none"}`
|
|
: `process tree ${rootPid}: no members reported`,
|
|
groups: observed.groups,
|
|
};
|
|
}
|
|
|
|
for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
|
|
process.on(signal, () => {
|
|
cancelled = true;
|
|
for (const pid of activeProcessGroups) {
|
|
const terminate = activeProcessTerminators.get(pid);
|
|
if (terminate) {
|
|
terminate();
|
|
continue;
|
|
}
|
|
activeProcessCleanup.set(pid, terminateProcessGroup(pid));
|
|
}
|
|
});
|
|
}
|
|
|
|
async function loadLocalEnvironment(target: NodeJS.ProcessEnv) {
|
|
const contents = await readFile(localEnvPath, "utf8").catch(
|
|
(error: NodeJS.ErrnoException) => {
|
|
if (error.code === "ENOENT") return null;
|
|
throw error;
|
|
},
|
|
);
|
|
if (contents === null) return;
|
|
for (const [index, rawLine] of contents.split(/\r?\n/).entries()) {
|
|
const line = rawLine.trim();
|
|
if (!line || line.startsWith("#")) continue;
|
|
const match = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line);
|
|
if (!match)
|
|
throw new Error(
|
|
`Invalid ${path.basename(localEnvPath)} line ${index + 1}`,
|
|
);
|
|
if (target[match[1]] !== undefined) continue;
|
|
let value = match[2].trim();
|
|
if (
|
|
(value.startsWith('"') && value.endsWith('"')) ||
|
|
(value.startsWith("'") && value.endsWith("'"))
|
|
) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
target[match[1]] = value;
|
|
}
|
|
}
|
|
|
|
async function prepareProviderPath(
|
|
temporaryRoot: string,
|
|
inheritedPath: string | undefined,
|
|
) {
|
|
const toolBin = path.join(temporaryRoot, "provider-bin");
|
|
await mkdir(toolBin, { recursive: true });
|
|
const runnerRequire = createRequire(
|
|
path.join(repositoryRoot, "packages/paperclip-runner/package.json"),
|
|
);
|
|
const codexAcpPackage = runnerRequire.resolve(
|
|
"@agentclientprotocol/codex-acp/package.json",
|
|
);
|
|
const codexRequire = createRequire(codexAcpPackage);
|
|
const codexPackage = codexRequire.resolve("@openai/codex/package.json");
|
|
const codexManifest = JSON.parse(await readFile(codexPackage, "utf8")) as {
|
|
bin?: string | Record<string, string>;
|
|
};
|
|
const codexBin =
|
|
typeof codexManifest.bin === "string"
|
|
? codexManifest.bin
|
|
: codexManifest.bin?.codex;
|
|
if (!codexBin)
|
|
throw new Error(
|
|
"Production Codex ACP dependency does not expose its pinned Codex executable",
|
|
);
|
|
await symlink(
|
|
path.resolve(path.dirname(codexPackage), codexBin),
|
|
path.join(toolBin, "codex"),
|
|
);
|
|
const packageBin = path.join(
|
|
repositoryRoot,
|
|
"packages/paperclip-runner/node_modules/.bin",
|
|
);
|
|
return [toolBin, packageBin, inheritedPath]
|
|
.filter(Boolean)
|
|
.join(path.delimiter);
|
|
}
|
|
|
|
async function runProcess(
|
|
invocation: { command: string; args: string[] },
|
|
env: NodeJS.ProcessEnv,
|
|
timeoutMs: number | null,
|
|
logPath: string,
|
|
completionPaths: readonly string[],
|
|
interactive: boolean,
|
|
) {
|
|
const log = createWriteStream(logPath, { flags: "a", mode: 0o600 });
|
|
const child = spawn(invocation.command, invocation.args, {
|
|
cwd: repositoryRoot,
|
|
env,
|
|
stdio: ["inherit", "pipe", "pipe"],
|
|
detached: process.platform !== "win32",
|
|
});
|
|
if (!child.pid) throw new Error("Failed to start Playwright");
|
|
activeProcessGroups.add(child.pid);
|
|
const processOwner = createProcessTreeOwner(child);
|
|
let outputTail = "";
|
|
const recordOutput = (chunk: Buffer, destination: NodeJS.WriteStream) => {
|
|
destination.write(chunk);
|
|
log.write(chunk);
|
|
outputTail += chunk.toString("utf8");
|
|
if (outputTail.length > 1024 * 1024)
|
|
outputTail = outputTail.slice(-1024 * 1024);
|
|
};
|
|
child.stdout?.on("data", (chunk: Buffer) =>
|
|
recordOutput(chunk, process.stdout),
|
|
);
|
|
child.stderr?.on("data", (chunk: Buffer) =>
|
|
recordOutput(chunk, process.stderr),
|
|
);
|
|
let childSettled = false;
|
|
let postResultStallError: string | null = null;
|
|
let boundedCleanup: Promise<string | null> | undefined;
|
|
let cleanupSettled!: () => void;
|
|
const cleanupFinished = new Promise<number>(resolve => { cleanupSettled = () => resolve(1); });
|
|
const stopChildTree = (_diagnostic?: ProcessTreeDiagnostic) => {
|
|
if (boundedCleanup) return;
|
|
boundedCleanup = stopOwnedProcessTree(child, processOwner, 45_000, 5_000, runnerE2EPlaywrightShutdownSignal)
|
|
.then(() => null, error => error instanceof Error ? error.message : String(error))
|
|
.finally(cleanupSettled);
|
|
activeProcessCleanup.set(child.pid!, boundedCleanup);
|
|
};
|
|
activeProcessTerminators.set(child.pid, stopChildTree);
|
|
const resultExitGuard = createResultExitGuard({
|
|
resultPaths: completionPaths,
|
|
interactive,
|
|
graceMs: completedResultExitGraceMs,
|
|
now: Date.now,
|
|
pathExists: (candidate) =>
|
|
access(candidate).then(
|
|
() => true,
|
|
() => false,
|
|
),
|
|
onExpired: async () => {
|
|
const diagnostic = await processTreeDiagnostic(child.pid!);
|
|
if (childSettled) return;
|
|
postResultStallError = `Playwright remained alive for ${completedResultExitGraceMs}ms after every result was written`;
|
|
recordOutput(
|
|
Buffer.from(
|
|
`\n${postResultStallError}; forcing bounded cleanup. ${diagnostic.summary}\n`,
|
|
),
|
|
process.stderr,
|
|
);
|
|
stopChildTree(diagnostic);
|
|
},
|
|
});
|
|
const completionPoll = resultExitGuard.enabled
|
|
? setInterval(() => {
|
|
void resultExitGuard.poll().catch(() => {
|
|
if (childSettled || postResultStallError) return;
|
|
postResultStallError =
|
|
"Completed-result exit guard failed during bounded inspection";
|
|
recordOutput(
|
|
Buffer.from(`\n${postResultStallError}; forcing cleanup.\n`),
|
|
process.stderr,
|
|
);
|
|
stopChildTree();
|
|
});
|
|
}, 500)
|
|
: undefined;
|
|
completionPoll?.unref();
|
|
let timedOut = false;
|
|
const timer =
|
|
timeoutMs === null
|
|
? undefined
|
|
: setTimeout(() => {
|
|
timedOut = true;
|
|
stopChildTree();
|
|
}, timeoutMs);
|
|
timer?.unref();
|
|
let spawnError: string | null = null;
|
|
const childExit = new Promise<number>((resolve, reject) => {
|
|
child.once("error", reject);
|
|
child.once("exit", (code) => {
|
|
childSettled = true;
|
|
resolve(code ?? 1);
|
|
});
|
|
}).catch((error) => {
|
|
childSettled = true;
|
|
spawnError = error instanceof Error ? error.message : String(error);
|
|
return 1;
|
|
});
|
|
const exitCode = await Promise.race([childExit, cleanupFinished]);
|
|
if (timer) clearTimeout(timer);
|
|
if (completionPoll) clearInterval(completionPoll);
|
|
// Even a successful launcher exit can leave an already-observed detached
|
|
// server behind. Retire that retained tree, never only the root group.
|
|
stopChildTree();
|
|
const processCleanupError = await boundedCleanup!;
|
|
processOwner.stopObserving();
|
|
// Even a failed direct-child kill must not hold cancellation forever or let
|
|
// inherited pipes write into an ended log. The cleanup error remains fatal.
|
|
if (processCleanupError) {
|
|
child.stdout?.destroy();
|
|
child.stderr?.destroy();
|
|
if (child.exitCode === null && child.signalCode === null) child.unref();
|
|
}
|
|
if (child.pid) {
|
|
activeProcessGroups.delete(child.pid);
|
|
activeProcessCleanup.delete(child.pid);
|
|
activeProcessTerminators.delete(child.pid);
|
|
}
|
|
await new Promise<void>((resolve) => log.end(resolve));
|
|
return {
|
|
exitCode,
|
|
timedOut,
|
|
postResultStallError,
|
|
processCleanupError,
|
|
spawnError,
|
|
outputTail,
|
|
};
|
|
}
|
|
|
|
function syntheticResult(
|
|
execution: MatrixExecution,
|
|
attempt: number,
|
|
startedAtMs: number,
|
|
error: string,
|
|
failureClass: RunnerE2EResult["failureClass"] = "transient_infrastructure",
|
|
): RunnerE2EResult {
|
|
const finishedAtMs = Date.now();
|
|
return {
|
|
schema: "paperclip.runner-e2e.result/v2",
|
|
executionId: execution.id,
|
|
suiteId: execution.suite.id,
|
|
suiteDefinitionHash: execution.suiteDefinitionHash,
|
|
source: resolveRunnerE2ESource(),
|
|
...(execution.profile.ranking
|
|
? { rankingSnapshot: execution.profile.ranking }
|
|
: {}),
|
|
attempt,
|
|
status: "failed",
|
|
failureClass,
|
|
error,
|
|
profileId: execution.profile.id,
|
|
environmentId: execution.environment.id,
|
|
caseId: execution.task.id,
|
|
provider: execution.profile.provider,
|
|
model: execution.profile.model,
|
|
runtimeMode: execution.profile.expectedRuntimeMode,
|
|
startedAt: new Date(startedAtMs).toISOString(),
|
|
finishedAt: new Date(finishedAtMs).toISOString(),
|
|
durationMs: finishedAtMs - startedAtMs,
|
|
cleanup: "not_started",
|
|
};
|
|
}
|
|
|
|
async function readResult(resultPath: string, fallback: RunnerE2EResult) {
|
|
try {
|
|
return JSON.parse(await readFile(resultPath, "utf8")) as RunnerE2EResult;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
|
|
async function runAttempt(input: {
|
|
executions: readonly MatrixExecution[];
|
|
attempt: number;
|
|
campaignId: string;
|
|
options: ReturnType<typeof parseRunnerSelectors>;
|
|
operations?: EvalOperations;
|
|
}): Promise<RunnerE2EResult[]> {
|
|
const { executions, attempt, campaignId, options } = input;
|
|
const accountingCampaignId = input.operations?.manifest.limits.campaignId ?? process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? campaignId;
|
|
const accountingAttemptId = randomBytes(16).toString("hex");
|
|
const traceConfig = evalBraintrustConfig({ ...process.env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined });
|
|
const execution = executions[0];
|
|
if (!execution) throw new Error("A runner E2E cell must contain a task case");
|
|
if (
|
|
executions.some(
|
|
(candidate) =>
|
|
candidate.profile.id !== execution.profile.id ||
|
|
candidate.environment.id !== execution.environment.id,
|
|
)
|
|
) {
|
|
throw new Error(
|
|
"One isolated runner E2E harness cannot mix profiles or environments",
|
|
);
|
|
}
|
|
if (process.env.GITHUB_ACTIONS === "true" && !input.operations && executions.some(candidate => requiresPrivateManualEvidence(candidate.id)) && !manualPrivateHandoff) throw new Error("Sensitive manual evidence requires verified restricted retention before setup");
|
|
await prepareRunnerE2ECampaignDirectory(resultsRoot, campaignId);
|
|
const upgrades = await readUpgradeArtifacts(process.env[upgradeArtifactKey]);
|
|
if (upgrades) {
|
|
// All attempts and the browser use A's installed fixture admission. The
|
|
// supervisor changes only the server artifact; the original sandbox lives on.
|
|
Object.assign(process.env, upgrades.artifacts.A.pins);
|
|
for (const artifact of [upgrades.artifacts.A, upgrades.artifacts.B]) {
|
|
await verifyUpgradeArtifact(artifact, process.env, executions);
|
|
}
|
|
}
|
|
assertInstalledCliSelection(process.env, executions);
|
|
const installedCli = await verifyInstalledCli(process.env, executions);
|
|
const installedPlugin = await verifyInstalledDaytonaPlugin(process.env, executions);
|
|
const startedAtMs = Date.now();
|
|
const sharedMemoryBaseline = snapshotDarwinSharedMemory();
|
|
const temporaryParent = await realpath(os.tmpdir());
|
|
const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent);
|
|
const privateDir = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "capture") : path.join(temporaryRoot, "artifacts-private");
|
|
const publishedResults: RunnerE2EResult[] = [];
|
|
const publishedResultPaths = new Map<string, string>();
|
|
let attemptSecrets: string[] = [];
|
|
let processCleanupFailed = false;
|
|
let startupReceiptPath: string | undefined;
|
|
let gateway: Awaited<ReturnType<EvalOperations["start"]>> | undefined;
|
|
const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = [];
|
|
try {
|
|
const paperclipHome = path.join(temporaryRoot, "paperclip-home");
|
|
const workspace = path.join(temporaryRoot, "workspace");
|
|
const instanceId = `runner-e2e-${randomBytes(8).toString("hex")}`;
|
|
const configPath = path.join(
|
|
paperclipHome,
|
|
"instances",
|
|
instanceId,
|
|
"config.json",
|
|
);
|
|
const port = await reserveRunnerE2EServerPort();
|
|
await Promise.all([
|
|
mkdir(paperclipHome, { recursive: true }),
|
|
mkdir(workspace, { recursive: true }),
|
|
mkdir(privateDir, { recursive: true }),
|
|
]);
|
|
const providerPath = await prepareProviderPath(
|
|
temporaryRoot,
|
|
process.env.PATH,
|
|
);
|
|
if (requiresCodexCiSandbox(execution)) {
|
|
await prepareCodexCiSandbox(repositoryRoot, temporaryRoot);
|
|
}
|
|
const agentJwtSecret = secret(48);
|
|
const decisionSigningSecret = secret(48);
|
|
const toolActionSigningSecret = secret(48);
|
|
const betterAuthSecret = secret(48);
|
|
gateway = await input.operations?.start(execution, accountingAttemptId, attempt);
|
|
const credentialSources = {
|
|
PAPERCLIP_E2E_GATEWAY_TOKEN: gateway?.token,
|
|
PAPERCLIP_E2E_OTLP_RELAY_TOKEN: gateway?.observability?.token,
|
|
...Object.fromEntries([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => [name, process.env[name]])),
|
|
PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,
|
|
PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,
|
|
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
|
|
BETTER_AUTH_SECRET: betterAuthSecret,
|
|
};
|
|
const credentials = normalizedSecrets(Object.values(credentialSources));
|
|
attemptSecrets = credentials;
|
|
if (manualPrivateHandoff) for (const credential of credentials) manualEvidenceSecrets.add(credential);
|
|
if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 });
|
|
if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 });
|
|
const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness(
|
|
executions,
|
|
repositoryRoot,
|
|
);
|
|
const childEnv: NodeJS.ProcessEnv = {
|
|
...buildRunnerE2EProcessEnvironment(process.env, executions),
|
|
PATH: providerPath,
|
|
PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify(
|
|
executions.map((candidate) => candidate.id),
|
|
),
|
|
PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt),
|
|
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: accountingCampaignId,
|
|
PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: accountingAttemptId,
|
|
PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined,
|
|
PAPERCLIP_RUNNER_E2E_PUBLIC_MCP: executions.some(candidate => candidate.task.flow === "public_mcp") ? "1" : "0",
|
|
PAPERCLIP_RUNNER_E2E_PORT: String(port),
|
|
PAPERCLIP_RUNNER_E2E_TEMP_ROOT: temporaryRoot,
|
|
PAPERCLIP_RUNNER_E2E_PRIVATE_DIR: privateDir,
|
|
PAPERCLIP_RUNNER_E2E_WORKSPACE: workspace,
|
|
PAPERCLIP_RUNNER_E2E_SERVER_LOG: path.join(privateDir, "server.log"),
|
|
PAPERCLIP_RUNNER_BINARY: runnerBinary,
|
|
PAPERCLIP_RUNNER_REMOTE_BINARY_PATH:
|
|
installedCli ? undefined : resolvePaperclipRemoteRunnerBinaryForHarness(executions, runnerBinary),
|
|
// Vite's optimized dependency cache embeds revision query strings. A
|
|
// private per-attempt cache prevents an earlier cell or local rebuild
|
|
// from producing `504 Outdated Optimize Dep` during browser bootstrap.
|
|
PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"),
|
|
PAPERCLIP_RUNNER_E2E_TEST_TIMEOUT_MS: String(
|
|
Math.max(
|
|
...executions.map(
|
|
(candidate) =>
|
|
runnerE2EPlaywrightCaseTimeoutMs(candidate.task.flow, candidate.task.attemptTimeoutMs[candidate.environment.id]),
|
|
),
|
|
),
|
|
),
|
|
PAPERCLIP_HOME: paperclipHome,
|
|
PAPERCLIP_INSTANCE_ID: instanceId,
|
|
PAPERCLIP_CONFIG: configPath,
|
|
PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret,
|
|
PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret,
|
|
PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret,
|
|
BETTER_AUTH_SECRET: betterAuthSecret,
|
|
};
|
|
if (gateway) {
|
|
applyEvalGatewayEnvironment(childEnv, execution.profile.credential, gateway);
|
|
// The fixture/server controller needs Daytona provisioning access. The
|
|
// remote provider still receives only the scoped model and OTLP tokens.
|
|
if (execution.environment.id === "daytona") childEnv.DAYTONA_API_KEY = process.env.DAYTONA_API_KEY;
|
|
}
|
|
// The database URLs are stripped here and again at the Playwright web-server
|
|
// boundary so a developer's shell can never redirect this paid test.
|
|
delete childEnv.DATABASE_URL;
|
|
delete childEnv.DATABASE_MIGRATION_URL;
|
|
|
|
const playwrightArgs = [
|
|
"test",
|
|
"--config",
|
|
"tests/runner-e2e/playwright.config.ts",
|
|
...(options.headed ? ["--headed"] : []),
|
|
...(options.ui ? ["--ui"] : []),
|
|
...(options.debug ? ["--debug"] : []),
|
|
];
|
|
const watchdog =
|
|
options.ui || options.debug
|
|
? null
|
|
: runnerE2EPlaywrightWatchdogMs(executions);
|
|
const processResult = await runProcess(
|
|
runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs),
|
|
childEnv,
|
|
watchdog,
|
|
path.join(privateDir, "playwright.log"),
|
|
executions.map((candidate) =>
|
|
path.join(privateDir, "cases", candidate.task.id, "result.json"),
|
|
),
|
|
options.ui || options.debug,
|
|
);
|
|
processCleanupFailed = processResult.processCleanupError !== null;
|
|
if (options.installedStartupOnly) {
|
|
const proofDir = path.join(resultsRoot, campaignId, "installed-startup-only");
|
|
await mkdir(proofDir, { recursive: true });
|
|
// This is private setup evidence, never a successful provider case/result.
|
|
await cp(privateDir, path.join(proofDir, "private"), { recursive: true });
|
|
const probe = JSON.parse(await readFile(path.join(privateDir, "installed-startup-only.json"), "utf8"));
|
|
if (processResult.exitCode !== 0 || processResult.timedOut || processResult.spawnError || processCleanupFailed || probe.status !== "health_ui_zero_company_passed" || probe.providerCalls !== 0 || probe.qualified !== false) throw new Error("Installed startup-only actual launch chain failed");
|
|
await assertEmbeddedDatabaseIsolation(configPath, temporaryRoot);
|
|
startupReceiptPath = path.join(proofDir, "receipt.json");
|
|
await writeFile(startupReceiptPath, `${JSON.stringify({ status: "installed_launch_chain_pending_scratch_cleanup", qualified: false, providerCalls: 0, credentialsLoaded: false, installedCli, processResult, probe }, null, 2)}\n`, { mode: 0o600 });
|
|
return [];
|
|
}
|
|
const processFailure = processResult.spawnError
|
|
? `Playwright failed to start: ${processResult.spawnError}`
|
|
: processResult.timedOut
|
|
? `Harness process exceeded ${Math.round((watchdog ?? 0) / 1000)} seconds`
|
|
: `Playwright exited ${processResult.exitCode} before writing a result`;
|
|
const processFailureClass =
|
|
processResult.spawnError || processResult.timedOut
|
|
? "transient_infrastructure"
|
|
: classifyFailure(
|
|
new Error(
|
|
processResult.outputTail
|
|
? `${processFailure}\n${processResult.outputTail}`
|
|
: processFailure,
|
|
),
|
|
);
|
|
const results = await Promise.all(
|
|
executions.map(async (candidate) => {
|
|
const resultPath = path.join(
|
|
privateDir,
|
|
"cases",
|
|
candidate.task.id,
|
|
"result.json",
|
|
);
|
|
const fallback = syntheticResult(
|
|
candidate,
|
|
attempt,
|
|
startedAtMs,
|
|
processFailure,
|
|
processFailureClass,
|
|
);
|
|
const result = await readResult(resultPath, fallback);
|
|
if (!result.workerTiming) result.workerTiming = await recoverWorkerTiming(path.join(path.dirname(resultPath), "snapshots", "worker-timing.json"),
|
|
{ executionId: candidate.id, attempt, startedAt: result.startedAt, finishedAt: result.finishedAt });
|
|
if (result === fallback) {
|
|
result.feedback = await readRecoveredFeedback(path.dirname(resultPath), {
|
|
campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,
|
|
});
|
|
if (traceConfig) result.tracing = await readInterruptedAttemptTrace(path.dirname(resultPath), {
|
|
projectId: traceConfig.projectId, executionId: candidate.id,
|
|
});
|
|
}
|
|
// Keep cleanup authority before evidence copying/publication can fail.
|
|
rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status });
|
|
return enforceResultProcessIntegrity(result, processResult);
|
|
}),
|
|
);
|
|
let isolationError: unknown;
|
|
try {
|
|
await assertEmbeddedDatabaseIsolation(configPath, temporaryRoot);
|
|
} catch (error) {
|
|
isolationError = error;
|
|
}
|
|
let persistedStateError: unknown;
|
|
let persistedStateFinding: { file: string; stateArea: string; reason: string; matchCategory?: string } | undefined;
|
|
// Onboarding evaluates behavior; credential persistence belongs to a separate layer.
|
|
// Keep artifact redaction/publication checks independent of this filesystem scan.
|
|
const persistenceCheckedExecutions = executions.filter(
|
|
(candidate) => candidate.suite.id !== "first-task",
|
|
);
|
|
if (persistenceCheckedExecutions.length > 0) {
|
|
try {
|
|
const expectedEphemeralCredentials = new Set<string>();
|
|
for (const [label, directory] of [
|
|
["Paperclip home", paperclipHome],
|
|
["workspace", workspace],
|
|
] as const) {
|
|
while (true) {
|
|
// The managed Codex home may legitimately contain upstream source-code
|
|
// fixtures with fake `sk-*` strings. Reject exact campaign credentials.
|
|
const leak = await findSecretLeakInDirectory(directory, credentials, {
|
|
includeShapes: false,
|
|
sources: credentialSources,
|
|
...(label === "Paperclip home" ? { privateCodexRuntimeIdentityRoot: paperclipHome } : {}),
|
|
ignoreFile: (file) => expectedEphemeralCredentials.has(file),
|
|
allowDisappearedFile: (file) =>
|
|
label === "Paperclip home" &&
|
|
isEphemeralPostgresScanFile(paperclipHome, file),
|
|
});
|
|
if (!leak) break;
|
|
const isManagedCodexRuntimeAuth =
|
|
label === "Paperclip home" &&
|
|
isEphemeralCodexRuntimeAuthFile(paperclipHome, leak.file);
|
|
if (isManagedCodexRuntimeAuth) {
|
|
const metadata = await lstat(leak.file);
|
|
if (metadata.isFile() && (metadata.mode & 0o777) === 0o600) {
|
|
// Codex CLI API-key mode requires this one runtime auth file. It
|
|
// lives only in the disposable cell root, is never published,
|
|
// must be owner-only, and is removed with the root below.
|
|
expectedEphemeralCredentials.add(leak.file);
|
|
continue;
|
|
}
|
|
}
|
|
persistedStateFinding = { ...leak, stateArea: label, file: path.relative(temporaryRoot, leak.file) };
|
|
throw new Error(`Secret leak in persisted ${label} state at ${persistedStateFinding.file}: ${leak.reason}${leak.matchCategory ? ` (${leak.matchCategory})` : ""}`);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
persistedStateError = error;
|
|
}
|
|
}
|
|
for (const [index, candidate] of executions.entries()) {
|
|
let result: RunnerE2EResult = {
|
|
...results[index],
|
|
accountingIdentity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId },
|
|
};
|
|
if (
|
|
isolationError &&
|
|
result.failureClass !== "cleanup_failure" &&
|
|
result.failureClass !== "secret_leak"
|
|
) {
|
|
const isolationMessage =
|
|
isolationError instanceof Error
|
|
? isolationError.message
|
|
: String(isolationError);
|
|
const configWasUnavailableDuringTransientBootstrap =
|
|
result.failureClass === "transient_infrastructure" &&
|
|
typeof isolationError === "object" &&
|
|
isolationError !== null &&
|
|
"code" in isolationError &&
|
|
isolationError.code === "ENOENT";
|
|
result = {
|
|
...result,
|
|
status: "failed",
|
|
failureClass: configWasUnavailableDuringTransientBootstrap
|
|
? result.failureClass
|
|
: "permanent_infrastructure",
|
|
error: configWasUnavailableDuringTransientBootstrap
|
|
? `${result.error}; isolated config could not be inspected after bootstrap failure: ${isolationMessage}`
|
|
: isolationMessage,
|
|
};
|
|
}
|
|
if (persistedStateError && persistenceCheckedExecutions.includes(candidate)) {
|
|
const persistedStateMessage =
|
|
persistedStateError instanceof Error
|
|
? persistedStateError.message
|
|
: String(persistedStateError);
|
|
const persistedStateClass = classifyFailure(persistedStateError);
|
|
if (
|
|
persistedStateClass === "secret_leak" ||
|
|
(result.failureClass !== "secret_leak" &&
|
|
result.failureClass !== "cleanup_failure")
|
|
) {
|
|
result = {
|
|
...result,
|
|
status: "failed",
|
|
failureClass:
|
|
persistedStateClass === "secret_leak"
|
|
? "secret_leak"
|
|
: "permanent_infrastructure",
|
|
error: persistedStateMessage,
|
|
};
|
|
} else {
|
|
result = {
|
|
...result,
|
|
error: `${result.error}; persisted-state scan also failed: ${persistedStateMessage}`,
|
|
};
|
|
}
|
|
}
|
|
const casePrivateDir = path.join(privateDir, "cases", candidate.task.id);
|
|
const uploadDir = path.join(
|
|
resultsRoot,
|
|
campaignId,
|
|
candidate.suite.id,
|
|
candidate.profile.id,
|
|
candidate.environment.id,
|
|
candidate.task.id,
|
|
`attempt-${attempt}`,
|
|
);
|
|
await mkdir(casePrivateDir, { recursive: true });
|
|
await copySharedEvidence(privateDir, casePrivateDir, temporaryRoot, credentials);
|
|
if (persistedStateError && persistenceCheckedExecutions.includes(candidate)) {
|
|
const snapshots = path.join(casePrivateDir, "snapshots");
|
|
await mkdir(snapshots, { recursive: true });
|
|
const original = results[index];
|
|
await writeFile(path.join(snapshots, "persisted-state-scan.json"), `${JSON.stringify(secretScanFailureEvidence({
|
|
scanError: persistedStateError instanceof Error ? persistedStateError.message : String(persistedStateError),
|
|
finding: persistedStateFinding,
|
|
originalResult: { status: original.status, failureClass: original.failureClass, error: original.error },
|
|
}, credentials), null, 2)}\n`, { mode: 0o600 });
|
|
}
|
|
const resultPath = path.join(casePrivateDir, "result.json");
|
|
if (manualPrivateHandoff) {
|
|
await preserveOriginalFrameworkResult(casePrivateDir);
|
|
await writeFile(
|
|
resultPath,
|
|
`${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
let evidence = await packageEvidence({
|
|
privateDir: casePrivateDir,
|
|
uploadDir,
|
|
secrets: credentials,
|
|
expectPassScreenshot: result.status === "passed",
|
|
privateCell: { owner: await privateCellEvidenceOwner(manualPrivateHandoff, candidate.id), layout: "original" as const },
|
|
});
|
|
const originalEvidence = evidence, originalFeaturePassed = result.status === "passed";
|
|
if (evidence.leaks.length > 0 || evidence.missing.length > 0) {
|
|
result = {
|
|
...result,
|
|
status: "failed",
|
|
failureClass:
|
|
evidence.leaks.length > 0
|
|
? "secret_leak"
|
|
: "permanent_infrastructure",
|
|
error:
|
|
evidence.leaks.length > 0
|
|
? `Secret leak rejected from evidence: ${evidence.leaks.map((leak) => leak.file).join(", ")}`
|
|
: `Required evidence missing: ${evidence.missing.join(", ")}`,
|
|
};
|
|
await writeFile(
|
|
resultPath,
|
|
`${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
evidence = await packageEvidence({
|
|
privateDir: casePrivateDir,
|
|
uploadDir,
|
|
secrets: credentials,
|
|
expectPassScreenshot: false,
|
|
privateCell: { owner: await privateCellEvidenceOwner(manualPrivateHandoff, candidate.id), layout: "original" as const },
|
|
});
|
|
}
|
|
await sealStandardOriginalEvidence({ context: manualPrivateHandoff,
|
|
executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed });
|
|
} else {
|
|
result = await publishRunnerE2EAttemptEvidence({
|
|
result,
|
|
resultsRoot,
|
|
privateDir: casePrivateDir,
|
|
uploadDir,
|
|
secrets: credentials,
|
|
});
|
|
}
|
|
console.log(
|
|
`${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`,
|
|
);
|
|
publishedResults.push(result);
|
|
publishedResultPaths.set(
|
|
candidate.id,
|
|
path.join(uploadDir, "result.json"),
|
|
);
|
|
}
|
|
return [...publishedResults];
|
|
} finally {
|
|
let gatewayCleanupError: unknown;
|
|
try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; }
|
|
if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline);
|
|
let cleanupError: unknown;
|
|
const resourceAdmissionStarted = await access(path.join(privateDir, "resource-admission-started"))
|
|
.then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT");
|
|
if (mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults })) {
|
|
// Remote allocation cleanup is journaled in this instance's database.
|
|
// Deleting it after the controller exits would strand uncertain creates.
|
|
await chmod(temporaryRoot, 0o700);
|
|
cleanupError = new Error(`Preserving private recovery state after unconfirmed cleanup: ${temporaryRoot}`);
|
|
} else if (shouldKeepFailedDiagnostics({
|
|
enabled: process.env.PAPERCLIP_RUNNER_E2E_KEEP_FAILED_PRIVATE === "1",
|
|
expectedResults: executions.length,
|
|
results: publishedResults,
|
|
})) {
|
|
// Explicit diagnosis only. Confirmed resource cleanup stays confirmed;
|
|
// keep private traces for investigation without publishing provider data.
|
|
await chmod(temporaryRoot, 0o700);
|
|
console.warn(`Retained private failed-case diagnostics: ${temporaryRoot}`);
|
|
} else if (
|
|
temporaryRoot.startsWith(`${temporaryParent}${path.sep}paperclip-runner-e2e-`)
|
|
) {
|
|
for (let cleanupAttempt = 1; cleanupAttempt <= 3; cleanupAttempt += 1) {
|
|
try {
|
|
await rm(temporaryRoot, { recursive: true, force: true });
|
|
cleanupError = undefined;
|
|
break;
|
|
} catch (error) {
|
|
cleanupError = error;
|
|
if (cleanupAttempt < 3) {
|
|
await makeDirectoryTreeRemovable(temporaryRoot).catch(() => {});
|
|
await new Promise((resolve) =>
|
|
setTimeout(resolve, cleanupAttempt * 250),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
cleanupError = new Error(
|
|
`Refusing to remove unexpected temporary path ${temporaryRoot}`,
|
|
);
|
|
}
|
|
if (gatewayCleanupError) cleanupError = new AggregateError([cleanupError, gatewayCleanupError].filter(Boolean), "Eval gateway cleanup failed");
|
|
if (startupReceiptPath) {
|
|
const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8"));
|
|
receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed";
|
|
receipt.temporaryRootRemoved = !cleanupError;
|
|
await writeFile(startupReceiptPath, `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 });
|
|
}
|
|
if (cleanupError) {
|
|
const message = `Temporary runner E2E state cleanup failed at ${temporaryRoot}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
|
|
for (const publishedResult of publishedResults) {
|
|
const publishedResultPath = publishedResultPaths.get(
|
|
publishedResult.executionId,
|
|
);
|
|
if (!publishedResultPath) continue;
|
|
Object.assign(publishedResult, {
|
|
status: "failed",
|
|
failureClass: "cleanup_failure",
|
|
error: publishedResult.error ? `${publishedResult.error}; ${message}` : message,
|
|
cleanup: "failed",
|
|
} satisfies Partial<RunnerE2EResult>);
|
|
const safeResult = `${JSON.stringify(
|
|
sanitizeJson(publishedResult, attemptSecrets),
|
|
null,
|
|
2,
|
|
)}\n`;
|
|
assertSecretFree(safeResult, attemptSecrets, publishedResultPath);
|
|
await writeFile(publishedResultPath, safeResult, "utf8");
|
|
}
|
|
// Cleanup failure is a failed cell, but must not suppress later cells in
|
|
// the same campaign. The result above carries the terminal failure.
|
|
console.error(message);
|
|
if (options.installedStartupOnly) throw new Error(message);
|
|
}
|
|
for (const result of publishedResults) {
|
|
const resultPath = publishedResultPaths.get(result.executionId);
|
|
const candidate = executions.find(candidate => candidate.id === result.executionId);
|
|
if (!traceConfig || !result.tracing || !resultPath || !candidate) continue;
|
|
const finalizationInput = {
|
|
receipt: result.tracing,
|
|
identity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id,
|
|
provider: candidate.profile.provider, model: candidate.profile.model, environment: candidate.environment.id },
|
|
provenance: { suiteId: candidate.suite.id, suiteDefinitionHash: candidate.suiteDefinitionHash, profileId: candidate.profile.id,
|
|
caseId: candidate.task.id, sourceRevision: resolveRunnerE2ESource().sha },
|
|
result,
|
|
};
|
|
result.tracing.finalization = input.operations
|
|
? await finalizeEvalAttemptThroughRelay(traceConfig, finalizationInput,
|
|
path.join(`${input.operations.manifest.ledgerPath}.otlp`, `${createHash("sha256").update(accountingAttemptId).digest("hex")}-finalization.jsonl`))
|
|
: await finalizeEvalAttemptTrace(traceConfig, finalizationInput);
|
|
const safeResult = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`;
|
|
assertSecretFree(safeResult, attemptSecrets, resultPath);
|
|
await writeFile(resultPath, safeResult, "utf8");
|
|
}
|
|
}
|
|
}
|
|
|
|
function printList(executions: readonly MatrixExecution[]) {
|
|
console.log("ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS");
|
|
for (const execution of executions) {
|
|
console.log(
|
|
[
|
|
execution.id,
|
|
execution.suite.id,
|
|
execution.profile.generation,
|
|
execution.profile.provider,
|
|
execution.profile.model,
|
|
execution.requiredCredentials.join(","),
|
|
].join("\t"),
|
|
);
|
|
}
|
|
}
|
|
|
|
async function runExecutionWithRetry(input: {
|
|
execution: MatrixExecution;
|
|
campaignId: string;
|
|
options: ReturnType<typeof parseRunnerSelectors>;
|
|
onAttempt: (result: RunnerE2EResult) => void;
|
|
operations?: EvalOperations;
|
|
}): Promise<RunnerE2EResult> {
|
|
const { execution, campaignId, options } = input;
|
|
return executeWithAutomaticRetry({
|
|
task: execution.task, options,
|
|
qualificationCandidate: execution.profile.qualificationCandidate !== undefined,
|
|
cancelled: () => cancelled,
|
|
onRetry: result => console.warn(
|
|
`Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`,
|
|
),
|
|
runAttempt: async attempt => {
|
|
const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options, operations: input.operations });
|
|
if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`);
|
|
input.onAttempt(result);
|
|
return result;
|
|
},
|
|
});
|
|
}
|
|
|
|
async function runWithConcurrency<T, R>(
|
|
values: readonly T[],
|
|
concurrency: number,
|
|
worker: (value: T) => Promise<R>,
|
|
): Promise<R[]> {
|
|
const results = new Array<R>(values.length);
|
|
let cursor = 0;
|
|
const workers = Array.from(
|
|
{ length: Math.min(concurrency, values.length) },
|
|
async () => {
|
|
while (true) {
|
|
const index = cursor;
|
|
cursor += 1;
|
|
if (index >= values.length) return;
|
|
if (cancelled) throw new Error("Runner E2E campaign cancelled");
|
|
results[index] = await worker(values[index]!);
|
|
}
|
|
},
|
|
);
|
|
const settled = await Promise.allSettled(workers);
|
|
const failures = settled.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
|
if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Eval campaign workers failed");
|
|
return results;
|
|
}
|
|
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
if (args.includes("--target-config") || args.includes("product-journeys")) {
|
|
const { launchProductJourneys } = await import("./product-journeys/launch.js");
|
|
return launchProductJourneys(args, repositoryRoot);
|
|
}
|
|
let options: ReturnType<typeof parseRunnerSelectors>;
|
|
try {
|
|
options = parseRunnerSelectors(process.argv.slice(2));
|
|
} catch (error) {
|
|
if (error instanceof RunnerSelectorError)
|
|
throw new Error(`${error.message}\nUse --list to inspect valid cells.`);
|
|
throw error;
|
|
}
|
|
const executions = selectRunnerExecutions(options, runnerMatrix);
|
|
manualPrivateHandoff = await readManualPrivateHandoff(process.env, executions.map(execution => execution.id));
|
|
resultsRoot = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "results") : resolveRunnerE2EResultsRoot(repositoryRoot);
|
|
if (options.installedStartupOnly) {
|
|
assertInstalledStartupOnly(process.env, executions, options);
|
|
assertRunnerE2EPrerequisites(executions);
|
|
const campaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `installed-startup-${Date.now()}`);
|
|
await runAttempt({ executions, attempt: 1, campaignId, options });
|
|
console.log(`PASS installed launch chain (no provider qualification) -> ${path.join(resultsRoot, campaignId, "installed-startup-only")}`);
|
|
return;
|
|
}
|
|
if (options.list) {
|
|
printList(executions);
|
|
return;
|
|
}
|
|
if (options.matrixJson) {
|
|
const jobs = buildMatrixJobs(executions);
|
|
console.log(
|
|
JSON.stringify({
|
|
include: jobs,
|
|
needsDaytona: jobs.some((job) => job.needsDaytona),
|
|
executionIds: executions.map((execution) => execution.id),
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST && executions.some(execution => execution.task.flow === "provider_connection")) throw new Error("Provider connection campaign has no budgeted routing qualification");
|
|
if (executions.some(execution => execution.task.flow === "antigravity_agent_journey")) {
|
|
if (options.connectionConfig || options.githubReviewConfig || options.provisionGitHubRepository) throw new Error("Use --antigravity-journey-config for the Antigravity journey.");
|
|
const { runAntigravityJourneyCampaign } = await import("./antigravity-journey-launch.js");
|
|
await runAntigravityJourneyCampaign({ executions, catalog: runnerMatrix, configFile: options.antigravityJourneyConfig, repositoryRoot,
|
|
preflight: options.antigravityJourneyPreflight, recoverFile: options.antigravityJourneyRecover });
|
|
return;
|
|
}
|
|
if (options.antigravityJourneyConfig || options.antigravityJourneyPreflight || options.antigravityJourneyRecover) throw new Error("Antigravity journey options require its explicit registered cell.");
|
|
if (executions.some(execution => execution.task.flow === "github_review_journey")) {
|
|
if (options.connectionConfig) throw new Error("Use --github-review-config for the GitHub journey.");
|
|
const { runGitHubReviewCampaign } = await import("./github-review-launch.js");
|
|
await runGitHubReviewCampaign({ executions, catalog: runnerMatrix, configFile: options.githubReviewConfig, repositoryRoot, provisionRepository: options.provisionGitHubRepository });
|
|
return;
|
|
}
|
|
if (options.githubReviewConfig || options.provisionGitHubRepository) throw new Error("GitHub journey options require an explicit GitHub review journey selection.");
|
|
if (executions.some(execution => execution.task.flow === "provider_connection")) {
|
|
const { runConnectionCampaign } = await import("./connection-launch.js");
|
|
await runConnectionCampaign({ executions, catalog: runnerMatrix, configFile: options.connectionConfig, repositoryRoot });
|
|
return;
|
|
}
|
|
if (options.connectionConfig) throw new Error("--connection-config is only supported by the provider-connections suite");
|
|
|
|
// Keep admission before local-env loading and credential checks. Pending
|
|
// profiles remain discoverable, but cannot reach a provider.
|
|
assertRunnerE2EPrerequisites(executions);
|
|
assertNativeCompletionSelection(executions);
|
|
assertNativeInstructionSelection(executions);
|
|
const campaignId = cleanId(
|
|
process.env.PAPERCLIP_E2E_CAMPAIGN_ID ??
|
|
`local-${new Date().toISOString().replace(/[:.]/g, "-")}`,
|
|
);
|
|
await loadLocalEnvironment(process.env);
|
|
resultsRoot = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "results") : resolveRunnerE2EResultsRoot(repositoryRoot);
|
|
const summaryDir = await prepareRunnerE2ECampaignDirectory(resultsRoot, campaignId);
|
|
if (executions.some(execution => execution.suite.id === "native-completion")) {
|
|
process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(summaryDir);
|
|
}
|
|
if (executions.some(execution => execution.suite.id === NATIVE_INSTRUCTION_SUITE)) {
|
|
process.env[NATIVE_INSTRUCTION_PREFLIGHT_ENV] = prepareNativeInstructionPreflight(summaryDir);
|
|
}
|
|
if (executions.some(execution => execution.suite.id === "stock-harness")) {
|
|
process.env[STOCK_PREFLIGHT_ENV] = prepareStockHarnessPreflight(summaryDir);
|
|
}
|
|
|
|
assertRemoteNativeEvidencePrerequisites(executions, process.env);
|
|
const missingCredentials = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST ? [] : [
|
|
...new Set(
|
|
executions.flatMap((execution) => execution.requiredCredentials),
|
|
),
|
|
].filter((name) => !process.env[name]?.trim());
|
|
if (missingCredentials.length > 0) {
|
|
throw new Error(
|
|
`Missing runner E2E credentials: ${missingCredentials.join(", ")}`,
|
|
);
|
|
}
|
|
if (
|
|
executions.some((execution) => execution.environment.id === "daytona") &&
|
|
!isImmutableDaytonaImage(process.env.PAPERCLIP_E2E_DAYTONA_IMAGE)
|
|
) {
|
|
throw new Error(
|
|
"PAPERCLIP_E2E_DAYTONA_IMAGE must be an immutable image@sha256 digest for Daytona cells",
|
|
);
|
|
}
|
|
|
|
|
|
await writeFile(
|
|
path.join(summaryDir, "invocation-policy.json"),
|
|
`${JSON.stringify(
|
|
{
|
|
version: 1,
|
|
maxAutomaticRetries: options.maxAutomaticRetries,
|
|
retryClasses: ["transient_infrastructure", "provider_variance"],
|
|
executions: executions.map(execution => ({
|
|
executionId: execution.id,
|
|
automaticRetryPolicy: execution.task.automaticRetryPolicy ?? "default",
|
|
maxAutomaticRetries: effectiveAutomaticRetryLimit(execution.task, options.maxAutomaticRetries),
|
|
})),
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
"utf8",
|
|
);
|
|
const requestedParallelism =
|
|
options.headed || options.ui || options.debug ? 1 : options.maxParallel;
|
|
console.log(
|
|
`Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`,
|
|
);
|
|
const attempts: RunnerE2EResult[] = [];
|
|
const operations = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST
|
|
? await openEvalOperations(process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, executions, options.maxAutomaticRetries, process.env) : undefined;
|
|
let finalResults: RunnerE2EResult[] = [];
|
|
let campaignFailed = false;
|
|
try {
|
|
if (operations) await writeFile(path.join(summaryDir, "operations-preflight.json"), `${JSON.stringify(operations.manifest, null, 2)}\n`, { mode: 0o600 });
|
|
finalResults = await runWithConcurrency(executions, operations ? Math.min(requestedParallelism, operations.manifest.limits.maxConcurrent) : requestedParallelism,
|
|
execution => runExecutionWithRetry({ execution, campaignId, options, operations, onAttempt: result => attempts.push(result) }));
|
|
} catch {
|
|
campaignFailed = true;
|
|
finalResults = [...new Map(attempts.map(result => [result.executionId, result])).values()];
|
|
} finally {
|
|
try { await operations?.close(); } catch { campaignFailed = true; }
|
|
}
|
|
|
|
const generatedAt = new Date().toISOString();
|
|
const operationsReport = operations ? await readEvalOperationsReport(operations.manifest.ledgerPath) : undefined;
|
|
if (operationsReport?.status === "unavailable") campaignFailed = true;
|
|
const campaign = buildRunnerCampaign({
|
|
campaignId,
|
|
generatedAt,
|
|
expected: executions.map((execution) => execution.id),
|
|
results: finalResults,
|
|
attempts,
|
|
...(operationsReport ? { operations: operationsReport } : {}),
|
|
});
|
|
const campaignSecrets = normalizedSecrets(
|
|
[...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => process.env[name]),
|
|
);
|
|
const safeCampaign = sanitizeJson(
|
|
campaign,
|
|
campaignSecrets,
|
|
) as typeof campaign;
|
|
const campaignText = `${JSON.stringify(safeCampaign, null, 2)}\n`;
|
|
assertSecretFree(campaignText, campaignSecrets, "campaign.json");
|
|
await writeFile(path.join(summaryDir, "campaign.json"), campaignText, "utf8");
|
|
const dashboard = renderRunnerE2EDashboard({
|
|
title: `Runner E2E · ${campaignId}`,
|
|
generatedAt,
|
|
expected: executions.map((execution) => execution.id),
|
|
catalog: runnerMatrix,
|
|
campaign: safeCampaign,
|
|
entries: safeCampaign.results.map((result) => ({
|
|
result,
|
|
valid: result.status === "passed" && result.cleanup === "passed",
|
|
errors:
|
|
result.status === "passed" && result.cleanup === "passed"
|
|
? []
|
|
: [
|
|
result.error ??
|
|
result.failureClass ??
|
|
`cleanup=${result.cleanup}`,
|
|
],
|
|
evidenceBaseHref: [
|
|
result.suiteId ?? "core-compatibility",
|
|
result.profileId,
|
|
result.environmentId,
|
|
result.caseId,
|
|
`attempt-${result.attempt}`,
|
|
].join("/"),
|
|
})),
|
|
});
|
|
assertSecretFree(dashboard, campaignSecrets, "dashboard.html");
|
|
await writeFile(path.join(summaryDir, "dashboard.html"), dashboard, "utf8");
|
|
console.log(
|
|
`Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`,
|
|
);
|
|
if (campaignFailed) console.error("Campaign finalization incomplete; inspect retained attempt artifacts and the operations ledger.");
|
|
if (campaignFailed || safeCampaign.failed > 0) process.exitCode = 1;
|
|
}
|
|
|
|
await main().catch((error) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
}).finally(async () => {
|
|
if (manualPrivateHandoff) await sealManualResultEvidence(manualPrivateHandoff, [...manualEvidenceSecrets]);
|
|
}).catch(() => { process.stderr.write("Private evidence finalization failed; no public fallback\n"); process.exitCode = 1; });
|