mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-07 16:11:46 +02:00
2867 lines
120 KiB
TypeScript
2867 lines
120 KiB
TypeScript
import { warmManagedFileEvidence } from "./warm-managed-files.js";
|
|
import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js";
|
|
import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js";
|
|
import { completionDelivery, type CompletionObservation } from "./completion-updates.js";
|
|
import { runInstructionPersistenceFlow } from "./instruction-persistence.js";
|
|
import { gradeApiResponsePaging, readResponseProof, responseEvidenceDescription } from "./api-response-reading.js";
|
|
import { runBlockerFlow } from "./blocker-flow.js";
|
|
import { largeJournalEvidence } from "./journal-evidence.js";
|
|
import { observeBrowserBootstrap } from "./browser-bootstrap-diagnostics.js";
|
|
import { runAccountingFlow } from "./accounting-flow.js";
|
|
import type { Issue } from "../../packages/shared/src/types/issue.js";
|
|
import { lifecycleLiveCase, gradeLifecycleRepair } from "./lifecycle-live-cases.js";
|
|
import { runContinuationFlow } from "./continuation-flow.js";
|
|
import { runEverydayFlow } from "./everyday-flow.js";
|
|
import { gradeTaskTitle } from "./task-titles.js";
|
|
import { runContextIntegrityFlow } from "./context-integrity-flow.js";
|
|
import { captureStockHarness, gradeStockHarness, gradeStockHire } from "./stock-harness.js";
|
|
import { createTaskThroughUi, submitTaskReply } from "./user-actions.js";
|
|
|
|
import { runFirstTaskFlow, setupFirstTaskFixtures } from "./first-task-flow.js";
|
|
import { runChatFlow } from "./chat-flow.js";
|
|
import { restartChatServer } from "./chat-restart.js";
|
|
import { matchesRunCount, minimumRunCount } from "./run-count.js";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { expect, test, type Page } from "@playwright/test";
|
|
import { RunnerApi, pollUntil } from "./api.js";
|
|
import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js";
|
|
import { runnerExecutionById } from "./catalog.js";
|
|
import { classifyFailure } from "./failure-classifier.js";
|
|
import { runnerE2EServerControlPaths } from "./harness-env.js";
|
|
import { setupConnectionReview } from "./connection-reviews.js";
|
|
import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
|
|
import { evaluateMatcher, persistedFinalRunMessage, type MatcherResult } from "./matchers.js";
|
|
import {
|
|
acceptedPlanSessionResetFailures,
|
|
collectRunEvents,
|
|
hasTerminalMalformedPlanConfirmation,
|
|
isControlPlaneGovernedResponseWait,
|
|
isNonExecutingReviewFenceRun,
|
|
isOpenRouterDeepSeekHelloTerminalVariance,
|
|
numberedPlanStepCount,
|
|
providerSessionContinuityFailures,
|
|
reasoningProjectionFailures,
|
|
} from "./run-observations.js";
|
|
import { resolveRunnerE2ESource } from "./source.js";
|
|
import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
|
|
import {
|
|
isPublicRunnerScreenshotRoute,
|
|
PUBLIC_RUNNER_SCREENSHOT_MARKER,
|
|
} from "./screenshot-policy.js";
|
|
import {
|
|
assertSecretFree,
|
|
findSecretLeakInJsonValues,
|
|
normalizedSecrets,
|
|
sanitizeJson,
|
|
} from "./redaction.js";
|
|
import {
|
|
CREDENTIAL_NAMES,
|
|
type CredentialName,
|
|
type FailureClass,
|
|
type RunnerE2EResult,
|
|
} from "./types.js";
|
|
import { assertRunnerE2EPrerequisites } from "./prerequisites.js";
|
|
|
|
interface IssueRecord {
|
|
id: string;
|
|
identifier?: string | null;
|
|
companyId: string;
|
|
title: string;
|
|
status: string;
|
|
workMode?: string;
|
|
assigneeAgentId?: string | null;
|
|
projectId?: string | null;
|
|
projectWorkspaceId?: string | null;
|
|
executionWorkspaceId?: string | null;
|
|
executionRunId?: string | null;
|
|
checkoutRunId?: string | null;
|
|
scheduledRetry?: unknown;
|
|
monitorNextCheckAt?: string | null;
|
|
}
|
|
|
|
interface CommentRecord {
|
|
id: string;
|
|
body?: string | null;
|
|
authorType?: string | null;
|
|
authorAgentId?: string | null;
|
|
authorUserId?: string | null;
|
|
createdByRunId?: string | null;
|
|
createdAt?: string;
|
|
}
|
|
|
|
interface RunRecord {
|
|
id: string;
|
|
companyId: string;
|
|
agentId: string;
|
|
status: string;
|
|
runtimeMode?: string;
|
|
continuationAttempt?: number;
|
|
retryOfRunId?: string | null;
|
|
runnerInstanceId?: string | null;
|
|
nativeSessionId?: string | null;
|
|
processPid?: number | null;
|
|
processStartedAt?: string | null;
|
|
contextSnapshot?: Record<string, unknown> | null;
|
|
runnerProfileJson?: Record<string, unknown> | null;
|
|
usageJson?: Record<string, unknown> | null;
|
|
resultJson?: Record<string, unknown> | null;
|
|
sessionIdBefore?: string | null;
|
|
sessionIdAfter?: string | null;
|
|
error?: string | null;
|
|
errorCode?: string | null;
|
|
startedAt?: string | null;
|
|
finishedAt?: string | null;
|
|
}
|
|
|
|
interface EnvironmentLeaseRecord {
|
|
id: string;
|
|
status?: string;
|
|
cleanupStatus?: string | null;
|
|
leasePolicy?: string;
|
|
providerLeaseId?: string | null;
|
|
executionWorkspaceId?: string | null;
|
|
metadata?: Record<string, unknown> | null;
|
|
issueId?: string | null;
|
|
heartbeatRunId?: string | null;
|
|
provider?: string | null;
|
|
acquiredAt?: string | null;
|
|
releasedAt?: string | null;
|
|
updatedAt?: string | null;
|
|
}
|
|
|
|
interface InteractionRecord {
|
|
id: string;
|
|
status: string;
|
|
kind?: string;
|
|
sourceRunId?: string | null;
|
|
continuationPolicy?: string;
|
|
payload?: {
|
|
version?: number;
|
|
acceptLabel?: string;
|
|
rejectLabel?: string;
|
|
target?: {
|
|
type?: string;
|
|
key?: string;
|
|
revisionId?: string;
|
|
revisionNumber?: number;
|
|
};
|
|
questions?: Array<{
|
|
id: string;
|
|
prompt: string;
|
|
selectionMode: "single" | "multi";
|
|
required?: boolean;
|
|
options: Array<{ id: string; label: string }>;
|
|
}>;
|
|
};
|
|
result?: {
|
|
version?: number;
|
|
answers?: Array<{
|
|
questionId?: string;
|
|
optionIds?: string[];
|
|
otherText?: string;
|
|
}>;
|
|
} | null;
|
|
}
|
|
interface IssueDocumentRecord {
|
|
id: string;
|
|
key: string;
|
|
body?: string | null;
|
|
latestRevisionId?: string | null;
|
|
latestRevisionNumber?: number;
|
|
}
|
|
interface RunEventRecord {
|
|
seq?: number;
|
|
eventType?: string;
|
|
payload?: Record<string, unknown> | null;
|
|
sourceInstanceId?: string | null;
|
|
sourceEventId?: string | null;
|
|
sourceSeq?: number | null;
|
|
protocolSchemaVersion?: number | null;
|
|
}
|
|
const TERMINAL_RUN_STATUSES = new Set([
|
|
"succeeded",
|
|
"interrupted",
|
|
"failed",
|
|
"cancelled",
|
|
"timed_out",
|
|
]);
|
|
const DEFINITIVE_FAILURE_RUN_STATUSES = new Set([
|
|
"failed",
|
|
"cancelled",
|
|
"timed_out",
|
|
]);
|
|
|
|
function definitiveRunFailure(runs: readonly RunRecord[]) {
|
|
const failed = runs.find((run) =>
|
|
DEFINITIVE_FAILURE_RUN_STATUSES.has(run.status),
|
|
);
|
|
if (!failed) return undefined;
|
|
return `heartbeat run ${failed.id} ended ${failed.status}${failed.errorCode ? ` (${failed.errorCode})` : ""}${failed.error ? `: ${failed.error}` : ""}`;
|
|
}
|
|
|
|
function chronologicalRunTime(run: RunRecord): number {
|
|
const value = run.startedAt ?? run.finishedAt;
|
|
if (!value) return 0;
|
|
const parsed = Date.parse(value);
|
|
return Number.isNaN(parsed) ? 0 : parsed;
|
|
}
|
|
|
|
function sortRunsChronologically(runs: readonly RunRecord[]): RunRecord[] {
|
|
return [...runs].sort(
|
|
(left, right) =>
|
|
chronologicalRunTime(left) - chronologicalRunTime(right) ||
|
|
left.id.localeCompare(right.id),
|
|
);
|
|
}
|
|
|
|
async function restartIsolatedPaperclipServer(input: {
|
|
api: RunnerApi;
|
|
requestId: string;
|
|
deadlineAt: number;
|
|
}): Promise<void> {
|
|
const {
|
|
controlDirectory,
|
|
restartRequestPath: requestPath,
|
|
restartAcknowledgementPath: acknowledgementPath,
|
|
} = runnerE2EServerControlPaths(temporaryRoot!);
|
|
await mkdir(controlDirectory, { recursive: true });
|
|
const temporaryRequestPath = `${requestPath}.${process.pid}.${input.requestId}.tmp`;
|
|
await writeFile(
|
|
temporaryRequestPath,
|
|
JSON.stringify({ requestId: input.requestId }),
|
|
{ encoding: "utf8", mode: 0o600 },
|
|
);
|
|
await rename(temporaryRequestPath, requestPath);
|
|
|
|
await pollUntil({
|
|
label: `isolated server restart ${input.requestId}`,
|
|
timeoutFailureClass: "transient_infrastructure",
|
|
deadlineAt: input.deadlineAt,
|
|
intervalMs: 250,
|
|
load: async () => {
|
|
try {
|
|
return JSON.parse(
|
|
await readFile(acknowledgementPath, "utf8"),
|
|
) as Record<string, unknown>;
|
|
} catch {
|
|
return {};
|
|
}
|
|
},
|
|
accept: (acknowledgement) =>
|
|
acknowledgement.requestId === input.requestId &&
|
|
acknowledgement.status === "ready",
|
|
reject: (acknowledgement) =>
|
|
acknowledgement.requestId === input.requestId &&
|
|
acknowledgement.status === "failed"
|
|
? String(acknowledgement.message ?? "replacement server failed")
|
|
: undefined,
|
|
});
|
|
await pollUntil({
|
|
label: `replacement server health ${input.requestId}`,
|
|
timeoutFailureClass: "transient_infrastructure",
|
|
deadlineAt: input.deadlineAt,
|
|
intervalMs: 250,
|
|
load: () => input.api.get<Record<string, unknown>>("/api/health"),
|
|
accept: (health) => Boolean(health),
|
|
});
|
|
}
|
|
|
|
const executionIds = (() => {
|
|
const encoded = process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS;
|
|
if (encoded) {
|
|
const parsed = JSON.parse(encoded) as unknown;
|
|
if (
|
|
!Array.isArray(parsed) ||
|
|
parsed.length === 0 ||
|
|
parsed.some((value) => typeof value !== "string")
|
|
) {
|
|
throw new Error(
|
|
"PAPERCLIP_RUNNER_E2E_EXECUTION_IDS must be a non-empty JSON string array",
|
|
);
|
|
}
|
|
return parsed;
|
|
}
|
|
const single = process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_ID;
|
|
if (!single)
|
|
throw new Error("PAPERCLIP_RUNNER_E2E_EXECUTION_IDS is required");
|
|
return [single];
|
|
})();
|
|
const executions = executionIds.map(runnerExecutionById);
|
|
// Direct Playwright invocation must enforce the same admission boundary as
|
|
// launch.ts before reading any provider credential from the environment.
|
|
assertRunnerE2EPrerequisites(executions);
|
|
const attempt = Number(process.env.PAPERCLIP_RUNNER_E2E_ATTEMPT ?? "1");
|
|
const temporaryRoot = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT;
|
|
const privateRoot = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR;
|
|
const workspacePath = process.env.PAPERCLIP_RUNNER_E2E_WORKSPACE;
|
|
if (!temporaryRoot || !privateRoot || !workspacePath)
|
|
throw new Error("Runner E2E temporary/private/workspace paths are required");
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function credentialValues(): Partial<Record<CredentialName, string>> {
|
|
return Object.fromEntries(
|
|
CREDENTIAL_NAMES.flatMap((name) => {
|
|
const value = process.env[name]?.trim();
|
|
return value ? [[name, value]] : [];
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function writeSanitizedJson(
|
|
directory: string,
|
|
name: string,
|
|
value: unknown,
|
|
secrets: readonly string[],
|
|
) {
|
|
const raw = `${JSON.stringify(value)}\n`;
|
|
// Public API payloads may contain harmless provider-shaped fixtures or
|
|
// opaque generated tokens. Reject exact campaign credentials before writing,
|
|
// then sanitize all known shapes and assert the published JSON is clean.
|
|
assertSecretFree(raw, secrets, name, { includeShapes: false });
|
|
const safe = `${JSON.stringify(sanitizeJson(value, secrets), null, 2)}\n`;
|
|
const leak = findSecretLeakInJsonValues(JSON.parse(safe), secrets);
|
|
if (leak) throw new Error(`Secret leak in ${name}: ${leak}`);
|
|
await mkdir(directory, { recursive: true });
|
|
await writeFile(path.join(directory, name), safe, "utf8");
|
|
}
|
|
|
|
async function submitTaskRevision(page: Page, body: string): Promise<number> {
|
|
const revise = page
|
|
.getByRole("button", { name: "Continue work", exact: true })
|
|
.last();
|
|
await expect(revise).toBeVisible({ timeout: 30_000 });
|
|
await revise.click();
|
|
const reason = page.getByPlaceholder("Add a short note").last();
|
|
await expect(reason).toBeVisible({ timeout: 30_000 });
|
|
await reason.fill(body);
|
|
const submittedAtMs = Date.now();
|
|
await page
|
|
.getByRole("button", { name: "Continue work", exact: true })
|
|
.last()
|
|
.click();
|
|
return submittedAtMs;
|
|
}
|
|
|
|
function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
|
|
const explicit = new Set(
|
|
[issue.executionRunId, issue.checkoutRunId].filter(Boolean),
|
|
);
|
|
return runs.filter((run) => {
|
|
if (isNonExecutingReviewFenceRun(run)) return false;
|
|
const context = record(run.contextSnapshot);
|
|
return (
|
|
context.issueId === issue.id ||
|
|
context.taskId === issue.id ||
|
|
explicit.has(run.id)
|
|
);
|
|
});
|
|
}
|
|
|
|
function isPendingPlanConfirmation(interaction: InteractionRecord) {
|
|
return (
|
|
interaction.kind === "request_confirmation" &&
|
|
interaction.status === "pending" &&
|
|
interaction.payload?.target?.type === "issue_document" &&
|
|
interaction.payload.target.key === "plan" &&
|
|
typeof interaction.payload.target.revisionId === "string"
|
|
);
|
|
}
|
|
|
|
function isPendingQuestion(interaction: InteractionRecord) {
|
|
return (
|
|
interaction.kind === "ask_user_questions" &&
|
|
interaction.status === "pending" &&
|
|
Boolean(interaction.payload?.questions?.length)
|
|
);
|
|
}
|
|
|
|
function isPendingWarmConfirmation(interaction: InteractionRecord) {
|
|
return (
|
|
interaction.kind === "request_confirmation" &&
|
|
interaction.status === "pending" &&
|
|
interaction.continuationPolicy === "wake_assignee" &&
|
|
interaction.payload?.target?.type === "custom"
|
|
);
|
|
}
|
|
|
|
function normalizePlanMarkdown(body: string | null | undefined) {
|
|
// Provider plan renderers may defensively escape underscores in plain-text
|
|
// markers. The rendered document and semantic marker are identical.
|
|
return (body ?? "").replaceAll("\\_", "_");
|
|
}
|
|
|
|
function nativeRunEventIntegrityFailures(
|
|
run: RunRecord,
|
|
events: readonly RunEventRecord[],
|
|
): string[] {
|
|
const failures: string[] = [];
|
|
let lastOuterSeq = 0;
|
|
const lastSourceSeq = new Map<string, number>();
|
|
const sourceEventIds = new Set<string>();
|
|
const runnerEventTypes: string[] = [];
|
|
const runTerminalSources: unknown[] = [];
|
|
const resultAcceptedSources: unknown[] = [];
|
|
|
|
for (const event of events) {
|
|
if (typeof event.seq !== "number" || event.seq <= lastOuterSeq) {
|
|
failures.push(
|
|
`run ${run.id} event sequence is not strictly monotonic at ${String(event.seq)}`,
|
|
);
|
|
} else {
|
|
lastOuterSeq = event.seq;
|
|
}
|
|
const envelope = record(event.payload?.prpEvent);
|
|
if (Object.keys(envelope).length === 0) continue;
|
|
if (!isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion)) {
|
|
failures.push(
|
|
`run ${run.id} exposed a malformed PRP envelope (schema=${String(envelope.schema)}, version=${String(envelope.schemaVersion)})`,
|
|
);
|
|
}
|
|
if (envelope.runId !== run.id) {
|
|
failures.push(
|
|
`run ${run.id} exposed an event bound to ${String(envelope.runId)}`,
|
|
);
|
|
}
|
|
if (envelope.eventType !== event.eventType) {
|
|
failures.push(
|
|
`run ${run.id} event discriminator changed from ${String(event.eventType)} to ${String(envelope.eventType)}`,
|
|
);
|
|
}
|
|
if (
|
|
envelope.sourceInstanceId !== event.sourceInstanceId ||
|
|
envelope.sourceEventId !== event.sourceEventId ||
|
|
envelope.sourceSeq !== event.sourceSeq
|
|
) {
|
|
failures.push(
|
|
`run ${run.id} event source identity changed during persistence/redaction`,
|
|
);
|
|
}
|
|
if (typeof event.sourceEventId === "string") {
|
|
if (sourceEventIds.has(event.sourceEventId)) {
|
|
failures.push(
|
|
`run ${run.id} duplicated source event ${event.sourceEventId}`,
|
|
);
|
|
}
|
|
sourceEventIds.add(event.sourceEventId);
|
|
}
|
|
if (
|
|
typeof event.sourceInstanceId === "string" &&
|
|
typeof event.sourceSeq === "number"
|
|
) {
|
|
const previous = lastSourceSeq.get(event.sourceInstanceId) ?? 0;
|
|
if (event.sourceSeq <= previous) {
|
|
failures.push(
|
|
`run ${run.id} source ${event.sourceInstanceId} sequence regressed from ${previous} to ${event.sourceSeq}`,
|
|
);
|
|
}
|
|
lastSourceSeq.set(event.sourceInstanceId, event.sourceSeq);
|
|
}
|
|
if (
|
|
envelope.sourceKind === "runner" &&
|
|
typeof event.eventType === "string"
|
|
) {
|
|
runnerEventTypes.push(event.eventType);
|
|
}
|
|
if (event.eventType === "run.terminal") {
|
|
runTerminalSources.push(envelope.sourceKind);
|
|
}
|
|
if (event.eventType === "run.result.accepted") {
|
|
resultAcceptedSources.push(envelope.sourceKind);
|
|
}
|
|
}
|
|
|
|
const runnerResultCount = runnerEventTypes.filter(
|
|
(value) => value === "run.result.proposed",
|
|
).length;
|
|
if (
|
|
runnerResultCount !== 1 &&
|
|
!(runnerResultCount === 0 && isControlPlaneGovernedResponseWait(events))
|
|
) {
|
|
failures.push(
|
|
`run ${run.id} must persist exactly one runner semantic result`,
|
|
);
|
|
}
|
|
if (resultAcceptedSources.length !== 1) {
|
|
failures.push(
|
|
`run ${run.id} must persist exactly one accepted semantic result`,
|
|
);
|
|
} else if (resultAcceptedSources[0] !== "control_plane") {
|
|
failures.push(
|
|
`run ${run.id} accepted semantic result must be control-plane authoritative`,
|
|
);
|
|
}
|
|
if (runTerminalSources.length !== 1) {
|
|
failures.push(`run ${run.id} must persist exactly one terminal event`);
|
|
} else if (runTerminalSources[0] !== "control_plane") {
|
|
failures.push(
|
|
`run ${run.id} terminal event must be control-plane authoritative`,
|
|
);
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
async function expectPlanStageVisible(
|
|
page: Page,
|
|
options: { native: boolean; revision: number | null },
|
|
) {
|
|
// A Plan flow is only qualified when the canonical saved document is
|
|
// projected as a card. Native profiles additionally require that card to be
|
|
// inside the runner turn, which proves write-boundary embedding rather than
|
|
// a standalone fallback. The API assertions own exact body-marker identity
|
|
// because the compact card intentionally shows only its first three lines.
|
|
const root = options.native
|
|
? page.locator('[data-testid="task-chat-turn"][data-settled="true"]')
|
|
: page.locator("body");
|
|
const preview = root.getByTestId("task-chat-plan-preview").last();
|
|
await expect(preview).toBeVisible({ timeout: 30_000 });
|
|
if (options.revision !== null) {
|
|
await expect(preview).toHaveAttribute(
|
|
"aria-label",
|
|
`Open Plan revision ${options.revision}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
for (const execution of executions) {
|
|
const privateDir = path.join(privateRoot, "cases", execution.task.id);
|
|
const resultPath = path.join(privateDir, "result.json");
|
|
const snapshotsDir = path.join(privateDir, "snapshots");
|
|
const deadlineMs = execution.task.attemptTimeoutMs[execution.environment.id];
|
|
|
|
test(`${execution.id} completes through the browser and public APIs`, async ({
|
|
page,
|
|
request,
|
|
}, testInfo) => {
|
|
test.setTimeout(
|
|
execution.task.flow === "warm_three_turn"
|
|
? deadlineMs
|
|
: deadlineMs + 90_000,
|
|
);
|
|
const startedAtMs = Date.now();
|
|
const startedAt = new Date(startedAtMs).toISOString();
|
|
const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
|
|
const marker = execution.task.buildVisibleMarker(nonce);
|
|
const title = execution.task.buildTitle(nonce);
|
|
let prompt = execution.task.buildPrompt(nonce);
|
|
let apiResponseSourceId: string | undefined;
|
|
let titleCreation: { issue: unknown; submitted: unknown } | undefined;
|
|
let lifecycleBlockerId: string | null = null;
|
|
const credentials = credentialValues();
|
|
const secrets = normalizedSecrets(Object.values(credentials));
|
|
const api = new RunnerApi(request);
|
|
const companyRunFlow = execution.suite.id === "task-titles" || ["blocker_guidance", "continuation_accounting", "continuation", "context_integrity", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence"].includes(execution.task.flow);
|
|
const consoleDiagnostics: Array<Record<string, unknown>> = [];
|
|
const networkDiagnostics: Array<Record<string, unknown>> = [];
|
|
const pageLifecycleDiagnostics: Array<Record<string, unknown>> = [];
|
|
const browserBootstrap = observeBrowserBootstrap(page);
|
|
let fixtures: LiveFixtureValues | undefined;
|
|
let reviewProvider: Awaited<ReturnType<typeof setupConnectionReview>> | undefined;
|
|
let issue: IssueRecord | undefined;
|
|
let selectedRuns: RunRecord[] = [];
|
|
let runtimeLeases: EnvironmentLeaseRecord[] = [];
|
|
let matcherResults: MatcherResult[] = [];
|
|
let downloadedResponseProof: Awaited<ReturnType<typeof readResponseProof>> | undefined;
|
|
const completionQuality: CompletionQualityRecord[] = [];
|
|
const completionEvidence = async (name: string, data: unknown) => {
|
|
await writeSanitizedJson(snapshotsDir, name, data, secrets);
|
|
if (!["completion-updates", "confirmation-replies"].includes(execution.suite.id) || !name.endsWith("completion-update.json")) return;
|
|
const probe = data as { observation?: CompletionObservation };
|
|
if (!probe.observation || !completionDelivery(probe.observation).checks.every(c => c.passed)) return;
|
|
if (!credentials.OPENAI_API_KEY) {
|
|
await writeSanitizedJson(snapshotsDir, "completion-quality-unqualified.json", { reason: "Judge credential unavailable in this provider-scoped job; run the separate judge against retained evidence." }, secrets);
|
|
return;
|
|
}
|
|
const samples = [{ name, purpose: "product" as const, expectedPass: true, observation: probe.observation },
|
|
...(execution.profile.id === "runner-codex" && execution.task.id === "handoff-completion-idle" ? completionQualityControls(probe.observation).map(c => ({ ...c, purpose: "calibration" as const, name: `${name}-${c.name}` })) : [])];
|
|
for (const sample of samples) {
|
|
const pending = { ...reserveCompletionQuality(sample.observation, 0.50, secrets), name: sample.name, purpose: sample.purpose, expectedPass: sample.expectedPass };
|
|
completionQuality.push(pending);
|
|
// Persist reservation and exact input before the one paid request. A crash
|
|
// leaves usage unknown, never zero, and the next campaign is a new attempt.
|
|
await writeSanitizedJson(snapshotsDir, `${sample.name}.quality-input.json`, sample, secrets);
|
|
await writeSanitizedJson(snapshotsDir, "completion-quality-ledger.json", completionQuality, secrets);
|
|
const result = await judgeCompletionQuality(sample.observation, pending, credentials.OPENAI_API_KEY ?? "", undefined, { approvedFixture: true, secrets });
|
|
completionQuality[completionQuality.length - 1] = { ...result, name: sample.name, purpose: sample.purpose, expectedPass: sample.expectedPass };
|
|
await writeSanitizedJson(snapshotsDir, "completion-quality-ledger.json", completionQuality, secrets);
|
|
}
|
|
};
|
|
let firstTaskEvidence: RunnerE2EResult["firstTask"];
|
|
let turnTimings: NonNullable<RunnerE2EResult["turnTimings"]> | undefined;
|
|
const turnSubmissionTimesMs: number[] = [];
|
|
const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
|
|
let primaryError: unknown;
|
|
let failureClassOverride: FailureClass | undefined;
|
|
let cleanup: RunnerE2EResult["cleanup"] = "not_started";
|
|
|
|
const capturePrivateScreenshot = async (id: string, file: string) => {
|
|
const screenshotPath = path.join(privateDir, file);
|
|
await page.screenshot({ path: screenshotPath, fullPage: true });
|
|
await testInfo.attach(id, {
|
|
path: screenshotPath,
|
|
contentType: "image/png",
|
|
});
|
|
};
|
|
|
|
const isReviewedFixtureScreenshotRoute = () =>
|
|
isPublicRunnerScreenshotRoute(page.url(), {
|
|
chatAgentId: execution.task.flow === "agent_chat" ? fixtures?.agent.id : undefined,
|
|
issuePrefix: fixtures?.company.issuePrefix,
|
|
issueId: issue?.id,
|
|
issueIdentifier: issue?.identifier,
|
|
});
|
|
|
|
const captureScreenshot = async (
|
|
id: string,
|
|
label: string,
|
|
file: string,
|
|
) => {
|
|
if (!isReviewedFixtureScreenshotRoute()) {
|
|
throw new Error(
|
|
`Public runner screenshot blocked on non-task route ${page.url()}`,
|
|
);
|
|
}
|
|
await capturePrivateScreenshot(id, file);
|
|
screenshots.push({
|
|
id,
|
|
label,
|
|
file,
|
|
publication: PUBLIC_RUNNER_SCREENSHOT_MARKER,
|
|
sha256: createHash("sha256")
|
|
.update(await readFile(path.join(privateDir, file)))
|
|
.digest("hex"),
|
|
});
|
|
};
|
|
|
|
const captureRuntimeLeases = async () => {
|
|
if (!fixtures || execution.environment.id !== "daytona") return;
|
|
const listed = await api.get<EnvironmentLeaseRecord[]>(
|
|
`/api/environments/${fixtures.environment.id}/leases`,
|
|
);
|
|
const selectedRunIds = new Set(selectedRuns.map((run) => run.id));
|
|
const relevant = listed.filter(
|
|
(lease) =>
|
|
lease.provider === "daytona" &&
|
|
(selectedRunIds.size === 0 ||
|
|
(lease.heartbeatRunId &&
|
|
selectedRunIds.has(lease.heartbeatRunId)) ||
|
|
(issue && lease.issueId === issue.id)),
|
|
);
|
|
runtimeLeases = relevant.length > 0 ? relevant : listed;
|
|
};
|
|
|
|
const cancelActiveRunsForCleanup = async () => {
|
|
if (!issue && !(companyRunFlow && fixtures)) return;
|
|
const cleanupIssueId = issue?.id;
|
|
const runs = await api.get<RunRecord[]>(
|
|
companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`,
|
|
);
|
|
const activeRunIds = [
|
|
...new Set(
|
|
runs
|
|
.filter((run) => !TERMINAL_RUN_STATUSES.has(run.status))
|
|
.map((run) => run.id)
|
|
.filter(
|
|
(runId): runId is string =>
|
|
typeof runId === "string" && runId.length > 0,
|
|
),
|
|
),
|
|
];
|
|
for (const runId of activeRunIds) {
|
|
await api.post(`/api/heartbeat-runs/${runId}/cancel`);
|
|
}
|
|
if (activeRunIds.length === 0) return;
|
|
const activeIds = new Set(activeRunIds);
|
|
await pollUntil({
|
|
label: `cleanup cancellation for issue ${cleanupIssueId}`,
|
|
deadlineAt: Date.now() + 45_000,
|
|
load: () => api.get<RunRecord[]>(companyRunFlow && fixtures ? `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100` : `/api/issues/${cleanupIssueId}/runs`),
|
|
accept: (currentRuns) =>
|
|
currentRuns
|
|
.filter((run) => activeIds.has(run.id))
|
|
.every((run) => TERMINAL_RUN_STATUSES.has(run.status)),
|
|
intervalMs: 500,
|
|
});
|
|
};
|
|
|
|
const captureFailureApiState = async () => {
|
|
if (!fixtures) return;
|
|
if (companyRunFlow && !issue) {
|
|
const companyRuns = await api.get<RunRecord[]>(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`);
|
|
selectedRuns = await Promise.all(companyRuns.map(run => api.get<RunRecord>(`/api/heartbeat-runs/${run.id}`)));
|
|
// Settings are already restored on failure, so chat resolution may be
|
|
// gated. Direct task access still permits evidence and usage capture.
|
|
const sourceId = selectedRuns.map(run => record(run.contextSnapshot).issueId).find(id => typeof id === "string");
|
|
if (typeof sourceId === "string") issue = await api.get<IssueRecord>(`/api/issues/${sourceId}`);
|
|
}
|
|
if (!issue) return;
|
|
const capture = async <T>(operation: () => Promise<T>) =>
|
|
operation().catch((error) => ({
|
|
evidenceCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
}));
|
|
const [currentIssue, listedRuns, comments, interactions] =
|
|
await Promise.all([
|
|
capture(() => api.get<IssueRecord>(`/api/issues/${issue!.id}`)),
|
|
capture(() =>
|
|
api.get<RunRecord[]>(
|
|
companyRunFlow
|
|
? `/api/companies/${fixtures!.company.id}/heartbeat-runs?limit=100`
|
|
: `/api/companies/${fixtures!.company.id}/heartbeat-runs?agentId=${fixtures!.agent.id}&limit=20`,
|
|
),
|
|
),
|
|
capture(() =>
|
|
api.get<CommentRecord[]>(
|
|
`/api/issues/${issue!.id}/comments?order=asc`,
|
|
),
|
|
),
|
|
capture(() =>
|
|
api.get<InteractionRecord[]>(
|
|
`/api/issues/${issue!.id}/interactions`,
|
|
),
|
|
),
|
|
]);
|
|
const taskRuns = Array.isArray(listedRuns)
|
|
? companyRunFlow ? listedRuns : matchingRuns(listedRuns, "id" in currentIssue ? currentIssue : issue)
|
|
: [];
|
|
const detailedRuns = await Promise.all(
|
|
taskRuns.map((candidate) =>
|
|
capture(() =>
|
|
api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`),
|
|
).then((value) => ("id" in value ? value : candidate)),
|
|
),
|
|
);
|
|
if (detailedRuns.length > 0) selectedRuns = detailedRuns;
|
|
const runEvidence = await Promise.all(
|
|
detailedRuns.map(async (candidate) => ({
|
|
runId: candidate.id,
|
|
workspaceOperations: await capture(() =>
|
|
api.get<unknown>(`/api/heartbeat-runs/${candidate.id}/workspace-operations`),
|
|
),
|
|
log: await capture(() =>
|
|
api.get<unknown>(
|
|
`/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
|
|
),
|
|
),
|
|
events: await capture(() =>
|
|
collectRunEvents<RunEventRecord>((afterSeq, limit) =>
|
|
api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
|
|
),
|
|
),
|
|
})),
|
|
);
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"api-state.json",
|
|
{
|
|
capturePhase: "failure",
|
|
issue: currentIssue,
|
|
runs: detailedRuns,
|
|
comments,
|
|
interactions,
|
|
runEvidence,
|
|
},
|
|
secrets,
|
|
);
|
|
};
|
|
|
|
page.on("console", (message) => {
|
|
if (["error", "warning"].includes(message.type())) {
|
|
consoleDiagnostics.push({
|
|
type: message.type(),
|
|
text: message.text(),
|
|
location: message.location(),
|
|
});
|
|
}
|
|
});
|
|
page.on("pageerror", (error) => {
|
|
pageLifecycleDiagnostics.push({
|
|
type: "pageerror",
|
|
message: error.message,
|
|
stack: error.stack ?? null,
|
|
url: page.url(),
|
|
});
|
|
});
|
|
page.on("framenavigated", (frame) => {
|
|
if (frame === page.mainFrame())
|
|
pageLifecycleDiagnostics.push({
|
|
type: "navigation",
|
|
url: frame.url(),
|
|
at: new Date().toISOString(),
|
|
});
|
|
});
|
|
page.on("requestfailed", (requestEvent) => {
|
|
networkDiagnostics.push({
|
|
method: requestEvent.method(),
|
|
url: requestEvent.url(),
|
|
failure: requestEvent.failure()?.errorText ?? null,
|
|
});
|
|
});
|
|
page.on("response", (response) => {
|
|
if (response.status() >= 400) {
|
|
networkDiagnostics.push({
|
|
method: response.request().method(),
|
|
url: response.url(),
|
|
status: response.status(),
|
|
statusText: response.statusText(),
|
|
});
|
|
}
|
|
});
|
|
|
|
try {
|
|
const experimental = await api.patch<{
|
|
enableNativeRunner: boolean;
|
|
}>("/api/instance/settings/experimental", {
|
|
enableNativeRunner: true,
|
|
...(["warm_three_turn", "everyday_workflow"].includes(execution.task.flow)
|
|
? { enableIsolatedWorkspaces: true }
|
|
: {}),
|
|
...(execution.profile.generation === "native" &&
|
|
execution.environment.id === "daytona"
|
|
? { enableRunnerPreviewIngress: true }
|
|
: {}),
|
|
});
|
|
expect(experimental.enableNativeRunner).toBe(true);
|
|
|
|
if (execution.suite.id === "daytona-git-streaming") {
|
|
await setupGitStreamingWorkspace(workspacePath);
|
|
}
|
|
fixtures = execution.task.flow === "first_task"
|
|
? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, observe: value => { fixtures = value; } })
|
|
: await setupLiveFixtures({
|
|
api,
|
|
execution,
|
|
executionNonce: nonce,
|
|
workspacePath,
|
|
credentials,
|
|
daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,
|
|
});
|
|
|
|
if (execution.suite.id === "stock-harness") {
|
|
const hire = await captureStockHarness({ api, companyId: fixtures.company.id,
|
|
agentId: fixtures.agent.id, generation: execution.profile.generation, runIds: [] });
|
|
const checks = gradeStockHire(hire);
|
|
await writeSanitizedJson(snapshotsDir, "stock-harness-hire.json", {
|
|
capturePhase: "before-provider", ...hire, checks,
|
|
}, secrets);
|
|
const failed = checks.filter(check => !check.passed);
|
|
if (failed.length) throw new Error(`Stock hire matcher failures: ${failed.map(check => check.id).join(", ")}`);
|
|
}
|
|
|
|
if (execution.suite.id === "api-response-reading") {
|
|
const source = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, {
|
|
title: `Synthetic diagnostic evidence ${nonce}`,
|
|
description: responseEvidenceDescription(nonce), status: "backlog",
|
|
});
|
|
apiResponseSourceId = source.id;
|
|
prompt = prompt.replaceAll("{{API_RESPONSE_SOURCE_ID}}", source.id);
|
|
}
|
|
|
|
if (execution.suite.id === "lifecycle-baseline" && lifecycleLiveCase(execution.task.id)?.family === "blocker") {
|
|
const prerequisite = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, {
|
|
title: `Supply dataset ${nonce}`,
|
|
description: "Fixture operator prerequisite: dataset has not been supplied.",
|
|
status: "backlog",
|
|
});
|
|
lifecycleBlockerId = prerequisite.id;
|
|
prompt = prompt.replaceAll("{{LIFECYCLE_BLOCKER_ID}}", prerequisite.id);
|
|
}
|
|
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"fixtures.json",
|
|
{
|
|
executionId: execution.id,
|
|
lifecycleBlockerId,
|
|
companyId: fixtures.company.id,
|
|
environmentId: fixtures.environment.id,
|
|
agentId: fixtures.agent.id,
|
|
secretIds: Object.fromEntries(
|
|
Object.entries(fixtures.secretRefs).map(([name, ref]) => [
|
|
name,
|
|
ref?.secretId,
|
|
]),
|
|
),
|
|
persistedAgent: await api.get<unknown>(
|
|
`/api/agents/${fixtures.agent.id}`,
|
|
),
|
|
persistedEnvironment: await api.get<unknown>(
|
|
`/api/environments/${fixtures.environment.id}`,
|
|
),
|
|
},
|
|
secrets,
|
|
);
|
|
|
|
if (execution.task.flow === "blocker_guidance") {
|
|
const blocker = await runBlockerFlow({ page, api, fixtures, execution, nonce, workspacePath,
|
|
deadlineAt: startedAtMs + deadlineMs - 30_000,
|
|
observe: (currentIssue, currentRuns, checks) => {
|
|
issue = currentIssue; selectedRuns = currentRuns;
|
|
matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `blocker.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
|
},
|
|
capture: captureScreenshot,
|
|
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
|
});
|
|
issue = blocker.issue as IssueRecord; selectedRuns = blocker.runs as RunRecord[];
|
|
} else if (execution.task.flow === "continuation_accounting") {
|
|
const accounting = await runAccountingFlow({
|
|
page, api, fixtures, execution, nonce, deadlineAt: startedAtMs + deadlineMs - 60_000,
|
|
restart: () => restartIsolatedPaperclipServer({ api, requestId: `accounting-${nonce}`, deadlineAt: startedAtMs + deadlineMs }),
|
|
observe: (currentIssue, currentRuns, checks) => {
|
|
issue = currentIssue; selectedRuns = currentRuns;
|
|
matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `accounting.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
|
},
|
|
capture: captureScreenshot,
|
|
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
|
});
|
|
issue = accounting.issue as IssueRecord; selectedRuns = accounting.runs as RunRecord[];
|
|
} else if (execution.task.flow === "instruction_persistence") {
|
|
const story = await runInstructionPersistenceFlow({
|
|
page, api, fixtures, execution, nonce, secrets, deadlineAt: startedAtMs + deadlineMs,
|
|
restart: () => restartIsolatedPaperclipServer({ api, requestId: `instructions-${nonce}`, deadlineAt: startedAtMs + deadlineMs }),
|
|
observe: (currentIssue, currentRuns) => { issue = currentIssue as IssueRecord; selectedRuns = currentRuns as RunRecord[]; },
|
|
capture: captureScreenshot,
|
|
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
|
});
|
|
issue = story.issue as IssueRecord; selectedRuns = story.runs as RunRecord[];
|
|
matcherResults = story.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `instructions.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
|
} else if (execution.task.flow === "continuation") {
|
|
const continuation = await runContinuationFlow({
|
|
page, api, fixtures, execution, nonce, secrets, workspacePath, deadlineAt: startedAtMs + deadlineMs - 60_000,
|
|
restart: () => restartIsolatedPaperclipServer({ api, requestId: `continuation-${nonce}`, deadlineAt: startedAtMs + deadlineMs }),
|
|
observe: (currentIssue, currentRuns, checks) => {
|
|
issue = currentIssue; selectedRuns = currentRuns;
|
|
matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `continuation.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
|
},
|
|
capture: captureScreenshot,
|
|
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
|
});
|
|
issue = continuation.issue as IssueRecord; selectedRuns = continuation.runs as RunRecord[];
|
|
} else if (execution.task.flow === "everyday_workflow") {
|
|
const story = await runEverydayFlow({
|
|
page, api, fixtures, execution, nonce, workspacePath, privateDir,
|
|
deadlineAt: startedAtMs + deadlineMs,
|
|
restart: () => restartIsolatedPaperclipServer({ api, requestId: `story-${nonce}`, deadlineAt: startedAtMs + deadlineMs }),
|
|
observe: (storyIssue, storyRuns) => { issue = storyIssue; selectedRuns = storyRuns; },
|
|
capture: captureScreenshot,
|
|
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
|
});
|
|
issue = story.issue; selectedRuns = story.runs;
|
|
matcherResults = story.evidence.checks.map(check => ({ matcher: { kind: "json_path" as const, path: check.id, expected: true }, passed: check.passed, detail: check.detail }));
|
|
} else if (execution.task.flow === "context_integrity") {
|
|
const contextIntegrity = await runContextIntegrityFlow({
|
|
page, api, fixtures, execution, nonce, deadlineAt: startedAtMs + deadlineMs,
|
|
capture: captureScreenshot,
|
|
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
|
observe: (currentIssue, currentRuns, checks) => {
|
|
issue = currentIssue as unknown as IssueRecord;
|
|
selectedRuns = currentRuns as unknown as RunRecord[];
|
|
matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: `contextIntegrity.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
|
},
|
|
});
|
|
issue = contextIntegrity.issue as unknown as IssueRecord;
|
|
selectedRuns = contextIntegrity.runs as unknown as RunRecord[];
|
|
} else if (execution.task.flow === "agent_chat") {
|
|
const chat = await runChatFlow({
|
|
page, api, fixtures, execution, nonce, workspacePath,
|
|
restart: () => restartChatServer(page, () => restartIsolatedPaperclipServer({ api, requestId: `chat-${nonce}`, deadlineAt: startedAtMs + deadlineMs })),
|
|
observe: (chatIssue, chatRuns) => { issue = chatIssue; selectedRuns = chatRuns; },
|
|
capture: captureScreenshot,
|
|
evidence: completionEvidence,
|
|
check: (id, passed, detail) => matcherResults.push({ matcher: { kind: "json_path", path: `chat.${id}`, expected: true }, passed, detail }),
|
|
});
|
|
issue = chat.issue; selectedRuns = chat.runs;
|
|
if (matcherResults.length === 0) matcherResults = [{ matcher: { kind: "issue_status", expected: "in_review" }, passed: true, detail: "Chat workflow and durable handoff/session assertions passed" }];
|
|
} else if (execution.task.flow === "first_task") {
|
|
const firstTask = await runFirstTaskFlow({
|
|
page, api, fixtures, execution, nonce, secrets,
|
|
observe: (currentIssue, runs, evidence) => {
|
|
issue = currentIssue; selectedRuns = runs; firstTaskEvidence = evidence;
|
|
matcherResults = evidence.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `firstTask.checks.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }));
|
|
},
|
|
createOrdinary: async (taskTitle, taskPrompt) => {
|
|
await createTaskThroughUi({ page, issuePrefix: fixtures!.company.issuePrefix!, agentName: fixtures!.agent.name, title: taskTitle, prompt: taskPrompt, workMode: "standard" });
|
|
const created = await pollUntil({ label: "ordinary UI-created task", deadlineAt: startedAtMs + deadlineMs,
|
|
load: async () => (await api.get<IssueRecord[]>(`/api/companies/${fixtures!.company.id}/issues?limit=100`)).find(row => row.title === taskTitle), accept: row => Boolean(row) });
|
|
if (!created) throw new Error("Missing ordinary task");
|
|
return created;
|
|
},
|
|
capture: captureScreenshot,
|
|
evidence: completionEvidence,
|
|
});
|
|
issue = firstTask.issue as IssueRecord; selectedRuns = firstTask.runs as RunRecord[];
|
|
} else {
|
|
const issuePrefix = fixtures.company.issuePrefix;
|
|
if (!issuePrefix)
|
|
throw new Error(
|
|
"Created fixture company did not return an issue prefix",
|
|
);
|
|
if (execution.task.flow === "governed_tool_review") {
|
|
reviewProvider = await setupConnectionReview({ page, api, prefix: issuePrefix, companyId: fixtures.company.id, agentId: fixtures.agent.id, marker });
|
|
}
|
|
// Capture the actual creation response: a generated title can change before
|
|
// a later list query, and the initial provisional state is part of the oracle.
|
|
const titleCreationResponse = execution.suite.id === "task-titles"
|
|
? page.waitForResponse(response => response.request().method() === "POST"
|
|
&& new URL(response.url()).pathname === `/api/companies/${fixtures!.company.id}/issues`,
|
|
{ timeout: Math.max(1, startedAtMs + deadlineMs - Date.now()) })
|
|
.catch((cause: unknown) => new Error("Could not capture task creation response", { cause }))
|
|
: null;
|
|
turnSubmissionTimesMs.push(
|
|
await createTaskThroughUi({
|
|
page,
|
|
issuePrefix,
|
|
agentName: fixtures.agent.name,
|
|
title,
|
|
prompt,
|
|
workMode: execution.task.workMode,
|
|
projectName: fixtures.project?.name,
|
|
}),
|
|
);
|
|
|
|
const deadlineAt = startedAtMs + deadlineMs;
|
|
if (titleCreationResponse) {
|
|
const response = await titleCreationResponse;
|
|
if (response instanceof Error) throw response;
|
|
expect(response.status()).toBe(201);
|
|
issue = await response.json() as IssueRecord;
|
|
titleCreation = { issue, submitted: response.request().postDataJSON() };
|
|
await writeSanitizedJson(snapshotsDir, "task-title-creation.json", titleCreation, secrets);
|
|
} else issue = await pollUntil({
|
|
label: `UI-created issue ${title}`,
|
|
deadlineAt,
|
|
load: async () => {
|
|
const issues = await api.get<IssueRecord[]>(
|
|
`/api/companies/${fixtures!.company.id}/issues?q=${encodeURIComponent(title)}&limit=50`,
|
|
);
|
|
return issues.find((candidate) => candidate.title === title);
|
|
},
|
|
accept: (candidate): candidate is IssueRecord => Boolean(candidate),
|
|
});
|
|
if (!issue) throw new Error(`Issue ${title} disappeared after creation`);
|
|
if (
|
|
issue.companyId !== fixtures.company.id ||
|
|
issue.assigneeAgentId !== fixtures.agent.id
|
|
) {
|
|
throw new Error(
|
|
"UI-created issue does not belong to the fixture company and agent",
|
|
);
|
|
}
|
|
if (issue.workMode !== execution.task.workMode) {
|
|
throw new Error(
|
|
`UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
|
|
);
|
|
}
|
|
if (fixtures.project) {
|
|
if (
|
|
issue.projectId !== fixtures.project.id ||
|
|
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace?.id
|
|
) {
|
|
throw new Error(
|
|
`Warm task did not retain its project/workspace scope: ${JSON.stringify({ projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId })}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
);
|
|
|
|
const loadTaskState = async () => {
|
|
const [currentIssue, runs, comments, interactions] = await Promise.all([
|
|
api.get<IssueRecord>(`/api/issues/${issue!.id}`),
|
|
api.get<RunRecord[]>(
|
|
companyRunFlow
|
|
? `/api/companies/${fixtures!.company.id}/heartbeat-runs?limit=100`
|
|
: `/api/companies/${fixtures!.company.id}/heartbeat-runs?agentId=${fixtures!.agent.id}&limit=20`,
|
|
),
|
|
api.get<CommentRecord[]>(
|
|
`/api/issues/${issue!.id}/comments?order=asc`,
|
|
),
|
|
api.get<InteractionRecord[]>(`/api/issues/${issue!.id}/interactions`),
|
|
]);
|
|
return {
|
|
currentIssue,
|
|
taskRuns: matchingRuns(runs, currentIssue),
|
|
comments,
|
|
interactions,
|
|
};
|
|
};
|
|
|
|
const rejectPlanConfirmationPoll = (input: {
|
|
taskRuns: RunRecord[];
|
|
interactions: InteractionRecord[];
|
|
minimumRunCount: number;
|
|
}) => {
|
|
const runFailure = definitiveRunFailure(input.taskRuns);
|
|
if (runFailure) return runFailure;
|
|
if (
|
|
!hasTerminalMalformedPlanConfirmation({
|
|
runs: input.taskRuns,
|
|
interactions: input.interactions,
|
|
minimumRunCount: input.minimumRunCount,
|
|
})
|
|
) {
|
|
return undefined;
|
|
}
|
|
failureClassOverride = "provider_variance";
|
|
return "succeeded heartbeat run created a pending request_confirmation without a revision-bound Plan target";
|
|
};
|
|
|
|
let planLifecycleEvidence: Record<string, unknown> | null = null;
|
|
let questionLifecycleEvidence: Record<string, unknown> | null = null;
|
|
let warmLifecycleEvidence: Record<string, unknown> | null = null;
|
|
let expectedQuestionResolution: {
|
|
interactionId: string;
|
|
optionId: string;
|
|
} | null = null;
|
|
if (execution.task.flow === "governed_tool_review") {
|
|
await expect(page.getByRole("button", { name: "Approve & run", exact: true })).toBeVisible({ timeout: Math.max(1, deadlineAt - Date.now()) });
|
|
expect(reviewProvider!.invocationCount()).toBe(0);
|
|
await pollUntil({ label: "governed waiting turn", deadlineAt, load: loadTaskState, accept: state => state.taskRuns.length === 1 && state.taskRuns.every(run => TERMINAL_RUN_STATUSES.has(run.status)) });
|
|
await captureScreenshot("tool-review-pending", "Connection review awaiting a human", "tool-review-pending.png");
|
|
await page.getByRole("button", { name: "Dismiss Approve tool action" }).click();
|
|
await page.getByRole("button", { name: "Review request", exact: true }).click();
|
|
if (execution.task.toolReviewDecision === "restart") {
|
|
await restartIsolatedPaperclipServer({ api, requestId: `tool-review-${nonce}`, deadlineAt });
|
|
await page.reload();
|
|
}
|
|
if (execution.task.toolReviewDecision === "always") {
|
|
await page.getByRole("button", { name: "Approval options", exact: true }).click();
|
|
await page.getByRole("menuitem", { name: "Always allow", exact: true }).click();
|
|
} else {
|
|
await page.getByRole("button", { name: execution.task.toolReviewDecision === "decline" ? "Decline" : "Approve & run", exact: true }).click();
|
|
}
|
|
} else if (execution.task.flow === "plan_revision_acceptance") {
|
|
const planMarkers = execution.task.buildPlanMarkers?.(nonce);
|
|
const revisionRequest = execution.task.buildRevisionRequest?.(nonce);
|
|
if (!planMarkers || !revisionRequest) {
|
|
throw new Error(
|
|
`Plan fixture ${execution.task.id} is missing lifecycle factories`,
|
|
);
|
|
}
|
|
const draftState = await pollUntil({
|
|
label: `initial plan confirmation for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 1 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(isPendingPlanConfirmation),
|
|
reject: ({ taskRuns, interactions }) =>
|
|
rejectPlanConfirmationPoll({
|
|
taskRuns,
|
|
interactions,
|
|
minimumRunCount: 1,
|
|
}),
|
|
});
|
|
const draftInteraction = draftState.interactions.find(
|
|
isPendingPlanConfirmation,
|
|
)!;
|
|
const draftPlan = await api.get<IssueDocumentRecord>(
|
|
`/api/issues/${issue.id}/documents/plan`,
|
|
);
|
|
if (
|
|
!normalizePlanMarkdown(draftPlan.body).includes(planMarkers.draft)
|
|
) {
|
|
throw new Error(
|
|
`Initial Plan document did not contain ${planMarkers.draft}`,
|
|
);
|
|
}
|
|
if (numberedPlanStepCount(draftPlan.body) !== 2) {
|
|
throw new Error(
|
|
"Initial Plan must contain exactly two numbered steps",
|
|
);
|
|
}
|
|
if (
|
|
draftInteraction.payload?.target?.revisionId !==
|
|
draftPlan.latestRevisionId
|
|
) {
|
|
throw new Error(
|
|
"Initial plan confirmation did not target the latest Plan revision",
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expectPlanStageVisible(page, {
|
|
native: execution.profile.expectedRuntimeMode === "native",
|
|
revision: draftPlan.latestRevisionNumber ?? null,
|
|
});
|
|
await captureScreenshot(
|
|
"plan-draft",
|
|
"Initial plan awaiting revision",
|
|
"plan-draft.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: draftInteraction.payload?.rejectLabel ?? "Reject",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
const revisionComposer = page
|
|
.getByTestId("plan-revision-composer")
|
|
.last();
|
|
await expect(revisionComposer).toBeVisible({ timeout: 10_000 });
|
|
await revisionComposer
|
|
.locator('[contenteditable="true"], textarea')
|
|
.first()
|
|
.fill(revisionRequest);
|
|
await page
|
|
.getByRole("button", {
|
|
name: draftInteraction.payload?.rejectLabel ?? "Reject",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
|
|
const revisedState = await pollUntil({
|
|
label: `revised plan confirmation for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 2 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(
|
|
(interaction) =>
|
|
isPendingPlanConfirmation(interaction) &&
|
|
interaction.id !== draftInteraction.id,
|
|
),
|
|
reject: ({ taskRuns, interactions }) =>
|
|
rejectPlanConfirmationPoll({
|
|
taskRuns,
|
|
interactions,
|
|
minimumRunCount: 2,
|
|
}),
|
|
});
|
|
const revisedInteraction = revisedState.interactions.find(
|
|
(interaction) =>
|
|
isPendingPlanConfirmation(interaction) &&
|
|
interaction.id !== draftInteraction.id,
|
|
)!;
|
|
const revisedPlan = await api.get<IssueDocumentRecord>(
|
|
`/api/issues/${issue.id}/documents/plan`,
|
|
);
|
|
const normalizedRevisedPlan = normalizePlanMarkdown(revisedPlan.body);
|
|
if (
|
|
!normalizedRevisedPlan.includes(planMarkers.revised) ||
|
|
normalizedRevisedPlan.includes(planMarkers.draft)
|
|
) {
|
|
throw new Error(
|
|
`Revised Plan must replace ${planMarkers.draft} with ${planMarkers.revised}`,
|
|
);
|
|
}
|
|
if (numberedPlanStepCount(revisedPlan.body) !== 3) {
|
|
throw new Error(
|
|
"Revised Plan must contain exactly three numbered steps",
|
|
);
|
|
}
|
|
if (
|
|
revisedInteraction.payload?.target?.revisionId !==
|
|
revisedPlan.latestRevisionId ||
|
|
revisedPlan.latestRevisionId === draftPlan.latestRevisionId
|
|
) {
|
|
throw new Error(
|
|
"Revised confirmation did not target a new latest Plan revision",
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expectPlanStageVisible(page, {
|
|
native: execution.profile.expectedRuntimeMode === "native",
|
|
revision: revisedPlan.latestRevisionNumber ?? null,
|
|
});
|
|
await captureScreenshot(
|
|
"plan-revised",
|
|
"Revised plan awaiting acceptance",
|
|
"plan-revised.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: revisedInteraction.payload?.acceptLabel ?? "Approve",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
planLifecycleEvidence = {
|
|
draftInteraction,
|
|
draftPlan,
|
|
revisedInteraction,
|
|
revisedPlan,
|
|
revisionRequest,
|
|
};
|
|
} else if (execution.task.flow === "question_resume_completion") {
|
|
const expectedAnswer = execution.task.buildQuestionAnswer?.(nonce);
|
|
if (!expectedAnswer) {
|
|
throw new Error(
|
|
`Question fixture ${execution.task.id} is missing its answer factory`,
|
|
);
|
|
}
|
|
const pendingState = await pollUntil({
|
|
label: `pending user question for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 1 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(isPendingQuestion),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
const questionInteractions = pendingState.interactions.filter(
|
|
(interaction) => interaction.kind === "ask_user_questions",
|
|
);
|
|
if (
|
|
questionInteractions.length !== 1 ||
|
|
!isPendingQuestion(questionInteractions[0]!)
|
|
) {
|
|
throw new Error(
|
|
`Expected exactly one pending question interaction; observed ${JSON.stringify(questionInteractions)}`,
|
|
);
|
|
}
|
|
const questionInteraction = questionInteractions[0]!;
|
|
const questions = questionInteraction.payload?.questions ?? [];
|
|
if (questionInteraction.payload?.version !== 1) {
|
|
throw new Error("Question interaction omitted payload version 1");
|
|
}
|
|
if (questions.length !== 1) {
|
|
throw new Error(
|
|
`Expected exactly one question; observed ${questions.length}`,
|
|
);
|
|
}
|
|
const question = questions[0]!;
|
|
if (
|
|
question.id !== "verification-word" ||
|
|
question.prompt !== "Choose the verification word." ||
|
|
question.selectionMode !== "single" ||
|
|
question.required !== true ||
|
|
JSON.stringify(question.options) !==
|
|
JSON.stringify([
|
|
{ id: "cobalt", label: "Cobalt" },
|
|
{ id: "amber", label: "Amber" },
|
|
])
|
|
) {
|
|
throw new Error(
|
|
`Question interaction did not preserve the exact verification contract: ${JSON.stringify(question)}`,
|
|
);
|
|
}
|
|
const selectedOption = question.options.find(
|
|
(option) => option.label === expectedAnswer.optionLabel,
|
|
);
|
|
if (!selectedOption) {
|
|
throw new Error(
|
|
`Question interaction omitted ${expectedAnswer.optionLabel}`,
|
|
);
|
|
}
|
|
expectedQuestionResolution = {
|
|
interactionId: questionInteraction.id,
|
|
optionId: selectedOption.id,
|
|
};
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expect(
|
|
page
|
|
.getByRole("radio", {
|
|
name: expectedAnswer.optionLabel,
|
|
exact: true,
|
|
})
|
|
.last(),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
await captureScreenshot(
|
|
"question-pending",
|
|
"Structured question awaiting an answer",
|
|
"question-pending.png",
|
|
);
|
|
if (execution.task.restartServerBeforeQuestionAnswer) {
|
|
const restartRequestId = `question-wait-${nonce}`;
|
|
await restartIsolatedPaperclipServer({
|
|
api,
|
|
requestId: restartRequestId,
|
|
deadlineAt,
|
|
});
|
|
const documentSentinel = `__paperclip_runner_restart_${nonce.replaceAll("-", "_")}`;
|
|
await page.evaluate(
|
|
(key) => Reflect.set(window, key, true),
|
|
documentSentinel,
|
|
);
|
|
try {
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
// The replacement Vite server can commit and render a fresh
|
|
// document while its navigation lifecycle remains unsettled.
|
|
// The sentinel and explicit assertions below prove the new
|
|
// document and durable state even when Playwright times out.
|
|
{ waitUntil: "commit" },
|
|
);
|
|
} catch (error) {
|
|
if (!(error instanceof Error) || error.name !== "TimeoutError") {
|
|
throw error;
|
|
}
|
|
}
|
|
await expect(
|
|
page
|
|
.getByRole("radio", {
|
|
name: expectedAnswer.optionLabel,
|
|
exact: true,
|
|
})
|
|
.last(),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
expect(
|
|
await page.evaluate(
|
|
(key) => Reflect.get(window, key) === true,
|
|
documentSentinel,
|
|
),
|
|
).toBe(false);
|
|
const reloadedInteractions = await api.get<InteractionRecord[]>(
|
|
`/api/issues/${issue.id}/interactions`,
|
|
);
|
|
const reloadedQuestions = reloadedInteractions.filter(
|
|
(interaction) => interaction.kind === "ask_user_questions",
|
|
);
|
|
if (
|
|
reloadedQuestions.length !== 1 ||
|
|
reloadedQuestions[0]?.id !== questionInteraction.id ||
|
|
!isPendingQuestion(reloadedQuestions[0])
|
|
) {
|
|
throw new Error(
|
|
`Server restart did not preserve the exact pending interaction ${questionInteraction.id}: ${JSON.stringify(reloadedQuestions)}`,
|
|
);
|
|
}
|
|
await captureScreenshot(
|
|
"question-pending-after-server-restart",
|
|
"Structured question preserved across server restart",
|
|
"question-pending-after-server-restart.png",
|
|
);
|
|
}
|
|
await page
|
|
.getByRole("radio", {
|
|
name: expectedAnswer.optionLabel,
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.check();
|
|
// The one-question form is on its last page, so selecting the radio
|
|
// records the answer and the existing form button submits it.
|
|
await page
|
|
.getByRole("button", { name: "Submit answers", exact: true })
|
|
.last()
|
|
.click();
|
|
questionLifecycleEvidence = {
|
|
interaction: questionInteraction,
|
|
answer: expectedAnswer.optionLabel,
|
|
expectedMarker: expectedAnswer.expectedMarker,
|
|
serverRestartedBeforeAnswer:
|
|
execution.task.restartServerBeforeQuestionAnswer ?? false,
|
|
};
|
|
} else if (execution.task.flow === "plan_approval_completion") {
|
|
const planMarkers = execution.task.buildPlanMarkers?.(nonce);
|
|
if (!planMarkers) {
|
|
throw new Error(
|
|
`Plan fixture ${execution.task.id} is missing its marker factory`,
|
|
);
|
|
}
|
|
const pendingState = await pollUntil({
|
|
label: `pending plan approval for issue ${issue.id}`,
|
|
deadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, interactions }) =>
|
|
taskRuns.length >= 1 &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
|
|
interactions.some(isPendingPlanConfirmation),
|
|
reject: ({ taskRuns, interactions }) =>
|
|
rejectPlanConfirmationPoll({
|
|
taskRuns,
|
|
interactions,
|
|
minimumRunCount: 1,
|
|
}),
|
|
});
|
|
const interaction = pendingState.interactions.find(
|
|
isPendingPlanConfirmation,
|
|
)!;
|
|
const plan = await api.get<IssueDocumentRecord>(
|
|
`/api/issues/${issue.id}/documents/plan`,
|
|
);
|
|
if (!normalizePlanMarkdown(plan.body).includes(planMarkers.draft)) {
|
|
throw new Error(`Plan document did not contain ${planMarkers.draft}`);
|
|
}
|
|
if (numberedPlanStepCount(plan.body) !== 2) {
|
|
throw new Error("Plan must contain exactly two numbered steps");
|
|
}
|
|
if (interaction.payload?.target?.revisionId !== plan.latestRevisionId) {
|
|
throw new Error(
|
|
"Plan confirmation did not target the latest Plan revision",
|
|
);
|
|
}
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await expectPlanStageVisible(page, {
|
|
native: execution.profile.expectedRuntimeMode === "native",
|
|
revision: plan.latestRevisionNumber ?? null,
|
|
});
|
|
await captureScreenshot(
|
|
"plan-pending",
|
|
"Plan awaiting approval",
|
|
"plan-pending.png",
|
|
);
|
|
await page
|
|
.getByRole("button", {
|
|
name: interaction.payload?.acceptLabel ?? "Approve",
|
|
exact: true,
|
|
})
|
|
.last()
|
|
.click();
|
|
planLifecycleEvidence = { interaction, plan };
|
|
} else if (execution.task.flow === "warm_three_turn") {
|
|
const followups = execution.task.buildFollowupMessages?.(nonce);
|
|
if (!followups || !fixtures.project?.primaryWorkspace?.id) {
|
|
throw new Error(
|
|
`Warm fixture ${execution.task.id} is missing its project or follow-up messages`,
|
|
);
|
|
}
|
|
const workspaceFile = path.join(
|
|
workspacePath,
|
|
`daytona-warm-${nonce}.txt`,
|
|
);
|
|
const turnEvidence: Array<Record<string, unknown>> = [];
|
|
for (const completedTurn of [1, 2] as const) {
|
|
const turnDeadlineAt = Math.min(
|
|
deadlineAt,
|
|
turnSubmissionTimesMs[completedTurn - 1]! +
|
|
(execution.task.turnTimeoutMs ?? 10 * 60_000),
|
|
);
|
|
const waitingState = await pollUntil({
|
|
label: `warm Daytona turn ${completedTurn} review state for issue ${issue.id}`,
|
|
deadlineAt: turnDeadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ currentIssue, taskRuns, interactions }) => {
|
|
const latestRun = sortRunsChronologically(taskRuns).at(-1);
|
|
const pendingConfirmations = interactions.filter(
|
|
isPendingWarmConfirmation,
|
|
);
|
|
return (
|
|
currentIssue.status === "in_review" &&
|
|
taskRuns.length === completedTurn &&
|
|
taskRuns.every((run) => run.status === "succeeded") &&
|
|
pendingConfirmations.length === 1 &&
|
|
pendingConfirmations[0]?.sourceRunId === latestRun?.id
|
|
);
|
|
},
|
|
reject: ({ taskRuns }) =>
|
|
definitiveRunFailure(taskRuns) ??
|
|
(taskRuns.length > completedTurn
|
|
? `warm turn ${completedTurn} dispatched duplicate runs`
|
|
: undefined),
|
|
});
|
|
if (execution.suite.id === "daytona-git-streaming") {
|
|
await writeSanitizedJson(snapshotsDir, `git-copyback-turn-${completedTurn}.json`, await gitStreamingEvidence(workspacePath, completedTurn), secrets);
|
|
const finalization = await gitFinalizationEvidence(api, issue.id, waitingState.taskRuns.map(run => run.id));
|
|
await writeSanitizedJson(snapshotsDir, `git-finalization-turn-${completedTurn}.json`, finalization, secrets);
|
|
expect(finalization.passed, finalization.failures.join("; ")).toBe(true);
|
|
}
|
|
if (execution.suite.id === "daytona-warm-continuity") {
|
|
const evidence = await warmManagedFileEvidence(api, fixtures.agent.id, sortRunsChronologically(waitingState.taskRuns).at(-1)!.id, completedTurn, nonce, execution.profile.generation === "native");
|
|
await writeSanitizedJson(snapshotsDir, `managed-warm-turn-${completedTurn}.json`, evidence, secrets);
|
|
expect(evidence.passed, JSON.stringify(evidence.checks)).toBe(true);
|
|
}
|
|
const expectedPrefix = `${Array.from(
|
|
{ length: completedTurn },
|
|
(_, index) => `T${index + 1}-${nonce}`,
|
|
).join("\n")}\n`;
|
|
const hostContent = await readFile(workspaceFile, "utf8");
|
|
if (hostContent !== expectedPrefix) {
|
|
throw new Error(
|
|
`Host workspace was not finalized after warm turn ${completedTurn}: expected ${JSON.stringify(expectedPrefix)}, observed ${JSON.stringify(hostContent)}`,
|
|
);
|
|
}
|
|
if (
|
|
waitingState.currentIssue.projectId !== fixtures.project.id ||
|
|
waitingState.currentIssue.projectWorkspaceId !==
|
|
fixtures.project.primaryWorkspace.id ||
|
|
!waitingState.currentIssue.executionWorkspaceId
|
|
) {
|
|
throw new Error(
|
|
`Warm turn ${completedTurn} lost its project execution-workspace scope`,
|
|
);
|
|
}
|
|
const chronologicalRuns = sortRunsChronologically(
|
|
waitingState.taskRuns,
|
|
);
|
|
const completedRunIds = new Set(
|
|
chronologicalRuns.map((candidate) => candidate.id),
|
|
);
|
|
const retainedTurnLeases = await pollUntil({
|
|
label: `retained Daytona leases after warm turn ${completedTurn}`,
|
|
deadlineAt: Math.min(turnDeadlineAt, Date.now() + 30_000),
|
|
intervalMs: 500,
|
|
load: () =>
|
|
api.get<EnvironmentLeaseRecord[]>(
|
|
`/api/environments/${fixtures!.environment.id}/leases`,
|
|
),
|
|
accept: (leases) => {
|
|
const runOrder = new Map(
|
|
chronologicalRuns.map((candidate, index) => [
|
|
candidate.id,
|
|
index,
|
|
]),
|
|
);
|
|
const completed = leases
|
|
.filter(
|
|
(lease) =>
|
|
lease.heartbeatRunId &&
|
|
completedRunIds.has(lease.heartbeatRunId),
|
|
)
|
|
.sort(
|
|
(left, right) =>
|
|
(runOrder.get(left.heartbeatRunId ?? "") ?? 0) -
|
|
(runOrder.get(right.heartbeatRunId ?? "") ?? 0),
|
|
);
|
|
return (
|
|
completed.length === completedTurn &&
|
|
completed
|
|
.slice(0, -1)
|
|
.every(
|
|
(lease) =>
|
|
lease.status === "expired" &&
|
|
lease.cleanupStatus === "success",
|
|
) &&
|
|
completed.at(-1)?.status === "retained" &&
|
|
completed.every(
|
|
(lease) =>
|
|
lease.leasePolicy === "reuse_by_environment" &&
|
|
typeof lease.providerLeaseId === "string" &&
|
|
record(lease.metadata).sandboxState === "started",
|
|
) &&
|
|
completed
|
|
.slice(1)
|
|
.every(
|
|
(lease) =>
|
|
record(lease.metadata).resumedFromState === "started",
|
|
)
|
|
);
|
|
},
|
|
});
|
|
const turnRunOrder = new Map(
|
|
chronologicalRuns.map((candidate, index) => [candidate.id, index]),
|
|
);
|
|
const completedLeases = retainedTurnLeases
|
|
.filter(
|
|
(lease) =>
|
|
lease.heartbeatRunId &&
|
|
completedRunIds.has(lease.heartbeatRunId),
|
|
)
|
|
.sort(
|
|
(left, right) =>
|
|
(turnRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
|
|
(turnRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
|
|
);
|
|
if (
|
|
new Set(completedLeases.map((lease) => lease.providerLeaseId))
|
|
.size !== 1
|
|
) {
|
|
throw new Error(
|
|
`Warm turn ${completedTurn} replaced its Daytona sandbox`,
|
|
);
|
|
}
|
|
const journalEvidence = execution.suite.id === "daytona-journal-continuity" && completedTurn === 1
|
|
? await largeJournalEvidence({
|
|
stateRoot: path.join(process.env.PAPERCLIP_HOME!, "instances", process.env.PAPERCLIP_INSTANCE_ID!, "runtime", "paperclip-runner", "durable-sessions"),
|
|
runId: chronologicalRuns.at(-1)!.id,
|
|
minimumBytes: 2 * 1024 * 1024,
|
|
minimumCompletedStimulusCalls: 240,
|
|
})
|
|
: undefined;
|
|
if (journalEvidence) {
|
|
await writeSanitizedJson(snapshotsDir, "large-journal-boundary.json", journalEvidence, secrets);
|
|
}
|
|
turnEvidence.push({
|
|
...(journalEvidence ? { journalEvidence } : {}),
|
|
turn: completedTurn,
|
|
issue: waitingState.currentIssue,
|
|
run: chronologicalRuns.at(-1),
|
|
hostContent,
|
|
leases: completedLeases,
|
|
});
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
await captureScreenshot(
|
|
`warm-turn-${completedTurn}`,
|
|
`Warm Daytona turn ${completedTurn} awaiting review`,
|
|
`warm-turn-${completedTurn}.png`,
|
|
);
|
|
turnSubmissionTimesMs.push(
|
|
await submitTaskRevision(page, followups[completedTurn - 1]),
|
|
);
|
|
}
|
|
warmLifecycleEvidence = { turns: turnEvidence };
|
|
}
|
|
|
|
const taskMatchers = execution.task.buildMatchers(nonce, execution);
|
|
const terminalDeadlineAt =
|
|
execution.task.flow === "warm_three_turn"
|
|
? Math.min(
|
|
deadlineAt,
|
|
turnSubmissionTimesMs.at(-1)! +
|
|
(execution.task.turnTimeoutMs ?? 10 * 60_000),
|
|
)
|
|
: deadlineAt;
|
|
let terminal = await pollUntil({
|
|
label: `issue ${issue.id} and heartbeat run terminal state`,
|
|
deadlineAt: terminalDeadlineAt,
|
|
load: loadTaskState,
|
|
accept: ({ currentIssue, taskRuns }) =>
|
|
currentIssue.status === execution.task.expectedTerminalState.issue &&
|
|
taskRuns.length >= minimumRunCount(execution.task) &&
|
|
taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)),
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
});
|
|
|
|
// Finalization commits the issue/run decision before the derived agent
|
|
// comment is guaranteed to be visible through the comments endpoint.
|
|
// Give that projection a short consistency window so a successful
|
|
// terminal response is not misclassified as an empty provider reply.
|
|
// If no comment arrives, retain the original terminal observation and
|
|
// let the ordinary message matcher report the product failure.
|
|
if (taskMatchers.some((matcher) => matcher.kind.startsWith("message_"))) {
|
|
terminal = await pollUntil({
|
|
label: `final agent comment for issue ${issue.id}`,
|
|
deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),
|
|
load: loadTaskState,
|
|
accept: ({ taskRuns, comments }) => {
|
|
if (!matchesRunCount(execution.task, taskRuns.length)) {
|
|
return false;
|
|
}
|
|
const finalRun = sortRunsChronologically(taskRuns).at(-1);
|
|
return comments.some(
|
|
(comment) =>
|
|
comment.createdByRunId === finalRun?.id &&
|
|
comment.authorAgentId === fixtures!.agent.id,
|
|
);
|
|
},
|
|
reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
|
|
}).catch(() => terminal);
|
|
}
|
|
|
|
issue = terminal.currentIssue;
|
|
selectedRuns = terminal.taskRuns;
|
|
if (lifecycleBlockerId && execution.profile.expectedRuntimeMode === "legacy") {
|
|
const blockedIssue = await api.get<Pick<Issue, "blockedBy">>(`/api/issues/${issue.id}`);
|
|
expect(blockedIssue.blockedBy?.map((blocker) => blocker.id)).toEqual([lifecycleBlockerId]);
|
|
expect((await api.get<{ status: string }>(`/api/issues/${lifecycleBlockerId}`)).status).toBe("backlog");
|
|
}
|
|
if (reviewProvider) {
|
|
expect(reviewProvider.invocationCount()).toBe(execution.task.toolReviewDecision === "decline" ? 0 : execution.task.toolReviewDecision === "always" ? 2 : 1);
|
|
const pending = await api.get<{ actionRequests: unknown[] }>(`/api/companies/${fixtures.company.id}/tools/action-requests?status=pending`);
|
|
expect(pending.actionRequests).toHaveLength(0);
|
|
await writeSanitizedJson(snapshotsDir, "connection-review.json", { source: "local MCP fixture", connectionId: reviewProvider.connectionId, providerCalls: reviewProvider.invocationCount(), decision: execution.task.toolReviewDecision, issueId: issue.id }, secrets);
|
|
}
|
|
if (!matchesRunCount(execution.task, selectedRuns.length)) {
|
|
const runLogs = await Promise.all(
|
|
selectedRuns.map(async (candidate) => ({
|
|
runId: candidate.id,
|
|
log: await api
|
|
.get<unknown>(
|
|
`/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
|
|
)
|
|
.catch((error) => ({
|
|
evidenceCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
})),
|
|
})),
|
|
);
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"api-state.json",
|
|
{ issue, runs: selectedRuns, runLogs, ...terminal },
|
|
secrets,
|
|
);
|
|
throw new Error(
|
|
`Expected ${minimumRunCount(execution.task)}..${execution.task.expectedRunCount} task heartbeat run(s); observed ${selectedRuns.length}`,
|
|
);
|
|
}
|
|
// The company run-list endpoint intentionally returns only a compact,
|
|
// allowlisted context summary. Hydrate each selected run through the
|
|
// public detail endpoint before asserting environment/lease metadata.
|
|
selectedRuns = await Promise.all(
|
|
selectedRuns.map((candidate) =>
|
|
api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`),
|
|
),
|
|
);
|
|
selectedRuns = sortRunsChronologically(selectedRuns);
|
|
if (execution.suite.id === "daytona-git-streaming") {
|
|
await writeSanitizedJson(snapshotsDir, "git-copyback-turn-3.json", await gitStreamingEvidence(workspacePath, 3), secrets);
|
|
const finalization = await gitFinalizationEvidence(api, issue.id, selectedRuns.map(run => run.id));
|
|
await writeSanitizedJson(snapshotsDir, "git-finalization-turn-3.json", finalization, secrets);
|
|
expect(finalization.passed, finalization.failures.join("; ")).toBe(true);
|
|
}
|
|
const lifecycleProbe = execution.suite.id === "lifecycle-baseline" ? lifecycleLiveCase(execution.task.id) : undefined;
|
|
if (lifecycleProbe?.family === "repair") {
|
|
const grade = gradeLifecycleRepair({ runs: selectedRuns, comments: terminal.comments,
|
|
agentId: fixtures.agent.id, narrative: lifecycleProbe.narrative });
|
|
await writeSanitizedJson(snapshotsDir, "lifecycle-repair.json", { grade, runs: selectedRuns, comments: terminal.comments }, secrets);
|
|
expect(grade.passed, grade.detail).toBe(true);
|
|
expect(terminal.currentIssue.scheduledRetry).toBeNull();
|
|
expect(terminal.currentIssue.monitorNextCheckAt).toBeNull();
|
|
expect(terminal.interactions.filter(i => i.status === "pending")).toHaveLength(0);
|
|
}
|
|
|
|
if (execution.task.flow === "warm_three_turn") {
|
|
turnTimings = selectedRuns.map((candidate, index) => {
|
|
const submittedAtMs = turnSubmissionTimesMs[index]!;
|
|
const runStartedAtMs = candidate.startedAt
|
|
? Date.parse(candidate.startedAt)
|
|
: Number.NaN;
|
|
const runFinishedAtMs = candidate.finishedAt
|
|
? Date.parse(candidate.finishedAt)
|
|
: Number.NaN;
|
|
const acquisition = record(
|
|
record(candidate.contextSnapshot).paperclipEnvironment,
|
|
).sandboxLeaseAcquisition;
|
|
const acquisitionOutcome = record(acquisition).outcome;
|
|
return {
|
|
turn: index + 1,
|
|
submittedAt: new Date(submittedAtMs).toISOString(),
|
|
runStartedAt: candidate.startedAt ?? null,
|
|
runFinishedAt: candidate.finishedAt ?? null,
|
|
schedulerLatencyMs: Number.isFinite(runStartedAtMs)
|
|
? Math.max(0, runStartedAtMs - submittedAtMs)
|
|
: null,
|
|
runDurationMs:
|
|
Number.isFinite(runStartedAtMs) &&
|
|
Number.isFinite(runFinishedAtMs)
|
|
? Math.max(0, runFinishedAtMs - runStartedAtMs)
|
|
: null,
|
|
responseLatencyMs: Number.isFinite(runFinishedAtMs)
|
|
? Math.max(0, runFinishedAtMs - submittedAtMs)
|
|
: null,
|
|
runId: candidate.id,
|
|
leaseAcquisitionOutcome:
|
|
acquisitionOutcome === "created" ||
|
|
acquisitionOutcome === "resumed" ||
|
|
acquisitionOutcome === "replacement"
|
|
? acquisitionOutcome
|
|
: "unknown",
|
|
};
|
|
});
|
|
}
|
|
if (execution.suite.id === "daytona-warm-continuity") {
|
|
const evidence = await warmManagedFileEvidence(api, fixtures.agent.id, selectedRuns.at(-1)!.id, 3, nonce, execution.profile.generation === "native");
|
|
await writeSanitizedJson(snapshotsDir, "managed-warm-turn-3.json", evidence, secrets);
|
|
expect(evidence.passed, JSON.stringify(evidence.checks)).toBe(true);
|
|
}
|
|
const finalRun = selectedRuns.at(-1)!;
|
|
const run =
|
|
selectedRuns.find(
|
|
(candidate) => candidate.id === issue!.executionRunId,
|
|
) ?? selectedRuns[0];
|
|
const [
|
|
persistedAgentValue,
|
|
persistedEnvironmentValue,
|
|
runLogs,
|
|
runEventsByRun,
|
|
] = await Promise.all([
|
|
api.get<unknown>(`/api/agents/${fixtures.agent.id}`),
|
|
api.get<unknown>(`/api/environments/${fixtures.environment.id}`),
|
|
Promise.all(
|
|
selectedRuns.map(async (candidate) => ({
|
|
runId: candidate.id,
|
|
log: await api
|
|
.get<unknown>(
|
|
`/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
|
|
)
|
|
.catch((error) => ({
|
|
evidenceCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
})),
|
|
})),
|
|
),
|
|
Promise.all(
|
|
selectedRuns.map(async (candidate) => {
|
|
try {
|
|
return {
|
|
runId: candidate.id,
|
|
events: await collectRunEvents<RunEventRecord>((afterSeq, limit) =>
|
|
api.get(`/api/heartbeat-runs/${candidate.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
|
|
),
|
|
error: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
runId: candidate.id,
|
|
events: [] as RunEventRecord[],
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
}
|
|
}),
|
|
),
|
|
]);
|
|
const runLog =
|
|
runLogs.find((candidate) => candidate.runId === run.id)?.log ?? {};
|
|
const persistedAgent = record(persistedAgentValue);
|
|
const persistedEnvironment = record(persistedEnvironmentValue);
|
|
const agentComments = terminal.comments.filter(
|
|
(comment) =>
|
|
comment.authorAgentId === fixtures!.agent.id ||
|
|
selectedRuns.some(
|
|
(candidate) => comment.createdByRunId === candidate.id,
|
|
),
|
|
);
|
|
// Message matchers intentionally use persisted agent comments only.
|
|
// A semantic finish summary can differ from the actual user-facing text;
|
|
// accepting it here would let backend metadata mask a truncated UI
|
|
// response. The browser assertion below remains the visible source of
|
|
// truth after the comment projection has settled.
|
|
const message = agentComments
|
|
.map((comment) => comment.body ?? "")
|
|
.join("\n");
|
|
const finalRunMessage = persistedFinalRunMessage(agentComments, finalRun);
|
|
const pendingInteractions = terminal.interactions.filter(
|
|
(interaction) => interaction.status === "pending",
|
|
);
|
|
const invariantFailures: string[] = reasoningProjectionFailures(
|
|
runEventsByRun.flatMap((entry) => entry.events),
|
|
);
|
|
if (pendingInteractions.length > 0)
|
|
invariantFailures.push(
|
|
`expected no unresolved interaction; observed ${pendingInteractions.length}`,
|
|
);
|
|
if (expectedQuestionResolution) {
|
|
const questionInteractions = terminal.interactions.filter(
|
|
(interaction) => interaction.kind === "ask_user_questions",
|
|
);
|
|
if (
|
|
questionInteractions.length !== 1 ||
|
|
questionInteractions[0]?.id !==
|
|
expectedQuestionResolution.interactionId
|
|
) {
|
|
invariantFailures.push(
|
|
`expected exactly one stable question interaction ${expectedQuestionResolution.interactionId}; observed ${JSON.stringify(questionInteractions)}`,
|
|
);
|
|
}
|
|
const resolvedQuestion = questionInteractions.find(
|
|
(interaction) =>
|
|
interaction.id === expectedQuestionResolution.interactionId,
|
|
);
|
|
const answers = resolvedQuestion?.result?.answers ?? [];
|
|
if (
|
|
resolvedQuestion?.status !== "answered" ||
|
|
resolvedQuestion.result?.version !== 1 ||
|
|
answers.length !== 1 ||
|
|
answers[0]?.questionId !== "verification-word" ||
|
|
JSON.stringify(answers[0]?.optionIds) !==
|
|
JSON.stringify([expectedQuestionResolution.optionId]) ||
|
|
answers[0]?.otherText !== undefined
|
|
) {
|
|
invariantFailures.push(
|
|
`expected interaction ${expectedQuestionResolution.interactionId} to resolve with exactly ${expectedQuestionResolution.optionId}; observed ${JSON.stringify(resolvedQuestion)}`,
|
|
);
|
|
}
|
|
const continuationRun = selectedRuns.at(-1);
|
|
const continuationContext = record(continuationRun?.contextSnapshot);
|
|
if (
|
|
!continuationRun ||
|
|
continuationContext.interactionId !==
|
|
expectedQuestionResolution.interactionId
|
|
) {
|
|
invariantFailures.push(
|
|
`expected the continuation run to consume interaction ${expectedQuestionResolution.interactionId}; observed ${String(continuationContext.interactionId)}`,
|
|
);
|
|
}
|
|
if (execution.profile.generation === "native" && continuationRun) {
|
|
const runnerProfile = record(continuationRun.runnerProfileJson);
|
|
const nativeExecutionInput = record(
|
|
runnerProfile.nativeExecutionInput,
|
|
);
|
|
const interactionResponses = Array.isArray(
|
|
nativeExecutionInput.interactionResponses,
|
|
)
|
|
? nativeExecutionInput.interactionResponses.map(record)
|
|
: [];
|
|
const matchingResponses = interactionResponses.filter(
|
|
(candidate) =>
|
|
candidate.interactionId ===
|
|
expectedQuestionResolution.interactionId,
|
|
);
|
|
const responseEnvelope = matchingResponses[0];
|
|
const response = record(responseEnvelope?.response);
|
|
const responseResult = record(response.result);
|
|
const responseAnswers = Array.isArray(responseResult.answers)
|
|
? responseResult.answers.map(record)
|
|
: [];
|
|
if (
|
|
matchingResponses.length !== 1 ||
|
|
responseEnvelope?.kind !== "ask_user_questions" ||
|
|
response.status !== "answered" ||
|
|
responseResult.version !== 1 ||
|
|
responseAnswers.length !== 1 ||
|
|
responseAnswers[0]?.questionId !== "verification-word" ||
|
|
JSON.stringify(responseAnswers[0]?.optionIds) !==
|
|
JSON.stringify([expectedQuestionResolution.optionId])
|
|
) {
|
|
invariantFailures.push(
|
|
`expected native continuation input to carry exactly one answered interaction ${expectedQuestionResolution.interactionId}; observed ${JSON.stringify(matchingResponses)}`,
|
|
);
|
|
}
|
|
}
|
|
questionLifecycleEvidence = {
|
|
...(questionLifecycleEvidence ?? {}),
|
|
resolvedInteraction: resolvedQuestion ?? null,
|
|
continuationRunId: continuationRun?.id ?? null,
|
|
};
|
|
}
|
|
for (const candidate of selectedRuns) {
|
|
if (
|
|
(candidate.continuationAttempt ?? 0) !== 0 ||
|
|
candidate.retryOfRunId
|
|
)
|
|
invariantFailures.push(
|
|
`expected run ${candidate.id} without a recovery continuation`,
|
|
);
|
|
}
|
|
for (const captured of runEventsByRun) {
|
|
if (captured.error) {
|
|
invariantFailures.push(
|
|
`run ${captured.runId} events query failed: ${captured.error}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
if (execution.suite.id === "api-response-reading") {
|
|
const paging = gradeApiResponsePaging(runEventsByRun.flatMap(captured => captured.events), apiResponseSourceId ?? "");
|
|
await writeSanitizedJson(snapshotsDir, "api-response-pagination.json", paging, secrets);
|
|
if (!paging.passed) invariantFailures.push(`Bounded response paging was not proven: ${paging.failure}`);
|
|
}
|
|
if (execution.suite.id === "task-titles") {
|
|
const titleEvidence = gradeTaskTitle({
|
|
initial: titleCreation?.issue, submitted: titleCreation?.submitted,
|
|
final: issue, prompt, explicitTitle: title, agentId: fixtures.agent.id,
|
|
runId: selectedRuns[0]!.id,
|
|
events: runEventsByRun.find(captured => captured.runId === selectedRuns[0]!.id)?.events ?? [],
|
|
activity: await api.get<unknown[]>(`/api/issues/${issue.id}/activity`),
|
|
});
|
|
await writeSanitizedJson(snapshotsDir, "task-title.json", titleEvidence, secrets);
|
|
invariantFailures.push(...titleEvidence.checks.filter(check => !check.passed).map(check => `${check.id}: ${check.detail}`));
|
|
}
|
|
|
|
const context = record(run.contextSnapshot);
|
|
const environmentContext = record(context.paperclipEnvironment);
|
|
const workspaceContext = record(context.paperclipWorkspace);
|
|
const environmentDriver =
|
|
environmentContext.driver ?? persistedEnvironment.driver;
|
|
const observedEnvironment =
|
|
environmentDriver === "sandbox" ? "daytona" : environmentDriver;
|
|
const observedRuntimeMode =
|
|
run.runtimeMode ??
|
|
(persistedAgent.adapterType === "paperclip_runner"
|
|
? "native"
|
|
: "legacy");
|
|
const matcherObservation = {
|
|
message,
|
|
issueStatus: issue.status,
|
|
runStatus: selectedRuns.every(
|
|
(candidate) =>
|
|
candidate.status === execution.task.expectedTerminalState.run,
|
|
)
|
|
? execution.task.expectedTerminalState.run
|
|
: selectedRuns.map((candidate) => candidate.status).join(","),
|
|
runtimeMode: observedRuntimeMode,
|
|
environment:
|
|
typeof observedEnvironment === "string"
|
|
? observedEnvironment
|
|
: undefined,
|
|
json: {
|
|
issue,
|
|
run,
|
|
comments: terminal.comments,
|
|
interactions: terminal.interactions,
|
|
},
|
|
};
|
|
const fileObservations = Object.fromEntries(
|
|
await Promise.all(
|
|
taskMatchers
|
|
.filter(
|
|
(matcher) =>
|
|
matcher.kind === "file_exists" ||
|
|
matcher.kind === "file_exact" ||
|
|
matcher.kind === "file_contains",
|
|
)
|
|
.map(async (matcher) => [
|
|
matcher.path,
|
|
await readFile(
|
|
path.isAbsolute(matcher.path)
|
|
? matcher.path
|
|
: path.join(workspacePath, matcher.path),
|
|
"utf8",
|
|
).catch(() => undefined),
|
|
]),
|
|
),
|
|
);
|
|
if (execution.suite.id === "api-response-reading") {
|
|
downloadedResponseProof = await readResponseProof(api, issue.id, run.id);
|
|
fileObservations["api-response-proof.txt"] = downloadedResponseProof.content;
|
|
await writeSanitizedJson(snapshotsDir, "downloaded-response-proof.json", downloadedResponseProof, secrets);
|
|
}
|
|
matcherResults = await Promise.all(
|
|
taskMatchers.map((matcher) =>
|
|
evaluateMatcher(matcher, {
|
|
...matcherObservation,
|
|
files: fileObservations,
|
|
// Multi-run tasks intentionally retain earlier waiting/revision
|
|
// replies. Exact completion text belongs to the chronological
|
|
// final run, while occurrence checks still span every agent
|
|
// comment so duplicate terminal markers cannot be hidden.
|
|
message:
|
|
matcher.kind === "message_exact" ? finalRunMessage : message,
|
|
}),
|
|
),
|
|
);
|
|
const failedMatchers = matcherResults.filter((result) => !result.passed);
|
|
const exactMessageMatcher = taskMatchers.find(
|
|
(matcher) => matcher.kind === "message_exact",
|
|
);
|
|
if (
|
|
execution.profile.id === "runner-opencode" &&
|
|
execution.task.id === "structured-question-restart-resume" &&
|
|
exactMessageMatcher?.kind === "message_exact" &&
|
|
finalRunMessage === exactMessageMatcher.expected.replace(/-\d+$/, "") &&
|
|
record(finalRun.resultJson).summary === exactMessageMatcher.expected
|
|
) {
|
|
// OpenCode can occasionally copy the complete marker into the
|
|
// accepted semantic result while dropping only the synthetic attempt
|
|
// suffix from its visible answer. Keep exact matching strict, but let
|
|
// the campaign retry this narrowly proven provider variance once in a
|
|
// fresh harness. A repeated near miss remains a failed cell.
|
|
failureClassOverride = "provider_variance";
|
|
}
|
|
const retryInteractiveTerminalProviderVariance =
|
|
(execution.suite.id === "openrouter-model-breadth" &&
|
|
(execution.task.id === "question-resume-complete" ||
|
|
execution.task.id === "plan-approve-complete")) ||
|
|
(execution.suite.id === "core-compatibility" &&
|
|
execution.profile.generation === "native" &&
|
|
execution.task.id === "plan-revise-accept");
|
|
if (
|
|
retryInteractiveTerminalProviderVariance &&
|
|
exactMessageMatcher?.kind === "message_exact" &&
|
|
selectedRuns.every((candidate) => candidate.status === "succeeded") &&
|
|
issue.status === "done" &&
|
|
finalRunMessage !== exactMessageMatcher.expected &&
|
|
record(finalRun.resultJson).summary === exactMessageMatcher.expected
|
|
) {
|
|
// An interactive breadth model can occasionally satisfy the durable
|
|
// terminal contract while paraphrasing the visible final response.
|
|
// Preserve the exact persisted-message and DOM assertions, but
|
|
// classify this narrowly proven provider variance for one
|
|
// fresh-harness retry.
|
|
failureClassOverride = "provider_variance";
|
|
}
|
|
const observedEnvironmentId =
|
|
environmentContext.id ??
|
|
(execution.environment.id === "local"
|
|
? persistedAgent.defaultEnvironmentId
|
|
: undefined);
|
|
if (observedEnvironmentId !== fixtures.environment.id) {
|
|
invariantFailures.push(
|
|
`Expected environment ${fixtures.environment.id}; observed ${String(observedEnvironmentId)}`,
|
|
);
|
|
}
|
|
if (
|
|
execution.environment.id === "daytona" &&
|
|
typeof environmentContext.leaseId !== "string"
|
|
) {
|
|
invariantFailures.push(
|
|
"expected a Daytona sandbox lease on the run context",
|
|
);
|
|
}
|
|
if (execution.task.flow === "warm_three_turn") {
|
|
const runEnvironmentContexts = selectedRuns.map((candidate) =>
|
|
record(record(candidate.contextSnapshot).paperclipEnvironment),
|
|
);
|
|
const leaseIds = runEnvironmentContexts.map((entry) => entry.leaseId);
|
|
const acquisitionOutcomes = runEnvironmentContexts.map(
|
|
(entry) => record(entry.sandboxLeaseAcquisition).outcome,
|
|
);
|
|
const projectWorkspaceIds = selectedRuns.map(
|
|
(candidate) =>
|
|
record(record(candidate.contextSnapshot).paperclipWorkspace)
|
|
.workspaceId,
|
|
);
|
|
const executionWorkspaceIds = selectedRuns.map(
|
|
(candidate) => record(candidate.contextSnapshot).executionWorkspaceId,
|
|
);
|
|
if (
|
|
leaseIds.some(
|
|
(leaseId) => typeof leaseId !== "string" || leaseId.length === 0,
|
|
)
|
|
) {
|
|
invariantFailures.push(
|
|
`expected a persisted Daytona lease row for every warm turn; observed ${JSON.stringify(leaseIds)}`,
|
|
);
|
|
}
|
|
if (
|
|
JSON.stringify(acquisitionOutcomes) !==
|
|
JSON.stringify(["created", "resumed", "resumed"])
|
|
) {
|
|
invariantFailures.push(
|
|
`expected warm lease outcomes created,resumed,resumed; observed ${JSON.stringify(acquisitionOutcomes)}`,
|
|
);
|
|
}
|
|
if (
|
|
!fixtures.project?.primaryWorkspace?.id ||
|
|
projectWorkspaceIds.some(
|
|
(workspaceId) =>
|
|
workspaceId !== fixtures!.project!.primaryWorkspace!.id,
|
|
) ||
|
|
new Set(executionWorkspaceIds).size !== 1 ||
|
|
executionWorkspaceIds[0] !== issue.executionWorkspaceId ||
|
|
issue.projectId !== fixtures.project.id ||
|
|
issue.projectWorkspaceId !== fixtures.project.primaryWorkspace.id ||
|
|
!issue.executionWorkspaceId
|
|
) {
|
|
invariantFailures.push(
|
|
`warm task did not preserve project, project-workspace, and execution-workspace identity: ${JSON.stringify({ projectWorkspaceIds, executionWorkspaceIds, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, executionWorkspaceId: issue.executionWorkspaceId })}`,
|
|
);
|
|
}
|
|
if (execution.profile.generation === "native") {
|
|
const stableIdentityFields: Array<{
|
|
label: string;
|
|
values: unknown[];
|
|
}> = [
|
|
{
|
|
label: "native session",
|
|
values: selectedRuns.map(
|
|
(candidate) => candidate.nativeSessionId,
|
|
),
|
|
},
|
|
{
|
|
label: "runner instance",
|
|
values: selectedRuns.map(
|
|
(candidate) => candidate.runnerInstanceId,
|
|
),
|
|
},
|
|
{
|
|
label: "provider session",
|
|
values: selectedRuns.map((candidate) => candidate.sessionIdAfter),
|
|
},
|
|
{
|
|
label: "runner pid",
|
|
values: selectedRuns.map((candidate) => candidate.processPid),
|
|
},
|
|
{
|
|
label: "runner process fingerprint",
|
|
values: selectedRuns.map(
|
|
(candidate) => candidate.processStartedAt,
|
|
),
|
|
},
|
|
];
|
|
for (const { label, values } of stableIdentityFields) {
|
|
if (
|
|
values.some(
|
|
(value) =>
|
|
value === null ||
|
|
value === undefined ||
|
|
String(value).length === 0,
|
|
) ||
|
|
new Set(values).size !== 1
|
|
) {
|
|
invariantFailures.push(
|
|
`expected one stable ${label} across native warm turns; observed ${JSON.stringify(values)}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
if (
|
|
turnTimings?.length !== 3 ||
|
|
turnTimings.some(
|
|
(timing) =>
|
|
timing.runStartedAt === null ||
|
|
timing.runFinishedAt === null ||
|
|
timing.schedulerLatencyMs === null ||
|
|
timing.runDurationMs === null ||
|
|
timing.responseLatencyMs === null ||
|
|
timing.responseLatencyMs >
|
|
(execution.task.turnTimeoutMs ?? 10 * 60_000),
|
|
)
|
|
) {
|
|
invariantFailures.push(
|
|
`warm turn timing data was incomplete or exceeded its structural deadline: ${JSON.stringify(turnTimings)}`,
|
|
);
|
|
}
|
|
const retainedLeases = await pollUntil({
|
|
label: `terminal warm Daytona lease history for issue ${issue.id}`,
|
|
deadlineAt: Math.min(deadlineAt, Date.now() + 30_000),
|
|
intervalMs: 500,
|
|
load: () =>
|
|
api.get<EnvironmentLeaseRecord[]>(
|
|
`/api/environments/${fixtures!.environment.id}/leases`,
|
|
),
|
|
accept: (leases) => {
|
|
const warmLeases = leases.filter(
|
|
(lease) =>
|
|
lease.issueId === issue!.id &&
|
|
selectedRuns.some(
|
|
(candidate) => candidate.id === lease.heartbeatRunId,
|
|
),
|
|
);
|
|
return (
|
|
warmLeases.length === 3 &&
|
|
warmLeases.every((lease) => {
|
|
const runIndex = selectedRuns.findIndex(
|
|
(candidate) => candidate.id === lease.heartbeatRunId,
|
|
);
|
|
return (
|
|
lease.status ===
|
|
(runIndex === selectedRuns.length - 1
|
|
? "retained"
|
|
: "expired") &&
|
|
lease.cleanupStatus === "success" &&
|
|
lease.leasePolicy === "reuse_by_environment" &&
|
|
typeof lease.providerLeaseId === "string" &&
|
|
record(lease.metadata).sandboxState === "started"
|
|
);
|
|
})
|
|
);
|
|
},
|
|
});
|
|
const selectedRunOrder = new Map(
|
|
selectedRuns.map((candidate, index) => [candidate.id, index]),
|
|
);
|
|
const warmLeases = retainedLeases
|
|
.filter(
|
|
(lease) =>
|
|
lease.issueId === issue!.id &&
|
|
selectedRunOrder.has(lease.heartbeatRunId ?? ""),
|
|
)
|
|
.sort(
|
|
(left, right) =>
|
|
(selectedRunOrder.get(left.heartbeatRunId ?? "") ?? 0) -
|
|
(selectedRunOrder.get(right.heartbeatRunId ?? "") ?? 0),
|
|
);
|
|
const providerLeaseIds = warmLeases.map(
|
|
(lease) => lease.providerLeaseId,
|
|
);
|
|
const resumedFromStates = warmLeases
|
|
.slice(1)
|
|
.map((lease) => record(lease.metadata).resumedFromState);
|
|
if (
|
|
new Set(providerLeaseIds).size !== 1 ||
|
|
typeof providerLeaseIds[0] !== "string" ||
|
|
JSON.stringify(resumedFromStates) !==
|
|
JSON.stringify(["started", "started"])
|
|
) {
|
|
invariantFailures.push(
|
|
`expected one continuously-started Daytona sandbox; observed ${JSON.stringify({ providerLeaseIds, resumedFromStates })}`,
|
|
);
|
|
}
|
|
warmLifecycleEvidence = {
|
|
...(warmLifecycleEvidence ?? {}),
|
|
leaseIds,
|
|
acquisitionOutcomes,
|
|
projectWorkspaceIds,
|
|
executionWorkspaceIds,
|
|
providerLeaseIds,
|
|
resumedFromStates,
|
|
retainedLeases: warmLeases,
|
|
turnTimings,
|
|
};
|
|
}
|
|
if (execution.environment.id === "local") {
|
|
const runLogContent = String(record(runLog).content ?? "");
|
|
// The log endpoint returns NDJSON, so quotes inside each `chunk` are
|
|
// escaped. Accept both that wire representation and a decoded chunk.
|
|
const fallbackWorkspace =
|
|
/Using fallback workspace \\\"([^"\\]+)\\\"/.exec(
|
|
runLogContent,
|
|
)?.[1] ??
|
|
/Using fallback workspace "([^"]+)"/.exec(runLogContent)?.[1];
|
|
const cwd = String(workspaceContext.cwd ?? fallbackWorkspace ?? "");
|
|
if (!cwd.startsWith(`${temporaryRoot}/`)) {
|
|
invariantFailures.push(
|
|
`local run workspace escaped the isolated root: ${cwd}`,
|
|
);
|
|
}
|
|
}
|
|
const runEvents =
|
|
runEventsByRun.find((candidate) => candidate.runId === run.id)
|
|
?.events ?? [];
|
|
if (execution.profile.generation === "native") {
|
|
for (const candidate of selectedRuns) {
|
|
const candidateEvents =
|
|
runEventsByRun.find((captured) => captured.runId === candidate.id)
|
|
?.events ?? [];
|
|
const runnerInstanceObserved =
|
|
Boolean(candidate.runnerInstanceId) ||
|
|
candidateEvents.some(
|
|
(event) =>
|
|
typeof event.sourceInstanceId === "string" &&
|
|
event.sourceInstanceId.length > 0 &&
|
|
!event.sourceInstanceId.endsWith(":control"),
|
|
);
|
|
if (!runnerInstanceObserved) {
|
|
invariantFailures.push(
|
|
`expected native run ${candidate.id} events from a runner instance`,
|
|
);
|
|
}
|
|
invariantFailures.push(
|
|
...nativeRunEventIntegrityFailures(candidate, candidateEvents),
|
|
);
|
|
}
|
|
if (
|
|
(execution.profile.provider === "codex" ||
|
|
execution.profile.provider === "opencode") &&
|
|
selectedRuns.length > 1
|
|
) {
|
|
invariantFailures.push(
|
|
...providerSessionContinuityFailures(
|
|
execution.profile.provider,
|
|
selectedRuns,
|
|
),
|
|
);
|
|
} else if (
|
|
execution.profile.provider === "acpx" &&
|
|
selectedRuns.length > 1
|
|
) {
|
|
for (let index = 1; index < selectedRuns.length; index += 1) {
|
|
const previousSessionId = selectedRuns[index - 1]?.sessionIdAfter;
|
|
const current = selectedRuns[index]!;
|
|
const currentSessionId = current.sessionIdAfter;
|
|
if (!previousSessionId || !currentSessionId) {
|
|
invariantFailures.push(
|
|
"expected ACPX to record provider session identity across all heartbeat runs",
|
|
);
|
|
continue;
|
|
}
|
|
const acceptedPlanResetFailures = acceptedPlanSessionResetFailures(
|
|
"acpx",
|
|
previousSessionId,
|
|
current,
|
|
);
|
|
if (acceptedPlanResetFailures) {
|
|
invariantFailures.push(...acceptedPlanResetFailures);
|
|
continue;
|
|
}
|
|
if (previousSessionId === currentSessionId) continue;
|
|
const currentEvents =
|
|
runEventsByRun.find((captured) => captured.runId === current.id)
|
|
?.events ?? [];
|
|
const recordedContinuityBreak = currentEvents.some((event) => {
|
|
const payload = record(event.payload);
|
|
return (
|
|
event.eventType === "run.performance.span" &&
|
|
payload.span === "provider.session.continuity_break" &&
|
|
payload.previousProviderSessionId === previousSessionId &&
|
|
payload.replacementProviderSessionId === currentSessionId
|
|
);
|
|
});
|
|
if (!recordedContinuityBreak) {
|
|
invariantFailures.push(
|
|
`ACPX provider session changed from ${previousSessionId} to ${currentSessionId} without a matching continuity event`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
const spanPayloads = runEvents
|
|
.filter((event) => event.eventType === "run.performance.span")
|
|
.map((event) => record(event.payload));
|
|
const selectedTransport = spanPayloads.find(
|
|
(payload) => payload.span === "runner.transport.selected",
|
|
);
|
|
const expectedTransport =
|
|
execution.environment.id === "daytona"
|
|
? "provider_ingress"
|
|
: "local_loopback";
|
|
if (selectedTransport?.mode !== expectedTransport) {
|
|
invariantFailures.push(
|
|
`Expected native runner transport ${expectedTransport}; observed ${String(selectedTransport?.mode)}`,
|
|
);
|
|
}
|
|
if (execution.environment.id === "daytona") {
|
|
const authenticated = spanPayloads.some(
|
|
(event) =>
|
|
event.span === "runner.prp.authenticate" &&
|
|
event.outcome === "ok",
|
|
);
|
|
if (!authenticated) {
|
|
invariantFailures.push(
|
|
"expected authenticated native Daytona runner preview ingress",
|
|
);
|
|
}
|
|
}
|
|
} else {
|
|
for (const candidate of selectedRuns) {
|
|
const candidateEvents =
|
|
runEventsByRun.find((captured) => captured.runId === candidate.id)
|
|
?.events ?? [];
|
|
if (
|
|
candidate.runtimeMode !== "legacy" ||
|
|
candidate.runnerInstanceId ||
|
|
candidateEvents.some(
|
|
(event) =>
|
|
Object.keys(record(event.payload?.prpEvent)).length > 0,
|
|
)
|
|
) {
|
|
invariantFailures.push(
|
|
`legacy run ${candidate.id} crossed into native runner persistence`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"api-state.json",
|
|
{
|
|
issue,
|
|
run,
|
|
comments: terminal.comments,
|
|
interactions: terminal.interactions,
|
|
planLifecycleEvidence,
|
|
questionLifecycleEvidence,
|
|
warmLifecycleEvidence,
|
|
matcherResults,
|
|
invariantFailures,
|
|
runEvents,
|
|
runEventsByRun,
|
|
runLogs,
|
|
},
|
|
secrets,
|
|
);
|
|
|
|
if (
|
|
exactMessageMatcher?.kind === "message_exact" &&
|
|
isOpenRouterDeepSeekHelloTerminalVariance({
|
|
suiteId: execution.suite.id,
|
|
profileId: execution.profile.id,
|
|
taskId: execution.task.id,
|
|
expectedMarker: exactMessageMatcher.expected,
|
|
finalRunMessage,
|
|
allAgentMessages: message,
|
|
semanticSummary: record(finalRun.resultJson).summary,
|
|
issueStatus: issue.status,
|
|
runStatuses: selectedRuns.map((candidate) => candidate.status),
|
|
matcherResults,
|
|
invariantFailures,
|
|
})
|
|
) {
|
|
// DeepSeek can occasionally complete the semantic finish correctly but
|
|
// expose its pre-tool acknowledgement as the visible final answer. Keep
|
|
// exact persisted-message, global occurrence, and DOM checks strict,
|
|
// while allowing one fresh-harness retry only for the zero-marker form.
|
|
failureClassOverride = "provider_variance";
|
|
}
|
|
|
|
// The backend polling above can observe a terminal transition before a
|
|
// websocket invalidation reaches the already-open task page. Reload the
|
|
// canonical task route so the screenshot and UI assertions prove the
|
|
// persisted final state, not a stale client cache.
|
|
await page.goto(
|
|
`/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
|
|
{ waitUntil: "domcontentloaded" },
|
|
);
|
|
const visibleAgentReplies = page
|
|
.getByTestId("task-chat-thread")
|
|
.getByTestId("task-chat-agent-bubble");
|
|
if (execution.suite.id === "task-titles") {
|
|
await expect(page.getByRole("heading", { name: issue.title.trim(), exact: true })).toBeVisible();
|
|
}
|
|
if (execution.suite.id === "api-response-reading") {
|
|
// File delivery renders a card instead of an exact summary bubble.
|
|
// Prove the visible link points to the same independently checked bytes.
|
|
const proofLink = page.getByRole("link", { name: "Open api-response-proof.txt", exact: true }).first();
|
|
await expect(proofLink).toBeVisible({ timeout: 30_000 });
|
|
await expect(proofLink).toHaveAttribute("href", `/api/attachments/${downloadedResponseProof!.attachmentId}/content`);
|
|
} else if (execution.suite.id === "task-titles") {
|
|
await expect(visibleAgentReplies.filter({ hasText: marker }).last()).toBeVisible({ timeout: 30_000 });
|
|
} else if (execution.task.flow === "warm_three_turn") {
|
|
// Prove the persisted user-facing response is visible, independently
|
|
// of the byte-for-byte workspace checks and lease continuity checks.
|
|
expect(finalRunMessage.trim()).not.toBe("");
|
|
await expect(visibleAgentReplies.filter({ hasText: finalRunMessage }).last())
|
|
.toBeVisible({ timeout: 30_000 });
|
|
} else {
|
|
const escapedMarker = marker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
const terminalAgentReplies = visibleAgentReplies.filter({
|
|
hasText: new RegExp(`^\\s*${escapedMarker}\\s*$`),
|
|
});
|
|
await expect(terminalAgentReplies).toHaveCount(1, { timeout: 30_000 });
|
|
await expect(terminalAgentReplies.first()).toBeVisible();
|
|
// A string-valued toHaveText assertion compares the complete rendered
|
|
// text while normalizing ordinary DOM whitespace. This keeps Markdown
|
|
// layout differences harmless without allowing prefixed, suffixed, or
|
|
// substituted provider prose to masquerade as the requested response.
|
|
await expect(terminalAgentReplies.first()).toHaveText(marker, {
|
|
useInnerText: true,
|
|
});
|
|
}
|
|
// The fixture owns the expected disposition; blocked workflows must prove
|
|
// their Blocked UI rather than inheriting the completion-only Done check.
|
|
const expectedStatus = execution.task.expectedTerminalState.issue;
|
|
const expectedStatusLabel = expectedStatus.replace(/_/g, " ").replace(/\b\w/g, (letter) => letter.toUpperCase());
|
|
await expect(
|
|
page.getByTestId("issue-detail-header").getByRole("button", {
|
|
// Blocked includes the live blocker-attention explanation in its
|
|
// accessible name. The exact persisted status is asserted separately.
|
|
name: `Change status (current: ${expectedStatusLabel}${expectedStatus === "blocked" ? "" : ")"}`,
|
|
exact: expectedStatus !== "blocked",
|
|
}),
|
|
).toBeVisible({ timeout: 30_000 });
|
|
if (execution.task.flow === "warm_three_turn") {
|
|
const continuedReceipts = page
|
|
.getByTestId("task-chat-interaction-receipt")
|
|
.filter({ hasText: "Selected “Continue work”" });
|
|
await expect(continuedReceipts).toHaveCount(2, { timeout: 30_000 });
|
|
await expect(
|
|
page.getByText("Declined request", { exact: true }),
|
|
).toHaveCount(0);
|
|
}
|
|
await captureScreenshot(
|
|
"final-state",
|
|
"Final visible task state",
|
|
"final-state.png",
|
|
);
|
|
|
|
if (failedMatchers.length > 0) {
|
|
throw new Error(
|
|
`Matcher failure: ${failedMatchers.map((result) => result.detail).join("; ")}; run error: ${run.errorCode ?? "none"} ${run.error ?? "none"}`,
|
|
);
|
|
}
|
|
if (invariantFailures.length > 0) {
|
|
throw new Error(
|
|
`Runtime invariant failure: ${invariantFailures.join("; ")}`,
|
|
);
|
|
}
|
|
}
|
|
if (runsCompletionUpdateProbe(execution) && credentials.OPENAI_API_KEY) {
|
|
const qualification = completionQualityStatus(completionQuality);
|
|
if (qualification === "unqualified") {
|
|
failureClassOverride = "permanent_infrastructure";
|
|
throw new Error("Completion accuracy is unqualified: missing, invalid, or miscalibrated judge evidence");
|
|
}
|
|
expect(qualification, "completion reply accuracy").toBe("passed");
|
|
}
|
|
} catch (error) {
|
|
primaryError = error;
|
|
if (execution.task.flow === "first_task" && !firstTaskEvidence && classifyFailure(error) === "candidate_failure") {
|
|
failureClassOverride = "permanent_infrastructure";
|
|
}
|
|
try {
|
|
await captureFailureApiState();
|
|
} catch (captureError) {
|
|
const failureClass = classifyFailure(captureError);
|
|
if (failureClass === "secret_leak") {
|
|
failureClassOverride = "secret_leak";
|
|
primaryError = new AggregateError(
|
|
[primaryError, captureError],
|
|
`Failure evidence contained unsafe data: ${captureError instanceof Error ? captureError.message : String(captureError)}`,
|
|
);
|
|
} else {
|
|
networkDiagnostics.push({
|
|
evidenceCaptureError:
|
|
captureError instanceof Error
|
|
? captureError.message
|
|
: String(captureError),
|
|
});
|
|
}
|
|
}
|
|
if (!page.isClosed()) {
|
|
if (isReviewedFixtureScreenshotRoute()) {
|
|
await captureScreenshot(
|
|
"failure",
|
|
"Task state at failure",
|
|
"failure.png",
|
|
).catch(() => undefined);
|
|
} else {
|
|
await capturePrivateScreenshot("failure", "failure.png").catch(
|
|
() => undefined,
|
|
);
|
|
}
|
|
}
|
|
} finally {
|
|
await reviewProvider?.close();
|
|
try {
|
|
await writeSanitizedJson(
|
|
snapshotsDir,
|
|
"browser-diagnostics.json",
|
|
{
|
|
console: consoleDiagnostics,
|
|
network: networkDiagnostics,
|
|
lifecycle: pageLifecycleDiagnostics,
|
|
bootstrap: await browserBootstrap.snapshot(),
|
|
},
|
|
secrets,
|
|
);
|
|
} catch (error) {
|
|
primaryError = new AggregateError(
|
|
[primaryError, error].filter(Boolean),
|
|
`Browser diagnostics contained unsafe data: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
failureClassOverride = "secret_leak";
|
|
}
|
|
browserBootstrap.dispose();
|
|
if (fixtures) {
|
|
await captureRuntimeLeases().catch((error) => {
|
|
networkDiagnostics.push({
|
|
runtimeBillingCaptureError:
|
|
error instanceof Error ? error.message : String(error),
|
|
});
|
|
});
|
|
try {
|
|
await cancelActiveRunsForCleanup();
|
|
if (companyRunFlow) {
|
|
const companyRuns = await api.get<RunRecord[]>(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`);
|
|
selectedRuns = await Promise.all(companyRuns.map(run => api.get<RunRecord>(`/api/heartbeat-runs/${run.id}`)));
|
|
await writeSanitizedJson(snapshotsDir, execution.task.flow === "first_task" ? "first-task-final-run-ledger.json" : "chat-final-run-ledger.json", selectedRuns, secrets);
|
|
}
|
|
if (execution.suite.id === "stock-harness") {
|
|
try {
|
|
const stock = await captureStockHarness({ api, companyId: fixtures.company.id,
|
|
agentId: fixtures.agent.id, generation: execution.profile.generation,
|
|
runIds: selectedRuns.map(run => run.id) });
|
|
const checks = gradeStockHarness(stock);
|
|
matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const,
|
|
path: `stockHarness.${check.id}`, expected: true }, passed: check.passed, detail: check.detail })));
|
|
await writeSanitizedJson(snapshotsDir, "stock-harness.json", { ...stock, checks }, secrets);
|
|
const failures = checks.filter(check => !check.passed);
|
|
if (failures.length && !primaryError) primaryError = new Error(`Stock harness matcher failures: ${failures.map(check => check.id).join(", ")}`);
|
|
} catch (error) {
|
|
await writeSanitizedJson(snapshotsDir, "stock-harness-evidence-error.json", {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
}, secrets);
|
|
if (!primaryError) primaryError = error;
|
|
}
|
|
}
|
|
await fixtures.teardown();
|
|
cleanup = "passed";
|
|
} catch (error) {
|
|
cleanup = "failed";
|
|
const priorFailureClass = primaryError
|
|
? classifyFailure(primaryError)
|
|
: undefined;
|
|
const cleanupFailureClass = classifyFailure(error);
|
|
failureClassOverride =
|
|
priorFailureClass === "secret_leak"
|
|
? priorFailureClass
|
|
: cleanupFailureClass === "cleanup_failure"
|
|
? cleanupFailureClass
|
|
: (priorFailureClass ?? cleanupFailureClass);
|
|
primaryError = new AggregateError(
|
|
[primaryError, error].filter(Boolean),
|
|
primaryError
|
|
? `${primaryError instanceof Error ? primaryError.message : String(primaryError)}; Cleanup also failed: ${error instanceof Error ? error.message : String(error)}`
|
|
: `Cleanup failed after test execution: ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
}
|
|
if (runtimeLeases.length > 0) {
|
|
runtimeLeases = await Promise.all(
|
|
runtimeLeases.map((lease) =>
|
|
api
|
|
.get<EnvironmentLeaseRecord>(
|
|
`/api/environment-leases/${lease.id}`,
|
|
)
|
|
.catch(() => lease),
|
|
),
|
|
);
|
|
}
|
|
} else {
|
|
cleanup =
|
|
primaryError &&
|
|
/(?:cleanup|teardown) failed/i.test(
|
|
primaryError instanceof Error
|
|
? primaryError.message
|
|
: String(primaryError),
|
|
)
|
|
? "failed"
|
|
: "passed";
|
|
}
|
|
|
|
const finishedAtMs = Date.now();
|
|
const runtimeUsage = buildRuntimeUsage({
|
|
environmentId: execution.environment.id,
|
|
runs: selectedRuns,
|
|
leases: runtimeLeases,
|
|
fallbackFinishedAt: new Date(finishedAtMs),
|
|
});
|
|
const resultWithoutBilling: RunnerE2EResult = {
|
|
schema: "paperclip.runner-e2e.result/v2",
|
|
executionId: execution.id,
|
|
suiteId: execution.suite.id,
|
|
suiteDefinitionHash: execution.suiteDefinitionHash,
|
|
source: resolveRunnerE2ESource(firstTaskEvidence?.source ? { ...firstTaskEvidence.source, workflowRunUrl: null } : undefined),
|
|
...(execution.profile.ranking
|
|
? { rankingSnapshot: execution.profile.ranking }
|
|
: {}),
|
|
attempt,
|
|
status: primaryError ? "failed" : "passed",
|
|
...(primaryError
|
|
? {
|
|
failureClass:
|
|
failureClassOverride ?? classifyFailure(primaryError),
|
|
error:
|
|
primaryError instanceof Error
|
|
? primaryError.message
|
|
: String(primaryError),
|
|
}
|
|
: {}),
|
|
profileId: execution.profile.id,
|
|
environmentId: execution.environment.id,
|
|
caseId: execution.task.id,
|
|
provider: execution.profile.provider,
|
|
model: firstTaskEvidence ? firstTaskEvidence.observedModels[0] ?? firstTaskEvidence.configuredModel ?? "provider-default (unreported)" : execution.profile.model,
|
|
...(firstTaskEvidence ? { firstTask: firstTaskEvidence } : {}),
|
|
...(completionQuality.length ? { completionQuality } : {}),
|
|
runtimeMode: execution.profile.expectedRuntimeMode,
|
|
issueId: issue?.id,
|
|
issueIdentifier: issue?.identifier ?? null,
|
|
runIds: selectedRuns.map((run) => run.id),
|
|
...(turnTimings ? { turnTimings } : {}),
|
|
startedAt,
|
|
finishedAt: new Date(finishedAtMs).toISOString(),
|
|
durationMs: finishedAtMs - startedAtMs,
|
|
usage:
|
|
selectedRuns.length === 1
|
|
? (selectedRuns[0]?.usageJson ?? null)
|
|
: {
|
|
runs: selectedRuns.map((candidate) => ({
|
|
runId: candidate.id,
|
|
usage: candidate.usageJson ?? null,
|
|
})),
|
|
},
|
|
runtimeUsage,
|
|
matcherResults,
|
|
screenshots,
|
|
cleanup,
|
|
};
|
|
const result: RunnerE2EResult = {
|
|
...resultWithoutBilling,
|
|
billing: summarizeExecutionBilling(resultWithoutBilling),
|
|
};
|
|
await mkdir(privateDir, { recursive: true });
|
|
await writeFile(
|
|
resultPath,
|
|
`${JSON.stringify(sanitizeJson(result, secrets), null, 2)}\n`,
|
|
"utf8",
|
|
);
|
|
await testInfo.attach("runner-e2e-result", {
|
|
path: resultPath,
|
|
contentType: "application/json",
|
|
});
|
|
}
|
|
|
|
if (primaryError) throw primaryError;
|
|
});
|
|
}
|