import type { RunnerE2EResult } from "./types.js";
import { isBlockedConnectionResult } from "./connection-evidence.js";
import { validateOnboardingConnectEvidence } from "./product-journeys/onboarding-connect.js";
const html = (value: unknown) =>
String(value ?? "").replace(
/[&<>"']/g,
(c) =>
({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[
c
]!,
);
function renderOnboardingConnect(result: RunnerE2EResult): string {
const receipt = result.productJourney?.onboardingConnect;
if (receipt === undefined) return "";
try {
validateOnboardingConnectEvidence(receipt);
} catch {
return "
Onboarding Connect diagnostics unavailable: invalid receipt.
";
}
return `Onboarding Connect diagnostics
Last checkpoint: ${html(receipt.checkpoint)}
${([["validation", "Native validation"], ["hire", "Agent hire"]] as const).map(([key, label]) => {
const response = receipt[key];
return `${label}
${[
["Response state", response.state], ["HTTP status", response.httpStatus],
["Test endpoint adapter", response.requestedAdapter], ["Returned adapter", response.adapter],
["Validation outcome", response.outcome], ["Server code", response.serverCode],
].map(([name, value]) => `- ${name}
- ${html(value ?? "Unavailable")}
`).join("")}
${response.checks.length ? `Validation checks:
${response.checks.map(check => `${html(check.code)} · ${html(check.level)} `).join("")}
` : ""}
${response.truncatedChecks ? "Only the first 32 validation checks were retained.
" : ""}`;
}).join("")} `;
}
/** An interrupted first-task journey is not evidence of failed downstream behavior. */
export function isIncompleteFirstTaskResult(
result: RunnerE2EResult,
errors: readonly string[] = [],
) {
if (result.productJourney && ["blocked", "unsupported"].includes(result.productJourney.outcome)) return result.cleanup === "passed" && errors.length === 0;
if (isBlockedConnectionResult(result)) return result.cleanup === "passed" && result.failureClass !== "secret_leak" && errors.length === 0;
const checks = result.firstTask?.checks ?? [];
return result.status === "failed" &&
result.failureClass === "candidate_failure" &&
result.cleanup === "passed" && errors.length === 0 &&
checks.some((c) => c.notReached) &&
checks.every((c) => c.passed || c.notReached);
}
export function renderCaseOutcome(
result: RunnerE2EResult,
valid: boolean,
errors: readonly string[],
) {
const checks =
(result.productJourney?.checks.map(c => ({ ...c, detail: c.id.replaceAll("-", " "), notReached: undefined as string | undefined })) ?? (result.firstTask?.checks.length ? result.firstTask.checks : undefined)) ??
(result.matcherResults ?? []).map((m) => ({
id: m.matcher.kind === "json_path" ? m.matcher.path : m.matcher.kind,
passed: m.passed,
notReached: isBlockedConnectionResult(result) && !m.passed ? "Blocked before this checkpoint" : undefined as string | undefined,
detail: m.detail,
}));
const failures = checks.filter((c) => !c.passed && !c.notReached);
const notReached = checks.filter((c) => c.notReached);
const evaluated = checks.length - notReached.length;
const passed = checks.filter((c) => c.passed).length;
const incomplete = isIncompleteFirstTaskResult(result, errors);
const failed = !valid || result.status === "failed";
const reason =
result.failureClass === "secret_leak"
? /persisted Paperclip home/.test(result.error ?? "")
? "Credential-persistence check failed"
: "Secret / evidence safety check failed"
: (result.failureClass?.replaceAll("_", " ") ??
(result.cleanup === "failed"
? "Cleanup failed"
: "Run or evidence validation failed"));
const messages = [
...new Set(
[result.error, ...errors].filter((s): s is string => Boolean(s)),
),
];
return `
${result.productJourney ? html(result.productJourney.outcome) : isBlockedConnectionResult(result) ? `Blocked: ${html(result.providerConnection!.outcome.replaceAll("_", " "))}` : incomplete ? "Incomplete journey" : failed ? "Overall failed" : "Overall passed"} · ${checks.length ? `${passed}/${evaluated} behavioral checks passed${notReached.length ? ` · ${notReached.length} not reached` : ""}` : "No behavioral checks recorded"}
${result.productJourney ? `Deployment: ${html(result.productJourney.deployment?.kind ?? "unavailable")} · ${html(result.productJourney.deployment?.targetId ?? "unavailable")} · ${html(result.productJourney.deployment?.commit ?? "revision unavailable")} · starting state: ${html(result.productJourney.startingState)} · resources: ${html(result.productJourney.resourceDisposition)}
${result.productJourney.phases.map(p => `- ${html(p.phase)}: ${html(p.outcome)}${p.code ? ` (${html(p.code)})` : ""}
`).join("")}
` : ""}
${result.productJourney ? `Deployment and setup evidence
${[
["Campaign", result.productJourney.campaignId], ["Attempt", result.productJourney.attemptId],
["Origin", result.productJourney.deployment?.origin], ["Cloud release", result.productJourney.deployment?.cloudRelease],
["Runtime image", result.productJourney.deployment?.runtimeImage], ["Effective configuration hash", result.productJourney.deployment?.configurationHash],
["Harness revision", result.source?.sha], ["Journey definition hash", result.suiteDefinitionHash],
].map(([label, value]) => `- ${html(label)}
- ${html(value ?? "Unavailable")}
`).join("")}
${result.productJourney.deployment?.executionEnvironment ? `Managed environment: ${html(result.productJourney.deployment.executionEnvironment.id)} · ${html(result.productJourney.deployment.executionEnvironment.driver)} · ${html(result.productJourney.deployment.executionEnvironment.status)}
` : ""}
${result.productJourney.deployment?.effectiveFeatures ? `Effective features:
${Object.entries(result.productJourney.deployment.effectiveFeatures).map(([key, value]) => `- ${html(key)}
- ${html(value)}
`).join("")}
` : ""} ` : ""}
${result.productJourney?.operations ? `Eval operations: ${html(result.productJourney.operations.campaignId)} · ${result.productJourney.operations.requestCount} requests · model cost $${html(result.productJourney.operations.modelCostUsd ?? "unknown")} (setup $${html(result.productJourney.operations.setupCostUsd ?? "unknown")}, counted once). Feedback: ${html(result.productJourney.operations.feedback.collection)} · ${result.productJourney.operations.feedback.complaints} complaints · ${result.productJourney.operations.feedback.suggestions} suggestions.
` : ""}
${result.productJourney?.operations?.traceIds.length ? `Trace correlation IDs
${result.productJourney.operations.traceIds.map(id => `${html(id)} `).join("")}
` : ""}
${renderOnboardingConnect(result)}
${result.providerConnection ? `Target: ${html(result.providerConnection.target.origin)} · ${html(result.providerConnection.target.commit ?? "revision unavailable")} · ${result.providerConnection.assisted ? "Assisted login" : "Unassisted"} · ${html(result.providerConnection.authFreshness)} browser · ${html(result.providerConnection.entry)} / ${html(result.providerConnection.method)} · stopped at ${html(result.providerConnection.phase)}
` : ""}
${failures.length ? `Failed checks:
${failures.map((c) => `- ${html(c.id)} — ${html(c.detail)}
`).join("")}
` : checks.length ? `No behavioral matcher failed.${notReached.length ? " The journey is incomplete; unexercised checks are not passes." : failed ? " The overall failure came from a separate run, cleanup, or evidence check." : ""}
` : ""}
${notReached.length ? `Not reached:
${notReached.map((c) => `- ${html(c.id)} — ${html(c.notReached)}
`).join("")}
` : ""}
${checks.length ? `See all behavioral checks (${checks.length})
| Result | Check | Detail |
${checks.map((c) => `| ${c.notReached ? "Not reached" : c.passed ? "Pass" : "Fail"} | ${html(c.id)} | ${html(c.detail)}${c.notReached ? ` — ${html(c.notReached)}` : ""} |
`).join("")}
` : ""}
${
failed
? `${html(incomplete ? "Recording stopped before the journey finished" : reason)}
${messages
.map((m) =>
m.length > 1200
? `Full failure details (${m.length.toLocaleString("en-US")} characters)
${html(m)}
`
: `${html(m)}
`,
)
.join("")}`
: ""
}
`;
}