mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
Dotta
b1e3950e6d
Preserve the frozen catalog and its historical qualification separately. Keep current supervisor ownership, installed-artifact and recovery controls, the shipped TestOps cell, and the historical OpenCode candidate's exact model. Production source matches master f2db909393. Focused support checks: 218 passed; full catalog typecheck passed.
410 lines
18 KiB
TypeScript
410 lines
18 KiB
TypeScript
import { renderPublicCampaignSummary } from "./public-summary-image.js";
|
|
import { execFile } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { afterEach, describe, expect, it } from "vitest";
|
|
import { runnerMatrix } from "./catalog.js";
|
|
import { discoverReportCatalog } from "./report-catalog.js";
|
|
import { buildRunnerCampaign } from "./history.js";
|
|
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
|
import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
|
|
import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js";
|
|
import { journeyRequiredChecks } from "./product-journeys/success-contract.js";
|
|
import type { RunnerE2EResult } from "./types.js";
|
|
|
|
const temporary: string[] = [];
|
|
afterEach(async () => {
|
|
await Promise.all(
|
|
temporary.splice(0).map((dir) => rm(dir, { recursive: true, force: true })),
|
|
);
|
|
});
|
|
const id = "branch-only-chat.branch-provider.local.conversation";
|
|
const missingId = "branch-only-chat.branch-provider.local.reset";
|
|
function result(): RunnerE2EResult {
|
|
return {
|
|
schema: "paperclip.runner-e2e.result/v2",
|
|
executionId: id,
|
|
suiteId: "branch-only-chat",
|
|
suiteDefinitionHash: "branch-definition",
|
|
profileId: "branch-provider",
|
|
environmentId: "local",
|
|
caseId: "conversation",
|
|
provider: '<img src=x onerror="alert(1)">',
|
|
model: "branch-model",
|
|
runtimeMode: "native",
|
|
attempt: 1,
|
|
status: "passed",
|
|
cleanup: "passed",
|
|
durationMs: 1000,
|
|
startedAt: "2026-09-11T21:00:00.000Z",
|
|
finishedAt: "2026-09-11T21:00:01.000Z",
|
|
screenshots: [
|
|
{
|
|
id: "final-state",
|
|
label: '<script>alert("label")</script>',
|
|
file: "final-state.png",
|
|
publication: "public-runner-fixture",
|
|
},
|
|
],
|
|
};
|
|
}
|
|
function campaign(results: RunnerE2EResult[] = [result()]) {
|
|
return buildRunnerCampaign({
|
|
campaignId: "branch-campaign",
|
|
generatedAt: "2026-09-11T21:01:00.000Z",
|
|
expected: [id, missingId],
|
|
results,
|
|
});
|
|
}
|
|
|
|
describe("trusted report catalog discovery", () => {
|
|
it("shows only the campaign selection while retaining a missing selected result", () => {
|
|
const known = runnerMatrix.find((execution) => execution.id === "core-compatibility.legacy-codex.local.ask-question")!;
|
|
const missing = runnerMatrix.find((execution) => execution.id === "core-compatibility.legacy-codex.local.message-marker")!;
|
|
const page = renderRunnerE2EDashboard({
|
|
title: "Narrow campaign", generatedAt: "2026-10-20T12:00:00Z",
|
|
expected: [known.id, missing.id], catalog: runnerMatrix,
|
|
entries: [{ result: { ...result(), executionId: known.id, suiteId: known.suite.id,
|
|
profileId: known.profile.id, environmentId: known.environment.id, caseId: known.task.id }, valid: true, errors: [] }],
|
|
});
|
|
expect(page).toContain(`data-execution-id="${known.id}"`);
|
|
expect(page).toContain(`data-execution-id="${missing.id}"`);
|
|
expect(page).toContain('class="case case-missing"');
|
|
expect(page).not.toContain('id="suite-first-task"');
|
|
expect(page).not.toContain('data-report-profile="runner-copilot"');
|
|
expect(page).not.toContain('class="suite-nav"');
|
|
expect(page.match(/data-report-case\s/g)).toHaveLength(2);
|
|
expect(page).toContain('data-report-features="questions-and-approvals"');
|
|
expect(page).toContain('title="questions-and-approvals/questions"');
|
|
expect(page).toContain("No mapped journey");
|
|
expect(page).toContain("0/1 passed · 1 missing");
|
|
});
|
|
|
|
it("leaves an unfamiliar journey unmapped even when its case name matches", () => {
|
|
const known = runnerMatrix.find((execution) => execution.id === "core-compatibility.legacy-codex.local.ask-question")!;
|
|
expect(journeyCapabilities(known).map((capability) => capability.id)).toEqual(["questions-and-approvals/questions"]);
|
|
expect(journeyCapabilities({ ...known, suite: { ...known.suite, id: "branch-only" } })).toEqual([]);
|
|
const product = { ...known, suite: { ...known.suite, id: "product-journeys" } };
|
|
expect(journeyCapabilities(product).map(capability => capability.id)).toEqual(["questions-and-approvals/questions"]);
|
|
expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-create" } }).map(capability => capability.id)).toEqual(["tasks/create"]);
|
|
expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-document-persistence" } }).map(capability => capability.id)).toEqual(["documents-artifacts/documents"]);
|
|
expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-search-filter" } }).map(capability => capability.id)).toEqual(["tasks/organize"]);
|
|
});
|
|
|
|
it.each([
|
|
{ caseId: "question-answer-resume", feature: "questions-and-approvals", capabilities: ["questions-and-approvals/questions", "questions-and-approvals/continuation"] },
|
|
{ caseId: "followup-while-running", feature: "steering", capabilities: ["steering/queue"] },
|
|
])("reports deployment journey $caseId in its feature coverage", ({ caseId, feature, capabilities }) => {
|
|
const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!;
|
|
const execution = { ...known, id: `product-journeys.${known.profile.id}.${known.environment.id}.${caseId}`,
|
|
suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: caseId } };
|
|
expect(journeyCapabilities(execution).map(capability => capability.id)).toEqual(capabilities);
|
|
expect(journeyFeatureIds(execution)).toEqual([feature]);
|
|
expect(journeyCapabilities({ ...execution, suite: { ...execution.suite, id: "unrecognized-suite" } })).toEqual([]);
|
|
const page = renderRunnerE2EDashboard({ title: "Deployment coverage", generatedAt: "2026-10-20T12:00:00Z",
|
|
expected: [execution.id], catalog: [execution], entries: [{ valid: true, errors: [],
|
|
result: { ...result(), executionId: execution.id, suiteId: execution.suite.id, profileId: execution.profile.id,
|
|
environmentId: execution.environment.id, caseId, provider: "opencode", runIds: ["run-1"],
|
|
productJourney: { targetId: "qa", outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained",
|
|
phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks[caseId]!.map(id => ({ id, passed: true })),
|
|
deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40),
|
|
cloudRelease: "b".repeat(40), runtimeImage: "immutable-image", configurationHash: "config" } } } }] });
|
|
expect(page).toContain(`data-execution-id="${execution.id}"`);
|
|
expect(page).toContain(`data-report-features="${feature}"`);
|
|
expect(page).toContain(`title="${capabilities.join(", ")}"`);
|
|
expect(page).toContain("1/1 passed");
|
|
});
|
|
|
|
it("counts the project/task deployment journey toward projects and task creation coverage", () => {
|
|
const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!;
|
|
const project = { ...known, suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: "project-task-context" } };
|
|
expect(journeyCapabilities(project).map(capability => capability.id)).toEqual(["projects/lifecycle", "projects/task-context", "tasks/create"]);
|
|
expect(journeyFeatureIds(project)).toEqual(["projects", "tasks"]);
|
|
expect(journeyCapabilities({ ...project, suite: { ...project.suite, id: "unrecognized-suite" } })).toEqual([]);
|
|
});
|
|
|
|
it("aggregates unknown suites and renders their cases, missing results, and screenshot gallery", () => {
|
|
const summary = campaign();
|
|
expect(summary.suites).toEqual([
|
|
expect.objectContaining({
|
|
suiteId: "branch-only-chat",
|
|
selected: 2,
|
|
executed: 1,
|
|
passed: 1,
|
|
failed: 1,
|
|
complete: false,
|
|
}),
|
|
]);
|
|
const page = renderRunnerE2EDashboard({
|
|
title: "Report",
|
|
generatedAt: summary.generatedAt,
|
|
expected: summary.expected,
|
|
catalog: [],
|
|
campaign: summary,
|
|
entries: [
|
|
{
|
|
result: result(),
|
|
valid: true,
|
|
errors: [],
|
|
evidenceBaseHref: `evidence/${id}/attempt-1`,
|
|
evidenceFiles: ["final-state.png"],
|
|
},
|
|
],
|
|
});
|
|
expect(page).toContain('id="suite-branch-only-chat"');
|
|
expect(page).toContain('class="suite-summary"');
|
|
expect(page).toContain(`data-execution-id="${id}"`);
|
|
expect(page).toContain(`data-execution-id="${missingId}"`);
|
|
expect(page).toContain('class="case case-missing"');
|
|
expect(page).toContain("No result artifact was uploaded");
|
|
expect(page).toContain(
|
|
`data-gallery-href="evidence/${id}/attempt-1/final-state.png"`,
|
|
);
|
|
expect(page).toContain("View gallery · 1");
|
|
expect(page).toContain("<img src=x onerror="alert(1)">");
|
|
expect(page).not.toContain("<img src=x");
|
|
expect(page).not.toContain('<script>alert("label")</script>');
|
|
});
|
|
|
|
it("renders selected branch suites in the public summary image without provider content", () => {
|
|
const page = renderPublicCampaignSummary(campaign());
|
|
expect(page).toContain("<strong>1/2</strong>");
|
|
expect(page).toContain("<span>branch-only-chat</span>");
|
|
expect(page).not.toContain("<strong>0/0</strong>");
|
|
expect(page).not.toContain("onerror");
|
|
expect(page).not.toContain("branch-model");
|
|
});
|
|
|
|
it("retains ordinary catalog metadata and marks no-results branch selections failed", () => {
|
|
const catalog = discoverReportCatalog({
|
|
catalog: runnerMatrix,
|
|
expected: [runnerMatrix[0]!.id],
|
|
results: [],
|
|
});
|
|
expect(catalog).toEqual(runnerMatrix);
|
|
expect(catalog[0]).toBe(runnerMatrix[0]);
|
|
expect(campaign([])).toMatchObject({
|
|
selected: 2,
|
|
executed: 0,
|
|
passed: 0,
|
|
failed: 2,
|
|
suites: [
|
|
expect.objectContaining({ suiteId: "branch-only-chat", failed: 2 }),
|
|
],
|
|
});
|
|
});
|
|
|
|
it("uses known profile metadata in a new suite and rejects forged identities", () => {
|
|
const known = runnerMatrix[0]!;
|
|
const knownProfileId = `branch-only-chat.${known.profile.id}.${known.environment.id}.conversation`;
|
|
expect(
|
|
discoverReportCatalog({
|
|
catalog: runnerMatrix,
|
|
expected: [knownProfileId],
|
|
results: [],
|
|
}).at(-1)?.profile,
|
|
).toBe(known.profile);
|
|
expect(() =>
|
|
discoverReportCatalog({
|
|
catalog: [],
|
|
expected: [id],
|
|
results: [{ ...result(), suiteId: "forged-suite" }],
|
|
}),
|
|
).toThrow("Result identity does not match");
|
|
expect(() =>
|
|
discoverReportCatalog({
|
|
catalog: [],
|
|
expected: ["../../private"],
|
|
results: [],
|
|
}),
|
|
).toThrow("Invalid report execution identity");
|
|
expect(() =>
|
|
discoverReportCatalog({ catalog: [], expected: [id, id], results: [] }),
|
|
).toThrow("must be unique");
|
|
});
|
|
|
|
it("uses the retained result model/provider for an unknown suite sharing a known profile ID", () => {
|
|
const known = runnerMatrix.find(row => row.profile.id === "runner-opencode")!;
|
|
const executionId = `branch-only-chat.${known.profile.id}.${known.environment.id}.conversation`;
|
|
const retained = { ...result(), executionId, profileId: known.profile.id, environmentId: known.environment.id,
|
|
provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", runtimeMode: "native" as const };
|
|
const oldCatalog = [{ ...known, profile: { ...known.profile, model: "obsolete-profile-default" } }];
|
|
const discovered = discoverReportCatalog({ catalog: oldCatalog, expected: [executionId], results: [retained] });
|
|
const profile = discovered.find(row => row.id === executionId)!.profile;
|
|
expect(profile.model).toBe(retained.model); expect(profile.provider).toBe(retained.provider);
|
|
expect(profile.label).toBe(known.profile.id);
|
|
const page = renderRunnerE2EDashboard({ title: "Retained source", generatedAt: retained.finishedAt,
|
|
expected: [executionId], catalog: oldCatalog, entries: [{ result: retained, valid: false, errors: [] }] });
|
|
expect(page).toContain(retained.model); expect(page).not.toContain("obsolete-profile-default");
|
|
expect(retained.source).toEqual(result().source);
|
|
});
|
|
|
|
it("regenerates branch-only cards and gallery from persisted JSON using trusted code", async () => {
|
|
const bundle = await mkdtemp(
|
|
path.join(os.tmpdir(), "branch-report-regenerate-"),
|
|
);
|
|
temporary.push(bundle);
|
|
const evidence = path.join(bundle, "evidence", id, "attempt-1");
|
|
await mkdir(evidence, { recursive: true });
|
|
await writeFile(path.join(evidence, "final-state.png"), "fixture");
|
|
await writeFile(
|
|
path.join(bundle, "normalized-results.json"),
|
|
JSON.stringify({ ...campaign([{ ...result(), targetUrl: "https://qa.example.test", matcherResults: [{ matcher: { kind: "json_path", path: "saved", expected: true }, passed: true, detail: "Saved after reload" }] }]), suites: [], presentation: { title: "QA basics", notes: ["Original measurement retained"], links: [{ label: "Provenance", href: "provenance.json" }, { label: "Unsafe destination", href: "javascript:alert(1)" }] } }),
|
|
);
|
|
await regenerateRunnerDashboard({ bundle, historyFile: null });
|
|
const page = await readFile(path.join(bundle, "index.html"), "utf8");
|
|
expect(page).toContain('data-gallery-target-url="https://qa.example.test/"');
|
|
expect(page).toContain('href="https://qa.example.test/" target="_blank"');
|
|
expect(page).toContain("Saved after reload");
|
|
expect(page).toContain("Original measurement retained");
|
|
expect(page).toContain('<a href="provenance.json">Provenance</a>');
|
|
expect(page).not.toContain("javascript:alert(1)");
|
|
expect(page).not.toContain("Unsafe destination");
|
|
expect(page).toContain(`data-execution-id="${id}"`);
|
|
expect(page).toContain(`data-execution-id="${missingId}"`);
|
|
expect(page).toContain(
|
|
`data-gallery-href="evidence/${id}/attempt-1/final-state.png"`,
|
|
);
|
|
const normalized = JSON.parse(
|
|
await readFile(path.join(bundle, "normalized-results.json"), "utf8"),
|
|
);
|
|
expect(normalized.presentation).toEqual({ title: "QA basics", notes: ["Original measurement retained"], links: [{ label: "Provenance", href: "provenance.json" }, { label: "Unsafe destination", href: "javascript:alert(1)" }] });
|
|
expect(normalized.results[0].targetUrl).toBe("https://qa.example.test");
|
|
expect(normalized.suites).toEqual([
|
|
expect.objectContaining({
|
|
suiteId: "branch-only-chat",
|
|
selected: 2,
|
|
passed: 1,
|
|
failed: 1,
|
|
}),
|
|
]);
|
|
});
|
|
|
|
it("generates an explicit failed result with the correct unknown identity when no artifact exists", async () => {
|
|
const root = await mkdtemp(
|
|
path.join(os.tmpdir(), "branch-report-missing-"),
|
|
);
|
|
temporary.push(root);
|
|
const out = path.join(root, "merged");
|
|
const repo = path.resolve(import.meta.dirname, "../..");
|
|
await promisify(execFile)(
|
|
process.execPath,
|
|
[
|
|
path.join(repo, "cli/node_modules/tsx/dist/cli.mjs"),
|
|
path.join(repo, "tests/runner-e2e/report.ts"),
|
|
],
|
|
{
|
|
cwd: repo,
|
|
env: {
|
|
...process.env,
|
|
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
|
|
PAPERCLIP_RUNNER_E2E_REPORT_OUT: out,
|
|
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([missingId]),
|
|
},
|
|
},
|
|
).catch((error: { code?: number }) => {
|
|
if (error.code !== 1) throw error;
|
|
});
|
|
const normalized = JSON.parse(
|
|
await readFile(path.join(out, "normalized-results.json"), "utf8"),
|
|
);
|
|
expect(normalized.results).toEqual([
|
|
expect.objectContaining({
|
|
executionId: missingId,
|
|
suiteId: "branch-only-chat",
|
|
profileId: "branch-provider",
|
|
caseId: "reset",
|
|
attempt: 0,
|
|
status: "failed",
|
|
}),
|
|
]);
|
|
expect(normalized.suites).toEqual([
|
|
expect.objectContaining({
|
|
suiteId: "branch-only-chat",
|
|
failed: 1,
|
|
executed: 0,
|
|
}),
|
|
]);
|
|
expect(await readFile(path.join(out, "index.html"), "utf8")).toContain(
|
|
`data-execution-id="${missingId}"`,
|
|
);
|
|
});
|
|
it.each([
|
|
"1/../../../escape",
|
|
"<img src=x onerror=alert(1)>",
|
|
-1,
|
|
1.5,
|
|
Infinity,
|
|
])("rejects malformed attempts before report staging: %s", (attempt) => {
|
|
expect(() =>
|
|
campaign([{ ...result(), attempt } as RunnerE2EResult]),
|
|
).toThrow("result.attempt");
|
|
});
|
|
|
|
it("validates nested timing and billing fields before rendering", () => {
|
|
const malicious = {
|
|
...result(),
|
|
turnTimings: [{ turn: "<script>alert(1)</script>" }],
|
|
} as unknown as RunnerE2EResult;
|
|
expect(() => campaign([malicious])).toThrow("result.turnTimings[0].turn");
|
|
const measured = campaign().results[0]!;
|
|
expect(() =>
|
|
campaign([
|
|
{
|
|
...measured,
|
|
billing: {
|
|
...measured.billing!,
|
|
llm: {
|
|
...measured.billing!.llm,
|
|
runCount: "<img>" as unknown as number,
|
|
},
|
|
},
|
|
},
|
|
]),
|
|
).toThrow("result.billing.llm.runCount");
|
|
});
|
|
|
|
it("does not certify a same-sized replacement matrix or changed suite definition", () => {
|
|
const known = runnerMatrix.filter(
|
|
(entry) => entry.suite.id === runnerMatrix[0]!.suite.id,
|
|
);
|
|
const receipts = known.map((entry) => ({
|
|
...result(),
|
|
executionId: entry.id,
|
|
suiteId: entry.suite.id,
|
|
suiteDefinitionHash: entry.suiteDefinitionHash,
|
|
profileId: entry.profile.id,
|
|
environmentId: entry.environment.id,
|
|
caseId: entry.task.id,
|
|
}));
|
|
const build = (results: RunnerE2EResult[]) =>
|
|
buildRunnerCampaign({
|
|
campaignId: "matrix-proof",
|
|
generatedAt: result().finishedAt,
|
|
expected: results.map((item) => item.executionId),
|
|
results,
|
|
});
|
|
expect(build(receipts).suites[0]!.complete).toBe(true);
|
|
const replacement = receipts.map((item, index) =>
|
|
index
|
|
? item
|
|
: {
|
|
...item,
|
|
caseId: "replacement",
|
|
executionId: item.executionId.replace(/[^.]+$/, "replacement"),
|
|
},
|
|
);
|
|
expect(build(replacement).suites[0]!.complete).toBe(false);
|
|
expect(
|
|
build(
|
|
receipts.map((item) => ({
|
|
...item,
|
|
suiteDefinitionHash: "different-definition",
|
|
})),
|
|
).suites[0]!.complete,
|
|
).toBe(false);
|
|
});
|
|
});
|