diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 0633fcee60c..990671279dd 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5264,7 +5264,7 @@ "chat-catalog-domains/internal-definition.ts", "chat-scenarios/skills.yaml" ], - "oracleVersion": "paperclip.chat-catalog.skill-studio.v2", + "oracleVersion": "paperclip.chat-catalog.skill-studio.v3", "expectedRunCount": 3, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.test.ts index 4ad7f696a27..2b26f3ba512 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.test.ts @@ -1,10 +1,13 @@ import { describe, expect, it } from "vitest"; import { gradeIntegratorSkillStudio, studioPageInput, studioNativeRunsComplete } from "./integrator-skill-studio.js"; +const record = (value: object) => "Checked the supplied page.\n\n```json\n" + JSON.stringify(value) + "\n```\n"; +const missingLabel = { check: "Every input has a visible associated label", element: '' }; +function setOutput(o: ReturnType, output: string) { const run = o.studioRuns[0]!; run.outputBody = output; run.outputSnapshot = output; run.documents[0]!.body = output; } function valid() { const skill = { id: "skill", companyId: "company", currentVersionId: "revision", markdown: "Inspect visible form labels", editable: true }; - const output = "The email input trial-email has no visible associated label. This accessibility check fails on the sample; add a label linked to its id."; + const output = record({ passed: 3, failed: 1, failures: [missingLabel] }); const run = { id: "test", companyId: "company", skillId: "skill", agentId: "morgan", skillVersionId: "revision", skillVersion: { id: "revision", companyId: "company", companySkillId: "skill", authorAgentId: null as string | null, authorUserId: "person", fileInventory: [{ path: "SKILL.md", kind: "markdown" }] }, - inputId: "input", inputSnapshot: studioPageInput, status: "succeeded", issueId: "harness", harnessIssue: { id: "harness", status: "done" }, harnessContent: { available: true }, + inputId: "input", inputSnapshot: studioPageInput, status: "succeeded", issueId: "harness", harnessIssue: { id: "harness", identifier: "COM-2", status: "done" }, harnessContent: { available: true }, outputBody: output, outputSnapshot: output, outputDocumentKey: "output", documents: [{ key: "output", body: output }], interactions: [] }; return { scenarioId: "skills.studio-test", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { skill: structuredClone(skill), input: { id: "input" }, worker: { id: "morgan" } }, tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], activity: [], links: [], @@ -14,7 +17,7 @@ function valid() { { id: "completion", companyId: "company", agentId: "ceo", runtimeMode: "native", status: "succeeded", startedAt: "2026-11-03T00:03:00Z", contextSnapshot: { issueId: "source", wakeReason: "chat_task_completed", wakeSource: "automation", chatCompletionDeliveryIds: ["delivery"], chatCompletionUpdates: [{ id: "harness", status: "done", completedAt: "2026-11-03T00:02:00Z" }] } as Record }, ], skill, studioRuns: [run], pinnedVersions: { test: { id: "revision", companyId: "company", companySkillId: "skill", authorAgentId: null as string | null, authorUserId: "person", fileInventory: [{ path: "SKILL.md", content: skill.markdown }] } }, sourceIssueId: "source", sourceComments: [{ id: "human", authorUserId: "person", body: "Test this skill on our sample landing page.", authorAgentId: null as string | null, createdByRunId: undefined as string | undefined, createdAt: "2026-11-03T00:00:00Z" }, - { id: "completed-reply", authorUserId: undefined, authorAgentId: "ceo", createdByRunId: "completion", createdAt: "2026-11-03T00:04:00Z", body: "The Studio result is saved for inspection." }], + { id: "completed-reply", authorUserId: undefined, authorAgentId: "ceo", createdByRunId: "completion", createdAt: "2026-11-03T00:04:00Z", body: "The test finished: [COM-2](/COM/issues/COM-2#document-output) has the saved result." }], reply: "Started one Studio test on the saved sample and selected accessibility revision. You can inspect the result there; this does not qualify other pages." }; } describe("one real inspectable Studio test", () => { @@ -35,17 +38,28 @@ describe("one real inspectable Studio test", () => { expect(gradeIntegratorSkillStudio(other).passed).toBe(false); expect(gradeIntegratorSkillStudio({ ...valid(), tasks: [{ id: "unrelated" }] }).passed).toBe(false); }); - it("accepts substantive paraphrase and rejects false sample or universal success", () => { - const o = valid(); - const paraphrase = "The email field is unlabeled; it fails the visible-label requirement. Associate a visible label with trial-email."; - o.studioRuns[0]!.outputBody = paraphrase; o.studioRuns[0]!.outputSnapshot = paraphrase; o.studioRuns[0]!.documents[0]!.body = paraphrase; - expect(gradeIntegratorSkillStudio(o).passed).toBe(true); - const falsePage = valid(); const wrong = "The email input trial-email has a visible label; no accessibility failures."; - falsePage.studioRuns[0]!.outputBody = wrong; falsePage.studioRuns[0]!.outputSnapshot = wrong; falsePage.studioRuns[0]!.documents[0]!.body = wrong; - expect(gradeIntegratorSkillStudio(falsePage).passed).toBe(false); - expect(gradeIntegratorSkillStudio({ ...o, reply: "Every production agent is qualified and all pages are verified." }).passed).toBe(false); + it("grades the requested result record and result link, not the review wording", () => { + const check = (o: ReturnType, id: string) => gradeIntegratorSkillStudio(o).checks.find(row => row.id === id)?.passed; + const reworded = valid(); setOutput(reworded, "# Any heading at all\n" + record({ passed: 0, failed: 1, failures: [{ ...missingLabel, check: "Form control naming" }] })); + expect(gradeIntegratorSkillStudio(reworded).passed).toBe(true); + const clean = valid(); setOutput(clean, record({ passed: 4, failed: 0, failures: [] })); + expect(check(clean, "saved-concrete-sample-failure")).toBe(false); + const wrongElement = valid(); setOutput(wrongElement, record({ passed: 3, failed: 1, failures: [{ check: "labels", element: '' }] })); + expect(check(wrongElement, "saved-concrete-sample-failure")).toBe(false); + const invented = valid(); setOutput(invented, record({ passed: 2, failed: 2, failures: [missingLabel, { check: "alt", element: '' }] })); + expect(check(invented, "saved-concrete-sample-failure")).toBe(false); + const miscounted = valid(); setOutput(miscounted, record({ passed: 3, failed: 2, failures: [missingLabel] })); + expect(check(miscounted, "saved-concrete-sample-failure")).toBe(false); + const prose = valid(); setOutput(prose, "The email input trial-email has no visible associated label."); + expect(check(prose, "saved-concrete-sample-failure")).toBe(false); + const unlinked = valid(); unlinked.sourceComments[1]!.body = "The test finished; COM-2 has the saved result."; + expect(check(unlinked, "reply-links-saved-result")).toBe(false); + const foreign = valid(); foreign.sourceComments[1]!.body = "See [COM-3](/COM/issues/COM-3)."; + expect(check(foreign, "reply-links-saved-result")).toBe(false); + const studio = valid(); studio.sourceComments[1]!.body = "Open the [Studio run](/COM/skills/studio/skill?run=test)."; + expect(check(studio, "reply-links-saved-result")).toBe(true); const failed = valid(); failed.studioRuns[0]!.status = "failed"; - expect(gradeIntegratorSkillStudio({ ...failed, reply: "The test passed." }).passed).toBe(false); + expect(gradeIntegratorSkillStudio(failed).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.ts index d8a1cc64ad7..fbab84bf7a3 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-studio.ts @@ -9,7 +9,39 @@ import { checks, observation, project, type Entity, type InternalObservation } f import { fixtureWorker } from "./internal-definition.js"; import { integratorDefinition, observeInternal } from "./integrator-support.js"; export const studioPageInput = 'Synthetic landing page

Launch trial

'; -const studioMarkdown = "---\nname: accessibility-review\ndescription: Review supplied synthetic HTML for accessibility.\n---\n\nInspect the provided input only. Check that every input has a visible associated label. Record the exact element and evidence for any failed check. Do not invent browser tests or claim production qualification. Save the requested test output and complete the test task; do not create other work."; +const studioMarkdown = "---\nname: accessibility-review\ndescription: Review supplied synthetic HTML for accessibility.\n---\n\nInspect the provided input only. Check that every input has a visible associated label. Record the exact element and evidence for any failed check. Do not invent browser tests or claim production qualification. Save the requested test output and complete the test task; do not create other work. End the output with a fenced json block recording the result exactly as {\"passed\": , \"failed\": , \"failures\": [{\"check\": \"\", \"element\": \"\"}]}."; +// The fixture skill asks for this literal record, so grading reads the record, +// not the wording of the review around it. +export interface StudioResultRecord { passed: number; failed: number; failures: Array<{ check: string; element: string }> } +export function studioResultRecord(output: string): StudioResultRecord | undefined { + const block = [...output.matchAll(/```json[^\n]*\n([\s\S]*?)```/gi)].at(-1)?.[1]; + if (!block) return undefined; + try { + const value = JSON.parse(block) as StudioResultRecord; + const count = (n: unknown) => Number.isInteger(n) && (n as number) >= 0; + return count(value.passed) && count(value.failed) && Array.isArray(value.failures) && value.failures.length === value.failed + && value.failures.every(row => typeof row?.check === "string" && !!row.check.trim() && typeof row.element === "string" && !!row.element.trim()) ? value : undefined; + } catch { return undefined; } +} +// The only form control in the synthetic input has no label; every reported +// failing element must be markup copied from the input. +export function recordFindsMissingLabel(record: StudioResultRecord | undefined) { + return !!record && record.failed >= 1 && record.failures.every(row => row.element.trim().startsWith("<") && studioPageInput.includes(row.element.trim())) + && record.failures.some(row => /^]*\bid="trial-email"/.test(row.element.trim())); +} +// The completion reply must link the saved result: the hidden harness task (or +// its output document) or the Studio run page. +export function completionReplyLinksResult(o: StudioObservation) { + const run = o.studioRuns[0]; if (!run) return false; + const completion = o.runs.find(row => row.agentId === o.leadId && (row.contextSnapshot?.chatCompletionUpdates as Array<{ id?: unknown }> | undefined)?.some(update => update?.id === run.issueId)); + const body = o.sourceComments.find(row => !!completion && row.authorAgentId === o.leadId && row.createdByRunId === completion.id)?.body ?? ""; + const targets = [run.issueId, run.harnessIssue?.identifier].filter((id): id is string => !!id); + return [...body.matchAll(/\]\(([^)\s]+)\)/g)].map(match => match[1]!).some(href => { + const url = new URL(href, "https://paperclip.invalid"); + return url.origin === "https://paperclip.invalid" && (targets.some(id => new RegExp(`^/[^/]+/issues/${id.replace(/[-]/g, "\\-")}$`).test(url.pathname)) + || (url.pathname.endsWith(`/skills/studio/${run.skillId}`) && url.searchParams.get("run") === run.id)); + }); +} const root = (ctx: ChatCatalogContext, skillId: string) => `/api/companies/${ctx.input.fixtures.company.id}/skills/${skillId}`; interface StudioObservation extends InternalObservation { skill: CompanySkillDetail; studioRuns: CompanySkillTestRunDetail[]; pinnedVersions: Record; sourceIssueId: string; sourceComments: Entity[] } async function observeStudio(ctx: ChatCatalogContext, state: Record): Promise { @@ -79,18 +111,13 @@ export function gradeIntegratorSkillStudio(value: unknown) { && run.harnessIssue?.id === run.issueId && run.harnessIssue.status === "done" && run.harnessContent.available, "Studio success is backed by a real completed native harness task; it is not a fabricated test row or another task's run."); const output = run?.outputBody ?? ""; - c.add("saved-concrete-sample-failure", !!run && output.length > 0 && output === run.outputSnapshot - && /trial-email|email (?:input|field)/i.test(output) && /\b(?:label|labelled|labeled)\b/i.test(output) - && /\b(?:missing|absent|no (?:visible |associated )?label|unlabelled|unlabeled|fails?|without)\b/i.test(output) - && !/\b(?:all checks pass|no accessibility (?:issues|failures)|(?:input|field) (?:has|includes) (?:a )?(?:visible |associated )?label)\b/i.test(output) + c.add("saved-concrete-sample-failure", !!run && output.length > 0 && output === run.outputSnapshot && recordFindsMissingLabel(studioResultRecord(output)) && run.documents.some(doc => doc.key === run.outputDocumentKey && doc.body === output), - "The saved inspectable output identifies the genuine missing label on the supplied email input; a successful test execution cannot turn a failing sample into a clean page."); + "The saved output's requested result record counts at least one failure and names the supplied unlabeled email input by its exact markup; a successful test execution cannot turn a failing sample into a clean page."); c.add("no-unrelated-work-or-skill-mutation", o.skill.currentVersionId === before?.currentVersionId && o.skill.markdown === before.markdown && o.tasks.every(task => task.id === run?.issueId) && !o.activity.some(event => event.agentId === o.leadId && event.action === "company.skill_file_updated"), "The test does not rewrite its tested revision or create unrelated production tasks."); - c.add("bounded-report-not-global-qualification", !/\b(?:all|every)\s+(?:production )?(?:agents?|pages?|websites?)\b[^.!?\n]{0,45}\b(?:qualified|passed|accessible|verified|enabled)\b/i.test(o.reply) - && !(/\b(?:test|run)\b[^.!?\n]{0,35}\b(?:passed|succeeded|completed)\b/i.test(o.reply) && run?.status !== "succeeded"), - "Chat does not elevate one test to universal qualification or misreport the actual test status."); + c.add("reply-links-saved-result", completionReplyLinksResult(o), "The completion reply links the saved Studio result (its test task, output document or Studio run) so the person can inspect it."); return c.result(); } export const integratorSkillStudioCases = [integratorDefinition("skills", "skills.studio-test", async ctx => { @@ -109,7 +136,7 @@ export const integratorSkillStudioCases = [integratorDefinition("skills", "skill await ctx.input.evidence("chat-skill-studio-visible-result.json", { skillId: o.skill.id, testRunId: run.id, status: run.status, issueId: run.issueId, interactions: run.interactions, url: ctx.input.page.url() }); await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, { authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md"), sourceFiles: ["chat-catalog-domains/integrator-skill-versions.ts", "../../server/src/services/chat-completion-delivery.ts", "chat-catalog-domains/integrator-skill-studio.ts", "chat-catalog-domains/integrator-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/internal-definition.ts", "chat-scenarios/skills.yaml"], - oracleVersion: "paperclip.chat-catalog.skill-studio.v2", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 540_000, + oracleVersion: "paperclip.chat-catalog.skill-studio.v3", expectedRunCount: 3, minimumRunCount: 2, attemptTimeoutMs: 540_000, async act(ctx, state) { await pollUntil({ label: "actual bounded skill Studio output", deadlineAt: Date.now() + 240_000, intervalMs: 1000, load: () => observeStudio(ctx, state), accept: o => studioNativeRunsComplete(o),