diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index 0b163da408b..f110d1959b6 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -32,9 +32,7 @@ export function grade_documents_artifacts(value: unknown) { const target = entity(o.state, "pricingThread"); const sibling = entity(o.state, "siblingThread"); const annotation = o.annotations?.find(row => row.id === target.id); const unrelated = o.annotations?.find(row => row.id === sibling.id); const plan = o.documents.find(row => row.key === "plan"); - c.add("substantive-pricing-fix", !!plan && /\$19/.test(plan.body ?? "") && /\b(?:monthly|month)\b/i.test(plan.body ?? "") - && /\bteam\b/i.test(plan.body ?? "") && /\b(?:total|entire|whole|per team|for the team)\b/i.test(plan.body ?? "") - && !/\b(?:per|each)\s+(?:user|person|seat)\b/i.test(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.supportText)), + c.add("substantive-pricing-fix", !!plan && monthlyTeamPricing(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.supportText)), "The actual current document unambiguously preserves $19 monthly total for the team and unrelated support content."); c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId && unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId, @@ -149,3 +147,22 @@ export function meaningfulOrchidPlan(body: string): boolean { return !clauses.some(clause => /\b(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause) && !negativeDirective.test(clause) && !/\b(?:not|no|without|avoid)\s+(?:a\s+)?(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause)); } + +export function monthlyTeamPricing(body: string): boolean { + const pricingClauses = body.split(/(?<=[.!?])\s+|[\n;]|\bbut\b/i); + if (!pricingClauses.some(clause => /\$19(?:\.00)?(?![\d.])/.test(clause) && /\b(?:monthly|month)\b/i.test(clause) + && /\bteam\b/i.test(clause) && /\b(?:total|entire|whole|per team|for the team)\b/i.test(clause))) return false; + const negated = (index: number) => /\b(?:not|never|no|rather than|instead of)(?:\s+(?:charged|billed|priced|calculated|applied))?(?:\s+(?:a|an|the))?(?:\s+\$[\d,.]+)?\s*$/i.test(body.slice(Math.max(0, index - 80), index)); + for (const amount of body.matchAll(/\$([\d,]+(?:\.\d+)?)/g)) { + if (Number(amount[1]!.replaceAll(",", "")) !== 19 && !negated(amount.index!)) return false; + if (Number(amount[1]!.replaceAll(",", "")) === 19 && negated(amount.index!) + && !/\bper[- ](?:user|person|seat)\b/i.test(body.slice(amount.index! + amount[0].length).split(/[.;\n]/)[0]!)) return false; + } + for (const scope of body.matchAll(/\b(?:per[- ]|each\s+|every\s+)(?:user|person|seat)\b/gi)) { + const suffix = body.slice(scope.index! + scope[0].length); + const deniedAfter = /^\s+(?:(?:pricing|billing|charge)\s+)?(?:is|would be)\s+(?:wrong|incorrect|not (?:correct|applicable|required))\b/i.test(suffix); + if (!negated(scope.index!) && !deniedAfter) return false; + } + for (const cadence of body.matchAll(/\b(?:yearly|annually|annual|per year|one[- ]time)\b/gi)) if (!negated(cadence.index!)) return false; + return !/\b(?:not|never|no)\s+(?:monthly|per month|(?:the\s+)?total|(?:(?:for|per)\s+(?:the\s+)?)?(?:(?:entire|whole)\s+)?team)\b/i.test(body); +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts new file mode 100644 index 00000000000..d300327f6ee --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts @@ -0,0 +1,43 @@ +import { expect, it } from "vitest"; +import { grade_documents_artifacts, monthlyTeamPricing } from "./internal-documents-artifacts.js"; + +it.each([ + "$19 monthly total for the entire team, not per user.", + "$19 monthly total for the entire team, not $19 per user.", + "$19 monthly total for the entire team, not a per-user charge.", + "$19 per month for the whole team. It is not billed per person.", + "The monthly total is $19 per team; no per-seat charge.", + "$19 monthly total for the team, not $49. Per user pricing is incorrect.", +])("accepts correct team pricing and scoped negation of the original ambiguity: %s", body => { + expect(monthlyTeamPricing(body)).toBe(true); +}); +it.each([ + "$19 monthly total for the team. Each user also pays $19.", + "$19 monthly total for the team, but $19 per user too.", + "$190 monthly total for the entire team, not per user.", + "Not $19 monthly total for the team; $49 instead.", + "$19 yearly total for the team. Monthly support is available.", + "$19 monthly per user.", + "$19 monthly total, not for the entire team.", + "The personal subscription costs $19. Monthly support is available for the entire team in total.", + "Monthly total for the entire team, not per user.", + "$19 monthly total, not for the team, but per user.", +])("rejects missing pricing, wrong amount/cadence or an actual per-person charge: %s", body => { + expect(monthlyTeamPricing(body)).toBe(false); +}); +it("preserves unrelated content, exact annotation relationships, saved revision and edit-before-resolution proof", () => { + const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" }; + const supportText = "Support is provided by the original launch team fixture-42."; + const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`; + const o = { scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo", + state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [], + documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }], + annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }], + activity: [{ id: "edit", action: "issue.document_updated", agentId: "ceo", createdAt: "2026-10-27T10:00:00Z" }, + { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-10-27T10:00:01Z" }], + reply: "Updated the pricing and resolved its review comment.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] }; + expect(grade_documents_artifacts(o).passed).toBe(true); + expect(grade_documents_artifacts({ ...o, documents: [{ ...o.documents[0], body: "$19 monthly total for the entire team, not per user." }] }).passed).toBe(false); + expect(grade_documents_artifacts({ ...o, annotations: o.annotations.map(row => ({ ...row, status: "resolved" })) }).passed).toBe(false); + expect(grade_documents_artifacts({ ...o, activity: [o.activity[1], { ...o.activity[0], createdAt: "2026-10-27T10:00:02Z" }] }).passed).toBe(false); +});