test(evals): distinguish team pricing from negated per-user charges

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts

@@ -32,9 +32,7 @@ export function grade_documents_artifacts(value: unknown) {
       const target = entity(o.state, "pricingThread"); const sibling = entity(o.state, "siblingThread");
       const annotation = o.annotations?.find(row => row.id === target.id); const unrelated = o.annotations?.find(row => row.id === sibling.id);
       const plan = o.documents.find(row => row.key === "plan");
-      c.add("substantive-pricing-fix", !!plan && /\$19/.test(plan.body ?? "") && /\b(?:monthly|month)\b/i.test(plan.body ?? "")
-        && /\bteam\b/i.test(plan.body ?? "") && /\b(?:total|entire|whole|per team|for the team)\b/i.test(plan.body ?? "")
-        && !/\b(?:per|each)\s+(?:user|person|seat)\b/i.test(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.supportText)),
+      c.add("substantive-pricing-fix", !!plan && monthlyTeamPricing(plan.body ?? "") && (plan.body ?? "").includes(String(o.state.supportText)),
         "The actual current document unambiguously preserves $19 monthly total for the team and unrelated support content.");
       c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId
         && unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId,
@@ -149,3 +147,22 @@ export function meaningfulOrchidPlan(body: string): boolean {
   return !clauses.some(clause => /\b(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause)
     && !negativeDirective.test(clause) && !/\b(?:not|no|without|avoid)\s+(?:a\s+)?(?:multi[- ]page|multiple\s+pages|separate\s+pages)\b/i.test(clause));
 }
+
+export function monthlyTeamPricing(body: string): boolean {
+  const pricingClauses = body.split(/(?<=[.!?])\s+|[\n;]|\bbut\b/i);
+  if (!pricingClauses.some(clause => /\$19(?:\.00)?(?![\d.])/.test(clause) && /\b(?:monthly|month)\b/i.test(clause)
+    && /\bteam\b/i.test(clause) && /\b(?:total|entire|whole|per team|for the team)\b/i.test(clause))) return false;
+  const negated = (index: number) => /\b(?:not|never|no|rather than|instead of)(?:\s+(?:charged|billed|priced|calculated|applied))?(?:\s+(?:a|an|the))?(?:\s+\$[\d,.]+)?\s*$/i.test(body.slice(Math.max(0, index - 80), index));
+  for (const amount of body.matchAll(/\$([\d,]+(?:\.\d+)?)/g)) {
+    if (Number(amount[1]!.replaceAll(",", "")) !== 19 && !negated(amount.index!)) return false;
+    if (Number(amount[1]!.replaceAll(",", "")) === 19 && negated(amount.index!)
+      && !/\bper[- ](?:user|person|seat)\b/i.test(body.slice(amount.index! + amount[0].length).split(/[.;\n]/)[0]!)) return false;
+  }
+  for (const scope of body.matchAll(/\b(?:per[- ]|each\s+|every\s+)(?:user|person|seat)\b/gi)) {
+    const suffix = body.slice(scope.index! + scope[0].length);
+    const deniedAfter = /^\s+(?:(?:pricing|billing|charge)\s+)?(?:is|would be)\s+(?:wrong|incorrect|not (?:correct|applicable|required))\b/i.test(suffix);
+    if (!negated(scope.index!) && !deniedAfter) return false;
+  }
+  for (const cadence of body.matchAll(/\b(?:yearly|annually|annual|per year|one[- ]time)\b/gi)) if (!negated(cadence.index!)) return false;
+  return !/\b(?:not|never|no)\s+(?:monthly|per month|(?:the\s+)?total|(?:(?:for|per)\s+(?:the\s+)?)?(?:(?:entire|whole)\s+)?team)\b/i.test(body);
+}

tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts

@@ -0,0 +1,43 @@
+import { expect, it } from "vitest";
+import { grade_documents_artifacts, monthlyTeamPricing } from "./internal-documents-artifacts.js";
+
+it.each([
+  "$19 monthly total for the entire team, not per user.",
+  "$19 monthly total for the entire team, not $19 per user.",
+  "$19 monthly total for the entire team, not a per-user charge.",
+  "$19 per month for the whole team. It is not billed per person.",
+  "The monthly total is $19 per team; no per-seat charge.",
+  "$19 monthly total for the team, not $49. Per user pricing is incorrect.",
+])("accepts correct team pricing and scoped negation of the original ambiguity: %s", body => {
+  expect(monthlyTeamPricing(body)).toBe(true);
+});
+it.each([
+  "$19 monthly total for the team. Each user also pays $19.",
+  "$19 monthly total for the team, but $19 per user too.",
+  "$190 monthly total for the entire team, not per user.",
+  "Not $19 monthly total for the team; $49 instead.",
+  "$19 yearly total for the team. Monthly support is available.",
+  "$19 monthly per user.",
+  "$19 monthly total, not for the entire team.",
+  "The personal subscription costs $19. Monthly support is available for the entire team in total.",
+  "Monthly total for the entire team, not per user.",
+  "$19 monthly total, not for the team, but per user.",
+])("rejects missing pricing, wrong amount/cadence or an actual per-person charge: %s", body => {
+  expect(monthlyTeamPricing(body)).toBe(false);
+});
+it("preserves unrelated content, exact annotation relationships, saved revision and edit-before-resolution proof", () => {
+  const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" };
+  const supportText = "Support is provided by the original launch team fixture-42.";
+  const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
+  const o = { scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
+    state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [],
+    documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }],
+    annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }],
+    activity: [{ id: "edit", action: "issue.document_updated", agentId: "ceo", createdAt: "2026-10-27T10:00:00Z" },
+      { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-10-27T10:00:01Z" }],
+    reply: "Updated the pricing and resolved its review comment.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] };
+  expect(grade_documents_artifacts(o).passed).toBe(true);
+  expect(grade_documents_artifacts({ ...o, documents: [{ ...o.documents[0], body: "$19 monthly total for the entire team, not per user." }] }).passed).toBe(false);
+  expect(grade_documents_artifacts({ ...o, annotations: o.annotations.map(row => ({ ...row, status: "resolved" })) }).passed).toBe(false);
+  expect(grade_documents_artifacts({ ...o, activity: [o.activity[1], { ...o.activity[0], createdAt: "2026-10-27T10:00:02Z" }] }).passed).toBe(false);
+});