From 8b82676ed6ca26347642985ba0769cab010ac7d9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:22:47 -0500 Subject: [PATCH] test(e2e): author native accounting and partial subscription journeys --- .../identity-accounting.test.ts | 30 ++++ .../identity-accounting.ts | 140 ++++++++++++++++++ .../identity-subscriptions.test.ts | 22 +++ .../identity-subscriptions.ts | 87 +++++++++++ 4 files changed, 279 insertions(+) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-accounting.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-subscriptions.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-subscriptions.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts new file mode 100644 index 00000000000..12ae92ac805 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { accountingIdentityCases } from "./identity-accounting.js"; +import type { IdentityObservation } from "./identity-support.js"; +function valid(scenarioId: string): IdentityObservation { + const repair = scenarioId.endsWith("repair-cost-recording"); + return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: [], + runs: [{ id: "original", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }], + links: ["/ACO/issues/morgan-receipt", "/ACO/issues/riley-receipt"], + reply: repair ? "Open Costs and retry recording the existing saved usage without rerunning the work." : "Launch this week: Morgan $1.37; Riley $2.63; recorded metered API total $4.00. Accounting is incomplete: unresolved usage is pending. Subscription fees stay separate.", + activity: repair ? [{ id: "retry", action: "accounting.retry_requested", entityId: "original", actorType: "user" }] : [], + state: { worker: { id: "morgan" }, project: { id: "launch" }, morganReceiptTask: { id: "morgan-receipt" }, rileyReceiptTask: { id: "riley-receipt" }, originalRunId: "original", + originalRun: { id: "original", status: "succeeded", usageJson: { inputTokens: 10 } }, afterOriginalRun: { id: "original", status: "succeeded", usageJson: { inputTokens: 10 } }, + originalUsagePreserved: true, originalLogPreserved: true, originalTaskPreserved: true, unresolvedBefore: { canRetry: true, runs: [{ runId: "original", reason: "retryable" }], events: [] }, + receiptOrigin: "public-imported-synthetic-fixture", weekFrom: "2026-10-26T00:00:00Z", weekTo: "2026-11-02T00:00:00Z", + issueCosts: [{ costCents: 137 }, { costCents: 263 }], projectCosts: [{ projectId: "launch", costCents: 400 }], agentRetryStatus: 403, humanRetryHttpStatus: 200, retryAccounted: false, retryReceiptVisible: true, + confirmedPricedRepair: false, afterStillUnresolved: true } }; +} +describe("actual saved native accounting contexts", () => { + for (const d of accountingIdentityCases) { + it(`${d.scenarioId}: accepts full observable contract and rejects absent evidence`, () => { const o = valid(d.scenarioId); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); }); + it(`${d.scenarioId}: fabricated missing usage is not a prerequisite`, () => { const o = valid(d.scenarioId); (o.state.originalRun as { usageJson: unknown }).usageJson = null; expect(d.grade(o).passed).toBe(false); }); + it(`${d.scenarioId}: wrong-run quality cannot substitute`, () => { const o = valid(d.scenarioId); o.state.unresolvedBefore = { runs: [{ runId: "different", reason: "retryable" }], events: [] }; expect(d.grade(o).passed).toBe(false); }); + it(`${d.scenarioId}: rerunning original work fails`, () => { const o = valid(d.scenarioId); o.runs.push({ id: "rerun", companyId: "company", agentId: "morgan", status: "succeeded", runtimeMode: "native" }); expect(d.grade(o).passed).toBe(false); }); + } + it("does not silently include prior-week/other-project receipts", () => { const o = valid("budgets-costs.spend-breakdown"); o.state.projectCosts = [{ projectId: "launch", costCents: 424 }]; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); }); + it("does not count unresolved native usage as free", () => { const o = valid("budgets-costs.spend-breakdown"); o.reply += " The pending usage is free."; expect(accountingIdentityCases[0]!.grade(o).passed).toBe(false); }); + it("a HTTP200 retry alone does not establish repaired accounting", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.afterStillUnresolved = false; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); + it("a genuine confirmed priced repair is distinct from still-unresolved", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.confirmedPricedRepair = true; o.state.afterStillUnresolved = false; o.state.retryAccounted = true; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(true); }); + it("an actual agent retry bypass fails", () => { const o = valid("budgets-costs.repair-cost-recording"); o.state.agentRetryStatus = 200; expect(accountingIdentityCases[1]!.grade(o).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts new file mode 100644 index 00000000000..54a005b9aa9 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-accounting.ts @@ -0,0 +1,140 @@ +import { createHash } from "node:crypto"; +import { expect } from "@playwright/test"; +import type { HeartbeatRun } from "../../../packages/shared/src/types/heartbeat.js"; +import type { CostDataQualityReport, CostByProject, CostEvent, IssueCostSummary } from "../../../packages/shared/src/types/cost.js"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { checks, entity, hasLink, project, task, sameTask, type Entity } from "./internal-support.js"; +import { identityDefinition, identityObservation, identityWorker, observeIdentity, openStandingChat, probeAgentPermission } from "./identity-support.js"; + +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); +interface NativeAccountingState { worker: Entity; project: Entity; preflightTask: Entity; originalRun: HeartbeatRun; originalRunId: string; originalUsageHash: string; originalLogHash: string | null; unresolvedBefore: CostDataQualityReport } +async function nativeUnresolvedContext(ctx: ChatCatalogContext, retryable: boolean): Promise { + await openStandingChat(ctx); const worker = await identityWorker(ctx); const p = await project(ctx, "Launch"); + const root = `/api/companies/${ctx.input.fixtures.company.id}`; + // Before any setup provider work, use real board policy controls. This is + // preparation for the genuine native prerequisite run, never a cost receipt. + for (const [scopeType, scopeId] of [["company", ctx.input.fixtures.company.id], ["agent", worker.id]]) await ctx.input.api.post(`${root}/budgets/policies`, { scopeType, scopeId, amount: 10000, unpricedUsagePolicy: "allow", windowKind: "calendar_month_utc" }); + await pollUntil({ label: "real accounting preflight worker ready", deadlineAt: Date.now() + 45_000, load: () => ctx.input.api.get<{ lifecycleState: string }>(`/api/agents/${worker.id}`), accept: row => row.lifecycleState === "ready" }); + const preflightTask = await task(ctx, "Native accounting prerequisite", p.id, { assigneeAgentId: worker.id, status: "todo", + description: "Return a concise readiness acknowledgement, then close this accounting preflight task. Do not create files, delegate, or start other tasks. This is a real native-provider prerequisite execution, not a simulated receipt." }); + const run = await pollUntil({ label: "genuine completed native accounting prerequisite", deadlineAt: Date.now() + 180_000, + load: () => ctx.allRuns(), accept: rows => rows.some(row => row.agentId === worker.id && row.status === "succeeded" && row.runtimeMode === "native") }).then(rows => rows.find(row => row.agentId === worker.id && row.status === "succeeded" && row.runtimeMode === "native")!); + const original = await ctx.input.api.get(`/api/heartbeat-runs/${run.id}`); + const quality = await ctx.input.api.get(`${root}/costs/data-quality?period=all`); + const matching = [...quality.runs, ...quality.events].filter(row => row.runId === run.id); + const eligible = original.usageJson != null && Object.keys(original.usageJson).length > 0 && matching.some(row => !retryable || row.reason === "retryable"); + await ctx.input.evidence("identity-native-accounting-context.json", { runId: run.id, taskId: preflightTask.id, workerId: worker.id, nativeRunActuallyCompleted: true, + usageEvidenceExists: original.usageJson != null && Object.keys(original.usageJson).length > 0, actualUnresolvedReasons: matching.map(row => row.reason), requiresRetryableSavedUsage: retryable, + prerequisiteMet: eligible, usageAccountingFabricated: false, setupNativeUsageMustCountInCampaignAccounting: true }); + if (!eligible) throw new Error(`Setup blocker: the actual completed native run has ${matching.length ? "no eligible saved retryable usage" : "fully settled accounting"}; the frozen unresolved context cannot be fabricated`); + return { worker, project: p, preflightTask: await ctx.input.api.get(`/api/issues/${preflightTask.id}`), originalRun: original, originalRunId: run.id, originalUsageHash: digest(original.usageJson), originalLogHash: original.logSha256, unresolvedBefore: quality }; +} +export function gradeAccounting(value: unknown) { + const c = checks(); const o = identityObservation(value); c.add("complete-native-accounting-evidence", !!o, "Actual completed native usage, public accounting projections and persisted independent readback are required."); if (!o) return c.result(); + const before = o.state.originalRun as HeartbeatRun | undefined; const after = o.state.afterOriginalRun as HeartbeatRun | undefined; + const quality = o.state.unresolvedBefore as CostDataQualityReport | undefined; + c.add("genuine-unresolved-native-context", !!before && before.id === o.state.originalRunId && before.status === "succeeded" && o.runs.some(run => run.id === before.id && run.runtimeMode === "native") && before.usageJson != null + && Object.keys(before.usageJson).length > 0 && !!quality && [...quality.runs, ...quality.events].some(row => row.runId === before.id), + "The frozen missing-accounting prerequisite comes from a real completed native run and actual saved usage/quality records, never a patched run or fabricated null receipt."); + c.add("original-work-never-rerun", after?.id === before?.id && after?.status === "succeeded" && o.state.originalUsagePreserved === true && o.state.originalLogPreserved === true + && o.runs.filter(run => run.agentId === entity(o.state, "worker").id).length === 1 && o.state.originalTaskPreserved === true, + "The actual prior worker run, provider usage and immutable log evidence remain, and no second worker execution or work mutation appears."); + if (o.scenarioId === "budgets-costs.spend-breakdown") { + const totals = o.state.issueCosts as IssueCostSummary[] | undefined; const projects = o.state.projectCosts as CostByProject[] | undefined; + c.add("independent-scoped-week-ledger", o.state.receiptOrigin === "public-imported-synthetic-fixture" && totals?.length === 2 && totals[0]?.costCents === 137 && totals[1]?.costCents === 263 + && projects?.find(row => row.projectId === entity(o.state, "project").id)?.costCents === 400 && typeof o.state.weekFrom === "string" && typeof o.state.weekTo === "string", + "Two explicitly imported synthetic USD API receipts total $4 in Launch's declared week, independently excluding the other-project and prior-week distractors; these are not live campaign/provider charges."); + c.add("actual-agent-breakdown-and-incomplete-total", /Morgan[\s\S]{0,120}(?:\$|USD\s*)?1\.37/i.test(o.reply) && /Riley[\s\S]{0,120}(?:\$|USD\s*)?2\.63/i.test(o.reply) + && /(?:\$|USD\s*)4\.00/.test(o.reply) && /week/i.test(o.reply) && /API|metered/i.test(o.reply) && /subscription|monthly fee/i.test(o.reply) + && /incomplete|unresolved|pending|not.*(?:complete|final)|missing.*cost/i.test(o.reply) && !/\b(?:unresolved|pending|missing)\b[^.\n]{0,50}\b(?:free|no charge|zero cost)\b/i.test(o.reply) + && hasLink(o, "issues", entity(o.state, "morganReceiptTask")) && hasLink(o, "issues", entity(o.state, "rileyReceiptTask")), + "The actual answer breaks down the recorded scoped API dollars by real agent with supporting records, keeps subscription fees separate and labels unresolved usage instead of counting it free."); + } else if (o.scenarioId === "budgets-costs.repair-cost-recording") { + c.add("real-human-saved-cost-retry", quality?.runs.some(row => row.runId === before?.id && row.reason === "retryable") === true && o.state.agentRetryStatus === 403 + && o.state.humanRetryHttpStatus === 200 && typeof o.state.retryAccounted === "boolean" && o.state.retryReceiptVisible === true + && o.activity.filter(row => row.action === "accounting.retry_requested" && row.entityId === before?.id && row.actorType === "user").length === 1, + "The authorized person clicks the actual saved-cost retry control for exactly that genuine retryable run; the agent cannot impersonate board accounting authority."); + c.add("honest-repair-or-unresolved-result", o.state.confirmedPricedRepair === true || o.state.afterStillUnresolved === true, + "Independent accounting readback distinguishes an actually repaired priced charge from a still-unresolved receipt/estimate; a successful HTTP request alone cannot claim repair."); + c.add("guides-accounting-without-rerun", /cost|accounting|usage/i.test(o.reply) && /retry|record/i.test(o.reply) && /saved|existing|without.*run|not.*rerun|no.*rerun/i.test(o.reply) + && !/\bI(?:'ve| have)?\s+(?:fixed|repaired|recorded|rerun|restarted)\b/i.test(o.reply), "The canonical response guides the real saved-usage control without claiming an unauthorized repair or rerunning the work."); + } else c.add("known-accounting-case", false, "No accounting oracle exists for this case."); + return c.result(); +} +const sources = ["chat-catalog-domains/identity-accounting.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/budgets-costs.yaml"]; +async function verifyUnresolvedSetup(ctx: ChatCatalogContext, state: Record) { + const quality = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/costs/data-quality?period=all`); + const c = checks(); const rows = [...quality.runs, ...quality.events].filter(row => row.runId === state.originalRunId); + c.add("genuine-unresolved-saved-native-usage", rows.some(row => ctx.scenario.id.endsWith("spend-breakdown") || row.reason === "retryable"), + "Immediately before the canonical query, real native saved-usage accounting is still unresolved in the public projection; an automatic repair makes this a setup blocker instead of a fabricated context."); + if (c.result().passed) state.unresolvedBefore = quality; return c.result(); +} +async function observeAccounting(ctx: ChatCatalogContext, state: Record) { + const run = await ctx.input.api.get(`/api/heartbeat-runs/${state.originalRunId}`); state.afterOriginalRun = run; + state.originalUsagePreserved = digest(run.usageJson) === state.originalUsageHash; state.originalLogPreserved = run.logSha256 === state.originalLogHash; + const task = await ctx.input.api.get(`/api/issues/${entity(state, "preflightTask").id}`); state.originalTaskPreserved = sameTask(entity(state, "preflightTask"), task); + if (ctx.scenario.id.endsWith("spend-breakdown")) { + state.issueCosts = await Promise.all(["morganReceiptTask", "rileyReceiptTask"].map(key => ctx.input.api.get(`/api/issues/${entity(state, key).id}/cost-summary`))); + state.projectCosts = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/costs/by-project?from=${encodeURIComponent(String(state.weekFrom))}&to=${encodeURIComponent(String(state.weekTo))}`); + } + return observeIdentity(ctx, state); +} +async function verifyAccountingLink(ctx: ChatCatalogContext, state: Record, value: unknown) { + const o = identityObservation(value); const source = ctx.input.page.url(); + const href = o?.links.find(link => { const url = new URL(link, ctx.input.api.baseURL); return url.origin === ctx.input.api.baseURL + && (ctx.scenario.id.endsWith("spend-breakdown") + ? [entity(state, "morganReceiptTask"), entity(state, "rileyReceiptTask")].some(row => [row.id, row.identifier].some(id => id && url.pathname.includes(`/issues/${id}`))) + : url.pathname.includes("/activity/costs") || url.pathname.includes(`/runs/${state.originalRunId}`)); }); + if (!href) throw new Error("Actual supporting cost/task/run link is required"); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + if (ctx.scenario.id.endsWith("spend-breakdown")) await expect(ctx.input.page.getByText(/Launch imported (?:Morgan|Riley) API usage/, { exact: true }).first()).toBeVisible(); + else await expect(ctx.input.page.getByText(/Cost|Usage|Native accounting prerequisite/i).first()).toBeVisible(); + await ctx.input.evidence("identity-accounting-support-link.json", { originalRunId: state.originalRunId, actualBrowserTarget: ctx.input.page.url(), actualSupportingRecordVisible: true }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); +} +export const accountingIdentityCases: readonly ChatCatalogCase[] = [ + identityDefinition("budgets-costs.spend-breakdown", async ctx => { + const state = await nativeUnresolvedContext(ctx, false); const riley = await identityWorker(ctx, "Riley"); + const date = new Date(); const week = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); week.setUTCDate(week.getUTCDate() - (week.getUTCDay() + 6) % 7); + const weekFrom = week.toISOString(); const weekTo = new Date(week.getTime() + 7 * 86400000).toISOString(); + const morganReceiptTask = await task(ctx, "Launch imported Morgan API usage", state.project.id, { assigneeAgentId: state.worker.id }); + const rileyReceiptTask = await task(ctx, "Launch imported Riley API usage", state.project.id, { assigneeAgentId: riley.id }); const other = await project(ctx, "Other project"); + const distractor = await task(ctx, "Other project imported API usage", other.id, { assigneeAgentId: state.worker.id }); + const priorWeek = await task(ctx, "Prior week Launch imported API usage", state.project.id, { assigneeAgentId: state.worker.id }); + const root = `/api/companies/${ctx.input.fixtures.company.id}`; const receipts: string[] = []; + for (const [work, agentId, cents, occurredAt] of [[morganReceiptTask, state.worker.id, 137, new Date().toISOString()], [rileyReceiptTask, riley.id, 263, new Date().toISOString()], + [distractor, state.worker.id, 11, new Date().toISOString()], [priorWeek, state.worker.id, 13, new Date(week.getTime() - 86400000).toISOString()]] as const) { + const receipt = await ctx.input.api.post(`${root}/cost-events`, { agentId, issueId: work.id, projectId: work.projectId, provider: "synthetic-eval-import", biller: "synthetic-eval-import", model: "imported-fixture-api-usage", + billingType: "metered_api", costStatus: "reported", costCents: cents, inputTokens: 0, outputTokens: 0, occurredAt, idempotencyKey: `identity-report:${ctx.input.nonce}:${receipts.length}` }); receipts.push(receipt.id); + } + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The requested reporting week is ${weekFrom} through ${weekTo} (UTC). Launch contains imported synthetic metered-API receipt tasks ${morganReceiptTask.identifier} (Morgan) and ${rileyReceiptTask.identifier} (Riley), plus actual completed native run ${state.originalRunId} with unresolved saved usage. Imported fixture receipts are not live provider/campaign billing. Other-project and earlier-week charges must be excluded; native subscription usage/fees are separate.` }); + return { ...state, riley, morganReceiptTask, rileyReceiptTask, weekFrom, weekTo, importedReceiptIds: receipts, receiptOrigin: "public-imported-synthetic-fixture" }; + }, gradeAccounting, { authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, + setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] } }), + identityDefinition("budgets-costs.repair-cost-recording", async ctx => { + const state = await nativeUnresolvedContext(ctx, true); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The completed native run requiring saved-cost recovery is ${state.originalRunId}, on task ${state.preflightTask.identifier}. Its public accounting quality record is actually retryable and retains saved usage. Only the person may use cost-recording retry; do not start or rerun the task.` }); + return { ...state }; + }, gradeAccounting, { + authentication: "human-session", sourceFiles: sources, expectedRunCount: 2, attemptTimeoutMs: 600_000, observe: observeAccounting, verifyResult: verifyAccountingLink, verifySetup: verifyUnresolvedSetup, + setupPolicy: { funding: "standard", readiness: "all-ready", invariantCheckIds: ["genuine-unresolved-saved-native-usage"] }, + async act(ctx, state) { + await pollUntil({ label: "accounting guidance source settled", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, + accept: rows => rows.length === 2 && rows.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting" }); + const root = `/api/companies/${ctx.input.fixtures.company.id}`; state.agentRetryStatus = await probeAgentPermission(ctx, `${root}/costs/retry-accounting`, "POST", { runId: state.originalRunId }); + const source = ctx.input.page.url(); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/activity/costs?period=all`, { waitUntil: "domcontentloaded" }); + await ctx.input.page.getByRole("button", { name: /run costs? unavailable|billing types? unknown/i }).first().click(); + const row = ctx.input.page.getByRole("listitem").filter({ has: ctx.input.page.locator(`a[href*="/runs/${state.originalRunId}"]`) }).filter({ has: ctx.input.page.getByRole("button", { name: "Retry cost recording", exact: true }) }).last(); + const response = ctx.input.page.waitForResponse(result => result.request().method() === "POST" && new URL(result.url()).pathname === `${root}/costs/retry-accounting` && result.request().postDataJSON()?.runId === state.originalRunId); + await row.getByRole("button", { name: "Retry cost recording", exact: true }).click(); const receipt = await response; state.humanRetryHttpStatus = receipt.status(); const result = await receipt.json() as { accounted: boolean }; state.retryAccounted = result.accounted; + await expect(ctx.input.page.getByText(result.accounted ? "Costs recorded." : "No new cost data is available.", { exact: true })).toBeVisible(); state.retryReceiptVisible = true; + await expect(ctx.input.page.getByText("The agent was not rerun.", { exact: true })).toBeVisible(); + const after = await ctx.input.api.get(`${root}/costs/data-quality?period=all`); state.afterStillUnresolved = [...after.runs, ...after.events].some(row => row.runId === state.originalRunId); + state.confirmedPricedRepair = result.accounted === true && state.afterStillUnresolved === false; + await ctx.input.evidence("identity-saved-usage-retry.json", { runId: state.originalRunId, actualHumanHttpStatus: receipt.status(), accounted: result.accounted, confirmedPricedRepair: state.confirmedPricedRepair, + afterStillUnresolved: state.afterStillUnresolved, originalRunNeverRerun: (await ctx.allRuns()).filter(run => run.agentId === entity(state, "worker").id).length === 1 }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + }), +]; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-subscriptions.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-subscriptions.test.ts new file mode 100644 index 00000000000..06e8de9f4d6 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-subscriptions.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import type { SubscriptionAccountReport, SubscriptionCostReport, SubscriptionUsage } from "../../../packages/shared/src/subscriptions.js"; +import type { IdentityObservation } from "./identity-support.js"; +import { subscriptionIdentityCases } from "./identity-subscriptions.js"; +const usage: SubscriptionUsage = { inputTokens: 10, cachedInputTokens: 0, outputTokens: 1, costCents: "0", eventCount: 1, estimatedEventCount: 0, unpricedEventCount: 0 }; +function account(id: string): SubscriptionAccountReport { return { id, provider: "openai", name: id, ownerUserId: "person", ownerName: "Actual owner", shared: false, identityVerified: false, detectedPlan: null, observedAt: null, lastCheckedAt: null, refreshStatus: null, canEdit: true, + price: { revision: 1, plan: "User recorded plan", amountCents: "2000", monthlyCents: "2000", currency: "USD", cadence: "month", status: "active", source: "user", sourceUrl: null, effectiveAt: "2026-10-01T00:00:00Z" }, usage, agents: [] }; } +function report(accounts: SubscriptionAccountReport[]): SubscriptionCostReport { return { accounts, canRefresh: true, asOf: "2026-10-26T00:00:00Z", monthlyTotals: [{ currency: "USD", amountCents: String(accounts.length * 2000), estimatedCount: 0 }], activeCount: accounts.length, unknownPriceCount: 0, unidentifiedAccountCount: 0, api: usage, subscription: usage, unknown: usage, unattributedSubscription: usage }; } +function valid(): IdentityObservation { const pair = [account("source"), account("target")]; return { scenarioId: "budgets-costs.subscription-fees", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", company: { id: "company" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], interactions: [], humanQueries: [], runs: [], + reply: "Open the subscription view to verify account ownership and the same paid seat. You can link duplicate records there. Tokens and recorded API usage do not establish the subscription fee.", links: ["/ACO/activity/costs"], + activity: [{ id: "price", action: "subscription.price_updated", entityId: "source", details: { revision: 1 } }, { id: "link", action: "subscription.linked", entityId: "source", actorType: "user", actorId: "person", details: { targetId: "target" } }], + state: { personId: "person", duplicateAccounts: pair, beforeReport: report(pair), afterReport: report([pair[1]!]), sameAccountProvenanceVerified: true, provenanceOrigin: "genuine-owned-provider-history", agentReadStatus: 403, agentLinkStatus: 403, noPreHumanMutation: true, humanLinkStatus: 200, ownershipVisible: true, linkDisclosureVisible: true, priceHistoryBefore: [{ id: "price", entityId: "source", details: { revision: 1 } }], archivedPriceHistoryVerified: true } }; } +describe("subscription duplicate contract is explicitly partial", () => { + const d = subscriptionIdentityCases[0]!; + it("full intended evidence calibrates; missing evidence fails", () => { const o = valid(); expect(d.grade(o), JSON.stringify(d.grade(o))).toMatchObject({ passed: true }); expect(d.grade(undefined).passed).toBe(false); expect(d.coverageGaps?.length).toBeGreaterThan(0); }); + it("two owned same-provider records do not prove ONE account", () => { const o = valid(); o.state.sameAccountProvenanceVerified = false; expect(d.grade(o).passed).toBe(false); }); + it("fabricated provenance is rejected", () => { const o = valid(); o.state.provenanceOrigin = "generated-fingerprint"; expect(d.grade(o).passed).toBe(false); }); + it("wrong actual owner fails even after successful human link", () => { const o = valid(); (o.state.duplicateAccounts as SubscriptionAccountReport[])[1]!.ownerUserId = "another-person"; expect(d.grade(o).passed).toBe(false); }); + it("deleting historical receipts cannot masquerade as deduplication", () => { const o = valid(); o.activity = o.activity.filter(row => row.id !== "price"); expect(d.grade(o).passed).toBe(false); }); + it("current target price alone cannot establish archived price preservation", () => { const o = valid(); o.state.archivedPriceHistoryVerified = false; expect(d.grade(o).passed).toBe(false); }); + it("ordinary agent merge or false fee calculation fails", () => { const o = valid(); o.state.agentLinkStatus = 200; o.reply += " I have merged them."; expect(d.grade(o).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-subscriptions.ts b/tests/runner-e2e/chat-catalog-domains/identity-subscriptions.ts new file mode 100644 index 00000000000..79294f1ddc3 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-subscriptions.ts @@ -0,0 +1,87 @@ +import { expect } from "@playwright/test"; +import type { SubscriptionAccountReport, SubscriptionCostReport } from "../../../packages/shared/src/subscriptions.js"; +import type { ChatCatalogCase } from "../chat-catalog-contract.js"; +import { checks, type Entity } from "./internal-support.js"; +import { identityDefinition, identityObservation, observeIdentity, openStandingChat, probeAgentPermission, waitForReply } from "./identity-support.js"; + +/** Intentionally partial. Provider/name/owner equality does not establish that + * two records represent ONE paid account. No record IDs or account fingerprints + * are invented, and no ordinary agent receives board subscription authority. */ +export function gradeSubscriptions(value: unknown) { + const c = checks(); const o = identityObservation(value); c.add("complete-subscription-evidence", !!o, "Actual owned account records, provenance and saved human linking are required."); if (!o) return c.result(); + const before = o.state.beforeReport as SubscriptionCostReport | undefined; const after = o.state.afterReport as SubscriptionCostReport | undefined; + const pair = o.state.duplicateAccounts as SubscriptionAccountReport[] | undefined; + c.add("one-actual-account-not-two-similar-records", pair?.length === 2 && pair[0]?.id !== pair[1]?.id && pair.every(row => row.canEdit && row.ownerUserId === o.state.personId) + && pair[0]?.provider === pair[1]?.provider && o.state.sameAccountProvenanceVerified === true && o.state.provenanceOrigin === "genuine-owned-provider-history", + "Both editable records must be owned by the actual human AND independently proven to describe one real provider account. Matching labels, two different accounts or fabricated fingerprints cannot qualify."); + c.add("board-only-subscription-actions", o.state.agentReadStatus === 403 && o.state.agentLinkStatus === 403 && o.state.noPreHumanMutation === true, + "The chat cannot read board subscription details or silently merge them, and the records remain distinct until the real human decision."); + c.add("human-link-persisted-with-target-price", !!pair && !!before && !!after && o.state.humanLinkStatus === 200 && o.state.ownershipVisible === true && o.state.linkDisclosureVisible === true + && after.accounts.length === before.accounts.length - 1 && !after.accounts.some(row => row.id === pair[0]?.id) + && JSON.stringify(after.accounts.find(row => row.id === pair[1]?.id)?.price) === JSON.stringify(pair[1]?.price) + && o.activity.some(row => row.action === "subscription.linked" && row.entityId === pair[0]?.id && row.actorType === "user" && row.actorId === o.state.personId && row.details?.targetId === pair[1]?.id), + "The actual human links the exact verified duplicate into the selected owned account; fresh report and audit retain its real existing price."); + const priceEvents = o.state.priceHistoryBefore as Entity[] | undefined; + c.add("historical-price-evidence-retained", !!priceEvents?.length && priceEvents.every(beforeEvent => o.activity.some(row => row.id === beforeEvent.id && row.entityId === beforeEvent.entityId && JSON.stringify(row.details) === JSON.stringify(beforeEvent.details))) + && o.state.archivedPriceHistoryVerified === true, + "Original price revision receipts remain unchanged AND archived historical prices require independent supported readback; a current price alone cannot prove the full history."); + c.add("routes-human-without-fabricated-fees", /subscription/i.test(o.reply) && /owner|ownership|same account|same.*seat/i.test(o.reply) && /link|duplicate/i.test(o.reply) + && /human|person|board|you can|open/i.test(o.reply) && !/\bI(?:'ve| have)?\s+(?:merged|linked|deduplicated)\b/i.test(o.reply) + && !/tokens?[^.\n]{0,50}(?:means? (?:a |the )?(?:fee|subscription)|subscription costs?)/i.test(o.reply) + && o.links.some(href => { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname.includes("/activity/costs"); }), + "The response routes the person to actual subscription controls, explains ownership and duplicate checks, and never fabricates a fee from token usage or claims an agent merge."); + return c.result(); +} +const sources = ["chat-catalog-domains/identity-subscriptions.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/budgets-costs.yaml", + "../../packages/shared/src/subscriptions.ts", "../../server/src/routes/costs.ts", "../../server/src/routes/ai-connections.ts", "../../server/src/services/subscriptions.ts", "../../ui/src/components/SubscriptionCostCard.tsx"]; +export const subscriptionIdentityCases: readonly ChatCatalogCase[] = [identityDefinition("budgets-costs.subscription-fees", async ctx => { + await openStandingChat(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; + const session = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); + const report = await ctx.input.api.get(`${root}/costs/subscriptions`); + const agentReadStatus = await probeAgentPermission(ctx, `${root}/costs/subscriptions`, "GET"); + await ctx.input.evidence("identity-subscription-context-blocker.json", { actualHumanId: session.user.id, visibleRecordIds: report.accounts.map(row => row.id), + ownedRecordIds: report.accounts.filter(row => row.ownerUserId === session.user.id && row.canEdit).map(row => row.id), agentReadStatus, + originalQuery: ctx.scenario.query, originalQuerySent: false, sameAccountProvenanceAvailable: false, setupBlocked: true, + unsupportedContext: "Public reports omit provider account identity. Local connection creation requires genuine browser login per session; no existing authentic duplicate-account backup/provenance was supplied. Current report also cannot read archived price revisions." }); + // Explicit setup blocker, not a successful refusal or arbitrary two-account + // substitute. A future admitted authentic owned backup needs public import, + // provenance verification and archived-price readback before removing gaps. + throw new Error("Setup blocker: two authentic records for ONE actual subscription and inspectable historical prices are not available through the current admitted public fixture path"); +}, gradeSubscriptions, { + authentication: "human-session", sourceFiles: sources, + coverageGaps: ["No supported admitted fixture establishes two subscription records with independently verified provenance for ONE actual account. Public current-price reports also omit archived price revisions; complete historical-price preservation cannot yet be independently verified. The original context remains blocked, never replaced by different accounts or a nonexistent-resource refusal."], + async act(ctx, state) { + await waitForReply(ctx); const pair = state.duplicateAccounts as SubscriptionAccountReport[] | undefined; + if (!pair || pair.length !== 2 || state.sameAccountProvenanceVerified !== true || state.provenanceOrigin !== "genuine-owned-provider-history") throw new Error("Actual same-account provenance is required before any link action"); + const [sourceAccount, targetAccount] = pair as [SubscriptionAccountReport, SubscriptionAccountReport]; + const root = `/api/companies/${ctx.input.fixtures.company.id}`; + const current = await ctx.input.api.get(`${root}/costs/subscriptions`); + state.noPreHumanMutation = pair.every(before => JSON.stringify(current.accounts.find(row => row.id === before.id)) === JSON.stringify(before)); + state.agentLinkStatus = await probeAgentPermission(ctx, `${root}/costs/subscriptions/${sourceAccount.id}/link`, "POST", { targetId: targetAccount.id, expectedRevision: sourceAccount.price.revision, targetRevision: targetAccount.price.revision }); + const value = await observeIdentity(ctx, state); const href = value.links.find(link => { const url = new URL(link, ctx.input.api.baseURL); return url.origin === ctx.input.api.baseURL && url.pathname.includes("/activity/costs"); }); + if (!href) throw new Error("Actual subscription control link is missing"); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + await ctx.input.page.getByRole("button", { name: "View details", exact: true }).click(); + const dialog = ctx.input.page.getByRole("dialog"); await expect(dialog.getByRole("heading", { name: "Subscriptions", exact: true })).toBeVisible(); + for (const account of pair) { if (!account.ownerName || account.ownerUserId !== state.personId || !account.canEdit) throw new Error("Actual matching account ownership is required"); await expect(dialog.getByText(account.ownerName, { exact: false }).first()).toBeVisible(); } + state.ownershipVisible = true; + const accountRow = dialog.locator("div.space-y-2.border-b").filter({ has: dialog.getByText(sourceAccount.name, { exact: true }) }); + await accountRow.getByRole("button", { name: "Edit price", exact: true }).click(); + await expect(dialog.getByLabel("Plan", { exact: true })).toHaveValue(sourceAccount.price.plan); + await expect(dialog.getByText("Linking counts these accounts once, combines their Paperclip usage, and keeps the selected account’s price. Only link the same subscription or paid seat.", { exact: true })).toBeVisible(); state.linkDisclosureVisible = true; + await dialog.getByLabel("Same subscription as another account?", { exact: true }).selectOption(targetAccount.id); + const response = ctx.input.page.waitForResponse(result => result.request().method() === "POST" && new URL(result.url()).pathname === `${root}/costs/subscriptions/${sourceAccount.id}/link` + && result.request().postDataJSON()?.targetId === targetAccount.id); + await dialog.getByRole("button", { name: "Link accounts", exact: true }).click(); state.humanLinkStatus = (await response).status(); + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + state.afterReport = await ctx.input.api.get(`${root}/costs/subscriptions`); + }, + async observe(ctx, state) { state.afterReport = await ctx.input.api.get(`/api/companies/${ctx.input.fixtures.company.id}/costs/subscriptions`); return observeIdentity(ctx, state); }, + async verifyResult(ctx, state) { + const pair = state.duplicateAccounts as SubscriptionAccountReport[] | undefined; if (!pair) throw new Error("Actual account context is absent"); + await ctx.input.page.getByRole("button", { name: "View details", exact: true }).click(); + await expect(ctx.input.page.getByRole("dialog").getByText(pair[1]!.name, { exact: true })).toBeVisible(); + if (pair[0]!.name !== pair[1]!.name) await expect(ctx.input.page.getByRole("dialog").getByText(pair[0]!.name, { exact: true })).toHaveCount(0); + await ctx.input.evidence("identity-subscription-human-link.json", { sourceId: pair[0]!.id, targetId: pair[1]!.id, actualReloadedSurvivorVisible: true, sourceRemoved: true }); + }, +})];