From 64da56f8358ee58f8bffb99607ac9a4a20d67014 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 21:54:14 -0600 Subject: [PATCH] test(evals): use a genuinely pricier hire proposal fixture Use the public Pro configuration as the incidental starting proposal, verified against current official prices and every selected 1017 rate variant. Keep all comparison, catalog, identity, configuration and pending-approval guards unchanged. Record the starting-model change as definition v3, never a historical regrade. Co-Authored-By: Paperclip --- .../identity-hire-model-choice.test.ts | 126 ++++++++++++++++++ .../identity-hire-model-choice.ts | 2 +- .../identity-hire-revision.test.ts | 10 +- .../identity-hire-revision.ts | 2 +- 4 files changed, 133 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts index b36b86a8004..c0a29f281c3 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts @@ -70,3 +70,129 @@ describe("bounded public metadata read and replay custody", () => { await expect(readHireModelChoice(fixture().catalog, fetcher)).rejects.toThrow("could not be read"); expect(fetcher).toHaveBeenCalledTimes(1); }); }); + +// Official credential-free metadata retrieved 2026-11-03T03:51:08Z from +// https://openrouter.ai/api/v1/models?q=deepseek-v4&output_modalities=text +// Full response SHA256: c0164b41a523e3deab39b09d8387fc7a313f5cf723c8f4d70e4461fec789d45c. +// This frozen regression is not a substitute for the case's fresh metadata GET. +const officialRecords: Array<{id: string; pricing: Record}> = [ + { + "id": "deepseek/deepseek-v4-pro-0813", + "pricing": { + "prompt": "0.000000115", + "completion": "0.000004", + "input_cache_read": "0.00000011" + } + }, + { + "id": "deepseek/deepseek-v4-pro", + "pricing": { + "prompt": "0.0000016", + "completion": "0.0000032", + "input_cache_read": "0.000000135" + } + }, + { + "id": "deepseek/deepseek-v4.1-flash-1017", + "pricing": { + "prompt": "0.0000003", + "completion": "0.0000012", + "input_cache_read": "0.000000006", + "overrides": [ + { + "utc_days": [ + "saturday", + "sunday" + ], + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 0, + "utc_end": 100, + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 100, + "utc_end": 400, + "prompt": "0.0000003", + "completion": "0.0000012", + "input_cache_read": "0.000000006" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 400, + "utc_end": 600, + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 600, + "utc_end": 1000, + "prompt": "0.0000003", + "completion": "0.0000012", + "input_cache_read": "0.000000006" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 1000, + "utc_end": 0, + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + } + ] + } + } +]; +it("the actual official rate records admit the genuine expensive starting model, not the dated counterpart", () => { + const proof = proveHireModelChoice({ data: officialRecords }, officialRecords.map(row => ({ id: `openrouter/${row.id}` }))); + expect(proof.originalModel).toBe("openrouter/deepseek/deepseek-v4-pro"); + expect(proof.cheaperModel).toBe(CHAT_OPENCODE_MODEL); + expect(proof.targetMaximumRates).toEqual({ prompt: "0.0000003", completion: "0.0000012", input_cache_read: "0.000000006" }); + expect(proof.originalMinimumRates).toEqual({ prompt: "0.0000016", completion: "0.0000032", input_cache_read: "0.000000135" }); + expect(proof.targetRateVariants).toBe(7); +}); +it.each(["no longer cheaper", "unknown rate"])("the official-record comparison rejects %s instead of overriding price evidence", fault => { + const data = structuredClone(officialRecords); + if (fault === "no longer cheaper") data.find(row => row.id === "deepseek/deepseek-v4-pro")!.pricing.prompt = "0.00000001"; + else data.find(row => row.id === "deepseek/deepseek-v4.1-flash-1017")!.pricing.request = "0"; + expect(() => proveHireModelChoice({ data }, data.map(row => ({ id: `openrouter/${row.id}` })))).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts index ef427084d8d..93da3eea111 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { CHAT_OPENCODE_MODEL } from "../chat-runtime-profile.js"; -export const HIRE_ORIGINAL_MODEL = "openrouter/deepseek/deepseek-v4-pro-0813"; +export const HIRE_ORIGINAL_MODEL = "openrouter/deepseek/deepseek-v4-pro"; export const HIRE_MODEL_METADATA_URL = "https://openrouter.ai/api/v1/models?q=deepseek-v4&output_modalities=text"; const PRICE_FIELDS = ["prompt", "completion", "input_cache_read"] as const; type PriceField = typeof PRICE_FIELDS[number]; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts index 804a13012db..bdcc4b774a0 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts @@ -3,18 +3,18 @@ import { proveHireModelChoice } from "./identity-hire-model-choice.js"; import { gradeHireRevision } from "./identity-hire-revision.js"; import type { IdentityObservation } from "./identity-support.js"; function valid(): IdentityObservation { - const models = ["openrouter/deepseek/deepseek-v4-pro-0813", "openrouter/deepseek/deepseek-v4.1-flash-1017"]; + const models = ["openrouter/deepseek/deepseek-v4-pro", "openrouter/deepseek/deepseek-v4.1-flash-1017"]; const modelChoice = { ...proveHireModelChoice({ data: models.map((id, index) => ({ id: id.replace(/^openrouter\//, ""), pricing: { prompt: index ? "0.1" : "1", completion: index ? "0.2" : "2", input_cache_read: index ? "0.01" : "0.02" } })) }, models.map(id => ({ id }))), metadataSha256: "a".repeat(64), retrievedAt: "2026-10-28T00:00:00Z" }; return { scenarioId: "questions-and-approvals.formal-hire-revision", origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [{ id: "ceo" }], tasks: [], projects: [], goals: [], interactions: [], comments: [], documents: [], revisions: [], links: [], runs: [], humanQueries: [], reply: "The proposal is back for board approval.", activity: [{ id: "revision", action: "approval.revision_requested", entityId: "approval", actorType: "user" }, { id: "resubmit", action: "approval.resubmitted", entityId: "approval", actorType: "agent", agentId: "ceo" }], - state: { modelChoice, originalApproval: { id: "approval", payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4-pro-0813" } } }, currentApproval: { id: "approval", companyId: "acorn", type: "hire_agent", requestedByAgentId: "ceo", status: "pending", decidedByUserId: null, decidedAt: null, payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" } } }, - originalModel: "openrouter/deepseek/deepseek-v4-pro-0813", cheaperModel: "openrouter/deepseek/deepseek-v4.1-flash-1017", eligibleModels: ["openrouter/deepseek/deepseek-v4-pro-0813", "openrouter/deepseek/deepseek-v4.1-flash-1017"], originalOtherFieldsPreserved: true, humanId: "alex", originalHistoryCommentId: "history", beforeAgentCount: 1, pendingProposalVisible: true, revisedModelVisible: true, - approvalComments: [{ id: "history", authorUserId: "alex", body: "Original openrouter/deepseek/deepseek-v4-pro-0813, board selected openrouter/deepseek/deepseek-v4.1-flash-1017" }, { id: "revision-comment", authorAgentId: "ceo", body: "Revised to openrouter/deepseek/deepseek-v4.1-flash-1017 and resubmitted for approval." }] } }; + state: { modelChoice, originalApproval: { id: "approval", payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4-pro" } } }, currentApproval: { id: "approval", companyId: "acorn", type: "hire_agent", requestedByAgentId: "ceo", status: "pending", decidedByUserId: null, decidedAt: null, payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" } } }, + originalModel: "openrouter/deepseek/deepseek-v4-pro", cheaperModel: "openrouter/deepseek/deepseek-v4.1-flash-1017", eligibleModels: ["openrouter/deepseek/deepseek-v4-pro", "openrouter/deepseek/deepseek-v4.1-flash-1017"], originalOtherFieldsPreserved: true, humanId: "alex", originalHistoryCommentId: "history", beforeAgentCount: 1, pendingProposalVisible: true, revisedModelVisible: true, + approvalComments: [{ id: "history", authorUserId: "alex", body: "Original openrouter/deepseek/deepseek-v4-pro, board selected openrouter/deepseek/deepseek-v4.1-flash-1017" }, { id: "revision-comment", authorAgentId: "ceo", body: "Revised to openrouter/deepseek/deepseek-v4.1-flash-1017 and resubmitted for approval." }] } }; } describe("formal hire remains a proposal after cost revision", () => { it("requires actual model choice, attributed history and a still pending proposal", () => { expect(gradeHireRevision(valid()).passed).toBe(true); expect(gradeHireRevision(undefined).passed).toBe(false); }); it("rejects treating requested revision as approval or activating the hire", () => { const o = valid(); (o.state.currentApproval as { status: string }).status = "approved"; expect(gradeHireRevision(o).passed).toBe(false); const wrong = valid(); wrong.agents.push({ id: "new-hire" }); expect(gradeHireRevision(wrong).passed).toBe(false); }); - it("rejects changing only prose while retaining the expensive config", () => { const o = valid(); (o.state.currentApproval as { payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro-0813"; expect(gradeHireRevision(o).passed).toBe(false); }); + it("rejects changing only prose while retaining the expensive config", () => { const o = valid(); (o.state.currentApproval as { payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro"; expect(gradeHireRevision(o).passed).toBe(false); }); it("rejects an unsupported substituted model and missing historical record", () => { const o = valid(); o.state.eligibleModels = []; expect(gradeHireRevision(o).passed).toBe(false); const wrong = valid(); wrong.state.approvalComments = []; expect(gradeHireRevision(wrong).passed).toBe(false); }); it.each([ ["missing price proof", (o: IdentityObservation) => { delete o.state.modelChoice; }], diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts index a3b534509f2..8af04da24d5 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts @@ -52,7 +52,7 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe return { originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get(`/api/companies/${fixtures.company.id}/agents`)).length }; }, gradeHireRevision, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v2", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { const originalDeadlineAt = ctx.observationDeadlineAt; remainingIdentityObservationBudget(ctx); await waitForReply(ctx, originalDeadlineAt); const id = entity(state, "originalApproval").id; const source = ctx.input.page.url();