diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts index b36b86a8004..c0a29f281c3 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.test.ts @@ -70,3 +70,129 @@ describe("bounded public metadata read and replay custody", () => { await expect(readHireModelChoice(fixture().catalog, fetcher)).rejects.toThrow("could not be read"); expect(fetcher).toHaveBeenCalledTimes(1); }); }); + +// Official credential-free metadata retrieved 2026-11-03T03:51:08Z from +// https://openrouter.ai/api/v1/models?q=deepseek-v4&output_modalities=text +// Full response SHA256: c0164b41a523e3deab39b09d8387fc7a313f5cf723c8f4d70e4461fec789d45c. +// This frozen regression is not a substitute for the case's fresh metadata GET. +const officialRecords: Array<{id: string; pricing: Record}> = [ + { + "id": "deepseek/deepseek-v4-pro-0813", + "pricing": { + "prompt": "0.000000115", + "completion": "0.000004", + "input_cache_read": "0.00000011" + } + }, + { + "id": "deepseek/deepseek-v4-pro", + "pricing": { + "prompt": "0.0000016", + "completion": "0.0000032", + "input_cache_read": "0.000000135" + } + }, + { + "id": "deepseek/deepseek-v4.1-flash-1017", + "pricing": { + "prompt": "0.0000003", + "completion": "0.0000012", + "input_cache_read": "0.000000006", + "overrides": [ + { + "utc_days": [ + "saturday", + "sunday" + ], + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 0, + "utc_end": 100, + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 100, + "utc_end": 400, + "prompt": "0.0000003", + "completion": "0.0000012", + "input_cache_read": "0.000000006" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 400, + "utc_end": 600, + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 600, + "utc_end": 1000, + "prompt": "0.0000003", + "completion": "0.0000012", + "input_cache_read": "0.000000006" + }, + { + "utc_days": [ + "monday", + "tuesday", + "wednesday", + "thursday", + "friday" + ], + "utc_start": 1000, + "utc_end": 0, + "prompt": "0.00000015", + "completion": "0.0000006", + "input_cache_read": "0.000000003" + } + ] + } + } +]; +it("the actual official rate records admit the genuine expensive starting model, not the dated counterpart", () => { + const proof = proveHireModelChoice({ data: officialRecords }, officialRecords.map(row => ({ id: `openrouter/${row.id}` }))); + expect(proof.originalModel).toBe("openrouter/deepseek/deepseek-v4-pro"); + expect(proof.cheaperModel).toBe(CHAT_OPENCODE_MODEL); + expect(proof.targetMaximumRates).toEqual({ prompt: "0.0000003", completion: "0.0000012", input_cache_read: "0.000000006" }); + expect(proof.originalMinimumRates).toEqual({ prompt: "0.0000016", completion: "0.0000032", input_cache_read: "0.000000135" }); + expect(proof.targetRateVariants).toBe(7); +}); +it.each(["no longer cheaper", "unknown rate"])("the official-record comparison rejects %s instead of overriding price evidence", fault => { + const data = structuredClone(officialRecords); + if (fault === "no longer cheaper") data.find(row => row.id === "deepseek/deepseek-v4-pro")!.pricing.prompt = "0.00000001"; + else data.find(row => row.id === "deepseek/deepseek-v4.1-flash-1017")!.pricing.request = "0"; + expect(() => proveHireModelChoice({ data }, data.map(row => ({ id: `openrouter/${row.id}` })))).toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts index ef427084d8d..93da3eea111 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-model-choice.ts @@ -1,7 +1,7 @@ import { createHash } from "node:crypto"; import { CHAT_OPENCODE_MODEL } from "../chat-runtime-profile.js"; -export const HIRE_ORIGINAL_MODEL = "openrouter/deepseek/deepseek-v4-pro-0813"; +export const HIRE_ORIGINAL_MODEL = "openrouter/deepseek/deepseek-v4-pro"; export const HIRE_MODEL_METADATA_URL = "https://openrouter.ai/api/v1/models?q=deepseek-v4&output_modalities=text"; const PRICE_FIELDS = ["prompt", "completion", "input_cache_read"] as const; type PriceField = typeof PRICE_FIELDS[number]; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts index 804a13012db..bdcc4b774a0 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts @@ -3,18 +3,18 @@ import { proveHireModelChoice } from "./identity-hire-model-choice.js"; import { gradeHireRevision } from "./identity-hire-revision.js"; import type { IdentityObservation } from "./identity-support.js"; function valid(): IdentityObservation { - const models = ["openrouter/deepseek/deepseek-v4-pro-0813", "openrouter/deepseek/deepseek-v4.1-flash-1017"]; + const models = ["openrouter/deepseek/deepseek-v4-pro", "openrouter/deepseek/deepseek-v4.1-flash-1017"]; const modelChoice = { ...proveHireModelChoice({ data: models.map((id, index) => ({ id: id.replace(/^openrouter\//, ""), pricing: { prompt: index ? "0.1" : "1", completion: index ? "0.2" : "2", input_cache_read: index ? "0.01" : "0.02" } })) }, models.map(id => ({ id }))), metadataSha256: "a".repeat(64), retrievedAt: "2026-10-28T00:00:00Z" }; return { scenarioId: "questions-and-approvals.formal-hire-revision", origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [{ id: "ceo" }], tasks: [], projects: [], goals: [], interactions: [], comments: [], documents: [], revisions: [], links: [], runs: [], humanQueries: [], reply: "The proposal is back for board approval.", activity: [{ id: "revision", action: "approval.revision_requested", entityId: "approval", actorType: "user" }, { id: "resubmit", action: "approval.resubmitted", entityId: "approval", actorType: "agent", agentId: "ceo" }], - state: { modelChoice, originalApproval: { id: "approval", payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4-pro-0813" } } }, currentApproval: { id: "approval", companyId: "acorn", type: "hire_agent", requestedByAgentId: "ceo", status: "pending", decidedByUserId: null, decidedAt: null, payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" } } }, - originalModel: "openrouter/deepseek/deepseek-v4-pro-0813", cheaperModel: "openrouter/deepseek/deepseek-v4.1-flash-1017", eligibleModels: ["openrouter/deepseek/deepseek-v4-pro-0813", "openrouter/deepseek/deepseek-v4.1-flash-1017"], originalOtherFieldsPreserved: true, humanId: "alex", originalHistoryCommentId: "history", beforeAgentCount: 1, pendingProposalVisible: true, revisedModelVisible: true, - approvalComments: [{ id: "history", authorUserId: "alex", body: "Original openrouter/deepseek/deepseek-v4-pro-0813, board selected openrouter/deepseek/deepseek-v4.1-flash-1017" }, { id: "revision-comment", authorAgentId: "ceo", body: "Revised to openrouter/deepseek/deepseek-v4.1-flash-1017 and resubmitted for approval." }] } }; + state: { modelChoice, originalApproval: { id: "approval", payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4-pro" } } }, currentApproval: { id: "approval", companyId: "acorn", type: "hire_agent", requestedByAgentId: "ceo", status: "pending", decidedByUserId: null, decidedAt: null, payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017" } } }, + originalModel: "openrouter/deepseek/deepseek-v4-pro", cheaperModel: "openrouter/deepseek/deepseek-v4.1-flash-1017", eligibleModels: ["openrouter/deepseek/deepseek-v4-pro", "openrouter/deepseek/deepseek-v4.1-flash-1017"], originalOtherFieldsPreserved: true, humanId: "alex", originalHistoryCommentId: "history", beforeAgentCount: 1, pendingProposalVisible: true, revisedModelVisible: true, + approvalComments: [{ id: "history", authorUserId: "alex", body: "Original openrouter/deepseek/deepseek-v4-pro, board selected openrouter/deepseek/deepseek-v4.1-flash-1017" }, { id: "revision-comment", authorAgentId: "ceo", body: "Revised to openrouter/deepseek/deepseek-v4.1-flash-1017 and resubmitted for approval." }] } }; } describe("formal hire remains a proposal after cost revision", () => { it("requires actual model choice, attributed history and a still pending proposal", () => { expect(gradeHireRevision(valid()).passed).toBe(true); expect(gradeHireRevision(undefined).passed).toBe(false); }); it("rejects treating requested revision as approval or activating the hire", () => { const o = valid(); (o.state.currentApproval as { status: string }).status = "approved"; expect(gradeHireRevision(o).passed).toBe(false); const wrong = valid(); wrong.agents.push({ id: "new-hire" }); expect(gradeHireRevision(wrong).passed).toBe(false); }); - it("rejects changing only prose while retaining the expensive config", () => { const o = valid(); (o.state.currentApproval as { payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro-0813"; expect(gradeHireRevision(o).passed).toBe(false); }); + it("rejects changing only prose while retaining the expensive config", () => { const o = valid(); (o.state.currentApproval as { payload: { adapterType: "paperclip_runner", adapterConfig: { provider: "opencode", model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro"; expect(gradeHireRevision(o).passed).toBe(false); }); it("rejects an unsupported substituted model and missing historical record", () => { const o = valid(); o.state.eligibleModels = []; expect(gradeHireRevision(o).passed).toBe(false); const wrong = valid(); wrong.state.approvalComments = []; expect(gradeHireRevision(wrong).passed).toBe(false); }); it.each([ ["missing price proof", (o: IdentityObservation) => { delete o.state.modelChoice; }], diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts index a3b534509f2..8af04da24d5 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts @@ -52,7 +52,7 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe return { originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get(`/api/companies/${fixtures.company.id}/agents`)).length }; }, gradeHireRevision, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v2", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { const originalDeadlineAt = ctx.observationDeadlineAt; remainingIdentityObservationBudget(ctx); await waitForReply(ctx, originalDeadlineAt); const id = entity(state, "originalApproval").id; const source = ctx.input.page.url();