From 6fcb3ddab6cbebecd01b33439b6fb9ead2d4a7ec Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 23:32:37 -0600 Subject: [PATCH] test(evals): grade the hire revision by saved state, not reply wording The formal hire revision oracle required the chat reply to name the selected model, and its side-effect check matched approval words in the reply. The query asks for no literal. The saved proposal, the resubmission audit and the request opened from the reply's own link already prove the revision. Accept a linked reply that does not repeat the model, and still reject one that names another catalog model. Bump oracleVersion to native1017.v6. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../identity-hire-revision.test.ts | 44 ++++++++++--------- .../identity-hire-revision.ts | 20 +++++++-- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 39f3fa315bd..13ac829e17b 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4215,7 +4215,7 @@ "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v5", + "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v6", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts index 53c225e1023..1119d51f6df 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts @@ -39,8 +39,10 @@ describe("formal hire remains a proposal after cost revision", () => { o.comments[0]!.body = body; (o.state.revisionHistoryComment as { body: string }).body = body; expect(gradeHireRevision(o).passed).toBe(true); - for (const wrong of ["deepseek-v4.1-flash", "deepseek-v4.1-flash-1017:batch", "other/deepseek/deepseek-v4.1-flash-1017"]) { + for (const wrong of ["openrouter/deepseek/deepseek-v4.1-flash", "openrouter/deepseek/deepseek-v4.1-flash-1017:batch", "other/deepseek/deepseek-v4.1-flash-1017"]) { + // A reply that names another catalog model as the revision states a wrong fact. const invalid = structuredClone(o), changed = body.replace(model, wrong); + (invalid.state.eligibleModels as string[]).push(wrong); invalid.reply = changed; invalid.comments[0]!.body = changed; (invalid.state.revisionHistoryComment as { body: string }).body = changed; @@ -51,25 +53,27 @@ describe("formal hire remains a proposal after cost revision", () => { const wrongProposal = structuredClone(o); (wrongProposal.state.currentApproval as { payload: { adapterConfig: { model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4.1-flash"; expect(gradeHireRevision(wrongProposal).passed).toBe(false); - if (!model.includes("/")) { - const ambiguous = structuredClone(o); - (ambiguous.state.eligibleModels as string[]).push("other/deepseek/" + model); - expect(gradeHireRevision(ambiguous).passed).toBe(false); - } }); - it("accepts a generic linked chat message only when the owned approval note supplies model-specific history", () => { - const o = validChatRevision(), body = "Revised and resubmitted for approval. [Pending proposal](/approvals/approval)."; - o.state.approvalComments = valid().state.approvalComments; - o.reply = body; - o.comments[0]!.body = body; - (o.state.revisionHistoryComment as { body: string }).body = body; - expect(gradeHireRevision(o).passed).toBe(true); - const missing = structuredClone(o); - missing.state.approvalComments = (missing.state.approvalComments as unknown[]).slice(0, 1); - expect(gradeHireRevision(missing).passed).toBe(false); - const foreign = structuredClone(o); - (foreign.state.approvalComments as Array<{ authorAgentId?: string }>)[1]!.authorAgentId = "other"; - expect(gradeHireRevision(foreign).passed).toBe(false); + it.each([ + "The revised hire is pending your approval. Review [the pending approval](/ACORN/approvals/approval) — it must be approved before the hire can proceed.", + "Done. [Open it](/approvals/approval).", + ])("accepts a linked reply that does not repeat the model, with or without an approval note: %s", body => { + // The saved proposal and the request behind the reply's link carry the revised model; wording is not graded. + for (const withNote of [false, true]) { + const o = validChatRevision(); + if (withNote) o.state.approvalComments = valid().state.approvalComments; + o.reply = body; + o.comments[0]!.body = body; + (o.state.revisionHistoryComment as { body: string }).body = body; + expect(gradeHireRevision(o).passed).toBe(true); + const expensive = structuredClone(o); + (expensive.state.currentApproval as { payload: { adapterConfig: { model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro"; + expect(gradeHireRevision(expensive).passed).toBe(false); + const unseen = structuredClone(o); unseen.state.revisedModelVisible = false; + expect(gradeHireRevision(unseen).passed).toBe(false); + const unlinked = structuredClone(o); unlinked.renderedLinkEvidence = []; + expect(gradeHireRevision(unlinked).passed).toBe(withNote); + } }); it("accepts a bound revision reply alongside progress from the same run", () => { const o = validChatRevision(); @@ -101,7 +105,7 @@ describe("formal hire remains a proposal after cost revision", () => { ["wrong comment source", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { issueId: string }).issueId = "other"; }], ["wrong comment actor", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { authorAgentId: string }).authorAgentId = "other"; }], ["wrong originating run", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { createdByRunId: string }).createdByRunId = "other"; }], - ["wrong model in reply", (o: IdentityObservation) => { o.reply = o.reply.replace(String(o.state.cheaperModel), "other-model"); (o.state.revisionHistoryComment as { body: string }).body = o.reply; }], + ["wrong model in reply", (o: IdentityObservation) => { (o.state.eligibleModels as string[]).push("openrouter/other/other-model"); o.reply = o.reply.replace(String(o.state.cheaperModel), "openrouter/other/other-model"); o.comments[0]!.body = o.reply; (o.state.revisionHistoryComment as { body: string }).body = o.reply; }], ["uncommitted run", (o: IdentityObservation) => { o.runs[0]!.resultJson = { finalizationPhase: "pending" }; }], ["foreign run company", (o: IdentityObservation) => { o.runs[0]!.companyId = "other"; }], ["foreign run actor", (o: IdentityObservation) => { o.runs[0]!.agentId = "other"; }], diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts index 0aac8a36afa..eaa7cc55e1e 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts @@ -18,6 +18,17 @@ function namesSelectedModel(body: string, model: string, catalog: unknown): bool return id === model || uniqueShortId && id === shortId; }); } +/** Catalog models the text names, by exact id or by a short id unique in the catalog. Wording is never graded. */ +function namedCatalogModels(body: string, catalog: unknown): string[] { + if (!Array.isArray(catalog)) return []; + const ids = catalog.filter((id): id is string => typeof id === "string"); + const identifiers = (body.match(/[A-Za-z0-9][A-Za-z0-9_./:@~-]*/g) ?? []).map(value => value.replace(/[.:]+$/, "")); + return ids.filter(model => { + const shortId = model.split("/").at(-1)!; + const uniqueShortId = ids.filter(id => id.split("/").at(-1) === shortId).length === 1; + return identifiers.some(id => id === model || uniqueShortId && id === shortId); + }); +} function hasBoundRevisionHistory(o: IdentityObservation, approval: Approval | undefined): boolean { const comment = o.state.revisionHistoryComment as Entity | undefined; const link = o.state.revisionHistoryLink as RenderedConversationLink | undefined; @@ -25,7 +36,10 @@ function hasBoundRevisionHistory(o: IdentityObservation, approval: Approval | un if (!approval || !comment || !link || typeof sourceId !== "string" || o.runs.length !== 1 || o.humanQueries.length !== 1 || comment.companyId !== o.companyId || comment.issueId !== sourceId || comment.authorAgentId !== o.leadId || comment.authorUserId || typeof comment.body !== "string" || typeof o.state.cheaperModel !== "string" - || !namesSelectedModel(comment.body, o.state.cheaperModel, o.state.eligibleModels)) return false; + // The saved proposal and the request the reply links to carry the revised + // model. The reply need not repeat it, but it must not name another + // catalog model than the original or the selected one. + || namedCatalogModels(comment.body, o.state.eligibleModels).some(model => model !== o.state.cheaperModel && model !== o.state.originalModel)) return false; const run = o.runs[0]!; if (run.id !== comment.createdByRunId || run.companyId !== o.companyId || run.agentId !== o.leadId || run.runtimeMode !== "native" || run.status !== "succeeded" || run.resultJson?.finalizationPhase !== "committed" || run.nativeIssueId !== sourceId @@ -68,7 +82,7 @@ export function gradeHireRevision(value: unknown) { && o.activity.filter(row => row.action === "approval.resubmitted" && row.entityId === approval?.id && row.actorType === "agent" && row.agentId === o.leadId).length === 1, "The retained public original/board-choice comment, agent approval note or bound native revision reply, and actual revision/resubmission audit form inspectable history."); c.add("no-hire-or-approval-side-effect", o.agents.length === o.state.beforeAgentCount && !o.activity.some(row => row.action === "approval.approved" && row.entityId === approval?.id) - && o.state.pendingProposalVisible === true && o.state.revisedModelVisible === true && /approval|review|pending|board/i.test(o.reply), + && o.state.pendingProposalVisible === true && o.state.revisedModelVisible === true, "The actual proposal remains visibly pending with the revised model and no employee is instantiated or activated under the revision request."); return c.result(); } @@ -92,7 +106,7 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe return { sourceIssueId: source.id, originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get(`/api/companies/${fixtures.company.id}/agents`)).length }; }, gradeHireRevision, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v5", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v6", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining();