diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 39f3fa315bd..13ac829e17b 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4215,7 +4215,7 @@ "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v5", + "oracleVersion": "paperclip.chat-catalog.hire-revision.native1017.v6", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts index 53c225e1023..1119d51f6df 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.test.ts @@ -39,8 +39,10 @@ describe("formal hire remains a proposal after cost revision", () => { o.comments[0]!.body = body; (o.state.revisionHistoryComment as { body: string }).body = body; expect(gradeHireRevision(o).passed).toBe(true); - for (const wrong of ["deepseek-v4.1-flash", "deepseek-v4.1-flash-1017:batch", "other/deepseek/deepseek-v4.1-flash-1017"]) { + for (const wrong of ["openrouter/deepseek/deepseek-v4.1-flash", "openrouter/deepseek/deepseek-v4.1-flash-1017:batch", "other/deepseek/deepseek-v4.1-flash-1017"]) { + // A reply that names another catalog model as the revision states a wrong fact. const invalid = structuredClone(o), changed = body.replace(model, wrong); + (invalid.state.eligibleModels as string[]).push(wrong); invalid.reply = changed; invalid.comments[0]!.body = changed; (invalid.state.revisionHistoryComment as { body: string }).body = changed; @@ -51,25 +53,27 @@ describe("formal hire remains a proposal after cost revision", () => { const wrongProposal = structuredClone(o); (wrongProposal.state.currentApproval as { payload: { adapterConfig: { model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4.1-flash"; expect(gradeHireRevision(wrongProposal).passed).toBe(false); - if (!model.includes("/")) { - const ambiguous = structuredClone(o); - (ambiguous.state.eligibleModels as string[]).push("other/deepseek/" + model); - expect(gradeHireRevision(ambiguous).passed).toBe(false); - } }); - it("accepts a generic linked chat message only when the owned approval note supplies model-specific history", () => { - const o = validChatRevision(), body = "Revised and resubmitted for approval. [Pending proposal](/approvals/approval)."; - o.state.approvalComments = valid().state.approvalComments; - o.reply = body; - o.comments[0]!.body = body; - (o.state.revisionHistoryComment as { body: string }).body = body; - expect(gradeHireRevision(o).passed).toBe(true); - const missing = structuredClone(o); - missing.state.approvalComments = (missing.state.approvalComments as unknown[]).slice(0, 1); - expect(gradeHireRevision(missing).passed).toBe(false); - const foreign = structuredClone(o); - (foreign.state.approvalComments as Array<{ authorAgentId?: string }>)[1]!.authorAgentId = "other"; - expect(gradeHireRevision(foreign).passed).toBe(false); + it.each([ + "The revised hire is pending your approval. Review [the pending approval](/ACORN/approvals/approval) — it must be approved before the hire can proceed.", + "Done. [Open it](/approvals/approval).", + ])("accepts a linked reply that does not repeat the model, with or without an approval note: %s", body => { + // The saved proposal and the request behind the reply's link carry the revised model; wording is not graded. + for (const withNote of [false, true]) { + const o = validChatRevision(); + if (withNote) o.state.approvalComments = valid().state.approvalComments; + o.reply = body; + o.comments[0]!.body = body; + (o.state.revisionHistoryComment as { body: string }).body = body; + expect(gradeHireRevision(o).passed).toBe(true); + const expensive = structuredClone(o); + (expensive.state.currentApproval as { payload: { adapterConfig: { model: string } } }).payload.adapterConfig.model = "openrouter/deepseek/deepseek-v4-pro"; + expect(gradeHireRevision(expensive).passed).toBe(false); + const unseen = structuredClone(o); unseen.state.revisedModelVisible = false; + expect(gradeHireRevision(unseen).passed).toBe(false); + const unlinked = structuredClone(o); unlinked.renderedLinkEvidence = []; + expect(gradeHireRevision(unlinked).passed).toBe(withNote); + } }); it("accepts a bound revision reply alongside progress from the same run", () => { const o = validChatRevision(); @@ -101,7 +105,7 @@ describe("formal hire remains a proposal after cost revision", () => { ["wrong comment source", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { issueId: string }).issueId = "other"; }], ["wrong comment actor", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { authorAgentId: string }).authorAgentId = "other"; }], ["wrong originating run", (o: IdentityObservation) => { (o.state.revisionHistoryComment as { createdByRunId: string }).createdByRunId = "other"; }], - ["wrong model in reply", (o: IdentityObservation) => { o.reply = o.reply.replace(String(o.state.cheaperModel), "other-model"); (o.state.revisionHistoryComment as { body: string }).body = o.reply; }], + ["wrong model in reply", (o: IdentityObservation) => { (o.state.eligibleModels as string[]).push("openrouter/other/other-model"); o.reply = o.reply.replace(String(o.state.cheaperModel), "openrouter/other/other-model"); o.comments[0]!.body = o.reply; (o.state.revisionHistoryComment as { body: string }).body = o.reply; }], ["uncommitted run", (o: IdentityObservation) => { o.runs[0]!.resultJson = { finalizationPhase: "pending" }; }], ["foreign run company", (o: IdentityObservation) => { o.runs[0]!.companyId = "other"; }], ["foreign run actor", (o: IdentityObservation) => { o.runs[0]!.agentId = "other"; }], diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts index 0aac8a36afa..eaa7cc55e1e 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts @@ -18,6 +18,17 @@ function namesSelectedModel(body: string, model: string, catalog: unknown): bool return id === model || uniqueShortId && id === shortId; }); } +/** Catalog models the text names, by exact id or by a short id unique in the catalog. Wording is never graded. */ +function namedCatalogModels(body: string, catalog: unknown): string[] { + if (!Array.isArray(catalog)) return []; + const ids = catalog.filter((id): id is string => typeof id === "string"); + const identifiers = (body.match(/[A-Za-z0-9][A-Za-z0-9_./:@~-]*/g) ?? []).map(value => value.replace(/[.:]+$/, "")); + return ids.filter(model => { + const shortId = model.split("/").at(-1)!; + const uniqueShortId = ids.filter(id => id.split("/").at(-1) === shortId).length === 1; + return identifiers.some(id => id === model || uniqueShortId && id === shortId); + }); +} function hasBoundRevisionHistory(o: IdentityObservation, approval: Approval | undefined): boolean { const comment = o.state.revisionHistoryComment as Entity | undefined; const link = o.state.revisionHistoryLink as RenderedConversationLink | undefined; @@ -25,7 +36,10 @@ function hasBoundRevisionHistory(o: IdentityObservation, approval: Approval | un if (!approval || !comment || !link || typeof sourceId !== "string" || o.runs.length !== 1 || o.humanQueries.length !== 1 || comment.companyId !== o.companyId || comment.issueId !== sourceId || comment.authorAgentId !== o.leadId || comment.authorUserId || typeof comment.body !== "string" || typeof o.state.cheaperModel !== "string" - || !namesSelectedModel(comment.body, o.state.cheaperModel, o.state.eligibleModels)) return false; + // The saved proposal and the request the reply links to carry the revised + // model. The reply need not repeat it, but it must not name another + // catalog model than the original or the selected one. + || namedCatalogModels(comment.body, o.state.eligibleModels).some(model => model !== o.state.cheaperModel && model !== o.state.originalModel)) return false; const run = o.runs[0]!; if (run.id !== comment.createdByRunId || run.companyId !== o.companyId || run.agentId !== o.leadId || run.runtimeMode !== "native" || run.status !== "succeeded" || run.resultJson?.finalizationPhase !== "committed" || run.nativeIssueId !== sourceId @@ -68,7 +82,7 @@ export function gradeHireRevision(value: unknown) { && o.activity.filter(row => row.action === "approval.resubmitted" && row.entityId === approval?.id && row.actorType === "agent" && row.agentId === o.leadId).length === 1, "The retained public original/board-choice comment, agent approval note or bound native revision reply, and actual revision/resubmission audit form inspectable history."); c.add("no-hire-or-approval-side-effect", o.agents.length === o.state.beforeAgentCount && !o.activity.some(row => row.action === "approval.approved" && row.entityId === approval?.id) - && o.state.pendingProposalVisible === true && o.state.revisedModelVisible === true && /approval|review|pending|board/i.test(o.reply), + && o.state.pendingProposalVisible === true && o.state.revisedModelVisible === true, "The actual proposal remains visibly pending with the revised model and no employee is instantiated or activated under the revision request."); return c.result(); } @@ -92,7 +106,7 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe return { sourceIssueId: source.id, originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get(`/api/companies/${fixtures.company.id}/agents`)).length }; }, gradeHireRevision, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v5", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v6", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { const remaining = () => remainingIdentityObservationBudget(ctx); remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining();