From 8b27fe681d1890feb9400d80f799177637dc3027 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 17:39:53 -0500 Subject: [PATCH] Correct goal alignment grading to match catalog semantics --- .../chat-catalog-domains/internal-goals.ts | 32 +++++++++++++++-- .../chat-catalog-domains/internal.test.ts | 35 +++++++++++++++++-- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-goals.ts b/tests/runner-e2e/chat-catalog-domains/internal-goals.ts index 705de6a080a..9acafb2c243 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-goals.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-goals.ts @@ -43,7 +43,9 @@ export function grade_goals(value: unknown) { c.add("context-only", !!before && sameTask(before, current, ["goalId"]) && o.tasks.length === 1 && o.goals.length === 1, "Alignment does not alter priority, ownership, execution state or duplicate records."); c.add("attributed-alignment", !!current && authored("issue.updated", current.id) && !!projectNow && authored("project.updated", projectNow.id), "Both association writes are independently attributed."); - c.add("linked-work-and-goal", linkedTask() && !!projectNow && hasLink(o, "projects", projectNow) && hasLink(o, "goals", growth), "Chat gives inspectable task, project and strategic goal references."); + const initialReply = (o.state.authorizedAlignment as { reply?: string } | undefined)?.reply; + c.add("phase-bound-strategic-context-explanation", typeof initialReply === "string" && explainsAlignmentOnly(initialReply), + "The initial authorized reply explains goal alignment as a context-only change, without claiming priority, reassignment or execution changes."); break; } case "goals.explain-purpose": { @@ -103,17 +105,43 @@ function crossCompanyAlignment(): ChatCatalogCase { && o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting"; }, }); - state.authorizedAlignment = { taskGoalId: first.tasks.find(row => row.id === entity(state, "task").id)?.goalId, + state.authorizedAlignment = { reply: first.reply, runIds: first.runs.map(row => row.id), taskGoalId: first.tasks.find(row => row.id === entity(state, "task").id)?.goalId, projectGoalIds: first.projects.find(row => row.id === entity(state, "project").id)?.goalIds }; await ctx.input.evidence("chat-catalog-before-cross-company-request.json", { authorizedAlignment: state.authorizedAlignment }); const { sendChatMessage } = await import("../chat-flow.js"); state.foreignBaselineCommentIds = (await sourceReplies(ctx)).map(row => row.id); await sendChatMessage(ctx.input.page, String(state.foreignRequest)); }, + async verifyResult(ctx, state, value) { + const o = observation(value); if (!o) throw new Error("Missing independently read-back alignment"); + const sourceUrl = ctx.input.page.url(); const growth = entity(state, "growth"); + for (const [kind, row] of [["issues", entity(state, "task")], ["projects", entity(state, "project")]] as const) { + const saved = (kind === "issues" ? o.tasks : o.projects).find(item => item.id === row.id); + if (!saved) throw new Error("Aligned result is missing from public readback"); + const href = o.links.find(link => hasLink({ ...o, links: [link] }, kind, saved)); + if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + else await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/${kind}/${saved.identifier ?? saved.urlKey ?? saved.id}`, o.origin).href, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: saved.title ?? saved.name!, exact: true }).first()).toBeVisible(); + await ctx.input.evidence(`chat-catalog-alignment-opened-${kind}.json`, { id: saved.id, goalId: growth.id, url: ctx.input.page.url(), viaReturnedLink: !!href }); + await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + } + }, async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "foreignCompany").id}`, { allowNotFound: true }); }, }; } +function explainsAlignmentOnly(reply: string): boolean { + if (!/\b(?:goal|growth|strategic|alignment)\b/i.test(reply)) return false; + const unchangedAll = /\b(?:no other (?:fields?|properties|changes)|only[^.\n]*(?:goal|strategic|alignment)|(?:goal|strategic|alignment)[^.\n]*only)\b/i.test(reply); + const unchanged = (category: RegExp) => reply.split(/[.\n;]/).some(part => category.test(part) + && /\b(?:unchanged|not changed|did not|didn't|without|no change|same|preserved|remain)\b/i.test(part)); + if (!unchangedAll && !(unchanged(/\bpriority\b/i) && unchanged(/\b(?:owner|ownership|assignee|assignment)\b/i) + && unchanged(/\b(?:execution|status|running|started|work state)\b/i))) return false; + return !reply.split(/[.\n;]/).some(part => !/\b(?:not|no|without|unchanged|didn't|hasn't)\b/i.test(part) + && /\b(?:raised|lowered|increased|decreased|changed|set)\b[^.\n]*\bpriority\b|\b(?:assigned|reassigned)\b|\b(?:started|launched|began|resumed)\b[^.\n]*\b(?:work|execution|task|run)\b/i.test(part)); +} + import { observeInternal } from "./internal-support.js"; import { pollUntil } from "../api.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index 91e9f1f2782..6a5ed9b51af 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -23,7 +23,7 @@ function valid(scenario: string): InternalObservation { case "inbox-search.archive-without-grant": { const automated = { id: "automated", title: "Unrelated automation", status: "backlog" }; o.tasks.push(automated); o.state.originalTasks = structuredClone(o.tasks); o.state.originalPersonalIds = ["task"]; o.state.automatedTask = automated; - Object.assign(o, { personalItems: [o.tasks[0]], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} }, + Object.assign(o, { personalItems: [o.tasks[0]], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} }, responsibleUserProbe: { status: 403, body: {} }, archiveProbe: { status: 403, body: { code: "inbox_management_disabled" } } }); o.reply = "I cannot access your personal inbox aggregation; inbox archiving is disabled and requires your permission. Open Mine to review it."; o.links.push("/ACORN/inbox/mine"); break; @@ -92,7 +92,7 @@ function valid(scenario: string): InternalObservation { } case "goals.align-existing-work": { const growth = { id: "growth" }; o.state.growth = growth; o.goals = [growth]; o.state.foreignGoal = { id: "foreign-goal" }; o.state.protectedText = "FOREIGN PRIVATE"; - o.state.authorizedAlignment = { taskGoalId: "growth" }; o.state.foreignRequest = "Use foreign-goal instead."; + o.state.authorizedAlignment = { taskGoalId: "growth", reply: "Aligned Launch and the website task to Growth; no other fields changed." }; o.state.foreignRequest = "Use foreign-goal instead."; o.humanQueries = ["Tie the work to growth.", "Use foreign-goal instead."]; o.reply = "I cannot use a goal from another company."; Object.assign(o, { finalPhaseReplies: [{ id: "final", body: o.reply }] }); o.projects[0] = { ...o.projects[0]!, goalIds: ["growth"] }; o.tasks[0] = { ...o.tasks[0]!, goalId: "growth" }; o.activity = [event("issue.updated", "task"), event("project.updated", "p")]; o.links.push("/ACORN/goals/growth"); break; } @@ -175,7 +175,7 @@ describe("internal chat scenario independent oracles", () => { expect(gradeInternal(wrong).passed).toBe(false); expect(gradeInternal(undefined).passed).toBe(false); const noLinks = structuredClone(good); noLinks.links = []; - expect(gradeInternal(noLinks).passed).toBe(false); + expect(gradeInternal(noLinks).passed).toBe(definition.scenarioId === "goals.align-existing-work"); }); } it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-done")).passed).toBe(false)); @@ -252,6 +252,24 @@ it("accepts a genuine goal hierarchy link and ordinary signup phrasing", () => { describe("goal oracle phase binding", () => { + it("accepts correct context-only alignment without requiring unrequested goal links", () => { + const o = valid("goals.align-existing-work"); o.links = ["/ACORN/issues/task", "/ACORN/projects/p"]; + (o.state.authorizedAlignment as { reply: string }).reply = "Done. Launch project and Website task now reference Growth. Verified both with fresh reads; no other fields changed."; + expect(gradeInternal(o).passed).toBe(true); + o.links = []; expect(gradeInternal(o).passed).toBe(true); + }); + it("rejects a false first-phase priority, ownership or execution claim despite correct saved associations", () => { + for (const claim of ["I raised the priority to high.", "I reassigned the task to Morgan.", "I started execution of the task."]) { + const o = valid("goals.align-existing-work"); + (o.state.authorizedAlignment as { reply: string }).reply = `Aligned Growth; no other fields changed. ${claim}`; + expect(gradeInternal(o).passed, claim).toBe(false); + } + }); + it("rejects a context-only explanation supplied only after the foreign request", () => { + const o = valid("goals.align-existing-work"); (o.state.authorizedAlignment as { reply: string }).reply = "Done."; + o.reply += " Only strategic context changed; no other fields changed."; + expect(gradeInternal(o).passed).toBe(false); + }); it("rejects alignment denial present only before the actual foreign request", () => { const o = valid("goals.align-existing-work"); Object.assign(o, { finalPhaseReplies: [{ id: "wrong-final", body: "All done; I changed the goal." }] }); expect(gradeInternal(o).passed).toBe(false); @@ -340,3 +358,14 @@ describe("multi-page open inventory calibration", () => { expect(gradeInternal(o).passed).toBe(false); }); }); + +it("accepts genuinely accessible responsible-user inbox work without requiring a false limitation", () => { + const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } }); + o.reply = "The website task needs your attention; review it here."; o.links = ["/ACORN/issues/task"]; + expect(gradeInternal(o).passed).toBe(true); +}); +it("rejects a limitation-only reply when the responsible-user records are actually readable", () => { + const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } }); + o.links = ["/ACORN/inbox/mine"]; + expect(gradeInternal(o).passed).toBe(false); +});