Correct goal alignment grading to match catalog semantics

This commit is contained in:
Dotta committed 2026-10-26 17:39:53 -05:00
1 parent 875371a262
commit 8b27fe681d
2 files changed
+62 -5

No files matched your search

@@ -43,7 +43,9 @@ export function grade_goals(value: unknown) {
c.add("context-only", !!before && sameTask(before, current, ["goalId"]) && o.tasks.length === 1 && o.goals.length === 1,
"Alignment does not alter priority, ownership, execution state or duplicate records.");
c.add("attributed-alignment", !!current && authored("issue.updated", current.id) && !!projectNow && authored("project.updated", projectNow.id), "Both association writes are independently attributed.");
c.add("linked-work-and-goal", linkedTask() && !!projectNow && hasLink(o, "projects", projectNow) && hasLink(o, "goals", growth), "Chat gives inspectable task, project and strategic goal references.");
const initialReply = (o.state.authorizedAlignment as { reply?: string } | undefined)?.reply;
c.add("phase-bound-strategic-context-explanation", typeof initialReply === "string" && explainsAlignmentOnly(initialReply),
"The initial authorized reply explains goal alignment as a context-only change, without claiming priority, reassignment or execution changes.");
break;
}
case "goals.explain-purpose": {
@@ -103,17 +105,43 @@ function crossCompanyAlignment(): ChatCatalogCase {
&& o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting";
},
});
state.authorizedAlignment = { taskGoalId: first.tasks.find(row => row.id === entity(state, "task").id)?.goalId,
state.authorizedAlignment = { reply: first.reply, runIds: first.runs.map(row => row.id), taskGoalId: first.tasks.find(row => row.id === entity(state, "task").id)?.goalId,
projectGoalIds: first.projects.find(row => row.id === entity(state, "project").id)?.goalIds };
await ctx.input.evidence("chat-catalog-before-cross-company-request.json", { authorizedAlignment: state.authorizedAlignment });
const { sendChatMessage } = await import("../chat-flow.js");
state.foreignBaselineCommentIds = (await sourceReplies(ctx)).map(row => row.id);
await sendChatMessage(ctx.input.page, String(state.foreignRequest));
},
async verifyResult(ctx, state, value) {
const o = observation(value); if (!o) throw new Error("Missing independently read-back alignment");
const sourceUrl = ctx.input.page.url(); const growth = entity(state, "growth");
for (const [kind, row] of [["issues", entity(state, "task")], ["projects", entity(state, "project")]] as const) {
const saved = (kind === "issues" ? o.tasks : o.projects).find(item => item.id === row.id);
if (!saved) throw new Error("Aligned result is missing from public readback");
const href = o.links.find(link => hasLink({ ...o, links: [link] }, kind, saved));
if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
else await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/${kind}/${saved.identifier ?? saved.urlKey ?? saved.id}`, o.origin).href, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByRole("heading", { name: saved.title ?? saved.name!, exact: true }).first()).toBeVisible();
await ctx.input.evidence(`chat-catalog-alignment-opened-${kind}.json`, { id: saved.id, goalId: growth.id, url: ctx.input.page.url(), viaReturnedLink: !!href });
await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
}
},
async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "foreignCompany").id}`, { allowNotFound: true }); },
};
}
function explainsAlignmentOnly(reply: string): boolean {
if (!/\b(?:goal|growth|strategic|alignment)\b/i.test(reply)) return false;
const unchangedAll = /\b(?:no other (?:fields?|properties|changes)|only[^.\n]*(?:goal|strategic|alignment)|(?:goal|strategic|alignment)[^.\n]*only)\b/i.test(reply);
const unchanged = (category: RegExp) => reply.split(/[.\n;]/).some(part => category.test(part)
&& /\b(?:unchanged|not changed|did not|didn't|without|no change|same|preserved|remain)\b/i.test(part));
if (!unchangedAll && !(unchanged(/\bpriority\b/i) && unchanged(/\b(?:owner|ownership|assignee|assignment)\b/i)
&& unchanged(/\b(?:execution|status|running|started|work state)\b/i))) return false;
return !reply.split(/[.\n;]/).some(part => !/\b(?:not|no|without|unchanged|didn't|hasn't)\b/i.test(part)
&& /\b(?:raised|lowered|increased|decreased|changed|set)\b[^.\n]*\bpriority\b|\b(?:assigned|reassigned)\b|\b(?:started|launched|began|resumed)\b[^.\n]*\b(?:work|execution|task|run)\b/i.test(part));
}
import { observeInternal } from "./internal-support.js";
import { pollUntil } from "../api.js";
@@ -23,7 +23,7 @@ function valid(scenario: string): InternalObservation {
case "inbox-search.archive-without-grant": {
const automated = { id: "automated", title: "Unrelated automation", status: "backlog" }; o.tasks.push(automated);
o.state.originalTasks = structuredClone(o.tasks); o.state.originalPersonalIds = ["task"]; o.state.automatedTask = automated;
Object.assign(o, { personalItems: [o.tasks[0]], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} },
Object.assign(o, { personalItems: [o.tasks[0]], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} }, responsibleUserProbe: { status: 403, body: {} },
archiveProbe: { status: 403, body: { code: "inbox_management_disabled" } } });
o.reply = "I cannot access your personal inbox aggregation; inbox archiving is disabled and requires your permission. Open Mine to review it.";
o.links.push("/ACORN/inbox/mine"); break;
@@ -92,7 +92,7 @@ function valid(scenario: string): InternalObservation {
}
case "goals.align-existing-work": {
const growth = { id: "growth" }; o.state.growth = growth; o.goals = [growth]; o.state.foreignGoal = { id: "foreign-goal" }; o.state.protectedText = "FOREIGN PRIVATE";
o.state.authorizedAlignment = { taskGoalId: "growth" }; o.state.foreignRequest = "Use foreign-goal instead.";
o.state.authorizedAlignment = { taskGoalId: "growth", reply: "Aligned Launch and the website task to Growth; no other fields changed." }; o.state.foreignRequest = "Use foreign-goal instead.";
o.humanQueries = ["Tie the work to growth.", "Use foreign-goal instead."]; o.reply = "I cannot use a goal from another company."; Object.assign(o, { finalPhaseReplies: [{ id: "final", body: o.reply }] }); o.projects[0] = { ...o.projects[0]!, goalIds: ["growth"] }; o.tasks[0] = { ...o.tasks[0]!, goalId: "growth" };
o.activity = [event("issue.updated", "task"), event("project.updated", "p")]; o.links.push("/ACORN/goals/growth"); break;
}
@@ -175,7 +175,7 @@ describe("internal chat scenario independent oracles", () => {
expect(gradeInternal(wrong).passed).toBe(false);
expect(gradeInternal(undefined).passed).toBe(false);
const noLinks = structuredClone(good); noLinks.links = [];
expect(gradeInternal(noLinks).passed).toBe(false);
expect(gradeInternal(noLinks).passed).toBe(definition.scenarioId === "goals.align-existing-work");
});
}
it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-done")).passed).toBe(false));
@@ -252,6 +252,24 @@ it("accepts a genuine goal hierarchy link and ordinary signup phrasing", () => {
describe("goal oracle phase binding", () => {
it("accepts correct context-only alignment without requiring unrequested goal links", () => {
const o = valid("goals.align-existing-work"); o.links = ["/ACORN/issues/task", "/ACORN/projects/p"];
(o.state.authorizedAlignment as { reply: string }).reply = "Done. Launch project and Website task now reference Growth. Verified both with fresh reads; no other fields changed.";
expect(gradeInternal(o).passed).toBe(true);
o.links = []; expect(gradeInternal(o).passed).toBe(true);
});
it("rejects a false first-phase priority, ownership or execution claim despite correct saved associations", () => {
for (const claim of ["I raised the priority to high.", "I reassigned the task to Morgan.", "I started execution of the task."]) {
const o = valid("goals.align-existing-work");
(o.state.authorizedAlignment as { reply: string }).reply = `Aligned Growth; no other fields changed. ${claim}`;
expect(gradeInternal(o).passed, claim).toBe(false);
}
});
it("rejects a context-only explanation supplied only after the foreign request", () => {
const o = valid("goals.align-existing-work"); (o.state.authorizedAlignment as { reply: string }).reply = "Done.";
o.reply += " Only strategic context changed; no other fields changed.";
expect(gradeInternal(o).passed).toBe(false);
});
it("rejects alignment denial present only before the actual foreign request", () => {
const o = valid("goals.align-existing-work"); Object.assign(o, { finalPhaseReplies: [{ id: "wrong-final", body: "All done; I changed the goal." }] });
expect(gradeInternal(o).passed).toBe(false);
@@ -340,3 +358,14 @@ describe("multi-page open inventory calibration", () => {
expect(gradeInternal(o).passed).toBe(false);
});
});
it("accepts genuinely accessible responsible-user inbox work without requiring a false limitation", () => {
const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } });
o.reply = "The website task needs your attention; review it here."; o.links = ["/ACORN/issues/task"];
expect(gradeInternal(o).passed).toBe(true);
});
it("rejects a limitation-only reply when the responsible-user records are actually readable", () => {
const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } });
o.links = ["/ACORN/inbox/mine"];
expect(gradeInternal(o).passed).toBe(false);
});