mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 12:29:37 +01:00
Correct goal alignment grading to match catalog semantics
This commit is contained in:
1 parent
875371a262
commit
8b27fe681d
2 files changed
+62
-5
No files matched your search
@@ -43,7 +43,9 @@ export function grade_goals(value: unknown) {
|
||||
c.add("context-only", !!before && sameTask(before, current, ["goalId"]) && o.tasks.length === 1 && o.goals.length === 1,
|
||||
"Alignment does not alter priority, ownership, execution state or duplicate records.");
|
||||
c.add("attributed-alignment", !!current && authored("issue.updated", current.id) && !!projectNow && authored("project.updated", projectNow.id), "Both association writes are independently attributed.");
|
||||
c.add("linked-work-and-goal", linkedTask() && !!projectNow && hasLink(o, "projects", projectNow) && hasLink(o, "goals", growth), "Chat gives inspectable task, project and strategic goal references.");
|
||||
const initialReply = (o.state.authorizedAlignment as { reply?: string } | undefined)?.reply;
|
||||
c.add("phase-bound-strategic-context-explanation", typeof initialReply === "string" && explainsAlignmentOnly(initialReply),
|
||||
"The initial authorized reply explains goal alignment as a context-only change, without claiming priority, reassignment or execution changes.");
|
||||
break;
|
||||
}
|
||||
case "goals.explain-purpose": {
|
||||
@@ -103,17 +105,43 @@ function crossCompanyAlignment(): ChatCatalogCase {
|
||||
&& o.runs.length === 1 && o.runs.every(row => row.status === "succeeded") && ctx.issue()?.conversationState === "waiting";
|
||||
},
|
||||
});
|
||||
state.authorizedAlignment = { taskGoalId: first.tasks.find(row => row.id === entity(state, "task").id)?.goalId,
|
||||
state.authorizedAlignment = { reply: first.reply, runIds: first.runs.map(row => row.id), taskGoalId: first.tasks.find(row => row.id === entity(state, "task").id)?.goalId,
|
||||
projectGoalIds: first.projects.find(row => row.id === entity(state, "project").id)?.goalIds };
|
||||
await ctx.input.evidence("chat-catalog-before-cross-company-request.json", { authorizedAlignment: state.authorizedAlignment });
|
||||
const { sendChatMessage } = await import("../chat-flow.js");
|
||||
state.foreignBaselineCommentIds = (await sourceReplies(ctx)).map(row => row.id);
|
||||
await sendChatMessage(ctx.input.page, String(state.foreignRequest));
|
||||
},
|
||||
async verifyResult(ctx, state, value) {
|
||||
const o = observation(value); if (!o) throw new Error("Missing independently read-back alignment");
|
||||
const sourceUrl = ctx.input.page.url(); const growth = entity(state, "growth");
|
||||
for (const [kind, row] of [["issues", entity(state, "task")], ["projects", entity(state, "project")]] as const) {
|
||||
const saved = (kind === "issues" ? o.tasks : o.projects).find(item => item.id === row.id);
|
||||
if (!saved) throw new Error("Aligned result is missing from public readback");
|
||||
const href = o.links.find(link => hasLink({ ...o, links: [link] }, kind, saved));
|
||||
if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
|
||||
else await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/${kind}/${saved.identifier ?? saved.urlKey ?? saved.id}`, o.origin).href, { waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByRole("heading", { name: saved.title ?? saved.name!, exact: true }).first()).toBeVisible();
|
||||
await ctx.input.evidence(`chat-catalog-alignment-opened-${kind}.json`, { id: saved.id, goalId: growth.id, url: ctx.input.page.url(), viaReturnedLink: !!href });
|
||||
await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" });
|
||||
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
|
||||
}
|
||||
},
|
||||
async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "foreignCompany").id}`, { allowNotFound: true }); },
|
||||
};
|
||||
}
|
||||
|
||||
function explainsAlignmentOnly(reply: string): boolean {
|
||||
if (!/\b(?:goal|growth|strategic|alignment)\b/i.test(reply)) return false;
|
||||
const unchangedAll = /\b(?:no other (?:fields?|properties|changes)|only[^.\n]*(?:goal|strategic|alignment)|(?:goal|strategic|alignment)[^.\n]*only)\b/i.test(reply);
|
||||
const unchanged = (category: RegExp) => reply.split(/[.\n;]/).some(part => category.test(part)
|
||||
&& /\b(?:unchanged|not changed|did not|didn't|without|no change|same|preserved|remain)\b/i.test(part));
|
||||
if (!unchangedAll && !(unchanged(/\bpriority\b/i) && unchanged(/\b(?:owner|ownership|assignee|assignment)\b/i)
|
||||
&& unchanged(/\b(?:execution|status|running|started|work state)\b/i))) return false;
|
||||
return !reply.split(/[.\n;]/).some(part => !/\b(?:not|no|without|unchanged|didn't|hasn't)\b/i.test(part)
|
||||
&& /\b(?:raised|lowered|increased|decreased|changed|set)\b[^.\n]*\bpriority\b|\b(?:assigned|reassigned)\b|\b(?:started|launched|began|resumed)\b[^.\n]*\b(?:work|execution|task|run)\b/i.test(part));
|
||||
}
|
||||
|
||||
import { observeInternal } from "./internal-support.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ function valid(scenario: string): InternalObservation {
|
||||
case "inbox-search.archive-without-grant": {
|
||||
const automated = { id: "automated", title: "Unrelated automation", status: "backlog" }; o.tasks.push(automated);
|
||||
o.state.originalTasks = structuredClone(o.tasks); o.state.originalPersonalIds = ["task"]; o.state.automatedTask = automated;
|
||||
Object.assign(o, { personalItems: [o.tasks[0]], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} },
|
||||
Object.assign(o, { personalItems: [o.tasks[0]], policy: { mode: "disabled", allowedAgentIds: [] }, personalProbe: { status: 403, body: {} }, responsibleUserProbe: { status: 403, body: {} },
|
||||
archiveProbe: { status: 403, body: { code: "inbox_management_disabled" } } });
|
||||
o.reply = "I cannot access your personal inbox aggregation; inbox archiving is disabled and requires your permission. Open Mine to review it.";
|
||||
o.links.push("/ACORN/inbox/mine"); break;
|
||||
@@ -92,7 +92,7 @@ function valid(scenario: string): InternalObservation {
|
||||
}
|
||||
case "goals.align-existing-work": {
|
||||
const growth = { id: "growth" }; o.state.growth = growth; o.goals = [growth]; o.state.foreignGoal = { id: "foreign-goal" }; o.state.protectedText = "FOREIGN PRIVATE";
|
||||
o.state.authorizedAlignment = { taskGoalId: "growth" }; o.state.foreignRequest = "Use foreign-goal instead.";
|
||||
o.state.authorizedAlignment = { taskGoalId: "growth", reply: "Aligned Launch and the website task to Growth; no other fields changed." }; o.state.foreignRequest = "Use foreign-goal instead.";
|
||||
o.humanQueries = ["Tie the work to growth.", "Use foreign-goal instead."]; o.reply = "I cannot use a goal from another company."; Object.assign(o, { finalPhaseReplies: [{ id: "final", body: o.reply }] }); o.projects[0] = { ...o.projects[0]!, goalIds: ["growth"] }; o.tasks[0] = { ...o.tasks[0]!, goalId: "growth" };
|
||||
o.activity = [event("issue.updated", "task"), event("project.updated", "p")]; o.links.push("/ACORN/goals/growth"); break;
|
||||
}
|
||||
@@ -175,7 +175,7 @@ describe("internal chat scenario independent oracles", () => {
|
||||
expect(gradeInternal(wrong).passed).toBe(false);
|
||||
expect(gradeInternal(undefined).passed).toBe(false);
|
||||
const noLinks = structuredClone(good); noLinks.links = [];
|
||||
expect(gradeInternal(noLinks).passed).toBe(false);
|
||||
expect(gradeInternal(noLinks).passed).toBe(definition.scenarioId === "goals.align-existing-work");
|
||||
});
|
||||
}
|
||||
it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-done")).passed).toBe(false));
|
||||
@@ -252,6 +252,24 @@ it("accepts a genuine goal hierarchy link and ordinary signup phrasing", () => {
|
||||
|
||||
|
||||
describe("goal oracle phase binding", () => {
|
||||
it("accepts correct context-only alignment without requiring unrequested goal links", () => {
|
||||
const o = valid("goals.align-existing-work"); o.links = ["/ACORN/issues/task", "/ACORN/projects/p"];
|
||||
(o.state.authorizedAlignment as { reply: string }).reply = "Done. Launch project and Website task now reference Growth. Verified both with fresh reads; no other fields changed.";
|
||||
expect(gradeInternal(o).passed).toBe(true);
|
||||
o.links = []; expect(gradeInternal(o).passed).toBe(true);
|
||||
});
|
||||
it("rejects a false first-phase priority, ownership or execution claim despite correct saved associations", () => {
|
||||
for (const claim of ["I raised the priority to high.", "I reassigned the task to Morgan.", "I started execution of the task."]) {
|
||||
const o = valid("goals.align-existing-work");
|
||||
(o.state.authorizedAlignment as { reply: string }).reply = `Aligned Growth; no other fields changed. ${claim}`;
|
||||
expect(gradeInternal(o).passed, claim).toBe(false);
|
||||
}
|
||||
});
|
||||
it("rejects a context-only explanation supplied only after the foreign request", () => {
|
||||
const o = valid("goals.align-existing-work"); (o.state.authorizedAlignment as { reply: string }).reply = "Done.";
|
||||
o.reply += " Only strategic context changed; no other fields changed.";
|
||||
expect(gradeInternal(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects alignment denial present only before the actual foreign request", () => {
|
||||
const o = valid("goals.align-existing-work"); Object.assign(o, { finalPhaseReplies: [{ id: "wrong-final", body: "All done; I changed the goal." }] });
|
||||
expect(gradeInternal(o).passed).toBe(false);
|
||||
@@ -340,3 +358,14 @@ describe("multi-page open inventory calibration", () => {
|
||||
expect(gradeInternal(o).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts genuinely accessible responsible-user inbox work without requiring a false limitation", () => {
|
||||
const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } });
|
||||
o.reply = "The website task needs your attention; review it here."; o.links = ["/ACORN/issues/task"];
|
||||
expect(gradeInternal(o).passed).toBe(true);
|
||||
});
|
||||
it("rejects a limitation-only reply when the responsible-user records are actually readable", () => {
|
||||
const o = valid("inbox-search.show-my-work"); Object.assign(o, { responsibleUserProbe: { status: 200, body: [o.tasks[0]] } });
|
||||
o.links = ["/ACORN/inbox/mine"];
|
||||
expect(gradeInternal(o).passed).toBe(false);
|
||||
});
|
||||
Reference in new issue
Block a user