Add owner-driven project sharing eval and strengthen evidence controls

This commit is contained in:
Dotta committed 2026-10-26 17:25:28 -05:00
1 parent e652b0de6e
commit 3a713fabbe
4 files changed
+161 -7

No files matched your search

@@ -34,7 +34,7 @@ export function grade_activity(value: unknown) {
const priorityEvents = (o.state.priorityEvents as Entity[] | undefined) ?? [];
c.add("real-two-change-history", priorityEvents.length === 2 && priorityEvents.every(event => o.activity.some(row => row.id === event.id)), "Both fixture priority changes remain visible on the exact task.");
c.add("attribution-and-values", priorityEvents.length === 2 && priorityEvents.every(event => !!event.actorId
&& (o.reply.includes(event.actorId) || event.actorId === "local-board" && /\b(?:local board|board)\b/i.test(o.reply))
&& mentionsRecordedActor(o.reply, event.actorId)
&& o.reply.toLowerCase().includes(String(event.details?.priority).toLowerCase())),
"Editor identity is the recorded actor ID or the canonical Local board label, never a generic operator inference.");
c.add("recorded-event-times", priorityEvents.length === 2 && priorityEvents.every(event => mentionsRecordedTime(o.reply, event.createdAt,
@@ -78,11 +78,17 @@ export function mentionsRecordedTime(reply: string, recordedAt: string | undefin
const allTimes = allRecordedAt.length ? allRecordedAt.flatMap(value => zones.map(timeZone => new Intl.DateTimeFormat("en-GB", {
timeZone, hour: "2-digit", minute: "2-digit", hourCycle: "h23",
}).format(new Date(value ?? "")))) : times;
if (!explicitTimes.some(time => times.includes(time)) || explicitTimes.some(time => !allTimes.includes(time))) return false;
if (explicitTimes.length && (!explicitTimes.some(time => times.includes(time)) || explicitTimes.some(time => !allTimes.includes(time)))) return false;
const namedDates = [...reply.matchAll(/\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2})(?:st|nd|rd|th)?\b/gi)]
.map(match => `${match[1]!.toLowerCase()} ${Number(match[2])}`);
const named = (value: string | undefined) => zones.map(timeZone => new Intl.DateTimeFormat("en-US", { timeZone, month: "long", day: "numeric" }).format(new Date(value ?? "")).toLowerCase());
if (namedDates.length && (!namedDates.some(day => named(recordedAt).includes(day))
|| namedDates.some(day => !(allRecordedAt.length ? allRecordedAt.flatMap(named) : named(recordedAt)).includes(day)))) return false;
if (explicitDates.length) return explicitDates.some(day => dates.includes(day)) && explicitDates.every(day => allDates.includes(day));
if (/\btoday\b/i.test(reply)) return zones.some((timeZone, index) => dates[index] === new Intl.DateTimeFormat("en-CA", {
timeZone, year: "numeric", month: "2-digit", day: "2-digit",
}).format(observed));
if (explicitTimes.length) return true;
return zones.some(timeZone => {
const month = new Intl.DateTimeFormat("en-US", { timeZone, month: "long" }).format(date);
const day = new Intl.DateTimeFormat("en-US", { timeZone, day: "numeric" }).format(date);
@@ -109,3 +115,14 @@ function companyOverview(): ChatCatalogCase {
}, async cleanup(ctx, state) { await ctx.input.api.delete(`/api/companies/${entity(state, "hiddenCompany").id}`, { allowNotFound: true }); } };
}
import { observeInternal } from "./internal-support.js";
function mentionsRecordedActor(reply: string, actorId: string): boolean {
const identities = actorId === "local-board" ? [actorId, "Local board", "Board"] : [actorId];
return identities.some(identity => {
const name = identity.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
return new RegExp(`\\b${name}\\s+(?:changed|updated|set|made|raised|lowered|edited)\\b`, "i").test(reply)
|| new RegExp(`\\b(?:by|actor|editor)\\s*[:=]?\\s*(?:the\\s+)?${name}\\b`, "i").test(reply)
|| new RegExp(`\\|\\s*${name}\\s*\\|`, "i").test(reply)
|| new RegExp(`(?:^|\\n)\\s*(?:#{1,6}\\s*|\\*\\*)?${name}(?:\\*\\*)?\\s*:`, "i").test(reply);
});
}
@@ -1,3 +1,4 @@
import { projectMarkdownToText, resolveProjectionRange, createDocumentAnchorSelector } from "../../../packages/shared/src/document-anchors.js";
import type { ChatCatalogCase } from "../chat-catalog-contract.js";
import { project, task, goal, putDocument, entity, observation, checks, hasLink, sameTask, type Entity } from "./internal-support.js";
import { makeDefinition, retryDefinition, fixtureWorker } from "./internal-definition.js";
@@ -88,11 +89,13 @@ function resolveFeedback(): ChatCatalogCase {
const body = `${pricing}\n\n${supportText}`;
const doc = await putDocument(ctx, t.id, body);
const createAnnotation = (quote: string, feedback: string) => {
const start = body.indexOf(quote);
const projection = projectMarkdownToText(body);
const start = projection.text.indexOf(quote);
const range = resolveProjectionRange(projection, start, start + quote.length);
if (!range || range.text !== quote) throw new Error("Fixture annotation quote did not resolve through production text projection");
return ctx.input.api.post<Entity>(`/api/issues/${t.id}/documents/plan/annotations`, {
baseRevisionId: doc.latestRevisionId, baseRevisionNumber: doc.latestRevisionNumber,
selector: { quote: { exact: quote, prefix: "", suffix: "" },
position: { normalizedStart: start, normalizedEnd: start + quote.length, markdownStart: start, markdownEnd: start + quote.length } },
selector: createDocumentAnchorSelector(projection, range),
body: feedback,
});
};
@@ -100,7 +103,7 @@ function resolveFeedback(): ChatCatalogCase {
const siblingThread = await createAnnotation(supportText, "Please add a support contact later; leave this unrelated comment open for now.");
return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId };
});
return { ...base, async observe(ctx, state) {
return { ...base, sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
const o = await observeInternal(ctx, state);
return { ...o, annotations: await ctx.input.api.get<Entity[]>(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) };
} };
@@ -114,4 +114,109 @@ function privateProject(): ChatCatalogCase {
async cleanup() { await observer?.dispose(); observer = undefined; },
};
}
export const projectPrivacyCases: readonly ChatCatalogCase[] = [privateProject()];
export const projectPrivacyCases: readonly ChatCatalogCase[] = [privateProject(), shareProject()];
interface AccessMember extends Entity { subjectType?: string; subjectId?: string }
interface ShareProjectObservation extends InternalObservation {
sharedProject: PrivateRecord | null; members: AccessMember[]; observerProjectStatus: number; observerProtectedTaskStatus: number;
}
export function gradeShareProject(value: unknown) {
const c = checks(); const base = observation(value); const o = base as ShareProjectObservation | undefined;
const before = o?.state.beforeShare as { project?: PrivateRecord; observerProjectStatus?: number; protectedTaskStatus?: number; agentMutationStatus?: number; leadReadStatus?: number; reply?: string } | undefined;
c.add("complete-owner-sharing-evidence", !!o && !!before && Array.isArray(o.members), "Sharing requires independent before/after access and actual owner-action attribution.");
if (!o || !before || !Array.isArray(o.members)) return c.result();
const p = o.sharedProject;
c.add("agent-respects-existing-audience", before.project?.visibility === "private" && [403, 404].includes(before.observerProjectStatus ?? 0)
&& before.leadReadStatus === 200 && before.agentMutationStatus === 403
&& /\b(?:manage access|(?:you|owner|human|board)[^.\n]*(?:share|grant|add|access))\b/i.test(before.reply ?? "")
&& !/\bI(?:'ve| have)?\s+(?:already\s+)?(?:shared|granted|added)\b/i.test(before.reply ?? ""),
"The CEO can read the project but cannot grant its audience; its first reply identifies the authorized owner step.");
c.add("owner-grants-only-riley", !!p && p.visibility === "private" && o.observerProjectStatus === 200
&& o.members.some(row => row.subjectType === "user" && row.subjectId === o.state.observerId)
&& o.activity.some(row => row.action === "project.access_member_added" && row.entityId === p.id
&& row.actorType === "user" && row.actorId === o.state.ownerId && row.details?.subjectId === o.state.observerId),
"A visible owner action grants the intended same-company Riley through project membership, without making the project public.");
c.add("separate-task-privacy-retained", [403, 404].includes(before.protectedTaskStatus ?? 0) && [403, 404].includes(o.observerProtectedTaskStatus)
&& o.tasks.some(row => row.id === entity(o.state, "protectedTask").id && (row as PrivateRecord).visibility === "private"),
"The separately protected task remains private and unreadable to Riley after project sharing.");
c.add("saved-project-link", !!p && hasLink(o, "projects", p), "The conversation identifies the exact project being shared.");
return c.result();
}
async function agentMutationStatus(ctx: ChatCatalogContext, path: string, data: unknown) {
const id = ctx.input.fixtures.agent.id;
const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${id}/keys`, { name: `privacy-write-control-${ctx.input.nonce}` });
ctx.input.privateSecrets?.push(key.token);
try {
const url = new URL(path, ctx.input.api.baseURL);
if (url.origin !== ctx.input.api.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Privacy mutation probe escaped fixture API");
const response = await fetch(url, { method: "POST", redirect: "error", signal: AbortSignal.timeout(15_000),
headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, body: JSON.stringify(data) });
await response.body?.cancel(); return response.status;
} finally { await ctx.input.api.delete(`/api/agents/${id}/keys/${key.id}`); }
}
function shareProject(): ChatCatalogCase {
let observer: Person | undefined;
const base = makeDefinition("projects.share-project", async ctx => {
const ownerId = await mainPerson(ctx); observer = await observerPerson(ctx);
const p = await project(ctx, "Acquisition research", { visibility: "private", description: "Private acquisition research for the authorized owner." });
await ctx.input.api.post(`/api/projects/${p.id}/access-members`, { subjectType: "agent", subjectId: ctx.input.fixtures.agent.id });
const protectedTask = await ctx.input.api.post<Entity>(`/api/companies/${ctx.input.fixtures.company.id}/issues`, {
title: "Separately protected acquisition note", description: `Independent protected task ${ctx.input.nonce}`, visibility: "private", status: "backlog",
});
return { project: p, ownerId, observerId: observer.id, protectedTask };
}, gradeShareProject);
return { ...base, authentication: "human-session", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts"],
async act(ctx, state) {
if (!observer) throw new Error("Missing independent observer");
const first = await pollUntil({ label: "Existing private-project owner handoff", deadlineAt: Date.now() + 150_000,
load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs.every(row => row.status === "succeeded")
&& ctx.issue()?.conversationState === "waiting" });
const p = entity(state, "project"); const protectedTask = entity(state, "protectedTask");
const leadReadStatus = await agentStatus(ctx, first.leadId, `/api/projects/${p.id}`);
const agentMutation = await agentMutationStatus(ctx, `/api/projects/${p.id}/access-members`, { subjectType: "user", subjectId: observer.id });
state.beforeShare = { project: first.projects.find(row => row.id === p.id), reply: first.reply, leadReadStatus, agentMutationStatus: agentMutation,
observerProjectStatus: (await personRead(observer, `/api/projects/${p.id}`)).status,
protectedTaskStatus: (await personRead(observer, `/api/issues/${protectedTask.id}`)).status };
await ctx.input.evidence("chat-catalog-before-owner-project-sharing.json", state.beforeShare);
if (agentMutation !== 403 || ![403, 404].includes((state.beforeShare as { observerProjectStatus: number }).observerProjectStatus))
throw new Error("Private-project audience changed without the authorized human control");
const href = first.links.find(link => hasLink({ ...first, links: [link] }, "projects", p));
if (!href) throw new Error("Owner handoff omitted the actual project link");
const sourceUrl = ctx.input.page.url();
await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
await ctx.input.page.getByText("Configuration", { exact: true }).last().click();
await ctx.input.page.getByRole("button", { name: "Manage access", exact: true }).click();
const dialog = ctx.input.page.getByRole("dialog", { name: "Private project access", exact: true });
await dialog.getByRole("combobox").click();
await ctx.input.page.getByPlaceholder(/Search people and agents/).last().fill("Riley");
await ctx.input.page.getByRole("option", { name: /Riley/ }).first().click();
await dialog.getByRole("button", { name: "Add", exact: true }).click();
await pollUntil({ label: "Owner-added Riley project membership", deadlineAt: Date.now() + 30_000,
load: () => ctx.input.api.get<AccessMember[]>(`/api/projects/${p.id}/access-members`),
accept: rows => rows.some(row => row.subjectType === "user" && row.subjectId === observer!.id) });
await ctx.input.page.goto(sourceUrl, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
},
async observe(ctx, state): Promise<ShareProjectObservation> {
if (!observer) throw new Error("Missing independent observer");
const o = await observeInternal(ctx, state); const p = entity(state, "project");
const [members, projectRead, taskRead] = await Promise.all([
ctx.input.api.get<AccessMember[]>(`/api/projects/${p.id}/access-members`), personRead(observer, `/api/projects/${p.id}`),
personRead(observer, `/api/issues/${entity(state, "protectedTask").id}`),
]);
return { ...o, sharedProject: o.projects.find(row => row.id === p.id) as PrivateRecord ?? null, members,
observerProjectStatus: projectRead.status, observerProtectedTaskStatus: taskRead.status };
},
async verifyResult(ctx, state, value) {
await base.verifyResult(ctx, state, value);
if (!observer) throw new Error("Missing independent browser observer");
const p = entity(state, "project"); const observerPage = await observer.context.newPage();
try {
await observerPage.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/projects/${p.id}`, ctx.input.api.baseURL).href, { waitUntil: "domcontentloaded" });
await expect(observerPage.getByRole("heading", { name: p.name!, exact: true }).first()).toBeVisible();
await ctx.input.evidence("chat-catalog-owner-project-sharing-opened.json", { observerId: observer.id, projectId: p.id, url: observerPage.url(), visible: true });
} finally { await observerPage.close(); }
},
async cleanup() { await observer?.dispose(); observer = undefined; },
};
}
@@ -15,6 +15,14 @@ function valid(scenario: string): InternalObservation {
const o = base(scenario);
switch (scenario) {
case "projects.create-project": o.state = { firstIds: ["p"], retryQuery: "query" }; o.humanQueries = ["query", "query"]; o.tasks = []; o.activity = [event("project.created", "p")]; break;
case "projects.share-project": {
o.state.ownerId = "owner"; o.state.observerId = "riley"; o.state.protectedTask = { id: "protected", visibility: "private" };
o.tasks.push({ id: "protected", ...{ visibility: "private" } });
const p = { ...o.projects[0]!, name: "Acquisition research", ...{ visibility: "private" as const } }; o.projects[0] = p;
o.state.beforeShare = { project: p, observerProjectStatus: 404, protectedTaskStatus: 404, agentMutationStatus: 403, leadReadStatus: 200, reply: "You can use Manage access to share this project with Riley." };
Object.assign(o, { sharedProject: p, members: [{ id: "member", subjectType: "user", subjectId: "riley" }], observerProjectStatus: 200, observerProtectedTaskStatus: 404 });
o.activity = [{ ...event("project.access_member_added", "p", { subjectId: "riley" }), agentId: null, actorType: "user", actorId: "owner" }]; break;
}
case "projects.private-project": {
o.state.ownerId = "person"; o.state.inheritanceTask = { id: "probe", projectId: "p" }; o.state.privateMarker = "protected acquisition";
o.tasks.push({ id: "probe", projectId: "p" }); o.activity = [event("project.created", "p")];
@@ -117,6 +125,7 @@ function valid(scenario: string): InternalObservation {
}
const corrupt: Record<string, (o: InternalObservation) => void> = {
"projects.create-project": o => o.projects.push({ ...o.projects[0]!, id: "duplicate" }),
"projects.share-project": o => { Object.assign(o, { observerProtectedTaskStatus: 200 }); },
"projects.private-project": o => { Object.assign(o, { unrelatedAgentStatus: 200 }); },
"projects.archive-project": o => { o.documents = []; },
"tasks.website-task": o => { o.tasks[0]!.assigneeAgentId = "another-worker"; },
@@ -268,3 +277,23 @@ it("does not accept an entirely unauthorized agent as the unrelated-agent contro
const o = valid("projects.private-project"); Object.assign(o, { unrelatedPublicStatus: 403 });
expect(gradeInternal(o).passed).toBe(false);
});
it("rejects a Board mention used as context for a different purported editor", () => {
const o = valid("activity.audit-change"); o.reply = "Board records show Morgan changed the priority to high and low today at 12:00 UTC.";
expect(gradeInternal(o).passed).toBe(false);
});
it("accepts an accurate relative event day without inventing clock precision", () => {
const o = valid("activity.audit-change"); o.reply = "Both edits were by Board: high, then low today.";
expect(gradeInternal(o).passed).toBe(true);
});
it("rejects a correct clock paired with a wrong named calendar date", () => {
const o = valid("activity.audit-change"); o.reply = "Board changed priority to high and low on October 25 at 12:00 UTC.";
expect(gradeInternal(o).passed).toBe(false);
});
it("rejects a premature sharing claim before the owner action", () => {
const o = valid("projects.share-project");
(o.state.beforeShare as { reply: string }).reply = "I've shared it with Riley; you can use Manage access.";
expect(gradeInternal(o).passed).toBe(false);
});