fix(evals): use public membership for incidental human fixtures

Keep invitation enrollment quarantine for invitation journeys. Establish the same real scoped non-admin identities through the existing public prerequisite helper for feedback, timeline, and primary-agent fixtures.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-auth.ts

@@ -104,6 +104,7 @@ export async function enrollAuthenticatedPerson(api: RunnerApi, companyId: strin
 }
 
 export const prerequisiteMembershipScenarioIds = Object.freeze([
+  "activity.agent-feedback", "activity.work-timeline", "agents.choose-primary",
   "assistant-connections.consent-read-only", "connection-setup.wrong-person-resolves",
   "documents-artifacts.private-download", "documents-artifacts.restore-document",
   "inbox-search.archive-done", "inbox-search.notification-audience", "inbox-search.restore-archive", "inbox-search.show-decisions",

tests/runner-e2e/chat-catalog-domains/identity-prerequisite-membership.test.ts

@@ -1,4 +1,5 @@
 import { afterEach, expect, it, vi } from "vitest";
+import { readFile } from "node:fs/promises";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import { grantsForHumanRole } from "../../../server/src/services/company-member-roles.js";
 import { enrollPrerequisitePerson, prerequisiteMembershipScenarioIds } from "./identity-auth.js";
@@ -39,7 +40,20 @@ it.each(["operator", "viewer"] as const)("establishes only the real public %s pr
   expect(receipt).toMatchObject({ userId: "person", companyId: "company", role, beforeStatus: 403, companyReadStatus: 200, instanceAdminReadStatus: 403 });
   expect(f.evidence).toHaveBeenCalledOnce(); expect([...f.reads, ...f.writes.map(row => row.route)]).not.toEqual(expect.arrayContaining([expect.stringContaining("invites")]));
 });
-it.each(["access.invite-teammate", "access.manage-membership", "activity.agent-feedback", "unknown"])("does not substitute public membership for unapproved/invitation journey %s", async scenarioId => {
+it.each([
+  ["activity.agent-feedback", "internal-agent-feedback.ts", "casey"],
+  ["activity.work-timeline", "internal-work-timeline.ts", "owner"],
+  ["agents.choose-primary", "runtime-primary.ts", "person"],
+])("uses scoped public membership for the incidental %s fixture", async (scenarioId, sourceFile, personName) => {
+  const f = fixture(); Object.assign(f.ctx.scenario, { id: scenarioId }); f.ctx.input.execution.id = `chat-features.runner-opencode.local.${scenarioId}`;
+  const receipt = await enrollPrerequisitePerson(f.ctx, f.person);
+  expect(receipt).toMatchObject({ role: "operator", enrollment: "public-company-access", instanceAdminReadStatus: 403 });
+  expect(f.writes).toHaveLength(2); expect(f.evidence).toHaveBeenCalledOnce();
+  const source = await readFile(new URL(sourceFile, import.meta.url), "utf8");
+  expect(source).toContain(`await enrollPrerequisitePerson(ctx, ${personName}, "operator")`);
+  expect(source).not.toContain("enrollAuthenticatedPerson");
+});
+it.each(["access.invite-teammate", "access.manage-membership", "unknown"])("does not substitute public membership for unapproved/invitation journey %s", async scenarioId => {
   const f = fixture(); Object.assign(f.ctx.scenario, { id: scenarioId }); f.ctx.input.execution.id = `chat-features.runner-opencode.local.${scenarioId}`;
   await expect(enrollPrerequisitePerson(f.ctx, f.person)).rejects.toThrow("not admitted"); expect(f.api.get).not.toHaveBeenCalled(); expect(f.writes).toHaveLength(0);
 });
@@ -75,6 +89,6 @@ it("a slow public access write cannot reset the original clock or start role/gra
   await expect(enrollPrerequisitePerson(f.ctx, f.person)).rejects.toThrow("budget"); expect(f.writes).toHaveLength(1); expect(f.api.patchSensitive).not.toHaveBeenCalled(); expect(f.evidence).not.toHaveBeenCalled();
 });
 it("the closed allowlist contains no access/invitation scenarios", () => {
-  expect(prerequisiteMembershipScenarioIds).toHaveLength(20);
+  expect(prerequisiteMembershipScenarioIds).toHaveLength(23);
   expect(prerequisiteMembershipScenarioIds.some(id => id.startsWith("access."))).toBe(false);
 });

tests/runner-e2e/chat-catalog-domains/internal-agent-feedback.ts

@@ -1,7 +1,7 @@
 import { expect } from "@playwright/test";
 import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { pollUntil } from "../api.js";
-import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
+import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js";
 import { fixtureWorker, makeDefinition } from "./internal-definition.js";
 import { mainPerson } from "./internal-privacy.js";
 import { checks, entity, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
@@ -45,7 +45,7 @@ function agentFeedback(): ChatCatalogCase {
     if (!browser || !ctx.input.privateSecrets) throw new Error("Feedback privacy needs a distinct real human and attempt-private redaction");
     casey = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" });
     try {
-      await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, casey, ctx.input.privateSecrets, "operator");
+      await enrollPrerequisitePerson(ctx, casey, "operator");
       const worker = await fixtureWorker(ctx); const privateWorker = await fixtureWorker(ctx);
       await ctx.input.api.patch(`/api/agents/${privateWorker.id}`, { name: "Riley" });
       const root = `/api/companies/${ctx.input.fixtures.company.id}`;

tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts

@@ -3,7 +3,7 @@ import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { pollUntil } from "../api.js";
 import { fixtureWorker, makeDefinition } from "./internal-definition.js";
 import { agentStatus, cleanupPrivacyProbes } from "./internal-privacy.js";
-import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
+import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js";
 import { startTimelineVerifier, type VerificationReceipt } from "./internal-timeline-verifier.js";
 import { localDay } from "./internal-routine-audit.js";
 import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
@@ -43,7 +43,7 @@ function timeline(): ChatCatalogCase {
   let verifier: Awaited<ReturnType<typeof startTimelineVerifier>> | undefined; let owner: Awaited<ReturnType<typeof createAuthenticatedPerson>> | undefined;
   const base = makeDefinition("activity.work-timeline", async ctx => {
     const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Timeline privacy context needs authenticated people");
-    owner = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" }); await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, owner, ctx.input.privateSecrets, "operator");
+    owner = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" }); await enrollPrerequisitePerson(ctx, owner, "operator");
     const response = await owner.request(`/api/companies/${ctx.input.fixtures.company.id}/issues`, "POST", { title: `Hidden acquisition launch ${ctx.input.nonce}`, description: "Protected acquisition research", status: "backlog", visibility: "private" }); if (!response.ok) throw new Error("Private timeline distractor was not created"); const hiddenTask = await response.json() as Entity;
     await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${hiddenTask.id}`);
     const p = await project(ctx, "Launch", { workspace: { name: "Launch timeline workspace", sourceType: "local_path", cwd: ctx.input.workspacePath, isPrimary: true } });

tests/runner-e2e/chat-catalog-domains/runtime-primary.ts

@@ -1,6 +1,6 @@
 import { expect } from "@playwright/test";
 import { checks, type Entity } from "./internal-support.js";
-import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
+import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js";
 import { probeAgentPermission, waitForReply } from "./identity-support.js";
 import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
@@ -42,7 +42,7 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a
   if (primaryBefore.primaryAgentId !== ctx.input.fixtures.agent.id) throw new Error("Catalog requires an existing primary agent");
   const worker = await runtimeWorker(ctx);
   const person = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets, name: "Second primary reader" }); people.set(ctx, person);
-  await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, person, secrets);
+  await enrollPrerequisitePerson(ctx, person, "operator");
   const otherCompany = await ctx.input.api.post<Entity>("/api/companies", { name: `Other primary company ${ctx.input.nonce}` });
   return { worker, primaryBefore, otherCompany, otherPersonBefore: await personPrimary(person, ctx.input.fixtures.company.id),
     otherCompanyBefore: await ctx.input.api.get<Primary>(endpoint(otherCompany.id)) };