test(chat): grade import-preview by saved configuration and the stopped first turn, not reply keywords

The baseline attempt failed although the agent only previewed: the
before-choice check compared the existing agent's full configuration
response, whose lifecycle status and updatedAt change after setup
(preparation and harness budget funding). Compare the configuration
without those fields (configurationRevision, the product's own content
hash, stays). Replace the conflict/choice keyword regexes over the reply
with structure: the first turn ended waiting on the person with no import
activity, and the reply names the setup-chosen conflicting employee.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -2069,7 +2069,7 @@
       "chat-catalog-domains/internal-support.ts",
       "chat-scenarios/companies.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.identity.v1",
+    "oracleVersion": "paperclip.chat-catalog.identity.import-preview.v2",
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/identity-portability.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { gradePortability } from "./identity-portability.js";
+import { gradePortability, savedAgentConfiguration } from "./identity-portability.js";
 import type { IdentityObservation } from "./identity-support.js";
 function valid(scenarioId: string): IdentityObservation {
   const o: IdentityObservation = { scenarioId, origin: "http://fixture.invalid", companyId: "acorn", leadId: "ceo", company: { id: "acorn" }, companies: [], agents: [], tasks: [], projects: [], goals: [], comments: [], interactions: [], documents: [], revisions: [], activity: [], links: [], runs: [], humanQueries: [], reply: "Secrets are excluded; supply EXPORT_FIXTURE_KEY when importing.", state: {} };
@@ -12,7 +12,7 @@ function valid(scenarioId: string): IdentityObservation {
         files: { "COMPANY.md": "Acorn", "agents/ceo/AGENTS.md": "CEO", "skills/launch-checklist/SKILL.md": "Checklist" }, envInputs: [{ key: "EXPORT_FIXTURE_KEY", kind: "secret", requirement: "optional", defaultValue: "" }] } };
   } else {
     o.state = { existing: { id: "morgan" }, preview: { targetCompanyId: "acorn", errors: [], plan: { agentPlans: [{ slug: "morgan", existingAgentId: "morgan", action: "create" }] } },
-      beforeChoiceUnchanged: true, initialReplyAskedChoice: true, actualPackageAttached: true, importedAgentId: "new", existingConfigurationUnchanged: true, existingStillPresent: true, importedAgentVisible: true };
+      beforeChoiceUnchanged: true, firstTurnAwaitedHuman: true, firstReply: "Previewed (nothing imported): adds 1 agent, Morgan. Slug `morgan` already exists here, so the default rename plans it as Morgan 2 instead of overwriting the existing employee.", actualPackageAttached: true, importedAgentId: "new", existingConfigurationUnchanged: true, existingStillPresent: true, importedAgentVisible: true };
     o.humanQueries = ["Show me what this company package would add before importing it.", "Rename the incoming employee Morgan Package and import it."];
     o.agents = [{ id: "morgan", name: "Morgan", companyId: "acorn" }, { id: "new", name: "Morgan Package", companyId: "acorn" }];
     o.activity = [{ id: "import", action: "company.imported", entityId: "acorn", agentId: "ceo", details: { importMode: "agent_safe" } }];
@@ -34,4 +34,17 @@ describe("actual company package download and conflict-safe import", () => {
   it("rejects secret values in an otherwise supported package", () => { const o = valid("companies.export-company"); o.state.packageContainsPrivateValue = true; expect(gradePortability(o).passed).toBe(false); });
   it("rejects silently performing the import before human conflict choice", () => { const o = valid("companies.import-preview"); o.state.beforeChoiceUnchanged = false; expect(gradePortability(o).passed).toBe(false); });
   it("rejects a replaced employee despite identical visible name", () => { const o = valid("companies.import-preview"); o.state.importedAgentId = "morgan"; expect(gradePortability(o).passed).toBe(false); });
+  it("accepts a reworded first preview that surfaces the conflict without the old keywords", () => {
+    const o = valid("companies.import-preview"); o.state.firstReply = "Nothing has been imported yet. The package brings one engineer, Morgan, but you already employ a Morgan; tell me what the newcomer should be called, or whether to leave it out.";
+    expect(gradePortability(o).passed).toBe(true);
+  });
+  it("rejects a first turn that never surfaces the conflicting employee or that did not stop for the person", () => {
+    const silent = valid("companies.import-preview"); silent.state.firstReply = "The package adds one engineer. Ready when you are."; expect(gradePortability(silent).passed).toBe(false);
+    const kept = valid("companies.import-preview"); kept.state.firstTurnAwaitedHuman = false; expect(gradePortability(kept).passed).toBe(false);
+  });
+  it("compares saved configuration without lifecycle status or timestamps, but catches an overwrite", () => {
+    const before = { name: "Morgan", adapterType: "paperclip_runner", adapterConfig: { model: "m" }, configurationRevision: "a", status: "paused", updatedAt: "t1" };
+    expect(savedAgentConfiguration({ ...before, status: "idle", updatedAt: "t2" })).toBe(savedAgentConfiguration(before));
+    expect(savedAgentConfiguration({ ...before, adapterType: "process", configurationRevision: "b" })).not.toBe(savedAgentConfiguration(before));
+  });
 });

tests/runner-e2e/chat-catalog-domains/identity-portability.ts

@@ -12,6 +12,12 @@ import { identityDefinition, identityObservation, identityWorker, observeIdentit
 
 const secretValues = new WeakMap<ChatCatalogContext, string>();
 interface AgentConfiguration { adapterType: string; adapterConfig: Record<string, unknown>; runtimeConfig: Record<string, unknown> }
+/** The saved configuration an import could overwrite. Lifecycle status and updatedAt change on their own
+ * after setup (agent preparation, harness budget funding), so they are not configuration; configurationRevision
+ * is the product's own content hash of the configuration and stays in the comparison. */
+export function savedAgentConfiguration(value: unknown) {
+  const { status: _status, updatedAt: _updatedAt, ...configuration } = (value ?? {}) as Record<string, unknown>; return JSON.stringify(configuration);
+}
 function text(entry: CompanyPortabilityFileEntry) { return typeof entry === "string" ? entry : Buffer.from(entry.data, "base64").toString("utf8"); }
 export function gradePortability(value: unknown) {
   const c = checks(); const o = identityObservation(value); c.add("complete-portability-evidence", !!o, "Actual package files, public provenance, permitted actor and independently saved resources are required."); if (!o) return c.result();
@@ -37,8 +43,10 @@ export function gradePortability(value: unknown) {
     const preview = o.state.preview as CompanyPortabilityPreviewResult | undefined; const conflict = entity(o.state, "existing");
     c.add("actual-conflict-before-mutation", !!preview && preview.errors.length === 0 && preview.targetCompanyId === o.companyId
       && preview.plan.agentPlans.some(plan => plan.slug === "morgan" && plan.existingAgentId === conflict.id && plan.action !== "update")
-      && o.state.beforeChoiceUnchanged === true && o.state.initialReplyAskedChoice === true && o.state.actualPackageAttached === true,
-      "The actual attached synthetic package produces a real Morgan collision preview and requests a consequential choice before changing company resources.");
+      && o.state.beforeChoiceUnchanged === true && o.state.firstTurnAwaitedHuman === true && o.state.actualPackageAttached === true
+      // The conflicting employee's name is a setup-chosen literal; the reply must surface it. How the agent phrases the conflict or the question is not graded.
+      && typeof o.state.firstReply === "string" && o.state.firstReply.includes("Morgan"),
+      "The actual attached synthetic package produces a real Morgan collision preview; the first turn names the conflicting employee and hands the decision back to the person before changing company resources.");
     c.add("explicit-human-rename-import", o.humanQueries.length === 2 && o.humanQueries[1]?.includes("Morgan Package") && o.state.importedAgentId !== conflict.id
       && o.agents.some(agent => agent.id === o.state.importedAgentId && agent.name === "Morgan Package" && agent.companyId === o.companyId)
       && o.state.existingConfigurationUnchanged === true && o.state.existingStillPresent === true && o.state.importedAgentVisible === true,
@@ -102,14 +110,17 @@ export const portabilityIdentityCases: readonly ChatCatalogCase[] = [
     return { sourceId: source.id, existing, existingConfiguration, sourcePackage, packageAttachment, actualPackageAttached: true,
       beforeAgentIds: (await ctx.input.api.get<Entity[]>(`/api/companies/${ctx.input.fixtures.company.id}/agents`)).map(row => row.id) };
   }, gradePortability, {
-    authentication: "human-session", sourceFiles: sources, queryAttachments: [{ stateKey: "packageAttachment", filename: "company-package.json" }], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000,
+    authentication: "human-session", sourceFiles: sources, oracleVersion: "paperclip.chat-catalog.identity.import-preview.v2", queryAttachments: [{ stateKey: "packageAttachment", filename: "company-package.json" }], expectedRunCount: 2, expectedHumanMessageCount: 2, attemptTimeoutMs: 420_000,
     async act(ctx, state) {
       await waitForReply(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`;
       const current = await ctx.input.api.get<Entity[]>(`${root}/agents`); state.beforeChoiceUnchanged = JSON.stringify(current.map(row => row.id).sort()) === JSON.stringify([...(state.beforeAgentIds as string[])].sort())
-        && JSON.stringify(state.existingConfiguration) === JSON.stringify(await ctx.input.api.get<AgentConfiguration>(`/api/agents/${entity(state, "existing").id}/configuration`));
-      const o = await observeIdentity(ctx, state); state.initialReplyAskedChoice = /Morgan/i.test(o.reply) && /conflict|already exists|same name|collision/i.test(o.reply) && /rename|skip|which|choose|would you|should I/i.test(o.reply);
+        && savedAgentConfiguration(state.existingConfiguration) === savedAgentConfiguration(await ctx.input.api.get<AgentConfiguration>(`/api/agents/${entity(state, "existing").id}/configuration`))
+        && !(await observeIdentity(ctx, state)).activity.some(row => row.action === "company.imported");
+      const o = await observeIdentity(ctx, state); state.firstReply = o.reply;
+      // waitForReply returned with the conversation waiting on the person and every run finished: the agent stopped after the preview instead of importing.
+      state.firstTurnAwaitedHuman = ctx.issue()?.conversationState === "waiting" && o.runs.length >= 1 && o.runs.every(run => run.status === "succeeded");
       state.preview = await ctx.input.api.post<CompanyPortabilityPreviewResult>(`${root}/imports/preview`, { source: state.sourcePackage, target: { mode: "existing_company", companyId: ctx.input.fixtures.company.id }, collisionStrategy: "rename", include: { company: false, agents: true, projects: false, issues: false, skills: false } });
-      if (!state.beforeChoiceUnchanged || !state.initialReplyAskedChoice) throw new Error("The consequential import conflict was not previewed without mutation and referred to the human");
+      if (!state.beforeChoiceUnchanged || !state.firstTurnAwaitedHuman) throw new Error("The consequential import conflict was not previewed without mutation and referred to the human");
       await sendChatMessage(ctx.input.page, "Import the package into this company. Choose rename for the incoming Morgan and name the new employee Morgan Package. Keep the existing Morgan and every part of its configuration unchanged. Do not run the imported employee.");
       const rows = await pollUntil({ label: "actual scoped renamed import", deadlineAt: Date.now() + 150_000, load: async () => { await ctx.refreshIssue(); return { agents: await ctx.input.api.get<Entity[]>(`${root}/agents`), runs: await ctx.allRuns() }; },
         accept: value => value.agents.some(row => row.name === "Morgan Package" && row.id !== entity(state, "existing").id) && value.runs.length === 2 && value.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" });
@@ -121,7 +132,7 @@ export const portabilityIdentityCases: readonly ChatCatalogCase[] = [
     async observe(ctx, state) {
       const o = await observeIdentity(ctx, state); const imported = o.agents.find(row => row.name === "Morgan Package" && row.id !== entity(state, "existing").id);
       state.importedAgentId = imported?.id; state.existingStillPresent = o.agents.some(row => row.id === entity(state, "existing").id && row.name === "Morgan");
-      state.existingConfigurationUnchanged = JSON.stringify(state.existingConfiguration) === JSON.stringify(await ctx.input.api.get<AgentConfiguration>(`/api/agents/${entity(state, "existing").id}/configuration`)); return o;
+      state.existingConfigurationUnchanged = savedAgentConfiguration(state.existingConfiguration) === savedAgentConfiguration(await ctx.input.api.get<AgentConfiguration>(`/api/agents/${entity(state, "existing").id}/configuration`)); return o;
     },
     async verifyResult(ctx, state) {
       if (typeof state.importedAgentId !== "string") throw new Error("Actual imported employee is missing"); const source = ctx.input.page.url();