fix(evals): preserve verified recovery and agent binding ownership

Co-Authored-By: Paperclip <paperclip@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-company-secret-question.test.ts

@@ -185,3 +185,8 @@ it("extracts only explicit environment identifiers and does not treat the actual
  expect(companyRequestedEnvironmentName("Once it's saved, I'll add a secret reference to Morgan's environment. I'll use DEPLOY_API_KEY as the variable name unless you specify another.")).toBeUndefined();
  expect(companyRequestedEnvironmentName("Bind Morgan's Environment AS `other_api_key`.")).toBe("other_api_key");
 });
+
+it("keeps a generic secure-save completion question with a later agent binding promise on the native path",()=>{
+ expect(companyOperatorBindingRequested("Have you configured the deployment secret in Settings Secrets? Once saved, I'll add its reference to Morgan's environment.")).toBe(false);
+ expect(companyOperatorBindingRequested("Have you configured the secret reference in Morgan's environment?")).toBe(true);
+});

tests/runner-e2e/chat-catalog-domains/external-company-secret-route.test.ts

@@ -4,7 +4,7 @@ import {companySecretSetupURL} from "./external-company-secret.js";
 const origin="https://paperclip.example",route="/RUN/company/settings/secrets";
 const body="I can't enter the key value myself — company secret values are board-only secure entry, and I won't ask you to paste a secret into chat.\n\nTo add it:\n1. Open `/RUN/company/settings/secrets`.\n2. Click **New secret**.\n3. Use name e.g. **Deployment API key**, env key **DEPLOYMENT_API_KEY**, and enter the deployment key value there.\n4. Save it, then tell me the name/key you used (not the value).\n\nOnce it exists I'll add a `DEPLOYMENT_API_KEY` secret reference to Morgan's agent environment and verify the reference is in place. Saving the secret only stores it; Morgan gets it on its next run after the binding.";
 it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("admits the actual4587 displayed inline route and rejects absent, foreign, cross-company or hidden routes in offline Chromium",async()=>{
- const browser=await chromium.launch({headless:true}),page=await browser.newPage();
+ const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage();
  const render=async(text:string,html?:string,id="native")=>{await page.setContent(`<article id="comment-${id}"><p>Use secure Settings Secrets for Morgan's deployment credential environment binding.</p>${html??`<code>${text}</code>`}</article>`);};
  try{
   await render(route);expect(await companySecretSetupURL(page,{body,commentId:"native"},origin,"RUN",1000)).toBe(origin+route);

tests/runner-e2e/chat-catalog-domains/external-company-secret.ts

@@ -54,7 +54,8 @@ export function companyRequestedEnvironmentName(text:string):string|false|undefi
  const unique=[...new Set(names)];return unique.length>1?false:unique[0];
 }
 export function companyOperatorBindingRequested(text:string){
- if(/\b(?:have you|did you)\b[^?]*\b(?:bound|configured)\b/i.test(text))return true;
+ const question=text.match(/\b(?:have you|did you)\b[^?\n]*/i)?.[0];
+ if(question&&(/\bbound\b/i.test(question)||/\bconfigured\b[^?\n]*\b(?:reference|binding|environment)\b/i.test(question)))return true;
  return text.split(/[.!;\n]+|\b(?:then|and)\b/i).some(clause=>/^\s*(?:please\s+)?(?:bind|configure|add|assign)\b[^.!;\n]*\b(?:reference|secret)\b[^.!;\n]*\bMorgan\b/i.test(clause));
 }
 export function companySecretFollowup(body:string,savedKey:string){

tests/runner-e2e/chat-catalog-domains/external-personal-secret-route.test.ts

@@ -4,7 +4,7 @@ import {personalSecretSetupURL} from "./external-personal-secret.js";
 const origin="https://paperclip.example",path="/RUN/company/settings/secrets",definition="current-definition";
 const body=`Open \`${path}?definition=${definition}\` → My secrets → PERSONAL_PUBLISHING_KEY → Set my value, then save. Please don't paste the key in chat. Morgan's PUBLISHING_API_KEY runtime reference is already configured.`;
 it.skipIf(process.env.PAPERCLIP_CHAT_EVAL_SECRET_ROUTE_CANARY!=="1")("admits the visible own-definition route from4716 and rejects unbound or unsafe routes",async()=>{
- const browser=await chromium.launch({headless:true}),page=await browser.newPage();
+ const browser=await chromium.launch({headless:true,...process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?{executablePath:process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE}:{}}),page=await browser.newPage();
  const render=async(text:string,html?:string)=>page.setContent(`<article id="comment-native"><p>Personal credential setup</p>${html??`<code>${text}</code>`}</article>`);
  try{
   await render(path+"?definition="+definition);

tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts

@@ -217,10 +217,10 @@ it("rejects an explicit current noninstallation claim after historical failure i
     expect(gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed).toBe(false);
   }
 });
-it("does not treat a historical negative alone as an asserted initial recovery", () => {
+it("does not require a present-tense recovery assertion when saved recovery is independently proven", () => {
   const { o } = recovered();
   o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was previously not installed.";
-  expect(gradeTeamConditions(o, "initial").passed).toBe(false);
+  expect(gradeTeamConditions(o, "initial").passed).toBe(true);
 });
 
 it.each(["It", "This skill", "That skill"])("accepts immediately bound %s recovery without repeating the skill name", subject => {
@@ -233,10 +233,10 @@ it("rejects an immediately bound pronoun claiming the recovered skill is still m
   o.phaseReplies[1]!.body = "Launch Checklist is installed. It is still missing and not installed.";
   expect(gradeTeamConditions(o).passed).toBe(false);
 });
-it.each(["Launch Checklist hit a conflict. Another skill was discussed. It is now installed.", "It is now installed. Launch Checklist hit a conflict.", "Launch Checklist hit a conflict.\n\nIt is now installed."])("does not credit interrupted or unbound pronoun recovery: %s", body => {
+it.each(["Launch Checklist hit a conflict. Another skill was discussed. It is now installed.", "It is now installed. Launch Checklist hit a conflict.", "Launch Checklist hit a conflict.\n\nIt is now installed."])("does not require unbound pronoun prose as evidence of independently proven recovery: %s", body => {
   const { o } = recovered();
   o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. ${body}`;
-  expect(gradeTeamConditions(o, "initial").passed).toBe(false);
+  expect(gradeTeamConditions(o, "initial").passed).toBe(true);
 });
 
 it("invalidates contradictory current failure in a contiguous bound pronoun chain in either phase", () => {
@@ -357,9 +357,9 @@ describe("reviewed skill assertion subject and polarity boundaries", () => {
   it.each([
     "Launch Checklist's slug is occupied and the team was installed successfully.",
     "Launch Checklist's conflict was resolved because the manager was installed.",
-  ])("does not credit an unrelated installation as skill recovery: %s", body => {
+  ])("does not require unrelated installation prose as evidence of independently proven recovery: %s", body => {
     const { o } = recovered(); o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. " + body;
-    expect(gradeTeamConditions(o).passed).toBe(false);
+    expect(gradeTeamConditions(o).passed).toBe(true);
   });
   it("rejects affirmed caveat resolution even beside a conditional unresolved reference", () => {
     const o = valid(); o.phaseReplies[1]!.body = "We resolved the launch-checklist skill caveat. No further action needed unless you want the launch-checklist skill caveat resolved.";
@@ -377,3 +377,19 @@ it("recognizes a Markdown-wrapped skill recovery subject", () => {
 it("rejects the same Markdown-wrapped installation claim while the skill is missing", () => {
   expect(truthfulTeamPartial("Launch Checklist's slug is occupied. **Launch Checklist** is now installed.", skill)).toBe(false);
 });
+
+
+it.each(["Launch Checklist has now been installed under another slug.", "Launch Checklist installation succeeded under a new slug.", "The installation of Launch Checklist succeeded under a new slug."])("admits independently verified recovery without a required success phrase: %s", body => {
+  const { o } = recovered(); o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. " + body;
+  expect(gradeTeamConditions(o).passed).toBe(true);
+});
+it("rejects an explicit verb-before-name denial of saved recovery", () => {
+  const { o } = recovered(); o.phaseReplies[1]!.body = "The same team exists. We have not installed Launch Checklist.";
+  expect(gradeTeamConditions(o).passed).toBe(false);
+});
+it("does not require an installation assertion from conflict-object prose when saved recovery is independently proven", () => {
+  const { o, condition } = recovered(); o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist hit a conflict. The conflict was resolved because the manager was installed.";
+  expect(gradeTeamConditions(o).passed).toBe(true);
+  condition.recovery.version.id = "foreign-revision";
+  expect(gradeTeamConditions(o).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts

@@ -205,12 +205,14 @@ function assertedTeamRecovery(clause: string, skill: Pick<CatalogSkill, "slug" |
   return [...clause.matchAll(claims)].some(match => !negatedTeamRecovery(clause.slice(0, match.index)));
 }
 function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
-  return namedTeamSkillClauses(reply, skill).some(clause => assertedTeamRecovery(clause, skill))
-    && !claimsRecoveredSkillMissing(reply, skill);
+  return !claimsRecoveredSkillMissing(reply, skill);
 }
 function claimsRecoveredSkillMissing(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
   const current = new RegExp(`\\b(?:${teamSkillSubject(skill)})\\s+(?:skill\\s+)?(?:(?:is|are|remains?|still)\\s+)?(?:(?:now|currently|still)\\s+)?(?:not (?:(?:successfully|fully|completely)\\s+){0,2}(?:installed|available|usable)|uninstalled|missing|unavailable|blocked)\\b`, "i");
-  return namedTeamSkillClauses(reply, skill, true).some(clause => current.test(clause));
+  const deniedObject = new RegExp(`\\binstalled\\s+(?:the\\s+)?(?:${teamSkillSubject(skill)})\\b`, "gi");
+  const deniedPast = new RegExp(`\\b(?:${teamSkillSubject(skill)})\\s+(?:was|were)\\s+not\\s+installed\\b`, "i");
+  return namedTeamSkillClauses(reply, skill, true).some(clause => current.test(clause) || deniedPast.test(clause)
+    || [...clause.matchAll(deniedObject)].some(match => negatedTeamRecovery(clause.slice(0, match.index))));
 }
 export async function observeTeamConditions(
   ctx: ChatCatalogContext,
@@ -375,7 +377,7 @@ export function gradeTeamConditions(
       (resolved(earlier) ? truthfulTeamRecovery(firstSkillReply, target) : truthfulTeamPartial(firstSkillReply, target)) &&
       (phase === "initial" || (resolved(later) ? !claimsRecoveredSkillMissing(repeatSkillReply, target) : truthfulTeamPartial(repeatSkillReply, target,
         earlier?.requiredCatalogStillMissing === true && later?.requiredCatalogStillMissing === true && truthfulTeamPartial(firstSkillReply, target)))),
-    "The first reply reports the actual renamed role and either the unresolved required skill or its independently verified recovery. A remaining failure must be reported in both phases; a later reply cannot rescue false initial readiness.",
+    "The first reply reports the actual renamed role and remaining skill failure; independently saved recovery needs no prescribed success phrase but neither phase may contradict the saved state. A later reply cannot rescue false initial readiness.",
   );
   return c.result();
 }