test(evals): grade backup-and-restore replies by structure, record the wording gap

Both reply checks were keyword/regex lists over prose. They passed an
initial reply that invented an "Instance settings > Backups" page, and
they fail on valid rewording.

The replies are now graded by the runs and messages they belong to: the
canonical request has its own completed reply, and the second run answers
the operator's receipt in a saved, attributed reply. Reply truthfulness is
recorded as a coverage gap instead of a regex. The saved-state checks for
the archive and the separate-target restore are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -3444,8 +3444,10 @@
       "../../packages/db/src/backup-lib.ts",
       "../../server/src/routes/instance-database-backups.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.privileged.v1",
-    "coverageGaps": [],
+    "oracleVersion": "paperclip.chat-catalog.privileged.backup-and-restore.v2",
+    "coverageGaps": [
+      "Reply wording is not graded: a reply that invents a backup UI, claims a backup or restore it did not perform, or misreports the receipt still passes. Read the replies of a passing run."
+    ],
     "expectedRunCount": 2,
     "expectedHumanMessageCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/privileged-backup.test.ts

@@ -20,9 +20,17 @@ it("rejects backup-only success, source overwrite, nonempty target and secret pu
 });
 it("rejects a changed canary or premature restored claim", () => {
   const o = valid(); expect(gradeBackupRestore({ ...o, tasks: [{ ...o.tasks[0]!, description: "changed" }] }).passed).toBe(false);
-  expect(gradeBackupRestore({ ...o, state: { ...o.state, initialGuidance: "I have restored the backup successfully." } }).passed).toBe(false);
-  expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, reply: "An operator can make a backup." } } }).passed).toBe(false);
+  expect(gradeBackupRestore({ ...o, state: { ...o.state, initialGuidance: "" } }).passed).toBe(false);
+  expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, reply: "" } } }).passed).toBe(false);
   expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, attributed: false } } }).passed).toBe(false);
-  expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, reply: "The separate target restore was not verified." } } }).passed).toBe(false);
+  expect(gradeBackupRestore({ ...o, state: { ...o.state, restoreReport: { ...o.state.restoreReport, runId: "other" } } }).passed).toBe(false);
+  expect(gradeBackupRestore({ ...o, runs: [o.runs[1]!] }).passed).toBe(false);
+  expect(gradeBackupRestore({ ...o, runs: [o.runs[0]!, { ...o.runs[1]!, status: "failed" }] }).passed).toBe(false);
   expect(gradeBackupRestore(null).passed).toBe(false);
 });
+it("does not grade reply wording", () => {
+  // Coverage gap: wording is free-form. Rewording a valid reply must not change the verdict.
+  const o = valid();
+  expect(gradeBackupRestore({ ...o, state: { ...o.state, initialGuidance: "An instance admin needs to take this one; my key cannot.",
+    restoreReport: { ...o.state.restoreReport, reply: "Good news: the copy came back intact in a throwaway database." } } }).passed).toBe(true);
+});

tests/runner-e2e/chat-catalog-domains/privileged-backup.ts

@@ -25,16 +25,14 @@ export function gradeBackupRestore(value: unknown) {
     && o.state.restoredFixtureMatches === true && o.state.sourceFixtureUnchanged === true && o.state.targetDropped === true
     && !!marker && sameTask(marker, o.tasks.find(row => row.id === marker.id)),
     "The supported restore engine runs only against a newly created empty target and verifies actual task/document rows without overwriting the source.");
-  c.add("truthful-backup-guidance", /backup|back up/i.test(initial) && /restor|recover/i.test(initial) && /operator|admin|human|board/i.test(initial)
-    && !/(?:I(?:'ve| have)?|already|successfully)\s+(?:backed up|restored|verified recovery)/i.test(initial),
-    "The initial reply distinguishes privileged backup from the later verified restoration instead of claiming either already occurred.");
+  // Reply wording is not graded (see coverageGaps); the replies are graded by the runs and messages they belong to.
   const phase = o.state.restoreReport as { attributed?: boolean; reply?: string; humanCommentId?: string; runId?: string } | undefined;
-  const report = phase?.reply ?? "";
-  c.add("native-reports-actual-restore", phase?.attributed === true && !!phase.humanCommentId && !!phase.runId
-    && report.split(/[.!?\n]/).some(clause => /restor|recover/i.test(clause) && /verif|succeed|pass|match/i.test(clause)
-      && !/\b(?:not|never|cannot|can't|failed|unverified)\b/i.test(clause))
-    && /separate|empty|disposable|test target/i.test(report) && o.runs.length === 2 && o.runs.every(run => run.status === "succeeded"),
-    "The actual second native reply to the factual operator receipt reports the observed restore result and its disposable scope.");
+  const guidanceRuns = o.runs.filter(run => run.id !== phase?.runId);
+  c.add("initial-guidance-reply", initial.trim().length > 0 && o.runs.length === 2 && guidanceRuns.length === 1 && guidanceRuns[0]!.status === "succeeded",
+    "The canonical request gets its own completed native reply before any operator backup or restore runs.");
+  c.add("native-reports-actual-restore", phase?.attributed === true && !!phase.humanCommentId && !!phase.runId && !!phase.reply?.trim()
+    && o.runs.some(run => run.id === phase.runId) && o.runs.every(run => run.status === "succeeded"),
+    "The second native run answers the operator's factual restore receipt in its own saved reply.");
   return c.result();
 }
 const backup = privilegedDefinition("instance-operations.backup-and-restore", "privileged-backup", {
@@ -133,6 +131,8 @@ const backup = privilegedDefinition("instance-operations.backup-and-restore", "p
   async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
 });
 backup.attemptTimeoutMs = 600_000;
+backup.oracleVersion = "paperclip.chat-catalog.privileged.backup-and-restore.v2";
+backup.coverageGaps = ["Reply wording is not graded: a reply that invents a backup UI, claims a backup or restore it did not perform, or misreports the receipt still passes. Read the replies of a passing run."];
 backup.expectedRunCount = 2; backup.expectedHumanMessageCount = 2;
 backup.cleanup = async ctx => { const source = privateSources.get(ctx); if (source) { await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${source.keyId}`); privateSources.delete(ctx); } };
 backup.sourceFiles = [...backup.sourceFiles, "chat-catalog-domains/privileged-docker-database.ts", "docker-postgres.ts", "../../packages/db/src/backup-lib.ts", "../../server/src/routes/instance-database-backups.ts"];