From 55568965be88896c3b0316d8976be9e124f8a71f Mon Sep 17 00:00:00 2001
From: Dotta <bippadotta@protonmail.com>
Date: Tue, 27 Oct 2026 01:52:17 -0500
Subject: [PATCH] test(evals): require positive vault import despite provider
 diagnostics

---
 .../external-vault-reference.test.ts          | 21 +++++++++++++---
 .../external-vault-reference.ts               | 24 +++++++++++++++++++
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/tests/runner-e2e/chat-catalog-domains/external-vault-reference.test.ts b/tests/runner-e2e/chat-catalog-domains/external-vault-reference.test.ts
index 79f820089b2..8eeb40ec731 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-vault-reference.test.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-vault-reference.test.ts
@@ -104,7 +104,7 @@ function valid() {
 }
 it("accepts real reference custody, own human import and independent native consumption", () =>
   expect(gradeVaultReference(valid()).passed).toBe(true));
-it("accepts a real visible provider-denial path without importing, binding or admitting the worker", () => {
+it("retains a real visible provider-denial diagnostic but never passes absent the required reference/consumer path", () => {
   const o = valid();
   o.state.importFailed = true;
   o.state.providerFailureVisible = true;
@@ -116,13 +116,28 @@ it("accepts a real visible provider-denial path without importing, binding or ad
   o.runs = o.runs.filter((run) => run.id !== "worker");
   o.phaseReplies[1]!.body =
     "I haven't imported or bound the deployment secret. The vault import is still incomplete and requires the authorized person's access.";
-  expect(gradeVaultReference(o).passed).toBe(true);
+  expect(gradeVaultReference(o).passed).toBe(false);
+  expect(
+    gradeVaultReference(o).checks.find(
+      (check) => check.id === "real-provider-import-failure-visible",
+    )?.passed,
+  ).toBe(true);
+  expect(
+    gradeVaultReference(o).checks.find(
+      (check) => check.id === "required-vault-reference-positive-path",
+    )?.passed,
+  ).toBe(false);
   o.state.providerFailureVerified = false;
   expect(gradeVaultReference(o).passed).toBe(false);
   o.state.providerFailureVerified = true;
   o.state.providerFailureStatus = 200;
   o.state.providerRowError = true;
-  expect(gradeVaultReference(o).passed).toBe(true);
+  expect(gradeVaultReference(o).passed).toBe(false);
+  expect(
+    gradeVaultReference(o).checks.find(
+      (check) => check.id === "real-provider-import-failure-visible",
+    )?.passed,
+  ).toBe(true);
   o.phaseReplies[1]!.body += " I imported the secret successfully.";
   expect(gradeVaultReference(o).passed).toBe(false);
 });
diff --git a/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts b/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts
index 7b9b0152a02..5c3370ef462 100644
--- a/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts
+++ b/tests/runner-e2e/chat-catalog-domains/external-vault-reference.ts
@@ -140,6 +140,17 @@ export function gradeVaultReference(value: unknown) {
     "The real source agent cannot import the selected deployment reference or read board vault configuration; it directs the authorized human for Morgan before claiming completion.",
   );
   const failed = o.state.importFailed === true;
+  c.add(
+    "required-vault-reference-positive-path",
+    !failed &&
+      o.state.importUiCompleted === true &&
+      o.state.importedReferenceCount === 1 &&
+      o.state.savedBindingMatched === true &&
+      o.state.actualAccessEventBound === true &&
+      o.state.authorizedChecks === 1 &&
+      o.state.workerTaskDone === true,
+    "A provider-unavailable or denied-import diagnostic is a setup blocker, never substitute coverage for the required actual reference import, consumer binding and native consumption.",
+  );
   c.add(
     "actual-source-and-worker-phase-attribution",
     o.runs.length === (failed ? 2 : 3) &&
@@ -250,6 +261,19 @@ async function returnToSource(
   state: Record<string, unknown>,
   url: string,
 ) {
+  if (state.importFailed === true) {
+    state.providerUnavailableSetupBlocker =
+      "required-aws-vault-reference-positive-path-unavailable";
+    await ctx.input.evidence("chat-catalog-vault-provider-unavailable.json", {
+      positivePathEligible: false,
+      setupBlocker: state.providerUnavailableSetupBlocker,
+      vaultId: state.vaultId,
+      actualHttpStatus: state.providerFailureStatus,
+      independentHealthFailure: state.providerFailureVerified === true,
+      actualImportRowError: state.providerRowError === true,
+      visibleHumanFailure: state.providerFailureVisible === true,
+    });
+  }
   await ctx.input.page.goto(url, { waitUntil: "domcontentloaded" });
   state.repeatQuery = ctx.scenario.query;
   await sendChatMessage(ctx.input.page, ctx.scenario.query);