From dd6632cc93732cc00cddd015c5410720f6cbbac6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Fri, 6 Nov 2026 14:48:57 -0600 Subject: [PATCH] test(evals): treat in-tree scenario YAML as the current catalog definition With the harness on master, the scenario YAML holds the repaired current values the runtime reads. The manifest keeps each previous value as history. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs index 754fd1b4eb4..39e3e57e64a 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs @@ -25,7 +25,8 @@ test('the manifest joins all244 source YAML scenarios without importing a domain const override = field => manifest.source.repairMetadata.changedFields.find(change => change.key === scenario.id && change.field === field); for (const [field, original] of [['originalQuery', scenario.query], ['originalContext', scenario.context], ['expectedOutcome', scenario.expected_outcome]]) { const change = override(field); - if (change) { assert.equal(change.previous, original); assert.equal(actual[field], change.current); } + // The scenario YAML in this tree is the current definition; the manifest keeps the previous value as history. + if (change) { assert.notEqual(change.previous, change.current); assert.equal(original, change.current); assert.equal(actual[field], change.current); } else assert.equal(actual[field], original); } assert.ok(!Object.hasOwn(actual, 'status') && !Object.hasOwn(actual, 'passed'));