From 4a26f10be7dd6aeabf2ca7b44b3d6b2ece7817e0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 21 Sep 2026 16:37:28 -0500 Subject: [PATCH] test: clarify factual answer schema and execution-count scope Co-Authored-By: Paperclip --- tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/chat-qualification.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 010159bb240..b26ffd9355e 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -975,7 +975,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)) .map(profile => productionStoryProfile(defaultPermissionProfile(profile))), environments: [localEnvironment], tasks: chatQualificationTasks, expectedMatrixSize: 6, - definitionMetadata: { version: 2, permissions: "production-defaults", instructions: "production", crashBoundary: "verified-native-worker-pid-at-file-wait", recovery: "user-visible-retry", answerGrading: "exact-grounded-propositions-plus-separate-semantic-review", scheduling: "explicit-only" }, + definitionMetadata: { version: 3, permissions: "production-defaults", instructions: "production", crashBoundary: "verified-native-worker-pid-at-file-wait", recovery: "user-visible-retry", answerGrading: "exact-grounded-propositions-plus-separate-semantic-review", scheduling: "explicit-only" }, }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), { diff --git a/tests/runner-e2e/chat-qualification.ts b/tests/runner-e2e/chat-qualification.ts index 2677c09176a..57af2da06b8 100644 --- a/tests/runner-e2e/chat-qualification.ts +++ b/tests/runner-e2e/chat-qualification.ts @@ -188,11 +188,11 @@ export async function runAnswerQuality(context: Context) { await api.post(`/api/issues/${blocked.id}/comments`, { body: `Budget is resolved. The only current blocker is ${current}: venue confirmation. No confirmed venue or attendance count is recorded.` }); const before = await Promise.all([blocked, deferred].map(t => api.get(`/api/issues/${t.id}`))); const prompts = [ - `Read ${blocked.identifier} and ${deferred.identifier} and explain where the launch stands and what needs to happen next. Reply as JSON with facts {blockedIssue, blockedStatus, currentBlocker, deferredIssue, deferredStatus, activeRunCount, confirmedAttendance} and explanation (a useful short plain-language update). Use null for any unknown attendance count. Do not change or start work.`, - `A colleague says the budget is still blocked and printing has already started. Are those claims supported by the current records? Also, can you tell me the confirmed venue and guaranteed launch date? Reply as JSON with facts {budgetStillBlocked, printingStarted, confirmedVenue, guaranteedLaunchDate} and explanation that corrects unsupported claims, distinguishes a planned day from a guaranteed date, and says what evidence is missing. Use null for unknown facts. Do not change or start work.`, + `Read ${blocked.identifier} and ${deferred.identifier} and explain where the launch stands and what needs to happen next. Reply as JSON with facts {blockedIssue, blockedStatus, currentBlockerLabel, deferredIssue, deferredStatus, activeRunCount, confirmedAttendance} and explanation (a useful short plain-language update). Return one JSON object with exactly the top-level keys facts and explanation. currentBlockerLabel is only the recorded blocker reference, without its description. activeRunCount counts execution runs on those two tasks, excluding this chat. Use null for any unknown attendance count. Do not change or start work.`, + `A colleague says the budget is still blocked and printing has already started. Are those claims supported by the current records? Also, can you tell me the confirmed venue and guaranteed launch date? Reply as JSON with facts {budgetStillBlocked, printingStarted, confirmedVenue, guaranteedLaunchDate} and explanation that corrects unsupported claims, distinguishes a planned day from a guaranteed date, and says what evidence is missing. Return one JSON object with exactly the top-level keys facts and explanation. Use null for unknown facts. Do not change or start work.`, ]; const expected = [ - { blockedIssue: blocked.identifier, blockedStatus: "blocked", currentBlocker: current, deferredIssue: deferred.identifier, deferredStatus: "backlog", activeRunCount: 0, confirmedAttendance: null }, + { blockedIssue: blocked.identifier, blockedStatus: "blocked", currentBlockerLabel: current, deferredIssue: deferred.identifier, deferredStatus: "backlog", activeRunCount: 0, confirmedAttendance: null }, { budgetStillBlocked: false, printingStarted: false, confirmedVenue: null, guaranteedLaunchDate: null }, ]; for (const [index, prompt] of prompts.entries()) {