Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-21 08:22:20 +02:00
Code Issues Packages Projects Releases Wiki Activity

test: clarify factual answer schema and execution-count scope

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-21 16:37:28 -05:00
1 parent 6039b02ed9
commit 4a26f10be7
2 files changed
+4 -4
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/catalog.ts
+1 -1
View File
@@ -975,7 +975,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id))
.map(profile => productionStoryProfile(defaultPermissionProfile(profile))),
environments: [localEnvironment], tasks: chatQualificationTasks, expectedMatrixSize: 6,
definitionMetadata: { version: 2, permissions: "production-defaults", instructions: "production", crashBoundary: "verified-native-worker-pid-at-file-wait", recovery: "user-visible-retry", answerGrading: "exact-grounded-propositions-plus-separate-semantic-review", scheduling: "explicit-only" },
definitionMetadata: { version: 3, permissions: "production-defaults", instructions: "production", crashBoundary: "verified-native-worker-pid-at-file-wait", recovery: "user-visible-retry", answerGrading: "exact-grounded-propositions-plus-separate-semantic-review", scheduling: "explicit-only" },
},
...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []),
{
tests/runner-e2e/chat-qualification.ts
+3 -3
View File
@@ -188,11 +188,11 @@ export async function runAnswerQuality(context: Context) {
await api.post(`/api/issues/${blocked.id}/comments`, { body: `Budget is resolved. The only current blocker is ${current}: venue confirmation. No confirmed venue or attendance count is recorded.` });
const before = await Promise.all([blocked, deferred].map(t => api.get<ChatIssue>(`/api/issues/${t.id}`)));
const prompts = [
`Read ${blocked.identifier} and ${deferred.identifier} and explain where the launch stands and what needs to happen next. Reply as JSON with facts {blockedIssue, blockedStatus, currentBlocker, deferredIssue, deferredStatus, activeRunCount, confirmedAttendance} and explanation (a useful short plain-language update). Use null for any unknown attendance count. Do not change or start work.`,
`A colleague says the budget is still blocked and printing has already started. Are those claims supported by the current records? Also, can you tell me the confirmed venue and guaranteed launch date? Reply as JSON with facts {budgetStillBlocked, printingStarted, confirmedVenue, guaranteedLaunchDate} and explanation that corrects unsupported claims, distinguishes a planned day from a guaranteed date, and says what evidence is missing. Use null for unknown facts. Do not change or start work.`,
`Read ${blocked.identifier} and ${deferred.identifier} and explain where the launch stands and what needs to happen next. Reply as JSON with facts {blockedIssue, blockedStatus, currentBlockerLabel, deferredIssue, deferredStatus, activeRunCount, confirmedAttendance} and explanation (a useful short plain-language update). Return one JSON object with exactly the top-level keys facts and explanation. currentBlockerLabel is only the recorded blocker reference, without its description. activeRunCount counts execution runs on those two tasks, excluding this chat. Use null for any unknown attendance count. Do not change or start work.`,
`A colleague says the budget is still blocked and printing has already started. Are those claims supported by the current records? Also, can you tell me the confirmed venue and guaranteed launch date? Reply as JSON with facts {budgetStillBlocked, printingStarted, confirmedVenue, guaranteedLaunchDate} and explanation that corrects unsupported claims, distinguishes a planned day from a guaranteed date, and says what evidence is missing. Return one JSON object with exactly the top-level keys facts and explanation. Use null for unknown facts. Do not change or start work.`,
];
const expected = [
{ blockedIssue: blocked.identifier, blockedStatus: "blocked", currentBlocker: current, deferredIssue: deferred.identifier, deferredStatus: "backlog", activeRunCount: 0, confirmedAttendance: null },
{ blockedIssue: blocked.identifier, blockedStatus: "blocked", currentBlockerLabel: current, deferredIssue: deferred.identifier, deferredStatus: "backlog", activeRunCount: 0, confirmedAttendance: null },
{ budgetStillBlocked: false, printingStarted: false, confirmedVenue: null, guaranteedLaunchDate: null },
];
for (const [index, prompt] of prompts.entries()) {