mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-26 18:31:34 +01:00
fix(evals): fail closed without maintained campaign
This commit is contained in:
1 parent
c3adbe751c
commit
653510aab4
2 files changed
+24
-7
No files matched your search
@@ -79,13 +79,7 @@ function parseRosterSelection(value) {
|
||||
|
||||
async function maintainedRosterSelection(programRoot) {
|
||||
const campaignPath = resolve(programRoot, "campaigns/live-direct-full.json");
|
||||
let campaign;
|
||||
try {
|
||||
campaign = await loadObject(campaignPath);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return null;
|
||||
throw error;
|
||||
}
|
||||
const campaign = await loadObject(campaignPath);
|
||||
if (
|
||||
campaign.schema !== "paperclip-runner/live-campaign/v1" ||
|
||||
!Array.isArray(campaign.lanes)
|
||||
|
||||
@@ -187,6 +187,29 @@ test("all selects the maintained enabled campaign and explicit diagnostics can s
|
||||
);
|
||||
});
|
||||
|
||||
test("all fails closed when the maintained campaign is missing", async () => {
|
||||
const { root, program } = await fixture();
|
||||
await rm(join(program, "campaigns/live-direct-full.json"));
|
||||
|
||||
await assert.rejects(
|
||||
buildProtocolEvalCatalog({
|
||||
evalsRoot: root,
|
||||
campaignId: "gha-42-1",
|
||||
}),
|
||||
/ENOENT/,
|
||||
);
|
||||
|
||||
const diagnostic = await buildProtocolEvalCatalog({
|
||||
evalsRoot: root,
|
||||
campaignId: "gha-42-2",
|
||||
rosterSelection: "protocol-live-opencode-model",
|
||||
});
|
||||
assert.deepEqual(
|
||||
diagnostic.rosters.map((roster) => roster.rosterId),
|
||||
["protocol-live-opencode-model"],
|
||||
);
|
||||
});
|
||||
|
||||
test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => {
|
||||
const { root, config, evalCase } = await fixture();
|
||||
const catalog = await buildProtocolEvalCatalog({
|
||||
|
||||
Reference in new issue
Block a user