mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-13 15:55:46 +01:00
DottaandClaude Opus 5.5
fa50b8d662
The fixture saved a built-in fake sandbox as its unsupported control. The product has refused to save fake sandboxes since #4449 ("reserved for internal probes"), so setup returned 422 before any agent run, and the test API withheld the reason. No saveable computer is advertised as unable to run Claude Code, so the case now contrasts advertised support with runtime health: the fixture saves an SSH computer on a reserved .invalid host that is advertised for Claude Code but never tested. The yaml context records this premise change (manifest changedFields pinned). The grader no longer reads reply keywords. It requires the agent boundary (403 on capabilities and probe), a rendered reply link to the environment settings page that the person then clicks to see both computers and the checked capabilities, no board-only SSH settings in the reply, unchanged targets and worker, and exactly one chat turn. Cookie-session API failures with 400/409/422 now report the product's own validation message, bounded and with request values and the session redacted. Details and other statuses stay withheld. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
39 lines
2.9 KiB
TypeScript
39 lines
2.9 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { validateChatCatalogGrade } from "./chat-catalog-contract.js";
|
|
import { chatCatalogScenarios, chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
|
|
import { runnerMatrix } from "./catalog.js";
|
|
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
|
|
import { chatCatalogCases } from "./chat-catalog-registry.js";
|
|
|
|
describe("catalog oracle contract", () => {
|
|
it("accepts an independently checked result and rejects missing or contradictory evidence", () => {
|
|
const checks = [{ id: "saved-owner", passed: true, detail: "Persisted assignee matches the requested agent" }];
|
|
expect(validateChatCatalogGrade({ passed: true, checks }).passed).toBe(true);
|
|
expect(validateChatCatalogGrade({ passed: false, checks: [{ ...checks[0]!, passed: false }] }).passed).toBe(false);
|
|
for (const value of [{ passed: true, checks: [] }, { passed: true, checks: [{ ...checks[0]!, passed: false }] },
|
|
{ passed: true, checks: [...checks, ...checks] }]) expect(() => validateChatCatalogGrade(value)).toThrow();
|
|
});
|
|
it("discovers each concrete implementation explicitly with its exact query and bounds", () => {
|
|
for (const implementation of chatCatalogCases) {
|
|
const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && cell.task.id === implementation.scenarioId);
|
|
expect(cells).toHaveLength(1);
|
|
const cell = cells[0]!;
|
|
expect(cell.task.buildPrompt("nonce")).toBe(chatCatalogScenarios.find(row => row.id === implementation.scenarioId)!.query);
|
|
expect(cell.task.automaticRetryPolicy).toBe("single_attempt");
|
|
expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(row => row.id)).toEqual([cell.id]);
|
|
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.id === cell.id)).toBe(false);
|
|
}
|
|
});
|
|
it("keeps every frozen scenario in the denominator independently of executable readiness", () => {
|
|
expect(chatCatalogScenarios).toHaveLength(244);
|
|
expect(new Set(chatCatalogScenarios.map(row => row.feature)).size).toBe(35);
|
|
expect(new Set(chatCatalogScenarios.flatMap(row => row.capabilities)).size).toBe(191);
|
|
// agent-chat.chat-disabled retargeted to the Conference Room gate (Agent Chat is always on since #19360);
|
|
// agents.restore-instructions retargeted to no saved history (instruction saves keep none since #14420).
|
|
// access.manage-membership retargeted to read-and-route; activity.routine-audit builds real same-morning history.
|
|
// gateways-profiles.create-gateway and rotate-token: agents keep delegated gateway/token authority and never receive token values.
|
|
expect(chatCatalogInventoryDigest).toBe("bb91452e7dfa40050bad20485c09a03e9c4ccb4befc243fc66d24c57abe13557");
|
|
for (const implementation of chatCatalogCases) expect(chatCatalogScenarios.some(row => row.id === implementation.scenarioId)).toBe(true);
|
|
});
|
|
});
|