mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-14 17:22:00 +01:00
DottaandClaude Opus 5.5
086af6dd8b
Agent Chat is always on since #19360, so "Enable Agent Chat on this instance." has no setting to change and the scripted human's "Toggle agent chat experimental setting" switch no longer exists. The case keeps its promise with an experiment that still exists and has a real route gate: Cases (enableCases), whose /<prefix>/cases route redirects to the dashboard while off. - YAML: new query/context/expected outcome for Cases; inventory digest, manifest rows and changedFields updated and pinned. - Setup starts Cases off (restored in cleanup), refuses a hidden setting, and probes GET/PATCH experimental settings with the agent's own key (403 board_access_required, setting unchanged). - The human checks Cases is still off after the agent's turn and its route is unavailable, then follows the Experimental settings link the reply rendered (no typed-URL fallback), turns Cases on, sees it persist after reload, and opens the Cases page. - The reply-wording regex is replaced by structural checks: a rendered link to Experimental settings that the person followed, and saved setting state. - oracleVersion privileged.instance-settings.v2; sourceFiles follow the code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
40 lines
3.0 KiB
TypeScript
40 lines
3.0 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { validateChatCatalogGrade } from "./chat-catalog-contract.js";
|
|
import { chatCatalogScenarios, chatCatalogInventoryDigest } from "./chat-catalog-inventory.js";
|
|
import { runnerMatrix } from "./catalog.js";
|
|
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
|
|
import { chatCatalogCases } from "./chat-catalog-registry.js";
|
|
|
|
describe("catalog oracle contract", () => {
|
|
it("accepts an independently checked result and rejects missing or contradictory evidence", () => {
|
|
const checks = [{ id: "saved-owner", passed: true, detail: "Persisted assignee matches the requested agent" }];
|
|
expect(validateChatCatalogGrade({ passed: true, checks }).passed).toBe(true);
|
|
expect(validateChatCatalogGrade({ passed: false, checks: [{ ...checks[0]!, passed: false }] }).passed).toBe(false);
|
|
for (const value of [{ passed: true, checks: [] }, { passed: true, checks: [{ ...checks[0]!, passed: false }] },
|
|
{ passed: true, checks: [...checks, ...checks] }]) expect(() => validateChatCatalogGrade(value)).toThrow();
|
|
});
|
|
it("discovers each concrete implementation explicitly with its exact query and bounds", () => {
|
|
for (const implementation of chatCatalogCases) {
|
|
const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && cell.task.id === implementation.scenarioId);
|
|
expect(cells).toHaveLength(1);
|
|
const cell = cells[0]!;
|
|
expect(cell.task.buildPrompt("nonce")).toBe(chatCatalogScenarios.find(row => row.id === implementation.scenarioId)!.query);
|
|
expect(cell.task.automaticRetryPolicy).toBe("single_attempt");
|
|
expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(row => row.id)).toEqual([cell.id]);
|
|
expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.id === cell.id)).toBe(false);
|
|
}
|
|
});
|
|
it("keeps every frozen scenario in the denominator independently of executable readiness", () => {
|
|
expect(chatCatalogScenarios).toHaveLength(244);
|
|
expect(new Set(chatCatalogScenarios.map(row => row.feature)).size).toBe(35);
|
|
expect(new Set(chatCatalogScenarios.flatMap(row => row.capabilities)).size).toBe(191);
|
|
// agent-chat.chat-disabled retargeted to the Conference Room gate (Agent Chat is always on since #19360);
|
|
// agents.restore-instructions retargeted to no saved history (instruction saves keep none since #14420).
|
|
// access.manage-membership retargeted to read-and-route; activity.routine-audit builds real same-morning history.
|
|
// gateways-profiles.create-gateway and rotate-token: agents keep delegated gateway/token authority and never receive token values.
|
|
// instance-operations.instance-settings retargeted from the removed Agent Chat toggle (always on since #19360) to the Cases experiment.
|
|
expect(chatCatalogInventoryDigest).toBe("a39eb193d579c18be6e5f6fae8061428c9f7996549cdca6537aa31baecfdd03e");
|
|
for (const implementation of chatCatalogCases) expect(chatCatalogScenarios.some(row => row.id === implementation.scenarioId)).toBe(true);
|
|
});
|
|
});
|