Close OAuth admission abuse paths and verify API and eval contracts

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-01 10:27:00 -05:00
1 parent 826de9e6c2
commit fb7946a4b8
13 files changed
+1096 -17

No files matched your search

+1 -1
View File
@@ -1102,7 +1102,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
extraConfig: { engine: "cli", maxTurnsPerRun: 16 },
})),
],
definitionMetadata: { version: 4, grader: publicMcpGraderVersion, workflowDigest: publicMcpWorkflowDigest, workerSkillDigest: publicMcpWorkerSkillDigest, workerInstructions: publicMcpWorkerInstructions, assistantMaxRequests: 16, assistantMaxEstimatedUsd: 2, instructions: "shipped-plugin-skills", scheduling: "explicit-only" },
definitionMetadata: { version: 5, grader: publicMcpGraderVersion, workflowDigest: publicMcpWorkflowDigest, workerSkillDigest: publicMcpWorkerSkillDigest, workerInstructions: publicMcpWorkerInstructions, assistantMaxRequests: 16, assistantMaxEstimatedUsd: 2, instructions: "shipped-plugin-skills", scheduling: "explicit-only" },
},
{
id: "blocker-guidance", label: "Direct blocker handling", manualOnly: true,
+11
View File
@@ -69,4 +69,15 @@ describe("external assistant evidence and billing", () => {
expect(snapshots).toEqual([{ calls: 0, requests: 1 }, { calls: 1, requests: 1 }, { calls: 1, requests: 1 }]);
expect(usage.estimatedCostUsd).toBeGreaterThan(0);
});
it("shares the 16-request ceiling across fresh conversations in one cell", async () => {
const fetchMock = vi.fn().mockImplementation(async () => Response.json({ model: "gpt-5.4-mini", usage: { input_tokens: 10, output_tokens: 10 }, output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }] }));
vi.stubGlobal("fetch", fetchMock);
const usage = assistantUsage("openai", "gpt-5.4-mini");
const input = { usage, credential: "unit-test-key", prompt: "Fresh conversation", tools: [], call: vi.fn(), deadlineAt: Date.now() + 10_000, observe: vi.fn() };
for (let i = 0; i < 16; i++) await runAssistant(input);
await expect(runAssistant(input)).rejects.toThrow("shared 16-request per-cell budget");
expect(fetchMock).toHaveBeenCalledTimes(16);
expect(usage.requests).toBe(16);
});
});
+1
View File
@@ -50,6 +50,7 @@ export async function runAssistant(input: {
const responseItems: any[] = [{ role: "user", content: input.prompt }];
try {
for (let step = 0; step < 16; step++) {
if (input.usage.requests >= 16) throw new Error("Assistant exceeded its shared 16-request per-cell budget");
if (Date.now() >= input.deadlineAt) throw new Error("Assistant workflow exceeded its bounded deadline");
const anthropic = input.usage.provider === "anthropic";
const response = await fetch(anthropic ? "https://api.anthropic.com/v1/messages" : "https://api.openai.com/v1/responses", {