From fce64fe2f2173e8001379e8ec870a9c5e9b43108 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 19:57:51 -0600 Subject: [PATCH] test(evals): keep CLI guidance inside the original clock Pass the already established case deadline into the two CLI guidance waits and validate it before and after observation. Preserve command grammar, scoped CLI actor and saved-state oracles. Co-Authored-By: Paperclip --- .../privileged-cli-observer.test.ts | 68 +++++++++++++++++++ .../privileged-cli.test.ts | 3 +- .../chat-catalog-domains/privileged-cli.ts | 10 ++- 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts new file mode 100644 index 00000000000..e1cbd89f908 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts @@ -0,0 +1,68 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { observeInternal } from "./internal-support.js"; +import { runOwnedCli } from "./privileged-cli-support.js"; +import { privilegedCliCases } from "./privileged-cli.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() })); +vi.mock("./internal-support.js", async original => ({ + ...await original(), observeInternal: vi.fn(), +})); +vi.mock("./privileged-cli-support.js", async original => ({ + ...await original(), runOwnedCli: vi.fn(), +})); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); +const start = 50_000; +function fixture(scenario: string) { + vi.useFakeTimers(); vi.setSystemTime(start); + let completesAt = start + 160_000, status = "succeeded", noWork = false; + const ctx = { observationDeadlineAt: start + 390_000, + issue: () => ({ conversationState: Date.now() >= completesAt ? "waiting" : "running" }), + input: { execution: { id: `chat-features.runner-opencode.local.${scenario}` }, + fixtures: { company: { id: "owned-company" } }, api: { baseURL: "http://fixture.test" } }, + } as unknown as ChatCatalogContext; + vi.mocked(observeInternal).mockImplementation(async () => ({ + tasks: [], runs: noWork ? [] : [{ status: Date.now() >= completesAt ? status : "running" }], + reply: Date.now() < completesAt ? "" : scenario.endsWith("choose-cli-context") + ? "```bash\npaperclipai context use acorn\npaperclipai context show --json\n```" + : '```bash\npaperclipai issue create --company-id --title "Website task" --project-id --assignee-agent-id \n```', + }) as never); + vi.mocked(runOwnedCli).mockRejectedValue(new Error("Reached actual owned CLI boundary")); + const d = privilegedCliCases.find(row => row.scenarioId === scenario)!; + const act = () => Promise.resolve(d.act!(ctx, { project: { id: "launch" }, worker: { id: "morgan" } })); + return { ctx, act, completesAt: (value: number) => { completesAt = value; }, status: (value: string) => { status = value; }, noWork: () => { noWork = true; } }; +} +for (const scenario of ["cli-operations.choose-cli-context", "cli-operations.create-from-cli"]) { + it(`${scenario}: reaches the genuine CLI boundary after160s only within its original clock`, async () => { + const f = fixture(scenario); const pending = f.act().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(160_000); + expect((await pending)?.message).toBe("Reached actual owned CLI boundary"); + expect(runOwnedCli).toHaveBeenCalledOnce(); expect(f.ctx.observationDeadlineAt).toBe(start + 390_000); + }); + it.each([undefined, NaN, Infinity, start, start - 1])(`${scenario}: rejects missing/invalid/expired deadline %s before observing or executing`, async deadline => { + const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: deadline }); + await expect(f.act()).rejects.toThrow(/Original privileged observation budget/); + expect(observeInternal).not.toHaveBeenCalled(); expect(runOwnedCli).not.toHaveBeenCalled(); + }); + it.each(["failed", "cancelled", "timed_out"])(`${scenario}: rejects stopped/failed native work %s`, async status => { + const f = fixture(scenario); f.completesAt(start); f.status(status); + await expect(f.act()).rejects.toThrow(/Canonical guidance run failed/); + expect(runOwnedCli).not.toHaveBeenCalled(); + }); + it(`${scenario}: a still-active or no-work turn cannot run CLI after its original deadline`, async () => { + for (const noWork of [false, true]) { + const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); + f.completesAt(start + 2000); if (noWork) { f.completesAt(start); f.noWork(); } + const pending = f.act().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(4000); + expect((await pending)?.message).toMatch(/Timed out waiting/); expect(runOwnedCli).not.toHaveBeenCalled(); + } + }); + it(`${scenario}: late observation cannot cross the CLI boundary`, async () => { + const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); f.completesAt(start); + const original = vi.mocked(observeInternal).getMockImplementation()!; + vi.mocked(observeInternal).mockImplementation(async (...args) => { vi.setSystemTime(start + 1001); return original(...args); }); + const pending = f.act().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(4000); + expect((await pending)?.message).toMatch(/Timed out waiting/); expect(runOwnedCli).not.toHaveBeenCalled(); + }); +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts index 9ef5dd5efed..8efc917bb89 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts @@ -1,7 +1,8 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { websiteCreateArgv, validContextGuidance } from "./privileged-cli-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { privilegedCliCases, gradeChooseCliContext, gradeCreateFromCli } from "./privileged-cli.js"; +vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() })); const target = { company: "owned-company", project: "launch", agent: "morgan", apiBase: "http://127.0.0.1:3100" }; const command = (text: string) => `Use the explicit owned context.\n\`\`\`bash\n${text}\n\`\`\``; const valid = 'paperclipai issue create --company-id --title "Website task" --project-id --assignee-agent-id '; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts index 21bb87c00de..4d3ae80b555 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts @@ -5,7 +5,7 @@ import type { RuntimeAgent } from "./runtime-support.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { checks, project, entity, type Entity } from "./internal-support.js"; import { fixtureWorker } from "./internal-definition.js"; -import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, remainingPrivilegedObservationBudget } from "./privileged-support.js"; import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli, websiteCreateArgv, validContextGuidance, contextGuidanceArgv } from "./privileged-cli-support.js"; export function gradeCreateFromCli(value: unknown) { @@ -38,7 +38,9 @@ const create = privilegedDefinition("cli-operations.create-from-cli", "privilege return { ...operator, project: launch, worker: paused, workerWasManuallyPaused: true }; }, async act(ctx, state) { - const reply = await settlePrivilegedChat(ctx); + remainingPrivilegedObservationBudget(ctx); + const reply = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); + remainingPrivilegedObservationBudget(ctx); state.tasksBeforeCli = reply.tasks.length; const fixture = { company: ctx.input.fixtures.company.id, project: entity(state, "project").id, agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL }; @@ -115,7 +117,9 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch }; }, async act(ctx, state) { - const reply = await settlePrivilegedChat(ctx); + remainingPrivilegedObservationBudget(ctx); + const reply = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); + remainingPrivilegedObservationBudget(ctx); const returned = contextGuidanceArgv(reply.reply, { company: ctx.input.fixtures.company.id, apiBase: ctx.input.api.baseURL }); if (!returned) throw new Error("Actual model context commands are unsupported or do not select and read back the intended profile"); for (const argv of returned.commands) await runOwnedCli(ctx, argv);