diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts new file mode 100644 index 00000000000..e1cbd89f908 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts @@ -0,0 +1,68 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { observeInternal } from "./internal-support.js"; +import { runOwnedCli } from "./privileged-cli-support.js"; +import { privilegedCliCases } from "./privileged-cli.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() })); +vi.mock("./internal-support.js", async original => ({ + ...await original(), observeInternal: vi.fn(), +})); +vi.mock("./privileged-cli-support.js", async original => ({ + ...await original(), runOwnedCli: vi.fn(), +})); +afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); }); +const start = 50_000; +function fixture(scenario: string) { + vi.useFakeTimers(); vi.setSystemTime(start); + let completesAt = start + 160_000, status = "succeeded", noWork = false; + const ctx = { observationDeadlineAt: start + 390_000, + issue: () => ({ conversationState: Date.now() >= completesAt ? "waiting" : "running" }), + input: { execution: { id: `chat-features.runner-opencode.local.${scenario}` }, + fixtures: { company: { id: "owned-company" } }, api: { baseURL: "http://fixture.test" } }, + } as unknown as ChatCatalogContext; + vi.mocked(observeInternal).mockImplementation(async () => ({ + tasks: [], runs: noWork ? [] : [{ status: Date.now() >= completesAt ? status : "running" }], + reply: Date.now() < completesAt ? "" : scenario.endsWith("choose-cli-context") + ? "```bash\npaperclipai context use acorn\npaperclipai context show --json\n```" + : '```bash\npaperclipai issue create --company-id --title "Website task" --project-id --assignee-agent-id \n```', + }) as never); + vi.mocked(runOwnedCli).mockRejectedValue(new Error("Reached actual owned CLI boundary")); + const d = privilegedCliCases.find(row => row.scenarioId === scenario)!; + const act = () => Promise.resolve(d.act!(ctx, { project: { id: "launch" }, worker: { id: "morgan" } })); + return { ctx, act, completesAt: (value: number) => { completesAt = value; }, status: (value: string) => { status = value; }, noWork: () => { noWork = true; } }; +} +for (const scenario of ["cli-operations.choose-cli-context", "cli-operations.create-from-cli"]) { + it(`${scenario}: reaches the genuine CLI boundary after160s only within its original clock`, async () => { + const f = fixture(scenario); const pending = f.act().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(160_000); + expect((await pending)?.message).toBe("Reached actual owned CLI boundary"); + expect(runOwnedCli).toHaveBeenCalledOnce(); expect(f.ctx.observationDeadlineAt).toBe(start + 390_000); + }); + it.each([undefined, NaN, Infinity, start, start - 1])(`${scenario}: rejects missing/invalid/expired deadline %s before observing or executing`, async deadline => { + const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: deadline }); + await expect(f.act()).rejects.toThrow(/Original privileged observation budget/); + expect(observeInternal).not.toHaveBeenCalled(); expect(runOwnedCli).not.toHaveBeenCalled(); + }); + it.each(["failed", "cancelled", "timed_out"])(`${scenario}: rejects stopped/failed native work %s`, async status => { + const f = fixture(scenario); f.completesAt(start); f.status(status); + await expect(f.act()).rejects.toThrow(/Canonical guidance run failed/); + expect(runOwnedCli).not.toHaveBeenCalled(); + }); + it(`${scenario}: a still-active or no-work turn cannot run CLI after its original deadline`, async () => { + for (const noWork of [false, true]) { + const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); + f.completesAt(start + 2000); if (noWork) { f.completesAt(start); f.noWork(); } + const pending = f.act().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(4000); + expect((await pending)?.message).toMatch(/Timed out waiting/); expect(runOwnedCli).not.toHaveBeenCalled(); + } + }); + it(`${scenario}: late observation cannot cross the CLI boundary`, async () => { + const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); f.completesAt(start); + const original = vi.mocked(observeInternal).getMockImplementation()!; + vi.mocked(observeInternal).mockImplementation(async (...args) => { vi.setSystemTime(start + 1001); return original(...args); }); + const pending = f.act().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(4000); + expect((await pending)?.message).toMatch(/Timed out waiting/); expect(runOwnedCli).not.toHaveBeenCalled(); + }); +} diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts index 9ef5dd5efed..8efc917bb89 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts @@ -1,7 +1,8 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { websiteCreateArgv, validContextGuidance } from "./privileged-cli-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { privilegedCliCases, gradeChooseCliContext, gradeCreateFromCli } from "./privileged-cli.js"; +vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() })); const target = { company: "owned-company", project: "launch", agent: "morgan", apiBase: "http://127.0.0.1:3100" }; const command = (text: string) => `Use the explicit owned context.\n\`\`\`bash\n${text}\n\`\`\``; const valid = 'paperclipai issue create --company-id --title "Website task" --project-id --assignee-agent-id '; diff --git a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts index 21bb87c00de..4d3ae80b555 100644 --- a/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts +++ b/tests/runner-e2e/chat-catalog-domains/privileged-cli.ts @@ -5,7 +5,7 @@ import type { RuntimeAgent } from "./runtime-support.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { checks, project, entity, type Entity } from "./internal-support.js"; import { fixtureWorker } from "./internal-definition.js"; -import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js"; +import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, remainingPrivilegedObservationBudget } from "./privileged-support.js"; import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli, websiteCreateArgv, validContextGuidance, contextGuidanceArgv } from "./privileged-cli-support.js"; export function gradeCreateFromCli(value: unknown) { @@ -38,7 +38,9 @@ const create = privilegedDefinition("cli-operations.create-from-cli", "privilege return { ...operator, project: launch, worker: paused, workerWasManuallyPaused: true }; }, async act(ctx, state) { - const reply = await settlePrivilegedChat(ctx); + remainingPrivilegedObservationBudget(ctx); + const reply = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); + remainingPrivilegedObservationBudget(ctx); state.tasksBeforeCli = reply.tasks.length; const fixture = { company: ctx.input.fixtures.company.id, project: entity(state, "project").id, agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL }; @@ -115,7 +117,9 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch }; }, async act(ctx, state) { - const reply = await settlePrivilegedChat(ctx); + remainingPrivilegedObservationBudget(ctx); + const reply = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt); + remainingPrivilegedObservationBudget(ctx); const returned = contextGuidanceArgv(reply.reply, { company: ctx.input.fixtures.company.id, apiBase: ctx.input.api.baseURL }); if (!returned) throw new Error("Actual model context commands are unsupported or do not select and read back the intended profile"); for (const argv of returned.commands) await runOwnedCli(ctx, argv);