test(evals): keep CLI guidance inside the original clock

Pass the already established case deadline into the two CLI guidance waits and validate it before and after observation. Preserve command grammar, scoped CLI actor and saved-state oracles.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/privileged-cli-observer.test.ts

@@ -0,0 +1,68 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { observeInternal } from "./internal-support.js";
+import { runOwnedCli } from "./privileged-cli-support.js";
+import { privilegedCliCases } from "./privileged-cli.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() }));
+vi.mock("./internal-support.js", async original => ({
+  ...await original<typeof import("./internal-support.js")>(), observeInternal: vi.fn(),
+}));
+vi.mock("./privileged-cli-support.js", async original => ({
+  ...await original<typeof import("./privileged-cli-support.js")>(), runOwnedCli: vi.fn(),
+}));
+afterEach(() => { vi.useRealTimers(); vi.restoreAllMocks(); vi.clearAllMocks(); });
+const start = 50_000;
+function fixture(scenario: string) {
+  vi.useFakeTimers(); vi.setSystemTime(start);
+  let completesAt = start + 160_000, status = "succeeded", noWork = false;
+  const ctx = { observationDeadlineAt: start + 390_000,
+    issue: () => ({ conversationState: Date.now() >= completesAt ? "waiting" : "running" }),
+    input: { execution: { id: `chat-features.runner-opencode.local.${scenario}` },
+      fixtures: { company: { id: "owned-company" } }, api: { baseURL: "http://fixture.test" } },
+  } as unknown as ChatCatalogContext;
+  vi.mocked(observeInternal).mockImplementation(async () => ({
+    tasks: [], runs: noWork ? [] : [{ status: Date.now() >= completesAt ? status : "running" }],
+    reply: Date.now() < completesAt ? "" : scenario.endsWith("choose-cli-context")
+      ? "```bash\npaperclipai context use acorn\npaperclipai context show --json\n```"
+      : '```bash\npaperclipai issue create --company-id <COMPANY_ID> --title "Website task" --project-id <PROJECT_ID> --assignee-agent-id <MORGAN_AGENT_ID>\n```',
+  }) as never);
+  vi.mocked(runOwnedCli).mockRejectedValue(new Error("Reached actual owned CLI boundary"));
+  const d = privilegedCliCases.find(row => row.scenarioId === scenario)!;
+  const act = () => Promise.resolve(d.act!(ctx, { project: { id: "launch" }, worker: { id: "morgan" } }));
+  return { ctx, act, completesAt: (value: number) => { completesAt = value; }, status: (value: string) => { status = value; }, noWork: () => { noWork = true; } };
+}
+for (const scenario of ["cli-operations.choose-cli-context", "cli-operations.create-from-cli"]) {
+  it(`${scenario}: reaches the genuine CLI boundary after160s only within its original clock`, async () => {
+    const f = fixture(scenario); const pending = f.act().then(() => null, error => error as Error);
+    await vi.advanceTimersByTimeAsync(160_000);
+    expect((await pending)?.message).toBe("Reached actual owned CLI boundary");
+    expect(runOwnedCli).toHaveBeenCalledOnce(); expect(f.ctx.observationDeadlineAt).toBe(start + 390_000);
+  });
+  it.each([undefined, NaN, Infinity, start, start - 1])(`${scenario}: rejects missing/invalid/expired deadline %s before observing or executing`, async deadline => {
+    const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: deadline });
+    await expect(f.act()).rejects.toThrow(/Original privileged observation budget/);
+    expect(observeInternal).not.toHaveBeenCalled(); expect(runOwnedCli).not.toHaveBeenCalled();
+  });
+  it.each(["failed", "cancelled", "timed_out"])(`${scenario}: rejects stopped/failed native work %s`, async status => {
+    const f = fixture(scenario); f.completesAt(start); f.status(status);
+    await expect(f.act()).rejects.toThrow(/Canonical guidance run failed/);
+    expect(runOwnedCli).not.toHaveBeenCalled();
+  });
+  it(`${scenario}: a still-active or no-work turn cannot run CLI after its original deadline`, async () => {
+    for (const noWork of [false, true]) {
+      const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 });
+      f.completesAt(start + 2000); if (noWork) { f.completesAt(start); f.noWork(); }
+      const pending = f.act().then(() => null, error => error as Error);
+      await vi.advanceTimersByTimeAsync(4000);
+      expect((await pending)?.message).toMatch(/Timed out waiting/); expect(runOwnedCli).not.toHaveBeenCalled();
+    }
+  });
+  it(`${scenario}: late observation cannot cross the CLI boundary`, async () => {
+    const f = fixture(scenario); Object.assign(f.ctx, { observationDeadlineAt: start + 1000 }); f.completesAt(start);
+    const original = vi.mocked(observeInternal).getMockImplementation()!;
+    vi.mocked(observeInternal).mockImplementation(async (...args) => { vi.setSystemTime(start + 1001); return original(...args); });
+    const pending = f.act().then(() => null, error => error as Error);
+    await vi.advanceTimersByTimeAsync(4000);
+    expect((await pending)?.message).toMatch(/Timed out waiting/); expect(runOwnedCli).not.toHaveBeenCalled();
+  });
+}

tests/runner-e2e/chat-catalog-domains/privileged-cli.test.ts

@@ -1,7 +1,8 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import { websiteCreateArgv, validContextGuidance } from "./privileged-cli-support.js";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import { privilegedCliCases, gradeChooseCliContext, gradeCreateFromCli } from "./privileged-cli.js";
+vi.mock("./internal-definition.js", () => ({ fixtureWorker: vi.fn() }));
 const target = { company: "owned-company", project: "launch", agent: "morgan", apiBase: "http://127.0.0.1:3100" };
 const command = (text: string) => `Use the explicit owned context.\n\`\`\`bash\n${text}\n\`\`\``;
 const valid = 'paperclipai issue create --company-id <COMPANY_ID> --title "Website task" --project-id <PROJECT_ID> --assignee-agent-id <MORGAN_AGENT_ID>';

tests/runner-e2e/chat-catalog-domains/privileged-cli.ts

@@ -5,7 +5,7 @@ import type { RuntimeAgent } from "./runtime-support.js";
 import type { ChatCatalogCase } from "../chat-catalog-contract.js";
 import { checks, project, entity, type Entity } from "./internal-support.js";
 import { fixtureWorker } from "./internal-definition.js";
-import { privilegedDefinition, privilegedObservation, settlePrivilegedChat } from "./privileged-support.js";
+import { privilegedDefinition, privilegedObservation, settlePrivilegedChat, remainingPrivilegedObservationBudget } from "./privileged-support.js";
 import { prepareOwnedCli, runOwnedCli, cleanupOwnedCli, websiteCreateArgv, validContextGuidance, contextGuidanceArgv } from "./privileged-cli-support.js";
 
 export function gradeCreateFromCli(value: unknown) {
@@ -38,7 +38,9 @@ const create = privilegedDefinition("cli-operations.create-from-cli", "privilege
     return { ...operator, project: launch, worker: paused, workerWasManuallyPaused: true };
   },
   async act(ctx, state) {
-    const reply = await settlePrivilegedChat(ctx);
+    remainingPrivilegedObservationBudget(ctx);
+    const reply = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt);
+    remainingPrivilegedObservationBudget(ctx);
     state.tasksBeforeCli = reply.tasks.length;
     const fixture = { company: ctx.input.fixtures.company.id, project: entity(state, "project").id,
       agent: entity(state, "worker").id, apiBase: ctx.input.api.baseURL };
@@ -115,7 +117,9 @@ const context = privilegedDefinition("cli-operations.choose-cli-context", "privi
     return { ...operator, initialProfile: before.currentProfile, otherProfile: before.profiles.birch };
   },
   async act(ctx, state) {
-    const reply = await settlePrivilegedChat(ctx);
+    remainingPrivilegedObservationBudget(ctx);
+    const reply = await settlePrivilegedChat(ctx, ctx.observationDeadlineAt);
+    remainingPrivilegedObservationBudget(ctx);
     const returned = contextGuidanceArgv(reply.reply, { company: ctx.input.fixtures.company.id, apiBase: ctx.input.api.baseURL });
     if (!returned) throw new Error("Actual model context commands are unsupported or do not select and read back the intended profile");
     for (const argv of returned.commands) await runOwnedCli(ctx, argv);