PaperClipAI/tests/runner-e2e/budgeted-setup.test.ts

import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { expect, it, vi } from "vitest";
import type { RunnerApi } from "./api.js";
import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js";
import { createKimiLifecycleTiming, finishBudgetedEvalSetup, type EvalSetupObservation } from "./budgeted-setup.js";

import { runKimiLifecycleFlow } from "./kimi-lifecycle.js";
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import { runnerE2EPlaywrightCaseTimeoutMs, runnerE2EPlaywrightWatchdogMs } from "./web-server-command.js";

const identity = { executionId: "fixture-execution", attemptId: "fixture-attempt" };
const ready = { id: "agent", companyId: "company", status: "idle", lifecycleState: "ready", lifecycleError: null };
async function fixture() {
  const directory = await mkdtemp(path.join(tmpdir(), "budgeted-setup-"));
  const file = path.join(directory, "phase.json");
  await createEvalAttemptPhase(file, identity);
  return { directory, file, env: { PAPERCLIP_E2E_GATEWAY_BASE_URL: "http://127.0.0.1:12345",
    PAPERCLIP_E2E_GATEWAY_PHASE_PATH: file, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId } };
}

it("keeps asynchronous hello probes in setup until the same agent is ready", async () => {
  const f = await fixture();
  const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying", adapterConfig: { private: true } }).mockResolvedValue(ready);
  let saved: EvalSetupObservation[] = [];
  try {
    await finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company",
      deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi,
      observe: async observations => {
        expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
        saved = structuredClone(observations);
      } }, f.env);
    expect(saved.map(row => row.lifecycleState)).toEqual(["verifying", "ready"]);
    expect(saved[0]).not.toHaveProperty("adapterConfig");
    expect(get).toHaveBeenCalledTimes(2);
    expect(get.mock.calls.every(([url, options]) => url === "/api/agents/agent" && options.timeout > 0 && options.timeout <= 5_000)).toBe(true);
    expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("agent");
  } finally { await rm(f.directory, { recursive: true, force: true }); }
});

it.each([
  [{ ...ready, lifecycleError: "harness_command_unavailable" }, "setup failed"],
  [{ ...ready, lifecycleHolds: ["approval"] }, "setup is held"],
  [{ ...ready, companyId: "other" }, "scope changed"],
  [{ ...ready, lifecycleState: "terminated" }, "setup stopped"],
] as const)("preserves setup attribution and the failed state when readiness is rejected: %s", async (state, reason) => {
  const f = await fixture(), observe = vi.fn(async () => {}), get = vi.fn().mockResolvedValue(state);
  try {
    await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company",
      deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, observe }, f.env)).rejects.toThrow(reason);
    expect(get).toHaveBeenCalledOnce();
    expect(observe).toHaveBeenCalledOnce();
    expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
  } finally { await rm(f.directory, { recursive: true, force: true }); }
});

it("does not start a readiness request after the attempt deadline", async () => {
  const f = await fixture(), get = vi.fn();
  try {
    await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company",
      deadlineAt: Date.now() - 1, api: { get } as unknown as RunnerApi, observe: async () => {} }, f.env)).rejects.toThrow("Timed out");
    expect(get).not.toHaveBeenCalled();
    expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
  } finally { await rm(f.directory, { recursive: true, force: true }); }
});

it("does not advance when saving readiness crosses the deadline", async () => {
  const f = await fixture(), get = vi.fn().mockResolvedValue(ready);
  const deadlineAt = Date.now() + 5_000;
  const now = vi.spyOn(Date, "now");
  try {
    await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company",
      deadlineAt, api: { get } as unknown as RunnerApi,
      observe: async () => { now.mockReturnValue(deadlineAt); } }, f.env)).rejects.toThrow("Timed out saving");
    expect(get).toHaveBeenCalledOnce();
    expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
  } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); }
});

it.each(["local", "daytona"] as const)("uses the bounded attempt window for remote setup while keeping the local limit: %s", async environmentId => {
  const f = await fixture();
  let time = Date.now();
  const now = vi.spyOn(Date, "now").mockImplementation(() => time);
  const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying" }).mockResolvedValue(ready);
  let observations = 0;
  try {
    const result = finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId, agentId: "agent", companyId: "company",
      deadlineAt: time + 15 * 60_000, api: { get } as unknown as RunnerApi,
      observe: async () => { if (++observations === 1) time += 46_000; } }, f.env);
    if (environmentId === "daytona") {
      await result;
      expect(get).toHaveBeenCalledTimes(2);
      expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("agent");
    } else {
      await expect(result).rejects.toThrow("Timed out");
      expect(get).toHaveBeenCalledOnce();
      expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup");
    }
  } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); }
});

it("waits for the same agent to be ready before changing Kimi lifecycle policy without a gateway", async () => {
  const execution = selectRunnerExecutions(parseRunnerSelectors(["--id", "kimi-qualification.runner-acpx-kimi.daytona.native-write-provider-loss"]))[0]!;
  const order: string[] = [], observations: EvalSetupObservation[][] = [];
  const configured = { ...ready, adapterConfig: { model: "kimi-for-coding" } };
  const get = vi.fn(async (url: string) => {
    if (url === "/api/health") return { deploymentMode: "local_trusted" };
    if (url === "/api/auth/get-session") return { session: { userId: "local-board", id: "paperclip:local_implicit:local-board" } };
    if (url === "/api/agents/agent") {
      order.push(order.length === 0 ? "verifying" : order.length === 1 ? "ready" : "policy-read");
      return order.length === 1 ? { ...configured, lifecycleState: "verifying" } : configured;
    }
    throw new Error(`Unexpected GET ${url}`);
  });
  const patch = vi.fn(async () => { order.push("policy-patch"); throw new Error("stop-after-policy-patch"); });
  await expect(runKimiLifecycleFlow({ page: {} as any, api: { get, patch } as unknown as RunnerApi,
    fixtures: { company: { id: "company" }, agent: { id: "agent" }, environment: { id: "environment" } } as LiveFixtureValues,
    execution, nonce: "unit", workspacePath: "/unused", deadlineAt: Date.now() + 5_000, remoteBootstrap: {} as any,
    registerCleanupAssertion: () => {}, registerBeforeEnvironmentTeardownAssertion: () => {}, observe: () => {}, capture: async () => {},
    evidence: async (name, value) => { if (name === "kimi-lifecycle-agent-ready.json") observations.push(structuredClone((value as { observations: EvalSetupObservation[] }).observations)); },
  })).rejects.toThrow("stop-after-policy-patch");
  expect(order).toEqual(["verifying", "ready", "policy-read", "policy-patch"]);
  expect(observations.at(-1)?.map(row => row.lifecycleState)).toEqual(["verifying", "ready"]);
  expect(observations.flat()).not.toContainEqual(expect.objectContaining({ adapterConfig: expect.anything() }));
  expect(patch).toHaveBeenCalledOnce();
});

it("does not change Kimi lifecycle policy after readiness fails", async () => {
  const execution = selectRunnerExecutions(parseRunnerSelectors(["--id", "kimi-qualification.runner-acpx-kimi.daytona.native-write-provider-loss"]))[0]!;
  const get = vi.fn(async (url: string) => url === "/api/health" ? { deploymentMode: "local_trusted" }
    : url === "/api/auth/get-session" ? { session: { userId: "local-board", id: "paperclip:local_implicit:local-board" } }
      : { ...ready, lifecycleError: "harness_command_unavailable" });
  const patch = vi.fn();
  await expect(runKimiLifecycleFlow({ page: {} as any, api: { get, patch } as unknown as RunnerApi,
    fixtures: { company: { id: "company" }, agent: { id: "agent" }, environment: { id: "environment" } } as LiveFixtureValues,
    execution, nonce: "unit", workspacePath: "/unused", deadlineAt: Date.now() + 5_000, remoteBootstrap: {} as any,
    registerCleanupAssertion: () => {}, registerBeforeEnvironmentTeardownAssertion: () => {}, observe: () => {}, capture: async () => {}, evidence: async () => {},
  })).rejects.toThrow("setup failed");
  expect(patch).not.toHaveBeenCalled();
});

it("keeps a finite setup cap and one full graded Kimi window after the recorded 242s readiness delay", () => {
  const startedAt = 1_000_000, deadlineMs = 300_000;
  const timing = createKimiLifecycleTiming({ startedAt, deadlineMs });
  expect(timing.setupDeadlineAt).toBe(startedAt + 300_000);
  expect(timing.playwrightTimeoutMs).toBe(690_000);
  expect(timing.cleanupDeadlineAt).toBe(startedAt + 660_000);
  expect(startedAt + timing.playwrightTimeoutMs - timing.cleanupDeadlineAt).toBe(30_000);
  const readyAt = startedAt + 242_000;
  const workDeadlineAt = timing.beginWork(readyAt);
  expect(workDeadlineAt - readyAt).toBe(300_000);
  expect(workDeadlineAt + 90_000 - startedAt).toBeLessThanOrEqual(timing.playwrightTimeoutMs);
  expect(() => timing.beginWork(readyAt + 1_000)).toThrow("already started");
  for (const elapsed of [300_000, 301_000]) {
    expect(() => createKimiLifecycleTiming({ startedAt, deadlineMs }).beginWork(startedAt + elapsed)).toThrow("setup deadline expired");
  }
});

it.each([60_000, 300_000])("lets Kimi cleanup and its final report finish after %sms of server startup", async startupMs => {
  vi.useFakeTimers();
  try {
    const execution = selectRunnerExecutions(parseRunnerSelectors(["--id", "kimi-qualification.runner-acpx-kimi.daytona.native-write-provider-loss"]))[0]!;
    const deadlineMs = execution.task.attemptTimeoutMs[execution.environment.id];
    const spawnedAt = Date.now();
    let killed = false, cleanupFinished = false, reportSaved = false;
    const watchdog = setTimeout(() => { killed = true; }, runnerE2EPlaywrightWatchdogMs([execution]));
    await vi.advanceTimersByTimeAsync(startupMs);
    const timing = createKimiLifecycleTiming({ startedAt: Date.now(), deadlineMs });
    setTimeout(() => { if (!killed) cleanupFinished = true; }, timing.cleanupDeadlineAt - Date.now() - 1);
    setTimeout(() => { if (!killed && cleanupFinished) reportSaved = true; }, timing.playwrightTimeoutMs - 1);
    await vi.advanceTimersByTimeAsync(timing.playwrightTimeoutMs - 1);
    expect({ killed, cleanupFinished, reportSaved }).toEqual({ killed: false, cleanupFinished: true, reportSaved: true });
    expect(runnerE2EPlaywrightCaseTimeoutMs(execution.task.flow, deadlineMs)).toBe(timing.playwrightTimeoutMs);
    expect(Date.now() - spawnedAt).toBe(startupMs + 690_000 - 1);
    await vi.advanceTimersByTimeAsync(300_000 - startupMs + 1);
    expect(killed).toBe(true);
    clearTimeout(watchdog);
  } finally { vi.clearAllTimers(); vi.useRealTimers(); }
});

it("keeps other launcher case and startup allowances unchanged", () => {
  const executions = selectRunnerExecutions(parseRunnerSelectors(["--suite", "kimi-qualification"]));
  const others = executions.filter(execution => execution.task.flow !== "kimi_native_lifecycle");
  expect(others.length).toBeGreaterThan(0);
  for (const execution of others) {
    const deadlineMs = execution.task.attemptTimeoutMs[execution.environment.id];
    expect(runnerE2EPlaywrightCaseTimeoutMs(execution.task.flow, deadlineMs)).toBe(deadlineMs + 90_000);
    expect(runnerE2EPlaywrightWatchdogMs([execution])).toBe(deadlineMs + 90_000 + 300_000);
  }
  expect(runnerE2EPlaywrightWatchdogMs(executions)).toBe(executions.reduce((total, execution) => total +
    (execution.task.flow === "kimi_native_lifecycle" ? 2 : 1) * execution.task.attemptTimeoutMs[execution.environment.id] + 90_000, 300_000));
});