diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index be61693fb07..53dfa5b570f 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -440,8 +440,11 @@ attempt. Installing the ACP package alone does not satisfy this setup path. Bounded cases keep the gateway in the setup phase until the created agent's asynchronous lifecycle verification reaches ready. This wait uses the remaining -attempt deadline, allows at most 45 seconds, and stops on an observed setup error +attempt deadline, allows at most 45 seconds locally and the bounded attempt window +for Daytona provisioning, and stops on an observed setup error or hold. The private `budgeted-setup.json` snapshot retains state transitions. +Budgeted Daytona fixtures give the saved-account hello probe five minutes inside +that attempt window; its former 90-second default could expire during cold startup. Only then does the case begin agent-phase accounting and create its task. Older evidence that advanced after fixture creation may label hello probes as agent cost. Its campaign totals still include those requests; retain that phase diff --git a/tests/runner-e2e/budgeted-setup.test.ts b/tests/runner-e2e/budgeted-setup.test.ts index f7ddaf2c5b0..c1c7e4dacb7 100644 --- a/tests/runner-e2e/budgeted-setup.test.ts +++ b/tests/runner-e2e/budgeted-setup.test.ts @@ -21,7 +21,7 @@ it("keeps asynchronous hello probes in setup until the same agent is ready", asy const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying", adapterConfig: { private: true } }).mockResolvedValue(ready); let saved: EvalSetupObservation[] = []; try { - await finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + await finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company", deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, observe: async observations => { expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); @@ -43,7 +43,7 @@ it.each([ ] as const)("preserves setup attribution and the failed state when readiness is rejected: %s", async (state, reason) => { const f = await fixture(), observe = vi.fn(async () => {}), get = vi.fn().mockResolvedValue(state); try { - await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company", deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, observe }, f.env)).rejects.toThrow(reason); expect(get).toHaveBeenCalledOnce(); expect(observe).toHaveBeenCalledOnce(); @@ -54,7 +54,7 @@ it.each([ it("does not start a readiness request after the attempt deadline", async () => { const f = await fixture(), get = vi.fn(); try { - await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company", deadlineAt: Date.now() - 1, api: { get } as unknown as RunnerApi, observe: async () => {} }, f.env)).rejects.toThrow("Timed out"); expect(get).not.toHaveBeenCalled(); expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); @@ -66,10 +66,32 @@ it("does not advance when saving readiness crosses the deadline", async () => { const deadlineAt = Date.now() + 5_000; const now = vi.spyOn(Date, "now"); try { - await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId: "local", agentId: "agent", companyId: "company", deadlineAt, api: { get } as unknown as RunnerApi, observe: async () => { now.mockReturnValue(deadlineAt); } }, f.env)).rejects.toThrow("Timed out saving"); expect(get).toHaveBeenCalledOnce(); expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } }); + +it.each(["local", "daytona"] as const)("uses the bounded attempt window for remote setup while keeping the local limit: %s", async environmentId => { + const f = await fixture(); + let time = Date.now(); + const now = vi.spyOn(Date, "now").mockImplementation(() => time); + const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying" }).mockResolvedValue(ready); + let observations = 0; + try { + const result = finishBudgetedEvalSetup({ executionId: identity.executionId, environmentId, agentId: "agent", companyId: "company", + deadlineAt: time + 15 * 60_000, api: { get } as unknown as RunnerApi, + observe: async () => { if (++observations === 1) time += 46_000; } }, f.env); + if (environmentId === "daytona") { + await result; + expect(get).toHaveBeenCalledTimes(2); + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("agent"); + } else { + await expect(result).rejects.toThrow("Timed out"); + expect(get).toHaveBeenCalledOnce(); + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); + } + } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/budgeted-setup.ts b/tests/runner-e2e/budgeted-setup.ts index 42dc216ea50..ee69773da95 100644 --- a/tests/runner-e2e/budgeted-setup.ts +++ b/tests/runner-e2e/budgeted-setup.ts @@ -1,3 +1,4 @@ +import type { RunnerEnvironmentId } from "./types.js"; import type { Agent } from "../../packages/shared/src/types/agent.js"; import { pollUntil, type RunnerApi } from "./api.js"; import { beginEvalAgentPhase } from "./attempt-phase.js"; @@ -7,6 +8,7 @@ export type EvalSetupObservation = SetupState & { observedAt: string }; export async function finishBudgetedEvalSetup(input: { executionId: string; + environmentId: RunnerEnvironmentId; agentId: string; companyId: string; deadlineAt: number; @@ -14,7 +16,7 @@ export async function finishBudgetedEvalSetup(input: { observe: (observations: EvalSetupObservation[]) => Promise; }, env: NodeJS.ProcessEnv = process.env) { if (env.PAPERCLIP_E2E_GATEWAY_BASE_URL) { - const deadlineAt = Math.min(input.deadlineAt, Date.now() + 45_000); + const deadlineAt = input.environmentId === "daytona" ? input.deadlineAt : Math.min(input.deadlineAt, Date.now() + 45_000); const observations: EvalSetupObservation[] = []; let previous: string | undefined; const scoped = (agent: SetupState) => agent.id === input.agentId && agent.companyId === input.companyId; diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 517b25b3d90..29cdb4ff214 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -385,6 +385,7 @@ export async function setupLiveFixtures(input: { if (managedHiring || savedCopilot || budgetedGateway) { const account = value(resolved, "ai-connection"); const config = agent.adapterConfig as Record; + if (budgetedGateway && execution.environment.id === "daytona") config.helloProbeTimeoutSec = 300; delete config.env; agent.runtimeConfig = { ...(agent.runtimeConfig as Record), diff --git a/tests/runner-e2e/operations.test.ts b/tests/runner-e2e/operations.test.ts index 8e03148dfdc..74cff67c76f 100644 --- a/tests/runner-e2e/operations.test.ts +++ b/tests/runner-e2e/operations.test.ts @@ -164,16 +164,21 @@ it.each([ cell.id, "feedback-persistence.runner-codex.local.complaint-and-suggestion", "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", + "core-compatibility.runner-acpx-claude.daytona.message-marker", ])("routes %s through a shared connection without ambient provider bindings", async executionId => { const selected = runnerMatrix.find(execution => execution.id === executionId)!; const anthropic = selected.profile.credential === "ANTHROPIC_API_KEY"; - const admitted = manifest(); admitted.executionIds = [selected.id]; + const remote = selected.environment.id === "daytona"; + const origin = remote ? "https://fixture.trycloudflare.com" : "http://127.0.0.1:43123"; + const admitted = manifest(); + if (remote) admitted.remoteGateway = "cloudflare"; admitted.executionIds = [selected.id]; admitted.models[0]!.provider = anthropic ? "anthropic" : "openai"; admitted.models[0]!.price.provider = admitted.models[0]!.provider; admitted.models[0]!.price.model = selected.profile.model; if (anthropic) Object.assign(admitted.models[0]!.price, { cacheWrite5mUsdPerMillion: "2", cacheWrite1hUsdPerMillion: "2" }); expect(validateEvalOperationsManifest(admitted, [selected], 0)).toBe(admitted); - vi.stubEnv("PAPERCLIP_E2E_GATEWAY_BASE_URL", "http://127.0.0.1:43123"); + vi.stubEnv("PAPERCLIP_E2E_GATEWAY_BASE_URL", origin); + if (remote) vi.stubEnv("PAPERCLIP_E2E_REMOTE_GATEWAY_ORIGIN", origin); vi.stubEnv("PAPERCLIP_E2E_GATEWAY_TOKEN", "ephemeral-fixture-token"); const id = "11111111-1111-4111-8111-111111111111", grantId = "22222222-2222-4222-8222-222222222222"; let connection: unknown, agent: Record | undefined, company: Record | undefined; @@ -181,12 +186,15 @@ it.each([ async get() { return [{ id, driver: "local" }]; }, async postSensitive(url: string, body: unknown) { if (url.endsWith("/ai-connections")) { connection = body; return { connectionId: id, grantId }; } return { id }; }, async post(url: string, body: Record) { if (url.endsWith("/agents")) agent = body; if (url === "/api/companies") company = body; return { id, name: "fixture" }; }, + async delete() {}, } as unknown as RunnerApi; - const fixtures = await setupLiveFixtures({ api, execution: selected, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { [selected.profile.credential]: "ephemeral-fixture-token" } }); + const fixtures = await setupLiveFixtures({ api, execution: selected, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { [selected.profile.credential]: "ephemeral-fixture-token", ...(remote ? { DAYTONA_API_KEY: "fixture-daytona" } : {}) }, + ...(remote ? { daytonaImage: `ghcr.io/paperclip/image@sha256:${"a".repeat(64)}` } : {}) }); await fixtures.teardown(); - expect(createAiConnectionSchema.parse(connection)).toMatchObject({ ownership: "shared", apiKey: "ephemeral-fixture-token", routing: { kind: "gateway", protocol: anthropic ? "messages" : "responses", baseUrl: `http://127.0.0.1:43123${anthropic ? "" : "/v1"}`, auth: "bearer" } }); + expect(createAiConnectionSchema.parse(connection)).toMatchObject({ ownership: "shared", apiKey: "ephemeral-fixture-token", routing: { kind: "gateway", protocol: anthropic ? "messages" : "responses", baseUrl: `${origin}${anthropic ? "" : "/v1"}`, auth: "bearer" } }); expect(aiConnectionBindingSchema.parse(agent!.runtimeConfig.aiConnection)).toEqual({ provider: anthropic ? "anthropic" : "openai", method: "api_key", mode: "shared", connectionId: id, grantId }); expect(agent!.adapterConfig.env).toBeUndefined(); + expect(agent!.adapterConfig.helloProbeTimeoutSec).toBe(remote ? 300 : undefined); if (selected.suite.id === "feedback-persistence") { expect(company!.budgetMonthlyCents).toBe(1_000); expect(agent!.budgetMonthlyCents).toBe(1_000); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 8836e27dfe8..1584725be40 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -957,7 +957,7 @@ for (const execution of executions) { daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, onCompanyCreated, }); - await finishBudgetedEvalSetup({ executionId: execution.id, agentId: fixtures.agent.id, + await finishBudgetedEvalSetup({ executionId: execution.id, environmentId: execution.environment.id, agentId: fixtures.agent.id, companyId: fixtures.company.id, deadlineAt: startedAtMs + deadlineMs, api, observe: observations => writeSanitizedJson(snapshotsDir, "budgeted-setup.json", { observations }, secrets) }); await workerTiming.enter("execution");