diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 3db1036b4f0..2943c23efe6 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1193,10 +1193,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ description: "Real user requests, useful downloaded work, and durable continuation using production instructions.", groups: ["native"], profiles: [...everydayProfiles, productionStoryProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!), - ].map(profile => ({ ...profile, buildAgent(input) { - const agent = profile.buildAgent(input); - return /\.(hire-reuse|delegate-feedback)$/.test(input.executionId) ? { ...agent, budgetMonthlyCents: 1_000 } : agent; - } })), environments: [localEnvironment, daytonaWarmEnvironment], + ], environments: [localEnvironment, daytonaWarmEnvironment], tasks: everydayTasks.map(task => ["hire-reuse", "delegate-feedback"].includes(task.id) ? { ...task, automaticRetryPolicy: "single_attempt" as const } : task), expectedMatrixSize: 52, excludedExecutionIds: [...everydayProfiles.flatMap(profile => everydayTasks @@ -1207,7 +1204,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]), ]), ], - definitionMetadata: { version: 5, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + definitionMetadata: { version: 6, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } }, }, { diff --git a/tests/runner-e2e/live-fixtures.test.ts b/tests/runner-e2e/live-fixtures.test.ts index 04be71d847f..a2c231dc3a3 100644 --- a/tests/runner-e2e/live-fixtures.test.ts +++ b/tests/runner-e2e/live-fixtures.test.ts @@ -4,6 +4,29 @@ import { runnerMatrix } from "./catalog.js"; import { setupLiveFixtures } from "./live-fixtures.js"; describe("live runner fixtures", () => { + it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile => + ["hire-reuse", "delegate-feedback"].map(task => [profile, task]))) ( + "applies %s %s budget stops in actual setup with a nonce and only its selected credential", async (profile, task) => { + const execution = runnerMatrix.find(row => row.id === `everyday-workflows.${profile}.local.${task}`)!; + let agentBody: any; + let companyBody: any; + const api = { + async get() { return [{ id: "local", driver: "local" }]; }, + async postSensitive(url: string) { return url.endsWith("/ai-connections") ? { connectionId: "account" } : { id: "secret" }; }, + async post(url: string, data: any) { + if (url === "/api/companies") { companyBody = data; return { id: "company", name: "Test" }; } + if (url.endsWith("/agents")) { agentBody = data; return { id: "lead", ...data }; } + throw new Error(`Unexpected POST ${url}`); + }, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "random-nonce", workspacePath: "/tmp/test", + credentials: { [execution.profile.credential]: "test-value" } }); + expect(companyBody.budgetMonthlyCents).toBe(1_000); + expect(agentBody.budgetMonthlyCents).toBe(1_000); + await fixtures.teardown(); + }, + ); + it.each(["runner-codex", "legacy-codex", "runner-acpx-claude", "legacy-opencode"])( "creates a production-default %s hire with company and agent budget stops", async (profile) => { const execution = runnerMatrix.find(row => row.suite.id === "stock-harness" && row.profile.id === profile)!; @@ -51,6 +74,7 @@ describe("live runner fixtures", () => { ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"], ["runner-codex", "everyday-workflows", "hire-reuse"], ["runner-acpx-claude", "everyday-workflows", "hire-reuse"], + ["runner-opencode", "everyday-workflows", "hire-reuse"], ["runner-codex", "agent-chat-hardening", "hire-delegate-reuse"], ["runner-acpx-claude", "agent-chat-hardening", "hire-delegate-reuse"], ])( @@ -64,7 +88,7 @@ describe("live runner fixtures", () => { e.environment.id === "local", )!; const provider = - profile === "runner-acpx-claude" ? "anthropic" : "openai"; + profile === "runner-acpx-claude" ? "anthropic" : profile === "runner-opencode" ? "openrouter" : "openai"; let connected = false; let agentBody: any; const api = { @@ -101,8 +125,7 @@ describe("live runner fixtures", () => { executionNonce: "nonce", workspacePath: "/tmp/test", credentials: { - OPENAI_API_KEY: "test-value", - ANTHROPIC_API_KEY: "test-value", + [execution.profile.credential]: "test-value", }, }); expect(connected).toBe(true); diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 94fd9506b80..96ccc5025e7 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -39,7 +39,7 @@ interface AgentRecord { interface ManagedAccountFixture { connectionId: string; binding: { - provider: "openai" | "anthropic"; + provider: "openai" | "anthropic" | "openrouter"; method: "api_key"; mode: "responsible_user"; }; @@ -245,10 +245,11 @@ export async function setupLiveFixtures(input: { dependencies: ["company"], async setup(resolved) { const company = value(resolved, "company"); - const provider = - execution.profile.provider === "acpx" ? "anthropic" : "openai"; - const key = - provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; + const key = execution.profile.credential; + const provider = key === "ANTHROPIC_API_KEY" ? "anthropic" + : key === "OPENROUTER_API_KEY" ? "openrouter" + : key === "OPENAI_API_KEY" ? "openai" : null; + if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`); const apiKey = input.credentials[key]; if (!apiKey) throw new Error(`Missing credential ${key}`); const account = await api.postSensitive<{ connectionId: string }>( @@ -291,7 +292,10 @@ export async function setupLiveFixtures(input: { secretRefs, executionId: input.executionNonce, }); - if (execution.suite.id === "stock-harness") agent.budgetMonthlyCents = 1_000; + if (execution.suite.id === "stock-harness" + || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { + agent.budgetMonthlyCents = 1_000; + } if (managedHiring) { const account = value(resolved, "ai-connection"); const config = agent.adapterConfig as Record; diff --git a/tests/runner-e2e/native-procedure-guidance.test.ts b/tests/runner-e2e/native-procedure-guidance.test.ts index 6aae7de272b..f84db225a10 100644 --- a/tests/runner-e2e/native-procedure-guidance.test.ts +++ b/tests/runner-e2e/native-procedure-guidance.test.ts @@ -19,13 +19,12 @@ describe("bounded native procedure comparison", () => { } expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(cell => cell.suite.id === suite.id)).toBe(false); }); - it("sets each lead's hard stop without fixture completion instructions or provider overrides", () => { + it("keeps procedure and completion directions out of the fixture persona", () => { for (const profile of suite.profiles) { - const input = { executionId: `everyday-workflows.${profile.id}.local.hire-reuse`, environmentId: "env", environmentFixtureId: "local" as const, workspacePath: "/workspace", secretRefs: { + const input = { executionId: "random-nonce", environmentId: "env", environmentFixtureId: "local" as const, workspacePath: "/workspace", secretRefs: { [profile.credential]: { type: "secret_ref" as const, secretId: "00000000-0000-4000-8000-000000000001", version: "latest" as const }, } }; const agent = profile.buildAgent(input); - expect(agent.budgetMonthlyCents).toBe(1_000); expect(JSON.stringify(agent.instructionsBundle)).not.toMatch(/paperclip_finish|paperclip_block|set_dependencies|hire_agent/); } });