test(evals): fix managed OpenCode auth and nonce-independent budget setup

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/catalog.ts

@@ -1193,10 +1193,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
     description: "Real user requests, useful downloaded work, and durable continuation using production instructions.",
     groups: ["native"], profiles: [...everydayProfiles,
       productionStoryProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!),
-    ].map(profile => ({ ...profile, buildAgent(input) {
-      const agent = profile.buildAgent(input);
-      return /\.(hire-reuse|delegate-feedback)$/.test(input.executionId) ? { ...agent, budgetMonthlyCents: 1_000 } : agent;
-    } })), environments: [localEnvironment, daytonaWarmEnvironment],
+    ], environments: [localEnvironment, daytonaWarmEnvironment],
     tasks: everydayTasks.map(task => ["hire-reuse", "delegate-feedback"].includes(task.id)
       ? { ...task, automaticRetryPolicy: "single_attempt" as const } : task), expectedMatrixSize: 52,
     excludedExecutionIds: [...everydayProfiles.flatMap(profile => everydayTasks
@@ -1207,7 +1204,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
         ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]),
       ]),
     ],
-    definitionMetadata: { version: 5, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only",
+    definitionMetadata: { version: 6, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only",
       procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } },
   },
   {

tests/runner-e2e/live-fixtures.test.ts

@@ -4,6 +4,29 @@ import { runnerMatrix } from "./catalog.js";
 import { setupLiveFixtures } from "./live-fixtures.js";
 
 describe("live runner fixtures", () => {
+  it.each(["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile =>
+    ["hire-reuse", "delegate-feedback"].map(task => [profile, task]))) (
+    "applies %s %s budget stops in actual setup with a nonce and only its selected credential", async (profile, task) => {
+      const execution = runnerMatrix.find(row => row.id === `everyday-workflows.${profile}.local.${task}`)!;
+      let agentBody: any;
+      let companyBody: any;
+      const api = {
+        async get() { return [{ id: "local", driver: "local" }]; },
+        async postSensitive(url: string) { return url.endsWith("/ai-connections") ? { connectionId: "account" } : { id: "secret" }; },
+        async post(url: string, data: any) {
+          if (url === "/api/companies") { companyBody = data; return { id: "company", name: "Test" }; }
+          if (url.endsWith("/agents")) { agentBody = data; return { id: "lead", ...data }; }
+          throw new Error(`Unexpected POST ${url}`);
+        },
+      } as unknown as RunnerApi;
+      const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "random-nonce", workspacePath: "/tmp/test",
+        credentials: { [execution.profile.credential]: "test-value" } });
+      expect(companyBody.budgetMonthlyCents).toBe(1_000);
+      expect(agentBody.budgetMonthlyCents).toBe(1_000);
+      await fixtures.teardown();
+    },
+  );
+
   it.each(["runner-codex", "legacy-codex", "runner-acpx-claude", "legacy-opencode"])(
     "creates a production-default %s hire with company and agent budget stops", async (profile) => {
       const execution = runnerMatrix.find(row => row.suite.id === "stock-harness" && row.profile.id === profile)!;
@@ -51,6 +74,7 @@ describe("live runner fixtures", () => {
     ["runner-acpx-claude", "hiring-templates", "hire-coder-template-reuse"],
     ["runner-codex", "everyday-workflows", "hire-reuse"],
     ["runner-acpx-claude", "everyday-workflows", "hire-reuse"],
+    ["runner-opencode", "everyday-workflows", "hire-reuse"],
     ["runner-codex", "agent-chat-hardening", "hire-delegate-reuse"],
     ["runner-acpx-claude", "agent-chat-hardening", "hire-delegate-reuse"],
   ])(
@@ -64,7 +88,7 @@ describe("live runner fixtures", () => {
           e.environment.id === "local",
       )!;
       const provider =
-        profile === "runner-acpx-claude" ? "anthropic" : "openai";
+        profile === "runner-acpx-claude" ? "anthropic" : profile === "runner-opencode" ? "openrouter" : "openai";
       let connected = false;
       let agentBody: any;
       const api = {
@@ -101,8 +125,7 @@ describe("live runner fixtures", () => {
         executionNonce: "nonce",
         workspacePath: "/tmp/test",
         credentials: {
-          OPENAI_API_KEY: "test-value",
-          ANTHROPIC_API_KEY: "test-value",
+          [execution.profile.credential]: "test-value",
         },
       });
       expect(connected).toBe(true);

tests/runner-e2e/live-fixtures.ts

@@ -39,7 +39,7 @@ interface AgentRecord {
 interface ManagedAccountFixture {
   connectionId: string;
   binding: {
-    provider: "openai" | "anthropic";
+    provider: "openai" | "anthropic" | "openrouter";
     method: "api_key";
     mode: "responsible_user";
   };
@@ -245,10 +245,11 @@ export async function setupLiveFixtures(input: {
       dependencies: ["company"],
       async setup(resolved) {
         const company = value<CompanyRecord>(resolved, "company");
-        const provider =
-          execution.profile.provider === "acpx" ? "anthropic" : "openai";
-        const key =
-          provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY";
+        const key = execution.profile.credential;
+        const provider = key === "ANTHROPIC_API_KEY" ? "anthropic"
+          : key === "OPENROUTER_API_KEY" ? "openrouter"
+          : key === "OPENAI_API_KEY" ? "openai" : null;
+        if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`);
         const apiKey = input.credentials[key];
         if (!apiKey) throw new Error(`Missing credential ${key}`);
         const account = await api.postSensitive<{ connectionId: string }>(
@@ -291,7 +292,10 @@ export async function setupLiveFixtures(input: {
         secretRefs,
         executionId: input.executionNonce,
       });
-      if (execution.suite.id === "stock-harness") agent.budgetMonthlyCents = 1_000;
+      if (execution.suite.id === "stock-harness"
+        || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) {
+        agent.budgetMonthlyCents = 1_000;
+      }
       if (managedHiring) {
         const account = value<ManagedAccountFixture>(resolved, "ai-connection");
         const config = agent.adapterConfig as Record<string, unknown>;

tests/runner-e2e/native-procedure-guidance.test.ts

@@ -19,13 +19,12 @@ describe("bounded native procedure comparison", () => {
     }
     expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(cell => cell.suite.id === suite.id)).toBe(false);
   });
-  it("sets each lead's hard stop without fixture completion instructions or provider overrides", () => {
+  it("keeps procedure and completion directions out of the fixture persona", () => {
     for (const profile of suite.profiles) {
-      const input = { executionId: `everyday-workflows.${profile.id}.local.hire-reuse`, environmentId: "env", environmentFixtureId: "local" as const, workspacePath: "/workspace", secretRefs: {
+      const input = { executionId: "random-nonce", environmentId: "env", environmentFixtureId: "local" as const, workspacePath: "/workspace", secretRefs: {
         [profile.credential]: { type: "secret_ref" as const, secretId: "00000000-0000-4000-8000-000000000001", version: "latest" as const },
       } };
       const agent = profile.buildAgent(input);
-      expect(agent.budgetMonthlyCents).toBe(1_000);
       expect(JSON.stringify(agent.instructionsBundle)).not.toMatch(/paperclip_finish|paperclip_block|set_dependencies|hire_agent/);
     }
   });