From 64cd2194a1d058356679b653c181f16404aefc80 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 5 Oct 2026 09:27:45 -0500 Subject: [PATCH] test(runner): freeze full tool payload and procedure comparison fixtures Co-Authored-By: Paperclip --- .../2026-10-05-native-procedure-guidance.md | 73 +++++++++ package.json | 2 +- .../native-procedure-measurement.test.ts | 147 ++++++++++++++++++ tests/runner-e2e/EVERYDAY-WORKFLOWS.md | 4 +- tests/runner-e2e/README.md | 25 +++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 20 ++- tests/runner-e2e/everyday.test.ts | 2 +- tests/runner-e2e/live-fixtures.ts | 3 +- .../native-instruction-consolidation.test.ts | 5 +- .../native-procedure-guidance.test.ts | 32 ++++ tests/runner-e2e/vitest.config.ts | 4 +- 12 files changed, 310 insertions(+), 13 deletions(-) create mode 100644 doc/plans/2026-10-05-native-procedure-guidance.md create mode 100644 server/src/__tests__/native-procedure-measurement.test.ts create mode 100644 tests/runner-e2e/native-procedure-guidance.test.ts diff --git a/doc/plans/2026-10-05-native-procedure-guidance.md b/doc/plans/2026-10-05-native-procedure-guidance.md new file mode 100644 index 00000000000..62908494410 --- /dev/null +++ b/doc/plans/2026-10-05-native-procedure-guidance.md @@ -0,0 +1,73 @@ +# Native hiring and dependency guidance + +Status: implementation and provider-free verification in progress. Live outcome +qualification is pending. This follows the merged completion slice in #15151. + +## Scope + +Use master `a386a599983519eb1d399f8b770bfccdb2a74762` as the common context. +Move hiring/reuse and assigned-worker guidance to the existing `hire_agent` and +`create_task` descriptions. Keep dependency discovery in the fixed prompt and +replacement semantics in `set_dependencies`. Return conditional workspace-yield +guidance after recording a nonempty blocker set. Empty sets do not need waiting +guidance. Preserve idempotency and the original dependency state transitions. + +Leave connection procedures and legacy skill/API instructions unchanged. Advance +the prompt and native-session catalog revisions so retained threads cannot use +the old declarations with the new instructions. + +## Measurement boundary + +Capture the actual standard-mode server tool authority with API tools enabled, +a local workspace, and no assigned external apps. Measure fixed instructions, +all advertised tool descriptions and input schemas, and the serialized task +input together. Scripted native start/resume/continuation checks cover Codex, +ACPX Claude and OpenCode; authenticated MCP tools/list checks the OpenCode +bridge separately. Export source identity, file hashes, bytes and tool counts. + +These receipts have zero provider calls. Bytes are not model tokens or invoice +costs. This boundary cannot establish the upstream SDK's private instructions, +lazy loading, truncation or cognitive use of returned guidance. The previous +completion-only capture selected a partial catalog and is not a full-tool +baseline. Preserve its historical evidence unchanged. + +## Matched live comparison + +Use the existing explicit-only `everyday-workflows` suite with original +`hire-reuse` and `delegate-feedback` requests and independent artifact oracles. +Select only native Codex `gpt-5.6-sol`, ACPX Claude `claude-sonnet-5`, and OpenCode +`openrouter/deepseek/deepseek-v4-flash-0731`, in local environments. + +Freeze common measurement, harness and test-runner repairs in both branches +before the candidate production changes. The historical branch keeps the +current production prompt and tool contracts. Record exact measured source +SHAs separately from the trusted master workflow definition and selected +runtime/artifact hashes. Never label a later source revision as the measured +revision without showing its delta. + +The plan has six cells per variant, one attempt per cell, a 12-minute cell +deadline, at most 12 story run records per cell, and a 1,000-cent company and +lead-agent hard stop per cell. All worker runs count toward the company budget. +Count every actual run, including notifications and failed attempts. No broad +matrix, automatic retries or baseline rerolls are part of this comparison. + +Inspect original grades and retained content for persistent teammate identity, +reuse on revision, worker ownership, independently tested delivered artifacts, +parent review, dependency release, and final ordering. Check per-pair changes; +equal totals are insufficient. Preserve every original failure and missing +evidence. Model-authored tests cannot replace the existing independent oracle. + +These two stories do not qualify arbitrary live resume, every existing-blocker +combination, general coding quality, or cost/speed trends. Explicitly report +missing evidence. Do not loosen the original oracle to create a pass. + +## Gates + +- [ ] Freeze common fixtures and both source revisions. +- [ ] Export comparable complete-payload measurements. +- [ ] Pass focused tool/session/measurement tests and eval support checks. +- [ ] Complete repository typecheck, tests and build. +- [ ] Run and inspect the bounded matched campaign. +- [ ] Verify current-head CI, review and conflicts before readiness. + +No merge is authorized. diff --git a/package.json b/package.json index 29223ed74ec..7799db91486 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "test:runner-recovery": "vitest run server/src/services/native-runtime/native-replacement-evidence.test.ts server/src/services/native-runtime/stopped-codex-turn.test.ts server/src/services/native-runtime/native-safe-replacement.test.ts", "test:e2e:browser-context": "playwright test --config tests/e2e/playwright-company-context.config.ts", "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts", - "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts", + "test:e2e:runner:unit": "vitest run --config tests/runner-e2e/vitest.config.ts && node --test tests/runner-e2e/native-completion-checks.test.mjs tests/runner-e2e/native-completion-git-source.test.mjs tests/runner-e2e/native-completion-source-contract.test.mjs", "test:e2e:runner:stock-harness": "node tests/runner-e2e/stock-harness-checks.mjs", "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", diff --git a/server/src/__tests__/native-procedure-measurement.test.ts b/server/src/__tests__/native-procedure-measurement.test.ts new file mode 100644 index 00000000000..a95d483d45f --- /dev/null +++ b/server/src/__tests__/native-procedure-measurement.test.ts @@ -0,0 +1,147 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { afterAll, describe, expect, it } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { buildNativeModelEnvelope, parseNativeExecutionInput, type NativeExecutionInput } from "../../../packages/paperclip-runner/src/contracts/native-execution.js"; +import type { PersistedNativeSession } from "../../../packages/paperclip-runner/src/contracts/native-session-backend.js"; +import { NATIVE_RUNTIME_ASSET_SCHEMA, PAPERCLIP_EXECUTION_PROMPT, PAPERCLIP_EXECUTION_PROMPT_REVISION, canonicalNativeRuntimeContextDigest, nativeRuntimePromptDigest } from "../../../packages/paperclip-runner/src/contracts/runtime-context.js"; +import { createRunnerdNativeSessionBackend } from "../../../packages/paperclip-runner/src/backends/codex-native-backend.js"; +import { resolveQualifiedAcpxProfile } from "../../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; +import { FakeCodexTransport, WORKSPACE } from "../../../packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.js"; +import { startOpenCodeMcpBridge } from "../../../packages/paperclip-runner/src/drivers/opencode/mcp-bridge.js"; + +// Production catalog selection, scripted provider transport: this measures what +// Paperclip delivers, not the vendor's private prompt, tokenizer or billing. +const root = new URL("../../../", import.meta.url); +const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); +const identity = { companyId: "company", agentId: "agent", issueId: "issue", runId: "run", sessionId: "session" }; +const tools = new PaperclipRunnerToolAuthority({} as Db, { + ...identity, apiToolsEnabled: true, workspaceRoot: WORKSPACE, +}).definitions(); +const { qualificationModel: _model, reportedModelId: _reportedModel, permissionPolicy: _policy, ...profile } = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"); +const providers: NativeExecutionInput["provider"][] = [ + { kind: "codex", model: "gpt-5.6-sol", approvalPolicy: "never" }, + { kind: "acpx", agent: "claude", model: "claude-sonnet-5", permissionMode: "approve-all", profile }, + { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731", permissionMode: "allow" }, +]; +const receipts: unknown[] = []; +let mcpReceipt: unknown; +function measure(value: unknown) { + const json = JSON.stringify(value).replaceAll(WORKSPACE, "/workspace"); + return { utf8Bytes: Buffer.byteLength(json), characters: json.length, sha256: sha(json) }; +} +function execution(provider: NativeExecutionInput["provider"]): NativeExecutionInput { + const entry = readFileSync(new URL("server/src/onboarding-assets/default/AGENTS.md", root), "utf8"); + writeFileSync(`${WORKSPACE}/AGENTS.md`, entry); + const context = { + prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() }, + instructions: { entryPath: "AGENTS.md", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, rootPath: WORKSPACE, digest: sha(entry), manifestDigest: sha(entry), fileCount: 1, totalBytes: Buffer.byteLength(entry) } }, + skills: [], mcp: { assignmentSetId: "none", digest: "0".repeat(64), bindingId: null }, + }; + return parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v5", binding: { companyId: identity.companyId, agentId: identity.agentId, issueId: identity.issueId, runId: identity.runId, executionWorkspaceId: "workspace" }, + task: { identifier: "TASK-1", title: "Review delegated work", description: "Review delegated work", prompt: "Review delegated work.", workMode: "standard" }, + workspace: { cwd: WORKSPACE, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: "session", driverKind: provider.kind === "acpx" ? "acpx_runtime" : provider.kind === "opencode" ? "opencode_server" : "codex_app_server", protocolVersion: 1 }, + provider, executionMode: "default", planningContext: null, + completionContract: { id: "contract", sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.completion-contract.v1", contract: { revision: "1", objective: "Review delegated work", criteria: [{ id: "objective", requirement: "Review complete" }] } }, + interactionResponses: [], credentialBindings: [], + runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }, + }); +} +function transport(provider: NativeExecutionInput["provider"]) { + return new FakeCodexTransport("thread-1", "provider-session-1", provider.kind === "acpx" ? { + kind: "acpx", normalizedSessionId: "session", acpxRecordId: "fixture-record", backendSessionId: "fixture-backend", + agentSessionId: "fixture-agent", profileDigest: `sha256:${"a".repeat(64)}`, workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: provider.model, effectiveModel: provider.model, permissionMode: "approve-all", + providerLifetimeFenceCandidates: [60001, 60002, 60003], + } : undefined); +} + +describe("production native procedure instruction measurement", () => { + it("uses the real standard-mode authority, including optional and connection tools", () => { + expect(tools.map(tool => tool.name)).toEqual(expect.arrayContaining([ + "hire_agent", "list_agents", "create_task", "set_dependencies", "connections_search", "connection_request", "register_deliverable", + ])); + expect(new Set(tools.map(tool => tool.name)).size).toBe(tools.length); + const withoutApi = new PaperclipRunnerToolAuthority({} as Db, { ...identity, apiToolsEnabled: false }).definitions(); + expect(withoutApi.some(tool => tool.name === "hire_agent")).toBe(false); + }); + + for (const provider of providers) it(`captures the full ${provider.kind} start/resume/continuation payload`, async () => { + const input = execution(provider); + let checkpoint: PersistedNativeSession | undefined; + for (const phase of ["start", "resume", "continuation"] as const) { + const captured = transport(provider); + captured.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, turns: [] } }; + const current = phase === "continuation" ? parseNativeExecutionInput({ ...input, continuationPrompt: "Review the teammate's saved result." }) : input; + const backend = createRunnerdNativeSessionBackend(current, { transportFactory: () => captured, dynamicTools: tools, environment: { PAPERCLIP_WORKSPACE_CWD: WORKSPACE } }); + const recovery = checkpoint ? await backend.recoverSession!(checkpoint, { signal: new AbortController().signal }) : null; + if (recovery) expect(recovery.recovered).toBe(true); + const session = recovery?.session ?? await backend.openSession({ identity, workingDirectory: WORKSPACE }); + try { + checkpoint ??= await session.snapshot(); + const envelope = buildNativeModelEnvelope(current, phase === "continuation" ? { resumedSession: true } : undefined); + if ("requestedSkills" in envelope && backend.preparedTaskConstraints) envelope.constraints = [...backend.preparedTaskConstraints]; + await session.startTurn({ message: { role: "user", text: JSON.stringify(envelope) } }); + const setup = captured.calls.find(call => call.method === (phase === "start" ? "thread/start" : "thread/resume"))!.params; + const turn = captured.calls.find(call => call.method === "turn/start")!.params; + const instructions = setup.developerInstructions ?? setup.baseInstructions; + const delivered = setup.dynamicTools as Record[]; + expect(delivered.filter(tool => !["paperclip_finish", "paperclip_block"].includes(String(tool.name)))).toEqual(tools); + expect(delivered).toHaveLength(tools.length + 2); + expect(instructions).toContain(PAPERCLIP_EXECUTION_PROMPT); + expect(JSON.parse((turn.input as {text: string}[])[0]!.text)).toEqual(envelope); + receipts.push({ provider: provider.kind, phase, instructions: measure(instructions), tools: measure(delivered), input: measure(turn.input), + fullProjection: measure({ instructions, tools: delivered, input: turn.input }), deliveredToolCount: delivered.length }); + } finally { await session.close({ reason: "measurement" }); } + } + }); + + it("preserves the full catalog through the actual OpenCode MCP tools/list bridge", async () => { + const bridge = await startOpenCodeMcpBridge({ tools, handler: async () => { throw new Error("No tool execution in measurement"); } }); + try { + const response = await fetch(bridge.url, { method: "POST", headers: { authorization: `Bearer ${bridge.secret}`, "content-type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }) }); + expect(response.ok).toBe(true); + const result = await response.json() as { result: { tools: Record[] } }; + expect(result.result.tools.filter(tool => !["paperclip_finish", "paperclip_block"].includes(String(tool.name)))).toEqual( + tools.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })), + ); + mcpReceipt = { boundary: "authenticated loopback MCP tools/list; no OpenCode/provider process", serverAnnotationsForwarded: false, tools: measure(result.result.tools), count: result.result.tools.length }; + } finally { await bridge.close(); } + }); +}); + +afterAll(() => { + const output = process.env.PAPERCLIP_NATIVE_PROCEDURE_MEASUREMENT; + if (!output) return; + if (receipts.length !== 9 || !mcpReceipt) throw new Error("Refusing incomplete instruction measurement"); + const paths = [ + "packages/paperclip-runner/src/contracts/runtime-context.ts", + "packages/paperclip-runner/src/protocol-actions/hire-agent.ts", + "packages/paperclip-runner/src/protocol-actions/create-task.ts", + "packages/paperclip-runner/src/protocol-actions/set-dependencies.ts", + "packages/paperclip-runner/src/catalog/semantic-action-catalog.ts", + "packages/paperclip-runner/generated/semantic-action-catalog.json", + "packages/paperclip-runner/generated/capability/semantic-tool-contracts.json", + "packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json", + "packages/paperclip-runner/protocol/manifest.json", + "packages/paperclip-runner/src/backends/codex-native-backend.ts", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts", + "server/src/services/native-runtime/paperclip-runner-tool-authority.ts", + "server/src/services/native-runtime/native-session-resume.ts", + "server/src/__tests__/native-procedure-measurement.test.ts", + ]; + writeFileSync(output, `${JSON.stringify({ schema: "paperclip.native-procedure-measurement.v1", + sourceSha: execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(), + sourceDirty: execFileSync("git", ["status", "--porcelain", "--", ...paths], { cwd: root, encoding: "utf8" }).trim().length > 0, + sourceHashes: Object.fromEntries(paths.map(path => [path, sha(readFileSync(new URL(path, root)))])), + boundary: "scripted runnerd RPC with production standard-mode authority, API tools enabled, local workspace, no assigned external apps", + providerCalls: 0, tokenCount: null, upstreamLoadingOrTruncation: "unverified", vendorStockPrompt: "unavailable", + toolDescriptions: tools.map(tool => ({ name: tool.name, characters: String(tool.description).length, utf8Bytes: Buffer.byteLength(String(tool.description)), inputSchema: measure(tool.inputSchema) })), + receipts, mcpReceipt, + }, null, 2)}\n`); +}); diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index cec2fe6fb5e..675c4ed8db8 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -40,7 +40,9 @@ See [controlled recovery tests](../runner-recovery/README.md). The local matrix has fourteen cases on native Codex `gpt-5.6-sol`, native ACPX Claude `claude-sonnet-5`, and native Codex `gpt-5.4-mini`: 42 cells. The two core profiles also declare build/revise, delegation, controller-restart, and skill-creation cases -on Daytona: eight cells. Remote runner-process killing is not supported. For remote controller +on Daytona: eight cells. OpenCode adds only local hiring/reuse and delegation, +for 52 cells total. Hiring/reuse and delegation have one attempt and a +1,000-cent company and lead-agent hard stop. Remote runner-process killing is not supported. For remote controller restart, a verified first download supplies the persistence checkpoint; the controller is interrupted during a subsequent revision with another queued requirement. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 397f33da086..5feba8909d3 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -12,6 +12,31 @@ scheduled execution gets a fresh Paperclip home, embedded Postgres database, instance configuration, port, workspace, company, encrypted secrets, environment, and agent. +## Native procedure guidance comparison (explicit only) + +Select `--suite everyday-workflows --environment local --case hire-reuse --case delegate-feedback +--profile runner-codex --profile runner-acpx-claude --profile runner-opencode` for the six comparison +cells on native Codex, ACPX Claude, and OpenCode. +Both variants use the same Studio Lead persona, original user requests, +artifact oracle, lifecycle checks, models and permissions. Each cell permits +one attempt, a 12-minute deadline, at most 12 story run records, and a +1,000-cent company hard stop; the lead also has a 1,000-cent hard stop. Worker +runs count toward the company budget. The suite is excluded from `--all`. + +Compare frozen branches on the same master with identical fixture sources. +Retain each original grade, source SHA, harness digest, actual run inventory, +downloaded artifacts and partial cost evidence. Check hiring identity/reuse, +worker ownership, delivered revisions, dependency release and parent completion +ordering independently of aggregate grades. These bounded stories do not +qualify every existing-blocker combination or arbitrary provider resume. + +The provider-free `native-procedure-measurement.test.ts` uses the server's real +tool authority and captures scripted start/resume/continuation delivery plus +the OpenCode MCP catalog. It includes descriptions and argument schemas. Its +byte counts are not model token counts or proof of an upstream harness's lazy +loading/truncation. The older completion measurement used a partial catalog; +do not use it as a full production tool-payload baseline. + The vocabulary is: a **campaign** is one workflow invocation against one SHA; a **suite** is a durable testing purpose; a **matrix** is that suite's profiles × environments × cases; an **execution/cell** is one parallel job; and an diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 8f44a3a160b..8e764f9f015 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,10 +136,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, + 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(476); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(476); + expect(validateRunnerCatalog()).toHaveLength(478); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(478); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index cc37562702b..3db1036b4f0 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1191,12 +1191,24 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ { id: "everyday-workflows", label: "Everyday Paperclip Work", manualOnly: true, description: "Real user requests, useful downloaded work, and durable continuation using production instructions.", - groups: ["native"], profiles: everydayProfiles, environments: [localEnvironment, daytonaWarmEnvironment], - tasks: everydayTasks, expectedMatrixSize: 50, + groups: ["native"], profiles: [...everydayProfiles, + productionStoryProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!), + ].map(profile => ({ ...profile, buildAgent(input) { + const agent = profile.buildAgent(input); + return /\.(hire-reuse|delegate-feedback)$/.test(input.executionId) ? { ...agent, budgetMonthlyCents: 1_000 } : agent; + } })), environments: [localEnvironment, daytonaWarmEnvironment], + tasks: everydayTasks.map(task => ["hire-reuse", "delegate-feedback"].includes(task.id) + ? { ...task, automaticRetryPolicy: "single_attempt" as const } : task), expectedMatrixSize: 52, excludedExecutionIds: [...everydayProfiles.flatMap(profile => everydayTasks .filter(task => !["build-revise", "delegate-feedback", "recover-controller", "create-skill-studio"].includes(task.id)) - .map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`))], - definitionMetadata: { version: 4, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only" }, + .map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`)), + ...everydayTasks.flatMap(task => [ + `everyday-workflows.runner-opencode.daytona.${task.id}`, + ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]), + ]), + ], + definitionMetadata: { version: 5, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } }, }, { id: NATIVE_INSTRUCTION_SUITE, label: "Native completion instruction consolidation", manualOnly: true, diff --git a/tests/runner-e2e/everyday.test.ts b/tests/runner-e2e/everyday.test.ts index 3afd8735fa9..88cbecddd72 100644 --- a/tests/runner-e2e/everyday.test.ts +++ b/tests/runner-e2e/everyday.test.ts @@ -212,7 +212,7 @@ describe("manual everyday workflow catalog", () => { const selected = selectRunnerExecutions( parseRunnerSelectors(["--suite", "everyday-workflows"]), ); - expect(selected).toHaveLength(50); + expect(selected).toHaveLength(52); expect(selected.every((e) => e.profile.generation === "native")).toBe(true); expect( selectRunnerExecutions( diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index ac5ce1bdc61..94fd9506b80 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -136,7 +136,8 @@ export async function setupLiveFixtures(input: { return api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", - budgetMonthlyCents: ["native-completion", "native-instruction-consolidation"].includes(execution.suite.id) ? NATIVE_COMPLETION_BUDGET_CENTS + budgetMonthlyCents: ["native-completion", "native-instruction-consolidation"].includes(execution.suite.id) + || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : execution.suite.id === "stock-harness" ? 1_000 : 0, }); diff --git a/tests/runner-e2e/native-instruction-consolidation.test.ts b/tests/runner-e2e/native-instruction-consolidation.test.ts index 6ba5ea2819f..407ed385ed1 100644 --- a/tests/runner-e2e/native-instruction-consolidation.test.ts +++ b/tests/runner-e2e/native-instruction-consolidation.test.ts @@ -69,7 +69,10 @@ describe("native instruction comparison admission", () => { const files = Object.keys(NATIVE_INSTRUCTION_VARIANTS.baseline); const baseline = new Map(files.map(file => [file, execFileSync("git", ["show", `${NATIVE_INSTRUCTION_BASE_SHA}:${file}`])])); expect(nativeInstructionVariant(file => baseline.get(file)!)).toBe("baseline"); - const candidate = new Map(files.map(file => [file, readFileSync(new URL(`../../${file}`, import.meta.url))])); + // This historical admission intentionally rejects later production changes. + // Exercise the retained qualified tree rather than requiring HEAD to remain + // one of this completed comparison's frozen variants forever. + const candidate = new Map(files.map(file => [file, execFileSync("git", ["show", `16ef1a5744b23bb043a8b286e6288921da1de98f:${file}`])])); const currentVariant = nativeInstructionVariant(file => candidate.get(file)!); expect(["baseline", "candidate", "corrected", "feedback", "opencodeFeedback", "opencodeFeedbackSettlement", "opencodeFeedbackResponses"]).toContain(currentVariant); candidate.set(files[0]!, currentVariant === "candidate" ? baseline.get(files[0]!)! : Buffer.from("unknown source")); diff --git a/tests/runner-e2e/native-procedure-guidance.test.ts b/tests/runner-e2e/native-procedure-guidance.test.ts new file mode 100644 index 00000000000..6aae7de272b --- /dev/null +++ b/tests/runner-e2e/native-procedure-guidance.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { runnerMatrix, runnerSuites } from "./catalog.js"; +import { everydayTasks } from "./everyday-cases.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; + +describe("bounded native procedure comparison", () => { + const suite = runnerSuites.find(suite => suite.id === "everyday-workflows")!; + it("selects only two original journeys on three native profiles, with no automatic retry", () => { + expect(suite.manualOnly).toBe(true); + const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", suite.id, + "--profile", "runner-codex", "--profile", "runner-acpx-claude", "--profile", "runner-opencode", + "--case", "hire-reuse", "--case", "delegate-feedback", "--environment", "local"])); + expect(selected).toHaveLength(6); + for (const { task } of selected) { + const original = everydayTasks.find(candidate => candidate.id === task.id)!; + expect(task).toEqual({ ...original, automaticRetryPolicy: "single_attempt" }); + expect(task.buildPrompt("nonce")).toBe(original.buildPrompt("nonce")); + expect(task.attemptTimeoutMs.local).toBe(720_000); + } + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"]), runnerMatrix).some(cell => cell.suite.id === suite.id)).toBe(false); + }); + it("sets each lead's hard stop without fixture completion instructions or provider overrides", () => { + for (const profile of suite.profiles) { + const input = { executionId: `everyday-workflows.${profile.id}.local.hire-reuse`, environmentId: "env", environmentFixtureId: "local" as const, workspacePath: "/workspace", secretRefs: { + [profile.credential]: { type: "secret_ref" as const, secretId: "00000000-0000-4000-8000-000000000001", version: "latest" as const }, + } }; + const agent = profile.buildAgent(input); + expect(agent.budgetMonthlyCents).toBe(1_000); + expect(JSON.stringify(agent.instructionsBundle)).not.toMatch(/paperclip_finish|paperclip_block|set_dependencies|hire_agent/); + } + }); +}); diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index fd13ba8fd34..7d0dcdbc0fe 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -1,10 +1,12 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { name: "runner-e2e-support", include: ["**/*.test.ts", "**/*.test.mjs"], + // These use node:test and run separately in test:e2e:runner:unit. + exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs"], testTimeout: 30_000, }, });