diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs index 759f5ef4c1b..be447aeb7ce 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs @@ -928,6 +928,18 @@ pub fn normalize_codex_notification(method: &str, params: &Value) -> Vec { ); expect(finish.inputSchema).toEqual(PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA); expect(block.inputSchema).toEqual(PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA); + expect(finish.description).toBe(PRP_COMPLETION_TOOL_DESCRIPTION); + expect(block.description).toBe(PRP_BLOCK_TOOL_DESCRIPTION); }); it("allows only a response-wake continuation when completion explicitly yields", () => { diff --git a/packages/paperclip-runner/src/contracts/completion-result.ts b/packages/paperclip-runner/src/contracts/completion-result.ts index dec93a2f1de..b727a942965 100644 --- a/packages/paperclip-runner/src/contracts/completion-result.ts +++ b/packages/paperclip-runner/src/contracts/completion-result.ts @@ -1,6 +1,11 @@ /** Provider-neutral semantic completion tools and their strict model-facing schemas. */ export const PRP_COMPLETION_TOOL_NAME = "paperclip_finish" as const; export const PRP_BLOCK_TOOL_NAME = "paperclip_block" as const; +/** Native Runner guidance; legacy adapters use their own skill/API completion paths. */ +export const PRP_COMPLETION_TOOL_DESCRIPTION = + "Report completed work (done), work requiring review (needs_review), or an explicit wait for the next response (yielded with response_wake). Use the current completion contract and supporting evidence. If rejected, correct the report and retry. After acceptance, read the returned outcome; do not claim completion while gated. Explain any required approval with its supplied link and action, then write the final response and end the turn without further tool calls."; +export const PRP_BLOCK_TOOL_DESCRIPTION = + "Report work that cannot continue because of a concrete blocker. Identify the blocker, its owner, and the action needed to unblock it; use the current completion contract and supporting evidence. If rejected, correct the report and retry. After acceptance, read the returned outcome, explain the blocker and any required action in the final response, and end the turn without further tool calls."; export const PRP_SEMANTIC_TOOL_NAMES = [ PRP_COMPLETION_TOOL_NAME, PRP_BLOCK_TOOL_NAME, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts index 81e8a3def93..c5ade4c19c2 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts @@ -1,3 +1,9 @@ +import { + PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_BLOCK_TOOL_DESCRIPTION, + PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_COMPLETION_TOOL_DESCRIPTION, +} from "../../contracts/completion-result.js"; import { CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, CODEX_INVALID_REQUEST, @@ -540,6 +546,10 @@ describe("Codex app-server Codex driver", () => { expect(started.developerInstructions).toBe(conversationMode === "direct" ? undefined : baseInstructions); expect(resumed.developerInstructions).toBe(conversationMode === "direct" ? "" : baseInstructions); expect(resumed.dynamicTools).toEqual(started.dynamicTools); + expect(started.dynamicTools).toEqual(conversationMode === "direct" ? [] : [ + { name: "paperclip_finish", description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA }, + { name: "paperclip_block", description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA }, + ]); expect(resumed.config).toEqual(started.config); await recovered.session?.close({ reason: "verified additive recovery" }); }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts index 68c32a3c9a6..5d4b9c9c7f2 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts @@ -1,5 +1,9 @@ import type { PersistedHarnessProviderIdentity } from "../../contracts/harness-driver.js"; import type { NativeUserMessage } from "../../contracts/types.js"; +import { + PRP_BLOCK_TOOL_DESCRIPTION, + PRP_COMPLETION_TOOL_DESCRIPTION, +} from "../../contracts/completion-result.js"; import { CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, CODEX_BLOCK_TOOL_NAME, @@ -232,8 +236,7 @@ export function differingJsonPaths( function finishToolSpec(): Record { return { name: CODEX_COMPLETION_TOOL_NAME, - description: - "Return the one semantic completion result for this task, including an explicit response_wake yield when waiting for the next response.", + description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: CODEX_RESULT_PROVIDER_INPUT_SCHEMA, }; } @@ -241,8 +244,7 @@ function finishToolSpec(): Record { function blockToolSpec(): Record { return { name: CODEX_BLOCK_TOOL_NAME, - description: - "Return the one semantic result when the task cannot continue.", + description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, }; } diff --git a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts index 6539a14b52a..07681400ddd 100644 --- a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts @@ -1,4 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { + PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_BLOCK_TOOL_DESCRIPTION, + PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_COMPLETION_TOOL_DESCRIPTION, +} from "../../contracts/completion-result.js"; import { canonicalOpenCodeMcpToolName, startOpenCodeMcpBridge, type OpenCodeMcpBridge } from "./mcp-bridge.js"; @@ -38,8 +44,8 @@ describe("OpenCode MCP bridge", () => { expect(await listed.json()).toMatchObject({ result: { tools: [ { name: "documents.read" }, - { name: "paperclip_finish" }, - { name: "paperclip_block" }, + { name: "paperclip_finish", description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA }, + { name: "paperclip_block", description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA }, ] }, }); }); diff --git a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts index 79ce6770375..1bf1555dd16 100644 --- a/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts +++ b/packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts @@ -4,8 +4,10 @@ import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; import { PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_BLOCK_TOOL_DESCRIPTION, PRP_BLOCK_TOOL_NAME, PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_COMPLETION_TOOL_DESCRIPTION, PRP_COMPLETION_TOOL_NAME, } from "../../contracts/completion-result.js"; @@ -117,8 +119,8 @@ function normalizeTools( seen.add(name); } if (includeTerminalTools) for (const terminal of [ - { name: PRP_COMPLETION_TOOL_NAME, description: "Return the semantic completion result.", inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA }, - { name: PRP_BLOCK_TOOL_NAME, description: "Return the semantic blocked result.", inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA }, + { name: PRP_COMPLETION_TOOL_NAME, description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA }, + { name: PRP_BLOCK_TOOL_NAME, description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA }, ]) { if (!seen.has(terminal.name)) tools.push(terminal); } diff --git a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts index b552adcb28a..5bf87169d24 100644 --- a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts +++ b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts @@ -1,6 +1,12 @@ import { connect } from "node:net"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { + PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_BLOCK_TOOL_DESCRIPTION, + PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_COMPLETION_TOOL_DESCRIPTION, +} from "../contracts/completion-result.js"; import { canonicalRunnerToolName, @@ -48,8 +54,8 @@ describe("runner semantic MCP bridge", () => { result: { tools: [ { name: "documents.read" }, - { name: "paperclip_finish" }, - { name: "paperclip_block" }, + { name: "paperclip_finish", description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA }, + { name: "paperclip_block", description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA }, ], }, }); diff --git a/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts b/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts index e59535171be..c8207c71833 100644 --- a/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts +++ b/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts @@ -10,8 +10,10 @@ import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; import { PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_BLOCK_TOOL_DESCRIPTION, PRP_BLOCK_TOOL_NAME, PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA, + PRP_COMPLETION_TOOL_DESCRIPTION, PRP_COMPLETION_TOOL_NAME, } from "../contracts/completion-result.js"; @@ -70,12 +72,12 @@ const MAX_RESULT_CHUNK_BYTES = 64 * 1024; const RESERVED_TOOLS: readonly RunnerToolDefinition[] = [ { name: PRP_COMPLETION_TOOL_NAME, - description: "Return the semantic completion result.", + description: PRP_COMPLETION_TOOL_DESCRIPTION, inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA, }, { name: PRP_BLOCK_TOOL_NAME, - description: "Return the semantic blocked result.", + description: PRP_BLOCK_TOOL_DESCRIPTION, inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, }, ]; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 8ab547d2211..50544213ea3 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -54,6 +54,10 @@ import { CODEX_SKILLLESS_BASE_INSTRUCTIONS, createCodexTaskEnvelope, } from "../contracts/codex.js"; +import { + PRP_BLOCK_TOOL_DESCRIPTION, + PRP_COMPLETION_TOOL_DESCRIPTION, +} from "../contracts/completion-result.js"; import { CodexAppServerDriver, codexSemanticToolSpecs, @@ -1336,14 +1340,19 @@ it("includes ACPX terminal tools in the authenticated bridge catalog", () => { }); }); -it("preserves answer and internal wait descriptions in the serialized native tool catalog", () => { +it.each(["codex", "opencode", "claude_managed", "aws_agentcore", "acpx"] as const)("preserves answer and internal wait descriptions in the serialized native %s tool catalog", (provider) => { const catalog = JSON.parse( - JSON.stringify(authorizedToolSetForProvider("codex", codexSemanticToolSpecs())), + JSON.stringify(authorizedToolSetForProvider(provider, codexSemanticToolSpecs())), ); const finish = catalog.operations.find( (operation: { operationId: string }) => operation.operationId === "paperclip_finish", ); + const block = catalog.operations.find( + (operation: { operationId: string }) => operation.operationId === "paperclip_block", + ); + expect(finish.description).toBe(PRP_COMPLETION_TOOL_DESCRIPTION); + expect(block.description).toBe(PRP_BLOCK_TOOL_DESCRIPTION); expect(finish.inputSchema.properties.summary.description).toContain( "complete user-facing answer", ); @@ -4154,7 +4163,9 @@ it("captures exact provider frames and correlates Rust and TypeScript interpreta ); const tracePath = join(traceDirectory, "trace.ndjson"); const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), + // Qualification builds a debug daemon for this exact source. Its selected + // binary must win over any separately staged product/runtime artifact. + runnerBinary: process.env.PAPERCLIP_STOCK_PREFLIGHT_RUNNERD ?? defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(traceDirectory, "--structured-activity"), stateDirectory: join(traceDirectory, "state"), diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index 59113de9b89..a8a893987cc 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -1875,6 +1875,12 @@ describe("rebindNativeSessionCheckpoint", () => { retainedFingerprint: "sha256:5b7b302db36f7ed6686f9ea1ba70bbf79ebd7fabf86953b548d966a2bc38b648", }, + { + contract: "native completion tool guidance", + // Deployed v13 local catalog before canonical finish/block descriptions. + retainedFingerprint: + "sha256:68a51d34e091c55ee5d0d2b563153454dd727d72db16e6a27c358d342ae489c9", + }, { contract: "task-bound human-input description", // Deployed v9 still advertises the generic mock-task question description. diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index d8a31f27460..bf49bea2b84 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -23,7 +23,7 @@ export function nativeToolContractFingerprintForTarget( return `sha256:${createHash("sha256") .update( JSON.stringify({ - schema: "paperclip.native-tool-contract.v13", + schema: "paperclip.native-tool-contract.v14", executionTargetKind, advertisementPolicy: { // Direct provider threads retain declarations from thread/start. diff --git a/tests/runner-e2e/NATIVE-COMPLETION.md b/tests/runner-e2e/NATIVE-COMPLETION.md new file mode 100644 index 00000000000..ca0ac463d25 --- /dev/null +++ b/tests/runner-e2e/NATIVE-COMPLETION.md @@ -0,0 +1,70 @@ +# Native completion guidance qualification + +This explicit-only suite independently qualifies PR #14961 on master +`59c07ede72dc08b8aba149a01cc11e0b7a204621`. It does not include PR #14948's +reduced manuals, shared prompts or operational-skill changes. The earlier +comparison on reduced defaults remains historical evidence, not admission for +this context. + +The six local cells pair the original `assigned-skill-explicit-invocation` +journey and `native-blocked-report` across `runner-codex`, +`runner-acpx-claude` and `runner-opencode`. Completion keeps the original task, +installed output skill, explicit invocation and independently saved issue +document checks. Blocking checks the durable whole-task blocker, exact external +owner/action, explanatory visible reply, unchanged workspace and absence of +extra tasks, agents, documents or interactions. + +Both use the actual production default hire: no custom QA instructions bundle. +Before execution, the public bundle receipt must hash to the unchanged master +AGENTS.md, and company/agent budgets must each be 1,000 cents. Profile model, +effort, credentials, skills, permissions and provider configuration are inherited. + +The native oracle requires exactly one succeeded native run attributed to this +issue, bound complete public events, one runner semantic proposal and authoritative +control-plane acceptance/terminal. A matched finishing tool start/result with the expected native terminal name +precedes the actual provider final; no new invocation follows the proposal. The final +must be persisted as the run's agent reply. Acceptance may precede or follow the +final. This proves observable ordering and durable disposition; it does not prove +the provider consumed feedback or establish general coding-quality equivalence. + +`single_attempt` is enforced by the launcher even when hosted invocation only +supplies `--id`. Every first failure, usage and cleanup receipt is retained; no +automatic infrastructure or behavioral retry occurs. The bounded comparison is +six cells per variant, twelve expected provider turns total, with the existing +per-cell deadlines (skill: original deadline; blocker: five minutes). + +Candidate and baseline use identical executable fixtures and public oracles. +Both include the same closed OpenCode compatibility projection: only actual +`tool_call` invocations named `paperclip_finish` or `paperclip_block` retain a +nested name, paired to the result by the same call ID. Arguments, arbitrary +tool names and results are not added. Old anonymous OpenCode streams cannot +retroactively satisfy this stricter identity check. +Only the five native description/fingerprint source files and their six +variant-specific unit tests may differ. The historical variant restores those +eleven files from master, including v13; candidate uses the archived native change +and v14. Source-contract hashes reject mixed revisions and changes to master +defaults, auth/runtime context or shared instructions. Derived capability metadata +is independently checked; it currently requires no regeneration in either variant. + +Credential-free admission runs before local environment/credential loading: + +```sh +node tests/runner-e2e/native-completion-checks.mjs --output-dir=/absolute/report/path +node tests/runner-e2e/native-completion-checks.mjs --verify=/absolute/report/path/preflight.json +pnpm test:e2e:runner -- --list --suite native-completion +``` + +Admission requires a committed clean exact source, variant-appropriate schema, +tool bridge/catalog and resume tests, independent oracle/default/retry calibrations, +original context-integrity calibrations, typecheck, manifest checks and exact +six-cell discovery. The Rust carrier calibration and built runner binary digest +are retained with source/build provenance. Generic stock-suite gates remain unchanged. Paid hosted runs +must dispatch the trusted default-branch workflow to distinct immutable target +branches using the exact six IDs, preserving authorization and secret boundaries. +The initial twelve hosted cells stopped at source admission with zero provider +calls. Their receipts and frozen refs remain separate from the recovery comparison. +Hosted depth-one checkouts use only the declared variant-specific immediate +parent proof when full ancestry is unavailable. The only admitted untracked +hydration is the exact build archive/checksum pair after checksum verification; +extra files, symlinks or source dirt remain failures. +Live results are pending until retained cell evidence is published. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index a9aa8109009..68eb10ae516 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -422,6 +422,8 @@ pnpm test:e2e:runner -- --list --suite agent-chat-hardening pnpm test:e2e:runner -- --id agent-chat-hardening.runner-codex.local.stop-startup-new-resume ``` +The independent, explicit-only `native-completion` suite qualifies native finish/block descriptions on unchanged master defaults. It preserves the original assigned-skill document journey and pairs it with whole-task blocking across three native profiles, with enforced single attempts. See [NATIVE-COMPLETION.md](NATIVE-COMPLETION.md) for admission, provenance and limits. + `context-integrity` is an explicit-only local suite with two bounded cases across ten listed legacy/native profiles (20 cells). Six cells are pending-prerequisite profiles and are listed for discovery but rejected before provider credentials are diff --git a/tests/runner-e2e/automatic-retry.test.ts b/tests/runner-e2e/automatic-retry.test.ts new file mode 100644 index 00000000000..eca2fc7c1f4 --- /dev/null +++ b/tests/runner-e2e/automatic-retry.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js"; +import type { FailureClass } from "./types.js"; + +const retryableClasses = ["transient_infrastructure", "provider_variance"] as const; +const allClasses: FailureClass[] = [...retryableClasses, "candidate_failure", "permanent_infrastructure", "secret_leak", "cleanup_failure"]; +const options = { maxAutomaticRetries: 1, ui: false, debug: false }; + +describe("explicit single-attempt fixture policy", () => { + for (const failureClass of allClasses) it(`retains the first ${failureClass} failure without a second attempt`, async () => { + const observedAttempts: number[] = [], retryNotifications: unknown[] = []; + const first = { status: "failed" as const, failureClass, attempt: 1, runIds: ["first-run"], cost: 3.25 }; + const result = await executeWithAutomaticRetry({ task: { automaticRetryPolicy: "single_attempt" }, options, + qualificationCandidate: false, cancelled: () => false, onRetry: result => retryNotifications.push(result), + runAttempt: async attempt => { observedAttempts.push(attempt); return first; } }); + expect(observedAttempts).toEqual([1]); + expect(retryNotifications).toEqual([]); + expect(result).toBe(first); + expect(result.runIds).toEqual(["first-run"]); + expect(result.cost).toBe(3.25); + }); + it("overrides any requested automatic retry count only for opted-in fixtures", () => { + expect(effectiveAutomaticRetryLimit({ automaticRetryPolicy: "single_attempt" }, 99)).toBe(0); + expect(effectiveAutomaticRetryLimit({}, 1)).toBe(1); + expect(effectiveAutomaticRetryLimit({}, 0)).toBe(0); + }); + for (const failureClass of retryableClasses) it(`preserves the ordinary ${failureClass} retry`, async () => { + const attempts: number[] = [], notifications: unknown[] = []; + const failed = { status: "failed" as const, failureClass, attempt: 1 }; + const passed = { status: "passed" as const, attempt: 2 }; + const result = await executeWithAutomaticRetry({ task: {}, options, qualificationCandidate: false, + cancelled: () => false, onRetry: result => notifications.push(result), + runAttempt: async attempt => { attempts.push(attempt); return attempt === 1 ? failed : passed; } }); + expect(attempts).toEqual([1, 2]); + expect(notifications).toEqual([failed]); + expect(result).toBe(passed); + }); + for (const restriction of ["qualificationCandidate", "ui", "debug", "zeroRetries"] as const) + it(`preserves the existing ${restriction} restriction`, async () => { + const attempts: number[] = []; + const first = { status: "failed" as const, failureClass: "provider_variance" as const }; + const result = await executeWithAutomaticRetry({ task: {}, options: { ...options, + ui: restriction === "ui", debug: restriction === "debug", maxAutomaticRetries: restriction === "zeroRetries" ? 0 : 1 }, + qualificationCandidate: restriction === "qualificationCandidate", cancelled: () => false, + onRetry: () => { throw new Error("Unexpected retry notification"); }, + runAttempt: async attempt => { attempts.push(attempt); return first; } }); + expect(attempts).toEqual([1]); expect(result).toBe(first); + }); + it("checks cancellation before an otherwise admitted second attempt", async () => { + const attempts: number[] = []; + await expect(executeWithAutomaticRetry({ task: {}, options, qualificationCandidate: false, + cancelled: () => true, onRetry: () => { throw new Error("Unexpected retry notification"); }, + runAttempt: async attempt => { attempts.push(attempt); return { status: "failed" as const, failureClass: "provider_variance" as const }; }, + })).rejects.toThrow("campaign cancelled"); + expect(attempts).toEqual([1]); + }); +}); diff --git a/tests/runner-e2e/automatic-retry.ts b/tests/runner-e2e/automatic-retry.ts new file mode 100644 index 00000000000..78fb35abab3 --- /dev/null +++ b/tests/runner-e2e/automatic-retry.ts @@ -0,0 +1,29 @@ +import { shouldRetryFailure } from "./failure-classifier.js"; +import type { FailureClass, RunnerTaskFixture } from "./types.js"; + +type RetryTask = Pick; +interface RetryableAttempt { status: "passed" | "failed"; failureClass?: FailureClass } +interface RetryOptions { maxAutomaticRetries: number; ui: boolean; debug: boolean } + +export function effectiveAutomaticRetryLimit(task: RetryTask, configuredLimit: number): number { + return task.automaticRetryPolicy === "single_attempt" ? 0 : configuredLimit; +} + +/** Keep the original failed result when the fixture admits exactly one attempt. */ +export async function executeWithAutomaticRetry(input: { + task: RetryTask; + options: RetryOptions; + qualificationCandidate: boolean; + runAttempt(attempt: 1 | 2): Promise; + cancelled(): boolean; + onRetry(result: Result): void; +}): Promise { + const firstResult = await input.runAttempt(1); + if (input.qualificationCandidate || input.options.ui || input.options.debug || + firstResult.status !== "failed" || !firstResult.failureClass || + !shouldRetryFailure(firstResult.failureClass, + effectiveAutomaticRetryLimit(input.task, input.options.maxAutomaticRetries))) return firstResult; + if (input.cancelled()) throw new Error("Runner E2E campaign cancelled"); + input.onRetry(firstResult); + return input.runAttempt(2); +} diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1e2a7a6561f..49748c8a6f6 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,10 +136,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 20, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, + 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 50, 6, 20, 52, 28, 18, 2, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(438); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(438); + expect(validateRunnerCatalog()).toHaveLength(444); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(444); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 15038c55de9..89301b7f111 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,3 +1,5 @@ +import { nativeCompletionTasks, nativeCompletionDefinitionDigest } from "./native-completion-cases.js"; +import { nativeCompletionProfile, NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; import { chatConfirmationTasks } from "./chat-cases.js"; import { hiringTemplateTasks, hiringTemplateProfile, hiringTemplateDefinitionDigest } from "./hiring-template-cases.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; @@ -1193,6 +1195,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ .map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`))], definitionMetadata: { version: 4, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only" }, }, + { + id: "native-completion", label: "Native completion guidance", manualOnly: true, + description: "Production-default assigned-skill document completion and concrete whole-task blocking, with independent native result/final ordering.", + groups: ["native", "local"], + profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude", "runner-opencode"].includes(profile.id)).map(nativeCompletionProfile), + environments: [localEnvironment], tasks: nativeCompletionTasks, expectedMatrixSize: 6, + definitionMetadata: { version: 1, fixtureDigest: nativeCompletionDefinitionDigest(), instructions: "unchanged-master-production-default", + providerTurns: 6, maximumAttemptsPerCell: 1, automaticRetryPolicy: "single_attempt", budgetMonthlyCents: NATIVE_COMPLETION_BUDGET_CENTS, + grading: "original-assigned-skill-durable-document-plus-native-disposition-and-observable-final-order", scheduling: "explicit-only" }, + }, { id: "context-integrity", label: "Context Integrity", diff --git a/tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json b/tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json new file mode 100644 index 00000000000..5f7e5144da3 --- /dev/null +++ b/tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json @@ -0,0 +1,55 @@ +{ + "schema": "paperclip.native-completion-terminal-carrier.fixture.v1", + "cases": [ + { + "name": "paperclip_finish", + "providerInput": { + "item": { + "id": "terminal-call", + "type": "tool_call", + "name": "paperclip_finish", + "status": "inProgress", + "arguments": { + "secret": "not-for-the-log" + } + } + }, + "normalizedPayload": { + "provider": "codex", + "itemId": "terminal-call", + "kind": "tool_call", + "status": "running", + "channel": "detail", + "text": null, + "item": { + "name": "paperclip_finish" + } + } + }, + { + "name": "paperclip_block", + "providerInput": { + "item": { + "id": "terminal-call", + "type": "tool_call", + "name": "paperclip_block", + "status": "inProgress", + "arguments": { + "secret": "not-for-the-log" + } + } + }, + "normalizedPayload": { + "provider": "codex", + "itemId": "terminal-call", + "kind": "tool_call", + "status": "running", + "channel": "detail", + "text": null, + "item": { + "name": "paperclip_block" + } + } + } + ] +} diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index ceb5927ec79..7cafc2c8f09 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -21,7 +21,8 @@ import { import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { packageEvidence } from "./evidence.js"; -import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js"; +import { classifyFailure } from "./failure-classifier.js"; +import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js"; import { buildRunnerCampaign } from "./history.js"; import { buildRunnerE2EProcessEnvironment, @@ -50,6 +51,7 @@ import { type RunnerE2EResult, } from "./types.js"; import { assertRunnerE2EPrerequisites } from "./prerequisites.js"; +import { assertNativeCompletionSelection, prepareNativeCompletionPreflight, NATIVE_COMPLETION_PREFLIGHT_ENV } from "./native-completion-admission.js"; import { reapNewDetachedDarwinSharedMemory, snapshotDarwinSharedMemory, @@ -1014,36 +1016,19 @@ async function runExecutionWithRetry(input: { options: ReturnType; }): Promise { const { execution, campaignId, options } = input; - const [firstResult] = await runAttempt({ - executions: [execution], - attempt: 1, - campaignId, - options, + return executeWithAutomaticRetry({ + task: execution.task, options, + qualificationCandidate: execution.profile.qualificationCandidate !== undefined, + cancelled: () => cancelled, + onRetry: result => console.warn( + `Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`, + ), + runAttempt: async attempt => { + const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options }); + if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`); + return result; + }, }); - if (!firstResult) throw new Error(`No result produced for ${execution.id}`); - if ( - execution.profile.qualificationCandidate !== undefined || - options.ui || - options.debug || - firstResult.status !== "failed" || - !firstResult.failureClass || - !shouldRetryFailure(firstResult.failureClass, options.maxAutomaticRetries) - ) { - return firstResult; - } - if (cancelled) throw new Error("Runner E2E campaign cancelled"); - console.warn( - `Retrying ${execution.id} in a fresh isolated harness after ${firstResult.failureClass.replaceAll("_", " ")}`, - ); - const [retryResult] = await runAttempt({ - executions: [execution], - attempt: 2, - campaignId, - options, - }); - if (!retryResult) - throw new Error(`No retry result produced for ${execution.id}`); - return retryResult; } async function runWithConcurrency( @@ -1098,6 +1083,15 @@ async function main() { // Keep admission before local-env loading and credential checks. Pending // profiles remain discoverable, but cannot reach a provider. assertRunnerE2EPrerequisites(executions); + assertNativeCompletionSelection(executions); + let nativeCampaignId: string | undefined; + if (executions.some(execution => execution.suite.id === "native-completion")) { + nativeCampaignId = cleanId(process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? + `local-${new Date().toISOString().replace(/[:.]/g, "-")}`); + const nativeCampaignDirectory = path.join(resultsRoot, nativeCampaignId); + await mkdir(nativeCampaignDirectory, { recursive: true }); + process.env[NATIVE_COMPLETION_PREFLIGHT_ENV] = prepareNativeCompletionPreflight(nativeCampaignDirectory); + } await loadLocalEnvironment(process.env); const missingCredentials = [ @@ -1119,7 +1113,7 @@ async function main() { ); } - const campaignId = cleanId( + const campaignId = nativeCampaignId ?? cleanId( process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? `local-${new Date().toISOString().replace(/[:.]/g, "-")}`, ); @@ -1132,6 +1126,11 @@ async function main() { version: 1, maxAutomaticRetries: options.maxAutomaticRetries, retryClasses: ["transient_infrastructure", "provider_variance"], + executions: executions.map(execution => ({ + executionId: execution.id, + automaticRetryPolicy: execution.task.automaticRetryPolicy ?? "default", + maxAutomaticRetries: effectiveAutomaticRetryLimit(execution.task, options.maxAutomaticRetries), + })), }, null, 2, diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index d8a4cbeccbe..da0aa8e19c9 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,3 +1,4 @@ +import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; import path from "node:path"; import { isManagedHiringCase } from "./chat-cases.js"; import { FixtureRegistry } from "./fixture-registry.js"; @@ -135,7 +136,7 @@ export async function setupLiveFixtures(input: { return api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", - budgetMonthlyCents: execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : 0, + budgetMonthlyCents: execution.suite.id === "native-completion" ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS : 0, }); }, async teardown() { diff --git a/tests/runner-e2e/native-blocker-visible.spec.ts b/tests/runner-e2e/native-blocker-visible.spec.ts new file mode 100644 index 00000000000..2a3fed5196c --- /dev/null +++ b/tests/runner-e2e/native-blocker-visible.spec.ts @@ -0,0 +1,28 @@ +import { expect, test } from "@playwright/test"; +import { assertNativeBlockerReply } from "./native-blocker-visible.js"; + +test("accepts a visible blocker explanation rather than requiring a marker-only reply", async ({ page }) => { + await page.setContent('
Deployment is blocked.
Release Owner must Grant deployment access. BLOCKED_probe
'); + await assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100); +}); + +test("accepts a future access condition while the task remains blocked", async ({ page }) => { + await page.setContent('
Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe
'); + await assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100); +}); + +for (const [name, body] of [ + ["marker only", "BLOCKED_probe"], + ["missing owner", "Blocked. Someone must Grant deployment access. BLOCKED_probe"], + ["missing action", "Blocked. Release Owner must take action. BLOCKED_probe"], + ["missing reason", "Release Owner: Grant deployment access. BLOCKED_probe"], + ["contradictory disposition", "Deployment is not blocked. Release Owner completed Grant deployment access. BLOCKED_probe"], + ["resolved blocker", "Deployment is no longer blocked. Release Owner must Grant deployment access. BLOCKED_probe"], + ["granted access", "Blocked. Access already granted. Release Owner must Grant deployment access. BLOCKED_probe"], + ["duplicate replies", "Blocked. Release Owner must Grant deployment access. BLOCKED_probe
BLOCKED_probe"], +]) { + test(`rejects ${name}`, async ({ page }) => { + await page.setContent(`
${body}
`); + await expect(assertNativeBlockerReply(page.getByTestId("reply"), "BLOCKED_probe", 100)).rejects.toThrow(); + }); +} diff --git a/tests/runner-e2e/native-blocker-visible.ts b/tests/runner-e2e/native-blocker-visible.ts new file mode 100644 index 00000000000..b9deff3d502 --- /dev/null +++ b/tests/runner-e2e/native-blocker-visible.ts @@ -0,0 +1,18 @@ +import { expect, type Locator } from "@playwright/test"; + +/** Blocker replies include an explanation; ordinary marker-only replies do not. */ +export async function assertNativeBlockerReply(agentReplies: Locator, marker: string, timeout = 30_000): Promise { + const reply = agentReplies.filter({ hasText: marker }); + await expect(reply).toHaveCount(1, { timeout }); + await expect(reply).toBeVisible({ timeout }); + await expect(reply).toContainText("Release Owner", { timeout }); + await expect(reply).toContainText("Grant deployment access", { timeout }); + await expect(reply).not.toContainText( + /\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\b/i, + { timeout }, + ); + await expect(reply).toContainText( + /\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\b/i, + { timeout }, + ); +} diff --git a/tests/runner-e2e/native-completion-admission.test.ts b/tests/runner-e2e/native-completion-admission.test.ts new file mode 100644 index 00000000000..7a9b1ef8204 --- /dev/null +++ b/tests/runner-e2e/native-completion-admission.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { spawnSync } from "node:child_process"; +import { assertNativeCompletionSelection, prepareNativeCompletionPreflight, nativeCompletionPreflightEnvironment, verifyNativeCompletionPreflight } from "./native-completion-admission.js"; +import { CREDENTIAL_NAMES, type MatrixExecution } from "./types.js"; + +vi.mock("node:child_process", () => ({ spawnSync: vi.fn() })); +const spawn = vi.mocked(spawnSync); +const campaignDirectory = "/fixture/results/gha-1-1-native-completion.fixture"; +beforeEach(() => { vi.resetAllMocks(); vi.unstubAllEnvs(); }); + +describe("native completion credential-free admission", () => { + it("allows toolchain paths and excludes every present or future credential", () => { + const source = { PATH: "/bin", HOME: "/home/fixture", CARGO_HOME: "/cargo", CI: "true", + GH_TOKEN: "secret", FUTURE_PROVIDER_API_KEY: "secret", NODE_OPTIONS: "ambient override", + ...Object.fromEntries(CREDENTIAL_NAMES.map(name => [name, "secret"])) }; + expect(nativeCompletionPreflightEnvironment(source)).toEqual({ PATH: "/bin", HOME: "/home/fixture", CARGO_HOME: "/cargo", CI: "true" }); + }); + it("carries only public hosted identity through actual prepare and verify subprocess options", () => { + const identity = { GITHUB_ACTIONS: "true", PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40), GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1" }; + for (const [name, value] of Object.entries(identity)) vi.stubEnv(name, value); + for (const name of [...CREDENTIAL_NAMES, "GH_TOKEN", "FUTURE_PROVIDER_API_KEY", "NODE_OPTIONS"]) vi.stubEnv(name, "secret"); + spawn.mockReturnValueOnce({ status: 0 } as ReturnType); + spawn.mockReturnValueOnce({ status: 0, stdout: JSON.stringify({ passed: true, sourceSha: identity.PAPERCLIP_RUNNER_E2E_SOURCE_SHA }) } as ReturnType); + prepareNativeCompletionPreflight(campaignDirectory); + expect(spawn).toHaveBeenCalledTimes(2); + for (const call of spawn.mock.calls) { + expect(call[2]?.env).toMatchObject(identity); + for (const name of [...CREDENTIAL_NAMES, "GH_TOKEN", "FUTURE_PROVIDER_API_KEY", "NODE_OPTIONS"]) expect(call[2]?.env).not.toHaveProperty(name); + } + expect(spawn.mock.calls[0]?.[1]?.some(arg => arg.startsWith("--output-dir="))).toBe(true); + expect(spawn.mock.calls[1]?.[1]?.some(arg => arg.startsWith("--verify="))).toBe(true); + }); + it("rejects missing receipts without invoking a process", () => { + expect(() => verifyNativeCompletionPreflight(undefined)).toThrow("no prerequisite receipt"); + expect(spawn).not.toHaveBeenCalled(); + }); + it("fails before provider execution when the prerequisite subprocess fails", () => { + spawn.mockReturnValue({ status: 1 } as ReturnType); + expect(() => prepareNativeCompletionPreflight(campaignDirectory)).toThrow("before provider execution"); + expect(spawn).toHaveBeenCalledTimes(1); + }); + it("retains credential-free prerequisites inside the exact campaign root and verifies them", () => { + const sourceSha = "c".repeat(40); + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", sourceSha); + vi.stubEnv("OPENAI_API_KEY", "secret"); vi.stubEnv("UNLISTED_PROVIDER_CREDENTIAL", "secret"); + spawn.mockReturnValueOnce({ status: 0 } as ReturnType); + spawn.mockReturnValueOnce({ status: 0, stdout: JSON.stringify({ passed: true, sourceSha }) } as ReturnType); + const receipt = prepareNativeCompletionPreflight(campaignDirectory); + expect(receipt).toMatch(/^\/fixture\/results\/gha-1-1-native-completion\.fixture\/native-completion-prerequisites\/[^/]+\/preflight.json$/); + expect(spawn.mock.calls[1]?.[1]).toContain(`--verify=${receipt}`); + for (const call of spawn.mock.calls) { + expect(call[2]?.env).not.toHaveProperty("OPENAI_API_KEY"); + expect(call[2]?.env).not.toHaveProperty("UNLISTED_PROVIDER_CREDENTIAL"); + } + }); + it("rejects a workflow source override that differs from the exact receipt SHA", () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "a".repeat(40)); + spawn.mockReturnValue({ status: 0, stdout: JSON.stringify({ passed: true, sourceSha: "b".repeat(40) }) } as ReturnType); + expect(() => verifyNativeCompletionPreflight("/fixture/preflight.json")).toThrow("source SHA differs"); + spawn.mockReturnValue({ status: 0, stdout: JSON.stringify({ passed: true, sourceSha: "a".repeat(40) }) } as ReturnType); + expect(verifyNativeCompletionPreflight("/fixture/preflight.json").sourceSha).toBe("a".repeat(40)); + }); + it("refuses failed or unreadable receipt verification", () => { + spawn.mockReturnValue({ status: 1, stderr: "stale source" } as ReturnType); + expect(() => verifyNativeCompletionPreflight("/fixture/preflight.json")).toThrow("stale source"); + spawn.mockReturnValue({ status: 0, stdout: "malformed" } as ReturnType); + expect(() => verifyNativeCompletionPreflight("/fixture/preflight.json")).toThrow(); + }); +}); + +function selectedFixture() { + const tasks = ["assigned-skill-explicit-invocation", "native-blocked-report"].map(id => + ({ id, expectedRunCount: 1, automaticRetryPolicy: "single_attempt" })); + return { id: "native-completion.runner-codex.local.assigned-skill-explicit-invocation", + profile: { id: "runner-codex", generation: "native" }, environment: { id: "local" }, task: tasks[0], + suite: { id: "native-completion", manualOnly: true, expectedMatrixSize: 6, tasks } }; +} +describe("native completion selected-fixture policy", () => { + it("admits an exact subset of the six declared one-turn single-attempt cells", () => { + const base = selectedFixture(); + const executions = ["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(id => base.suite.tasks.map(task => + ({ ...structuredClone(base), profile: { id, generation: "native" }, task, + id: `native-completion.${id}.local.${task.id}` }))); + expect(() => assertNativeCompletionSelection(executions as unknown as MatrixExecution[])).not.toThrow(); + expect(() => assertNativeCompletionSelection([executions[0]!] as unknown as MatrixExecution[])).not.toThrow(); + }); + const invalid: Array<[string, (fixture: ReturnType) => void]> = [ + ["a legacy generation", f => { f.profile.generation = "legacy"; }], + ["an unlisted profile", f => { f.profile.id = "runner-other"; }], + ["a remote environment", f => { f.environment.id = "daytona"; }], + ["an unexpected case", f => { f.task!.id = "other-task"; }], + ["an extra expected turn", f => { f.task!.expectedRunCount = 2; }], + ["a missing single-attempt policy", f => { f.task!.automaticRetryPolicy = "default"; }], + ["a forged execution ID", f => { f.id = "other-cell"; }], + ["an automatic suite", f => { f.suite.manualOnly = false; }], + ["an expanded matrix", f => { f.suite.expectedMatrixSize = 7; }], + ["a missing original completion case", f => { f.suite.tasks.shift(); }], + ["an extra task", f => { f.suite.tasks.push({ ...f.suite.tasks[1]!, id: "extra-task" }); }], + ]; + for (const [label, mutate] of invalid) it(`rejects ${label} before credentials`, () => { + const f = selectedFixture(); mutate(f); + expect(() => assertNativeCompletionSelection([f] as unknown as MatrixExecution[])).toThrow("before provider execution"); + }); + it("leaves ordinary suites and their retry policies unchanged", () => { + const f = selectedFixture(); f.suite.id = "ordinary-suite"; f.task!.automaticRetryPolicy = "default"; + expect(() => assertNativeCompletionSelection([f] as unknown as MatrixExecution[])).not.toThrow(); + }); +}); diff --git a/tests/runner-e2e/native-completion-admission.ts b/tests/runner-e2e/native-completion-admission.ts new file mode 100644 index 00000000000..ca429c405c5 --- /dev/null +++ b/tests/runner-e2e/native-completion-admission.ts @@ -0,0 +1,60 @@ +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import path from "node:path"; +import type { MatrixExecution } from "./types.js"; + +const root = path.resolve(import.meta.dirname, "../.."); +export const NATIVE_COMPLETION_PREFLIGHT_ENV = "PAPERCLIP_RUNNER_E2E_NATIVE_COMPLETION_PREFLIGHT"; + +/** Explicit fixture policy is enforced even when the workflow supplies only --id. */ +export function assertNativeCompletionSelection(executions: readonly MatrixExecution[]): void { + const profiles = ["runner-codex", "runner-acpx-claude", "runner-opencode"]; + const tasks = ["assigned-skill-explicit-invocation", "native-blocked-report"]; + for (const execution of executions.filter(execution => execution.suite.id === "native-completion")) { + const suite = execution.suite; + if (!profiles.includes(execution.profile.id) || execution.profile.generation !== "native" || + execution.profile.qualificationCandidate !== undefined || execution.environment.id !== "local" || + !tasks.includes(execution.task.id) || execution.task.expectedRunCount !== 1 || + (execution.task.minimumExpectedRunCount !== undefined && execution.task.minimumExpectedRunCount !== 1) || + execution.task.automaticRetryPolicy !== "single_attempt" || + execution.id !== `native-completion.${execution.profile.id}.local.${execution.task.id}` || + suite.manualOnly !== true || suite.expectedMatrixSize !== 6 || suite.tasks.length !== 2 || + !tasks.every(id => suite.tasks.filter(task => task.id === id && task.expectedRunCount === 1 && + task.automaticRetryPolicy === "single_attempt").length === 1)) { + throw new Error("Native completion requires the declared local native fixtures, one turn and one attempt before provider execution."); + } + } +} + +/** Future credentials and ambient auth overrides stay outside prerequisite children. */ +export function nativeCompletionPreflightEnvironment(source: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return Object.fromEntries([ + "PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL", + "CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS", + "PAPERCLIP_RUNNER_E2E_SOURCE_SHA", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", + ].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]])); +} + +export function verifyNativeCompletionPreflight(receiptPath: string | undefined) { + if (!receiptPath) throw new Error("Native completion has no prerequisite receipt; use the live launcher."); + const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/native-completion-checks.mjs"), `--verify=${receiptPath}`], { + cwd: root, env: nativeCompletionPreflightEnvironment(process.env), encoding: "utf8", timeout: 90_000, + }); + if (run.status !== 0) throw new Error(`Native completion prerequisite verification failed: ${run.stderr || run.error?.message || run.status}`); + const receipt = JSON.parse(run.stdout) as Record; + const expectedSourceSha = process.env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA?.trim(); + if (expectedSourceSha && receipt.sourceSha !== expectedSourceSha) + throw new Error("Native completion prerequisite source SHA differs from the requested immutable source."); + return receipt; +} + +export function prepareNativeCompletionPreflight(campaignDirectory: string) { + const output = path.join(campaignDirectory, "native-completion-prerequisites", randomUUID()); + const receipt = path.join(output, "preflight.json"); + const run = spawnSync(process.execPath, [path.join(root, "tests/runner-e2e/native-completion-checks.mjs"), `--output-dir=${output}`], { + cwd: root, env: nativeCompletionPreflightEnvironment(process.env), stdio: "inherit", timeout: 35 * 60_000, + }); + if (run.status !== 0) throw new Error(`Native completion prerequisites failed before provider execution. Retained receipt: ${receipt}`); + verifyNativeCompletionPreflight(receipt); + return receipt; +} diff --git a/tests/runner-e2e/native-completion-cases.ts b/tests/runner-e2e/native-completion-cases.ts new file mode 100644 index 00000000000..8cea25b7c93 --- /dev/null +++ b/tests/runner-e2e/native-completion-cases.ts @@ -0,0 +1,47 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerTaskFixture } from "./types.js"; +import { contextIntegrityTasks } from "./context-integrity-cases.js"; + +export const nativeBlockedReportTask: RunnerTaskFixture = { + id: "native-blocked-report", label: "Native concrete blocker report", groups: ["native-completion"], + workMode: "standard", flow: "single_turn", expectedRunCount: 1, automaticRetryPolicy: "single_attempt", + attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 }, + expectedTerminalState: { issue: "blocked", run: "succeeded" }, + buildTitle: nonce => `Native blocker report ${nonce}`, + buildVisibleMarker: nonce => `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}`, + buildPrompt: nonce => [ + "Deployment cannot proceed because required release access has not been granted.", + "Release Owner is the external owner of that grant. Do not attempt deployment, run shell/process commands, change files, invoke external services, create documents, agents, interactions or subtasks, or claim the access exists.", + 'Record the whole task as blocked and identify the owner as Release Owner. The exact unblock action is "Grant deployment access" (without quotation marks or trailing punctuation).', + `Include BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")} exactly once in your final explanation.`, + ].join("\n"), + buildMatchers: nonce => [ + { kind: "issue_status", expected: "blocked" }, { kind: "run_status", expected: "succeeded" }, + { kind: "runtime_mode", expected: "native" }, + { kind: "message_contains", expected: `BLOCKED_${nonce.replace(/[^a-z0-9]/gi, "")}` }, + { kind: "message_contains", expected: "Release Owner" }, + { kind: "message_contains", expected: "Grant deployment access" }, + { kind: "message_regex", pattern: "^(?![\\s\\S]*\\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\\b)[\\s\\S]*\\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\\b", flags: "i" }, + { kind: "json_path", path: "run.resultJson.nativeResult.reportedWorkDisposition", expected: "blocked" }, + { kind: "json_path", path: "run.resultJson.nativeResult.blocker.owner.name", expected: "Release Owner" }, + { kind: "json_path", path: "run.resultJson.nativeResult.blocker.unblockAction", expected: "Grant deployment access" }, + { kind: "json_path", path: "run.resultJson.nativeResult.blocker.scope", expected: "task_wide" }, + ], +}; + +const assignedSkill = contextIntegrityTasks.find(task => task.id === "assigned-skill-explicit-invocation"); +if (!assignedSkill) throw new Error("Missing original assigned-skill completion journey"); +// Preserve the original task prompt, skill, durable-document oracle and deadline. +export const nativeCompletionTasks: readonly RunnerTaskFixture[] = [ + { ...assignedSkill, automaticRetryPolicy: "single_attempt" }, nativeBlockedReportTask, +]; +export function nativeCompletionDefinitionDigest() { + const hash = createHash("sha256"); + for (const file of ["native-completion-cases.ts", "native-completion-scoring.ts", "native-completion-defaults.ts", + "native-blocker-visible.ts", "native-completion-admission.ts", "native-completion-checks.mjs", + "native-completion-source-contract.mjs", "automatic-retry.ts", "context-integrity-cases.ts", "context-integrity-flow.ts", "context-integrity-scoring.ts", "types.ts", "catalog.ts", "live-fixtures.ts", "runner.spec.ts", "launch.ts"]) { + hash.update(file); hash.update(readFileSync(new URL(file, import.meta.url))); + } + return hash.digest("hex"); +} diff --git a/tests/runner-e2e/native-completion-checks.mjs b/tests/runner-e2e/native-completion-checks.mjs new file mode 100644 index 00000000000..7d326e3bec5 --- /dev/null +++ b/tests/runner-e2e/native-completion-checks.mjs @@ -0,0 +1,297 @@ +// Independent, credential-free admission for the native completion comparison. +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { + NATIVE_COMPLETION_SOURCE_CONTRACT, NATIVE_COMPLETION_SOURCE_FILES, + nativeCompletionSourceFingerprint, nativeSourceSha256, +} from "./native-completion-source-contract.mjs"; + +import { inspectNativeCompletionSourceMetadata, inspectNativeCompletionRunnerd, NATIVE_COMPLETION_RUNNERD_PATH } from "./native-completion-git-source.mjs"; + +const root = resolve(import.meta.dirname, "../.."); +export const NATIVE_COMPLETION_PREFLIGHT_SCHEMA = "paperclip.native-completion-preflight.v1"; +export const NATIVE_COMPLETION_CELL_IDS = ["runner-codex", "runner-acpx-claude", "runner-opencode"].flatMap(profile => + ["assigned-skill-explicit-invocation", "native-blocked-report"].map(task => `native-completion.${profile}.local.${task}`)); +export function nativeCompletionGates(variant) { + if (!["candidate", "historical"].includes(variant)) throw new Error("Unknown native completion source variant"); + return [ + { id: "NC-schemas", name: "Variant-native completion schemas", cwd: "packages/paperclip-runner", + files: ["src/contracts/completion-result.test.ts"], + required: ["distinguishes user-facing answer content", "propagates answer and wait descriptions", "requires a reason code for verification that was not run"] }, + { id: "NC-tools", name: "Native tool delivery and retained provider catalogs", cwd: "packages/paperclip-runner", files: [ + "src/drivers/codex/codex-app-server-driver.lifecycle.test.ts", "src/drivers/runner-tool-bridge.test.ts", + "src/drivers/opencode/mcp-bridge.test.ts", "src/live/runnerd-codex-transport.test.ts"], + testPattern: "binds to loopback|preserves stock Codex instructions on|includes ACPX terminal tools|preserves answer and internal wait descriptions", + required: ["runner semantic MCP bridge binds to loopback", "OpenCode MCP bridge binds to loopback", + "preserves stock Codex instructions on task recovery", "preserves stock Codex instructions on prepared recovery", + "preserves stock Codex instructions on direct recovery", "includes ACPX terminal tools in the authenticated bridge catalog", + ...(variant === "candidate" ? ["codex", "opencode", "claude_managed", "aws_agentcore", "acpx"].map(provider => + `preserves answer and internal wait descriptions in the serialized native ${provider} tool catalog`) : [])] }, + { id: "NC-resume", name: "Variant-native fingerprint and checkpoint refresh", cwd: ".", + files: ["server/src/services/native-runtime/native-session-resume.test.ts"], testPattern: "refreshes retained", + required: ["refreshes retained 'completion descriptions'", "refreshes retained 'task-bound human-input description'", + ...(variant === "candidate" ? ["native completion tool guidance"] : [])] }, + { id: "NC-eval", name: "Independent native oracle, defaults, admission and attempt policy", cwd: ".", + config: "tests/runner-e2e/vitest.config.ts", files: [ + "tests/runner-e2e/native-completion-scoring.test.ts", "tests/runner-e2e/native-completion-defaults.test.ts", + "tests/runner-e2e/native-completion-admission.test.ts", "tests/runner-e2e/automatic-retry.test.ts", + "tests/runner-e2e/context-integrity.test.ts", "tests/runner-e2e/context-integrity-evidence.test.ts", + "tests/runner-e2e/context-integrity-flow-logs.test.ts"], + required: ["allows toolchain paths and excludes every present or future credential", + "retains credential-free prerequisites inside the exact campaign root", + "retains the first transient_infrastructure failure without a second attempt", + "retains the first provider_variance failure without a second attempt"] }, + ]; +} +export const NATIVE_COMPLETION_COMMAND_GATE_IDS = ["NC-node", "NC-typecheck", "NC-manifest", "NC-discovery", "NC-rust-carrier"]; +export function nativeCompletionCommandGateIds(mode) { + return NATIVE_COMPLETION_COMMAND_GATE_IDS.map(id => mode === "trusted_hosted_archive" && id === "NC-rust-carrier" ? "NC-hosted-runnerd" : id); +} +export function gradeNativeCompletionGate(gate, report, exitCode) { + const assertions = (report?.testResults ?? []).flatMap(file => file.assertionResults ?? []); + const requirements = gate.required.map(name => ({ name, passed: assertions.some(assertion => + assertion.fullName?.includes(name) && assertion.status === "passed") })); + const files = gate.files.map(file => ({ file, passed: (report?.testResults ?? []).some(result => + result.name?.endsWith(file) && result.status === "passed" && result.assertionResults?.some(assertion => assertion.status === "passed") && + result.assertionResults.every(assertion => assertion.status === "passed" || (gate.testPattern && assertion.status === "skipped" && + !gate.required.some(name => assertion.fullName?.includes(name))))) })); + return { id: gate.id, name: gate.name, passed: exitCode === 0 && requirements.every(row => row.passed) && files.every(row => row.passed), + exitCode, files, requirements, total: report?.numTotalTests ?? 0, passedTests: report?.numPassedTests ?? 0, + failedTests: report?.numFailedTests ?? 0, pendingTests: report?.numPendingTests ?? 0 }; +} + +export function gradeNativeCompletionDiscovery(output, exitCode) { + const lines = output.trim().split(/\r?\n/).filter(Boolean); + const rows = lines.slice(1).map(line => line.split("\t")); + const ids = rows.map(row => row[0]); + return { passed: exitCode === 0 && lines[0] === "ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS" && + rows.length === 6 && new Set(ids).size === 6 && NATIVE_COMPLETION_CELL_IDS.every(id => ids.includes(id)) && + rows.every(row => row.length === 6 && row[1] === "native-completion" && row[2] === "native" && row[4]?.trim() && row[5]?.trim()), + executionIds: ids, expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1 }; +} +/** capture() stores compact provenance JSON on its first line, then diagnostics. */ +export function gradeNativeCompletionHostedRunnerdEvidence(text, expected) { + try { return JSON.stringify(JSON.parse(text.split(/\r?\n/, 1)[0])) === JSON.stringify(expected); } + catch { return false; } +} +export function gradeNativeCompletionRustCarrier(output, exitCode) { + return exitCode === 0 && /^test provider_events::tests::preserves_closed_compatibility_terminal_tool_identity \.\.\. ok$/m.test(output) && + /test result: ok\. 1 passed; 0 failed;/.test(output); +} +export function nativeCompletionPrerequisiteEnvironment(source) { + return Object.fromEntries(["PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SYSTEMROOT", "LANG", "LC_ALL", + "CARGO_HOME", "RUSTUP_HOME", "CI", "GITHUB_ACTIONS", "GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT", "PAPERCLIP_RUNNER_E2E_SOURCE_SHA"].flatMap(name => source[name] === undefined ? [] : [[name, source[name]]])); +} +function currentSource() { + const source = nativeCompletionSourceFingerprint(); + return { ...source, ...inspectNativeCompletionSourceMetadata({ repositoryRoot: root, + sourceFiles: NATIVE_COMPLETION_SOURCE_FILES, baseSha: source.baseSha, variant: source.variant, + shallowParentAnchors: NATIVE_COMPLETION_SOURCE_CONTRACT.shallowParentAnchors, + }) }; +} +function buildOutputFingerprint() { + const hash = createHash("sha256"); + function visit(relative) { + for (const entry of readdirSync(join(root, relative), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const file = join(relative, entry.name); + if (entry.isDirectory()) visit(file); + else if (entry.isFile()) { const bytes = readFileSync(join(root, file)); hash.update(JSON.stringify([file, bytes.length])).update(bytes); } + else throw new Error(`Unexpected build output: ${file}`); + } + } + for (const directory of ["packages/shared/dist", "packages/plugins/sdk/dist", "packages/paperclip-runner/dist"]) { + if (!existsSync(join(root, directory, "index.js"))) throw new Error(`Missing prerequisite build output: ${directory}`); + visit(directory); + } + const daemon = runnerdBinary(); + const bytes = readFileSync(daemon); + hash.update(JSON.stringify(["debug/paperclip-runnerd", bytes.length])).update(bytes); + return hash.digest("hex"); +} +function runnerdBinary() { + return join(root, `${NATIVE_COMPLETION_RUNNERD_PATH}${process.platform === "win32" ? ".exe" : ""}`); +} +function runnerdProvenance(source, env) { + return inspectNativeCompletionRunnerd({ repositoryRoot: root, sourceSha: source.sha, + sourceFingerprint: source.fingerprint, environment: env }); +} +export function assertNativeCompletionPreflightReceipt(report, current) { + const expected = [...nativeCompletionGates(current.variant).map(gate => gate.id), ...nativeCompletionCommandGateIds(current.runnerdProvenance?.mode)]; + const hostedGate = report?.gates?.find(gate => gate.id === "NC-hosted-runnerd"); + const truthfulHosted = current.runnerdProvenance?.mode !== "trusted_hosted_archive" || + hostedGate?.executed === false && hostedGate?.calibration === "not_executed" && hostedGate?.total === 0 && + hostedGate?.passedTests === 0 && hostedGate?.reuse === "trusted_same_run_build"; + if (!truthfulHosted || report?.schema !== NATIVE_COMPLETION_PREFLIGHT_SCHEMA || report.passed !== true || report.providerCalls !== 0 || + report.live !== "not_run" || report.sourceSha !== current.sha || !/^[a-f0-9]{40}$/.test(current.sha ?? "") || + ![current.fingerprint, current.fixtureFingerprint, current.manifestFingerprint, current.buildOutputFingerprint] + .every(value => typeof value === "string" && /^[a-f0-9]{64}$/.test(value)) || + report.sourceFingerprint !== current.fingerprint || report.fixtureFingerprint !== current.fixtureFingerprint || + report.manifestFingerprint !== current.manifestFingerprint || report.variant !== current.variant || + report.baseSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha || report.archiveSha !== NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha || + report.layering !== true || current.layering !== true || report.immutable !== true || current.immutable !== true || + !/^[a-f0-9]{64}$/.test(current.sourceMetadataFingerprint ?? "") || + report.sourceMetadataFingerprint !== current.sourceMetadataFingerprint || + !Array.isArray(report.sourceMetadataErrors) || report.sourceMetadataErrors.length !== 0 || current.sourceMetadataErrors?.length !== 0 || + !Array.isArray(report.sourceErrors) || report.sourceErrors.length !== 0 || current.sourceErrors?.length !== 0 || + report.setup?.passed !== true || report.setup.sdkExitCode !== 0 || report.setup.runnerTypeScriptExitCode !== 0 || + report.setup.runnerdExitCode !== (current.runnerdProvenance?.mode === "trusted_hosted_archive" ? null : 0) || + current.runnerdProvenance?.passed !== true || !["trusted_hosted_archive", "fresh_local_build"].includes(current.runnerdProvenance?.mode) || + JSON.stringify(report.setup.runnerdProvenance) !== JSON.stringify(current.runnerdProvenance) || + report.setup.runnerdSelectedPath !== current.runnerdProvenance.selectedPath || + report.setup.runnerdSha256 !== current.runnerdProvenance.binarySha256 || + !/^[a-f0-9]{64}$/.test(current.runnerdProvenance.binarySha256 ?? "") || + report.setup.runnerdSourceSha !== current.sha || report.setup.runnerdSourceFingerprint !== current.fingerprint || + report.setup.buildOutputFingerprint !== current.buildOutputFingerprint || + !Array.isArray(report.setup.evidence) || report.setup.evidence.length !== 3 || + ["setup-sdk.txt", "setup-runner-typescript.txt", "setup-runnerd.txt"].some(file => report.setup.evidence.filter(row => + row.file === file && /^[a-f0-9]{64}$/.test(row.sha256 ?? "")).length !== 1) || + !Array.isArray(report.gates) || report.gates.length !== expected.length || + expected.some(id => report.gates.filter(gate => gate.id === id && gate.passed === true && gate.exitCode === 0).length !== 1) || + report.expectedCells !== 6 || report.expectedTurns !== 6 || report.maximumAttemptsPerCell !== 1) { + throw new Error("Native completion requires passing prerequisites for this exact immutable source SHA, variant and fingerprint."); + } + return report; +} +function runCommand(command, args, env, timeout = 10 * 60_000, cwd = root) { + return spawnSync(command, args, { cwd, env, encoding: "utf8", timeout }); +} +function capture(output, name, run) { + const file = `${name}.txt`, bytes = `${run?.stdout ?? ""}\n${run?.stderr ?? ""}`; + writeFileSync(join(output, file), bytes); + return { file, sha256: nativeSourceSha256(bytes) }; +} +export function assertRetainedNativeCompletionEvidence(output, evidence) { + if (!evidence || typeof evidence.file !== "string" || !/^[A-Za-z0-9_.-]+$/.test(evidence.file) || + nativeSourceSha256(readFileSync(join(output, evidence.file))) !== evidence.sha256) + throw new Error("Native completion retained prerequisite evidence changed or is unavailable."); + return readFileSync(join(output, evidence.file), "utf8"); +} +function manifestCommands(env) { + return ["generate-capability-contract.mjs", "check-capability-inventory.mjs"].map(file => runCommand(process.execPath, + [join(root, "packages/paperclip-runner/scripts", file), ...(file.startsWith("generate-") ? ["--check"] : [])], env)); +} + +export function main(args = process.argv.slice(2)) { + if (args.includes("--list")) { + console.log(JSON.stringify({ variants: ["candidate", "historical"], gates: nativeCompletionGates("candidate"), + commands: { local: nativeCompletionCommandGateIds("fresh_local_build"), hosted: nativeCompletionCommandGateIds("trusted_hosted_archive") }, + cells: NATIVE_COMPLETION_CELL_IDS, providerCalls: 0 }, null, 2)); return; + } + if (args.some(arg => !arg.startsWith("--output-dir=") && !arg.startsWith("--verify="))) + throw new Error("Use --list, --output-dir= or --verify=; never paid providers."); + const source = currentSource(), env = nativeCompletionPrerequisiteEnvironment(process.env); + const verify = args.find(arg => arg.startsWith("--verify="))?.slice("--verify=".length); + if (verify) { + const report = assertNativeCompletionPreflightReceipt(JSON.parse(readFileSync(verify, "utf8")), { + ...source, buildOutputFingerprint: buildOutputFingerprint(), runnerdProvenance: runnerdProvenance(source, env), + }); + const output = resolve(verify, ".."); + for (const evidence of report.setup.evidence) assertRetainedNativeCompletionEvidence(output, evidence); + for (const gate of nativeCompletionGates(source.variant)) { + const retained = report.gates.find(row => row.id === gate.id); + const grade = gradeNativeCompletionGate(gate, JSON.parse(assertRetainedNativeCompletionEvidence(output, retained.evidence)), 0); + if (!grade.passed) throw new Error(`Missing or failed retained native prerequisite assertions: ${gate.id}`); + } + for (const id of nativeCompletionCommandGateIds(report.setup.runnerdProvenance.mode)) { + const retained = report.gates.find(row => row.id === id), text = assertRetainedNativeCompletionEvidence(output, retained.evidence); + if (id === "NC-node" && (!/# fail 0\b/.test(text) || !/# tests [1-9]\d*\b/.test(text))) + throw new Error("Missing passing native admission calibrations."); + if (id === "NC-discovery" && !gradeNativeCompletionDiscovery(text.split("\n\n")[0], 0).passed) + throw new Error("Native completion six-cell discovery changed."); + if (id === "NC-rust-carrier" && !gradeNativeCompletionRustCarrier(text, 0)) + throw new Error("Missing passing retained Rust terminal-tool carrier calibration."); + if (id === "NC-hosted-runnerd" && (retained.executed !== false || retained.calibration !== "not_executed" || + retained.total !== 0 || retained.passedTests !== 0 || + !gradeNativeCompletionHostedRunnerdEvidence(text, report.setup.runnerdProvenance))) + throw new Error("Hosted runnerd reuse must retain exact binary proof and report Rust calibration not executed."); + if (id === "NC-manifest" && manifestCommands(env).some(run => run.status !== 0)) + throw new Error("Generated native capability manifests are stale."); + } + console.log(JSON.stringify(report)); return report; + } + const output = resolve(args.find(arg => arg.startsWith("--output-dir="))?.slice("--output-dir=".length) ?? + join(root, "tests/runner-e2e/results", `native-completion-preflight-${new Date().toISOString().replaceAll(":", "-")}`)); + mkdirSync(output, { recursive: true }); + const report = { schema: NATIVE_COMPLETION_PREFLIGHT_SCHEMA, sourceSha: source.sha, sourceFingerprint: source.fingerprint, + fixtureFingerprint: source.fixtureFingerprint, manifestFingerprint: source.manifestFingerprint, + variant: source.variant, baseSha: source.baseSha, archiveSha: source.archiveSha, + sourceErrors: source.sourceErrors, immutable: source.immutable, layering: source.layering, + sourceMetadata: source.sourceMetadata, sourceMetadataErrors: source.sourceMetadataErrors, + sourceMetadataFingerprint: source.sourceMetadataFingerprint, + measuredAt: new Date().toISOString(), providerCalls: 0, live: "not_run", expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1, + setup: { passed: false, sdkExitCode: null, runnerTypeScriptExitCode: null, runnerdExitCode: null, evidence: [] }, gates: [], passed: false }; + if (!source.sha || source.sourceErrors.length || !source.layering || !source.immutable) { + writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); + console.error(`Native completion source admission failed before providers: ${[...source.sourceErrors, ...source.sourceMetadataErrors].join("; ")}`); + process.exitCode = 1; return report; + } + const sdk = runCommand(process.execPath, [join(root, "scripts/ensure-plugin-build-deps.mjs")], env, 5 * 60_000); + report.setup.sdkExitCode = sdk.status; report.setup.evidence.push(capture(output, "setup-sdk", sdk)); + const runner = sdk.status === 0 ? runCommand("pnpm", ["--filter", "@paperclipai/paperclip-runner", "build:typescript"], env) : null; + report.setup.runnerTypeScriptExitCode = runner?.status ?? null; + report.setup.evidence.push(capture(output, "setup-runner-typescript", runner)); + const hosted = env.GITHUB_ACTIONS === "true"; + const runnerd = sdk.status === 0 && runner?.status === 0 && !hosted ? runCommand("cargo", + ["build", "--locked", "--offline", "--workspace", "--bins"], env, + 10 * 60_000, join(root, "packages/paperclip-runner/runner")) : null; + report.setup.runnerdExitCode = runnerd?.status ?? null; + report.setup.runnerdProvenance = runnerdProvenance(source, env); + report.setup.evidence.push(capture(output, "setup-runnerd", hosted + ? { stdout: JSON.stringify(report.setup.runnerdProvenance), stderr: "Rust build and unit calibration not executed in this hosted cell; reusing trusted same-run build." } + : runnerd)); + report.setup.passed = sdk.status === 0 && runner?.status === 0 && (hosted || runnerd?.status === 0) && report.setup.runnerdProvenance.passed; + if (report.setup.passed) { + report.setup.buildOutputFingerprint = buildOutputFingerprint(); + report.setup.runnerdSha256 = report.setup.runnerdProvenance.binarySha256; + report.setup.runnerdSelectedPath = report.setup.runnerdProvenance.selectedPath; + report.setup.runnerdSourceSha = source.sha; + report.setup.runnerdSourceFingerprint = source.fingerprint; + for (const gate of nativeCompletionGates(source.variant)) { + console.log(`Checking ${gate.id}: ${gate.name}`); + const file = `${gate.id}.json`, run = runCommand(process.execPath, [join(root, "node_modules/vitest/vitest.mjs"), "run", ...gate.files, + ...(gate.config ? ["--config", gate.config] : []), ...(gate.testPattern ? ["--testNamePattern", gate.testPattern] : []), + "--reporter=default", "--reporter=json", `--outputFile.json=${join(output, file)}`], env, 10 * 60_000, resolve(root, gate.cwd)); + capture(output, gate.id, run); + let result; try { result = JSON.parse(readFileSync(join(output, file), "utf8")); } catch { /* Missing discovery fails closed. */ } + report.gates.push({ ...gradeNativeCompletionGate(gate, result, run.status), + evidence: existsSync(join(output, file)) ? { file, sha256: nativeSourceSha256(readFileSync(join(output, file))) } : null }); + } + const commands = [ + { id: "NC-node", command: process.execPath, args: ["--test", "--test-reporter=tap", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs"] }, + { id: "NC-typecheck", command: process.execPath, args: ["node_modules/typescript/bin/tsc", "-p", "tests/runner-e2e/tsconfig.json"] }, + ...(!hosted ? [{ id: "NC-rust-carrier", command: "cargo", args: ["test", "--locked", "--offline", + "-p", "paperclip-runner-core", "--lib", "provider_events::tests::preserves_closed_compatibility_terminal_tool_identity", "--", "--exact"], + cwd: join(root, "packages/paperclip-runner/runner") }] : []), + { id: "NC-discovery", command: process.execPath, args: ["--import", "./server/node_modules/tsx/dist/loader.mjs", "tests/runner-e2e/launch.ts", "--list", "--suite", "native-completion"] }, + ]; + for (const command of commands) { + console.log(`Checking ${command.id}`); + const run = runCommand(command.command, command.args, env, 10 * 60_000, command.cwd ?? root); + report.gates.push({ id: command.id, passed: run.status === 0 && + (command.id !== "NC-discovery" || gradeNativeCompletionDiscovery(run.stdout, run.status).passed) && + (command.id !== "NC-rust-carrier" || gradeNativeCompletionRustCarrier(`${run.stdout ?? ""}\n${run.stderr ?? ""}`, run.status)) && + (command.id !== "NC-node" || /# fail 0\b/.test(run.stdout) && /# tests [1-9]\d*\b/.test(run.stdout)), + exitCode: run.status, evidence: capture(output, command.id, run) }); + } + if (hosted) report.gates.push({ id: "NC-hosted-runnerd", name: "Trusted hosted runnerd provenance; Rust calibration not executed", + passed: report.setup.runnerdProvenance.passed, exitCode: 0, executed: false, calibration: "not_executed", + reuse: "trusted_same_run_build", total: 0, passedTests: 0, evidence: capture(output, "NC-hosted-runnerd", { + stdout: JSON.stringify(report.setup.runnerdProvenance), stderr: "Rust unit calibration must be qualified by the separate exact-source local receipt and normal CI." }) }); + const manifest = manifestCommands(env), combined = { stdout: manifest.map(run => run.stdout ?? "").join("\n"), stderr: manifest.map(run => run.stderr ?? "").join("\n") }; + report.gates.push({ id: "NC-manifest", passed: manifest.every(run => run.status === 0), + exitCode: manifest.find(run => run.status !== 0)?.status ?? 0, evidence: capture(output, "NC-manifest", combined) }); + const after = currentSource(); + report.passed = report.gates.every(gate => gate.passed) && after.immutable && after.layering && + after.sha === source.sha && after.fingerprint === source.fingerprint && after.sourceErrors.length === 0 && + after.sourceMetadataFingerprint === source.sourceMetadataFingerprint && after.sourceMetadataErrors.length === 0 && + buildOutputFingerprint() === report.setup.buildOutputFingerprint && + JSON.stringify(runnerdProvenance(after, env)) === JSON.stringify(report.setup.runnerdProvenance); + } + writeFileSync(join(output, "preflight.json"), JSON.stringify(report, null, 2) + "\n"); + console.log(`Native completion prerequisite evidence: ${join(output, "preflight.json")}`); + if (!report.passed) process.exitCode = 1; + return report; +} +if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) main(); diff --git a/tests/runner-e2e/native-completion-checks.test.mjs b/tests/runner-e2e/native-completion-checks.test.mjs new file mode 100644 index 00000000000..36ca65c0c71 --- /dev/null +++ b/tests/runner-e2e/native-completion-checks.test.mjs @@ -0,0 +1,182 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + nativeCompletionGates, gradeNativeCompletionGate, gradeNativeCompletionDiscovery, gradeNativeCompletionRustCarrier, gradeNativeCompletionHostedRunnerdEvidence, + assertNativeCompletionPreflightReceipt, assertRetainedNativeCompletionEvidence, nativeCompletionPrerequisiteEnvironment, + NATIVE_COMPLETION_PREFLIGHT_SCHEMA, NATIVE_COMPLETION_CELL_IDS, NATIVE_COMPLETION_COMMAND_GATE_IDS, nativeCompletionCommandGateIds, +} from "./native-completion-checks.mjs"; +import { nativeSourceSha256, NATIVE_COMPLETION_SOURCE_CONTRACT } from "./native-completion-source-contract.mjs"; + +const gate = { id: "NC-test", name: "Boundary", files: ["boundary.test.ts"], required: ["native assertion"] }; +const report = () => ({ testResults: [{ name: "/repo/boundary.test.ts", status: "passed", assertionResults: [ + { fullName: "Boundary native assertion", status: "passed" }] }], numTotalTests: 1, numPassedTests: 1 }); +test("native completion prerequisite gates exclude reduced manual and shared legacy assertions", () => { + for (const variant of ["candidate", "historical"]) { + const gates = nativeCompletionGates(variant); + assert.deepEqual(gates.map(gate => gate.id), ["NC-schemas", "NC-tools", "NC-resume", "NC-eval"]); + assert.ok(gates.every(gate => gate.files.length && gate.required.length)); + assert.ok(!gates.flatMap(gate => gate.files).some(file => /agent-skills-routes|onboarding-first-task-assets|server-utils\.test|prompt-sections/.test(file))); + } + assert.ok(nativeCompletionGates("candidate")[1].required.some(name => name.includes("serialized native codex"))); + assert.ok(!nativeCompletionGates("historical")[1].required.some(name => name.includes("serialized native codex"))); + assert.ok(!nativeCompletionGates("historical")[2].required.includes("native completion tool guidance")); + assert.throws(() => nativeCompletionGates("unknown")); +}); +test("native completion prerequisite admits an executed passing boundary", () => assert.equal(gradeNativeCompletionGate(gate, report(), 0).passed, true)); +for (const status of ["pending", "skipped", "failed", "todo"]) test(`native completion prerequisite rejects a ${status} required assertion`, () => { + const observation = report(); observation.testResults[0].assertionResults[0].status = status; + assert.equal(gradeNativeCompletionGate(gate, observation, 0).passed, false); +}); +test("native completion prerequisite rejects missing discovery, unrelated assertions and failed commands", () => { + assert.equal(gradeNativeCompletionGate(gate, undefined, 0).passed, false); + const different = report(); different.testResults[0].assertionResults[0].fullName = "Other"; + assert.equal(gradeNativeCompletionGate(gate, different, 0).passed, false); + assert.equal(gradeNativeCompletionGate(gate, report(), 1).passed, false); + assert.equal(gradeNativeCompletionGate({ ...gate, files: [...gate.files, "undiscovered.test.ts"] }, report(), 0).passed, false); +}); +test("native completion prerequisite allows filtering unrelated assertions but never the required assertion", () => { + const observation = report(); observation.testResults[0].assertionResults.push({ fullName: "Unrelated", status: "skipped" }); + assert.equal(gradeNativeCompletionGate({ ...gate, testPattern: "native assertion" }, observation, 0).passed, true); + assert.equal(gradeNativeCompletionGate(gate, observation, 0).passed, false); + observation.testResults[0].assertionResults[0].status = "skipped"; + assert.equal(gradeNativeCompletionGate({ ...gate, testPattern: "native assertion" }, observation, 0).passed, false); +}); +const discovery = () => ["ID\tSUITE\tGENERATION\tPROVIDER\tMODEL\tCREDENTIALS", ...NATIVE_COMPLETION_CELL_IDS.map(id => + `${id}\tnative-completion\tnative\tprovider\tmodel\tAPI_KEY`)].join("\n"); +test("native completion prerequisite discovers exactly six native single-turn cells", () => { + assert.deepEqual(gradeNativeCompletionDiscovery(discovery(), 0), { passed: true, executionIds: NATIVE_COMPLETION_CELL_IDS, + expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1 }); +}); +for (const kind of ["missing", "duplicate", "legacy", "extra", "failed", "missing-model", "wrong-task"]) test(`native completion prerequisite rejects ${kind} discovery`, () => { + let output = discovery(); + if (kind === "missing") output = output.split("\n").slice(0, -1).join("\n"); + if (kind === "duplicate") output = output.replace(NATIVE_COMPLETION_CELL_IDS[1], NATIVE_COMPLETION_CELL_IDS[0]); + if (kind === "legacy") output = output.replace("\tnative\t", "\tlegacy\t"); + if (kind === "extra") output += `\n${output.split("\n")[1]}`; + if (kind === "missing-model") output = output.replace("\tmodel\t", "\t\t"); + if (kind === "wrong-task") output = output.replace("assigned-skill-explicit-invocation", "other-case"); + assert.equal(gradeNativeCompletionDiscovery(output, kind === "failed" ? 1 : 0).passed, false); +}); +test("native completion prerequisite excludes future credentials and auth overrides from every child", () => { + assert.deepEqual(nativeCompletionPrerequisiteEnvironment({ PATH: "/bin", HOME: "/fixture", CI: "true", + OPENAI_API_KEY: "secret", ANTHROPIC_API_KEY: "secret", FUTURE_TOKEN: "secret", NODE_OPTIONS: "secret", GH_TOKEN: "secret" }), + { PATH: "/bin", HOME: "/fixture", CI: "true" }); +}); +const runnerdProof = () => ({ schema: "paperclip.native-completion-runnerd-provenance.v1", mode: "fresh_local_build", passed: true, + selectedPath: "packages/paperclip-runner/runner/target/debug/paperclip-runnerd", binarySha256: "9".repeat(64), + sourceSha: "a".repeat(40), sourceFingerprint: "b".repeat(64), archiveSha256: null, workflowRunId: null, + workflowRunAttempt: null, artifactName: null, errors: [] }); +const current = variant => ({ runnerdProvenance: runnerdProof(), sha: "a".repeat(40), fingerprint: "b".repeat(64), fixtureFingerprint: "c".repeat(64), + manifestFingerprint: "d".repeat(64), buildOutputFingerprint: "e".repeat(64), runnerdSha256: "9".repeat(64), variant, immutable: true, layering: true, sourceErrors: [], sourceMetadataFingerprint: "f".repeat(64), sourceMetadataErrors: [] }); +const receipt = variant => ({ schema: NATIVE_COMPLETION_PREFLIGHT_SCHEMA, passed: true, providerCalls: 0, live: "not_run", + sourceSha: current(variant).sha, sourceFingerprint: current(variant).fingerprint, fixtureFingerprint: current(variant).fixtureFingerprint, + manifestFingerprint: current(variant).manifestFingerprint, variant, + baseSha: NATIVE_COMPLETION_SOURCE_CONTRACT.baseSha, archiveSha: NATIVE_COMPLETION_SOURCE_CONTRACT.archiveSha, immutable: true, layering: true, sourceErrors: [], + sourceMetadataFingerprint: current(variant).sourceMetadataFingerprint, sourceMetadataErrors: [], + expectedCells: 6, expectedTurns: 6, maximumAttemptsPerCell: 1, + setup: { passed: true, sdkExitCode: 0, runnerTypeScriptExitCode: 0, buildOutputFingerprint: current(variant).buildOutputFingerprint, + runnerdExitCode: 0, runnerdSha256: current(variant).runnerdSha256, runnerdProvenance: runnerdProof(), + runnerdSelectedPath: runnerdProof().selectedPath, + runnerdSourceSha: current(variant).sha, runnerdSourceFingerprint: current(variant).fingerprint, + evidence: ["setup-sdk.txt", "setup-runner-typescript.txt", "setup-runnerd.txt"].map(file => ({ file, sha256: "f".repeat(64) })) }, + gates: [...nativeCompletionGates(variant).map(gate => gate.id), ...NATIVE_COMPLETION_COMMAND_GATE_IDS].map(id => ({ id, passed: true, exitCode: 0 })) }); +for (const variant of ["candidate", "historical"]) test(`native completion prerequisite admits only the exact immutable ${variant} receipt`, () => { + assert.equal(assertNativeCompletionPreflightReceipt(receipt(variant), current(variant)).passed, true); +}); +const negatives = [ + ["old SHA", r => { r.sourceSha = "f".repeat(40); }], ["changed source", r => { r.sourceFingerprint = "f".repeat(64); }], + ["changed fixtures", r => { r.fixtureFingerprint = "f".repeat(64); }], ["stale manifests", r => { r.manifestFingerprint = "f".repeat(64); }], + ["wrong variant", r => { r.variant = "historical"; }], ["wrong base", r => { r.baseSha = "f".repeat(40); }], + ["wrong archive", r => { r.archiveSha = "f".repeat(40); }], ["dirty source", r => { r.immutable = false; }], + ["wrong master layering", r => { r.layering = false; }], ["source errors", r => { r.sourceErrors.push("missing.ts"); }], + ["paid provider calls", r => { r.providerCalls = 1; }], ["paid live run", r => { r.live = "run"; }], + ["missing gate", r => { r.gates.pop(); }], ["duplicate gate", r => { r.gates[1] = r.gates[0]; }], + ["failed gate", r => { r.gates[0].passed = false; }], ["nonzero exit", r => { r.gates[0].exitCode = 1; }], + ["failed SDK build", r => { r.setup.sdkExitCode = 1; }], ["missing Runner build", r => { delete r.setup.runnerTypeScriptExitCode; }], + ["changed build output", r => { r.setup.buildOutputFingerprint = "f".repeat(64); }], + ["extra cells", r => { r.expectedCells = 7; }], ["extra turns", r => { r.expectedTurns = 7; }], + ["additional attempts", r => { r.maximumAttemptsPerCell = 2; }], + ["missing runnerd provenance", r => { delete r.setup.runnerdProvenance; }], + ["wrong selected runnerd path", r => { r.setup.runnerdSelectedPath = "packages/paperclip-runner/dist/bin/paperclip-runnerd"; }], + ["claimed hosted reuse of local binary", r => { r.setup.runnerdProvenance.mode = "trusted_hosted_archive"; r.setup.runnerdExitCode = null; }], + ["failed runnerd build", r => { r.setup.runnerdExitCode = 1; }], + ["stale runnerd binary", r => { r.setup.runnerdSha256 = "8".repeat(64); }], + ["wrong runnerd source SHA", r => { r.setup.runnerdSourceSha = "b".repeat(40); }], + ["wrong runnerd source fingerprint", r => { r.setup.runnerdSourceFingerprint = "8".repeat(64); }], + ["changed hosted source metadata", r => { r.sourceMetadataFingerprint = "a".repeat(64); }], + ["source metadata errors", r => { r.sourceMetadataErrors.push("unknown lineage"); }], + ["missing setup evidence", r => { r.setup.evidence.pop(); }], + ["duplicate setup evidence", r => { r.setup.evidence[1] = r.setup.evidence[0]; }], +]; +for (const [name, mutate] of negatives) test(`native completion prerequisite rejects ${name} before providers`, () => { + const r = receipt("candidate"); mutate(r); + assert.throws(() => assertNativeCompletionPreflightReceipt(r, current("candidate")), /exact immutable source SHA, variant and fingerprint/); +}); +test("native completion prerequisite rejects a current dirty source and unavailable digests", () => { + assert.throws(() => assertNativeCompletionPreflightReceipt(receipt("candidate"), { ...current("candidate"), immutable: false })); + assert.throws(() => assertNativeCompletionPreflightReceipt(receipt("candidate"), { ...current("candidate"), fingerprint: null })); +}); + +test("native completion prerequisite rejects changed or missing retained evidence and unsafe paths", () => { + const output = mkdtempSync(join(tmpdir(), "native-completion-retained-")); + try { + const bytes = '{"passing":"native assertions"}', evidence = { file: "NC-tools.json", sha256: nativeSourceSha256(bytes) }; + writeFileSync(join(output, evidence.file), bytes); + assert.equal(assertRetainedNativeCompletionEvidence(output, evidence), bytes); + writeFileSync(join(output, evidence.file), "Changed retained assertions"); + assert.throws(() => assertRetainedNativeCompletionEvidence(output, evidence), /changed or is unavailable/); + assert.throws(() => assertRetainedNativeCompletionEvidence(output, { file: "missing.json", sha256: evidence.sha256 })); + assert.throws(() => assertRetainedNativeCompletionEvidence(output, { file: "../elsewhere.json", sha256: evidence.sha256 })); + assert.throws(() => assertRetainedNativeCompletionEvidence(output, null)); + } finally { rmSync(output, { recursive: true, force: true }); } +}); + +test("native completion prerequisite requires the executed exact Rust carrier calibration", () => { + const output = "test provider_events::tests::preserves_closed_compatibility_terminal_tool_identity ... ok\ntest result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;"; + assert.equal(gradeNativeCompletionRustCarrier(output, 0), true); + assert.equal(gradeNativeCompletionRustCarrier(output, 1), false); + assert.equal(gradeNativeCompletionRustCarrier(output.replace(" ... ok", " ... FAILED"), 0), false); + assert.equal(gradeNativeCompletionRustCarrier(output.replace("1 passed", "0 passed"), 0), false); + assert.equal(gradeNativeCompletionRustCarrier(output.replace("preserves_closed_compatibility_terminal_tool_identity", "other_test"), 0), false); +}); + +function hostedFixture() { + const c = current("candidate"), r = receipt("candidate"); + c.runnerdProvenance = { ...runnerdProof(), mode: "trusted_hosted_archive", archiveSha256: "7".repeat(64), + workflowRunId: "12345", workflowRunAttempt: "2", artifactName: `runner-e2e-build-${c.sha}-12345-2` }; + r.setup.runnerdProvenance = structuredClone(c.runnerdProvenance); r.setup.runnerdExitCode = null; + r.gates = r.gates.map(gate => gate.id === "NC-rust-carrier" ? { ...gate, id: "NC-hosted-runnerd", executed: false, + calibration: "not_executed", total: 0, passedTests: 0, reuse: "trusted_same_run_build" } : gate); + return { c, r }; +} +test("native hosted receipt explicitly reuses verified runnerd bytes without claiming Rust calibration execution", () => { + const {c, r} = hostedFixture(); assert.equal(assertNativeCompletionPreflightReceipt(r, c).passed, true); + assert.equal(nativeCompletionCommandGateIds("trusted_hosted_archive").includes("NC-rust-carrier"), false); + assert.equal(nativeCompletionCommandGateIds("trusted_hosted_archive").includes("NC-hosted-runnerd"), true); +}); +for (const [name, mutate] of [ + ["executed Rust calibration", r => { r.gates.at(-1).executed = true; }], + ["positive Rust test count", r => { r.gates.at(-1).passedTests = 1; }], + ["missing explicit not-executed status", r => { delete r.gates.at(-1).calibration; }], + ["fresh Rust build claim", r => { r.setup.runnerdExitCode = 0; }], + ["different archive bytes", r => { r.setup.runnerdProvenance.archiveSha256 = "8".repeat(64); }], + ["different workflow run", r => { r.setup.runnerdProvenance.workflowRunId = "999"; }], +]) test(`native hosted receipt rejects ${name}`, () => { + const {c,r} = hostedFixture(); + // The hosted provenance gate is last in this deterministic fixture. + r.gates.sort((a,b) => a.id === "NC-hosted-runnerd" ? 1 : b.id === "NC-hosted-runnerd" ? -1 : 0); + mutate(r); assert.throws(() => assertNativeCompletionPreflightReceipt(r,c)); +}); + +test("hosted runnerd evidence verifies the exact compact JSON capture before its diagnostic line", () => { + const proof = hostedFixture().c.runnerdProvenance; + const capture = `${JSON.stringify(proof)}\nRust unit calibration must be qualified by the separate exact-source local receipt and normal CI.`; + assert.equal(gradeNativeCompletionHostedRunnerdEvidence(capture, proof), true); + assert.equal(gradeNativeCompletionHostedRunnerdEvidence(capture.replace("\n", "\r\n"), proof), true); + assert.equal(gradeNativeCompletionHostedRunnerdEvidence(capture, { ...proof, binarySha256: "f".repeat(64) }), false); + assert.equal(gradeNativeCompletionHostedRunnerdEvidence(`not-json\n${JSON.stringify(proof)}`, proof), false); + assert.equal(gradeNativeCompletionHostedRunnerdEvidence("", proof), false); +}); diff --git a/tests/runner-e2e/native-completion-defaults.test.ts b/tests/runner-e2e/native-completion-defaults.test.ts new file mode 100644 index 00000000000..21c5c6dc082 --- /dev/null +++ b/tests/runner-e2e/native-completion-defaults.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { captureNativeDefault, gradeNativeDefault, nativeCompletionProfile, nativeCompletionWorkspaceDigest, NATIVE_MASTER_DEFAULT_SHA256, type NativeDefaultReceipt } from "./native-completion-defaults.js"; +import { mkdtemp, mkdir, writeFile, symlink, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { RunnerProfileFixture } from "./types.js"; +import { runnerSuites } from "./catalog.js"; +import { contextIntegrityTasks } from "./context-integrity-cases.js"; + +const receipt = (): NativeDefaultReceipt => ({ schema: "paperclip.native-completion-default.v1", agentId: "agent", companyId: "company", + entryFile: "AGENTS.md", files: [{ path: "AGENTS.md", sha256: NATIVE_MASTER_DEFAULT_SHA256, bytes: 100 }], budgets: { company: 1000, agent: 1000 } }); +describe("native production defaults", () => { + it("hashes scoped file and symlink identity without reading an outside target", async () => { + const temporary = await mkdtemp(path.join(tmpdir(), "native-workspace-calibration-")); + try { + const root = path.join(temporary, "workspace"), outside = path.join(temporary, "outside"); + await mkdir(root); await mkdir(outside); + await writeFile(path.join(root, "file"), "initial"); await writeFile(path.join(outside, "private"), "first"); + await symlink(outside, path.join(root, "skill-link")); + const first = await nativeCompletionWorkspaceDigest(root); + await writeFile(path.join(outside, "private"), "changed outside"); + expect(await nativeCompletionWorkspaceDigest(root)).toBe(first); + await writeFile(path.join(root, "file"), "changed fixture"); + expect(await nativeCompletionWorkspaceDigest(root)).not.toBe(first); + } finally { await rm(temporary, { recursive: true, force: true }); } + }); + it("preserves the original assigned-skill prompt flow and oracle in exactly six explicit cells", () => { + const suite = runnerSuites.find(value => value.id === "native-completion")!; + const original = contextIntegrityTasks.find(value => value.id === "assigned-skill-explicit-invocation")!; + const completion = suite.tasks.find(value => value.id === original.id)!; + expect({ ...completion, automaticRetryPolicy: undefined }).toEqual({ ...original, automaticRetryPolicy: undefined }); + expect(suite.manualOnly).toBe(true); + expect(suite.expectedMatrixSize).toBe(6); + expect(suite.tasks).toHaveLength(2); + expect(suite.tasks.every(task => task.expectedRunCount === 1 && task.automaticRetryPolicy === "single_attempt")).toBe(true); + expect(suite.environments.map(value => value.id)).toEqual(["local"]); + expect(suite.profiles.map(value => value.id).sort()).toEqual(["runner-acpx-claude", "runner-codex", "runner-opencode"]); + }); + it("accepts only the unchanged master default and bounded budgets", () => expect(gradeNativeDefault(receipt()).passed).toBe(true)); + it.each(["qa", "reduced", "extra-file", "wrong-entry", "company-unbounded", "agent-unbounded"])("rejects %s override", name => { + const value = receipt(); + if (name === "qa" || name === "reduced") value.files[0]!.sha256 = name; + if (name === "extra-file") value.files.push({ path: "HEARTBEAT.md", sha256: "x", bytes: 1 }); + if (name === "wrong-entry") value.entryFile = "OTHER.md"; + if (name === "company-unbounded") value.budgets.company = 0; + if (name === "agent-unbounded") value.budgets.agent = 0; + expect(gradeNativeDefault(value).passed).toBe(false); + }); + it("strips only the fixture bundle and preserves auth skills permissions and model", () => { + const original = { instructionsBundle: { entryFile: "AGENTS.md", files: { "AGENTS.md": "QA" } }, + adapterConfig: { model: "same", env: { KEY: { type: "secret_ref", secretId: "id" } } }, + runtimeConfig: { permissionMode: "same" }, skills: ["same"], budgetMonthlyCents: 0 }; + const profile = { generation: "native", buildAgent: () => original } as unknown as RunnerProfileFixture; + const wrapped = nativeCompletionProfile(profile).buildAgent({} as never); + expect(wrapped).toEqual({ ...original, instructionsBundle: undefined, budgetMonthlyCents: 1000 }); + expect(Object.hasOwn(wrapped, "instructionsBundle")).toBe(false); + expect(original.instructionsBundle).toBeDefined(); + expect(() => nativeCompletionProfile({ ...profile, generation: "legacy" })).toThrow(); + }); + it("hashes actual public bundle bytes before execution without publishing the body", async () => { + const content = readFileSync(new URL("../../server/src/onboarding-assets/default/AGENTS.md", import.meta.url), "utf8"); + const called: string[] = []; + const api = { async get(path: string): Promise { + called.push(path); + return (path.endsWith("instructions-bundle") ? { entryFile: "AGENTS.md", files: [{ path: "AGENTS.md" }] } + : path.includes("instructions-bundle/file") ? { content } : { budgetMonthlyCents: 1000 }) as T; + } }; + const actual = await captureNativeDefault({ api, agentId: "agent", companyId: "company" }); + expect(gradeNativeDefault(actual).passed).toBe(true); + expect(called).toContain("/api/agents/agent/instructions-bundle/file?path=AGENTS.md"); + expect(JSON.stringify(actual)).not.toContain(content); + }); +}); diff --git a/tests/runner-e2e/native-completion-defaults.ts b/tests/runner-e2e/native-completion-defaults.ts new file mode 100644 index 00000000000..53efae177ba --- /dev/null +++ b/tests/runner-e2e/native-completion-defaults.ts @@ -0,0 +1,70 @@ +import { createHash } from "node:crypto"; +import { lstat, readFile, readdir, readlink } from "node:fs/promises"; +import path from "node:path"; +import type { RunnerProfileFixture } from "./types.js"; + +export const NATIVE_COMPLETION_BUDGET_CENTS = 1_000; +// Independent, immutable current-master default; not the reduced #14948 manual. +export const NATIVE_MASTER_DEFAULT_SHA256 = "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f"; + +/** Scoped blocker proof: hash symlink identity without following any target. */ +export async function nativeCompletionWorkspaceDigest(root: string): Promise { + const hash = createHash("sha256"); + async function visit(relative: string) { + const entries = await readdir(path.join(root, relative), { withFileTypes: true }); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const file = path.join(relative, entry.name), absolute = path.join(root, file), stat = await lstat(absolute); + hash.update(JSON.stringify([file, stat.isSymbolicLink() ? "symlink" : stat.isDirectory() ? "directory" : stat.isFile() ? "file" : "other"])); + if (stat.isSymbolicLink()) hash.update(await readlink(absolute)); + else if (stat.isDirectory()) await visit(file); + else if (stat.isFile()) hash.update(await readFile(absolute)); + } + } + await visit(""); return hash.digest("hex"); +} + +export function nativeCompletionProfile(profile: RunnerProfileFixture): RunnerProfileFixture { + if (profile.generation !== "native") throw new Error("Native completion requires a native profile"); + return { ...profile, buildAgent(input) { + const { instructionsBundle: _fixtureBundle, ...agent } = profile.buildAgent(input); + return { ...agent, budgetMonthlyCents: NATIVE_COMPLETION_BUDGET_CENTS }; + } }; +} + +export interface NativeDefaultReceipt { + schema: "paperclip.native-completion-default.v1"; + agentId: string; + companyId: string; + entryFile: unknown; + files: Array<{ path: string; sha256: string; bytes: number }>; + budgets: { company: unknown; agent: unknown }; +} + +export function gradeNativeDefault(receipt: NativeDefaultReceipt) { + const checks = [ + { id: "production-default-bundle", passed: receipt.entryFile === "AGENTS.md" + && receipt.files.length === 1 && receipt.files[0]?.path === "AGENTS.md" + && receipt.files[0]?.sha256 === NATIVE_MASTER_DEFAULT_SHA256, + detail: "The served public hire bundle is the unchanged master default, with no QA or reduced-manual injection." }, + { id: "bounded-company-and-agent", passed: receipt.budgets.company === NATIVE_COMPLETION_BUDGET_CENTS + && receipt.budgets.agent === NATIVE_COMPLETION_BUDGET_CENTS, + detail: "Both public budgets enforce the declared 1,000-cent hard stop." }, + ]; + return { passed: checks.every(check => check.passed), checks }; +} + +export async function captureNativeDefault(input: { + api: { get(path: string): Promise }; agentId: string; companyId: string; +}): Promise { + const [bundle, company, agent] = await Promise.all([ + input.api.get<{ entryFile: unknown; files: Array<{ path: string }> }>(`/api/agents/${input.agentId}/instructions-bundle`), + input.api.get<{ budgetMonthlyCents: unknown }>(`/api/companies/${input.companyId}`), + input.api.get<{ budgetMonthlyCents: unknown }>(`/api/agents/${input.agentId}`), + ]); + const files = await Promise.all(bundle.files.map(async file => { + const detail = await input.api.get<{ content: string }>(`/api/agents/${input.agentId}/instructions-bundle/file?path=${encodeURIComponent(file.path)}`); + return { path: file.path, sha256: createHash("sha256").update(detail.content).digest("hex"), bytes: Buffer.byteLength(detail.content) }; + })); + return { schema: "paperclip.native-completion-default.v1", agentId: input.agentId, companyId: input.companyId, + entryFile: bundle.entryFile, files, budgets: { company: company.budgetMonthlyCents, agent: agent.budgetMonthlyCents } }; +} diff --git a/tests/runner-e2e/native-completion-git-source.mjs b/tests/runner-e2e/native-completion-git-source.mjs new file mode 100644 index 00000000000..0da48024d44 --- /dev/null +++ b/tests/runner-e2e/native-completion-git-source.mjs @@ -0,0 +1,166 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { closeSync, constants, fstatSync, lstatSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, posix } from "node:path"; + +const hydrationDirectory = "runner-e2e-build"; +const archiveName = "runner-e2e-build-bundle.tar.gz"; +const checksumName = `${archiveName}.sha256`; +const hydrationPaths = [archiveName, checksumName].map(file => `${hydrationDirectory}/${file}`); +const sha256 = bytes => createHash("sha256").update(bytes).digest("hex"); + +/** A shallow commit still carries its hash-bound immediate parent in raw bytes. */ +export function verifyNativeCompletionParentAnchor({ sha, rawCommit, anchor }) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !/^[a-f0-9]{40}$/.test(anchor ?? "")) return false; + const bytes = Buffer.isBuffer(rawCommit) ? rawCommit : Buffer.from(rawCommit ?? ""); + const observed = createHash("sha1").update(`commit ${bytes.length}\0`).update(bytes).digest("hex"); + const parents = bytes.toString("utf8").split("\n\n", 1)[0].split("\n").filter(line => line.startsWith("parent ")); + return observed === sha && parents.length === 1 && parents[0] === `parent ${anchor}`; +} +function regularFile(path) { + const stat = lstatSync(path); + if (!stat.isFile() || stat.nlink !== 1) throw new Error("entry is not a regular, independently downloaded file"); + return stat; +} +function archiveDigest(path) { + const before = regularFile(path), fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + try { + const opened = fstatSync(fd); + if (!opened.isFile() || opened.nlink !== 1 || opened.ino !== before.ino || opened.dev !== before.dev) + throw new Error("archive changed during verification"); + const hash = createHash("sha256"), buffer = Buffer.alloc(1024 * 1024); + let count; + while ((count = readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, count)); + const after = fstatSync(fd); + if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs) throw new Error("archive changed during verification"); + return hash.digest("hex"); + } finally { closeSync(fd); } +} +export function inspectNativeCompletionBuildHydration(repositoryRoot) { + const directory = join(repositoryRoot, hydrationDirectory), errors = []; + let stat; + try { stat = lstatSync(directory); } + catch (error) { + if (error.code === "ENOENT") return { present: false, verified: false, archiveSha256: null, checksumSha256: null, errors }; + return { present: true, verified: false, archiveSha256: null, checksumSha256: null, errors: ["build hydration directory is unreadable"] }; + } + let archiveSha256 = null, checksumSha256 = null; + try { + if (!stat.isDirectory()) throw new Error("directory must be a real directory, not a symlink or file"); + const entries = readdirSync(directory).sort(); + if (JSON.stringify(entries) !== JSON.stringify([archiveName, checksumName].sort())) + throw new Error("directory must contain exactly the archive and its checksum, with no extra entries"); + const checksumPath = join(directory, checksumName), checksumStat = regularFile(checksumPath); + if (checksumStat.size > 128) throw new Error("checksum record is malformed"); + const checksum = readFileSync(checksumPath), match = /^([a-f0-9]{64}) {2}runner-e2e-build-bundle\.tar\.gz\n?$/.exec(checksum.toString("utf8")); + if (!match) throw new Error("checksum must be one exact SHA256 record for the downloaded archive"); + checksumSha256 = sha256(checksum); archiveSha256 = archiveDigest(join(directory, archiveName)); + if (archiveSha256 !== match[1]) throw new Error("archive checksum verification failed"); + } catch (error) { errors.push(`build hydration ${error.message}`); } + return { present: true, verified: errors.length === 0, archiveSha256, checksumSha256, errors }; +} + +export const NATIVE_COMPLETION_RUNNERD_PATH = "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"; +/** Mirror the production default selector, and reject staged binaries that would take precedence. */ +export function inspectNativeCompletionRunnerd({ repositoryRoot, sourceSha, sourceFingerprint, environment }) { + const errors = [], hosted = environment.GITHUB_ACTIONS === "true"; + const selectedPath = `${NATIVE_COMPLETION_RUNNERD_PATH}${process.platform === "win32" ? ".exe" : ""}`; + const staged = join(repositoryRoot, `packages/paperclip-runner/dist/bin/paperclip-runnerd${process.platform === "win32" ? ".exe" : ""}`); + let binarySha256 = null, binaryBytes = null, archiveSha256 = null, archiveMemberBytes = null; + try { + try { lstatSync(staged); throw new Error("staged dist/bin runnerd would override the admitted debug executable"); } + catch (error) { if (error.code !== "ENOENT") throw error; } + let directory = repositoryRoot; + for (const part of selectedPath.split("/").slice(0, -1)) { + directory = join(directory, part); + if (!lstatSync(directory).isDirectory()) throw new Error("selected runnerd ancestor must be a real directory"); + } + const binary = join(repositoryRoot, selectedPath), stat = regularFile(binary); + if (process.platform !== "win32" && !(stat.mode & 0o111)) throw new Error("selected runnerd is not executable"); + binaryBytes = stat.size; + binarySha256 = archiveDigest(binary); + if (hosted) { + if (environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA !== sourceSha || !/^[a-f0-9]{40}$/.test(sourceSha ?? "")) + throw new Error("trusted hosted target SHA does not match admitted Git HEAD"); + if (![environment.GITHUB_RUN_ID, environment.GITHUB_RUN_ATTEMPT].every(value => typeof value === "string" && value === value.trim() && /^[1-9]\d*$/.test(value))) + throw new Error("trusted hosted workflow run and attempt are unavailable"); + const hydration = inspectNativeCompletionBuildHydration(repositoryRoot); + if (!hydration.verified) throw new Error(`hosted build archive is unverified: ${hydration.errors.join("; ")}`); + archiveSha256 = hydration.archiveSha256; + const archive = join(repositoryRoot, hydrationDirectory, archiveName); + const tar = args => spawnSync("tar", args, { encoding: "utf8", timeout: 120_000, maxBuffer: 16 * 1024 * 1024 }); + const names = tar(["-tzf", archive]); + if (names.status !== 0) throw new Error("hosted build archive cannot be listed"); + const matching = names.stdout.split(/\r?\n/).filter(name => name && posix.normalize(name.replace(/^\/+/, "")) === selectedPath); + if (matching.length !== 1 || matching[0] !== selectedPath) + throw new Error("hosted archive must contain exactly one canonical selected runnerd member"); + const details = tar(["-tvzf", archive, selectedPath]); + const rows = details.stdout.trim().split(/\r?\n/); + if (details.status !== 0 || rows.length !== 1 || !rows[0].startsWith("-") || !rows[0].endsWith(` ${selectedPath}`)) + throw new Error("hosted selected runnerd member must be a regular file"); + // Keep daemon bytes out of JavaScript memory; the selected executable supplies the exact expected size. + const temporary = mkdtempSync(join(tmpdir(), "native-completion-runnerd-member-")), member = join(temporary, "runnerd"); + let fd; + try { + fd = openSync(member, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | (constants.O_NOFOLLOW ?? 0), 0o600); + const extracted = spawnSync("tar", ["-xOzf", archive, selectedPath], { timeout: 120_000, stdio: ["ignore", fd, "pipe"] }); + archiveMemberBytes = fstatSync(fd).size; + closeSync(fd); fd = undefined; + if (extracted.status !== 0 || archiveMemberBytes !== binaryBytes || archiveDigest(member) !== binarySha256) + throw new Error("selected runnerd byte count or hash differs from the verified hosted archive member"); + } finally { + if (fd !== undefined) closeSync(fd); + rmSync(temporary, { recursive: true, force: true }); + } + if (archiveDigest(archive) !== archiveSha256) throw new Error("hosted build archive changed during runnerd verification"); + } + } catch (error) { errors.push(`runnerd provenance ${error.message}`); } + return { schema: "paperclip.native-completion-runnerd-provenance.v1", mode: hosted ? "trusted_hosted_archive" : "fresh_local_build", + passed: errors.length === 0, selectedPath, binarySha256, binaryBytes, archiveMemberBytes, sourceSha, sourceFingerprint, archiveSha256, + workflowRunId: hosted ? environment.GITHUB_RUN_ID ?? null : null, + workflowRunAttempt: hosted ? environment.GITHUB_RUN_ATTEMPT ?? null : null, + artifactName: hosted ? `runner-e2e-build-${sourceSha}-${environment.GITHUB_RUN_ID}-${environment.GITHUB_RUN_ATTEMPT}` : null, + errors }; +} + +/** Preserve tracked-source cleanliness; allow only the verified workflow download. */ +export function inspectNativeCompletionSourceMetadata({ repositoryRoot, sourceFiles, baseSha, variant, shallowParentAnchors }) { + const git = (args, raw = false) => spawnSync("git", ["--no-replace-objects", ...args], { + cwd: repositoryRoot, encoding: raw ? undefined : "utf8", timeout: 30_000, + }); + const head = git(["rev-parse", "--verify", "HEAD"]), sha = head.status === 0 ? head.stdout.trim() : null; + const ancestor = git(["merge-base", "--is-ancestor", baseSha, "HEAD"]); + const shallow = git(["rev-parse", "--is-shallow-repository"]), isShallow = shallow.status === 0 && shallow.stdout.trim() === "true"; + const errors = []; + if (!/^[a-f0-9]{40}$/.test(sha ?? "")) errors.push("Git HEAD is unavailable or is not an immutable SHA1 commit"); + let layering = ancestor.status === 0, strategy = layering ? "full_ancestry" : null; + const anchor = shallowParentAnchors?.[variant] ?? null; + if (!layering && isShallow && anchor && sha) { + const commit = git(["cat-file", "commit", "HEAD"], true); + layering = commit.status === 0 && verifyNativeCompletionParentAnchor({ sha, rawCommit: commit.stdout, anchor }); + if (layering) strategy = "shallow_immediate_parent_anchor"; + } + if (!layering) errors.push(isShallow + ? "shallow Git HEAD does not have the sole hash-verified immediate parent admitted for this source variant" + : "full Git history does not prove the declared master base is an ancestor of HEAD"); + const tracked = git(["ls-files", "--error-unmatch", "--", ...sourceFiles]); + const clean = git(["diff", "--quiet", "HEAD", "--"]); + const status = git(["status", "--porcelain=v1", "-z", "--untracked-files=all"]); + const entries = status.status === 0 ? status.stdout.split("\0").filter(Boolean) : []; + const unexpected = entries.filter(entry => entry.slice(0, 3) !== "?? " || !hydrationPaths.includes(entry.slice(3))); + const hydration = inspectNativeCompletionBuildHydration(repositoryRoot); + if (tracked.status !== 0) errors.push("one or more admitted source paths are not tracked at HEAD"); + if (clean.status !== 0) errors.push("tracked source differs from HEAD"); + if (status.status !== 0) errors.push("Git worktree status is unavailable"); + if (unexpected.length) errors.push(`unexpected worktree entries: ${unexpected.join(", ")}`); + errors.push(...hydration.errors); + const immutable = tracked.status === 0 && clean.status === 0 && status.status === 0 && unexpected.length === 0 && + (!hydration.present || hydration.verified); + const metadata = { schema: "paperclip.native-completion-source-metadata.v1", + lineage: { strategy, shallow: isShallow, sha, baseSha, anchor: strategy === "shallow_immediate_parent_anchor" ? anchor : null }, + hydration: { present: hydration.present, verified: hydration.verified, archiveSha256: hydration.archiveSha256, checksumSha256: hydration.checksumSha256 }, + worktreeEntries: entries, errors }; + return { sha, layering, immutable, sourceMetadata: metadata, sourceMetadataErrors: errors, + sourceMetadataFingerprint: sha256(JSON.stringify(metadata)) }; +} diff --git a/tests/runner-e2e/native-completion-git-source.test.mjs b/tests/runner-e2e/native-completion-git-source.test.mjs new file mode 100644 index 00000000000..03681ee3454 --- /dev/null +++ b/tests/runner-e2e/native-completion-git-source.test.mjs @@ -0,0 +1,174 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { after, test } from "node:test"; +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { inspectNativeCompletionSourceMetadata, inspectNativeCompletionBuildHydration, verifyNativeCompletionParentAnchor, + inspectNativeCompletionRunnerd, NATIVE_COMPLETION_RUNNERD_PATH } from "./native-completion-git-source.mjs"; + +const roots = []; +after(() => { for (const root of roots) rmSync(root, { recursive: true, force: true }); }); +const temporary = () => { const root = mkdtempSync(join(tmpdir(), "native-source-metadata-")); roots.push(root); return root; }; +const git = (root, args, input) => execFileSync("git", args, { cwd: root, input, encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] }).trim(); +const digest = bytes => createHash("sha256").update(bytes).digest("hex"); +const archiveName = "runner-e2e-build-bundle.tar.gz", checksumName = `${archiveName}.sha256`; +function commit(root, tree, parents, message) { + const raw = `tree ${tree}\n${parents.map(parent => `parent ${parent}\n`).join("")}author Fixture 1 +0000\ncommitter Fixture 1 +0000\n\n${message}\n`; + return { raw, sha: git(root, ["hash-object", "-t", "commit", "-w", "--stdin"], raw) }; +} +function fixture(shallow = false, parentCount = 1) { + const source = temporary(); git(source, ["init", "--quiet"]); + writeFileSync(join(source, "source.txt"), "Immutable admitted source\n"); + const blob = git(source, ["hash-object", "-w", "source.txt"]); + const tree = git(source, ["mktree"], `100644 blob ${blob}\tsource.txt\n`); + const base = commit(source, tree, [], "Admitted base"), anchor = commit(source, tree, [base.sha], "Admitted native parent"); + const other = commit(source, tree, [base.sha], "Different parent"); + const head = commit(source, tree, parentCount === 2 ? [anchor.sha, other.sha] : [anchor.sha], "Native-only repair"); + git(source, ["update-ref", "refs/heads/fixture", head.sha]); git(source, ["checkout", "--quiet", "--force", "fixture"]); + let root = source; + if (shallow) { + root = join(temporary(), "checkout"); + git(source, ["clone", "--quiet", "--depth", "1", "--branch", "fixture", pathToFileURL(source).href, root]); + } + const input = { repositoryRoot: root, sourceFiles: ["source.txt"], baseSha: base.sha, variant: "candidate", + shallowParentAnchors: { candidate: anchor.sha, historical: other.sha } }; + return { root, source, base, anchor, other, head, input }; +} +function hydrate(root, bytes = Buffer.from("Exact downloaded build archive bytes")) { + const directory = join(root, "runner-e2e-build"); mkdirSync(directory); + writeFileSync(join(directory, archiveName), bytes); + writeFileSync(join(directory, checksumName), `${digest(bytes)} ${archiveName}\n`); + return directory; +} +test("native hosted-source metadata retains full ancestry and clean sources", () => { + const f = fixture(), result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.layering, true); assert.equal(result.immutable, true); assert.deepEqual(result.sourceMetadataErrors, []); + assert.equal(result.sourceMetadata.lineage.strategy, "full_ancestry"); +}); +test("native hosted-source metadata accepts an actual shallow clone with the exact raw parent anchor", () => { + const f = fixture(true); hydrate(f.root); + assert.equal(git(f.root, ["rev-parse", "--is-shallow-repository"]), "true"); + assert.throws(() => git(f.root, ["merge-base", "--is-ancestor", f.base.sha, "HEAD"])); + const result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.sha, f.head.sha); assert.equal(result.layering, true); assert.equal(result.immutable, true); + assert.deepEqual(result.sourceMetadataErrors, []); + assert.equal(result.sourceMetadata.lineage.strategy, "shallow_immediate_parent_anchor"); + assert.equal(result.sourceMetadata.lineage.anchor, f.anchor.sha); + assert.equal(result.sourceMetadata.hydration.verified, true); +}); +test("native hosted-source metadata requires the variant-specific shallow anchor", () => { + const f = fixture(true), result = inspectNativeCompletionSourceMetadata({ ...f.input, variant: "historical" }); + assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("sole hash-verified immediate parent"))); +}); +test("native hosted-source metadata never substitutes an anchor for missing full-clone ancestry", () => { + const f = fixture(), result = inspectNativeCompletionSourceMetadata({ ...f.input, baseSha: "f".repeat(40) }); + assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("full Git history"))); +}); +test("native hosted-source metadata rejects a shallow merge even with the admitted parent", () => { + const f = fixture(true, 2), result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.layering, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("sole hash-verified"))); +}); +test("native hosted-source metadata cryptographically binds all raw HEAD bytes and the sole parent", () => { + const f = fixture(); + assert.equal(verifyNativeCompletionParentAnchor({ sha: f.head.sha, rawCommit: f.head.raw, anchor: f.anchor.sha }), true); + for (const input of [ + { sha: "f".repeat(40), rawCommit: f.head.raw, anchor: f.anchor.sha }, + { sha: f.head.sha, rawCommit: `${f.head.raw}altered`, anchor: f.anchor.sha }, + { sha: f.head.sha, rawCommit: f.head.raw, anchor: f.other.sha }, + { sha: f.base.sha, rawCommit: f.base.raw, anchor: f.anchor.sha }, + { sha: null, rawCommit: f.head.raw, anchor: f.anchor.sha }, + ]) assert.equal(verifyNativeCompletionParentAnchor(input), false); +}); +test("native hosted-source metadata rejects dirty tracked and untracked admitted source", () => { + const f = fixture(); hydrate(f.root); writeFileSync(join(f.root, "source.txt"), "Changed admitted source\n"); + let result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.includes("tracked source differs from HEAD")); + git(f.root, ["checkout", "HEAD", "--", "source.txt"]); writeFileSync(join(f.root, "extra-source.mjs"), "Unexpected code"); + result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("extra-source.mjs"))); + assert.equal(inspectNativeCompletionSourceMetadata({ ...f.input, sourceFiles: ["extra-source.mjs"] }).immutable, false); +}); +test("native hosted-source metadata verifies only the exact two downloaded regular files", () => { + const f = fixture(), directory = hydrate(f.root), first = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(first.immutable, true); assert.equal(first.sourceMetadata.hydration.verified, true); + assert.equal(inspectNativeCompletionSourceMetadata(f.input).sourceMetadataFingerprint, first.sourceMetadataFingerprint); + writeFileSync(join(directory, archiveName), "Different verified archive"); + writeFileSync(join(directory, checksumName), `${digest("Different verified archive")} ${archiveName}\n`); + const changed = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(changed.immutable, true); assert.notEqual(changed.sourceMetadataFingerprint, first.sourceMetadataFingerprint); +}); +const negatives = [ + ["missing checksum", (root, directory) => rmSync(join(directory, checksumName))], + ["checksum mismatch", (root, directory) => writeFileSync(join(directory, archiveName), "Unverified replacement")], + ["extra file", (root, directory) => writeFileSync(join(directory, "extra.mjs"), "Unexpected code")], + ["extra nested directory", (root, directory) => mkdirSync(join(directory, "nested"))], + ["malformed checksum", (root, directory) => writeFileSync(join(directory, checksumName), "malformed")], + ["wrong checksum filename", (root, directory) => writeFileSync(join(directory, checksumName), `${"a".repeat(64)} other.tar.gz\n`)], + ["multiple checksum records", (root, directory) => writeFileSync(join(directory, checksumName), readFileSync(join(directory, checksumName), "utf8").repeat(2))], + ["symlink archive", (root, directory) => { rmSync(join(directory, archiveName)); symlinkSync(join(root, "source.txt"), join(directory, archiveName)); }], + ["symlink checksum", (root, directory) => { rmSync(join(directory, checksumName)); symlinkSync(join(root, "source.txt"), join(directory, checksumName)); }], + ["symlink directory", (root, directory) => { rmSync(directory, { recursive: true }); symlinkSync(root, directory); }], +]; +for (const [name, mutate] of negatives) test(`native hosted-source metadata rejects ${name} hydration`, () => { + const f = fixture(), directory = hydrate(f.root); mutate(f.root, directory); + const hydration = inspectNativeCompletionBuildHydration(f.root), result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(hydration.verified, false); assert.ok(hydration.errors.length > 0); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("build hydration"))); +}); +test("native hosted-source metadata rejects unrelated dirty trees despite verified hydration", () => { + const f = fixture(); hydrate(f.root); mkdirSync(join(f.root, "untrusted")); writeFileSync(join(f.root, "untrusted", "anything"), "extra"); + const result = inspectNativeCompletionSourceMetadata(f.input); + assert.equal(result.immutable, false); assert.ok(result.sourceMetadataErrors.some(error => error.includes("untrusted/anything"))); +}); + +function runnerdFixture() { + const root = temporary(), binary = join(root, NATIVE_COMPLETION_RUNNERD_PATH); + mkdirSync(join(binary, ".."), { recursive: true }); writeFileSync(binary, "Exact built runnerd fixture bytes"); chmodSync(binary, 0o755); + const directory = join(root, "runner-e2e-build"); mkdirSync(directory); + function pack(members = [NATIVE_COMPLETION_RUNNERD_PATH]) { + execFileSync("tar", ["-czf", join(directory, archiveName), ...members], { cwd: root }); + writeFileSync(join(directory, checksumName), `${digest(readFileSync(join(directory, archiveName)))} ${archiveName}\n`); + } + pack(); + const input = { repositoryRoot: root, sourceSha: "a".repeat(40), sourceFingerprint: "b".repeat(64), environment: { + GITHUB_ACTIONS: "true", PAPERCLIP_RUNNER_E2E_SOURCE_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "2" } }; + return { root, binary, directory, pack, input }; +} +test("native hosted runnerd proof binds the actual selected debug executable to the trusted same-run archive", () => { + const f = runnerdFixture(), proof = inspectNativeCompletionRunnerd(f.input); + assert.equal(proof.passed, true); assert.equal(proof.mode, "trusted_hosted_archive"); + assert.equal(proof.selectedPath, NATIVE_COMPLETION_RUNNERD_PATH); + assert.equal(proof.binarySha256, digest(readFileSync(f.binary))); + assert.equal(proof.binaryBytes, readFileSync(f.binary).length); assert.equal(proof.archiveMemberBytes, proof.binaryBytes); + assert.equal(proof.artifactName, `runner-e2e-build-${f.input.sourceSha}-12345-2`); + assert.deepEqual(proof.errors, []); + const local = inspectNativeCompletionRunnerd({ ...f.input, environment: {} }); + assert.equal(local.passed, true); assert.equal(local.mode, "fresh_local_build"); + assert.equal(local.archiveSha256, null); assert.equal(local.workflowRunId, null); + assert.equal(local.binaryBytes, readFileSync(f.binary).length); assert.equal(local.archiveMemberBytes, null); +}); +for (const [name, mutate] of [ + ["wrong trusted source SHA", f => { f.input.environment.PAPERCLIP_RUNNER_E2E_SOURCE_SHA = "c".repeat(40); }], + ["missing workflow run", f => { delete f.input.environment.GITHUB_RUN_ID; }], + ["malformed workflow attempt", f => { f.input.environment.GITHUB_RUN_ATTEMPT = "../2"; }], + ["newline in workflow run", f => { f.input.environment.GITHUB_RUN_ID = "12345\n"; }], + ["unverified archive", f => { writeFileSync(join(f.directory, archiveName), "Replacement"); }], + ["different selected executable", f => { writeFileSync(f.binary, "Stale binary"); }], + ["missing selected executable", f => { rmSync(f.binary); }], + ["nonexecutable selected binary", f => { chmodSync(f.binary, 0o644); }], + ["same-size different binary hash", f => { writeFileSync(f.binary, Buffer.alloc(readFileSync(f.binary).length, 0x58)); }], + ["symlink selected executable", f => { const bytes = readFileSync(f.binary); rmSync(f.binary); writeFileSync(join(f.root, "other"), bytes); symlinkSync(join(f.root, "other"), f.binary); }], + ["symlink executable ancestor", f => { const bytes = readFileSync(f.binary), directory = join(f.binary, ".."); rmSync(directory, { recursive: true }); mkdirSync(join(f.root, "other")); writeFileSync(join(f.root, "other", "paperclip-runnerd"), bytes); chmodSync(join(f.root, "other", "paperclip-runnerd"), 0o755); symlinkSync(join(f.root, "other"), directory); }], + ["staged executable override", f => { const staged = join(f.root, "packages/paperclip-runner/dist/bin"); mkdirSync(staged, { recursive: true }); writeFileSync(join(staged, "paperclip-runnerd"), "Unexpected staged binary"); }], + ["duplicate canonical archive members", f => { f.pack([NATIVE_COMPLETION_RUNNERD_PATH, NATIVE_COMPLETION_RUNNERD_PATH]); }], + ["noncanonical archive member", f => { f.pack([`./${NATIVE_COMPLETION_RUNNERD_PATH}`]); }], + ["missing archive member", f => { writeFileSync(join(f.root, "other"), "Other"); f.pack(["other"]); }], + ["symlink archive member", f => { rmSync(f.binary); writeFileSync(join(f.root, "other"), "Other"); symlinkSync(join(f.root, "other"), f.binary); f.pack(); rmSync(f.binary); writeFileSync(f.binary, "Other"); chmodSync(f.binary, 0o755); }], +]) test(`native hosted runnerd proof rejects ${name}`, () => { + const f = runnerdFixture(); mutate(f); + const proof = inspectNativeCompletionRunnerd(f.input); + assert.equal(proof.passed, false); assert.ok(proof.errors.length > 0); +}); diff --git a/tests/runner-e2e/native-completion-scoring.test.ts b/tests/runner-e2e/native-completion-scoring.test.ts new file mode 100644 index 00000000000..8c476cbc32e --- /dev/null +++ b/tests/runner-e2e/native-completion-scoring.test.ts @@ -0,0 +1,118 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { rehydrateRunnerdItemNotification } from "../../packages/paperclip-runner/src/live/runnerd-codex-transport.js"; +import { gradeNativeCompletion, type NativeCompletionObservation } from "./native-completion-scoring.js"; +type Row = Record; +function sample(blocked = true, compatibility = false): NativeCompletionObservation { + const body = blocked ? "Deployment remains blocked until access is granted. Release Owner must Grant deployment access. BLOCKED_probe" : "Saved the requested document."; + const result = { reportedWorkDisposition: blocked ? "blocked" : "done", ...(blocked ? { blocker: { owner: { name: "Release Owner" }, unblockAction: "Grant deployment access", scope: "task_wide" } } : {}) }; + const event = (seq: number, eventType: string, payload: Row, sourceKind = "runner"): Row => ({ seq, eventType, protocolSchemaVersion: 1, + sourceEventId: `event${seq}`, sourceInstanceId: sourceKind, sourceSeq: seq, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, runId: "run", eventType, + sourceKind, sourceEventId: `event${seq}`, sourceInstanceId: sourceKind, sourceSeq: seq, payload } } }); + const tool = blocked ? "paperclip_block" : "paperclip_finish"; + return { caseId: blocked ? "native-blocked-report" : "assigned-skill-explicit-invocation", companyId: "company", agentId: "agent", marker: "BLOCKED_probe", + issue: { id: "issue", companyId: "company", assigneeAgentId: "agent", status: result.reportedWorkDisposition }, + runs: [{ id: "run", nativeIssueId: "issue", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native", resultJson: { nativeResult: result } }], + comments: [{ createdByRunId: "run", authorAgentId: "agent", body }], + initial: { issueIds: [], agentIds: ["agent"] }, state: { issueIds: ["issue"], agentIds: ["agent"], documentCount: blocked ? 0 : 1, interactionCount: 0 }, workspaceChanged: false, + events: [compatibility ? event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "tool", name: tool } }) : event(1, "tool.execution.started", { name: tool, executionId: "tool" }), event(2, "run.result.proposed", result), + compatibility ? event(3, "item.completed", { kind: "tool_result", item: { type: "tool_result", status: "completed", id: "tool" } }) : event(3, "tool.execution.completed", { name: tool, status: "completed", executionId: "tool" }), + event(4, "item.completed", { kind: "agentMessage", channel: "final", item: { type: "agentMessage", phase: "final_answer", text: body, channel: "final" } }), + event(5, "run.result.accepted", { result }, "control_plane"), event(6, "run.terminal", { runTerminalState: "succeeded", turnTerminalState: "completed" }, "control_plane")], + }; +} +function payload(input: NativeCompletionObservation, index: number): Row { return ((input.events[index]!.payload as Row).prpEvent as Row).payload as Row; } +describe("native completion independent oracle", () => { + it.each([[true, false], [true, true], [false, false], [false, true]])("accepts disposition %s compatibility %s with final before late control-plane acceptance", (blocked, compatibility) => expect(gradeNativeCompletion(sample(blocked, compatibility)).passed).toBe(true)); + it.each(["paperclip_finish", "paperclip_block"])("carries normalized %s identity through rehydration into the exact-call oracle", name => { + // The Rust normalization calibration asserts these exact fixture bytes. + const fixture = JSON.parse(readFileSync(new URL("./fixtures/native-completion/terminal-tool-carrier.json", import.meta.url), "utf8")) as { + cases: Array<{ name: string; normalizedPayload: Row }>; + }; + const normalized = fixture.cases.find(value => value.name === name)!.normalizedPayload; + const started = rehydrateRunnerdItemNotification(normalized, "thread", "turn"); + expect(started.item).toMatchObject({ id: "terminal-call", type: "tool_call", name }); + const completed = rehydrateRunnerdItemNotification({ provider: "codex", itemId: "terminal-call", kind: "tool_result", status: "completed", channel: "detail", text: null }, "thread", "turn"); + const value = sample(name === "paperclip_block", true); + payload(value, 0).item = started.item; + payload(value, 2).item = completed.item; + expect(gradeNativeCompletion(value).passed).toBe(true); + (payload(value, 2).item as Row).id = "different-call"; + expect(gradeNativeCompletion(value).checks.find(check => check.id === "final-after-tool-result")?.passed).toBe(false); + }); + it("accepts authoritative acceptance before the provider final", () => { + const value = sample(); + const events = [...value.events]; + value.events = [events[0]!, events[1]!, events[2]!, events[4]!, events[3]!, events[5]!].map((event, index) => ({ ...event, seq: index + 1 })); + expect(gradeNativeCompletion(value).passed).toBe(true); + }); + it("accepts complete public streams containing non-PRP lifecycle rows", () => { + const value = sample(); + value.events = [{ seq: 1, eventType: "lifecycle" }, ...value.events.map(event => ({ ...event, seq: Number(event.seq) + 1 }))]; + expect(gradeNativeCompletion(value).passed).toBe(true); + }); + it.each(["canonical", "compatibility"])("rejects an unmatched %s terminal result", kind => { + const value = sample(true, kind === "compatibility"); + if (kind === "canonical") payload(value, 2).executionId = "unmatched"; + else (payload(value, 2).item as Row).id = "unmatched"; + expect(gradeNativeCompletion(value).passed).toBe(false); + }); + it.each(["missing-name", "unrelated-named-tool", "contradictory-result-name"])("rejects compatibility finishing identity %s despite a matching ID", name => { + const value = sample(true, true); + if (name === "missing-name") delete (payload(value, 0).item as Row).name; + if (name === "unrelated-named-tool") (payload(value, 0).item as Row).name = "write_document"; + if (name === "contradictory-result-name") (payload(value, 2).item as Row).name = "write_document"; + expect(gradeNativeCompletion(value).checks.find(check => check.id === "final-after-tool-result")?.passed).toBe(false); + }); + it.each(["extra-run", "retry", "continuation", "wrong-account", "wrong-agent", "wrong-disposition", "punctuated-action", "wrong-scope", "missing-final", "summary-fallback", "pre-tool-final", "post-admission-call", "missing-acceptance", "runner-acceptance", "failed-terminal", "event-gap", "binding-mismatch", "extra-task", "extra-agent", "extra-document", "interaction", "changed-workspace", "process-call", "marker-only", "contradiction", "wrong-reply-run"])("rejects %s", name => { + const value = sample(); const run = value.runs[0]!; + if (name === "extra-run") value.runs = [run, { ...run, id: "extra" }]; + if (name === "retry") run.retryOfRunId = "prior"; + if (name === "continuation") run.continuationAttempt = 1; + if (name === "wrong-account") run.companyId = "other"; + if (name === "wrong-agent") run.agentId = "other"; + if (name === "wrong-disposition") value.issue.status = "done"; + if (name === "punctuated-action") ((run.resultJson as Row).nativeResult as {blocker: {unblockAction: string}}).blocker.unblockAction = "Grant deployment access."; + if (name === "wrong-scope") ((run.resultJson as Row).nativeResult as {blocker: {scope: string}}).blocker.scope = "step"; + if (name === "missing-final" || name === "summary-fallback") payload(value, 3).kind = "summary", (payload(value, 3).item as Row).type = "summary"; + if (name === "pre-tool-final") (payload(value, 2).name = "other"); + if (name === "post-admission-call") value.events[3]!.eventType = "tool.execution.started"; + if (name === "missing-acceptance") value.events[4]!.eventType = "other"; + if (name === "runner-acceptance") ((value.events[4]!.payload as Row).prpEvent as Row).sourceKind = "runner"; + if (name === "failed-terminal") payload(value, 5).runTerminalState = "failed"; + if (name === "event-gap") value.events[1]!.seq = 20; + if (name === "binding-mismatch") ((value.events[1]!.payload as Row).prpEvent as Row).runId = "other"; + if (name === "extra-task") value.state.issueIds.push("extra"); + if (name === "extra-agent") value.state.agentIds.push("extra"); + if (name === "extra-document") value.state.documentCount = 1; + if (name === "interaction") value.state.interactionCount = 1; + if (name === "changed-workspace") value.workspaceChanged = true; + if (name === "process-call") payload(value, 0).transport = "process"; + if (name === "marker-only" || name === "contradiction") { + const text = name === "marker-only" ? "BLOCKED_probe" : "Deployment is not blocked. Release Owner completed Grant deployment access. BLOCKED_probe"; + (payload(value, 3).item as Row).text = text; value.comments[0]!.body = text; + } + if (name === "wrong-reply-run") value.comments[0]!.createdByRunId = "other"; + expect(gradeNativeCompletion(value).passed).toBe(false); + }); + it.each([undefined, "other-issue"])("rejects missing or wrong native issue attribution %s", nativeIssueId => { + const value = sample(); value.runs[0]!.nativeIssueId = nativeIssueId; + expect(gradeNativeCompletion(value).passed).toBe(false); + }); + it.each(["missing-action-envelope", "wrong-action-source", "missing-source-identity", "extra-wrong-source-result"])("rejects %s", name => { + const value = sample(); + if (name === "missing-action-envelope") value.events[0]!.payload = {}; + if (name === "wrong-action-source") ((value.events[0]!.payload as Row).prpEvent as Row).sourceKind = "control_plane"; + if (name === "missing-source-identity") { + delete value.events[0]!.sourceEventId; + delete ((value.events[0]!.payload as Row).prpEvent as Row).sourceEventId; + } + if (name === "extra-wrong-source-result") { + const extra = structuredClone(value.events[4]!); + ((extra.payload as Row).prpEvent as Row).sourceKind = "runner"; + value.events = [...value.events, { ...extra, seq: 7 }]; + } + expect(gradeNativeCompletion(value).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/native-completion-scoring.ts b/tests/runner-e2e/native-completion-scoring.ts new file mode 100644 index 00000000000..86740c45808 --- /dev/null +++ b/tests/runner-e2e/native-completion-scoring.ts @@ -0,0 +1,135 @@ +import { isValidNativePrpEnvelope } from "./native-event-envelope.js"; + +type Row = Record; +const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; +const forbiddenKinds = new Set(["commandExecution", "process", "fileChange", "file_change", "apply_patch"]); +const toolKinds = new Set(["dynamicToolCall", "mcpToolCall", "commandExecution", "tool_call", "tool_use"]); +const terminalName = (value: unknown, expected: string) => typeof value === "string" && (value === expected || value.endsWith(`.${expected}`)); + +export interface NativeCompletionObservation { + caseId: "assigned-skill-explicit-invocation" | "native-blocked-report"; + companyId: string; agentId: string; issue: Row; runs: readonly Row[]; + comments: readonly Row[]; events: readonly Row[]; + state: { issueIds: string[]; agentIds: string[]; documentCount: number; interactionCount: number }; + initial: { issueIds: string[]; agentIds: string[] }; + workspaceChanged: boolean; + marker: string; +} + +/** Observable tool/result/final sequence, not a claim that the provider consumed feedback. */ +export function gradeNativeCompletion(input: NativeCompletionObservation) { + const checks: Array<{ id: string; passed: boolean; detail: string }> = []; + const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail }); + const blocked = input.caseId === "native-blocked-report"; + const run = input.runs[0] ?? {}; + const expectedDisposition = blocked ? "blocked" : "done"; + const expectedTool = blocked ? "paperclip_block" : "paperclip_finish"; + const nativeResult = row(row(run.resultJson).nativeResult); + check("one-native-attempt", input.runs.length === 1 && run.status === "succeeded" && run.runtimeMode === "native" + && typeof run.id === "string" && run.id.length > 0 && typeof input.issue.id === "string" && input.issue.id.length > 0 + && run.nativeIssueId === input.issue.id + && run.companyId === input.companyId && run.agentId === input.agentId + && !run.retryOfRunId && !run.continuationAttempt && !input.issue.scheduledRetry, + "Exactly one succeeded native run, with no continuation/retry or uncounted extra work."); + check("durable-disposition", input.issue.companyId === input.companyId && input.issue.assigneeAgentId === input.agentId + && input.issue.status === expectedDisposition && nativeResult.reportedWorkDisposition === expectedDisposition, + "Public durable issue status and the native semantic result agree with the requested disposition."); + if (blocked) { + const blocker = row(nativeResult.blocker); + check("exact-blocker", row(blocker.owner).name === "Release Owner" && blocker.unblockAction === "Grant deployment access" + && blocker.scope === "task_wide", "The structured owner, exact action and whole-task scope are independently checked."); + } + + const events = input.events.map(event => ({ event, envelope: row(row(event.payload).prpEvent) })); + const bound = events.length > 0 && events.every(({ event, envelope }, index) => { + const action = /^(?:tool\.execution\.|item\.)/.test(String(event.eventType)); + const required = action || ["run.result.proposed", "run.result.accepted", "run.terminal"].includes(String(event.eventType)); + return event.seq === index + 1 + && (Object.keys(envelope).length === 0 ? !required : isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion as number | undefined) + && (!action || envelope.sourceKind === "runner") + && typeof envelope.sourceEventId === "string" && envelope.sourceEventId.length > 0 + && typeof envelope.sourceInstanceId === "string" && envelope.sourceInstanceId.length > 0 + && Number.isSafeInteger(envelope.sourceSeq) && Number(envelope.sourceSeq) > 0 + && envelope.runId === run.id && envelope.eventType === event.eventType + && envelope.sourceEventId === event.sourceEventId && envelope.sourceInstanceId === event.sourceInstanceId + && envelope.sourceSeq === event.sourceSeq); + }); + check("complete-bound-events", bound, "A complete contiguous public PRP stream is bound to this run and persisted source identity."); + const selected = (type: string) => events.filter(({ event }) => event.eventType === type); + const proposed = selected("run.result.proposed"); + const accepted = selected("run.result.accepted"); + const terminal = selected("run.terminal"); + const acceptedResult = row(row(accepted[0]?.envelope.payload).result); + const terminalResult = row(terminal[0]?.envelope.payload); + check("authoritative-native-result", proposed.length === 1 && accepted.length === 1 && terminal.length === 1 + && proposed[0]?.envelope.sourceKind === "runner" && accepted[0]?.envelope.sourceKind === "control_plane" && terminal[0]?.envelope.sourceKind === "control_plane" + && row(proposed[0]?.envelope.payload).reportedWorkDisposition === expectedDisposition + && acceptedResult.reportedWorkDisposition === expectedDisposition + && terminalResult.runTerminalState === "succeeded" && terminalResult.turnTerminalState === "completed", + "One runner proposal is independently accepted and terminated by the control plane."); + const proposalSeq = Number(proposed[0]?.event.seq ?? -1); + const finishes = events.filter(({ event, envelope }) => { + const payload = row(envelope.payload), item = row(payload.item); + const completion = envelope.sourceKind === "runner" && Number(event.seq) > proposalSeq && ( + event.eventType === "tool.execution.completed" && terminalName(payload.name, expectedTool) && payload.status === "completed" + || event.eventType === "item.completed" && (payload.kind === "tool_result" || item.type === "tool_result") && item.status === "completed" + ); + if (!completion) return false; + return events.some(({ event: start, envelope: startEnvelope }) => { + const started = row(startEnvelope.payload), startedItem = row(started.item); + return startEnvelope.sourceKind === "runner" && Number(start.seq) < proposalSeq && ( + event.eventType === "tool.execution.completed" && start.eventType === "tool.execution.started" + && typeof payload.executionId === "string" && payload.executionId.length > 0 && started.executionId === payload.executionId && terminalName(started.name, expectedTool) + || event.eventType === "item.completed" && start.eventType === "item.started" + && (started.kind === "tool_call" || startedItem.type === "tool_call") + && typeof item.id === "string" && item.id.length > 0 && item.id === startedItem.id + && terminalName(startedItem.name, expectedTool) + && (item.name === undefined || item.name === null || terminalName(item.name, expectedTool)) + ); + }); + }); + const finals = events.filter(({ event, envelope }) => { + const payload = row(envelope.payload), item = row(payload.item); + return envelope.sourceKind === "runner" && event.eventType === "item.completed" + && (payload.kind === "agentMessage" || item.type === "agentMessage") + && (payload.channel === "final" || item.channel === "final") && item.phase === "final_answer" + && typeof item.text === "string" && item.text.trim().length > 0; + }); + const finishSeq = Number(finishes.at(-1)?.event.seq ?? -1); + const finalSeq = Number(finals[0]?.event.seq ?? -1); + const starts = events.filter(({ event, envelope }) => envelope.sourceKind === "runner" && ( + event.eventType === "tool.execution.started" || event.eventType === "item.started" + && toolKinds.has(String(row(envelope.payload).kind ?? row(row(envelope.payload).item).type)))); + check("final-after-tool-result", proposed.length === 1 && finishes.length > 0 && finals.length === 1 + && proposalSeq < finishSeq && finishSeq < finalSeq + && proposalSeq < Number(accepted[0]?.event.seq ?? -1) && Number(accepted[0]?.event.seq ?? -1) < Number(terminal[0]?.event.seq ?? -1) + && !starts.some(({ event }) => Number(event.seq) > proposalSeq), + "The provider final follows the observed finishing result; no new invocation follows admission. Control-plane acceptance precedes its terminal receipt independently."); + const finalText = String(row(row(finals[0]?.envelope.payload).item).text ?? "").trim(); + const replies = input.comments.filter(comment => comment.createdByRunId === run.id && comment.authorAgentId === input.agentId + && typeof comment.body === "string" && comment.body.trim() === finalText); + check("persisted-provider-final", finalText.length > 0 && replies.length === 1, + "A nonempty provider final is durably projected as this run's agent reply; semantic-summary fallback is insufficient."); + if (blocked) check("visible-blocker-content", finalText.split(input.marker).length === 2 + && finalText.includes("Release Owner") && finalText.includes("Grant deployment access") + && /\b(?:blocked|cannot proceed|can't proceed|missing|required access|not (?:yet )?granted|awaiting|waiting|unavailable)\b/i.test(finalText) + && !/\b(?:not blocked|no longer blocked|access (?:is |has been |was )?already granted|completed Grant deployment access)\b/i.test(finalText), + "The final explains the actual unresolved blocker and its owner/action, rather than supplying only a marker."); + const same = (a: string[], b: string[]) => a.length === b.length && a.every(id => b.includes(id)) && new Set(a).size === a.length; + check("bounded-durable-work", same(input.state.issueIds, [...input.initial.issueIds, String(input.issue.id)]) + && same(input.state.agentIds, input.initial.agentIds) && input.state.documentCount === (blocked ? 0 : 1) + && input.state.interactionCount === 0, + "No additional tasks, agents or interactions; completion permits its one required document and blocker permits none."); + if (blocked) { + const forbidden = events.some(({ event, envelope }) => { + const payload = row(envelope.payload), item = row(payload.item); + return envelope.sourceKind === "runner" && (forbiddenKinds.has(String(payload.kind ?? item.type)) + || String(event.eventType).startsWith("tool.execution.") && payload.transport === "process" + || /(?:write_document|create_task|hire_agent|register_deliverable|apply_patch|exec_command)/.test(String(payload.name ?? item.name ?? ""))); + }); + check("no-deployment-or-file-work", !input.workspaceChanged && !forbidden, + "The fixture workspace is unchanged and no process/file/deployment or extra-work tool is observed."); + } + return { schema: "paperclip.native-completion-observation.v2", passed: checks.every(value => value.passed), checks, + limitations: ["Exact provider feedback identity/consumption is not measured by the public sequence."] }; +} diff --git a/tests/runner-e2e/native-completion-source-contract.mjs b/tests/runner-e2e/native-completion-source-contract.mjs new file mode 100644 index 00000000000..6c6ea682ab1 --- /dev/null +++ b/tests/runner-e2e/native-completion-source-contract.mjs @@ -0,0 +1,155 @@ +// Exact native-only comparison boundary: executable fixtures stay byte-identical. +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +export const NATIVE_COMPLETION_SOURCE_CONTRACT = { + "schema": "paperclip.native-completion-source-contract.v1", + "baseSha": "59c07ede72dc08b8aba149a01cc11e0b7a204621", + "archiveSha": "9138f570c341c251a5727c32d6615ce238bc8e03", + "shallowParentAnchors": { + "candidate": "0a9c5a75164cd0b02115ff12273aadb6a86c9464", + "historical": "00a761b967f9f73b0f45069c0ba828fae277a76e" + }, + "variants": { + "candidate": { + "packages/paperclip-runner/src/contracts/completion-result.ts": "bd4bb79d2be5c4ee3765df67dfb4da98ef95bc532cac872be40cb41a6bff6cda", + "packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts": "aff3b3d2d626fbaf504a7e46adb27cb8127d0afbc62a8ed58111938f1293de7d", + "packages/paperclip-runner/src/drivers/runner-tool-bridge.ts": "4cca0d7837e0f5a1f709abb0ba3982227288a01abd16a08107337416fde4a9e2", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "ec1cd94c17be22d0556d4586fa6407b28e2309ed7c2437d95a32beeb0755cc4f", + "server/src/services/native-runtime/native-session-resume.ts": "5b27465e1a328412770fc27c4bc637e88b62022adc6a67709bb74187eadc465b", + "packages/paperclip-runner/src/contracts/completion-result.test.ts": "42bd18105505da7fd7963311bf862002df7158f8c0cf2a81b927dff4c47ac500", + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts": "2b15425514d8e3316baec48b8ce118394acf67d97b76cbab2a99916fc4e8e1d2", + "packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts": "097d4973ac4ca200e82af31aece0f9d5531e1756ffe739d4422bd8f65d479232", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts": "533dd12ea00c61e012f5d596e03080fa1aaf63375ad7602c0de4e0a6c8370400", + "packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts": "8546181e5b3a63a194ee9ea0fc99c2bd97562f0c105fffde298fa6d74b22e83b", + "server/src/services/native-runtime/native-session-resume.test.ts": "57b498d76bc89801edc81d700cd87660a16074ae8bb8a43aa566205cc377d7b4" + }, + "historical": { + "packages/paperclip-runner/src/contracts/completion-result.ts": "00a684f9aea87f68eabf0974681c52461bc308631a1252a89c4ed087d7158b28", + "packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts": "87929dd81b6fdc44fb46e5b14ac775ca7244354587028b3bd356a617dcf31907", + "packages/paperclip-runner/src/drivers/runner-tool-bridge.ts": "966696d39eb9554e5f40bd1d89d31efc426c3c01d2ce883b3b6e968db8480088", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts": "7080ae2896fe7a81fdf1a1c4f7fcd1773611cbd3336896b5f9e861c5b0aa0ed4", + "server/src/services/native-runtime/native-session-resume.ts": "72b4f2e19c01a83ccd52e21102d2c5422265eec84b589ea75a7cffe40d1d0b8d", + "packages/paperclip-runner/src/contracts/completion-result.test.ts": "7c615dc36ad07b7659c3605f94fec89ae592727ce078a62fa45eb1bb549f26d5", + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts": "0b7e8b7a552d6a3d74d761e0e68b720ed02d621ded41a1c2076ca3fda0f96d0c", + "packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts": "1ce59108e0cfef78fa71f0da9b46af70fd7bf3887d4a662556afc072d2351653", + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts": "bb1b0447cca30f3a2984e9e832949108c43cefa35d765440d07ee8098be4ef0c", + "packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts": "624bbac680e1dfc87fff9fab71501e8208e0bdbc6aefcb45a008b8e6c9030a27", + "server/src/services/native-runtime/native-session-resume.test.ts": "ddd60db81cdb8d17a2b31ea934779687708b927f877f3fb6d0c249ceb805be62" + } + }, + "fixedContext": { + "server/src/onboarding-assets/default/AGENTS.md": "e4d2375d722602cd744403292d99f6e6c9b7e8014d9cdfa6f9811ace03428e5f", + "packages/adapter-utils/src/server-utils.ts": "916e879f70a90d31c89bc1486d5f84807c4ecafeab62961f67d84d1ebec7a173", + "skills/paperclip/SKILL.md": "19b4e17bc6588b087efce63673c3f2839ac09dba9c072f56b8e90924a66856de", + "server/src/services/onboarding-first-task-assets.ts": "c7fd1db7ed97850bd67e48a4d8c1470c8ff801f895f4f665b84690978f19b47e", + "server/src/services/default-agent-instructions.ts": "8bd3b59f900c132db5ed89b96524f35ba7c8dafb9d58db1dac888430f08c4d63", + "packages/paperclip-runner/src/contracts/runtime-context.ts": "a17e30a69f3cb38fba00c37e1a06753d723651a004e6050c9e77ae202a61264b", + "packages/paperclip-runner/src/backends/runtime-context.ts": "e7c46e81cc9c93f9a4f805b5091ef08c59ea70c62f208cfda70d6edeed363898", + "server/src/services/native-runtime/native-execution-input.ts": "e346bf3f163dfc5374c3c90fb515c1db6847bbab69fb59a49d73227478cef70d", + "server/src/services/native-runtime/completion-contracts.ts": "b1f4f7a9b04fea902c23313adb296e67f202ee5878424f85a7d0cebcb775ab69", + "server/src/services/native-runtime/runtime-context.ts": "13f71f749ad6a3a0612e3e268fdc2bc58ee26f70399dd120b1341abf00d1847b", + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts": "1a3b80e9b155a1e7e1f72313ad41bb0a397d6d42cb6aedf23704c1cf9121c0b8" + } +}; +export const NATIVE_COMPLETION_FIXTURE_FILES = [ + "tests/runner-e2e/native-completion-cases.ts", + "tests/runner-e2e/native-completion-scoring.ts", + "tests/runner-e2e/native-completion-scoring.test.ts", + "tests/runner-e2e/native-completion-defaults.ts", + "tests/runner-e2e/native-completion-defaults.test.ts", + "tests/runner-e2e/native-blocker-visible.ts", + "tests/runner-e2e/native-blocker-visible.spec.ts", + "tests/runner-e2e/native-completion-admission.ts", + "tests/runner-e2e/native-completion-admission.test.ts", + "tests/runner-e2e/native-completion-checks.mjs", + "tests/runner-e2e/native-completion-checks.test.mjs", + "tests/runner-e2e/native-completion-git-source.mjs", + "tests/runner-e2e/native-completion-git-source.test.mjs", + "tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json", + "tests/runner-e2e/native-completion-source-contract.mjs", + "tests/runner-e2e/native-completion-source-contract.test.mjs", + "tests/runner-e2e/automatic-retry.ts", + "tests/runner-e2e/automatic-retry.test.ts", + "tests/runner-e2e/launch.ts", + "tests/runner-e2e/types.ts", + "tests/runner-e2e/catalog.ts", + "tests/runner-e2e/catalog.test.ts", + "tests/runner-e2e/runner.spec.ts", + "tests/runner-e2e/playwright-support.config.ts", + "tests/runner-e2e/context-integrity-cases.ts", + "tests/runner-e2e/context-integrity.test.ts", + "tests/runner-e2e/context-integrity-evidence.test.ts", + "tests/runner-e2e/context-integrity-flow-logs.test.ts", + "tests/runner-e2e/live-fixtures.ts", + "tests/runner-e2e/context-integrity-flow.ts", + "tests/runner-e2e/context-integrity-scoring.ts", + "tests/runner-e2e/selectors.ts", + "tests/runner-e2e/harness-env.ts", + "tests/runner-e2e/tsconfig.json", + "tests/runner-e2e/vitest.config.ts" +]; +export const NATIVE_COMPLETION_MANIFEST_FILES = [ + "packages/paperclip-runner/scripts/generate-capability-contract.mjs", + "packages/paperclip-runner/scripts/check-capability-inventory.mjs", + "packages/paperclip-runner/scripts/lib/capability-inventory.mjs", + "packages/paperclip-runner/spec/capability/source-contract.json", + "packages/paperclip-runner/spec/capability/capabilities.yaml", + "packages/paperclip-runner/spec/capability/eval-traceability.yaml", + "packages/paperclip-runner/spec/capability/mcp-tool-map.yaml", + "packages/paperclip-runner/spec/capability/inventory.schema.json", + "packages/paperclip-runner/src/generated/capability-contract.ts", + "packages/paperclip-runner/docs/capability-contract.md", + ...["capabilities.yaml", "mcp-tool-map.yaml", "eval-traceability.yaml", "capability-contract.md", "downstream-handoff.md"] + .map(file => `packages/paperclip-runner/generated/capability/${file}`), +]; +export const NATIVE_COMPLETION_SOURCE_FILES = [...new Set([ + ...Object.keys(NATIVE_COMPLETION_SOURCE_CONTRACT.variants.candidate), + ...Object.keys(NATIVE_COMPLETION_SOURCE_CONTRACT.fixedContext), + ...NATIVE_COMPLETION_FIXTURE_FILES, ...NATIVE_COMPLETION_MANIFEST_FILES, + "scripts/ensure-plugin-build-deps.mjs", + "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs", + "packages/paperclip-runner/src/live/runnerd-codex-transport.ts", + "packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts", + "packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts", + "packages/adapters/codex-local/src/index.ts", "packages/adapters/claude-local/src/index.ts", + "packages/paperclip-runner/package.json", "tsconfig.base.json", + "packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs", + "packages/paperclip-runner/runner/Cargo.toml", "packages/paperclip-runner/runner/Cargo.lock", + "packages/paperclip-runner/runner/crates/runner-core/Cargo.toml", +])].sort(); + +export const nativeSourceSha256 = bytes => createHash("sha256").update(bytes).digest("hex"); +function fingerprint(paths, read) { + const hash = createHash("sha256"); + for (const file of [...paths].sort()) { + const bytes = read(file); + hash.update(JSON.stringify([file, bytes.length])).update(bytes); + } + return hash.digest("hex"); +} + +/** Infer a variant only from all eleven matching production/assertion files. */ +export function nativeCompletionSourceFingerprint(read = file => readFileSync(join( + resolve(import.meta.dirname, "../.."), file)), contract = NATIVE_COMPLETION_SOURCE_CONTRACT) { + const observations = new Map(), sourceErrors = []; + for (const file of NATIVE_COMPLETION_SOURCE_FILES) { + try { observations.set(file, read(file)); } catch { sourceErrors.push(file); } + } + const matches = Object.entries(contract.variants).filter(([, expected]) => Object.entries(expected) + .every(([file, digest]) => observations.has(file) && nativeSourceSha256(observations.get(file)) === digest)); + const variant = matches.length === 1 ? matches[0][0] : null; + if (!variant) sourceErrors.push("native-completion: mixed or unknown production/assertion variant"); + for (const [file, digest] of Object.entries(contract.fixedContext)) + if (!observations.has(file) || nativeSourceSha256(observations.get(file)) !== digest) + sourceErrors.push(`native-completion: changed master context ${file}`); + const fromObservation = file => observations.get(file); + const complete = NATIVE_COMPLETION_SOURCE_FILES.every(file => observations.has(file)); + return { variant, sourceErrors, + fingerprint: complete ? fingerprint(NATIVE_COMPLETION_SOURCE_FILES, fromObservation) : null, + fixtureFingerprint: complete ? fingerprint(NATIVE_COMPLETION_FIXTURE_FILES, fromObservation) : null, + manifestFingerprint: complete ? fingerprint(NATIVE_COMPLETION_MANIFEST_FILES, fromObservation) : null, + baseSha: contract.baseSha, archiveSha: contract.archiveSha, + }; +} diff --git a/tests/runner-e2e/native-completion-source-contract.test.mjs b/tests/runner-e2e/native-completion-source-contract.test.mjs new file mode 100644 index 00000000000..d6044b2b51f --- /dev/null +++ b/tests/runner-e2e/native-completion-source-contract.test.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + NATIVE_COMPLETION_SOURCE_CONTRACT as original, NATIVE_COMPLETION_SOURCE_FILES, + nativeCompletionSourceFingerprint, nativeSourceSha256, +} from "./native-completion-source-contract.mjs"; + +function fixture(variant = "candidate") { + const bytes = new Map(NATIVE_COMPLETION_SOURCE_FILES.map(file => [file, Buffer.from(`common:${file}`)])); + const variantFiles = Object.keys(original.variants.candidate), fixed = Object.keys(original.fixedContext); + const contract = { ...original, variants: Object.fromEntries(["candidate", "historical"].map(name => [name, + Object.fromEntries(variantFiles.map(file => [file, nativeSourceSha256(Buffer.from(`${name}:${file}`))]))])), + fixedContext: Object.fromEntries(fixed.map(file => [file, nativeSourceSha256(bytes.get(file))])) }; + for (const file of variantFiles) bytes.set(file, Buffer.from(`${variant}:${file}`)); + const read = file => { if (!bytes.has(file)) throw new Error("Missing source"); return bytes.get(file); }; + return { bytes, contract, read }; +} +for (const variant of ["candidate", "historical"]) test(`native source contract accepts the complete ${variant} source map`, () => { + const f = fixture(variant), result = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.equal(result.variant, variant); assert.deepEqual(result.sourceErrors, []); + for (const key of ["fingerprint", "fixtureFingerprint", "manifestFingerprint"]) assert.match(result[key], /^[a-f0-9]{64}$/); +}); +test("native source contract keeps fixtures identical across candidate and historical variants", () => { + const a = fixture("candidate"), b = fixture("historical"); + const candidate = nativeCompletionSourceFingerprint(a.read, a.contract), historical = nativeCompletionSourceFingerprint(b.read, b.contract); + assert.equal(candidate.fixtureFingerprint, historical.fixtureFingerprint); + assert.equal(candidate.manifestFingerprint, historical.manifestFingerprint); + assert.notEqual(candidate.fingerprint, historical.fingerprint); +}); +for (const file of Object.keys(original.variants.candidate)) test(`native source contract rejects a mixed production/assertion source: ${file}`, () => { + const f = fixture(); f.bytes.set(file, Buffer.from(`historical:${file}`)); + const result = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.equal(result.variant, null); assert.ok(result.sourceErrors.some(error => error.includes("mixed or unknown"))); +}); +for (const file of Object.keys(original.fixedContext)) test(`native source contract rejects changed master context: ${file}`, () => { + const f = fixture(); f.bytes.set(file, Buffer.from("Reduced or replaced context")); + const result = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.ok(result.sourceErrors.includes(`native-completion: changed master context ${file}`)); +}); +test("native source contract rejects missing files and changes its digest when a fixture changes", () => { + const f = fixture(), before = nativeCompletionSourceFingerprint(f.read, f.contract); + f.bytes.set("tests/runner-e2e/native-completion-cases.ts", Buffer.from("Changed behavior")); + const changed = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.notEqual(before.fixtureFingerprint, changed.fixtureFingerprint); + f.bytes.delete("tests/runner-e2e/native-completion-cases.ts"); + const missing = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.equal(missing.fingerprint, null); assert.ok(missing.sourceErrors.includes("tests/runner-e2e/native-completion-cases.ts")); +}); +test("native source contract rejects an ambiguous identical variant map", () => { + const f = fixture(); f.contract.variants.historical = f.contract.variants.candidate; + assert.equal(nativeCompletionSourceFingerprint(f.read, f.contract).variant, null); +}); +test("native source contract binds common Rust carrier source and the shared behavioral projection fixture", () => { + const f = fixture(), before = nativeCompletionSourceFingerprint(f.read, f.contract); + f.bytes.set("packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs", Buffer.from("Changed carrier")); + const carrier = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.equal(carrier.variant, "candidate"); assert.notEqual(carrier.fingerprint, before.fingerprint); + f.bytes.set("tests/runner-e2e/fixtures/native-completion/terminal-tool-carrier.json", Buffer.from("Changed projection")); + const projection = nativeCompletionSourceFingerprint(f.read, f.contract); + assert.notEqual(projection.fingerprint, carrier.fingerprint); assert.notEqual(projection.fixtureFingerprint, carrier.fixtureFingerprint); +}); +test("native source contract binds exactly five production and six variant assertion files", () => { + const files = Object.keys(original.variants.candidate); + assert.equal(files.length, 11); assert.equal(files.filter(file => file.endsWith(".test.ts")).length, 6); + assert.deepEqual(files, Object.keys(original.variants.historical)); + assert.ok(files.every(file => original.variants.candidate[file] !== original.variants.historical[file])); + assert.equal(original.baseSha, "59c07ede72dc08b8aba149a01cc11e0b7a204621"); + assert.equal(original.archiveSha, "9138f570c341c251a5727c32d6615ce238bc8e03"); + assert.deepEqual(original.shallowParentAnchors, { + candidate: "0a9c5a75164cd0b02115ff12273aadb6a86c9464", + historical: "00a761b967f9f73b0f45069c0ba828fae277a76e", + }); + assert.ok(!NATIVE_COMPLETION_SOURCE_FILES.some(file => file.includes("stock-harness") || file.endsWith("issue-documents.md"))); +}); diff --git a/tests/runner-e2e/playwright-support.config.ts b/tests/runner-e2e/playwright-support.config.ts index 11a00cb0529..6d2d2976a5e 100644 --- a/tests/runner-e2e/playwright-support.config.ts +++ b/tests/runner-e2e/playwright-support.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test"; /** Browser-only harness regressions: no Paperclip instance or provider credentials. */ export default defineConfig({ testDir: ".", - testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts"], + testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts"], workers: 1, retries: 0, timeout: 10_000, diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 3833d3cef02..46cc8e96aaa 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,7 @@ +import { assertNativeCompletionSelection, NATIVE_COMPLETION_PREFLIGHT_ENV, verifyNativeCompletionPreflight } from "./native-completion-admission.js"; +import { captureNativeDefault, gradeNativeDefault, nativeCompletionWorkspaceDigest } from "./native-completion-defaults.js"; +import { gradeNativeCompletion } from "./native-completion-scoring.js"; +import { assertNativeBlockerReply } from "./native-blocker-visible.js"; import { warmManagedFileEvidence } from "./warm-managed-files.js"; import { gitFinalizationEvidence, gitStreamingEvidence, setupGitStreamingWorkspace } from "./daytona-git-streaming.js"; import { runsCompletionUpdateProbe, completionQualityControls, completionQualityStatus, judgeCompletionQuality, reserveCompletionQuality, type CompletionQualityRecord } from "./completion-quality.js"; @@ -552,6 +556,10 @@ for (const execution of executions) { let apiResponseSourceId: string | undefined; let titleCreation: { issue: unknown; submitted: unknown } | undefined; let lifecycleBlockerId: string | null = null; + if (execution.suite.id === "native-completion") { + assertNativeCompletionSelection([execution]); + verifyNativeCompletionPreflight(process.env[NATIVE_COMPLETION_PREFLIGHT_ENV]); + } const credentials = credentialValues(); const secrets = normalizedSecrets(Object.values(credentials)); const api = new RunnerApi(request); @@ -560,6 +568,9 @@ for (const execution of executions) { const networkDiagnostics: Array> = []; const pageLifecycleDiagnostics: Array> = []; const browserBootstrap = observeBrowserBootstrap(page); + let nativeInitial: { issueIds: string[]; agentIds: string[]; workspaceDigest: string | null } | undefined; + const nativeWorkspaceDigest = () => execution.task.id === "native-blocked-report" + ? nativeCompletionWorkspaceDigest(workspacePath!) : Promise.resolve(null); let fixtures: LiveFixtureValues | undefined; let reviewProvider: Awaited> | undefined; let issue: IssueRecord | undefined; @@ -842,6 +853,18 @@ for (const execution of executions) { daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, }); + if (execution.suite.id === "native-completion") { + const receipt = await captureNativeDefault({ api, agentId: fixtures.agent.id, companyId: fixtures.company.id }); + const grade = gradeNativeDefault(receipt); + await writeSanitizedJson(snapshotsDir, "native-default-before-execution.json", { receipt, grade }, secrets); + if (!grade.passed) throw new Error("Native production-default hire admission failed before execution"); + const [issues, agents, workspaceDigest] = await Promise.all([ + api.get>(`/api/companies/${fixtures.company.id}/issues?limit=100`), + api.get>(`/api/companies/${fixtures.company.id}/agents`), nativeWorkspaceDigest(), + ]); + nativeInitial = { issueIds: issues.map(value => value.id), agentIds: agents.map(value => value.id), workspaceDigest }; + } + if (execution.suite.id === "api-response-reading") { const source = await api.post<{ id: string }>(`/api/companies/${fixtures.company.id}/issues`, { title: `Synthetic diagnostic evidence ${nonce}`, @@ -2574,6 +2597,8 @@ for (const execution of executions) { await expect(proofLink).toHaveAttribute("href", `/api/attachments/${downloadedResponseProof!.attachmentId}/content`); } else if (execution.suite.id === "task-titles") { await expect(visibleAgentReplies.filter({ hasText: marker }).last()).toBeVisible({ timeout: 30_000 }); + } else if (execution.suite.id === "native-completion" && execution.task.id === "native-blocked-report") { + await assertNativeBlockerReply(visibleAgentReplies, marker); } else if (execution.task.flow === "warm_three_turn") { // Prove the persisted user-facing response is visible, independently // of the byte-for-byte workspace checks and lease continuity checks. @@ -2633,6 +2658,31 @@ for (const execution of executions) { ); } } + if (execution.suite.id === "native-completion") { + if (!issue || !nativeInitial) throw new Error("Missing native qualification issue or pre-execution receipt"); + const [currentIssue, companyRuns, issues, agents, comments, documents, interactions, workspaceDigest] = await Promise.all([ + api.get(`/api/issues/${issue.id}`), + api.get(`/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`), + api.get>(`/api/companies/${fixtures.company.id}/issues?limit=100`), + api.get>(`/api/companies/${fixtures.company.id}/agents`), + api.get(`/api/issues/${issue.id}/comments`), + api.get(`/api/issues/${issue.id}/documents`), + api.get(`/api/issues/${issue.id}/interactions`), nativeWorkspaceDigest(), + ]); + const detailedRuns = await Promise.all(companyRuns.map(candidate => api.get(`/api/heartbeat-runs/${candidate.id}`))); + selectedRuns = detailedRuns; issue = currentIssue; + const events = detailedRuns.length === 1 ? await collectRunEvents((afterSeq, limit) => api.get(`/api/heartbeat-runs/${detailedRuns[0]!.id}/events?afterSeq=${afterSeq}&limit=${limit}`)) : []; + const observation = { caseId: execution.task.id as "assigned-skill-explicit-invocation" | "native-blocked-report", + companyId: fixtures.company.id, agentId: fixtures.agent.id, issue: currentIssue as unknown as Record, + runs: detailedRuns as unknown as Record[], comments: comments as unknown as Record[], events: events as unknown as Record[], + initial: nativeInitial, state: { issueIds: issues.map(value => value.id), agentIds: agents.map(value => value.id), documentCount: documents.length, interactionCount: interactions.length }, + workspaceChanged: workspaceDigest !== nativeInitial.workspaceDigest, marker }; + const grade = gradeNativeCompletion(observation); + const integrity = detailedRuns.flatMap(candidate => nativeRunEventIntegrityFailures(candidate, events)); + await writeSanitizedJson(snapshotsDir, "native-completion.json", { observation, grade, integrity }, secrets); + matcherResults.push(...grade.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `nativeCompletion.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }))); + if (!grade.passed || integrity.length) throw new Error(`Native completion qualification failed: ${[...grade.checks.filter(check => !check.passed).map(check => check.id), ...integrity].join("; ")}`); + } if (runsCompletionUpdateProbe(execution) && credentials.OPENAI_API_KEY) { const qualification = completionQualityStatus(completionQuality); if (qualification === "unqualified") { diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index 0d230d87437..b0e3c1608a9 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -138,6 +138,8 @@ export interface RunnerTaskFixture { expectedRunCount: number; /** Optional lower bound; expectedRunCount remains the maximum/cost estimate. */ minimumExpectedRunCount?: number; + /** Admit only the first attempt, including provider or infrastructure failures. */ + automaticRetryPolicy?: "single_attempt"; attemptTimeoutMs: Readonly>; expectedTerminalState: { issue: "done" | "in_review" | "blocked" | "in_progress";