feat(runner): add remote execution substrate (#12638)

## Thinking Path

> - Paperclip is the open source app people use to manage AI agents for
work.
> - Paperclip Runner gives native runs a durable and governed execution
path.
> - The current native path runs on the control-plane host.
> - Remote environments need an authenticated execution-target contract.
> - The contract must not change direct adapters or enable new runtimes
by default.
> - This pull request adds the remote execution substrate and Daytona
ingress.
> - The benefit is a bounded base for later remote runner transport
work.

## Linked Issues or Issue Description

Refs #12616.
Refs #12352.

**Subsystem affected**

Cross-cutting. This change touches runner transport, server
orchestration, plugin contracts, and shared settings.

**Problem or motivation**

Native execution cannot resolve an authenticated runner ingress through
a remote environment. The server also lacks one provider-neutral
contract for remote execution targets.

**Proposed solution**

Add a default-off runner preview ingress capability. Add
transport-neutral runner connectivity. Add remote execution target and
lifecycle handling. Add a Daytona ingress implementation with redacted
credentials.

**Alternatives considered**

A provider-specific server path would duplicate orchestration and
authorization. A public endpoint without an environment contract would
weaken the trust boundary.

**Roadmap alignment**

This work supports the Cloud and Sandbox agents milestone. It also
supports self-healing runs and governed tool access.

## What Changed

- Added execution-target traits for local, SSH, and sandbox
environments.
- Added plugin RPC contracts for runner ingress endpoints.
- Added authenticated Daytona preview ingress.
- Added transport-neutral PRP outbound connections.
- Added remote runner artifact verification and fail-closed provider
selection.
- Added bounded native session resume, cancellation, and lifecycle
recovery.
- Preserved Codex-only selection for fresh experimental runner starts.
- Preserved all direct adapter execution and finalization paths.
- Removed stale Pi provider-pack requirements that security review
rejected.
- Kept the rollout controls off by default.
- Did not change pnpm-lock.yaml, Cargo, database migrations, or GitHub
workflows.

## Verification

- GitHub Actions will run the repository test, typecheck, build,
security, and policy gates.
- Focused tests cover ingress validation, redaction, execution targets,
remote lifecycle, cancellation, resume, and legacy adapter selection.
- Local tests were not run. The requested verification policy uses
GitHub Actions for this series.
- `git diff --check origin/master...HEAD` passes.
- The diff contains 52 files.

## Risks

- Remote execution crosses a trust boundary.
- The implementation validates target capabilities, artifact digests,
provider-pack pins, and connection metadata.
- The feature remains default-off.
- Fresh native selection remains Codex-only.
- Existing direct adapters remain on the legacy path.
- This PR does not yet make remote Codex runnable. The next PR adds the
Rust WSS and TLS transport.

## Model Used

OpenAI Codex with GPT-5.6. The work used high-reasoning agent mode,
repository tools, GitHub tools, and parallel code-audit agents.

## Checklist

- [x] I have included a thinking path that traces from project context
to this change
- [x] I have specified the model used (with version and capability
details)
- [x] I have checked ROADMAP.md and confirmed this PR does not duplicate
planned core work
- [x] I have searched GitHub for duplicate or related PRs and linked
them above
- [x] I have either (a) linked existing issues with Fixes: / Closes /
Refs OR (b) described the issue in-PR following the relevant issue
template
- [x] I have not referenced internal or instance-local Paperclip issues
or links
- [x] My branch name describes the change and contains no internal
Paperclip ticket id
- [ ] I have run tests locally and they pass
- [x] I have added or updated tests where applicable
- [x] I have updated relevant documentation to reflect my changes
- [x] I have considered and documented risks above
- [ ] All Paperclip CI gates are green
- [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups
- [x] I will address all Greptile and reviewer comments before
requesting merge

server/src/services/native-runtime/native-execution-input.ts

@@ -1,12 +1,18 @@
 import type {
+  NativeAcpxAgent,
+  NativeAcpxPermissionMode,
   NativeCodexApprovalPolicy,
   NativeExecutionInputV4,
   NativeInteractionResponseEnvelope,
+  NativeOpenCodePermissionMode,
   NativePlanningContext,
   NativeRuntimeContextSnapshot,
   StrictCompletionContractInput,
 } from "../../vendor/paperclip-runner/index.js";
-import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
+import {
+  parseNativeExecutionInput,
+  resolveQualifiedAcpxProfile,
+} from "../../vendor/paperclip-runner/index.js";
 import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils";
 
 /** Closed constructor: callers cannot spread legacy context or environment data. */
@@ -38,7 +44,11 @@ export function buildNativeExecutionInput(input: {
     branchName: string | null;
   };
   normalizedSessionId: string | null;
+  provider?: "codex" | "opencode" | "acpx";
+  acpxAgent?: NativeAcpxAgent;
   codexApprovalPolicy?: NativeCodexApprovalPolicy;
+  opencodePermissionMode?: NativeOpenCodePermissionMode;
+  acpxPermissionMode?: NativeAcpxPermissionMode;
   model?: string | null;
   lifecyclePolicy?: NativeExecutionInputV4["session"]["lifecyclePolicy"];
   executionMode?: "default" | "plan";
@@ -57,6 +67,12 @@ export function buildNativeExecutionInput(input: {
   }
   const executionMode = input.executionMode
     ?? (input.issue.workMode === "planning" ? "plan" : "default");
+  const acpxProfile = input.provider === "acpx"
+    ? resolveQualifiedAcpxProfile(
+        input.acpxAgent ?? "pi",
+        input.model ?? "",
+      )
+    : null;
   const wakePrompt = renderPaperclipWakePrompt(input.wakePayload, {
     resumedSession: input.resumedSession === true,
     suppressIssueDescription: input.taskPrompt.trim().length > 0,
@@ -90,15 +106,44 @@ export function buildNativeExecutionInput(input: {
     },
     session: {
       normalizedSessionId: input.normalizedSessionId,
-      driverKind: "codex_app_server",
+      driverKind: input.provider === "opencode"
+        ? "opencode_server"
+        : input.provider === "acpx"
+          ? "acpx_runtime"
+          : "codex_app_server",
       protocolVersion: 1,
       lifecyclePolicy: input.lifecyclePolicy ?? { mode: "per_turn", idleTimeoutMs: null },
     },
-    provider: {
-      kind: "codex",
-      model: input.model ?? null,
-      approvalPolicy: input.codexApprovalPolicy ?? "never",
-    },
+    provider: input.provider === "acpx"
+      ? {
+          kind: "acpx",
+          agent: acpxProfile!.agent,
+          model: input.model,
+          permissionMode: input.acpxPermissionMode ?? "approve-all",
+          profile: {
+            driverKind: acpxProfile!.driverKind,
+            protocolVersion: acpxProfile!.protocolVersion,
+            acpxVersion: acpxProfile!.acpxVersion,
+            agent: acpxProfile!.agent,
+            agentProfileVersion: acpxProfile!.agentProfileVersion,
+            agentServerPackage: acpxProfile!.agentServerPackage,
+            agentServerVersion: acpxProfile!.agentServerVersion,
+            agentRuntimePackage: acpxProfile!.agentRuntimePackage,
+            agentRuntimeVersion: acpxProfile!.agentRuntimeVersion,
+            commandDigest: acpxProfile!.commandDigest,
+          },
+        }
+      : input.provider === "opencode"
+        ? {
+            kind: "opencode",
+            model: input.model,
+            permissionMode: input.opencodePermissionMode ?? "allow",
+          }
+        : {
+            kind: "codex",
+            model: input.model ?? null,
+            approvalPolicy: input.codexApprovalPolicy ?? "never",
+          },
     completionContract: input.completionContract,
     interactionResponses: input.interactionResponses ?? [],
     credentialBindings: [],

server/src/services/native-runtime/native-harness-backup-stamp.ts

@@ -0,0 +1,137 @@
+import { createHash } from "node:crypto";
+import {
+  existsSync,
+  lstatSync,
+  readFileSync,
+  readdirSync,
+  readlinkSync,
+} from "node:fs";
+import { resolve } from "node:path";
+import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
+
+export interface NativeHarnessBackupStamp {
+  schema: "paperclip.native-harness-backup-stamp.v1";
+  normalizedSessionId: string;
+  runnerInstanceId: string;
+  manifestSha256: string;
+  completedAt: string;
+}
+
+function stateRoot(normalizedSessionId: string): string {
+  return resolve(
+    process.env.PAPERCLIP_RUNNER_STATE_DIR
+      ?? resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "durable-sessions"),
+    createHash("sha256").update(normalizedSessionId).digest("hex"),
+  );
+}
+
+function digestDirectory(directory: string): { sha256: string; bytes: number } {
+  const hash = createHash("sha256");
+  let bytes = 0;
+  const visit = (current: string, relative: string) => {
+    const entries = readdirSync(current, { withFileTypes: true })
+      .sort((left, right) => left.name.localeCompare(right.name));
+    if (entries.length === 0) hash.update(`directory:${relative}\0`);
+    for (const entry of entries) {
+      const path = resolve(current, entry.name);
+      const relativePath = relative ? `${relative}/${entry.name}` : entry.name;
+      const stats = lstatSync(path);
+      if (entry.isDirectory()) {
+        hash.update(`directory:${relativePath}:${stats.mode & 0o777}\0`);
+        visit(path, relativePath);
+      } else if (entry.isSymbolicLink()) {
+        hash.update(`symlink:${relativePath}:${readlinkSync(path)}\0`);
+      } else if (entry.isFile()) {
+        const contents = readFileSync(path);
+        bytes += contents.byteLength;
+        hash.update(`file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`);
+        hash.update(contents);
+      } else {
+        throw new Error(`runner_harness_backup_unsupported_entry:${relativePath}`);
+      }
+    }
+  };
+  visit(directory, "");
+  return { sha256: `sha256:${hash.digest("hex")}`, bytes };
+}
+
+export function createNativeHarnessBackupStamp(input: {
+  manifestPath: string;
+  normalizedSessionId: string;
+  runnerInstanceId: string;
+  completedAt: string;
+}): NativeHarnessBackupStamp {
+  const manifestBytes = readFileSync(input.manifestPath);
+  return {
+    schema: "paperclip.native-harness-backup-stamp.v1",
+    normalizedSessionId: input.normalizedSessionId,
+    runnerInstanceId: input.runnerInstanceId,
+    manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`,
+    completedAt: input.completedAt,
+  };
+}
+
+export function verifyNativeHarnessBackupStamp(
+  value: unknown,
+  expectedProviderLeaseId: string,
+): boolean {
+  if (!expectedProviderLeaseId) return false;
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const stamp = value as Record<string, unknown>;
+  if (
+    stamp.schema !== "paperclip.native-harness-backup-stamp.v1" ||
+    typeof stamp.normalizedSessionId !== "string" || !stamp.normalizedSessionId ||
+    typeof stamp.runnerInstanceId !== "string" || !stamp.runnerInstanceId ||
+    typeof stamp.manifestSha256 !== "string" || !stamp.manifestSha256.startsWith("sha256:")
+  ) return false;
+  const backupRoot = resolve(stateRoot(stamp.normalizedSessionId), "failover-backups");
+  for (const candidate of [resolve(backupRoot, "current"), resolve(backupRoot, "previous")]) {
+    const manifestPath = resolve(candidate, "manifest.json");
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const bytes = readFileSync(manifestPath);
+      const manifestSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+      if (manifestSha256 !== stamp.manifestSha256) continue;
+      const manifest = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
+      if (
+        manifest.schema !== "paperclip.native-harness-backup.v1" ||
+        manifest.normalizedSessionId !== stamp.normalizedSessionId ||
+        manifest.runnerInstanceId !== stamp.runnerInstanceId ||
+        manifest.sourceProviderLeaseId !== expectedProviderLeaseId ||
+        !Array.isArray(manifest.directories) ||
+        manifest.directories.length === 0
+      ) continue;
+      let valid = true;
+      for (const entry of manifest.directories) {
+        if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+          valid = false;
+          break;
+        }
+        const declared = entry as Record<string, unknown>;
+        if (
+          typeof declared.name !== "string" ||
+          !/^[A-Za-z0-9._-]+$/.test(declared.name) ||
+          typeof declared.sha256 !== "string" ||
+          typeof declared.bytes !== "number"
+        ) {
+          valid = false;
+          break;
+        }
+        const directory = resolve(candidate, declared.name);
+        if (!existsSync(directory)) {
+          valid = false;
+          break;
+        }
+        const actual = digestDirectory(directory);
+        if (actual.sha256 !== declared.sha256 || actual.bytes !== declared.bytes) {
+          valid = false;
+          break;
+        }
+      }
+      if (valid) return true;
+    } catch {
+      // Try the previous atomically-published backup.
+    }
+  }
+  return false;
+}

server/src/services/native-runtime/native-session-resume.test.ts

@@ -20,6 +20,7 @@ function execution(runId: string, cwd = "/workspace") {
     agentId,
     workspace: { id: runId, cwd, repoUrl: null, repoRef: null, branchName: null },
     normalizedSessionId,
+    provider: "codex",
     completionContract: {
       id: "70000000-0000-4000-8000-000000000007",
       sha256: `sha256:${"a".repeat(64)}`,
@@ -43,6 +44,7 @@ function planningExecution(runId: string, revisionId: string) {
     agentId,
     workspace: { id: runId, cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
     normalizedSessionId,
+    provider: "codex",
     executionMode: "plan",
     planningContext: {
       documentId: "80000000-0000-4000-8000-000000000008",
@@ -189,7 +191,7 @@ describe("rebindNativeSessionCheckpoint", () => {
 });
 
 describe("buildNativeExecutionInput wake projection", () => {
-  it("writes native v4 and pins the complete Codex configuration", () => {
+  it("writes native v4 and pins every provider's complete effective configuration", () => {
     const common = {
       companyId,
       runId: currentRunId,
@@ -203,14 +205,36 @@ describe("buildNativeExecutionInput wake projection", () => {
     } as const;
     const codex = buildNativeExecutionInput({
       ...common,
+      provider: "codex",
       codexApprovalPolicy: "on-request",
     });
+    const opencode = buildNativeExecutionInput({
+      ...common,
+      provider: "opencode",
+      model: "openrouter/z-ai/glm-5.2",
+      opencodePermissionMode: "ask",
+    });
+    const acpx = buildNativeExecutionInput({
+      ...common,
+      provider: "acpx",
+      acpxAgent: "claude",
+      model: "claude-sonnet-5",
+      acpxPermissionMode: "deny-all",
+    });
 
     expect(codex).toMatchObject({
       schema: "paperclip.native-execution-input.v4",
       provider: { kind: "codex", approvalPolicy: "on-request" },
     });
-    expect(JSON.stringify(codex))
+    expect(opencode).toMatchObject({
+      schema: "paperclip.native-execution-input.v4",
+      provider: { kind: "opencode", permissionMode: "ask" },
+    });
+    expect(acpx).toMatchObject({
+      schema: "paperclip.native-execution-input.v4",
+      provider: { kind: "acpx", permissionMode: "deny-all" },
+    });
+    expect(JSON.stringify([codex, opencode, acpx]))
       .not.toMatch(/OPENAI_API_KEY|ANTHROPIC_API_KEY|AWS_SECRET_ACCESS_KEY|PAPERCLIP_API_KEY/);
   });
 
@@ -257,6 +281,8 @@ describe("buildNativeExecutionInput wake projection", () => {
         branchName: null,
       },
       normalizedSessionId,
+      provider: "opencode",
+      model: "openrouter/z-ai/glm-5.2",
       completionContract: {
         id: "70000000-0000-4000-8000-000000000007",
         sha256: `sha256:${"a".repeat(64)}`,

server/src/services/native-runtime/runtime-mode.test.ts

@@ -172,6 +172,21 @@ describe("resolveNativeRuntimeMode", () => {
     }));
   });
 
+  it("keeps a persisted OpenCode recovery on its immutable driver", () => {
+    expect(resolveHeartbeatNativeRuntimeMode({
+      ...eligible,
+      enabled: false,
+      persisted: {
+        runtimeMode: "native",
+        runtimeModeReason: "eligible_opt_in",
+        runtimeModeResolvedAt: new Date(),
+        driverKind: "opencode_server",
+      },
+    })).toEqual(expect.objectContaining({
+      profile: { mode: "native", backend: "opencode_server", protocolVersion: 1 },
+    }));
+  });
+
   it("rejects an explicit native profile outside the approved boundary", () => {
     expect(resolveNativeRuntimeMode({ ...eligible, agent: { ...eligible.agent, adapterType: "claude_local" } }))
       .toEqual(expect.objectContaining({ kind: "legacy", reason: "direct_adapter" }));
@@ -179,16 +194,19 @@ describe("resolveNativeRuntimeMode", () => {
       .toThrow(NativeRuntimeEligibilityError);
   });
 
-  it("rejects remote targets for fresh paperclip_runner starts", () => {
-    expect(() => resolveNativeRuntimeMode({
+  it("admits remote targets only through paperclip_runner", () => {
+    expect(resolveNativeRuntimeMode({
       ...eligible,
       target: { kind: "remote" },
       runtimeConfig: {},
       adapterConfig: { provider: "codex" },
       agent: { ...eligible.agent, adapterType: "paperclip_runner" },
-    })).toThrow(expect.objectContaining({
-      code: "paperclip_runner_environment_unsupported",
-    }));
+    })).toMatchObject({ kind: "native" });
+    expect(resolveNativeRuntimeMode({
+      ...eligible,
+      target: { kind: "remote" },
+      agent: { ...eligible.agent, adapterType: "codex_local" },
+    })).toMatchObject({ kind: "legacy", reason: "direct_adapter" });
   });
 
   it("allows paperclip_runner to use a transient local workspace for projectless issues", () => {

server/src/services/native-runtime/runtime-mode.ts

@@ -40,7 +40,7 @@ export type NativeRuntimeResolution =
       reason: "eligible_opt_in";
       profile: {
         mode: "native";
-        backend: "codex_app_server";
+        backend: "codex_app_server" | "opencode_server" | "acpx_runtime";
         protocolVersion: 1;
       };
       authorityDecision: NativeStatusDecision;
@@ -129,12 +129,6 @@ export function resolveNativeRuntimeMode(input: {
       "Paperclip Runner requires a standard, planning, or ask task.",
     );
   }
-  if (!input.target || input.target.kind !== "local") {
-    throw ineligible(
-      "paperclip_runner_environment_unsupported",
-      "Paperclip Runner currently requires a local execution environment.",
-    );
-  }
   const rollout = resolveNativeMigrationStatus({
     facts: { applicationEnabled: true },
     priorIssueStatus: "in_progress",
@@ -267,13 +261,17 @@ export function resolveHeartbeatNativeRuntimeMode(input: {
         );
       }
       const driverKind = input.persisted.driverKind;
-      const backend = driverKind === null
-        || driverKind === undefined
-        || driverKind === "codex"
-        || driverKind === "codex_app_server"
-        ? "codex_app_server"
-        : null;
-      if (!backend) {
+      const backend = driverKind === "opencode_server"
+        ? "opencode_server"
+        : driverKind === "acpx_runtime"
+          ? "acpx_runtime"
+          : driverKind === null
+              || driverKind === undefined
+              || driverKind === "codex"
+              || driverKind === "codex_app_server"
+            ? "codex_app_server"
+            : null;
+      if (backend === null) {
         throw ineligible(
           "paperclip_runner_driver_unsupported",
           `Persisted Paperclip Runner driver is unsupported: ${driverKind}`,