mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
feat(runner): add remote execution substrate (#12638)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - Paperclip Runner gives native runs a durable and governed execution path. > - The current native path runs on the control-plane host. > - Remote environments need an authenticated execution-target contract. > - The contract must not change direct adapters or enable new runtimes by default. > - This pull request adds the remote execution substrate and Daytona ingress. > - The benefit is a bounded base for later remote runner transport work. ## Linked Issues or Issue Description Refs #12616. Refs #12352. **Subsystem affected** Cross-cutting. This change touches runner transport, server orchestration, plugin contracts, and shared settings. **Problem or motivation** Native execution cannot resolve an authenticated runner ingress through a remote environment. The server also lacks one provider-neutral contract for remote execution targets. **Proposed solution** Add a default-off runner preview ingress capability. Add transport-neutral runner connectivity. Add remote execution target and lifecycle handling. Add a Daytona ingress implementation with redacted credentials. **Alternatives considered** A provider-specific server path would duplicate orchestration and authorization. A public endpoint without an environment contract would weaken the trust boundary. **Roadmap alignment** This work supports the Cloud and Sandbox agents milestone. It also supports self-healing runs and governed tool access. ## What Changed - Added execution-target traits for local, SSH, and sandbox environments. - Added plugin RPC contracts for runner ingress endpoints. - Added authenticated Daytona preview ingress. - Added transport-neutral PRP outbound connections. - Added remote runner artifact verification and fail-closed provider selection. - Added bounded native session resume, cancellation, and lifecycle recovery. - Preserved Codex-only selection for fresh experimental runner starts. - Preserved all direct adapter execution and finalization paths. - Removed stale Pi provider-pack requirements that security review rejected. - Kept the rollout controls off by default. - Did not change pnpm-lock.yaml, Cargo, database migrations, or GitHub workflows. ## Verification - GitHub Actions will run the repository test, typecheck, build, security, and policy gates. - Focused tests cover ingress validation, redaction, execution targets, remote lifecycle, cancellation, resume, and legacy adapter selection. - Local tests were not run. The requested verification policy uses GitHub Actions for this series. - `git diff --check origin/master...HEAD` passes. - The diff contains 52 files. ## Risks - Remote execution crosses a trust boundary. - The implementation validates target capabilities, artifact digests, provider-pack pins, and connection metadata. - The feature remains default-off. - Fresh native selection remains Codex-only. - Existing direct adapters remain on the legacy path. - This PR does not yet make remote Codex runnable. The next PR adds the Rust WSS and TLS transport. ## Model Used OpenAI Codex with GPT-5.6. The work used high-reasoning agent mode, repository tools, GitHub tools, and parallel code-audit agents. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with Fixes: / Closes / Refs OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal Paperclip ticket id - [ ] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
Dotta authored and GitHub committed
1 parent
209680409d
commit
0a422fda52
52 files changed
+6920
-223
No files matched your search
@@ -77,6 +77,13 @@ const fakeSandboxEnvironmentConfigSchema = z.object({
|
||||
.default("ubuntu:24.04"),
|
||||
reuseLease: z.boolean().optional().default(false),
|
||||
streamRunLogs: z.boolean().optional(),
|
||||
runnerLifecycleMode: z.enum(["inherit", "per_turn", "warm"]).optional(),
|
||||
runnerIdleTimeoutMs: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.min(1_000)
|
||||
.max(86_400_000)
|
||||
.optional(),
|
||||
archiveOnRelease: z.boolean().optional(),
|
||||
}).strict();
|
||||
|
||||
@@ -93,6 +100,13 @@ const pluginSandboxEnvironmentConfigSchema = z.object({
|
||||
timeoutMs: z.coerce.number().int().min(1).max(86_400_000).optional(),
|
||||
reuseLease: z.boolean().optional().default(false),
|
||||
streamRunLogs: z.boolean().optional(),
|
||||
runnerLifecycleMode: z.enum(["inherit", "per_turn", "warm"]).optional(),
|
||||
runnerIdleTimeoutMs: z.coerce
|
||||
.number()
|
||||
.int()
|
||||
.min(1_000)
|
||||
.max(86_400_000)
|
||||
.optional(),
|
||||
archiveOnRelease: z.boolean().optional(),
|
||||
}).catchall(z.unknown());
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ export interface EnvironmentDriverCapabilitySupport {
|
||||
|
||||
const NO_CAPABILITY_SUPPORT: ReadonlySet<SandboxCapabilityKey> = new Set<SandboxCapabilityKey>();
|
||||
|
||||
// The eight capability keys, written out here as literal strings. This module
|
||||
// The nine capability keys, written out here as literal strings. This module
|
||||
// does not import `environment-runtime.ts` as a value (see the module comment
|
||||
// above), so it cannot read `SANDBOX_CAPABILITY_KEYS` from there. Keep this
|
||||
// list equal to that list.
|
||||
@@ -50,6 +50,7 @@ const ALL_CAPABILITY_SUPPORT: ReadonlySet<SandboxCapabilityKey> = new Set<Sandbo
|
||||
"incrementalSessionOutput",
|
||||
"concurrentSyncOperations",
|
||||
"duplexCommandStream",
|
||||
"runnerWebSocketIngress",
|
||||
]);
|
||||
|
||||
/**
|
||||
|
||||
@@ -4,6 +4,7 @@ import { adapterSupportsRemoteManagedEnvironments } from "@paperclipai/shared";
|
||||
import {
|
||||
adapterExecutionTargetToRemoteSpec,
|
||||
type AdapterExecutionTarget,
|
||||
type SandboxLeaseAcquisition,
|
||||
} from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability";
|
||||
import {
|
||||
@@ -71,6 +72,40 @@ function toBoolean(value: unknown): boolean | undefined {
|
||||
return typeof value === "boolean" ? value : undefined;
|
||||
}
|
||||
|
||||
function sandboxLeaseAcquisitionFromMetadata(
|
||||
value: unknown,
|
||||
providerLeaseId: string | null | undefined,
|
||||
): SandboxLeaseAcquisition | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||||
const candidate = value as Record<string, unknown>;
|
||||
if (
|
||||
candidate.outcome !== "created" &&
|
||||
candidate.outcome !== "resumed" &&
|
||||
candidate.outcome !== "replacement"
|
||||
) return null;
|
||||
const resolvedProviderLeaseId =
|
||||
typeof candidate.providerLeaseId === "string" && candidate.providerLeaseId
|
||||
? candidate.providerLeaseId
|
||||
: providerLeaseId;
|
||||
if (!resolvedProviderLeaseId) return null;
|
||||
const reason = candidate.reason;
|
||||
if (
|
||||
reason !== undefined &&
|
||||
reason !== "not_found" &&
|
||||
reason !== "expired" &&
|
||||
reason !== "identity_mismatch" &&
|
||||
reason !== "resume_failed"
|
||||
) return null;
|
||||
return {
|
||||
outcome: candidate.outcome,
|
||||
providerLeaseId: resolvedProviderLeaseId,
|
||||
...(typeof candidate.previousProviderLeaseId === "string"
|
||||
? { previousProviderLeaseId: candidate.previousProviderLeaseId }
|
||||
: {}),
|
||||
...(reason ? { reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the tail of `final` that the provider did NOT already stream.
|
||||
*
|
||||
@@ -331,11 +366,39 @@ export async function resolveEnvironmentExecutionTarget(input: {
|
||||
shellCommand,
|
||||
remoteCwd,
|
||||
enableSandboxDuplexBridge,
|
||||
runnerLifecyclePolicy:
|
||||
parsed.config.runnerLifecycleMode === "warm"
|
||||
? {
|
||||
mode: "warm",
|
||||
idleTimeoutMs:
|
||||
typeof parsed.config.runnerIdleTimeoutMs === "number"
|
||||
? parsed.config.runnerIdleTimeoutMs
|
||||
: 300_000,
|
||||
}
|
||||
: parsed.config.runnerLifecycleMode === "per_turn"
|
||||
? { mode: "per_turn", idleTimeoutMs: null }
|
||||
: null,
|
||||
reusableLeaseConfigured: parsed.config.reuseLease === true,
|
||||
sandboxLeaseAcquisition: sandboxLeaseAcquisitionFromMetadata(
|
||||
input.lease?.metadata?.sandboxLeaseAcquisition,
|
||||
input.lease?.providerLeaseId,
|
||||
),
|
||||
// Attach the host duplex observability recorder next to the runner. The bridge
|
||||
// binds it to the fixed observability surface. Absent keeps the no-op
|
||||
// default, so the surface stays inert on a run with no injected recorder.
|
||||
duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null,
|
||||
...(effectiveCapabilities ? { effectiveCapabilities: Object.freeze({ ...effectiveCapabilities }) } : {}),
|
||||
...(input.environmentRuntime?.getRunnerIngressEndpoint && input.lease
|
||||
? {
|
||||
getRunnerIngressEndpoint: ({ port, path }) =>
|
||||
input.environmentRuntime!.getRunnerIngressEndpoint({
|
||||
environment: input.environment as Environment,
|
||||
lease: input.lease!,
|
||||
port,
|
||||
path,
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
environmentId: input.environment.id ?? null,
|
||||
leaseId: input.leaseId ?? null,
|
||||
timeoutMs,
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
buildEnvironmentLeaseContext,
|
||||
type EnvironmentRuntimeLeaseRecord,
|
||||
type EnvironmentRuntimeService,
|
||||
type ProviderResourceDisposition,
|
||||
} from "./environment-runtime.js";
|
||||
import { ENVIRONMENT_DRIVER_TRAITS } from "./environment-driver-traits.js";
|
||||
import {
|
||||
@@ -583,6 +584,17 @@ export function environmentRunOrchestrator(
|
||||
agentId: string;
|
||||
status?: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed">;
|
||||
failureReason?: string;
|
||||
/** Explicit paperclip_runner resource lifecycle. Omitted for legacy adapters. */
|
||||
providerResourceDisposition?: ProviderResourceDisposition;
|
||||
nativeLifecycleTelemetry?: {
|
||||
provider: string;
|
||||
harness: string;
|
||||
lifecycleMode: "per_turn" | "warm";
|
||||
sandboxResource:
|
||||
| "keep_running"
|
||||
| "stop_and_reuse"
|
||||
| "destroy_after_turn";
|
||||
};
|
||||
}): Promise<EnvironmentReleaseResult> {
|
||||
const status = input.status ?? "released";
|
||||
const result: EnvironmentReleaseResult = { released: [], errors: [] };
|
||||
@@ -593,6 +605,7 @@ export function environmentRunOrchestrator(
|
||||
input.heartbeatRunId,
|
||||
status,
|
||||
(leaseId, error) => result.errors.push({ leaseId, error }),
|
||||
input.providerResourceDisposition,
|
||||
);
|
||||
} catch (err) {
|
||||
result.errors.push({ leaseId: "*", error: err });
|
||||
@@ -621,6 +634,8 @@ export function environmentRunOrchestrator(
|
||||
status: released.lease.status,
|
||||
cleanupStatus: released.lease.cleanupStatus,
|
||||
failureReason: input.failureReason ?? released.lease.failureReason,
|
||||
providerResourceDisposition:
|
||||
input.providerResourceDisposition ?? "legacy_default",
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
|
||||
@@ -15,6 +15,7 @@ import type {
|
||||
} from "@paperclipai/shared";
|
||||
import { resolveDeclaredSandboxCapabilities } from "@paperclipai/shared";
|
||||
import type { EffectiveExecutionCapabilities } from "@paperclipai/adapter-utils/execution-target";
|
||||
import type { RunnerIngressEndpoint } from "@paperclipai/adapter-utils/runner-connectivity";
|
||||
import type {
|
||||
CommandManagedDuplexChannel,
|
||||
} from "@paperclipai/adapter-utils/command-managed-runtime";
|
||||
@@ -34,6 +35,7 @@ import {
|
||||
} from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
|
||||
import { environmentService } from "./environments.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { verifyNativeHarnessBackupStamp } from "./native-runtime/native-harness-backup-stamp.js";
|
||||
import {
|
||||
collectEnvironmentSecretRefs,
|
||||
parseEnvironmentDriverConfig,
|
||||
@@ -107,6 +109,7 @@ export const SANDBOX_CAPABILITY_KEYS = [
|
||||
"incrementalSessionOutput",
|
||||
"concurrentSyncOperations",
|
||||
"duplexCommandStream",
|
||||
"runnerWebSocketIngress",
|
||||
] as const;
|
||||
|
||||
export type SandboxCapabilityKey = (typeof SANDBOX_CAPABILITY_KEYS)[number];
|
||||
@@ -124,6 +127,7 @@ const SANDBOX_CAPABILITY_OPT_IN_KEYS: ReadonlySet<SandboxCapabilityKey> = new Se
|
||||
"incrementalSessionOutput",
|
||||
"concurrentSyncOperations",
|
||||
"duplexCommandStream",
|
||||
"runnerWebSocketIngress",
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -180,6 +184,7 @@ const SANDBOX_CAPABILITY_PREREQUISITE_METHODS: Record<SandboxCapabilityKey, read
|
||||
incrementalSessionOutput: [["environmentExecute"]],
|
||||
concurrentSyncOperations: [["environmentSyncIn"], ["environmentSyncOut"]],
|
||||
duplexCommandStream: [["duplexChannelOpen"]],
|
||||
runnerWebSocketIngress: [["environmentRunnerIngressEndpoint"]],
|
||||
};
|
||||
|
||||
function capabilityIsVerified(
|
||||
@@ -290,6 +295,7 @@ export function classifyEnvironmentCapabilities(input: {
|
||||
incrementalSessionOutput: resolve("incrementalSessionOutput"),
|
||||
concurrentSyncOperations: resolve("concurrentSyncOperations"),
|
||||
duplexCommandStream: resolve("duplexCommandStream"),
|
||||
runnerWebSocketIngress: resolve("runnerWebSocketIngress"),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -556,6 +562,12 @@ export interface EnvironmentDriverOpenDuplexChannelInput extends EnvironmentDriv
|
||||
command: readonly string[];
|
||||
}
|
||||
|
||||
export interface EnvironmentDriverRunnerIngressInput
|
||||
extends EnvironmentDriverLeaseInput {
|
||||
port: number;
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface EnvironmentRuntimeDriver {
|
||||
readonly driver: string;
|
||||
acquireRunLease(input: EnvironmentDriverAcquireInput): Promise<EnvironmentLease>;
|
||||
@@ -583,6 +595,9 @@ export interface EnvironmentRuntimeDriver {
|
||||
openDuplexChannel?(
|
||||
input: EnvironmentDriverOpenDuplexChannelInput,
|
||||
): Promise<CommandManagedDuplexChannel>;
|
||||
getRunnerIngressEndpoint?(
|
||||
input: EnvironmentDriverRunnerIngressInput,
|
||||
): Promise<RunnerIngressEndpoint>;
|
||||
/** True when the lease's plugin worker advertises both sync verbs. */
|
||||
supportsSync?(input: EnvironmentDriverLeaseInput): boolean;
|
||||
/**
|
||||
@@ -678,12 +693,57 @@ export class SandboxOrphanCleanupWriteError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** A reusable sandbox could not be resumed, but has not been proven lost. */
|
||||
export class ReusableSandboxResumeError extends Error {
|
||||
readonly provider: string;
|
||||
readonly providerLeaseId: string;
|
||||
|
||||
constructor(input: {
|
||||
provider: string;
|
||||
providerLeaseId: string;
|
||||
cause?: unknown;
|
||||
}) {
|
||||
super(
|
||||
`Reusable sandbox lease "${input.providerLeaseId}" could not be resumed; ` +
|
||||
"the lease was preserved and no replacement was created.",
|
||||
input.cause === undefined ? undefined : { cause: input.cause },
|
||||
);
|
||||
this.name = "ReusableSandboxResumeError";
|
||||
this.provider = input.provider;
|
||||
this.providerLeaseId = input.providerLeaseId;
|
||||
}
|
||||
}
|
||||
|
||||
export class RunnerHarnessBackupUnavailableError extends Error {
|
||||
readonly providerLeaseId: string;
|
||||
|
||||
constructor(providerLeaseId: string) {
|
||||
super(
|
||||
`runner_harness_backup_unavailable: reusable sandbox "${providerLeaseId}" ` +
|
||||
"was confirmed lost, but no complete verified failover backup is available",
|
||||
);
|
||||
this.name = "RunnerHarnessBackupUnavailableError";
|
||||
this.providerLeaseId = providerLeaseId;
|
||||
}
|
||||
}
|
||||
|
||||
export interface EnvironmentRuntimeLeaseRecord {
|
||||
environment: Environment;
|
||||
lease: EnvironmentLease;
|
||||
leaseContext: ReturnType<typeof buildEnvironmentLeaseContext>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Host-side decision for the provider resource after a run. This is kept
|
||||
* separate from heartbeat status: a failed turn can still leave a reusable
|
||||
* sandbox resumable, while a disposable successful turn must destroy it.
|
||||
* An omitted disposition preserves the legacy adapter behavior.
|
||||
*/
|
||||
export type ProviderResourceDisposition =
|
||||
| "keep_running"
|
||||
| "stop_and_retain"
|
||||
| "destroy";
|
||||
|
||||
const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS = 5_000;
|
||||
const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS = 100;
|
||||
|
||||
@@ -704,6 +764,24 @@ function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function transientSandboxResumeFailure(error: unknown): boolean {
|
||||
const candidate = error && typeof error === "object"
|
||||
? error as Record<string, unknown>
|
||||
: {};
|
||||
const status = typeof candidate.status === "number"
|
||||
? candidate.status
|
||||
: typeof candidate.statusCode === "number"
|
||||
? candidate.statusCode
|
||||
: null;
|
||||
if (status === 429 || (status !== null && status >= 500)) return true;
|
||||
const code = typeof candidate.code === "string" ? candidate.code.toUpperCase() : "";
|
||||
if (["ETIMEDOUT", "ECONNRESET", "ECONNREFUSED", "EAI_AGAIN", "ENETUNREACH"].includes(code)) {
|
||||
return true;
|
||||
}
|
||||
const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
|
||||
return /\b(timeout|timed out|rate limit|temporar|network|connection reset|service unavailable)\b/.test(message);
|
||||
}
|
||||
|
||||
function getLeaseDriverKey(
|
||||
lease: Pick<EnvironmentLease, "metadata">,
|
||||
environment: Pick<Environment, "driver"> | null,
|
||||
@@ -1794,23 +1872,40 @@ function createSandboxEnvironmentDriver(
|
||||
: null;
|
||||
|
||||
let providerLease: PluginEnvironmentLease | null = null;
|
||||
let replacementReason:
|
||||
| "not_found"
|
||||
| "expired"
|
||||
| "identity_mismatch"
|
||||
| "resume_failed"
|
||||
| undefined;
|
||||
if (reusableLease?.providerLeaseId) {
|
||||
// The `supportsReusableLeases` check above reads a snapshot of the
|
||||
// worker methods. The runtime then does asynchronous database work
|
||||
// (list, fingerprint, obsolete-lease cleanup) before this dispatch. A
|
||||
// worker restart in that window can drop `environmentResumeLease`
|
||||
// while the snapshot still marks the method verified. Re-check the
|
||||
// live worker here and fail closed when the method is absent: skip the
|
||||
// resume, destroy the stale reusable lease, and acquire a fresh lease
|
||||
// below. The runtime never dispatches a resume the live worker cannot
|
||||
// serve.
|
||||
// live worker here and fail closed when the method is absent. The
|
||||
// runtime preserves the recorded lease and never dispatches a resume
|
||||
// the live worker cannot serve.
|
||||
const workerVerifiesResume = pluginWorkerVerifiesLifecycleMethod(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentResumeLease",
|
||||
);
|
||||
if (workerVerifiesResume) {
|
||||
try {
|
||||
const resumed = await pluginWorkerManager.call(
|
||||
if (!workerVerifiesResume) {
|
||||
throw new ReusableSandboxResumeError({
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: reusableLease.providerLeaseId,
|
||||
});
|
||||
}
|
||||
try {
|
||||
const resumeDeadline = Date.now() + 60_000;
|
||||
const configuredResumeTimeoutMs =
|
||||
resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000;
|
||||
let retryDelayMs = 250;
|
||||
let resumed: PluginEnvironmentLease;
|
||||
while (true) {
|
||||
try {
|
||||
resumed = await pluginWorkerManager.call(
|
||||
pluginProvider.resolved.plugin.id,
|
||||
"environmentResumeLease",
|
||||
{
|
||||
@@ -1822,21 +1917,65 @@ function createSandboxEnvironmentDriver(
|
||||
providerLeaseId: reusableLease.providerLeaseId,
|
||||
leaseMetadata: reusableLease.metadata ?? undefined,
|
||||
},
|
||||
resolvePluginSandboxRpcTimeoutMs(workerConfig),
|
||||
Math.min(
|
||||
configuredResumeTimeoutMs,
|
||||
Math.max(1, resumeDeadline - Date.now()),
|
||||
),
|
||||
);
|
||||
providerLease =
|
||||
typeof resumed.providerLeaseId === "string" && resumed.providerLeaseId.length > 0
|
||||
? resumed
|
||||
: null;
|
||||
} catch {
|
||||
providerLease = null;
|
||||
break;
|
||||
} catch (error) {
|
||||
if (
|
||||
!transientSandboxResumeFailure(error) ||
|
||||
Date.now() + retryDelayMs * 1.25 >= resumeDeadline
|
||||
) throw error;
|
||||
const jitteredDelayMs = Math.max(
|
||||
1,
|
||||
Math.round(retryDelayMs * (0.75 + Math.random() * 0.5)),
|
||||
);
|
||||
await delay(jitteredDelayMs);
|
||||
retryDelayMs = Math.min(retryDelayMs * 2, 5_000);
|
||||
}
|
||||
}
|
||||
providerLease =
|
||||
typeof resumed.providerLeaseId === "string" && resumed.providerLeaseId.length > 0
|
||||
? resumed
|
||||
: null;
|
||||
if (!providerLease) {
|
||||
const sentinel = isRecord(resumed.metadata?.workspaceSentinel)
|
||||
? resumed.metadata.workspaceSentinel
|
||||
: null;
|
||||
replacementReason = sentinel && sentinel.result !== "matched"
|
||||
? "identity_mismatch"
|
||||
: resumed.metadata?.expired === true
|
||||
? "not_found"
|
||||
: "expired";
|
||||
}
|
||||
} catch (error) {
|
||||
throw new ReusableSandboxResumeError({
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: reusableLease.providerLeaseId,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
if (!providerLease) {
|
||||
if (
|
||||
input.adapterType === "paperclip_runner" &&
|
||||
!verifyNativeHarnessBackupStamp(
|
||||
reusableLease.metadata?.nativeHarnessBackup,
|
||||
reusableLease.providerLeaseId,
|
||||
)
|
||||
) {
|
||||
throw new RunnerHarnessBackupUnavailableError(
|
||||
reusableLease.providerLeaseId,
|
||||
);
|
||||
}
|
||||
// The verified, lease-bound backup authorizes destructive
|
||||
// replacement. Keep the existing sandbox intact when validation
|
||||
// fails so the only recoverable provider state is not lost.
|
||||
await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: reusableLease,
|
||||
failureReason: workerVerifiesResume ? "resume_failed" : "resume_capability_lost",
|
||||
failureReason: replacementReason ?? "resume_failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1906,6 +2045,21 @@ function createSandboxEnvironmentDriver(
|
||||
sandboxProviderPlugin: true,
|
||||
...sandboxConfigForLeaseMetadata(storedConfig),
|
||||
...sanitizedProviderMetadata,
|
||||
sandboxLeaseAcquisition: providerLease
|
||||
? {
|
||||
outcome: "resumed",
|
||||
}
|
||||
: reusableLease?.providerLeaseId
|
||||
? {
|
||||
outcome: "replacement",
|
||||
reason: replacementReason ?? "resume_failed",
|
||||
}
|
||||
: {
|
||||
outcome: "created",
|
||||
},
|
||||
...(reusableLease?.metadata?.nativeHarnessBackup
|
||||
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
|
||||
: {}),
|
||||
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
|
||||
};
|
||||
try {
|
||||
@@ -2072,10 +2226,10 @@ function createSandboxEnvironmentDriver(
|
||||
});
|
||||
} catch (error) {
|
||||
if (reusableLease) {
|
||||
await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
lease: reusableLease,
|
||||
failureReason: "resume_failed",
|
||||
throw new ReusableSandboxResumeError({
|
||||
provider: parsed.config.provider,
|
||||
providerLeaseId: reusableLease.providerLeaseId!,
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
@@ -2112,6 +2266,22 @@ function createSandboxEnvironmentDriver(
|
||||
driver: input.environment.driver,
|
||||
executionWorkspaceMode: input.executionWorkspaceMode,
|
||||
...providerLease.metadata,
|
||||
sandboxLeaseAcquisition:
|
||||
reusableLease && providerLease.providerLeaseId === reusableLease.providerLeaseId
|
||||
? {
|
||||
outcome: "resumed",
|
||||
}
|
||||
: reusableLease?.providerLeaseId
|
||||
? {
|
||||
outcome: "replacement",
|
||||
reason: "resume_failed",
|
||||
}
|
||||
: {
|
||||
outcome: "created",
|
||||
},
|
||||
...(reusableLease?.metadata?.nativeHarnessBackup
|
||||
? { nativeHarnessBackup: reusableLease.metadata.nativeHarnessBackup }
|
||||
: {}),
|
||||
...(reusableScope ? { reusableSandboxLease: reusableScope } : {}),
|
||||
};
|
||||
try {
|
||||
@@ -2573,6 +2743,91 @@ function createSandboxEnvironmentDriver(
|
||||
return await resolveSandboxCapabilitiesForLease(input);
|
||||
},
|
||||
|
||||
async getRunnerIngressEndpoint(input) {
|
||||
if (!input.lease.metadata?.sandboxProviderPlugin || !pluginWorkerManager) {
|
||||
throw new Error("Sandbox driver does not support runner ingress for this lease.");
|
||||
}
|
||||
const pluginId = readString(input.lease.metadata.pluginId);
|
||||
const providerKey = readString(input.lease.metadata.provider);
|
||||
if (!pluginId || !providerKey || !input.lease.providerLeaseId) {
|
||||
throw new Error("Sandbox runner ingress is missing its provider identity.");
|
||||
}
|
||||
const config = await resolvePluginSandboxRuntimeConfig({
|
||||
environment: input.environment,
|
||||
lease: input.lease,
|
||||
provider: providerKey,
|
||||
});
|
||||
const sanitizedConfig = stripSandboxProviderEnvelope(
|
||||
config as SandboxEnvironmentConfig,
|
||||
);
|
||||
const acquire = async (): Promise<RunnerIngressEndpoint> => {
|
||||
const result = await pluginWorkerManager.call(
|
||||
pluginId,
|
||||
"environmentRunnerIngressEndpoint",
|
||||
{
|
||||
driverKey: providerKey,
|
||||
companyId: input.lease.companyId,
|
||||
environmentId: input.environment.id,
|
||||
issueId: input.lease.issueId,
|
||||
config: sanitizedConfig,
|
||||
lease: {
|
||||
providerLeaseId: input.lease.providerLeaseId,
|
||||
metadata: input.lease.metadata ?? undefined,
|
||||
expiresAt: input.lease.expiresAt?.toISOString() ?? null,
|
||||
},
|
||||
port: input.port,
|
||||
path: input.path,
|
||||
},
|
||||
resolvePluginSandboxRpcTimeoutMs(sanitizedConfig),
|
||||
);
|
||||
const endpointUrl = new URL(result.websocketUrl);
|
||||
if (
|
||||
result.kind !== "authenticated_websocket" ||
|
||||
endpointUrl.protocol !== "wss:" ||
|
||||
endpointUrl.username ||
|
||||
endpointUrl.password ||
|
||||
endpointUrl.search ||
|
||||
endpointUrl.hash ||
|
||||
endpointUrl.pathname !== input.path ||
|
||||
!result.generation
|
||||
) {
|
||||
throw new Error("Sandbox provider returned an invalid runner ingress endpoint.");
|
||||
}
|
||||
const secretHeaders = result.secretHeaders.map((header) => {
|
||||
if (!/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(header.name) || !header.value) {
|
||||
throw new Error("Sandbox provider returned an invalid runner ingress secret header.");
|
||||
}
|
||||
const secretHeader = { name: header.name } as {
|
||||
name: string;
|
||||
readonly value: string;
|
||||
toJSON(): { name: string; value: "[REDACTED]" };
|
||||
};
|
||||
Object.defineProperty(secretHeader, "value", {
|
||||
enumerable: false,
|
||||
configurable: false,
|
||||
writable: false,
|
||||
value: header.value,
|
||||
});
|
||||
Object.defineProperty(secretHeader, "toJSON", {
|
||||
enumerable: false,
|
||||
configurable: false,
|
||||
writable: false,
|
||||
value: () => ({ name: header.name, value: "[REDACTED]" as const }),
|
||||
});
|
||||
return Object.freeze(secretHeader);
|
||||
});
|
||||
return {
|
||||
kind: "authenticated_websocket",
|
||||
websocketUrl: endpointUrl.toString(),
|
||||
secretHeaders: Object.freeze(secretHeaders),
|
||||
generation: result.generation,
|
||||
refresh: acquire,
|
||||
close: async () => undefined,
|
||||
};
|
||||
};
|
||||
return await acquire();
|
||||
},
|
||||
|
||||
async destroyRunLease(input) {
|
||||
return await destroyReusableSandboxLease({
|
||||
environment: input.environment,
|
||||
@@ -2897,6 +3152,8 @@ const INTERNAL_PLUGIN_SANDBOX_CONFIG_KEYS = new Set([
|
||||
"remoteCwd",
|
||||
"shellCommand",
|
||||
"sandboxProviderPlugin",
|
||||
"sandboxLeaseAcquisition",
|
||||
"nativeHarnessBackup",
|
||||
]);
|
||||
|
||||
// Drop the host-internal and per-lease runtime keys from a sandbox config
|
||||
@@ -3386,6 +3643,7 @@ export function environmentRuntimeService(
|
||||
heartbeatRunId: string,
|
||||
status: Extract<EnvironmentLeaseStatus, "released" | "expired" | "failed"> = "released",
|
||||
onLeaseReleaseError?: (leaseId: string, error: unknown) => void,
|
||||
providerResourceDisposition?: ProviderResourceDisposition,
|
||||
): Promise<EnvironmentRuntimeLeaseRecord[]> {
|
||||
const leaseRows = await db
|
||||
.select()
|
||||
@@ -3413,14 +3671,72 @@ export function environmentRuntimeService(
|
||||
if (!environment) continue;
|
||||
|
||||
const leaseSnapshot = toEnvironmentLeaseSnapshot(leaseRow);
|
||||
if (
|
||||
providerResourceDisposition === "keep_running" &&
|
||||
leaseSnapshot.leasePolicy === "reuse_by_environment"
|
||||
) {
|
||||
const lease = await environmentsSvc.releaseLease(
|
||||
leaseRow.id,
|
||||
"retained",
|
||||
{ cleanupStatus: "success" },
|
||||
);
|
||||
if (lease) {
|
||||
released.push({
|
||||
environment,
|
||||
lease,
|
||||
leaseContext: {
|
||||
executionWorkspaceId: lease.executionWorkspaceId,
|
||||
executionWorkspaceMode:
|
||||
(lease.metadata?.executionWorkspaceMode as ExecutionWorkspace["mode"] | null | undefined) ?? null,
|
||||
},
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const driver = getDriver(getLeaseDriverKey(leaseSnapshot, environment));
|
||||
const lease = driver
|
||||
? await driver.releaseRunLease({
|
||||
if (
|
||||
providerResourceDisposition === "keep_running" &&
|
||||
leaseSnapshot.leasePolicy !== "reuse_by_environment"
|
||||
) {
|
||||
throw new Error(
|
||||
`Cannot keep non-reusable environment lease "${leaseSnapshot.id}" running.`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
providerResourceDisposition === "destroy" &&
|
||||
leaseSnapshot.metadata?.sandboxLeaseAcquisition &&
|
||||
(!leaseSnapshot.providerLeaseId ||
|
||||
!verifyNativeHarnessBackupStamp(
|
||||
leaseSnapshot.metadata.nativeHarnessBackup,
|
||||
leaseSnapshot.providerLeaseId,
|
||||
))
|
||||
) {
|
||||
throw new RunnerHarnessBackupUnavailableError(
|
||||
leaseSnapshot.providerLeaseId ?? leaseSnapshot.id,
|
||||
);
|
||||
}
|
||||
const lease = providerResourceDisposition === "destroy" && driver?.destroyRunLease
|
||||
? await driver.destroyRunLease({
|
||||
environment,
|
||||
lease: leaseSnapshot,
|
||||
status,
|
||||
failureReason: "paperclip_runner_destroy_after_turn",
|
||||
})
|
||||
: await environmentsSvc.releaseLease(leaseRow.id, status);
|
||||
: driver
|
||||
? await driver.releaseRunLease({
|
||||
environment,
|
||||
lease: leaseSnapshot,
|
||||
// A stopped reusable provider resource must remain eligible
|
||||
// for exact-lease resume independently of turn outcome.
|
||||
status:
|
||||
providerResourceDisposition === "stop_and_retain" &&
|
||||
leaseSnapshot.leasePolicy === "reuse_by_environment"
|
||||
? "released"
|
||||
: status,
|
||||
})
|
||||
: await environmentsSvc.releaseLease(
|
||||
leaseRow.id,
|
||||
providerResourceDisposition === "destroy" ? "expired" : status,
|
||||
);
|
||||
if (!lease) continue;
|
||||
|
||||
released.push({
|
||||
@@ -3763,6 +4079,26 @@ export function environmentRuntimeService(
|
||||
}
|
||||
return await driver.openDuplexChannel(input);
|
||||
},
|
||||
|
||||
async getRunnerIngressEndpoint(
|
||||
input: EnvironmentDriverRunnerIngressInput,
|
||||
): Promise<RunnerIngressEndpoint> {
|
||||
const driver = requireDriverKey(
|
||||
getLeaseDriverKey(input.lease, input.environment),
|
||||
);
|
||||
const effective = await driver.resolveCapabilities(input);
|
||||
if (effective.runnerWebSocketIngress !== true) {
|
||||
throw new Error(
|
||||
"Sandbox lease does not grant runner WebSocket ingress.",
|
||||
);
|
||||
}
|
||||
if (!driver.getRunnerIngressEndpoint) {
|
||||
throw new Error(
|
||||
`Environment driver "${driver.driver}" does not support runner ingress.`,
|
||||
);
|
||||
}
|
||||
return await driver.getRunnerIngressEndpoint(input);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -392,7 +392,10 @@ import { extractSkillMentionIds, isUuidLike } from "@paperclipai/shared";
|
||||
import { evaluateCodexCredentialReadiness } from "@paperclipai/adapter-codex-local/server";
|
||||
import { environmentService } from "./environments.js";
|
||||
import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js";
|
||||
import { environmentRuntimeService } from "./environment-runtime.js";
|
||||
import {
|
||||
environmentRuntimeService,
|
||||
type ProviderResourceDisposition,
|
||||
} from "./environment-runtime.js";
|
||||
import { skillVersionSelectionMap } from "./runtime-skill-selections.js";
|
||||
import { environmentRunOrchestrator } from "./environment-run-orchestrator.js";
|
||||
import { isUnsafeSessionWorkspaceCwd } from "./session-workspace-cwd.js";
|
||||
@@ -1674,6 +1677,48 @@ export function leaseReleaseStatusForRunStatus(
|
||||
return status === "failed" || status === "timed_out" ? "failed" : "released";
|
||||
}
|
||||
|
||||
export interface NativeSandboxLifecycle {
|
||||
runnerProcess: "per_turn" | "warm";
|
||||
sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
|
||||
failoverBackup: "verified";
|
||||
}
|
||||
|
||||
export function resolveNativeSandboxLifecycle(input: {
|
||||
adapterType: string;
|
||||
lifecyclePolicy:
|
||||
| { mode: "per_turn"; idleTimeoutMs: null }
|
||||
| { mode: "warm"; idleTimeoutMs: number };
|
||||
target: {
|
||||
kind: "local" | "remote";
|
||||
transport?: string;
|
||||
reusableLeaseConfigured?: boolean;
|
||||
effectiveCapabilities?: { reusableLeases: boolean } | null;
|
||||
} | null;
|
||||
}): NativeSandboxLifecycle | null {
|
||||
if (
|
||||
input.adapterType !== "paperclip_runner" ||
|
||||
input.target?.kind !== "remote" ||
|
||||
input.target.transport !== "sandbox"
|
||||
)
|
||||
return null;
|
||||
const reusableLease =
|
||||
input.target.reusableLeaseConfigured === true &&
|
||||
input.target.effectiveCapabilities?.reusableLeases === true;
|
||||
if (input.lifecyclePolicy.mode === "warm" && !reusableLease) {
|
||||
throw new Error("runner_warm_lifecycle_requires_reusable_provider_lease");
|
||||
}
|
||||
return {
|
||||
runnerProcess: input.lifecyclePolicy.mode,
|
||||
sandboxResource:
|
||||
input.lifecyclePolicy.mode === "warm"
|
||||
? "keep_running"
|
||||
: reusableLease
|
||||
? "stop_and_reuse"
|
||||
: "destroy_after_turn",
|
||||
failoverBackup: "verified",
|
||||
};
|
||||
}
|
||||
|
||||
export function applyPersistedExecutionWorkspaceConfig(input: {
|
||||
config: Record<string, unknown>;
|
||||
workspaceConfig: ExecutionWorkspaceConfig | null;
|
||||
@@ -8611,6 +8656,13 @@ export function heartbeatService(
|
||||
agentId: string;
|
||||
status: string | null | undefined;
|
||||
failureReason?: string | null;
|
||||
providerResourceDisposition?: ProviderResourceDisposition;
|
||||
nativeLifecycleTelemetry?: {
|
||||
provider: string;
|
||||
harness: string;
|
||||
lifecycleMode: "per_turn" | "warm";
|
||||
sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn";
|
||||
};
|
||||
}) {
|
||||
const releaseResult = await envOrchestrator
|
||||
.releaseForRun({
|
||||
@@ -8619,6 +8671,8 @@ export function heartbeatService(
|
||||
agentId: input.agentId,
|
||||
status: leaseReleaseStatusForRunStatus(input.status),
|
||||
failureReason: input.failureReason ?? undefined,
|
||||
providerResourceDisposition: input.providerResourceDisposition,
|
||||
nativeLifecycleTelemetry: input.nativeLifecycleTelemetry,
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.warn(
|
||||
@@ -17408,6 +17462,17 @@ export function heartbeatService(
|
||||
activeRunExecutions.add(run.id);
|
||||
let runScratch: HeartbeatRunScratch | null = null;
|
||||
let nativeSessionResumeScheduled = false;
|
||||
let providerResourceDispositionForRun:
|
||||
ProviderResourceDisposition | undefined;
|
||||
let nativeLifecycleTelemetryForRun:
|
||||
| {
|
||||
provider: string;
|
||||
harness: string;
|
||||
lifecycleMode: "per_turn" | "warm";
|
||||
sandboxResource:
|
||||
"keep_running" | "stop_and_reuse" | "destroy_after_turn";
|
||||
}
|
||||
| undefined;
|
||||
let providerTraceCapture: Awaited<
|
||||
ReturnType<typeof traceStore.prepare>
|
||||
> | null = null;
|
||||
@@ -19879,7 +19944,26 @@ export function heartbeatService(
|
||||
: 300_000,
|
||||
}
|
||||
: { mode: "per_turn" as const, idleTimeoutMs: null };
|
||||
const effectiveLifecyclePolicy = agentLifecyclePolicy;
|
||||
const environmentLifecyclePolicy =
|
||||
executionTarget?.kind === "remote" &&
|
||||
executionTarget.transport === "sandbox"
|
||||
? executionTarget.runnerLifecyclePolicy ?? null
|
||||
: null;
|
||||
const effectiveLifecyclePolicy =
|
||||
environmentLifecyclePolicy ?? agentLifecyclePolicy;
|
||||
if (
|
||||
effectiveLifecyclePolicy.mode === "warm" &&
|
||||
executionTarget?.kind === "remote" &&
|
||||
executionTarget.transport === "sandbox" &&
|
||||
(
|
||||
executionTarget.reusableLeaseConfigured !== true ||
|
||||
executionTarget.effectiveCapabilities?.reusableLeases !== true
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
"runner_warm_environment_requires_reusable_lease",
|
||||
);
|
||||
}
|
||||
const persistedProfile = persistedRunnerProfile;
|
||||
if (persistedNativeExecutionInput) {
|
||||
nativeExecution = persistedNativeExecutionInput;
|
||||
@@ -19977,10 +20061,30 @@ export function heartbeatService(
|
||||
: {},
|
||||
}
|
||||
: null,
|
||||
provider:
|
||||
nativeRuntimeResolution.profile.backend === "opencode_server"
|
||||
? "opencode"
|
||||
: nativeRuntimeResolution.profile.backend === "acpx_runtime"
|
||||
? "acpx"
|
||||
: "codex",
|
||||
...(nativeRuntimeResolution.profile.backend === "acpx_runtime"
|
||||
? {
|
||||
acpxAgent: parseObject(runtimeConfig).acpxAgent as
|
||||
"pi" | "claude" | "codex",
|
||||
}
|
||||
: {}),
|
||||
codexApprovalPolicy: resolvePaperclipRunnerPermissionMode(
|
||||
"codex",
|
||||
parseObject(agent.adapterConfig).codexPermissionMode,
|
||||
) as "never" | "on-request" | "untrusted",
|
||||
opencodePermissionMode: resolvePaperclipRunnerPermissionMode(
|
||||
"opencode",
|
||||
parseObject(runtimeConfig).opencodePermissionMode,
|
||||
) as "allow" | "ask" | "deny",
|
||||
acpxPermissionMode: resolvePaperclipRunnerPermissionMode(
|
||||
"acpx",
|
||||
parseObject(runtimeConfig).acpxPermissionMode,
|
||||
) as "approve-all" | "approve-reads" | "deny-all",
|
||||
model:
|
||||
typeof parseObject(agent.adapterConfig).model === "string"
|
||||
? String(parseObject(agent.adapterConfig).model)
|
||||
@@ -20015,6 +20119,44 @@ export function heartbeatService(
|
||||
}
|
||||
}
|
||||
}
|
||||
const nativeSandboxLifecycle = resolveNativeSandboxLifecycle({
|
||||
adapterType: agent.adapterType,
|
||||
lifecyclePolicy: nativeExecution.session.lifecyclePolicy,
|
||||
target: executionTarget,
|
||||
});
|
||||
if (nativeSandboxLifecycle) {
|
||||
nativeLifecycleTelemetryForRun = {
|
||||
provider: nativeExecution.provider.kind,
|
||||
harness: nativeExecution.session.driverKind,
|
||||
lifecycleMode: nativeExecution.session.lifecyclePolicy.mode,
|
||||
sandboxResource: nativeSandboxLifecycle.sandboxResource,
|
||||
};
|
||||
const selectedLifecycleSpan = getStartupTracer(
|
||||
"paperclip.environment-lifecycle",
|
||||
).startSpan("sandbox.lifecycle.selected", {
|
||||
attributes: {
|
||||
"paperclip.native.span.provider": nativeExecution.provider.kind,
|
||||
"paperclip.native.span.harness":
|
||||
nativeExecution.session.driverKind,
|
||||
"paperclip.native.span.lifecycle_mode":
|
||||
nativeExecution.session.lifecyclePolicy.mode,
|
||||
"paperclip.native.span.sandbox_resource":
|
||||
nativeSandboxLifecycle.sandboxResource,
|
||||
"paperclip.native.span.outcome": "selected",
|
||||
"paperclip.native.span.bytes_transferred": 0,
|
||||
},
|
||||
});
|
||||
selectedLifecycleSpan.end();
|
||||
}
|
||||
providerResourceDispositionForRun =
|
||||
nativeSandboxLifecycle?.sandboxResource === "keep_running"
|
||||
? "keep_running"
|
||||
: nativeSandboxLifecycle?.sandboxResource === "stop_and_reuse"
|
||||
? "stop_and_retain"
|
||||
: nativeSandboxLifecycle?.sandboxResource ===
|
||||
"destroy_after_turn"
|
||||
? "destroy"
|
||||
: undefined;
|
||||
await db.transaction(async (tx) => {
|
||||
const lockedRun = await tx
|
||||
.select()
|
||||
@@ -20473,6 +20615,27 @@ export function heartbeatService(
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
runnerExecutionTarget: executionTarget,
|
||||
enableRunnerPreviewIngress:
|
||||
resolvedInstanceSettings.experimental
|
||||
.enableRunnerPreviewIngress === true,
|
||||
runnerPublicUrl:
|
||||
runtimeEnv.PAPERCLIP_RUNNER_PUBLIC_URL?.trim() || null,
|
||||
runnerCaBundlePath:
|
||||
runtimeEnv.PAPERCLIP_RUNNER_CA_BUNDLE_PATH?.trim() ||
|
||||
null,
|
||||
runnerRemoteBinaryPath:
|
||||
runtimeEnv.PAPERCLIP_RUNNER_REMOTE_BINARY_PATH?.trim() ||
|
||||
null,
|
||||
runnerRemoteCodexPath:
|
||||
runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_PATH?.trim() ||
|
||||
null,
|
||||
runnerRemoteCodexNpmSpec:
|
||||
runtimeEnv.PAPERCLIP_RUNNER_REMOTE_CODEX_NPM_SPEC?.trim() ||
|
||||
null,
|
||||
runnerRemoteProviderPackPath:
|
||||
runtimeEnv.PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH?.trim() ||
|
||||
null,
|
||||
enqueueWakeup,
|
||||
onSpawn: async (meta) => {
|
||||
markDispatchStarted();
|
||||
@@ -21786,6 +21949,8 @@ export function heartbeatService(
|
||||
agentId: run.agentId,
|
||||
status: latestRun?.status,
|
||||
failureReason: latestRun?.error ?? undefined,
|
||||
providerResourceDisposition: providerResourceDispositionForRun,
|
||||
nativeLifecycleTelemetry: nativeLifecycleTelemetryForRun,
|
||||
});
|
||||
await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
|
||||
}
|
||||
|
||||
@@ -249,6 +249,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
enableWorkspaceDirtyQuarantineRepair: parsed.data.enableWorkspaceDirtyQuarantineRepair ?? true,
|
||||
enableOwnerInstanceAdmin: parsed.data.enableOwnerInstanceAdmin ?? false,
|
||||
enableSandboxDuplexBridge: parsed.data.enableSandboxDuplexBridge ?? false,
|
||||
enableRunnerPreviewIngress: parsed.data.enableRunnerPreviewIngress ?? false,
|
||||
enableWorktreeRunExecution: parsed.data.enableWorktreeRunExecution ?? false,
|
||||
worktreeRunExecutionActivatedAt: parsed.data.worktreeRunExecutionActivatedAt ?? null,
|
||||
worktreeRunExecutionActivationInstanceId:
|
||||
@@ -288,6 +289,7 @@ export function normalizeExperimentalSettings(raw: unknown): InstanceExperimenta
|
||||
enableWorkspaceDirtyQuarantineRepair: true,
|
||||
enableOwnerInstanceAdmin: false,
|
||||
enableSandboxDuplexBridge: false,
|
||||
enableRunnerPreviewIngress: false,
|
||||
enableWorktreeRunExecution: false,
|
||||
worktreeRunExecutionActivatedAt: null,
|
||||
worktreeRunExecutionActivationInstanceId: null,
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
import type {
|
||||
NativeAcpxAgent,
|
||||
NativeAcpxPermissionMode,
|
||||
NativeCodexApprovalPolicy,
|
||||
NativeExecutionInputV4,
|
||||
NativeInteractionResponseEnvelope,
|
||||
NativeOpenCodePermissionMode,
|
||||
NativePlanningContext,
|
||||
NativeRuntimeContextSnapshot,
|
||||
StrictCompletionContractInput,
|
||||
} from "../../vendor/paperclip-runner/index.js";
|
||||
import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
|
||||
import {
|
||||
parseNativeExecutionInput,
|
||||
resolveQualifiedAcpxProfile,
|
||||
} from "../../vendor/paperclip-runner/index.js";
|
||||
import { renderPaperclipWakePrompt } from "@paperclipai/adapter-utils/server-utils";
|
||||
|
||||
/** Closed constructor: callers cannot spread legacy context or environment data. */
|
||||
@@ -38,7 +44,11 @@ export function buildNativeExecutionInput(input: {
|
||||
branchName: string | null;
|
||||
};
|
||||
normalizedSessionId: string | null;
|
||||
provider?: "codex" | "opencode" | "acpx";
|
||||
acpxAgent?: NativeAcpxAgent;
|
||||
codexApprovalPolicy?: NativeCodexApprovalPolicy;
|
||||
opencodePermissionMode?: NativeOpenCodePermissionMode;
|
||||
acpxPermissionMode?: NativeAcpxPermissionMode;
|
||||
model?: string | null;
|
||||
lifecyclePolicy?: NativeExecutionInputV4["session"]["lifecyclePolicy"];
|
||||
executionMode?: "default" | "plan";
|
||||
@@ -57,6 +67,12 @@ export function buildNativeExecutionInput(input: {
|
||||
}
|
||||
const executionMode = input.executionMode
|
||||
?? (input.issue.workMode === "planning" ? "plan" : "default");
|
||||
const acpxProfile = input.provider === "acpx"
|
||||
? resolveQualifiedAcpxProfile(
|
||||
input.acpxAgent ?? "pi",
|
||||
input.model ?? "",
|
||||
)
|
||||
: null;
|
||||
const wakePrompt = renderPaperclipWakePrompt(input.wakePayload, {
|
||||
resumedSession: input.resumedSession === true,
|
||||
suppressIssueDescription: input.taskPrompt.trim().length > 0,
|
||||
@@ -90,15 +106,44 @@ export function buildNativeExecutionInput(input: {
|
||||
},
|
||||
session: {
|
||||
normalizedSessionId: input.normalizedSessionId,
|
||||
driverKind: "codex_app_server",
|
||||
driverKind: input.provider === "opencode"
|
||||
? "opencode_server"
|
||||
: input.provider === "acpx"
|
||||
? "acpx_runtime"
|
||||
: "codex_app_server",
|
||||
protocolVersion: 1,
|
||||
lifecyclePolicy: input.lifecyclePolicy ?? { mode: "per_turn", idleTimeoutMs: null },
|
||||
},
|
||||
provider: {
|
||||
kind: "codex",
|
||||
model: input.model ?? null,
|
||||
approvalPolicy: input.codexApprovalPolicy ?? "never",
|
||||
},
|
||||
provider: input.provider === "acpx"
|
||||
? {
|
||||
kind: "acpx",
|
||||
agent: acpxProfile!.agent,
|
||||
model: input.model,
|
||||
permissionMode: input.acpxPermissionMode ?? "approve-all",
|
||||
profile: {
|
||||
driverKind: acpxProfile!.driverKind,
|
||||
protocolVersion: acpxProfile!.protocolVersion,
|
||||
acpxVersion: acpxProfile!.acpxVersion,
|
||||
agent: acpxProfile!.agent,
|
||||
agentProfileVersion: acpxProfile!.agentProfileVersion,
|
||||
agentServerPackage: acpxProfile!.agentServerPackage,
|
||||
agentServerVersion: acpxProfile!.agentServerVersion,
|
||||
agentRuntimePackage: acpxProfile!.agentRuntimePackage,
|
||||
agentRuntimeVersion: acpxProfile!.agentRuntimeVersion,
|
||||
commandDigest: acpxProfile!.commandDigest,
|
||||
},
|
||||
}
|
||||
: input.provider === "opencode"
|
||||
? {
|
||||
kind: "opencode",
|
||||
model: input.model,
|
||||
permissionMode: input.opencodePermissionMode ?? "allow",
|
||||
}
|
||||
: {
|
||||
kind: "codex",
|
||||
model: input.model ?? null,
|
||||
approvalPolicy: input.codexApprovalPolicy ?? "never",
|
||||
},
|
||||
completionContract: input.completionContract,
|
||||
interactionResponses: input.interactionResponses ?? [],
|
||||
credentialBindings: [],
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
existsSync,
|
||||
lstatSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
readlinkSync,
|
||||
} from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
|
||||
|
||||
export interface NativeHarnessBackupStamp {
|
||||
schema: "paperclip.native-harness-backup-stamp.v1";
|
||||
normalizedSessionId: string;
|
||||
runnerInstanceId: string;
|
||||
manifestSha256: string;
|
||||
completedAt: string;
|
||||
}
|
||||
|
||||
function stateRoot(normalizedSessionId: string): string {
|
||||
return resolve(
|
||||
process.env.PAPERCLIP_RUNNER_STATE_DIR
|
||||
?? resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "durable-sessions"),
|
||||
createHash("sha256").update(normalizedSessionId).digest("hex"),
|
||||
);
|
||||
}
|
||||
|
||||
function digestDirectory(directory: string): { sha256: string; bytes: number } {
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
const visit = (current: string, relative: string) => {
|
||||
const entries = readdirSync(current, { withFileTypes: true })
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
if (entries.length === 0) hash.update(`directory:${relative}\0`);
|
||||
for (const entry of entries) {
|
||||
const path = resolve(current, entry.name);
|
||||
const relativePath = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
const stats = lstatSync(path);
|
||||
if (entry.isDirectory()) {
|
||||
hash.update(`directory:${relativePath}:${stats.mode & 0o777}\0`);
|
||||
visit(path, relativePath);
|
||||
} else if (entry.isSymbolicLink()) {
|
||||
hash.update(`symlink:${relativePath}:${readlinkSync(path)}\0`);
|
||||
} else if (entry.isFile()) {
|
||||
const contents = readFileSync(path);
|
||||
bytes += contents.byteLength;
|
||||
hash.update(`file:${relativePath}:${stats.mode & 0o777}:${contents.byteLength}\0`);
|
||||
hash.update(contents);
|
||||
} else {
|
||||
throw new Error(`runner_harness_backup_unsupported_entry:${relativePath}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(directory, "");
|
||||
return { sha256: `sha256:${hash.digest("hex")}`, bytes };
|
||||
}
|
||||
|
||||
export function createNativeHarnessBackupStamp(input: {
|
||||
manifestPath: string;
|
||||
normalizedSessionId: string;
|
||||
runnerInstanceId: string;
|
||||
completedAt: string;
|
||||
}): NativeHarnessBackupStamp {
|
||||
const manifestBytes = readFileSync(input.manifestPath);
|
||||
return {
|
||||
schema: "paperclip.native-harness-backup-stamp.v1",
|
||||
normalizedSessionId: input.normalizedSessionId,
|
||||
runnerInstanceId: input.runnerInstanceId,
|
||||
manifestSha256: `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`,
|
||||
completedAt: input.completedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export function verifyNativeHarnessBackupStamp(
|
||||
value: unknown,
|
||||
expectedProviderLeaseId: string,
|
||||
): boolean {
|
||||
if (!expectedProviderLeaseId) return false;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
||||
const stamp = value as Record<string, unknown>;
|
||||
if (
|
||||
stamp.schema !== "paperclip.native-harness-backup-stamp.v1" ||
|
||||
typeof stamp.normalizedSessionId !== "string" || !stamp.normalizedSessionId ||
|
||||
typeof stamp.runnerInstanceId !== "string" || !stamp.runnerInstanceId ||
|
||||
typeof stamp.manifestSha256 !== "string" || !stamp.manifestSha256.startsWith("sha256:")
|
||||
) return false;
|
||||
const backupRoot = resolve(stateRoot(stamp.normalizedSessionId), "failover-backups");
|
||||
for (const candidate of [resolve(backupRoot, "current"), resolve(backupRoot, "previous")]) {
|
||||
const manifestPath = resolve(candidate, "manifest.json");
|
||||
if (!existsSync(manifestPath)) continue;
|
||||
try {
|
||||
const bytes = readFileSync(manifestPath);
|
||||
const manifestSha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
|
||||
if (manifestSha256 !== stamp.manifestSha256) continue;
|
||||
const manifest = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
|
||||
if (
|
||||
manifest.schema !== "paperclip.native-harness-backup.v1" ||
|
||||
manifest.normalizedSessionId !== stamp.normalizedSessionId ||
|
||||
manifest.runnerInstanceId !== stamp.runnerInstanceId ||
|
||||
manifest.sourceProviderLeaseId !== expectedProviderLeaseId ||
|
||||
!Array.isArray(manifest.directories) ||
|
||||
manifest.directories.length === 0
|
||||
) continue;
|
||||
let valid = true;
|
||||
for (const entry of manifest.directories) {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
const declared = entry as Record<string, unknown>;
|
||||
if (
|
||||
typeof declared.name !== "string" ||
|
||||
!/^[A-Za-z0-9._-]+$/.test(declared.name) ||
|
||||
typeof declared.sha256 !== "string" ||
|
||||
typeof declared.bytes !== "number"
|
||||
) {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
const directory = resolve(candidate, declared.name);
|
||||
if (!existsSync(directory)) {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
const actual = digestDirectory(directory);
|
||||
if (actual.sha256 !== declared.sha256 || actual.bytes !== declared.bytes) {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (valid) return true;
|
||||
} catch {
|
||||
// Try the previous atomically-published backup.
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -20,6 +20,7 @@ function execution(runId: string, cwd = "/workspace") {
|
||||
agentId,
|
||||
workspace: { id: runId, cwd, repoUrl: null, repoRef: null, branchName: null },
|
||||
normalizedSessionId,
|
||||
provider: "codex",
|
||||
completionContract: {
|
||||
id: "70000000-0000-4000-8000-000000000007",
|
||||
sha256: `sha256:${"a".repeat(64)}`,
|
||||
@@ -43,6 +44,7 @@ function planningExecution(runId: string, revisionId: string) {
|
||||
agentId,
|
||||
workspace: { id: runId, cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null },
|
||||
normalizedSessionId,
|
||||
provider: "codex",
|
||||
executionMode: "plan",
|
||||
planningContext: {
|
||||
documentId: "80000000-0000-4000-8000-000000000008",
|
||||
@@ -189,7 +191,7 @@ describe("rebindNativeSessionCheckpoint", () => {
|
||||
});
|
||||
|
||||
describe("buildNativeExecutionInput wake projection", () => {
|
||||
it("writes native v4 and pins the complete Codex configuration", () => {
|
||||
it("writes native v4 and pins every provider's complete effective configuration", () => {
|
||||
const common = {
|
||||
companyId,
|
||||
runId: currentRunId,
|
||||
@@ -203,14 +205,36 @@ describe("buildNativeExecutionInput wake projection", () => {
|
||||
} as const;
|
||||
const codex = buildNativeExecutionInput({
|
||||
...common,
|
||||
provider: "codex",
|
||||
codexApprovalPolicy: "on-request",
|
||||
});
|
||||
const opencode = buildNativeExecutionInput({
|
||||
...common,
|
||||
provider: "opencode",
|
||||
model: "openrouter/z-ai/glm-5.2",
|
||||
opencodePermissionMode: "ask",
|
||||
});
|
||||
const acpx = buildNativeExecutionInput({
|
||||
...common,
|
||||
provider: "acpx",
|
||||
acpxAgent: "claude",
|
||||
model: "claude-sonnet-5",
|
||||
acpxPermissionMode: "deny-all",
|
||||
});
|
||||
|
||||
expect(codex).toMatchObject({
|
||||
schema: "paperclip.native-execution-input.v4",
|
||||
provider: { kind: "codex", approvalPolicy: "on-request" },
|
||||
});
|
||||
expect(JSON.stringify(codex))
|
||||
expect(opencode).toMatchObject({
|
||||
schema: "paperclip.native-execution-input.v4",
|
||||
provider: { kind: "opencode", permissionMode: "ask" },
|
||||
});
|
||||
expect(acpx).toMatchObject({
|
||||
schema: "paperclip.native-execution-input.v4",
|
||||
provider: { kind: "acpx", permissionMode: "deny-all" },
|
||||
});
|
||||
expect(JSON.stringify([codex, opencode, acpx]))
|
||||
.not.toMatch(/OPENAI_API_KEY|ANTHROPIC_API_KEY|AWS_SECRET_ACCESS_KEY|PAPERCLIP_API_KEY/);
|
||||
});
|
||||
|
||||
@@ -257,6 +281,8 @@ describe("buildNativeExecutionInput wake projection", () => {
|
||||
branchName: null,
|
||||
},
|
||||
normalizedSessionId,
|
||||
provider: "opencode",
|
||||
model: "openrouter/z-ai/glm-5.2",
|
||||
completionContract: {
|
||||
id: "70000000-0000-4000-8000-000000000007",
|
||||
sha256: `sha256:${"a".repeat(64)}`,
|
||||
|
||||
@@ -172,6 +172,21 @@ describe("resolveNativeRuntimeMode", () => {
|
||||
}));
|
||||
});
|
||||
|
||||
it("keeps a persisted OpenCode recovery on its immutable driver", () => {
|
||||
expect(resolveHeartbeatNativeRuntimeMode({
|
||||
...eligible,
|
||||
enabled: false,
|
||||
persisted: {
|
||||
runtimeMode: "native",
|
||||
runtimeModeReason: "eligible_opt_in",
|
||||
runtimeModeResolvedAt: new Date(),
|
||||
driverKind: "opencode_server",
|
||||
},
|
||||
})).toEqual(expect.objectContaining({
|
||||
profile: { mode: "native", backend: "opencode_server", protocolVersion: 1 },
|
||||
}));
|
||||
});
|
||||
|
||||
it("rejects an explicit native profile outside the approved boundary", () => {
|
||||
expect(resolveNativeRuntimeMode({ ...eligible, agent: { ...eligible.agent, adapterType: "claude_local" } }))
|
||||
.toEqual(expect.objectContaining({ kind: "legacy", reason: "direct_adapter" }));
|
||||
@@ -179,16 +194,19 @@ describe("resolveNativeRuntimeMode", () => {
|
||||
.toThrow(NativeRuntimeEligibilityError);
|
||||
});
|
||||
|
||||
it("rejects remote targets for fresh paperclip_runner starts", () => {
|
||||
expect(() => resolveNativeRuntimeMode({
|
||||
it("admits remote targets only through paperclip_runner", () => {
|
||||
expect(resolveNativeRuntimeMode({
|
||||
...eligible,
|
||||
target: { kind: "remote" },
|
||||
runtimeConfig: {},
|
||||
adapterConfig: { provider: "codex" },
|
||||
agent: { ...eligible.agent, adapterType: "paperclip_runner" },
|
||||
})).toThrow(expect.objectContaining({
|
||||
code: "paperclip_runner_environment_unsupported",
|
||||
}));
|
||||
})).toMatchObject({ kind: "native" });
|
||||
expect(resolveNativeRuntimeMode({
|
||||
...eligible,
|
||||
target: { kind: "remote" },
|
||||
agent: { ...eligible.agent, adapterType: "codex_local" },
|
||||
})).toMatchObject({ kind: "legacy", reason: "direct_adapter" });
|
||||
});
|
||||
|
||||
it("allows paperclip_runner to use a transient local workspace for projectless issues", () => {
|
||||
|
||||
@@ -40,7 +40,7 @@ export type NativeRuntimeResolution =
|
||||
reason: "eligible_opt_in";
|
||||
profile: {
|
||||
mode: "native";
|
||||
backend: "codex_app_server";
|
||||
backend: "codex_app_server" | "opencode_server" | "acpx_runtime";
|
||||
protocolVersion: 1;
|
||||
};
|
||||
authorityDecision: NativeStatusDecision;
|
||||
@@ -129,12 +129,6 @@ export function resolveNativeRuntimeMode(input: {
|
||||
"Paperclip Runner requires a standard, planning, or ask task.",
|
||||
);
|
||||
}
|
||||
if (!input.target || input.target.kind !== "local") {
|
||||
throw ineligible(
|
||||
"paperclip_runner_environment_unsupported",
|
||||
"Paperclip Runner currently requires a local execution environment.",
|
||||
);
|
||||
}
|
||||
const rollout = resolveNativeMigrationStatus({
|
||||
facts: { applicationEnabled: true },
|
||||
priorIssueStatus: "in_progress",
|
||||
@@ -267,13 +261,17 @@ export function resolveHeartbeatNativeRuntimeMode(input: {
|
||||
);
|
||||
}
|
||||
const driverKind = input.persisted.driverKind;
|
||||
const backend = driverKind === null
|
||||
|| driverKind === undefined
|
||||
|| driverKind === "codex"
|
||||
|| driverKind === "codex_app_server"
|
||||
? "codex_app_server"
|
||||
: null;
|
||||
if (!backend) {
|
||||
const backend = driverKind === "opencode_server"
|
||||
? "opencode_server"
|
||||
: driverKind === "acpx_runtime"
|
||||
? "acpx_runtime"
|
||||
: driverKind === null
|
||||
|| driverKind === undefined
|
||||
|| driverKind === "codex"
|
||||
|| driverKind === "codex_app_server"
|
||||
? "codex_app_server"
|
||||
: null;
|
||||
if (backend === null) {
|
||||
throw ineligible(
|
||||
"paperclip_runner_driver_unsupported",
|
||||
`Persisted Paperclip Runner driver is unsupported: ${driverKind}`,
|
||||
|
||||
Reference in new issue
Block a user