mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 11:20:42 +01:00
fix(runner): preserve native work through controller outages (#17594)
Keep remote native work under its authenticated connection lease through controller outages. Hold durable ACPX tools through reconnection, align the native Claude HTTP MCP client budget, and wait for the recovered handler before dispatch. Preserve Stop, cancellation, exact identity and artifact checks. Verified with real-process regressions and a real Claude Opus 5.5 Daytona workflow: 125 seconds offline during useful work, original process and lease retained, no failed native tool calls, queued follow-up completed, independently verified ZIP and committed workspace receipts. All current-head CI gates pass and Greptile is 5/5 with all findings resolved. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
3536b3649f
commit
aec86c794b
37 files changed
+628
-75
No files matched your search
@@ -88,6 +88,7 @@ packages/grok-acp/bin
|
||||
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v27-identity.json
|
||||
|
||||
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json
|
||||
!packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json
|
||||
|
||||
!packages/paperclip-runner/test/fixtures/copilot-profile-v23-identity.json
|
||||
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# Paperclip evaluation guide
|
||||
|
||||
The explicit-only [Daytona restart continuity journey](../tests/runner-e2e/README.md)
|
||||
extends the existing native restart oracle to a 125-second controller outage,
|
||||
using the recorded Claude and Codex models in disposable development fixtures.
|
||||
It requires same-process adoption, saved files and finalization without replay;
|
||||
local regression checks and catalog discovery are separate from live acceptance.
|
||||
|
||||
The explicit-only [GitHub review bot journey](../tests/runner-e2e/GITHUB-REVIEW-JOURNEY.md)
|
||||
converts the Ponytail walkthrough into attended App setup, scoped installation,
|
||||
automatic PR review, and independent two-service checks. It supports attached
|
||||
|
||||
@@ -1676,6 +1676,30 @@ waiting for provider startup. An abrupt controller exit during session opening
|
||||
can then recover through the same exact process-identity checks as an active turn.
|
||||
|
||||
Both graceful and hot restarts detach the old controller from native sessions.
|
||||
Remote Runner processes use their authenticated connection lease as the bound
|
||||
on disconnected lifetime. The controller's shorter connection-recovery budget
|
||||
limits its own attempt; it must not terminate the remote owner while another
|
||||
controller is starting or classifying runs. Lease expiry still shuts down the
|
||||
provider, and exact identity, authentication, Stop and revocation checks remain
|
||||
required before any recovery or new execution.
|
||||
Native ACPX semantic tool calls wait on their durable PRP invocation across a
|
||||
controller outage. Their provider-side MCP bridge does not impose a separate
|
||||
request deadline that could make the provider retry an already queued write.
|
||||
Runner lease expiry, turn cancellation and bridge closure still end those waits.
|
||||
Ordinary bridge callers retain their request deadline. Repeating the same call
|
||||
identity returns its one retained result; it does not execute the operation again.
|
||||
The native Claude sidecar also sets its HTTP MCP client budget to one hour,
|
||||
matching the Runner connection lease budget. This prevents Claude's default
|
||||
60-second HTTP deadline from aborting an otherwise pending durable request.
|
||||
The setting applies to that native Claude session's MCP client; other provider
|
||||
sessions and ordinary bridge deadlines remain unchanged. Explicit cancellation
|
||||
and Runner shutdown still take precedence over that client budget.
|
||||
An adopted transport can receive authenticated queued semantic calls before
|
||||
the driver publishes its recovered session and tool handler. It holds those
|
||||
calls until the handler is installed, then rechecks the original authority and
|
||||
turn epoch before dispatch. Transport failure, close or controller detachment
|
||||
ends that wait without invoking a late handler. Missing startup wiring is not
|
||||
reported to the provider as the result of an external operation.
|
||||
If shutdown begins while a provider session is opening, its eventual publication
|
||||
honors the pending detachment before dispatching a turn. Once detached, an old
|
||||
execution finalizer cannot suspend or signal the durable runner: the next
|
||||
|
||||
@@ -14,12 +14,12 @@
|
||||
"requiresProviderPolicy": false
|
||||
},
|
||||
"pi": {
|
||||
"agentProfileVersion": 28,
|
||||
"agentProfileVersion": 29,
|
||||
"agentServerPackage": "pi-acp",
|
||||
"agentServerVersion": "0.0.33",
|
||||
"agentRuntimePackage": "@earendil-works/pi-coding-agent",
|
||||
"agentRuntimeVersion": "1.0.0",
|
||||
"commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c",
|
||||
"commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
|
||||
"requiresProviderPolicy": true
|
||||
},
|
||||
"cursor": {
|
||||
@@ -32,12 +32,12 @@
|
||||
"requiresProviderPolicy": true
|
||||
},
|
||||
"copilot": {
|
||||
"agentProfileVersion": 58,
|
||||
"agentProfileVersion": 59,
|
||||
"agentServerPackage": "@github/copilot",
|
||||
"agentServerVersion": "1.0.88",
|
||||
"agentRuntimePackage": null,
|
||||
"agentRuntimeVersion": null,
|
||||
"commandDigest": "sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94",
|
||||
"commandDigest": "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40",
|
||||
"requiresProviderPolicy": true
|
||||
},
|
||||
"claude": {
|
||||
|
||||
@@ -3212,7 +3212,7 @@ mod tests {
|
||||
#[test]
|
||||
fn pi_profile_matches_published_identity_and_rejects_prior_profiles() {
|
||||
let published: Value = serde_json::from_str(include_str!(
|
||||
"../../../../test-fixtures/pi-acp/profile-v28-identity.json"
|
||||
"../../../../test-fixtures/pi-acp/profile-v29-identity.json"
|
||||
))
|
||||
.unwrap();
|
||||
let mut value = descriptor("codex");
|
||||
@@ -3229,6 +3229,7 @@ mod tests {
|
||||
current.validate(&context()).unwrap();
|
||||
|
||||
for prior in [
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v28-identity.json"),
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v24-identity.json"),
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v23-identity.json"),
|
||||
include_str!("../../../../test-fixtures/pi-acp/profile-v15-identity.json"),
|
||||
|
||||
@@ -29,7 +29,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option<AcpxReleaseProfile> {
|
||||
agent_runtime_package: Some("@earendil-works/pi-coding-agent"),
|
||||
agent_runtime_version: Some("1.0.0"),
|
||||
command_digest:
|
||||
"sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c",
|
||||
"sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
|
||||
requires_provider_policy: true,
|
||||
},
|
||||
"cursor" => AcpxReleaseProfile {
|
||||
@@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option<AcpxReleaseProfile> {
|
||||
agent_runtime_package: None,
|
||||
agent_runtime_version: None,
|
||||
command_digest:
|
||||
"sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94",
|
||||
"sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40",
|
||||
requires_provider_policy: true,
|
||||
},
|
||||
"claude" => AcpxReleaseProfile {
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ use sha2::{Digest, Sha256};
|
||||
const CODEX_ACPX_DIGEST: &str =
|
||||
"sha256:443491f4e814501a5dc47347e4510e84faeec9c9382e2d4e53bfe69386d5aabc";
|
||||
const PI_ACPX_DIGEST: &str =
|
||||
"sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c";
|
||||
"sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06";
|
||||
|
||||
fn temporary_directory(label: &str) -> PathBuf {
|
||||
let nonce = SystemTime::now()
|
||||
|
||||
@@ -7,9 +7,9 @@ import { fileURLToPath } from "node:url";
|
||||
import { resolve, dirname } from "node:path";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const fixture = resolve(root, "test/fixtures/copilot-profile-v58-identity.json");
|
||||
const fixture = resolve(root, "test/fixtures/copilot-profile-v59-identity.json");
|
||||
const historical = JSON.parse(readFileSync(resolve(root, "test/fixtures/copilot-profile-v12-identity.json"), "utf8"));
|
||||
const declaration = { ...historical.declaration, agentProfileVersion: 58 };
|
||||
const declaration = { ...historical.declaration, agentProfileVersion: 59 };
|
||||
delete declaration.permissionIdentitySourceSha256;
|
||||
const sources = {
|
||||
displayEventSourceSha256: "src/drivers/acpx/copilot-events.ts",
|
||||
|
||||
@@ -12,6 +12,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createAcpxSidecarHostEnvironment, createSanitizedAcpxSpawnInput } from "../drivers/acpx/environment.js";
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
@@ -93,6 +94,22 @@ afterEach(async () => {
|
||||
});
|
||||
|
||||
describe("qualified ACPX runtime sidecar", () => {
|
||||
it("gives native Claude HTTP tools the Runner lease budget without changing sibling providers", () => {
|
||||
const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8");
|
||||
const start = source.indexOf(" environment:");
|
||||
const end = source.indexOf(" expectedIdentity:", start);
|
||||
expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);
|
||||
const create = new Function("process", "params", "createAcpxSidecarHostEnvironment", `return ({ ${source.slice(start, end)} }).environment;`);
|
||||
const original = { MCP_TOOL_TIMEOUT: "1000", ANTHROPIC_API_KEY: "selected-key", UNRELATED_SECRET: "private" };
|
||||
const claude = create({ env: original }, { agent: "claude", normalizedSessionId: "session" }, createAcpxSidecarHostEnvironment);
|
||||
expect(createSanitizedAcpxSpawnInput(claude, "claude").env).toMatchObject({ MCP_TOOL_TIMEOUT: "3600000", ANTHROPIC_API_KEY: "selected-key" });
|
||||
expect(createSanitizedAcpxSpawnInput(claude, "claude").env).not.toHaveProperty("UNRELATED_SECRET");
|
||||
expect(original.MCP_TOOL_TIMEOUT).toBe("1000");
|
||||
const codex = create({ env: original }, { agent: "codex", normalizedSessionId: "session" }, createAcpxSidecarHostEnvironment);
|
||||
expect(codex).toBe(original);
|
||||
expect(createSanitizedAcpxSpawnInput(codex, "codex").env).not.toHaveProperty("MCP_TOOL_TIMEOUT");
|
||||
});
|
||||
|
||||
it("projects the actual sidecar terminal diagnostic block without authoritative accounting", () => {
|
||||
const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8");
|
||||
const start = source.indexOf(" try {", source.indexOf(" const usageAfter = await readSidecarHostStatusWithin(activeHost);"));
|
||||
@@ -279,6 +296,42 @@ describe("qualified ACPX runtime sidecar", () => {
|
||||
for (const agent of ["cursor", "codex", "pi"]) expect(create({ agent, tools: [] }, evidence, () => undefined).captureSemanticReceipt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps a native tool pending through controller downtime without a duplicate side effect", async () => {
|
||||
const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8");
|
||||
const start = source.indexOf(" semanticTools: {");
|
||||
const end = source.indexOf(" onGoalUpdate:", start);
|
||||
expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);
|
||||
const create = new Function("params", "activeCopilotEvidence", "waitForTool", `return ({ ${source.slice(start, end)} }).semanticTools;`);
|
||||
let admitted!: () => void, complete!: (value: unknown) => void;
|
||||
const started = new Promise<void>(resolve => { admitted = resolve; });
|
||||
const result = new Promise(resolve => { complete = resolve; });
|
||||
let signal: AbortSignal | undefined;
|
||||
const handler = vi.fn((call: RunnerToolCall) => { signal = call.signal; admitted(); return result; });
|
||||
const bridge = await startRunnerToolBridge(create({ agent: "claude", tools: [{ name: "register_deliverable", inputSchema: { type: "object" } }] }, undefined, handler));
|
||||
const call = () => fetch(bridge.url, { method: "POST", headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: "saved-call", method: "tools/call", params: { name: "register_deliverable", arguments: {} } }) }).then(response => response.json());
|
||||
let first: Promise<any> | undefined;
|
||||
try {
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
first = call();
|
||||
await started;
|
||||
await vi.advanceTimersByTimeAsync(125_000);
|
||||
expect(signal?.aborted).toBe(false);
|
||||
// The same pending invocation can be read again after reconnect. It must
|
||||
// share the original result rather than create another external write.
|
||||
const duplicate = call();
|
||||
complete({ artifactId: "saved-artifact" });
|
||||
expect(await first).toMatchObject({ result: { content: [{ text: '{"artifactId":"saved-artifact"}' }] } });
|
||||
expect(await duplicate).toEqual(await first);
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
complete({ artifactId: "saved-artifact" });
|
||||
await first?.catch(() => undefined);
|
||||
await bridge.close();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["forwarded", "emit-failed"])("binds native v2 receipts to actual sidecar-normalized input: %s", async mode => {
|
||||
const raw = { reportedWorkDisposition: "done", summary: "Complete", evidence: [], verification: [],
|
||||
completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] } };
|
||||
|
||||
@@ -305,12 +305,22 @@ async function dispatch(
|
||||
providerPolicy: params.providerPolicy,
|
||||
systemInstructions: params.systemInstructions,
|
||||
runtimeContext: params.runtimeContext,
|
||||
environment: createAcpxSidecarHostEnvironment(process.env, params.agent, params.normalizedSessionId),
|
||||
environment: params.agent === "claude" ? {
|
||||
...createAcpxSidecarHostEnvironment(process.env, params.agent, params.normalizedSessionId),
|
||||
// Claude's HTTP MCP client otherwise aborts at 60s, before the
|
||||
// native Runner connection lease. Cancellation and Runner shutdown
|
||||
// still revoke the pending call; this is not a new authority lease.
|
||||
MCP_TOOL_TIMEOUT: String(60 * 60 * 1_000),
|
||||
} : createAcpxSidecarHostEnvironment(process.env, params.agent, params.normalizedSessionId),
|
||||
expectedIdentity: params.expectedIdentity,
|
||||
assignedMcpTransport: "prp",
|
||||
semanticTools: {
|
||||
tools: params.tools,
|
||||
handler: waitForTool,
|
||||
// These calls are durable PRP requests. A controller outage must not
|
||||
// turn a queued write into a provider-visible failure and retry.
|
||||
// Runner lease expiry, turn cancellation and bridge close own them.
|
||||
timeoutMs: null,
|
||||
...(params.agent === "copilot" ? { captureSemanticReceipt: () => activeCopilotEvidence?.captureSemanticReceipt() } : {}),
|
||||
},
|
||||
onGoalUpdate: (goal) => {
|
||||
|
||||
@@ -47,6 +47,10 @@ export interface RunnerExecutionDriver {
|
||||
}) => Promise<Record<string, unknown>>;
|
||||
/** Active-connection recovery budget. Omitted for the existing local mode. */
|
||||
runnerReconnectGraceMs?: number;
|
||||
/** Process lifetime while disconnected. Defaults to the active-connection
|
||||
* recovery budget for compatibility. Null uses the authenticated connection
|
||||
* lease as the lifetime bound, so controller downtime cannot kill its owner. */
|
||||
runnerProcessReconnectGraceMs?: number | null;
|
||||
/**
|
||||
* A verified Runner that outlived its controller. Adoption registers
|
||||
* the durable authority and waits for this exact process to reconnect; it
|
||||
|
||||
@@ -87,9 +87,19 @@ vi.mock("./installation-integrity.js", () => ({ verifyNativeAcpxInstallation: vi
|
||||
|
||||
describe("Copilot build-owned installation", () => {
|
||||
it("admits the current profile with environment launch preparation", () => {
|
||||
expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(58);
|
||||
expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(59);
|
||||
expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined();
|
||||
expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94");
|
||||
expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40");
|
||||
});
|
||||
|
||||
it("preserves v58 while binding the separate remote process reconnect policy in v59", () => {
|
||||
const prior = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v58-identity.json", import.meta.url), "utf8"));
|
||||
const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v59-identity.json", import.meta.url), "utf8"));
|
||||
expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([
|
||||
"agentProfileVersion", "credentialBindingSourceSha256", "nativeInstructionsSourceSha256", "nativeSessionRuntimeSourceSha256", "profileCompatibilitySourceSha256", "runtimeRequestBridgeSourceSha256", "semanticBridgeSourceSha256", "semanticSidecarSourceSha256", "transportContractSourceSha256",
|
||||
]);
|
||||
expect(isSupportedAcpxProfileVersion("copilot", 58)).toBe(true);
|
||||
expect(current.commandDigest).not.toBe(prior.commandDigest);
|
||||
});
|
||||
|
||||
it("preserves v57 while binding the shared Pi setup and launch repairs in v58", () => {
|
||||
|
||||
@@ -6,6 +6,13 @@ afterEach(() => vi.unstubAllEnvs());
|
||||
import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxCredentialBinding, createAcpxSidecarHostEnvironment, createSanitizedAcpxSpawnInput } from "./environment.js";
|
||||
|
||||
describe("ACPX launch environment", () => {
|
||||
it("retains an explicit Claude MCP timeout without inheriting an ambient or sibling setting", () => {
|
||||
vi.stubEnv("MCP_TOOL_TIMEOUT", "999999");
|
||||
expect(createSanitizedAcpxSpawnInput(undefined, "claude").env).not.toHaveProperty("MCP_TOOL_TIMEOUT");
|
||||
expect(createSanitizedAcpxSpawnInput({ MCP_TOOL_TIMEOUT: "120000" }, "claude").env.MCP_TOOL_TIMEOUT).toBe("120000");
|
||||
expect(createSanitizedAcpxSpawnInput({ MCP_TOOL_TIMEOUT: "120000" }, "codex").env).not.toHaveProperty("MCP_TOOL_TIMEOUT");
|
||||
});
|
||||
|
||||
it("excludes general IAM from the Pi sidecar and launch, including selected task values", () => {
|
||||
const iam = { AWS_ACCESS_KEY_ID: "general-key", AWS_SECRET_ACCESS_KEY: "general-secret", AWS_SESSION_TOKEN: "general-session" };
|
||||
const source = { ...iam, ...configuredEnvironmentProjection({ ...iam, CUSTOM_SETTING: "allowed" }), AWS_BEARER_TOKEN_BEDROCK: "provider-key" };
|
||||
|
||||
@@ -131,6 +131,7 @@ export function createSanitizedAcpxSpawnInput(
|
||||
...(environment === undefined ? [] : ["PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY"]),
|
||||
...credentialNames,
|
||||
...(agent === "claude" ? CLAUDE_ROUTING_ENV_KEYS : []),
|
||||
...(agent === "claude" && environment !== undefined ? ["MCP_TOOL_TIMEOUT"] : []),
|
||||
...(usesPreconfiguredHome(source) ? [...PRECONFIGURED_HOME_KEYS, "PAPERCLIP_RUNNER_HOME_MODE", "ACPX_CLAUDE_INCLUDE_USER_SETTINGS"] : []),
|
||||
]);
|
||||
let retainedBytes = 0;
|
||||
|
||||
@@ -22,12 +22,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = {
|
||||
"protocolVersion": 1,
|
||||
"acpxVersion": "0.13.1",
|
||||
"agent": "pi",
|
||||
"agentProfileVersion": 28,
|
||||
"agentProfileVersion": 29,
|
||||
"agentServerPackage": "pi-acp",
|
||||
"agentServerVersion": "0.0.33",
|
||||
"agentRuntimePackage": "@earendil-works/pi-coding-agent",
|
||||
"agentRuntimeVersion": "1.0.0",
|
||||
"commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c",
|
||||
"commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
|
||||
"permissionPolicy": "interactive"
|
||||
},
|
||||
"cursor": {
|
||||
@@ -48,12 +48,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = {
|
||||
"protocolVersion": 1,
|
||||
"acpxVersion": "0.13.1",
|
||||
"agent": "copilot",
|
||||
"agentProfileVersion": 58,
|
||||
"agentProfileVersion": 59,
|
||||
"agentServerPackage": "@github/copilot",
|
||||
"agentServerVersion": "1.0.88",
|
||||
"agentRuntimePackage": null,
|
||||
"agentRuntimeVersion": null,
|
||||
"commandDigest": "sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94",
|
||||
"commandDigest": "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40",
|
||||
"permissionPolicy": "interactive"
|
||||
},
|
||||
"claude": {
|
||||
|
||||
@@ -155,11 +155,20 @@ describe("Pi installation factory", () => {
|
||||
expect(current.commandDigest).not.toBe(prior.commandDigest);
|
||||
});
|
||||
|
||||
it("preserves profile 28 while binding the Claude-only explicit MCP client budget source in profile 29", async () => {
|
||||
const read = async (version: number) => JSON.parse(await readFile(new URL(`../../../test-fixtures/pi-acp/profile-v${version}-identity.json`, import.meta.url), "utf8"));
|
||||
const prior = await read(28), current = await read(29);
|
||||
expect(Object.keys(current.declaration).filter(key => JSON.stringify(current.declaration[key]) !== JSON.stringify(prior.declaration[key])).sort())
|
||||
.toEqual(["agentProfileVersion", "credentialEnvironmentSourceSha256"]);
|
||||
expect(current.declaration.closure).toEqual(prior.declaration.closure);
|
||||
expect(current.commandDigest).not.toBe(prior.commandDigest);
|
||||
});
|
||||
|
||||
it("binds the profile declaration to the reviewed patch and platform closure pins", async () => {
|
||||
const hash = (bytes: string | Uint8Array) => createHash("sha256").update(bytes).digest("hex");
|
||||
const canonical = (value: any): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]`
|
||||
: value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}` : JSON.stringify(value);
|
||||
const identity = JSON.parse(await readFile(new URL("../../../test-fixtures/pi-acp/profile-v28-identity.json", import.meta.url), "utf8"));
|
||||
const identity = JSON.parse(await readFile(new URL("../../../test-fixtures/pi-acp/profile-v29-identity.json", import.meta.url), "utf8"));
|
||||
const declaration = identity.declaration;
|
||||
expect(declaration.sharedRuntimeContract).toBe("paperclip.acpx-runtime-contract.v1");
|
||||
expect(identity.commandDigest).toBe(`sha256:${hash(canonical(declaration))}`);
|
||||
@@ -190,21 +199,21 @@ describe("Pi installation factory", () => {
|
||||
|
||||
it("rejects legacy profiles and tampered executable identities", () => {
|
||||
expect(() => assertPiInstallationProfile(candidate())).not.toThrow();
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 1 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 2 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 3 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 4 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 5 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 6 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 7 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 8 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 9 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 10 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 11 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 12 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 13 })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 14 })).toThrow("version 28");
|
||||
for (const priorVersion of [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]) expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: priorVersion })).toThrow("version 28");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 1 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 2 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 3 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 4 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 5 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 6 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 7 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 8 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 9 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 10 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 11 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 12 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 13 })).toThrow("version 29");
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 14 })).toThrow("version 29");
|
||||
for (const priorVersion of [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]) expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: priorVersion })).toThrow("version 29");
|
||||
for (const changed of [{ agentServerVersion: "latest" }, { commandDigest: `sha256:${"0".repeat(64)}` }, { agentRuntimePackage: "ambient-pi" }]) {
|
||||
expect(() => assertPiInstallationProfile({ ...candidate(), ...changed })).toThrow("trusted declaration");
|
||||
}
|
||||
|
||||
@@ -2,13 +2,13 @@ import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js";
|
||||
|
||||
/** Decodable historical profile revisions, not permission to launch them.
|
||||
* Exact current profile/package/digest admission is checked separately. */
|
||||
type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57;
|
||||
type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58;
|
||||
type CurrentAcpxProfileVersion = typeof QUALIFIED_ACPX_PROFILE_DATA[keyof typeof QUALIFIED_ACPX_PROFILE_DATA]["agentProfileVersion"];
|
||||
export type AcpxProfileVersion = HistoricalAcpxProfileVersion | CurrentAcpxProfileVersion;
|
||||
|
||||
const historicalVersions: Readonly<Record<string, readonly HistoricalAcpxProfileVersion[]>> = {
|
||||
pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5],
|
||||
grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57],
|
||||
pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5],
|
||||
grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],
|
||||
cursor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
||||
};
|
||||
|
||||
|
||||
@@ -246,7 +246,7 @@ describe("runner semantic MCP bridge", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("honors MCP cancellation after the handler starts", async () => {
|
||||
it.each([60_000, null])("honors MCP cancellation after the handler starts (timeout=%s)", async timeoutMs => {
|
||||
let markStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
@@ -254,7 +254,7 @@ describe("runner semantic MCP bridge", () => {
|
||||
const aborted = vi.fn();
|
||||
const bridge = await startRunnerToolBridge({
|
||||
tools: [tool("documents.read")],
|
||||
timeoutMs: 60_000,
|
||||
timeoutMs,
|
||||
handler: ({ signal }) => new Promise((_resolve, reject) => {
|
||||
signal.addEventListener("abort", () => {
|
||||
aborted();
|
||||
@@ -291,6 +291,23 @@ describe("runner semantic MCP bridge", () => {
|
||||
expect(aborted).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("cancels a lease-owned native tool when its bridge closes", async () => {
|
||||
let admitted!: () => void;
|
||||
const started = new Promise<void>(resolve => { admitted = resolve; });
|
||||
let signal: AbortSignal | undefined;
|
||||
const bridge = await startRunnerToolBridge({ tools: [tool("documents.read")], timeoutMs: null,
|
||||
handler: call => { signal = call.signal; admitted(); return new Promise(() => {}); },
|
||||
});
|
||||
bridges.push(bridge);
|
||||
const pending = rpc(bridge, { id: "close-me", method: "tools/call", params: { name: "documents.read", arguments: {} } });
|
||||
await started;
|
||||
await bridge.close();
|
||||
expect(signal?.aborted).toBe(true);
|
||||
expect(await (await pending).json()).toMatchObject({ result: { isError: true,
|
||||
content: [{ text: "Paperclip tool call cancelled" }],
|
||||
} });
|
||||
});
|
||||
|
||||
it("preserves successful mutation identity when the result is oversized", async () => {
|
||||
const result = { text: `snowman-${"☃".repeat(65 * 1024)}` };
|
||||
const handler = vi.fn(async () => result);
|
||||
|
||||
@@ -45,7 +45,9 @@ export interface RunnerToolBridgeOptions {
|
||||
handler(call: RunnerToolCall): Promise<unknown>;
|
||||
/** Copilot-only opt-in; capture the active invocation scope before dispatch. */
|
||||
captureSemanticReceipt?: () => ((receipt: SemanticToolReceipt) => void) | undefined;
|
||||
timeoutMs?: number;
|
||||
/** Null delegates the wait to an owning durable runtime's lease and close.
|
||||
* Ordinary callers retain the bounded request timeout. */
|
||||
timeoutMs?: number | null;
|
||||
privateToolTimeoutMs?: number;
|
||||
maxBodyBytes?: number;
|
||||
/** Internal test seam; production bounds headers and body reads to 30 seconds. */
|
||||
@@ -143,7 +145,7 @@ export async function startRunnerToolBridge(
|
||||
controllers,
|
||||
handler: options.handler,
|
||||
captureSemanticReceipt: options.captureSemanticReceipt,
|
||||
timeoutMs: positiveBoundedInteger(
|
||||
timeoutMs: options.timeoutMs === null ? null : positiveBoundedInteger(
|
||||
options.timeoutMs,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
1,
|
||||
@@ -214,7 +216,7 @@ async function handleRequest(
|
||||
controllers: Map<string, AbortController>;
|
||||
handler: RunnerToolBridgeOptions["handler"];
|
||||
captureSemanticReceipt: RunnerToolBridgeOptions["captureSemanticReceipt"];
|
||||
timeoutMs: number;
|
||||
timeoutMs: number | null;
|
||||
privateToolTimeoutMs: number;
|
||||
privateToolNames: ReadonlySet<string>;
|
||||
maxBodyBytes: number;
|
||||
@@ -599,24 +601,24 @@ function closeServer(server: Server): Promise<void> {
|
||||
function withCancellationAndTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
controller: AbortController,
|
||||
timeoutMs: number,
|
||||
timeoutMs: number | null,
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const finish = (callback: () => void) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
controller.signal.removeEventListener("abort", onAbort);
|
||||
callback();
|
||||
};
|
||||
const onAbort = () =>
|
||||
finish(() => reject(new Error("Paperclip tool call cancelled")));
|
||||
const timer = setTimeout(() => {
|
||||
const timer = timeoutMs === null ? undefined : setTimeout(() => {
|
||||
finish(() => reject(new Error("Paperclip tool call timed out")));
|
||||
controller.abort();
|
||||
}, timeoutMs);
|
||||
timer.unref();
|
||||
timer?.unref();
|
||||
controller.signal.addEventListener("abort", onAbort, { once: true });
|
||||
promise.then(
|
||||
(value) => finish(() => resolve(value)),
|
||||
|
||||
@@ -857,6 +857,11 @@ function runnerCloseGraceMs(options: Pick<CapabilityRunnerdCodexTransportOptions
|
||||
return options.closeGraceMs ?? (options.provider === "acpx" && options.acpxAgent === "pi" ? 45_000 : 10_000);
|
||||
}
|
||||
|
||||
function runnerProcessReconnectGraceMs(options: CapabilityRunnerdCodexTransportOptions): number | undefined {
|
||||
if (options.runnerProcessReconnectGraceMs === null) return undefined;
|
||||
return options.runnerProcessReconnectGraceMs ?? options.runnerReconnectGraceMs;
|
||||
}
|
||||
|
||||
function runnerCloseDeadlines(
|
||||
startedAtMs: number,
|
||||
graceMs: number,
|
||||
@@ -3494,6 +3499,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
},
|
||||
],
|
||||
});
|
||||
#handlerInstalled = false;
|
||||
readonly #handlerReady: Promise<void>;
|
||||
#resolveHandlerReady!: () => void;
|
||||
#core: DurablePrpControlPlane | null = null;
|
||||
#handle: RunnerProcessHandle | null = null;
|
||||
#adoptedRunnerMonitor: NodeJS.Timeout | null = null;
|
||||
@@ -3622,6 +3630,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
this.#failureSignal = new Promise<never>((_resolve, reject) => {
|
||||
this.#rejectFailureSignal = reject;
|
||||
});
|
||||
this.#handlerReady = new Promise<void>(resolve => {
|
||||
this.#resolveHandlerReady = resolve;
|
||||
});
|
||||
this.#closedSignal = new Promise<void>(resolve => {
|
||||
this.#resolveClosedSignal = resolve;
|
||||
});
|
||||
@@ -3997,6 +4008,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
|
||||
setServerRequestHandler(handler: CodexServerRequestHandler): void {
|
||||
this.#handler = handler;
|
||||
this.#handlerInstalled = true;
|
||||
this.#resolveHandlerReady();
|
||||
}
|
||||
|
||||
async #awaitWarmRunAttachmentReady(): Promise<void> {
|
||||
@@ -5074,7 +5087,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
|
||||
p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
|
||||
maxRuntimeMs: 0,
|
||||
reconnectGraceMs: this.options.runnerReconnectGraceMs,
|
||||
reconnectGraceMs: runnerProcessReconnectGraceMs(this.options),
|
||||
lifecyclePolicy: this.options.lifecyclePolicy,
|
||||
runnerBinaryPath,
|
||||
runnerVersion: runnerArtifact.version,
|
||||
@@ -5759,7 +5772,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES,
|
||||
p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES,
|
||||
maxRuntimeMs: 0,
|
||||
reconnectGraceMs: this.options.runnerReconnectGraceMs,
|
||||
reconnectGraceMs: runnerProcessReconnectGraceMs(this.options),
|
||||
lifecyclePolicy: this.options.lifecyclePolicy,
|
||||
runnerBinaryPath,
|
||||
runnerVersion: runnerArtifact.version,
|
||||
@@ -5915,6 +5928,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
: await Promise.race([admission.settled, this.#failureSignal]).catch(() => {
|
||||
throw new SemanticToolNotDispatchedError();
|
||||
});
|
||||
// An adopted Runner can authenticate and deliver queued tools before the
|
||||
// driver has finished reconstructing its session. Handler publication is
|
||||
// the dispatch barrier; the default RPC diagnostic is not a tool outcome.
|
||||
// The owning startup/transport lifetime bounds this wait, and the original
|
||||
// authority/epoch checks below still apply after it becomes ready.
|
||||
if (!this.#handlerInstalled) {
|
||||
await Promise.race([
|
||||
this.#handlerReady,
|
||||
this.#failureSignal,
|
||||
this.#closedSignal.then(() => { throw new SemanticToolNotDispatchedError(); }),
|
||||
]).catch(() => { throw new SemanticToolNotDispatchedError(); });
|
||||
}
|
||||
try {
|
||||
this.#throwIfFailed();
|
||||
} catch {
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { createServer } from "node:http";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
|
||||
import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js";
|
||||
import {
|
||||
createCapabilityRunnerdCodexTransport,
|
||||
defaultCapabilityRunnerdBinary,
|
||||
} from "./runnerd-codex-transport.js";
|
||||
|
||||
it.each([
|
||||
{ policy: "connection lease", processGraceMs: null, survives: true },
|
||||
{ policy: "explicit process deadline", processGraceMs: 500, survives: false },
|
||||
])("preserves the $policy policy across controller downtime beyond the connection recovery budget", async ({ processGraceMs, survives }) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-controller-outage-"));
|
||||
const server = createServer();
|
||||
let authority: DurablePrpControlPlane | null = null;
|
||||
server.on("upgrade", (request, socket, head) => {
|
||||
if (!authority) { socket.destroy(); return; }
|
||||
authority.handleUpgrade(request, socket, "/runner", head);
|
||||
});
|
||||
await new Promise<void>(resolveListen => server.listen(0, "127.0.0.1", resolveListen));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Missing outage test listener");
|
||||
const registration = async (next: DurablePrpControlPlane) => {
|
||||
authority = next;
|
||||
return {
|
||||
connectUrl: `ws://127.0.0.1:${address.port}/runner`,
|
||||
release: () => { if (authority === next) authority = null; },
|
||||
};
|
||||
};
|
||||
const shared = {
|
||||
runnerBinary: defaultCapabilityRunnerdBinary(),
|
||||
codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"),
|
||||
codexArgs: ["--state-file", join(root, "provider-state.json"), "--hold-first-durable-turn"],
|
||||
stateDirectory: root,
|
||||
prpIdentity: {
|
||||
runnerInstanceId: "runner-outage", environmentLeaseId: "lease-outage", runId: "run-outage",
|
||||
normalizedSessionId: "session-outage", turnId: "turn-outage", itemId: "item-outage",
|
||||
},
|
||||
// Scale the production 120-second reconnect budget down. The live process
|
||||
// must be bounded by its authenticated connection lease, independently of
|
||||
// how long a controller spends retrying a connection or booting its server.
|
||||
runnerReconnectGraceMs: 500,
|
||||
runnerProcessReconnectGraceMs: processGraceMs,
|
||||
controlPlaneRegistration: registration,
|
||||
};
|
||||
const first = createCapabilityRunnerdCodexTransport(shared);
|
||||
let adopted: ReturnType<typeof createCapabilityRunnerdCodexTransport> | undefined;
|
||||
let pid: number | null = null;
|
||||
try {
|
||||
const opened = await first.transport.request("thread/start", {
|
||||
cwd: root, dynamicTools: codexSemanticToolSpecs(),
|
||||
});
|
||||
await first.transport.request("turn/start", { input: [{ type: "text", text: "Keep this task active." }] });
|
||||
for await (const notification of first.transport.notifications()) {
|
||||
if (notification.method === "turn/started") break;
|
||||
}
|
||||
pid = first.evidence().runnerPid;
|
||||
const providerPid = first.evidence().codexPid;
|
||||
expect(pid).toEqual(expect.any(Number));
|
||||
expect(providerPid).toEqual(expect.any(Number));
|
||||
await writeFile(join(root, "saved-work.txt"), "completed work survives controller replacement");
|
||||
await first.detachControllerForRestart();
|
||||
// The deliberate fault is an offline controller, not a slow test assertion.
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 1_250));
|
||||
const runnerState = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8"));
|
||||
if (!survives) {
|
||||
expect(runnerState.recoverableFailure).toBe("transport_reconnect_grace_exceeded");
|
||||
await vi.waitFor(() => expect(() => process.kill(pid!, 0)).toThrow());
|
||||
return;
|
||||
}
|
||||
expect(runnerState.recoverableFailure).not.toBe("transport_reconnect_grace_exceeded");
|
||||
expect(() => process.kill(pid!, 0)).not.toThrow();
|
||||
expect(() => process.kill(providerPid!, 0)).not.toThrow();
|
||||
const duplicateLaunch = vi.fn(() => { throw new Error("Must adopt the existing runner"); });
|
||||
const thread = opened.thread as Record<string, unknown>;
|
||||
adopted = createCapabilityRunnerdCodexTransport({
|
||||
...shared,
|
||||
runnerReconnectGraceMs: 5_000,
|
||||
resumeDynamicTools: codexSemanticToolSpecs(),
|
||||
resumeProviderSession: { driverSessionId: String(thread.id), providerSessionId: String(thread.sessionId) },
|
||||
runnerProcessLauncher: duplicateLaunch,
|
||||
adoptExistingRunner: {
|
||||
pid: pid!, processGroupId: pid, startedAt: new Date().toISOString(),
|
||||
isAlive: () => { try { process.kill(pid!, 0); return true; } catch { return false; } },
|
||||
signal: signal => { try { process.kill(-pid!, signal); return true; } catch { return false; } },
|
||||
},
|
||||
});
|
||||
adopted.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] }));
|
||||
const restored = await adopted.transport.request("thread/read", {});
|
||||
expect(restored.thread).toMatchObject({ id: thread.id, sessionId: thread.sessionId });
|
||||
expect(adopted.evidence()).toMatchObject({ runnerPid: pid, codexPid: providerPid, runnerExited: false });
|
||||
expect(duplicateLaunch).not.toHaveBeenCalled();
|
||||
expect(await readFile(join(root, "saved-work.txt"), "utf8")).toBe("completed work survives controller replacement");
|
||||
// A real Stop still reaches the adopted owner; the lease-only disconnect
|
||||
// policy cannot turn a detached process into an uncancellable executor.
|
||||
await adopted.transport.close();
|
||||
expect(adopted.evidence().runnerExited).toBe(true);
|
||||
} finally {
|
||||
await first.transport.close().catch(() => undefined);
|
||||
await adopted?.transport.close().catch(() => undefined);
|
||||
if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* already stopped */ } }
|
||||
server.closeAllConnections();
|
||||
if (server.listening) await new Promise<void>(resolveClose => server.close(() => resolveClose()));
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 30_000);
|
||||
@@ -0,0 +1,54 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js";
|
||||
import type { CodexRpcServerRequest } from "../drivers/codex/app-server-transport.js";
|
||||
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
|
||||
|
||||
it.each(["install", "close", "detach"] as const)("holds an authenticated semantic call until handler readiness or %s", async mode => {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-handler-readiness-"));
|
||||
let core!: DurablePrpControlPlane;
|
||||
const bundle = createCapabilityRunnerdCodexTransport({
|
||||
runnerBinary: defaultCapabilityRunnerdBinary(),
|
||||
codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"),
|
||||
codexArgs: ["--state-file", join(root, "provider.json"), "--emit-tool-call"],
|
||||
stateDirectory: root,
|
||||
controlPlaneRegistration: async authority => {
|
||||
core = authority;
|
||||
await authority.start();
|
||||
return { release: () => undefined };
|
||||
},
|
||||
});
|
||||
const handler = vi.fn(async (_request: CodexRpcServerRequest) => ({ success: true, contentItems: [{ type: "inputText", text: "one retained result" }] }));
|
||||
let pid: number | null = null;
|
||||
try {
|
||||
await bundle.transport.request("thread/start", { cwd: root, dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] });
|
||||
await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Read the task." }] });
|
||||
pid = bundle.evidence().runnerPid;
|
||||
expect(bundle.evidence().runnerProcessGroupId).toBe(pid);
|
||||
await vi.waitFor(() => expect(core.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true));
|
||||
// The authenticated provider is already asking for a tool, but the driver
|
||||
// has not published its recovered session/handler yet. No error result may
|
||||
// be manufactured in this gap, and no external effect has been dispatched.
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 50));
|
||||
expect(core.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0);
|
||||
expect(core.semanticToolResultsSettled()).toBe(false);
|
||||
if (mode === "close") await bundle.transport.close();
|
||||
if (mode === "detach") await bundle.detachControllerForRestart();
|
||||
bundle.transport.setServerRequestHandler(handler);
|
||||
if (mode === "install") {
|
||||
await vi.waitFor(() => expect(handler).toHaveBeenCalledOnce());
|
||||
await vi.waitFor(() => expect(core.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(1));
|
||||
expect(core.store.state.commands.find(command => command.type === "semantic_tool.result")?.payload).toMatchObject({ isError: false });
|
||||
expect(handler.mock.calls[0]?.[0]).toMatchObject({ method: "item/tool/call", params: { tool: "get_task_context" } });
|
||||
} else {
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 50));
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
}
|
||||
} finally {
|
||||
await bundle.transport.close().catch(() => undefined);
|
||||
if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* owned child already stopped */ } }
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 20_000);
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06",
|
||||
"declaration": {
|
||||
"schema": "paperclip.acpx_profile_declaration.v1",
|
||||
"agent": "pi",
|
||||
"agentProfileVersion": 29,
|
||||
"acpxVersion": "0.13.1",
|
||||
"agentServerVersion": "0.0.33",
|
||||
"agentRuntimeVersion": "1.0.0",
|
||||
"nodeVersion": "24.21.0",
|
||||
"closure": {
|
||||
"darwin-arm64": "282022db10150c6632b3444df421342e7d534bdf5d5fb1097a2e79d0625a2bcf",
|
||||
"darwin-x64": "64e251e19009f755c0b04f73ce2138246faab71a961b0f13d75ebfcc34bef12e",
|
||||
"linux-x64": "713b1fdff42fb56a1518bdc084f181d70bee8ebadc3e4b1d76321ed9108c8410"
|
||||
},
|
||||
"wrapperSha256": "9d129b3d38772e93e97080aa6c4e574ac5df4e47ce5bc331a484a9fbf8188b36",
|
||||
"helperSha256": "41e0490b617da0d60e0c8ec58ef311236f945129d99f816cff6897f78da7f91d",
|
||||
"extensionSha256": "f5818573a355702388072b28926897db546a5cf346b5171f646582ce5f9102a4",
|
||||
"agentFilesBinding": "registered-runtime-context-v1",
|
||||
"nativeQuestions": "select-confirm-input-editor-v4-root-field-types",
|
||||
"pathPolicy": "sdk-normalization-v1",
|
||||
"mcpToolNames": "provider-safe-exact-source-v1",
|
||||
"acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1",
|
||||
"nativeAssistantMessages": "pi1-rpc-deltas-v3-bound-message-and-tool-receipts",
|
||||
"nativeNotices": "session-extension-v2-bounded-runtime-failure",
|
||||
"messageProjectionSha256": "17d1e91b948dfa6a56e1905645add52b502041ad843c8d50a13b07956f22aadd",
|
||||
"noticeProjectionSha256": "014485adc690998e395d334ea3ae67f72be6060a2fc78a1185df7b8d1da28fbf",
|
||||
"sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1",
|
||||
"nativeInputDisposition": "queued-only-v1",
|
||||
"cacheWarming": "disabled-owned-decision-v1",
|
||||
"thinkingLevel": {
|
||||
"supported": [
|
||||
"off",
|
||||
"low",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"newConfigurationDefault": "low",
|
||||
"admission": "explicit-native-effective-before-prompt-v1",
|
||||
"recovery": "mode-bound-v1"
|
||||
},
|
||||
"modelSelection": "caller-selected-native-verified-v1",
|
||||
"providerConfigurationSourceSha256": "75a9a13f13f224dc092bd8ebf49e8747162172943761e44476c8eb18d80c52d4",
|
||||
"credentialEnvironmentSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161",
|
||||
"runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea",
|
||||
"recoveryIdentitySourceSha256": "d2c9fc161fcc135fb9760110bfa3d984ee2bb38d87fa5ce7cb7418a49dd89e66",
|
||||
"nativeCancellation": "acknowledged-cancelled-terminal-preserves-usage-v1",
|
||||
"dependencySecurityPatchSha256": "5273a195b952f233336122842faaf650193c04f36b9359d20d4d63079b0ce814",
|
||||
"openRouterAttributionSourceSha256": "1211191529d6073b733b311c3d69cada4973eede0f3e540c46e261f78e0b90f9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"commandDigest": "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40",
|
||||
"declaration": {
|
||||
"schema": "paperclip.acpx_profile_declaration.v1",
|
||||
"agent": "copilot",
|
||||
"agentProfileVersion": 59,
|
||||
"acpxVersion": "0.13.1",
|
||||
"agentServerVersion": "1.0.88",
|
||||
"protocolPolicyRevision": "copilot-agent-manual-v1",
|
||||
"acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1",
|
||||
"policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3",
|
||||
"distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107",
|
||||
"agentFilesBinding": "registered-runtime-context-v1",
|
||||
"systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1",
|
||||
"sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1",
|
||||
"innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a",
|
||||
"upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0",
|
||||
"patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679",
|
||||
"messageIdentityContract": "copilot-native-message-id-v1",
|
||||
"ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1",
|
||||
"permissionContextContract": "copilot-edit-permission-context-v1",
|
||||
"permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b",
|
||||
"permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668",
|
||||
"permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9",
|
||||
"permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f",
|
||||
"permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4",
|
||||
"semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2",
|
||||
"semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791",
|
||||
"semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5",
|
||||
"toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706",
|
||||
"semanticNormalizedInputContract": "validated-forwarded-input-commit-v1",
|
||||
"semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629",
|
||||
"semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103",
|
||||
"semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975",
|
||||
"semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e",
|
||||
"semanticCompletionContractSourceSha256": "3343df6362252d44d12b079ff78c8f6539d1544f3ce5ec36ff36db70ab0fac31",
|
||||
"semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9",
|
||||
"semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603",
|
||||
"semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236",
|
||||
"semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1",
|
||||
"displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877",
|
||||
"displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e",
|
||||
"configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff",
|
||||
"configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994",
|
||||
"profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282",
|
||||
"profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571",
|
||||
"profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88",
|
||||
"attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2",
|
||||
"singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f",
|
||||
"nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce",
|
||||
"nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842",
|
||||
"nativeRuntimePromptsSourceSha256": "103b83066da72434602e9e461e6278dfb40bb993d2f3b49b2e93df81bbb08b72",
|
||||
"nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90",
|
||||
"harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412",
|
||||
"harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249",
|
||||
"harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94",
|
||||
"transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523",
|
||||
"nativeBackendSourceSha256": "972c08e6e320d1384221e180391fe817ea1ba632db687201e457d64e68ef4360",
|
||||
"nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44",
|
||||
"nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a",
|
||||
"providerLaunchSourceSha256": "d4f4a69baeeb2f3b2f350840f81f711e40f08588c28ea28039748dde90302a51",
|
||||
"launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296",
|
||||
"runtimeRequestBridgeSourceSha256": "7497083b6cf1fc0268187aff948352f3bce6e054ac6e845005846473efdaff15",
|
||||
"runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295",
|
||||
"runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211",
|
||||
"guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f",
|
||||
"nativeExecutionSourceSha256": "fc7c6568feb0d8d709e9baa51839420c9b90336b4865a9b8405aba06e4d1e9c4",
|
||||
"profileCompatibilitySourceSha256": "6a26b102b78f282137ff93b0f75eb8788c603fc177802c58715c5ad7b2f3b9e3",
|
||||
"installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d",
|
||||
"assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c",
|
||||
"assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f",
|
||||
"credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161",
|
||||
"sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80",
|
||||
"profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456",
|
||||
"metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727",
|
||||
"permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1",
|
||||
"semanticContinuationSourceSha256": "d248e47a0fe7f2536d1e64cea0b0b410f34104800cd643d995b4473104ecc8e0",
|
||||
"runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec",
|
||||
"runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8",
|
||||
"runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea",
|
||||
"cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca",
|
||||
"agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba",
|
||||
"nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b"
|
||||
}
|
||||
}
|
||||
@@ -52,8 +52,8 @@ assert.equal(JSON.parse(await readFile(join(server, 'package.json'), 'utf8')).na
|
||||
const { resolvePaperclipRunnerBinary } = await import(pathToFileURL(join(server, 'dist/services/native-runtime/native-codex-runner.js')));
|
||||
const { createCapabilityRunnerdCodexTransport } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/live/runnerd-codex-transport.js')));
|
||||
const { QUALIFIED_ACPX_PROFILES } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/drivers/acpx/qualified-profiles.js')));
|
||||
assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 28);
|
||||
assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c');
|
||||
assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 29);
|
||||
assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06');
|
||||
assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'qualificationModel'), false);
|
||||
assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'reportedModelId'), false);
|
||||
const daemon = resolvePaperclipRunnerBinary();
|
||||
|
||||
@@ -11999,7 +11999,8 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
claudeEffort: "claudeEffort" in provider ? provider.claudeEffort : undefined,
|
||||
acpxPermissionModePinned: true } : {}),
|
||||
...(provider.kind === "opencode" ? { opencodePermissionMode: provider.permissionMode } : {}),
|
||||
runnerReconnectGraceMs: 120_000, lifecyclePolicy: input.execution.session.lifecyclePolicy,
|
||||
runnerReconnectGraceMs: 120_000, runnerProcessReconnectGraceMs: null,
|
||||
lifecyclePolicy: input.execution.session.lifecyclePolicy,
|
||||
runtimeContext: "runtimeContext" in input.execution ? input.execution.runtimeContext : null,
|
||||
externallySandboxed: true, baseInstructions: recovery?.baseInstructions,
|
||||
resumeDynamicTools: dynamicTools, resumeCompletionContract: { revision: input.execution.completionContract.contract.revision,
|
||||
@@ -13494,6 +13495,10 @@ async function createRunnerdBackendWithinSessionClaim(
|
||||
? resolveSourceCodexHome(resolveNativeProviderEnvironment(input.execution.provider, runnerEnvironment))
|
||||
: undefined,
|
||||
runnerReconnectGraceMs: remoteTarget ? 120_000 : undefined,
|
||||
// A controller's bounded reconnect attempt is not the remote owner's
|
||||
// lifetime. Retain it through restarts until its authenticated lease
|
||||
// expires or explicit Stop/suspension proves that execution ended.
|
||||
runnerProcessReconnectGraceMs: remoteTarget ? null : undefined,
|
||||
environment: effectiveRunnerEnvironment,
|
||||
observability: input.observability,
|
||||
onDiagnostic: (message) => {
|
||||
|
||||
@@ -16,7 +16,7 @@ exact execution ID explicitly.
|
||||
| Decide on an installed service action | `service-approve`, `service-decline` | Assign an authenticated local MCP fixture with Ask first; match its tool action and connection ID; no provider call before approval; exactly one after approval and a verified document; none after decline. |
|
||||
| Decline a new connection | `connection-decline` | Start without service connections; match a Notion connection intent; click Not now; verify the saved rejection, no new connection or repeated request, and an explanation followed by Done. |
|
||||
| Request an email address | `agentmail-setup` | Enable Chat connectors, ask for an email address, and require a durable AgentMail card for the requesting agent and user. Reload and verify one password field, the direct API-key link, and no access selectors or modal. Decline and verify no connection or repeated request. |
|
||||
| Continue work after a controller restart | `recover-controller` | Observe saved source, persist a user message, restart the isolated controller, and independently test the delivered result. |
|
||||
| Continue work after a controller restart | `recover-controller` | Observe saved source, persist a user message, restart the isolated controller, and independently test the delivered result. Native Daytona cases keep the controller offline for 125 seconds before restarting. |
|
||||
| Stop work and change direction | `stop-redirect` | Click Stop, send one new request, reload, observe exactly one stored user message and the new answer, and reach Done. |
|
||||
| Create and edit a company skill | `create-skill-studio` | Create one skill through the runner, verify its persisted library entry and activity-feed card, open Skill Studio, save an edit, and verify the edit after returning. Local Codex, local ACPX Claude, and warm Daytona cells are explicit. |
|
||||
|
||||
@@ -47,8 +47,10 @@ restart, a verified first download supplies the persistence checkpoint; the
|
||||
controller is interrupted during a subsequent revision with another queued
|
||||
requirement.
|
||||
|
||||
The native Codex Daytona restart cell now requires a completed native tool
|
||||
receipt while that revision is still running. Before and after restart it pins
|
||||
The native Codex and ACPX Daytona restart cells require a completed native tool
|
||||
receipt while that revision is still running. They keep the controller offline
|
||||
for 125 seconds after its health listener stops, then start its replacement.
|
||||
Before and after restart each pins
|
||||
the same run, normalized session, Runner identity, PID/start time, and active
|
||||
sandbox lease. The original receipt prefix must remain unchanged. A run that
|
||||
ends before either active observation leaves the fault boundary unexercised;
|
||||
@@ -56,7 +58,10 @@ it cannot pass from the final ZIP alone. The cell no longer exempts an original
|
||||
Codex remote run that fails with `process_lost` and is replaced.
|
||||
|
||||
Completion requires one original provider turn and control-plane terminal,
|
||||
the same native receipt prefix, no duplicate completed tool execution IDs,
|
||||
the same native receipt prefix, no failed native tool receipts, and consistent
|
||||
results for every stable completed tool identity. Reattached activity can
|
||||
redeliver an identical start or tool receipt; changed content or a second
|
||||
submission fails. It also requires
|
||||
a committed run and successful `workspace_finalize` receipt, and no pending
|
||||
workspace operation or recovery. Existing independent ZIP and preserved-download
|
||||
checks still apply. These receipt checks do not prove that arbitrary external
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
# Paid runner full-stack E2E
|
||||
|
||||
The explicit-only `daytona-restart-continuity` suite runs the existing
|
||||
`recover-controller` journey with the recorded Codex and Claude staging models
|
||||
on fresh Daytona fixtures. After a completed task tool in an active revision
|
||||
turn, it keeps the owned development controller offline for 125 seconds, beyond
|
||||
the controller's two-minute reconnect budget. The existing independent oracle
|
||||
requires the original process, native session, sandbox lease and receipt prefix,
|
||||
one completion and successful workspace finalization, plus verified downloaded
|
||||
work. It permits one attempt per cell and retains failed evidence. This is
|
||||
development API-key acceptance, distinct from staging subscription recovery.
|
||||
|
||||
The explicit-only [GitHub review bot journey](GITHUB-REVIEW-JOURNEY.md) tests
|
||||
attended bot setup and an automatic PR review against an attached deployment.
|
||||
Private account bindings and setup receipts live in the private eval repository.
|
||||
|
||||
@@ -53,13 +53,17 @@ changes.
|
||||
|
||||
## GitHub authorization
|
||||
|
||||
The Everyday remote Codex restart oracle reads only the existing public run,
|
||||
The Everyday remote native restart oracle reads only the existing public run,
|
||||
event, environment-lease, workspace-operation and recovery APIs. It does not
|
||||
execute commands in a provider sandbox or signal a provider process. Its new
|
||||
evidence projects closed receipt metadata and hashes the provider sandbox ID
|
||||
and full native event content. Provider text, tool arguments and command output
|
||||
do not enter these snapshots. Existing image admission, credentials, billing,
|
||||
retention and cleanup rules continue to apply.
|
||||
The isolated server's private restart request may delay its replacement by at
|
||||
most five minutes. This fault injection never stops a hosted instance or a
|
||||
provider sandbox. The explicit Daytona continuity suite has one attempt and
|
||||
1,000-cent company and agent hard stops.
|
||||
|
||||
Set `RUNNER_E2E_ALLOWED_ACTOR_IDS` to a non-empty JSON array of numeric GitHub
|
||||
user IDs. Keep the list equal to the owners of `.github/**` in
|
||||
|
||||
@@ -219,10 +219,10 @@ describe("runner E2E catalog", () => {
|
||||
expect(localIntegrityTasks).toHaveLength(2);
|
||||
expect(openRouterBreadthTasks).toHaveLength(3);
|
||||
expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
|
||||
1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2,
|
||||
2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2,
|
||||
]);
|
||||
expect(validateRunnerCatalog()).toHaveLength(765);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(765);
|
||||
expect(validateRunnerCatalog()).toHaveLength(767);
|
||||
expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(767);
|
||||
expect(
|
||||
runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
|
||||
).toHaveLength(48);
|
||||
|
||||
@@ -1155,6 +1155,17 @@ const repositoryModelProfiles = [
|
||||
];
|
||||
|
||||
export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
{
|
||||
id: "daytona-restart-continuity", label: "Daytona controller outage continuity", manualOnly: true,
|
||||
description: "Keep the original active native runner, task tools and saved files through more than two minutes without a controller, then commit the result without replay.",
|
||||
groups: [], profiles: repositoryModelProfiles.map(({ native }) => productionStoryProfile(native)),
|
||||
environments: runnerEnvironments.filter(environment => environment.id === "daytona"),
|
||||
tasks: everydayTasks.filter(task => task.id === "recover-controller")
|
||||
.map(task => ({ ...task, automaticRetryPolicy: "single_attempt" as const })), expectedMatrixSize: 2,
|
||||
definitionMetadata: { version: 2, scheduling: "explicit-only", controllerDowntimeMs: 125_000,
|
||||
maximumAttemptsPerCell: 1, budgetMonthlyCents: 1000, instructions: "production", grading: "same-active-process-lease-native-prefix-and-workspace-receipt",
|
||||
authentication: "api-key", stagingAuthentication: "subscription", stagingQualification: false },
|
||||
},
|
||||
{
|
||||
id: "codie-latency-diagnostic",
|
||||
label: "Native Codex latency diagnostic",
|
||||
@@ -1514,7 +1525,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]),
|
||||
]),
|
||||
],
|
||||
definitionMetadata: { version: 7, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only",
|
||||
definitionMetadata: { version: 9, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only",
|
||||
remoteControllerDowntimeMs: 125_000,
|
||||
remoteCodexRestart: "same-active-process-lease-native-prefix-and-workspace-receipt",
|
||||
procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } },
|
||||
},
|
||||
|
||||
@@ -89,7 +89,7 @@ interface Input {
|
||||
workspacePath: string;
|
||||
privateDir: string;
|
||||
deadlineAt: number;
|
||||
restart(): Promise<void>;
|
||||
restart(controllerDowntimeMs?: number): Promise<void>;
|
||||
observe(issue: StoryIssue, runs: StoryRun[]): void;
|
||||
capture(id: string, label: string, file: string): Promise<void>;
|
||||
evidence(name: string, value: unknown): Promise<void>;
|
||||
@@ -169,7 +169,8 @@ export async function runEverydayFlow(input: Input) {
|
||||
let project = fixtures.project;
|
||||
const caseId = execution.task.id;
|
||||
const requireRemoteRestart = caseId === "recover-controller" && execution.environment.id === "daytona"
|
||||
&& execution.profile.generation === "native" && execution.profile.provider === "codex";
|
||||
&& execution.profile.generation === "native" && (execution.profile.provider === "codex"
|
||||
|| execution.profile.provider === "acpx");
|
||||
const providerChoice = caseId === "provider-decline" || caseId === "provider-second";
|
||||
const nativeProviderCase = caseId === "provider-native";
|
||||
let aggregatorFixture: Awaited<ReturnType<typeof setupAggregatorFixture>> | undefined;
|
||||
@@ -949,7 +950,7 @@ export async function runEverydayFlow(input: Input) {
|
||||
ev.remoteRestart = { before };
|
||||
await input.evidence("remote-restart-before.json", before);
|
||||
} else ev.allowedInterruptedRuns.push(active.id);
|
||||
await input.restart();
|
||||
await input.restart(requireRemoteRestart ? 125_000 : undefined);
|
||||
note("controller-restarted");
|
||||
await openParent();
|
||||
if (requireRemoteRestart) {
|
||||
@@ -1643,7 +1644,9 @@ export async function runEverydayFlow(input: Input) {
|
||||
ev.remoteRestart!.final = final;
|
||||
return final;
|
||||
}, accept: final => gradeRemoteRestart({ ...ev.remoteRestart, final }).every(check => check.passed),
|
||||
reject: final => ["failed", "cancelled", "timed_out"].includes(final.run.status) ? "Original remote run failed after restart" : undefined });
|
||||
reject: final => ["failed", "cancelled", "timed_out"].includes(final.run.status) ? "Original remote run failed after restart"
|
||||
: final.run.status === "succeeded" && final.run.nativePhase === "committed"
|
||||
&& !gradeRemoteRestart({ ...ev.remoteRestart, final })[1]!.passed ? "Original remote turn completed with failed tools or conflicting/repeated execution receipts" : undefined });
|
||||
await input.evidence("remote-restart-final.json", final);
|
||||
}
|
||||
ev.checks.push(
|
||||
|
||||
@@ -111,11 +111,27 @@ describe("remote restart lifecycle evidence", () => {
|
||||
expect(gradeRemoteRestart(missingPrefix)[1]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["replacement", "second submission", "duplicate tool receipt", "missing terminal"])("rejects %s at completion", fault => {
|
||||
it("rejects failed native tool calls even when the original turn eventually succeeds", () => {
|
||||
const { evidence, event } = fixture();
|
||||
const rows = [...fixture().afterRows, event(6, "tool.execution.completed", { executionId: "failed-write", status: "failed", output: "private timeout" }),
|
||||
event(7, "turn.completed"), event(8, "run.result.accepted", {}, "control_plane", 1), event(9, "run.terminal", {}, "control_plane", 2)];
|
||||
evidence.final.events = projectRemoteRestartEvents(evidence.final.run, rows);
|
||||
expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts identical activity receipt redelivery without treating it as another submission", () => {
|
||||
const { evidence } = fixture();
|
||||
evidence.final.events.push({ ...evidence.final.events[1]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" },
|
||||
{ ...evidence.final.events[3]!, seq: 11, sourceSeq: 11, sourceEventId: "runner:run:11" });
|
||||
expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["replacement", "second submission", "changed tool receipt", "changed start receipt", "missing terminal"])("rejects %s at completion", fault => {
|
||||
const { evidence } = fixture();
|
||||
if (fault === "replacement") evidence.final.run.id = "replacement";
|
||||
if (fault === "second submission") evidence.final.events.push({ ...evidence.final.events[0]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" });
|
||||
if (fault === "duplicate tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" });
|
||||
if (fault === "changed tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10", payloadSha256: "0".repeat(64) });
|
||||
if (fault === "changed start receipt") evidence.final.events.push({ ...evidence.final.events[1]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10", payloadSha256: "0".repeat(64) });
|
||||
if (fault === "missing terminal") evidence.final.events.pop();
|
||||
expect(gradeRemoteRestart(evidence)[1]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
@@ -12,7 +12,8 @@ const digest = (value: unknown) => createHash("sha256").update(canonicalJson(val
|
||||
export interface RemoteRestartEvent {
|
||||
seq: number; sourceSeq: number; sourceEventId: string; eventType: string;
|
||||
sourceKind: "runner" | "control_plane"; turnId: string | null;
|
||||
toolExecutionId: string | null; toolCompleted: boolean; sha256: string;
|
||||
toolExecutionId: string | null; toolCompleted: boolean; toolFailed: boolean;
|
||||
payloadSha256: string; sha256: string;
|
||||
}
|
||||
export interface RemoteRestartBoundary {
|
||||
identity: { runId: string; companyId: string; agentId: string; issueId: string;
|
||||
@@ -78,6 +79,8 @@ export function projectRemoteRestartEvents(run: StoryRun, rows: readonly Row[]):
|
||||
turnId: id(event.turnId) ? event.turnId : null,
|
||||
toolExecutionId: id(payload.executionId) ? payload.executionId : null,
|
||||
toolCompleted: event.eventType === "tool.execution.completed" && payload.status === "completed",
|
||||
toolFailed: event.eventType === "tool.execution.completed" && payload.status !== "completed",
|
||||
payloadSha256: digest(payload),
|
||||
sha256: digest(event) };
|
||||
});
|
||||
}
|
||||
@@ -132,12 +135,23 @@ export function gradeRemoteRestart(evidence: RemoteRestartEvidence | undefined):
|
||||
const events = final?.events ?? [];
|
||||
const turnId = before?.events.find(event => event.eventType === "turn.started")?.turnId;
|
||||
const once = (kind: string, source: RemoteRestartEvent["sourceKind"]) => events.filter(event => event.eventType === kind && event.sourceKind === source).length === 1;
|
||||
const completedTools = events.filter(event => event.toolCompleted).map(event => event.toolExecutionId);
|
||||
// Reattachment can redeliver an already-observed activity notification. Its
|
||||
// stable identity and exact payload must agree; a new submission, changed
|
||||
// result or failed tool is not receipt replay and must never pass this cell.
|
||||
const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner");
|
||||
const tools = events.filter(event => event.toolCompleted);
|
||||
const toolResults = new Map<string, string>();
|
||||
let consistentToolResults = true;
|
||||
for (const event of tools) {
|
||||
if (!event.toolExecutionId || (toolResults.has(event.toolExecutionId) && toolResults.get(event.toolExecutionId) !== event.payloadSha256)) consistentToolResults = false;
|
||||
else toolResults.set(event.toolExecutionId, event.payloadSha256);
|
||||
}
|
||||
const nativeCompletion = finalIdentity && Boolean(before && after && retainsPrefix(after.events, events))
|
||||
&& once("turn.submitted", "runner") && once("turn.started", "runner") && once("turn.completed", "runner")
|
||||
&& once("turn.submitted", "runner") && starts.length > 0 && new Set(starts.map(event => event.payloadSha256)).size === 1
|
||||
&& once("turn.completed", "runner")
|
||||
&& once("run.result.accepted", "control_plane") && once("run.terminal", "control_plane")
|
||||
&& events.filter(event => event.eventType.startsWith("turn.")).every(event => event.turnId === turnId || (event.eventType === "turn.submitted" && event.turnId === null))
|
||||
&& new Set(completedTools).size === completedTools.length;
|
||||
&& consistentToolResults && !events.some(event => event.toolFailed);
|
||||
const operations = final?.operations ?? [];
|
||||
const saved = finalIdentity && run?.status === "succeeded" && run.nativePhase === "committed"
|
||||
&& run.resultJson?.finalizationPhase === "committed" && run.resultJson.workspaceFinalizeStatus === "succeeded"
|
||||
|
||||
@@ -171,7 +171,7 @@ export async function setupLiveFixtures(input: {
|
||||
: ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id)
|
||||
|| (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS
|
||||
: execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS
|
||||
: ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence"].includes(execution.suite.id) ? 1_000 : 0,
|
||||
: ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence", "daytona-restart-continuity"].includes(execution.suite.id) ? 1_000 : 0,
|
||||
});
|
||||
await input.onCompanyCreated?.(company.id);
|
||||
return company;
|
||||
@@ -365,7 +365,7 @@ export async function setupLiveFixtures(input: {
|
||||
setupProbe.defaultRunner = selected.defaultRunner;
|
||||
}
|
||||
}
|
||||
if (["stock-harness", "native-connection-guidance", "feedback-persistence"].includes(execution.suite.id)
|
||||
if (["stock-harness", "native-connection-guidance", "feedback-persistence", "daytona-restart-continuity"].includes(execution.suite.id)
|
||||
|| (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) {
|
||||
agent.budgetMonthlyCents = 1_000;
|
||||
}
|
||||
|
||||
@@ -264,6 +264,7 @@ async function restartIsolatedPaperclipServer(input: {
|
||||
deadlineAt: number;
|
||||
preserveRunner?: RestartRunnerIdentity;
|
||||
crashAfterResult?: FinalizationCrashBinding;
|
||||
controllerDowntimeMs?: number;
|
||||
}): Promise<void> {
|
||||
const {
|
||||
controlDirectory,
|
||||
@@ -274,7 +275,8 @@ async function restartIsolatedPaperclipServer(input: {
|
||||
const temporaryRequestPath = `${requestPath}.${process.pid}.${input.requestId}.tmp`;
|
||||
await writeFile(
|
||||
temporaryRequestPath,
|
||||
JSON.stringify({ requestId: input.requestId, ...(input.preserveRunner ? { preserveRunner: input.preserveRunner } : {}),
|
||||
JSON.stringify({ requestId: input.requestId, ...(input.controllerDowntimeMs !== undefined ? { controllerDowntimeMs: input.controllerDowntimeMs } : {}),
|
||||
...(input.preserveRunner ? { preserveRunner: input.preserveRunner } : {}),
|
||||
...(input.crashAfterResult ? { crashAfterResult: input.crashAfterResult } : {}) }),
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
@@ -1240,7 +1242,7 @@ for (const execution of executions) {
|
||||
const story = await runEverydayFlow({
|
||||
page, api, fixtures, execution, nonce, workspacePath, privateDir,
|
||||
deadlineAt: startedAtMs + deadlineMs,
|
||||
restart: () => restartIsolatedPaperclipServer({ api, requestId: `story-${nonce}`, deadlineAt: startedAtMs + deadlineMs }),
|
||||
restart: controllerDowntimeMs => restartIsolatedPaperclipServer({ api, requestId: `story-${nonce}`, deadlineAt: startedAtMs + deadlineMs, controllerDowntimeMs }),
|
||||
observe: (storyIssue, storyRuns) => { issue = storyIssue; selectedRuns = storyRuns; },
|
||||
capture: captureScreenshot,
|
||||
evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets),
|
||||
|
||||
@@ -293,6 +293,7 @@ async function waitForHealthToStop() {
|
||||
|
||||
interface RestartRequest {
|
||||
requestId: string;
|
||||
controllerDowntimeMs?: number;
|
||||
preserveRunner?: RestartRunnerIdentity;
|
||||
crashAfterResult?: FinalizationCrashBinding;
|
||||
}
|
||||
@@ -321,9 +322,15 @@ async function readRestartRequest(): Promise<RestartRequest | null> {
|
||||
return null;
|
||||
}
|
||||
const preserveRunner = (value as { preserveRunner?: unknown }).preserveRunner;
|
||||
const controllerDowntimeMs = (value as { controllerDowntimeMs?: unknown }).controllerDowntimeMs;
|
||||
if (controllerDowntimeMs !== undefined && (!Number.isSafeInteger(controllerDowntimeMs)
|
||||
|| Number(controllerDowntimeMs) < 0 || Number(controllerDowntimeMs) > 300_000)) {
|
||||
throw new Error("Controller downtime must be an integer between zero and five minutes");
|
||||
}
|
||||
const crashAfterResult = (value as { crashAfterResult?: unknown }).crashAfterResult;
|
||||
if (crashAfterResult !== undefined && preserveRunner !== undefined) throw new Error("A finalization crash cannot preserve a running provider");
|
||||
return { requestId, ...(preserveRunner === undefined ? {} : { preserveRunner: parseRestartRunnerIdentity(preserveRunner) }),
|
||||
return { requestId, ...(controllerDowntimeMs === undefined ? {} : { controllerDowntimeMs: Number(controllerDowntimeMs) }),
|
||||
...(preserveRunner === undefined ? {} : { preserveRunner: parseRestartRunnerIdentity(preserveRunner) }),
|
||||
...(crashAfterResult === undefined ? {} : { crashAfterResult: parseFinalizationCrashBinding(crashAfterResult) }) };
|
||||
}
|
||||
|
||||
@@ -348,7 +355,7 @@ async function writeRestartAck(
|
||||
await rename(temporaryAckPath, restartAckPath);
|
||||
}
|
||||
|
||||
async function restartServer({ requestId, preserveRunner, crashAfterResult }: RestartRequest) {
|
||||
async function restartServer({ requestId, preserveRunner, crashAfterResult, controllerDowntimeMs = 0 }: RestartRequest) {
|
||||
activeRestartRequestId = requestId;
|
||||
appendLog(`\nRestart request ${requestId}: stopping Paperclip\n`);
|
||||
const previous = child;
|
||||
@@ -365,6 +372,11 @@ async function restartServer({ requestId, preserveRunner, crashAfterResult }: Re
|
||||
// Do not mistake an orphaned old server for a healthy replacement. The port
|
||||
// must stop answering before the next launcher is allowed to start.
|
||||
await waitForHealthToStop();
|
||||
const downtimeDeadline = Date.now() + controllerDowntimeMs;
|
||||
while (Date.now() < downtimeDeadline) {
|
||||
if (shutdownRequested()) throw new Error("Wrapper shutdown interrupted controller downtime");
|
||||
await delay(Math.min(250, downtimeDeadline - Date.now()));
|
||||
}
|
||||
if (shutdownRequested()) {
|
||||
throw new Error("Wrapper shutdown interrupted the Paperclip restart");
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user