diff --git a/.dockerignore b/.dockerignore index 83ef4daeb0d..3bf8e79b7b1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -88,6 +88,7 @@ packages/grok-acp/bin !packages/paperclip-runner/test-fixtures/pi-acp/profile-v27-identity.json !packages/paperclip-runner/test-fixtures/pi-acp/profile-v28-identity.json +!packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json !packages/paperclip-runner/test/fixtures/copilot-profile-v23-identity.json diff --git a/doc/evals.md b/doc/evals.md index 2c0af7e5ec7..78304500307 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -1,5 +1,11 @@ # Paperclip evaluation guide +The explicit-only [Daytona restart continuity journey](../tests/runner-e2e/README.md) +extends the existing native restart oracle to a 125-second controller outage, +using the recorded Claude and Codex models in disposable development fixtures. +It requires same-process adoption, saved files and finalization without replay; +local regression checks and catalog discovery are separate from live acceptance. + The explicit-only [GitHub review bot journey](../tests/runner-e2e/GITHUB-REVIEW-JOURNEY.md) converts the Ponytail walkthrough into attended App setup, scoped installation, automatic PR review, and independent two-service checks. It supports attached diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index b4d63b5687f..52dbcac821e 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1676,6 +1676,30 @@ waiting for provider startup. An abrupt controller exit during session opening can then recover through the same exact process-identity checks as an active turn. Both graceful and hot restarts detach the old controller from native sessions. +Remote Runner processes use their authenticated connection lease as the bound +on disconnected lifetime. The controller's shorter connection-recovery budget +limits its own attempt; it must not terminate the remote owner while another +controller is starting or classifying runs. Lease expiry still shuts down the +provider, and exact identity, authentication, Stop and revocation checks remain +required before any recovery or new execution. +Native ACPX semantic tool calls wait on their durable PRP invocation across a +controller outage. Their provider-side MCP bridge does not impose a separate +request deadline that could make the provider retry an already queued write. +Runner lease expiry, turn cancellation and bridge closure still end those waits. +Ordinary bridge callers retain their request deadline. Repeating the same call +identity returns its one retained result; it does not execute the operation again. +The native Claude sidecar also sets its HTTP MCP client budget to one hour, +matching the Runner connection lease budget. This prevents Claude's default +60-second HTTP deadline from aborting an otherwise pending durable request. +The setting applies to that native Claude session's MCP client; other provider +sessions and ordinary bridge deadlines remain unchanged. Explicit cancellation +and Runner shutdown still take precedence over that client budget. +An adopted transport can receive authenticated queued semantic calls before +the driver publishes its recovered session and tool handler. It holds those +calls until the handler is installed, then rechecks the original authority and +turn epoch before dispatch. Transport failure, close or controller detachment +ends that wait without invoking a late handler. Missing startup wiring is not +reported to the provider as the result of an external operation. If shutdown begins while a provider session is opening, its eventual publication honors the pending detachment before dispatching a turn. Once detached, an old execution finalizer cannot suspend or signal the durable runner: the next diff --git a/packages/paperclip-runner/acpx-profiles.json b/packages/paperclip-runner/acpx-profiles.json index 6b28b8dad51..fdf1b06255c 100644 --- a/packages/paperclip-runner/acpx-profiles.json +++ b/packages/paperclip-runner/acpx-profiles.json @@ -14,12 +14,12 @@ "requiresProviderPolicy": false }, "pi": { - "agentProfileVersion": 28, + "agentProfileVersion": 29, "agentServerPackage": "pi-acp", "agentServerVersion": "0.0.33", "agentRuntimePackage": "@earendil-works/pi-coding-agent", "agentRuntimeVersion": "1.0.0", - "commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", + "commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06", "requiresProviderPolicy": true }, "cursor": { @@ -32,12 +32,12 @@ "requiresProviderPolicy": true }, "copilot": { - "agentProfileVersion": 58, + "agentProfileVersion": 59, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94", + "commandDigest": "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40", "requiresProviderPolicy": true }, "claude": { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs index c3f53e20b84..212852476cd 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_backend.rs @@ -3212,7 +3212,7 @@ mod tests { #[test] fn pi_profile_matches_published_identity_and_rejects_prior_profiles() { let published: Value = serde_json::from_str(include_str!( - "../../../../test-fixtures/pi-acp/profile-v28-identity.json" + "../../../../test-fixtures/pi-acp/profile-v29-identity.json" )) .unwrap(); let mut value = descriptor("codex"); @@ -3229,6 +3229,7 @@ mod tests { current.validate(&context()).unwrap(); for prior in [ + include_str!("../../../../test-fixtures/pi-acp/profile-v28-identity.json"), include_str!("../../../../test-fixtures/pi-acp/profile-v24-identity.json"), include_str!("../../../../test-fixtures/pi-acp/profile-v23-identity.json"), include_str!("../../../../test-fixtures/pi-acp/profile-v15-identity.json"), diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs index 128ec81306e..eb6c82cefcb 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs @@ -29,7 +29,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option { agent_runtime_package: Some("@earendil-works/pi-coding-agent"), agent_runtime_version: Some("1.0.0"), command_digest: - "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", + "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06", requires_provider_policy: true, }, "cursor" => AcpxReleaseProfile { @@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option { agent_runtime_package: None, agent_runtime_version: None, command_digest: - "sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94", + "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40", requires_provider_policy: true, }, "claude" => AcpxReleaseProfile { diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs index 872935a3a03..615a2e74839 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/native_provider_backend.rs @@ -17,7 +17,7 @@ use sha2::{Digest, Sha256}; const CODEX_ACPX_DIGEST: &str = "sha256:443491f4e814501a5dc47347e4510e84faeec9c9382e2d4e53bfe69386d5aabc"; const PI_ACPX_DIGEST: &str = - "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c"; + "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06"; fn temporary_directory(label: &str) -> PathBuf { let nonce = SystemTime::now() diff --git a/packages/paperclip-runner/scripts/copilot-profile-identity.mjs b/packages/paperclip-runner/scripts/copilot-profile-identity.mjs index 4d55c9f8367..7f43d83e8af 100644 --- a/packages/paperclip-runner/scripts/copilot-profile-identity.mjs +++ b/packages/paperclip-runner/scripts/copilot-profile-identity.mjs @@ -7,9 +7,9 @@ import { fileURLToPath } from "node:url"; import { resolve, dirname } from "node:path"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const fixture = resolve(root, "test/fixtures/copilot-profile-v58-identity.json"); +const fixture = resolve(root, "test/fixtures/copilot-profile-v59-identity.json"); const historical = JSON.parse(readFileSync(resolve(root, "test/fixtures/copilot-profile-v12-identity.json"), "utf8")); -const declaration = { ...historical.declaration, agentProfileVersion: 58 }; +const declaration = { ...historical.declaration, agentProfileVersion: 59 }; delete declaration.permissionIdentitySourceSha256; const sources = { displayEventSourceSha256: "src/drivers/acpx/copilot-events.ts", diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts index 62dc7e15321..28a0c5814b2 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -12,6 +12,7 @@ import { createHash, randomUUID } from "node:crypto"; import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; +import { createAcpxSidecarHostEnvironment, createSanitizedAcpxSpawnInput } from "../drivers/acpx/environment.js"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -93,6 +94,22 @@ afterEach(async () => { }); describe("qualified ACPX runtime sidecar", () => { + it("gives native Claude HTTP tools the Runner lease budget without changing sibling providers", () => { + const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8"); + const start = source.indexOf(" environment:"); + const end = source.indexOf(" expectedIdentity:", start); + expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); + const create = new Function("process", "params", "createAcpxSidecarHostEnvironment", `return ({ ${source.slice(start, end)} }).environment;`); + const original = { MCP_TOOL_TIMEOUT: "1000", ANTHROPIC_API_KEY: "selected-key", UNRELATED_SECRET: "private" }; + const claude = create({ env: original }, { agent: "claude", normalizedSessionId: "session" }, createAcpxSidecarHostEnvironment); + expect(createSanitizedAcpxSpawnInput(claude, "claude").env).toMatchObject({ MCP_TOOL_TIMEOUT: "3600000", ANTHROPIC_API_KEY: "selected-key" }); + expect(createSanitizedAcpxSpawnInput(claude, "claude").env).not.toHaveProperty("UNRELATED_SECRET"); + expect(original.MCP_TOOL_TIMEOUT).toBe("1000"); + const codex = create({ env: original }, { agent: "codex", normalizedSessionId: "session" }, createAcpxSidecarHostEnvironment); + expect(codex).toBe(original); + expect(createSanitizedAcpxSpawnInput(codex, "codex").env).not.toHaveProperty("MCP_TOOL_TIMEOUT"); + }); + it("projects the actual sidecar terminal diagnostic block without authoritative accounting", () => { const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8"); const start = source.indexOf(" try {", source.indexOf(" const usageAfter = await readSidecarHostStatusWithin(activeHost);")); @@ -279,6 +296,42 @@ describe("qualified ACPX runtime sidecar", () => { for (const agent of ["cursor", "codex", "pi"]) expect(create({ agent, tools: [] }, evidence, () => undefined).captureSemanticReceipt).toBeUndefined(); }); + it("keeps a native tool pending through controller downtime without a duplicate side effect", async () => { + const source = readFileSync(new URL("./acpx-runtime-sidecar.ts", import.meta.url), "utf8"); + const start = source.indexOf(" semanticTools: {"); + const end = source.indexOf(" onGoalUpdate:", start); + expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); + const create = new Function("params", "activeCopilotEvidence", "waitForTool", `return ({ ${source.slice(start, end)} }).semanticTools;`); + let admitted!: () => void, complete!: (value: unknown) => void; + const started = new Promise(resolve => { admitted = resolve; }); + const result = new Promise(resolve => { complete = resolve; }); + let signal: AbortSignal | undefined; + const handler = vi.fn((call: RunnerToolCall) => { signal = call.signal; admitted(); return result; }); + const bridge = await startRunnerToolBridge(create({ agent: "claude", tools: [{ name: "register_deliverable", inputSchema: { type: "object" } }] }, undefined, handler)); + const call = () => fetch(bridge.url, { method: "POST", headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: "saved-call", method: "tools/call", params: { name: "register_deliverable", arguments: {} } }) }).then(response => response.json()); + let first: Promise | undefined; + try { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + first = call(); + await started; + await vi.advanceTimersByTimeAsync(125_000); + expect(signal?.aborted).toBe(false); + // The same pending invocation can be read again after reconnect. It must + // share the original result rather than create another external write. + const duplicate = call(); + complete({ artifactId: "saved-artifact" }); + expect(await first).toMatchObject({ result: { content: [{ text: '{"artifactId":"saved-artifact"}' }] } }); + expect(await duplicate).toEqual(await first); + expect(handler).toHaveBeenCalledOnce(); + } finally { + complete({ artifactId: "saved-artifact" }); + await first?.catch(() => undefined); + await bridge.close(); + vi.useRealTimers(); + } + }); + it.each(["forwarded", "emit-failed"])("binds native v2 receipts to actual sidecar-normalized input: %s", async mode => { const raw = { reportedWorkDisposition: "done", summary: "Complete", evidence: [], verification: [], completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] } }; diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index 2b728b2027e..bc1ffb523c3 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -305,12 +305,22 @@ async function dispatch( providerPolicy: params.providerPolicy, systemInstructions: params.systemInstructions, runtimeContext: params.runtimeContext, - environment: createAcpxSidecarHostEnvironment(process.env, params.agent, params.normalizedSessionId), + environment: params.agent === "claude" ? { + ...createAcpxSidecarHostEnvironment(process.env, params.agent, params.normalizedSessionId), + // Claude's HTTP MCP client otherwise aborts at 60s, before the + // native Runner connection lease. Cancellation and Runner shutdown + // still revoke the pending call; this is not a new authority lease. + MCP_TOOL_TIMEOUT: String(60 * 60 * 1_000), + } : createAcpxSidecarHostEnvironment(process.env, params.agent, params.normalizedSessionId), expectedIdentity: params.expectedIdentity, assignedMcpTransport: "prp", semanticTools: { tools: params.tools, handler: waitForTool, + // These calls are durable PRP requests. A controller outage must not + // turn a queued write into a provider-visible failure and retry. + // Runner lease expiry, turn cancellation and bridge close own them. + timeoutMs: null, ...(params.agent === "copilot" ? { captureSemanticReceipt: () => activeCopilotEvidence?.captureSemanticReceipt() } : {}), }, onGoalUpdate: (goal) => { diff --git a/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts b/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts index 06e94abc778..7a55b20b843 100644 --- a/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts +++ b/packages/paperclip-runner/src/control-plane/runner-execution-driver.ts @@ -47,6 +47,10 @@ export interface RunnerExecutionDriver { }) => Promise>; /** Active-connection recovery budget. Omitted for the existing local mode. */ runnerReconnectGraceMs?: number; + /** Process lifetime while disconnected. Defaults to the active-connection + * recovery budget for compatibility. Null uses the authenticated connection + * lease as the lifetime bound, so controller downtime cannot kill its owner. */ + runnerProcessReconnectGraceMs?: number | null; /** * A verified Runner that outlived its controller. Adoption registers * the durable authority and waits for this exact process to reconnect; it diff --git a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts index 6df0af8afb3..8bdf9b3950d 100644 --- a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts @@ -87,9 +87,19 @@ vi.mock("./installation-integrity.js", () => ({ verifyNativeAcpxInstallation: vi describe("Copilot build-owned installation", () => { it("admits the current profile with environment launch preparation", () => { - expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(58); + expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(59); expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined(); - expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94"); + expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40"); + }); + + it("preserves v58 while binding the separate remote process reconnect policy in v59", () => { + const prior = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v58-identity.json", import.meta.url), "utf8")); + const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v59-identity.json", import.meta.url), "utf8")); + expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "credentialBindingSourceSha256", "nativeInstructionsSourceSha256", "nativeSessionRuntimeSourceSha256", "profileCompatibilitySourceSha256", "runtimeRequestBridgeSourceSha256", "semanticBridgeSourceSha256", "semanticSidecarSourceSha256", "transportContractSourceSha256", + ]); + expect(isSupportedAcpxProfileVersion("copilot", 58)).toBe(true); + expect(current.commandDigest).not.toBe(prior.commandDigest); }); it("preserves v57 while binding the shared Pi setup and launch repairs in v58", () => { diff --git a/packages/paperclip-runner/src/drivers/acpx/environment.test.ts b/packages/paperclip-runner/src/drivers/acpx/environment.test.ts index 3bc038a101e..3c7cdee2fe9 100644 --- a/packages/paperclip-runner/src/drivers/acpx/environment.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/environment.test.ts @@ -6,6 +6,13 @@ afterEach(() => vi.unstubAllEnvs()); import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxCredentialBinding, createAcpxSidecarHostEnvironment, createSanitizedAcpxSpawnInput } from "./environment.js"; describe("ACPX launch environment", () => { + it("retains an explicit Claude MCP timeout without inheriting an ambient or sibling setting", () => { + vi.stubEnv("MCP_TOOL_TIMEOUT", "999999"); + expect(createSanitizedAcpxSpawnInput(undefined, "claude").env).not.toHaveProperty("MCP_TOOL_TIMEOUT"); + expect(createSanitizedAcpxSpawnInput({ MCP_TOOL_TIMEOUT: "120000" }, "claude").env.MCP_TOOL_TIMEOUT).toBe("120000"); + expect(createSanitizedAcpxSpawnInput({ MCP_TOOL_TIMEOUT: "120000" }, "codex").env).not.toHaveProperty("MCP_TOOL_TIMEOUT"); + }); + it("excludes general IAM from the Pi sidecar and launch, including selected task values", () => { const iam = { AWS_ACCESS_KEY_ID: "general-key", AWS_SECRET_ACCESS_KEY: "general-secret", AWS_SESSION_TOKEN: "general-session" }; const source = { ...iam, ...configuredEnvironmentProjection({ ...iam, CUSTOM_SETTING: "allowed" }), AWS_BEARER_TOKEN_BEDROCK: "provider-key" }; diff --git a/packages/paperclip-runner/src/drivers/acpx/environment.ts b/packages/paperclip-runner/src/drivers/acpx/environment.ts index 1fb79afba46..f11aeacfa08 100644 --- a/packages/paperclip-runner/src/drivers/acpx/environment.ts +++ b/packages/paperclip-runner/src/drivers/acpx/environment.ts @@ -131,6 +131,7 @@ export function createSanitizedAcpxSpawnInput( ...(environment === undefined ? [] : ["PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY"]), ...credentialNames, ...(agent === "claude" ? CLAUDE_ROUTING_ENV_KEYS : []), + ...(agent === "claude" && environment !== undefined ? ["MCP_TOOL_TIMEOUT"] : []), ...(usesPreconfiguredHome(source) ? [...PRECONFIGURED_HOME_KEYS, "PAPERCLIP_RUNNER_HOME_MODE", "ACPX_CLAUDE_INCLUDE_USER_SETTINGS"] : []), ]); let retainedBytes = 0; diff --git a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts index ca6e53d79d7..36f2bdf29e1 100644 --- a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts +++ b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts @@ -22,12 +22,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = { "protocolVersion": 1, "acpxVersion": "0.13.1", "agent": "pi", - "agentProfileVersion": 28, + "agentProfileVersion": 29, "agentServerPackage": "pi-acp", "agentServerVersion": "0.0.33", "agentRuntimePackage": "@earendil-works/pi-coding-agent", "agentRuntimeVersion": "1.0.0", - "commandDigest": "sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c", + "commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06", "permissionPolicy": "interactive" }, "cursor": { @@ -48,12 +48,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = { "protocolVersion": 1, "acpxVersion": "0.13.1", "agent": "copilot", - "agentProfileVersion": 58, + "agentProfileVersion": 59, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:d490dacdab02051d24c84aae49ac6fb12b8226cf8a47b014ec53f3d3a8410a94", + "commandDigest": "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40", "permissionPolicy": "interactive" }, "claude": { diff --git a/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts b/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts index 7695e64219c..8d81addb3cf 100644 --- a/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts @@ -155,11 +155,20 @@ describe("Pi installation factory", () => { expect(current.commandDigest).not.toBe(prior.commandDigest); }); + it("preserves profile 28 while binding the Claude-only explicit MCP client budget source in profile 29", async () => { + const read = async (version: number) => JSON.parse(await readFile(new URL(`../../../test-fixtures/pi-acp/profile-v${version}-identity.json`, import.meta.url), "utf8")); + const prior = await read(28), current = await read(29); + expect(Object.keys(current.declaration).filter(key => JSON.stringify(current.declaration[key]) !== JSON.stringify(prior.declaration[key])).sort()) + .toEqual(["agentProfileVersion", "credentialEnvironmentSourceSha256"]); + expect(current.declaration.closure).toEqual(prior.declaration.closure); + expect(current.commandDigest).not.toBe(prior.commandDigest); + }); + it("binds the profile declaration to the reviewed patch and platform closure pins", async () => { const hash = (bytes: string | Uint8Array) => createHash("sha256").update(bytes).digest("hex"); const canonical = (value: any): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(",")}}` : JSON.stringify(value); - const identity = JSON.parse(await readFile(new URL("../../../test-fixtures/pi-acp/profile-v28-identity.json", import.meta.url), "utf8")); + const identity = JSON.parse(await readFile(new URL("../../../test-fixtures/pi-acp/profile-v29-identity.json", import.meta.url), "utf8")); const declaration = identity.declaration; expect(declaration.sharedRuntimeContract).toBe("paperclip.acpx-runtime-contract.v1"); expect(identity.commandDigest).toBe(`sha256:${hash(canonical(declaration))}`); @@ -190,21 +199,21 @@ describe("Pi installation factory", () => { it("rejects legacy profiles and tampered executable identities", () => { expect(() => assertPiInstallationProfile(candidate())).not.toThrow(); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 1 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 2 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 3 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 4 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 5 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 6 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 7 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 8 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 9 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 10 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 11 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 12 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 13 })).toThrow("version 28"); - expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 14 })).toThrow("version 28"); - for (const priorVersion of [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27]) expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: priorVersion })).toThrow("version 28"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 1 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 2 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 3 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 4 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 5 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 6 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 7 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 8 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 9 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 10 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 11 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 12 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 13 })).toThrow("version 29"); + expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: 14 })).toThrow("version 29"); + for (const priorVersion of [15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28]) expect(() => assertPiInstallationProfile({ ...candidate(), agentProfileVersion: priorVersion })).toThrow("version 29"); for (const changed of [{ agentServerVersion: "latest" }, { commandDigest: `sha256:${"0".repeat(64)}` }, { agentRuntimePackage: "ambient-pi" }]) { expect(() => assertPiInstallationProfile({ ...candidate(), ...changed })).toThrow("trusted declaration"); } diff --git a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts index 1119594c0ed..56a5878aa39 100644 --- a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts +++ b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts @@ -2,13 +2,13 @@ import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js"; /** Decodable historical profile revisions, not permission to launch them. * Exact current profile/package/digest admission is checked separately. */ -type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57; +type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58; type CurrentAcpxProfileVersion = typeof QUALIFIED_ACPX_PROFILE_DATA[keyof typeof QUALIFIED_ACPX_PROFILE_DATA]["agentProfileVersion"]; export type AcpxProfileVersion = HistoricalAcpxProfileVersion | CurrentAcpxProfileVersion; const historicalVersions: Readonly> = { - pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5], - grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57], + pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5], + grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58], cursor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], }; diff --git a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts index 152b8a9b6ea..4b212476c49 100644 --- a/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts +++ b/packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts @@ -246,7 +246,7 @@ describe("runner semantic MCP bridge", () => { }); }); - it("honors MCP cancellation after the handler starts", async () => { + it.each([60_000, null])("honors MCP cancellation after the handler starts (timeout=%s)", async timeoutMs => { let markStarted!: () => void; const started = new Promise((resolve) => { markStarted = resolve; @@ -254,7 +254,7 @@ describe("runner semantic MCP bridge", () => { const aborted = vi.fn(); const bridge = await startRunnerToolBridge({ tools: [tool("documents.read")], - timeoutMs: 60_000, + timeoutMs, handler: ({ signal }) => new Promise((_resolve, reject) => { signal.addEventListener("abort", () => { aborted(); @@ -291,6 +291,23 @@ describe("runner semantic MCP bridge", () => { expect(aborted).toHaveBeenCalledOnce(); }); + it("cancels a lease-owned native tool when its bridge closes", async () => { + let admitted!: () => void; + const started = new Promise(resolve => { admitted = resolve; }); + let signal: AbortSignal | undefined; + const bridge = await startRunnerToolBridge({ tools: [tool("documents.read")], timeoutMs: null, + handler: call => { signal = call.signal; admitted(); return new Promise(() => {}); }, + }); + bridges.push(bridge); + const pending = rpc(bridge, { id: "close-me", method: "tools/call", params: { name: "documents.read", arguments: {} } }); + await started; + await bridge.close(); + expect(signal?.aborted).toBe(true); + expect(await (await pending).json()).toMatchObject({ result: { isError: true, + content: [{ text: "Paperclip tool call cancelled" }], + } }); + }); + it("preserves successful mutation identity when the result is oversized", async () => { const result = { text: `snowman-${"☃".repeat(65 * 1024)}` }; const handler = vi.fn(async () => result); diff --git a/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts b/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts index 3a82d69d209..624569fa195 100644 --- a/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts +++ b/packages/paperclip-runner/src/drivers/runner-tool-bridge.ts @@ -45,7 +45,9 @@ export interface RunnerToolBridgeOptions { handler(call: RunnerToolCall): Promise; /** Copilot-only opt-in; capture the active invocation scope before dispatch. */ captureSemanticReceipt?: () => ((receipt: SemanticToolReceipt) => void) | undefined; - timeoutMs?: number; + /** Null delegates the wait to an owning durable runtime's lease and close. + * Ordinary callers retain the bounded request timeout. */ + timeoutMs?: number | null; privateToolTimeoutMs?: number; maxBodyBytes?: number; /** Internal test seam; production bounds headers and body reads to 30 seconds. */ @@ -143,7 +145,7 @@ export async function startRunnerToolBridge( controllers, handler: options.handler, captureSemanticReceipt: options.captureSemanticReceipt, - timeoutMs: positiveBoundedInteger( + timeoutMs: options.timeoutMs === null ? null : positiveBoundedInteger( options.timeoutMs, DEFAULT_TIMEOUT_MS, 1, @@ -214,7 +216,7 @@ async function handleRequest( controllers: Map; handler: RunnerToolBridgeOptions["handler"]; captureSemanticReceipt: RunnerToolBridgeOptions["captureSemanticReceipt"]; - timeoutMs: number; + timeoutMs: number | null; privateToolTimeoutMs: number; privateToolNames: ReadonlySet; maxBodyBytes: number; @@ -599,24 +601,24 @@ function closeServer(server: Server): Promise { function withCancellationAndTimeout( promise: Promise, controller: AbortController, - timeoutMs: number, + timeoutMs: number | null, ): Promise { return new Promise((resolve, reject) => { let settled = false; const finish = (callback: () => void) => { if (settled) return; settled = true; - clearTimeout(timer); + if (timer !== undefined) clearTimeout(timer); controller.signal.removeEventListener("abort", onAbort); callback(); }; const onAbort = () => finish(() => reject(new Error("Paperclip tool call cancelled"))); - const timer = setTimeout(() => { + const timer = timeoutMs === null ? undefined : setTimeout(() => { finish(() => reject(new Error("Paperclip tool call timed out"))); controller.abort(); }, timeoutMs); - timer.unref(); + timer?.unref(); controller.signal.addEventListener("abort", onAbort, { once: true }); promise.then( (value) => finish(() => resolve(value)), diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 21452870d1a..b70954e9aa8 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -857,6 +857,11 @@ function runnerCloseGraceMs(options: Pick; + #resolveHandlerReady!: () => void; #core: DurablePrpControlPlane | null = null; #handle: RunnerProcessHandle | null = null; #adoptedRunnerMonitor: NodeJS.Timeout | null = null; @@ -3622,6 +3630,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#failureSignal = new Promise((_resolve, reject) => { this.#rejectFailureSignal = reject; }); + this.#handlerReady = new Promise(resolve => { + this.#resolveHandlerReady = resolve; + }); this.#closedSignal = new Promise(resolve => { this.#resolveClosedSignal = resolve; }); @@ -3997,6 +4008,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { setServerRequestHandler(handler: CodexServerRequestHandler): void { this.#handler = handler; + this.#handlerInstalled = true; + this.#resolveHandlerReady(); } async #awaitWarmRunAttachmentReady(): Promise { @@ -5074,7 +5087,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 0, - reconnectGraceMs: this.options.runnerReconnectGraceMs, + reconnectGraceMs: runnerProcessReconnectGraceMs(this.options), lifecyclePolicy: this.options.lifecyclePolicy, runnerBinaryPath, runnerVersion: runnerArtifact.version, @@ -5759,7 +5772,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { maxOutboxBytes: RUNNERD_MAX_OUTBOX_BYTES, p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, maxRuntimeMs: 0, - reconnectGraceMs: this.options.runnerReconnectGraceMs, + reconnectGraceMs: runnerProcessReconnectGraceMs(this.options), lifecyclePolicy: this.options.lifecyclePolicy, runnerBinaryPath, runnerVersion: runnerArtifact.version, @@ -5915,6 +5928,18 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { : await Promise.race([admission.settled, this.#failureSignal]).catch(() => { throw new SemanticToolNotDispatchedError(); }); + // An adopted Runner can authenticate and deliver queued tools before the + // driver has finished reconstructing its session. Handler publication is + // the dispatch barrier; the default RPC diagnostic is not a tool outcome. + // The owning startup/transport lifetime bounds this wait, and the original + // authority/epoch checks below still apply after it becomes ready. + if (!this.#handlerInstalled) { + await Promise.race([ + this.#handlerReady, + this.#failureSignal, + this.#closedSignal.then(() => { throw new SemanticToolNotDispatchedError(); }), + ]).catch(() => { throw new SemanticToolNotDispatchedError(); }); + } try { this.#throwIfFailed(); } catch { diff --git a/packages/paperclip-runner/src/live/runnerd-controller-outage.test.ts b/packages/paperclip-runner/src/live/runnerd-controller-outage.test.ts new file mode 100644 index 00000000000..2988e4845a9 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-controller-outage.test.ts @@ -0,0 +1,110 @@ +import { createServer } from "node:http"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { + createCapabilityRunnerdCodexTransport, + defaultCapabilityRunnerdBinary, +} from "./runnerd-codex-transport.js"; + +it.each([ + { policy: "connection lease", processGraceMs: null, survives: true }, + { policy: "explicit process deadline", processGraceMs: 500, survives: false }, +])("preserves the $policy policy across controller downtime beyond the connection recovery budget", async ({ processGraceMs, survives }) => { + const root = await mkdtemp(join(tmpdir(), "runner-controller-outage-")); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + server.on("upgrade", (request, socket, head) => { + if (!authority) { socket.destroy(); return; } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing outage test listener"); + const registration = async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + release: () => { if (authority === next) authority = null; }, + }; + }; + const shared = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "provider-state.json"), "--hold-first-durable-turn"], + stateDirectory: root, + prpIdentity: { + runnerInstanceId: "runner-outage", environmentLeaseId: "lease-outage", runId: "run-outage", + normalizedSessionId: "session-outage", turnId: "turn-outage", itemId: "item-outage", + }, + // Scale the production 120-second reconnect budget down. The live process + // must be bounded by its authenticated connection lease, independently of + // how long a controller spends retrying a connection or booting its server. + runnerReconnectGraceMs: 500, + runnerProcessReconnectGraceMs: processGraceMs, + controlPlaneRegistration: registration, + }; + const first = createCapabilityRunnerdCodexTransport(shared); + let adopted: ReturnType | undefined; + let pid: number | null = null; + try { + const opened = await first.transport.request("thread/start", { + cwd: root, dynamicTools: codexSemanticToolSpecs(), + }); + await first.transport.request("turn/start", { input: [{ type: "text", text: "Keep this task active." }] }); + for await (const notification of first.transport.notifications()) { + if (notification.method === "turn/started") break; + } + pid = first.evidence().runnerPid; + const providerPid = first.evidence().codexPid; + expect(pid).toEqual(expect.any(Number)); + expect(providerPid).toEqual(expect.any(Number)); + await writeFile(join(root, "saved-work.txt"), "completed work survives controller replacement"); + await first.detachControllerForRestart(); + // The deliberate fault is an offline controller, not a slow test assertion. + await new Promise(resolveWait => setTimeout(resolveWait, 1_250)); + const runnerState = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8")); + if (!survives) { + expect(runnerState.recoverableFailure).toBe("transport_reconnect_grace_exceeded"); + await vi.waitFor(() => expect(() => process.kill(pid!, 0)).toThrow()); + return; + } + expect(runnerState.recoverableFailure).not.toBe("transport_reconnect_grace_exceeded"); + expect(() => process.kill(pid!, 0)).not.toThrow(); + expect(() => process.kill(providerPid!, 0)).not.toThrow(); + const duplicateLaunch = vi.fn(() => { throw new Error("Must adopt the existing runner"); }); + const thread = opened.thread as Record; + adopted = createCapabilityRunnerdCodexTransport({ + ...shared, + runnerReconnectGraceMs: 5_000, + resumeDynamicTools: codexSemanticToolSpecs(), + resumeProviderSession: { driverSessionId: String(thread.id), providerSessionId: String(thread.sessionId) }, + runnerProcessLauncher: duplicateLaunch, + adoptExistingRunner: { + pid: pid!, processGroupId: pid, startedAt: new Date().toISOString(), + isAlive: () => { try { process.kill(pid!, 0); return true; } catch { return false; } }, + signal: signal => { try { process.kill(-pid!, signal); return true; } catch { return false; } }, + }, + }); + adopted.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + const restored = await adopted.transport.request("thread/read", {}); + expect(restored.thread).toMatchObject({ id: thread.id, sessionId: thread.sessionId }); + expect(adopted.evidence()).toMatchObject({ runnerPid: pid, codexPid: providerPid, runnerExited: false }); + expect(duplicateLaunch).not.toHaveBeenCalled(); + expect(await readFile(join(root, "saved-work.txt"), "utf8")).toBe("completed work survives controller replacement"); + // A real Stop still reaches the adopted owner; the lease-only disconnect + // policy cannot turn a detached process into an uncancellable executor. + await adopted.transport.close(); + expect(adopted.evidence().runnerExited).toBe(true); + } finally { + await first.transport.close().catch(() => undefined); + await adopted?.transport.close().catch(() => undefined); + if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* already stopped */ } } + server.closeAllConnections(); + if (server.listening) await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-semantic-handler-readiness.test.ts b/packages/paperclip-runner/src/live/runnerd-semantic-handler-readiness.test.ts new file mode 100644 index 00000000000..f669c4d4aca --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-semantic-handler-readiness.test.ts @@ -0,0 +1,54 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import type { CodexRpcServerRequest } from "../drivers/codex/app-server-transport.js"; +import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js"; + +it.each(["install", "close", "detach"] as const)("holds an authenticated semantic call until handler readiness or %s", async mode => { + const root = await mkdtemp(join(tmpdir(), "runner-handler-readiness-")); + let core!: DurablePrpControlPlane; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "provider.json"), "--emit-tool-call"], + stateDirectory: root, + controlPlaneRegistration: async authority => { + core = authority; + await authority.start(); + return { release: () => undefined }; + }, + }); + const handler = vi.fn(async (_request: CodexRpcServerRequest) => ({ success: true, contentItems: [{ type: "inputText", text: "one retained result" }] })); + let pid: number | null = null; + try { + await bundle.transport.request("thread/start", { cwd: root, dynamicTools: [{ name: "get_task_context", description: "Read the task.", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] }); + await bundle.transport.request("turn/start", { input: [{ type: "text", text: "Read the task." }] }); + pid = bundle.evidence().runnerPid; + expect(bundle.evidence().runnerProcessGroupId).toBe(pid); + await vi.waitFor(() => expect(core.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true)); + // The authenticated provider is already asking for a tool, but the driver + // has not published its recovered session/handler yet. No error result may + // be manufactured in this gap, and no external effect has been dispatched. + await new Promise(resolveWait => setTimeout(resolveWait, 50)); + expect(core.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); + expect(core.semanticToolResultsSettled()).toBe(false); + if (mode === "close") await bundle.transport.close(); + if (mode === "detach") await bundle.detachControllerForRestart(); + bundle.transport.setServerRequestHandler(handler); + if (mode === "install") { + await vi.waitFor(() => expect(handler).toHaveBeenCalledOnce()); + await vi.waitFor(() => expect(core.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(1)); + expect(core.store.state.commands.find(command => command.type === "semantic_tool.result")?.payload).toMatchObject({ isError: false }); + expect(handler.mock.calls[0]?.[0]).toMatchObject({ method: "item/tool/call", params: { tool: "get_task_context" } }); + } else { + await new Promise(resolveWait => setTimeout(resolveWait, 50)); + expect(handler).not.toHaveBeenCalled(); + } + } finally { + await bundle.transport.close().catch(() => undefined); + if (pid) { try { process.kill(-pid, "SIGKILL"); } catch { /* owned child already stopped */ } } + await rm(root, { recursive: true, force: true }); + } +}, 20_000); diff --git a/packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json b/packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json new file mode 100644 index 00000000000..664c0ab00c7 --- /dev/null +++ b/packages/paperclip-runner/test-fixtures/pi-acp/profile-v29-identity.json @@ -0,0 +1,51 @@ +{ + "commandDigest": "sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "pi", + "agentProfileVersion": 29, + "acpxVersion": "0.13.1", + "agentServerVersion": "0.0.33", + "agentRuntimeVersion": "1.0.0", + "nodeVersion": "24.21.0", + "closure": { + "darwin-arm64": "282022db10150c6632b3444df421342e7d534bdf5d5fb1097a2e79d0625a2bcf", + "darwin-x64": "64e251e19009f755c0b04f73ce2138246faab71a961b0f13d75ebfcc34bef12e", + "linux-x64": "713b1fdff42fb56a1518bdc084f181d70bee8ebadc3e4b1d76321ed9108c8410" + }, + "wrapperSha256": "9d129b3d38772e93e97080aa6c4e574ac5df4e47ce5bc331a484a9fbf8188b36", + "helperSha256": "41e0490b617da0d60e0c8ec58ef311236f945129d99f816cff6897f78da7f91d", + "extensionSha256": "f5818573a355702388072b28926897db546a5cf346b5171f646582ce5f9102a4", + "agentFilesBinding": "registered-runtime-context-v1", + "nativeQuestions": "select-confirm-input-editor-v4-root-field-types", + "pathPolicy": "sdk-normalization-v1", + "mcpToolNames": "provider-safe-exact-source-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "nativeAssistantMessages": "pi1-rpc-deltas-v3-bound-message-and-tool-receipts", + "nativeNotices": "session-extension-v2-bounded-runtime-failure", + "messageProjectionSha256": "17d1e91b948dfa6a56e1905645add52b502041ad843c8d50a13b07956f22aadd", + "noticeProjectionSha256": "014485adc690998e395d334ea3ae67f72be6060a2fc78a1185df7b8d1da28fbf", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "nativeInputDisposition": "queued-only-v1", + "cacheWarming": "disabled-owned-decision-v1", + "thinkingLevel": { + "supported": [ + "off", + "low", + "high", + "max" + ], + "newConfigurationDefault": "low", + "admission": "explicit-native-effective-before-prompt-v1", + "recovery": "mode-bound-v1" + }, + "modelSelection": "caller-selected-native-verified-v1", + "providerConfigurationSourceSha256": "75a9a13f13f224dc092bd8ebf49e8747162172943761e44476c8eb18d80c52d4", + "credentialEnvironmentSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "recoveryIdentitySourceSha256": "d2c9fc161fcc135fb9760110bfa3d984ee2bb38d87fa5ce7cb7418a49dd89e66", + "nativeCancellation": "acknowledged-cancelled-terminal-preserves-usage-v1", + "dependencySecurityPatchSha256": "5273a195b952f233336122842faaf650193c04f36b9359d20d4d63079b0ce814", + "openRouterAttributionSourceSha256": "1211191529d6073b733b311c3d69cada4973eede0f3e540c46e261f78e0b90f9" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v59-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v59-identity.json new file mode 100644 index 00000000000..b087d433186 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v59-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:2f39ffe54a6af832d044b5098c31df9e08d4d27aee0ac675ba4b0c9da7445c40", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 59, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629", + "semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "3343df6362252d44d12b079ff78c8f6539d1544f3ce5ec36ff36db70ab0fac31", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "103b83066da72434602e9e461e6278dfb40bb993d2f3b49b2e93df81bbb08b72", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "972c08e6e320d1384221e180391fe817ea1ba632db687201e457d64e68ef4360", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a", + "providerLaunchSourceSha256": "d4f4a69baeeb2f3b2f350840f81f711e40f08588c28ea28039748dde90302a51", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "7497083b6cf1fc0268187aff948352f3bce6e054ac6e845005846473efdaff15", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "fc7c6568feb0d8d709e9baa51839420c9b90336b4865a9b8405aba06e4d1e9c4", + "profileCompatibilitySourceSha256": "6a26b102b78f282137ff93b0f75eb8788c603fc177802c58715c5ad7b2f3b9e3", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "d248e47a0fe7f2536d1e64cea0b0b410f34104800cd643d995b4473104ecc8e0", + "runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec", + "runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +} diff --git a/scripts/pi-public-install-probe.mjs b/scripts/pi-public-install-probe.mjs index 5e6672cc68f..945d8c902de 100644 --- a/scripts/pi-public-install-probe.mjs +++ b/scripts/pi-public-install-probe.mjs @@ -52,8 +52,8 @@ assert.equal(JSON.parse(await readFile(join(server, 'package.json'), 'utf8')).na const { resolvePaperclipRunnerBinary } = await import(pathToFileURL(join(server, 'dist/services/native-runtime/native-codex-runner.js'))); const { createCapabilityRunnerdCodexTransport } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/live/runnerd-codex-transport.js'))); const { QUALIFIED_ACPX_PROFILES } = await import(pathToFileURL(join(server, 'dist/vendor/paperclip-runner/drivers/acpx/qualified-profiles.js'))); -assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 28); -assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:4918a4d5472d66b22b1ec642b68b5c60a8092a39c84ec1ab1c9ad0289cb8c68c'); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, 29); +assert.equal(QUALIFIED_ACPX_PROFILES.pi.commandDigest, 'sha256:e80c6dc7bde12790371345ee5a12f0201d30941d26f1a780aec3d090545b2a06'); assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'qualificationModel'), false); assert.equal(Object.hasOwn(QUALIFIED_ACPX_PROFILES.pi, 'reportedModelId'), false); const daemon = resolvePaperclipRunnerBinary(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index ccf020f577b..62118309e9d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11999,7 +11999,8 @@ async function createRunnerdBackendWithinSessionClaim( claudeEffort: "claudeEffort" in provider ? provider.claudeEffort : undefined, acpxPermissionModePinned: true } : {}), ...(provider.kind === "opencode" ? { opencodePermissionMode: provider.permissionMode } : {}), - runnerReconnectGraceMs: 120_000, lifecyclePolicy: input.execution.session.lifecyclePolicy, + runnerReconnectGraceMs: 120_000, runnerProcessReconnectGraceMs: null, + lifecyclePolicy: input.execution.session.lifecyclePolicy, runtimeContext: "runtimeContext" in input.execution ? input.execution.runtimeContext : null, externallySandboxed: true, baseInstructions: recovery?.baseInstructions, resumeDynamicTools: dynamicTools, resumeCompletionContract: { revision: input.execution.completionContract.contract.revision, @@ -13494,6 +13495,10 @@ async function createRunnerdBackendWithinSessionClaim( ? resolveSourceCodexHome(resolveNativeProviderEnvironment(input.execution.provider, runnerEnvironment)) : undefined, runnerReconnectGraceMs: remoteTarget ? 120_000 : undefined, + // A controller's bounded reconnect attempt is not the remote owner's + // lifetime. Retain it through restarts until its authenticated lease + // expires or explicit Stop/suspension proves that execution ended. + runnerProcessReconnectGraceMs: remoteTarget ? null : undefined, environment: effectiveRunnerEnvironment, observability: input.observability, onDiagnostic: (message) => { diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index 567f365adeb..a9e395f53bb 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -16,7 +16,7 @@ exact execution ID explicitly. | Decide on an installed service action | `service-approve`, `service-decline` | Assign an authenticated local MCP fixture with Ask first; match its tool action and connection ID; no provider call before approval; exactly one after approval and a verified document; none after decline. | | Decline a new connection | `connection-decline` | Start without service connections; match a Notion connection intent; click Not now; verify the saved rejection, no new connection or repeated request, and an explanation followed by Done. | | Request an email address | `agentmail-setup` | Enable Chat connectors, ask for an email address, and require a durable AgentMail card for the requesting agent and user. Reload and verify one password field, the direct API-key link, and no access selectors or modal. Decline and verify no connection or repeated request. | -| Continue work after a controller restart | `recover-controller` | Observe saved source, persist a user message, restart the isolated controller, and independently test the delivered result. | +| Continue work after a controller restart | `recover-controller` | Observe saved source, persist a user message, restart the isolated controller, and independently test the delivered result. Native Daytona cases keep the controller offline for 125 seconds before restarting. | | Stop work and change direction | `stop-redirect` | Click Stop, send one new request, reload, observe exactly one stored user message and the new answer, and reach Done. | | Create and edit a company skill | `create-skill-studio` | Create one skill through the runner, verify its persisted library entry and activity-feed card, open Skill Studio, save an edit, and verify the edit after returning. Local Codex, local ACPX Claude, and warm Daytona cells are explicit. | @@ -47,8 +47,10 @@ restart, a verified first download supplies the persistence checkpoint; the controller is interrupted during a subsequent revision with another queued requirement. -The native Codex Daytona restart cell now requires a completed native tool -receipt while that revision is still running. Before and after restart it pins +The native Codex and ACPX Daytona restart cells require a completed native tool +receipt while that revision is still running. They keep the controller offline +for 125 seconds after its health listener stops, then start its replacement. +Before and after restart each pins the same run, normalized session, Runner identity, PID/start time, and active sandbox lease. The original receipt prefix must remain unchanged. A run that ends before either active observation leaves the fault boundary unexercised; @@ -56,7 +58,10 @@ it cannot pass from the final ZIP alone. The cell no longer exempts an original Codex remote run that fails with `process_lost` and is replaced. Completion requires one original provider turn and control-plane terminal, -the same native receipt prefix, no duplicate completed tool execution IDs, +the same native receipt prefix, no failed native tool receipts, and consistent +results for every stable completed tool identity. Reattached activity can +redeliver an identical start or tool receipt; changed content or a second +submission fails. It also requires a committed run and successful `workspace_finalize` receipt, and no pending workspace operation or recovery. Existing independent ZIP and preserved-download checks still apply. These receipt checks do not prove that arbitrary external diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 129f75c4e31..c6ed181c3e5 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1,5 +1,15 @@ # Paid runner full-stack E2E +The explicit-only `daytona-restart-continuity` suite runs the existing +`recover-controller` journey with the recorded Codex and Claude staging models +on fresh Daytona fixtures. After a completed task tool in an active revision +turn, it keeps the owned development controller offline for 125 seconds, beyond +the controller's two-minute reconnect budget. The existing independent oracle +requires the original process, native session, sandbox lease and receipt prefix, +one completion and successful workspace finalization, plus verified downloaded +work. It permits one attempt per cell and retains failed evidence. This is +development API-key acceptance, distinct from staging subscription recovery. + The explicit-only [GitHub review bot journey](GITHUB-REVIEW-JOURNEY.md) tests attended bot setup and an automatic PR review against an attached deployment. Private account bindings and setup receipts live in the private eval repository. diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index afbcf8dd494..555a6673b20 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -53,13 +53,17 @@ changes. ## GitHub authorization -The Everyday remote Codex restart oracle reads only the existing public run, +The Everyday remote native restart oracle reads only the existing public run, event, environment-lease, workspace-operation and recovery APIs. It does not execute commands in a provider sandbox or signal a provider process. Its new evidence projects closed receipt metadata and hashes the provider sandbox ID and full native event content. Provider text, tool arguments and command output do not enter these snapshots. Existing image admission, credentials, billing, retention and cleanup rules continue to apply. +The isolated server's private restart request may delay its replacement by at +most five minutes. This fault injection never stops a hosted instance or a +provider sandbox. The explicit Daytona continuity suite has one attempt and +1,000-cent company and agent hard stops. Set `RUNNER_E2E_ALLOWED_ACTOR_IDS` to a non-empty JSON array of numeric GitHub user IDs. Keep the list equal to the owners of `.github/**` in diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 4f36708a091..4b87cac677d 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -219,10 +219,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, + 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(765); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(765); + expect(validateRunnerCatalog()).toHaveLength(767); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(767); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 84c733fe035..a824e9b9dd4 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1155,6 +1155,17 @@ const repositoryModelProfiles = [ ]; export const runnerSuites: readonly RunnerSuiteFixture[] = [ + { + id: "daytona-restart-continuity", label: "Daytona controller outage continuity", manualOnly: true, + description: "Keep the original active native runner, task tools and saved files through more than two minutes without a controller, then commit the result without replay.", + groups: [], profiles: repositoryModelProfiles.map(({ native }) => productionStoryProfile(native)), + environments: runnerEnvironments.filter(environment => environment.id === "daytona"), + tasks: everydayTasks.filter(task => task.id === "recover-controller") + .map(task => ({ ...task, automaticRetryPolicy: "single_attempt" as const })), expectedMatrixSize: 2, + definitionMetadata: { version: 2, scheduling: "explicit-only", controllerDowntimeMs: 125_000, + maximumAttemptsPerCell: 1, budgetMonthlyCents: 1000, instructions: "production", grading: "same-active-process-lease-native-prefix-and-workspace-receipt", + authentication: "api-key", stagingAuthentication: "subscription", stagingQualification: false }, + }, { id: "codie-latency-diagnostic", label: "Native Codex latency diagnostic", @@ -1514,7 +1525,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]), ]), ], - definitionMetadata: { version: 7, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + definitionMetadata: { version: 9, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + remoteControllerDowntimeMs: 125_000, remoteCodexRestart: "same-active-process-lease-native-prefix-and-workspace-receipt", procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } }, }, diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index 3dc84a29769..bebc81eacff 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -89,7 +89,7 @@ interface Input { workspacePath: string; privateDir: string; deadlineAt: number; - restart(): Promise; + restart(controllerDowntimeMs?: number): Promise; observe(issue: StoryIssue, runs: StoryRun[]): void; capture(id: string, label: string, file: string): Promise; evidence(name: string, value: unknown): Promise; @@ -169,7 +169,8 @@ export async function runEverydayFlow(input: Input) { let project = fixtures.project; const caseId = execution.task.id; const requireRemoteRestart = caseId === "recover-controller" && execution.environment.id === "daytona" - && execution.profile.generation === "native" && execution.profile.provider === "codex"; + && execution.profile.generation === "native" && (execution.profile.provider === "codex" + || execution.profile.provider === "acpx"); const providerChoice = caseId === "provider-decline" || caseId === "provider-second"; const nativeProviderCase = caseId === "provider-native"; let aggregatorFixture: Awaited> | undefined; @@ -949,7 +950,7 @@ export async function runEverydayFlow(input: Input) { ev.remoteRestart = { before }; await input.evidence("remote-restart-before.json", before); } else ev.allowedInterruptedRuns.push(active.id); - await input.restart(); + await input.restart(requireRemoteRestart ? 125_000 : undefined); note("controller-restarted"); await openParent(); if (requireRemoteRestart) { @@ -1643,7 +1644,9 @@ export async function runEverydayFlow(input: Input) { ev.remoteRestart!.final = final; return final; }, accept: final => gradeRemoteRestart({ ...ev.remoteRestart, final }).every(check => check.passed), - reject: final => ["failed", "cancelled", "timed_out"].includes(final.run.status) ? "Original remote run failed after restart" : undefined }); + reject: final => ["failed", "cancelled", "timed_out"].includes(final.run.status) ? "Original remote run failed after restart" + : final.run.status === "succeeded" && final.run.nativePhase === "committed" + && !gradeRemoteRestart({ ...ev.remoteRestart, final })[1]!.passed ? "Original remote turn completed with failed tools or conflicting/repeated execution receipts" : undefined }); await input.evidence("remote-restart-final.json", final); } ev.checks.push( diff --git a/tests/runner-e2e/everyday-remote-restart.test.ts b/tests/runner-e2e/everyday-remote-restart.test.ts index 938c9b30bf6..c492e4f9e72 100644 --- a/tests/runner-e2e/everyday-remote-restart.test.ts +++ b/tests/runner-e2e/everyday-remote-restart.test.ts @@ -111,11 +111,27 @@ describe("remote restart lifecycle evidence", () => { expect(gradeRemoteRestart(missingPrefix)[1]!.passed).toBe(false); }); - it.each(["replacement", "second submission", "duplicate tool receipt", "missing terminal"])("rejects %s at completion", fault => { + it("rejects failed native tool calls even when the original turn eventually succeeds", () => { + const { evidence, event } = fixture(); + const rows = [...fixture().afterRows, event(6, "tool.execution.completed", { executionId: "failed-write", status: "failed", output: "private timeout" }), + event(7, "turn.completed"), event(8, "run.result.accepted", {}, "control_plane", 1), event(9, "run.terminal", {}, "control_plane", 2)]; + evidence.final.events = projectRemoteRestartEvents(evidence.final.run, rows); + expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(false); + }); + + it("accepts identical activity receipt redelivery without treating it as another submission", () => { + const { evidence } = fixture(); + evidence.final.events.push({ ...evidence.final.events[1]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" }, + { ...evidence.final.events[3]!, seq: 11, sourceSeq: 11, sourceEventId: "runner:run:11" }); + expect(gradeRemoteRestart(evidence).every(check => check.passed)).toBe(true); + }); + + it.each(["replacement", "second submission", "changed tool receipt", "changed start receipt", "missing terminal"])("rejects %s at completion", fault => { const { evidence } = fixture(); if (fault === "replacement") evidence.final.run.id = "replacement"; if (fault === "second submission") evidence.final.events.push({ ...evidence.final.events[0]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" }); - if (fault === "duplicate tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10" }); + if (fault === "changed tool receipt") evidence.final.events.push({ ...evidence.final.events[3]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10", payloadSha256: "0".repeat(64) }); + if (fault === "changed start receipt") evidence.final.events.push({ ...evidence.final.events[1]!, seq: 10, sourceSeq: 10, sourceEventId: "runner:run:10", payloadSha256: "0".repeat(64) }); if (fault === "missing terminal") evidence.final.events.pop(); expect(gradeRemoteRestart(evidence)[1]!.passed).toBe(false); }); diff --git a/tests/runner-e2e/everyday-remote-restart.ts b/tests/runner-e2e/everyday-remote-restart.ts index 59c1db429e8..9b50934eec6 100644 --- a/tests/runner-e2e/everyday-remote-restart.ts +++ b/tests/runner-e2e/everyday-remote-restart.ts @@ -12,7 +12,8 @@ const digest = (value: unknown) => createHash("sha256").update(canonicalJson(val export interface RemoteRestartEvent { seq: number; sourceSeq: number; sourceEventId: string; eventType: string; sourceKind: "runner" | "control_plane"; turnId: string | null; - toolExecutionId: string | null; toolCompleted: boolean; sha256: string; + toolExecutionId: string | null; toolCompleted: boolean; toolFailed: boolean; + payloadSha256: string; sha256: string; } export interface RemoteRestartBoundary { identity: { runId: string; companyId: string; agentId: string; issueId: string; @@ -78,6 +79,8 @@ export function projectRemoteRestartEvents(run: StoryRun, rows: readonly Row[]): turnId: id(event.turnId) ? event.turnId : null, toolExecutionId: id(payload.executionId) ? payload.executionId : null, toolCompleted: event.eventType === "tool.execution.completed" && payload.status === "completed", + toolFailed: event.eventType === "tool.execution.completed" && payload.status !== "completed", + payloadSha256: digest(payload), sha256: digest(event) }; }); } @@ -132,12 +135,23 @@ export function gradeRemoteRestart(evidence: RemoteRestartEvidence | undefined): const events = final?.events ?? []; const turnId = before?.events.find(event => event.eventType === "turn.started")?.turnId; const once = (kind: string, source: RemoteRestartEvent["sourceKind"]) => events.filter(event => event.eventType === kind && event.sourceKind === source).length === 1; - const completedTools = events.filter(event => event.toolCompleted).map(event => event.toolExecutionId); + // Reattachment can redeliver an already-observed activity notification. Its + // stable identity and exact payload must agree; a new submission, changed + // result or failed tool is not receipt replay and must never pass this cell. + const starts = events.filter(event => event.eventType === "turn.started" && event.sourceKind === "runner"); + const tools = events.filter(event => event.toolCompleted); + const toolResults = new Map(); + let consistentToolResults = true; + for (const event of tools) { + if (!event.toolExecutionId || (toolResults.has(event.toolExecutionId) && toolResults.get(event.toolExecutionId) !== event.payloadSha256)) consistentToolResults = false; + else toolResults.set(event.toolExecutionId, event.payloadSha256); + } const nativeCompletion = finalIdentity && Boolean(before && after && retainsPrefix(after.events, events)) - && once("turn.submitted", "runner") && once("turn.started", "runner") && once("turn.completed", "runner") + && once("turn.submitted", "runner") && starts.length > 0 && new Set(starts.map(event => event.payloadSha256)).size === 1 + && once("turn.completed", "runner") && once("run.result.accepted", "control_plane") && once("run.terminal", "control_plane") && events.filter(event => event.eventType.startsWith("turn.")).every(event => event.turnId === turnId || (event.eventType === "turn.submitted" && event.turnId === null)) - && new Set(completedTools).size === completedTools.length; + && consistentToolResults && !events.some(event => event.toolFailed); const operations = final?.operations ?? []; const saved = finalIdentity && run?.status === "succeeded" && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed" && run.resultJson.workspaceFinalizeStatus === "succeeded" diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index cc861c6232c..c5d27531b43 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -171,7 +171,7 @@ export async function setupLiveFixtures(input: { : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS - : ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence"].includes(execution.suite.id) ? 1_000 : 0, + : ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence", "daytona-restart-continuity"].includes(execution.suite.id) ? 1_000 : 0, }); await input.onCompanyCreated?.(company.id); return company; @@ -365,7 +365,7 @@ export async function setupLiveFixtures(input: { setupProbe.defaultRunner = selected.defaultRunner; } } - if (["stock-harness", "native-connection-guidance", "feedback-persistence"].includes(execution.suite.id) + if (["stock-harness", "native-connection-guidance", "feedback-persistence", "daytona-restart-continuity"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { agent.budgetMonthlyCents = 1_000; } diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 420d8c97cf5..8836e27dfe8 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -264,6 +264,7 @@ async function restartIsolatedPaperclipServer(input: { deadlineAt: number; preserveRunner?: RestartRunnerIdentity; crashAfterResult?: FinalizationCrashBinding; + controllerDowntimeMs?: number; }): Promise { const { controlDirectory, @@ -274,7 +275,8 @@ async function restartIsolatedPaperclipServer(input: { const temporaryRequestPath = `${requestPath}.${process.pid}.${input.requestId}.tmp`; await writeFile( temporaryRequestPath, - JSON.stringify({ requestId: input.requestId, ...(input.preserveRunner ? { preserveRunner: input.preserveRunner } : {}), + JSON.stringify({ requestId: input.requestId, ...(input.controllerDowntimeMs !== undefined ? { controllerDowntimeMs: input.controllerDowntimeMs } : {}), + ...(input.preserveRunner ? { preserveRunner: input.preserveRunner } : {}), ...(input.crashAfterResult ? { crashAfterResult: input.crashAfterResult } : {}) }), { encoding: "utf8", mode: 0o600 }, ); @@ -1240,7 +1242,7 @@ for (const execution of executions) { const story = await runEverydayFlow({ page, api, fixtures, execution, nonce, workspacePath, privateDir, deadlineAt: startedAtMs + deadlineMs, - restart: () => restartIsolatedPaperclipServer({ api, requestId: `story-${nonce}`, deadlineAt: startedAtMs + deadlineMs }), + restart: controllerDowntimeMs => restartIsolatedPaperclipServer({ api, requestId: `story-${nonce}`, deadlineAt: startedAtMs + deadlineMs, controllerDowntimeMs }), observe: (storyIssue, storyRuns) => { issue = storyIssue; selectedRuns = storyRuns; }, capture: captureScreenshot, evidence: (name, data) => writeSanitizedJson(snapshotsDir, name, data, secrets), diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 8956ac3f5aa..451ed91e034 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -293,6 +293,7 @@ async function waitForHealthToStop() { interface RestartRequest { requestId: string; + controllerDowntimeMs?: number; preserveRunner?: RestartRunnerIdentity; crashAfterResult?: FinalizationCrashBinding; } @@ -321,9 +322,15 @@ async function readRestartRequest(): Promise { return null; } const preserveRunner = (value as { preserveRunner?: unknown }).preserveRunner; + const controllerDowntimeMs = (value as { controllerDowntimeMs?: unknown }).controllerDowntimeMs; + if (controllerDowntimeMs !== undefined && (!Number.isSafeInteger(controllerDowntimeMs) + || Number(controllerDowntimeMs) < 0 || Number(controllerDowntimeMs) > 300_000)) { + throw new Error("Controller downtime must be an integer between zero and five minutes"); + } const crashAfterResult = (value as { crashAfterResult?: unknown }).crashAfterResult; if (crashAfterResult !== undefined && preserveRunner !== undefined) throw new Error("A finalization crash cannot preserve a running provider"); - return { requestId, ...(preserveRunner === undefined ? {} : { preserveRunner: parseRestartRunnerIdentity(preserveRunner) }), + return { requestId, ...(controllerDowntimeMs === undefined ? {} : { controllerDowntimeMs: Number(controllerDowntimeMs) }), + ...(preserveRunner === undefined ? {} : { preserveRunner: parseRestartRunnerIdentity(preserveRunner) }), ...(crashAfterResult === undefined ? {} : { crashAfterResult: parseFinalizationCrashBinding(crashAfterResult) }) }; } @@ -348,7 +355,7 @@ async function writeRestartAck( await rename(temporaryAckPath, restartAckPath); } -async function restartServer({ requestId, preserveRunner, crashAfterResult }: RestartRequest) { +async function restartServer({ requestId, preserveRunner, crashAfterResult, controllerDowntimeMs = 0 }: RestartRequest) { activeRestartRequestId = requestId; appendLog(`\nRestart request ${requestId}: stopping Paperclip\n`); const previous = child; @@ -365,6 +372,11 @@ async function restartServer({ requestId, preserveRunner, crashAfterResult }: Re // Do not mistake an orphaned old server for a healthy replacement. The port // must stop answering before the next launcher is allowed to start. await waitForHealthToStop(); + const downtimeDeadline = Date.now() + controllerDowntimeMs; + while (Date.now() < downtimeDeadline) { + if (shutdownRequested()) throw new Error("Wrapper shutdown interrupted controller downtime"); + await delay(Math.min(250, downtimeDeadline - Date.now())); + } if (shutdownRequested()) { throw new Error("Wrapper shutdown interrupted the Paperclip restart"); }