diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 25d01ce03f5..7bc0d578290 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1404,6 +1404,35 @@ same recovery generation and request. Every continuation carries the triggering request, ordered user direction, interaction outcomes, completed work, and explicit history coverage. A delivered message remains part of the task's request after its connection or approval resolves. The original title is background; a completed Notion read does not satisfy a later Gmail request. Author and source-trust boundaries survive rendering into both native and legacy prompts. Missing required history must be fetched before dispatch rather than described as complete. +### Remote runner process evidence + +A failed remote health request does not prove that runnerd stopped. Read-only +process checks retry transient transport failures up to three times within one +30-second budget. Missing or changed identity, an unknown process generation, +and exhausted health reads retain uncertainty. They do not authorize a signal, +replacement launch, or provider replay. Confirmed predecessor exit permits the +transport to attempt recovery within its existing reconnect deadline. Repeated +listener conflicts use that same deadline and port; they do not grant ownership +of another listener. + +The remote launcher retains a stop signal received while artifact staging is +pending. It checks cancellation again immediately before dispatch. A cancelled +launch that has not been dispatched settles without starting a process. If a +launch RPC is already in flight, signalling waits for its exact identity and +uses the existing marker, nonce, PID and runner identity checks. The signal +request is not a termination receipt. Monitor evidence still decides whether +an already dispatched process stopped. + +A runner receipt that its reconnect grace expired ends that incident. The +transport recognizes both the coded error and the runner's plain-text message. +It must not open another recovery window after that receipt. + +A bounded transport close can return before artifact staging settles. That +return does not prove cleanup completed or release the retained-state obligation. +Launch attempts, authenticated runner connections, provider process entries, +provider turns, and external effects are different counts. Transport recovery +alone does not prove the shared provider-attempt budget or authorize a new turn. + ### Interrupted conversation continuation Before provider dispatch, chat-control admission retries transient database lock diff --git a/doc/native-workspace-finalization-recovery.md b/doc/native-workspace-finalization-recovery.md index 5b1ffb5ce98..c4e8328f7d9 100644 --- a/doc/native-workspace-finalization-recovery.md +++ b/doc/native-workspace-finalization-recovery.md @@ -80,6 +80,32 @@ database statements. It does not disable the database timeout or reconnect a failed lock session. Publication still requires the physical connection and the exact durable owner token. A disabled timeout adds no periodic probe. +## Remote startup cancellation and retained uncertainty + +Remote artifact staging can outlast the transport's shutdown preparation wait. +The remote launcher keeps cancellation until staging settles and checks it +before launch dispatch. A cancelled, undispatched launch starts no process. A +stop during an in-flight launch waits for the exact process identity before the +existing guarded signal path can run. An accepted signal request does not prove +that the process stopped. + +Read-only health retries are bounded by three attempts and 30 seconds. An +unavailable health RPC, missing identity, or changed process generation retains +the allocation and its files. A healthy listener alone is not identity or stop +proof. Confirmed predecessor exit can permit bounded transport recovery, but an +unrelated listener must remain untouched. A close timeout does not certify +cleanup settlement, file restoration, or provider continuation admission. + +A runner receipt that its reconnect grace expired ends that incident. The +transport recognizes both the coded error and the runner's plain-text message. +It must not open another recovery window after that receipt. + +Verify file restoration independently of runner connection and provider activity. +A file hash receipt proves the restored bytes only. The normal continuation gates +must still check current ownership, pause, approval and budget state. Retained +unknown effects remain unknown; successful file recovery does not authorize +replay of a historical turn. + ## Automatic cloud recovery New Railway copyback receipts record the exact project, environment, service and diff --git a/packages/paperclip-runner/acpx-profiles.json b/packages/paperclip-runner/acpx-profiles.json index c65d9d1dd40..885c8d2a6be 100644 --- a/packages/paperclip-runner/acpx-profiles.json +++ b/packages/paperclip-runner/acpx-profiles.json @@ -32,12 +32,12 @@ "requiresProviderPolicy": true }, "copilot": { - "agentProfileVersion": 64, + "agentProfileVersion": 65, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:22e31f95ef972bcead98a4a707c9f33433558fd111e390e027a2bbb4346e112a", + "commandDigest": "sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47", "requiresProviderPolicy": true }, "claude": { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs index b804b781bc6..794c7a195bf 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/generated_acpx_profiles.rs @@ -47,7 +47,7 @@ pub(crate) fn acpx_release_profile(agent: &str) -> Option { agent_runtime_package: None, agent_runtime_version: None, command_digest: - "sha256:22e31f95ef972bcead98a4a707c9f33433558fd111e390e027a2bbb4346e112a", + "sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47", requires_provider_policy: true, }, "claude" => AcpxReleaseProfile { diff --git a/packages/paperclip-runner/scripts/copilot-profile-identity.mjs b/packages/paperclip-runner/scripts/copilot-profile-identity.mjs index 90033995a3f..c6f81d00497 100644 --- a/packages/paperclip-runner/scripts/copilot-profile-identity.mjs +++ b/packages/paperclip-runner/scripts/copilot-profile-identity.mjs @@ -7,9 +7,9 @@ import { fileURLToPath } from "node:url"; import { resolve, dirname } from "node:path"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const fixture = resolve(root, "test/fixtures/copilot-profile-v64-identity.json"); +const fixture = resolve(root, "test/fixtures/copilot-profile-v65-identity.json"); const historical = JSON.parse(readFileSync(resolve(root, "test/fixtures/copilot-profile-v12-identity.json"), "utf8")); -const declaration = { ...historical.declaration, agentProfileVersion: 64 }; +const declaration = { ...historical.declaration, agentProfileVersion: 65 }; delete declaration.permissionIdentitySourceSha256; const sources = { displayEventSourceSha256: "src/drivers/acpx/copilot-events.ts", diff --git a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts index 56c0f1308fe..808384a8b9b 100644 --- a/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts @@ -87,9 +87,38 @@ vi.mock("./installation-integrity.js", () => ({ verifyNativeAcpxInstallation: vi describe("Copilot build-owned installation", () => { it("admits the current profile with environment launch preparation", () => { - expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(64); + expect(QUALIFIED_ACPX_PROFILES.copilot.agentProfileVersion).toBe(65); expect(QUALIFIED_ACPX_PROFILES.copilot.qualificationStatus).toBeUndefined(); - expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:22e31f95ef972bcead98a4a707c9f33433558fd111e390e027a2bbb4346e112a"); + expect(QUALIFIED_ACPX_PROFILES.copilot.commandDigest).toBe("sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47"); + }); + + it("preserves published v64 while binding exhausted reconnect cancellation in v65", async () => { + const priorBytes = readFileSync(new URL("../../../test/fixtures/copilot-profile-v64-identity.json", import.meta.url)); + expect(createHash("sha256").update(priorBytes).digest("hex")).toBe("9ee015b4968d8df8d38364626d413f69f6c0e846de20fe1d5f72c7f1706837a9"); + const prior = JSON.parse(priorBytes.toString("utf8")); + const current = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v65-identity.json", import.meta.url), "utf8")); + expect(Object.keys(current.declaration).filter(key => current.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "profileCompatibilitySourceSha256", "runtimeRequestBridgeSourceSha256", + ]); + expect(Object.keys(current.declaration).sort()).toEqual(Object.keys(prior.declaration).sort()); + expect(isSupportedAcpxProfileVersion("copilot", 64)).toBe(true); + await expect(verifyCopilotInstallation({ ...QUALIFIED_ACPX_PROFILES.copilot, + agentProfileVersion: 64, commandDigest: prior.commandDigest })).rejects.toThrow("exact pinned"); + }); + + it("preserves the published startup cancellation candidate as historical v63 evidence", async () => { + const bytes = readFileSync(new URL("../../../test/fixtures/copilot-profile-v63-reconnect-candidate-historical-identity.json", import.meta.url)); + expect(createHash("sha256").update(bytes).digest("hex")).toBe("1ed0ac81863a588f3a2ff997b0e3f3bf0b64440454eb19282bdcd05d35650207"); + const candidate = JSON.parse(bytes.toString("utf8")); + const prior = JSON.parse(readFileSync(new URL("../../../test/fixtures/copilot-profile-v62-identity.json", import.meta.url), "utf8")); + expect(Object.keys(candidate.declaration).filter(key => candidate.declaration[key] !== prior.declaration[key]).sort()).toEqual([ + "agentProfileVersion", "profileCompatibilitySourceSha256", "runtimeRequestBridgeSourceSha256", + ]); + const sorted = Object.fromEntries(Object.entries(candidate.declaration).sort(([a], [b]) => a.localeCompare(b))); + expect(`sha256:${createHash("sha256").update(JSON.stringify(sorted)).digest("hex")}`).toBe(candidate.commandDigest); + expect(isSupportedAcpxProfileVersion("copilot", 63)).toBe(true); + await expect(verifyCopilotInstallation({ ...QUALIFIED_ACPX_PROFILES.copilot, + agentProfileVersion: 63, commandDigest: candidate.commandDigest })).rejects.toThrow("exact pinned"); }); it("preserves published v63 while binding delegated connection guidance in v64", async () => { diff --git a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts index 508f5a24c1d..ca6b08b1e7c 100644 --- a/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts +++ b/packages/paperclip-runner/src/drivers/acpx/generated-profiles.ts @@ -48,12 +48,12 @@ export const QUALIFIED_ACPX_PROFILE_DATA = { "protocolVersion": 1, "acpxVersion": "0.13.1", "agent": "copilot", - "agentProfileVersion": 64, + "agentProfileVersion": 65, "agentServerPackage": "@github/copilot", "agentServerVersion": "1.0.88", "agentRuntimePackage": null, "agentRuntimeVersion": null, - "commandDigest": "sha256:22e31f95ef972bcead98a4a707c9f33433558fd111e390e027a2bbb4346e112a", + "commandDigest": "sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47", "permissionPolicy": "interactive" }, "claude": { diff --git a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts index 2eb8b782321..08db0ba384d 100644 --- a/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts +++ b/packages/paperclip-runner/src/drivers/acpx/profile-compatibility.ts @@ -2,13 +2,13 @@ import { QUALIFIED_ACPX_PROFILE_DATA } from "./generated-profiles.js"; /** Decodable historical profile revisions, not permission to launch them. * Exact current profile/package/digest admission is checked separately. */ -type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63; +type HistoricalAcpxProfileVersion = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64; type CurrentAcpxProfileVersion = typeof QUALIFIED_ACPX_PROFILE_DATA[keyof typeof QUALIFIED_ACPX_PROFILE_DATA]["agentProfileVersion"]; export type AcpxProfileVersion = HistoricalAcpxProfileVersion | CurrentAcpxProfileVersion; const historicalVersions: Readonly> = { pi: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28], claude: [1, 2, 3, 4, 5], codex: [1, 2, 3, 4, 5], - grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63], + grok: [1, 2, 3, 4, 5], copilot: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64], cursor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], }; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index fb329070a4e..d2b054ef787 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -351,6 +351,51 @@ it.each(["resolved", "rejected"] as const)( 15_000, ); +it.each([ + "transport_reconnect_grace_exceeded", + "transport reconnect grace exceeded; durable state is preserved", +])("does not restart an exhausted reconnect incident: %s", async detail => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-exhausted-grace-")); + const children: ReturnType[] = []; + const exits: Promise[] = []; + const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { + cwd: spec.cwd, env: spec.environment, stdio: "ignore", + }); + children.push(child); + const completion = new Promise((resolveExit, rejectExit) => { + child.once("error", rejectExit); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: detail })); + }); + exits.push(completion); + return { child, completion, isExitConfirmed: () => child.exitCode !== null || child.signalCode !== null }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: process.execPath, + codexArgs: [fileURLToPath(new URL("../../test/fixtures/fake-durable-codex-app-server.mjs", import.meta.url)), join(directory, "fake-state.json")], stateDirectory: directory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 1_000, runnerProcessLauncher: launch, + }); + try { + await bundle.transport.request("thread/start", { cwd: directory }); + const failure = (async () => { + for await (const _event of bundle.transport.notifications()) {} + })().catch(error => error as Error); + children[0]!.kill("SIGKILL"); + // The runner's terminal receipt is the exhausted incident authority. + const result = await Promise.race([failure, new Promise(resolve => setTimeout(() => resolve("still active"), 2_000))]); + expect(result).toMatchObject({ message: expect.stringContaining("transport_reconnect_grace_exceeded") }); + expect(launch).toHaveBeenCalledTimes(1); + } finally { + const providerPid = bundle.evidence().codexPid; + if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } + await bundle.transport.detachControllerForRestart!(); + for (const child of children) child.kill("SIGKILL"); + await Promise.allSettled(exits); + await rm(directory, { recursive: true, force: true }); + } +}, 15_000); + it("preserves the cause when a replacement runner's health cannot be verified", async () => { const directory = await mkdtemp(join(tmpdir(), "runnerd-replacement-health-")); const probeError = Object.assign(new Error("remote recovery process_verification: probe_timeout"), { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 06903c8327a..aa4f781f06f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -7018,7 +7018,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ? "provider_frame_too_large" : /runner_ingress_bind_conflict/i.test(detail) ? "runner_ingress_bind_conflict" - : /transport_reconnect_grace_exceeded/i.test(detail) + : /transport[_ ]reconnect[_ ]grace[_ ]exceeded/i.test(detail) ? "transport_reconnect_grace_exceeded" : this.#startupComplete ? "native_runner_process_exited" diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v63-reconnect-candidate-historical-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v63-reconnect-candidate-historical-identity.json new file mode 100644 index 00000000000..0a6c463f55e --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v63-reconnect-candidate-historical-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:3ca9ffc81abda9733eadfcf61ed37cdc6fe80251de744f33bdc8be73ab3e9170", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 63, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629", + "semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "3343df6362252d44d12b079ff78c8f6539d1544f3ce5ec36ff36db70ab0fac31", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "103b83066da72434602e9e461e6278dfb40bb993d2f3b49b2e93df81bbb08b72", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "e7b12d4418c7a71f5f2474b046697d3009eae3613b284bec381df5e9965a1c37", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a", + "providerLaunchSourceSha256": "fa8db1c42c378898287c141d8cea9069166981e369c9b9c51422cb6177de5a2c", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "b8c0c38d45f7625c051387136f60fdc2ca1b83e7e58e92fc08360d057b97f95a", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "3a30d52bffad68c7f02349193be84761050eb99039fc0556c2b449891a36e918", + "profileCompatibilitySourceSha256": "5b00c3f78ad12321d48e66a90338af2d53d59dd17bb4b72b10ccdb4240b5a150", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "d248e47a0fe7f2536d1e64cea0b0b410f34104800cd643d995b4473104ecc8e0", + "runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec", + "runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +} diff --git a/packages/paperclip-runner/test/fixtures/copilot-profile-v65-identity.json b/packages/paperclip-runner/test/fixtures/copilot-profile-v65-identity.json new file mode 100644 index 00000000000..084c61a32b0 --- /dev/null +++ b/packages/paperclip-runner/test/fixtures/copilot-profile-v65-identity.json @@ -0,0 +1,85 @@ +{ + "commandDigest": "sha256:277894e850e07e6801cc24299c237513f58b41eb93c24f751d984fc4ddb04c47", + "declaration": { + "schema": "paperclip.acpx_profile_declaration.v1", + "agent": "copilot", + "agentProfileVersion": 65, + "acpxVersion": "0.13.1", + "agentServerVersion": "1.0.88", + "protocolPolicyRevision": "copilot-agent-manual-v1", + "acpxPatchSha256": "00d6af17216cb0cad902b48633eb19bf0ec6b05837a984cb24661fca593f09c1", + "policySha256": "a4be69be43db84052560420a3ca546a665c9165573c24d543eb33d144ba138a3", + "distributionSourceSha256": "c0a65e247fc47b263bd369a3d5ab505c4b323d4e6493be8a9ecbeb0e9dbb9107", + "agentFilesBinding": "registered-runtime-context-v1", + "systemInstructionDelivery": "COPILOT_HOME/copilot-instructions.md:replace-under-lifetime-lease-before-launch:v1", + "sharedRuntimeContract": "paperclip.acpx-runtime-contract.v1", + "innerDistributionSourceSha256": "9bb08347eed01f1ab44e065c8b06ead9be3a25973d6e6329dc4c0a099096279a", + "upstreamAppSha256": "7b48282a19b5b0814a0c96ad5e3a125173d792cc55f9525b2effea962f6063c0", + "patchedAppSha256": "61f8eebe0c30cc03b311c6818b2acb158a6d16ca58c8bfac5ebd42db095c2679", + "messageIdentityContract": "copilot-native-message-id-v1", + "ownedDistributionDelivery": "COPILOT_CLI_DIST_DIR:lease-owned-guarded-inner-distribution:v1", + "permissionContextContract": "copilot-edit-permission-context-v1", + "permissionContextSourceSha256": "de2d38d702b3c574161af0c0c04790fb6f6347916eb3916ce7a131a8c229121b", + "permissionAdapterSourceSha256": "ee5929ed2ee647ad0369465dee9d51a00b2dd28e1ff735b43e969ddc9c164668", + "permissionLocationsSourceSha256": "f39a81da395b6e23aa2a1a45572a28846e5e81bfd3d764d9a456f74a79c752e9", + "permissionRedactionSourceSha256": "ad6770c69269087951790e2e87e22ac088e3d432769adf17ebc865ed512e6c4f", + "permissionClassifierSourceSha256": "1161eabd82e26133e3b3db3ab567f65d52808b03f176e93497e02c8330349fb4", + "semanticToolReceiptContract": "paperclip.semantic_tool_receipt.v2", + "semanticReceiptSourceSha256": "b2795726b2234a663fec1a6ca11f0551ae69882fe9e82aa39686e952d036e791", + "semanticBridgeSourceSha256": "8764b8c9032fdc7d1415b01e75a9fc57c550e7a5e11da4bed7d9100082872ae5", + "toolEvidenceSourceSha256": "6eef5039afc75bb851eaf9cd82a6f099e4bd398ca2deb59dd66f5b2a81a7a706", + "semanticNormalizedInputContract": "validated-forwarded-input-commit-v1", + "semanticSidecarSourceSha256": "17361d1782c0dcebb2b7972635f45c890f2a4081f8bc0ef2d2276550e0b91629", + "semanticDirectDriverSourceSha256": "ff572638f2cb5d9f8249ff8e2f412311c1e3af209ef0bf7e5ea9f10a21d91103", + "semanticValidationSourceSha256": "02fbb92681c78c5de5d2f1aae25aabc25a0e5df6b0ff4baccac652192a8da975", + "semanticNormalizationSourceSha256": "8b628b82e8eacc91cdcc6fdca2210ba4cf282f01e9c6f74e95ff271655b5047e", + "semanticCompletionContractSourceSha256": "f49398b682a5623812f5d3c33d6bb9a749008e47cc0ed969072041615f66df59", + "semanticValidatorsSourceSha256": "f22d9caf2035b600abc1a6625ae23ca97f1409d7e29ed5b7264bb8f9784c7dc9", + "semanticSchemaBundleSourceSha256": "016391a087cc217aafed4ba1f4efafc6e4881a7525952dce824b7f4d24530603", + "semanticSidecarProtocolSourceSha256": "03c0c95b900f9dc18f29f6be27b8c79bc0bcab29780190d46a74e95328081236", + "semanticSidecarReceiptCommitContract": "correlated-tool-resolve-v1", + "displayEventSourceSha256": "d72f9a6035c247bc6a42c067c05753076c54322c4b58984e93922f3325dc6877", + "displayProjectionSourceSha256": "75952606b26dd93bbc582c1f0a4424487377272ea745fb3a27ceae031fd5dc7e", + "configuredEnvironmentSourceSha256": "5c9008d121979a3a8987e6b8a5752172eafdc4a2b5d725a73f3bbb8fb83be8ff", + "configuredEnvironmentRustSourceSha256": "85c87fae62b8a9377e4f0a4c00302c7519fae4397b1c48253f8f2b7ba7faf994", + "profileActivitySourceSha256": "4fdb6b0f9125fd8372ba6f27945fa5c8a3d3b2ec891df544092e3dc7a75d5282", + "profileExtensionsSourceSha256": "a80ddc95eb4505095bfdbbf85c758199949ab20be9c84b34c77f762eb7180571", + "profileInstallationSourceSha256": "3e61c4cdd4b78e38fcb16eaadae4f1fd65ce3c7127184ea7a80948c8e5d77f88", + "attachmentPolicySourceSha256": "4f7638b7cc64e18f6a4ba33fd46de2b3d49225e32aeab41f5c695b268fa4f8e2", + "singleReadEvidenceSourceSha256": "4069393405089bde613cecec04f5123c426594811da4c668bd96f0e7e283550f", + "nativeSessionRuntimeSourceSha256": "1fe6fa9e6b96d84d70432b85a57c473f833606d4a521563d5022b4d3c41576ce", + "nativeRuntimeContextSourceSha256": "71607f80dbb8dc5cc6c123d0b8f3995d6c5407277695a8c82cbedc90fb8f4842", + "nativeRuntimePromptsSourceSha256": "fff5755dba9b7dce8a5f10ff6ac6d5ed06b92ed079ce1e0a44ac72f2ce7869c1", + "nativeSessionContractSourceSha256": "22a8c96ce41ba4f03e976fc37f0cc7c5d05cee99fa789941da31ee1dab440b90", + "harnessSessionContractSourceSha256": "03f4cd7f76f34f6d82e009b66aa51f18f6e23dc235b660778060e35b6ea07412", + "harnessBackendSourceSha256": "dc827e281ae6731545b166ee4d709a4f67f594a556dcbe63c70fe8f1aca26249", + "harnessSessionSourceSha256": "81361733e86b56dd5a66415aefc6f72e69be24acc8d3417370154ebb7fef6e94", + "transportContractSourceSha256": "03fca93d36a89b0cc0917335e293ad097fdf04356d21b307827bb0ea4c13f523", + "nativeBackendSourceSha256": "e7b12d4418c7a71f5f2474b046697d3009eae3613b284bec381df5e9965a1c37", + "nativeBackendFactorySourceSha256": "7e431c926b76d53c4c08e3a4105353dcd34bd940f6588a1a68f79631f6feeb44", + "nativeInstructionsSourceSha256": "9173ebccf09cb56a781d2bf5c7eb27fdafb13d3450d99342c6aff86b42d6b43a", + "providerLaunchSourceSha256": "fa8db1c42c378898287c141d8cea9069166981e369c9b9c51422cb6177de5a2c", + "launchPreparationStoreSourceSha256": "ff6025568bbc0aa6ce545772d73dfca2d3da226740bd3a7b386b069e76cff296", + "runtimeRequestBridgeSourceSha256": "b8c0c38d45f7625c051387136f60fdc2ca1b83e7e58e92fc08360d057b97f95a", + "runtimeRequestFacadeSourceSha256": "d0ed28e0c5ddad34db8f437e704c8a755eb0b47f27e08e44185f66d1d2205295", + "runtimeRequestNotificationSourceSha256": "6196c6175f11c69295dc00fc773201f4fdeca26c500653870253c43b81e65211", + "guardedInstallationSourceSha256": "56a177d3e702914818dca4ca50e061f0d25f988bfc4c0582c36ce23c1696d82f", + "nativeExecutionSourceSha256": "3a30d52bffad68c7f02349193be84761050eb99039fc0556c2b449891a36e918", + "profileCompatibilitySourceSha256": "c89ee3a3831170e044d715b2c26d1b5deb05f4e56481ba971284a7fddbf117a6", + "installationSourceSha256": "a1c7c6f103b898216973e77ffbac0ef7572e250aa574d1a4065270e9dfa0dc2d", + "assetOwnershipSourceSha256": "1785e3f31e5dad5a13b53e25d8cf9482a9bd89a27b6c127830aed6b81acc605c", + "assetInstallerSourceSha256": "d3bd9514e5920d0cc3567aa50487fb534c57a6f66b8da0d3f22c5f6e499c841f", + "credentialBindingSourceSha256": "7586d44b47c83395498dc8806287cd3538ef58177ba7c5949f5766ed105a0161", + "sidecarErrorSourceSha256": "b09af4577f8bf42eee7036b2ad9842202cfc763e72b53c3287f82c6090cdad80", + "profileSourceSha256": "f5538b3dbefa7634a332b87d5ededcefb56aa7e5790cede1c7a0d948a548a456", + "metadataProbeSourceSha256": "be4c8416edd27061109bc1f33d0c26071b4facbb3f651beaae6b3c0275cce727", + "permissionToolIdentitySourceSha256": "770b47305e571a81f344883057d0e4a482a1cd59ac77d045836593a8e739deb1", + "semanticContinuationSourceSha256": "c9cc0fd516b3df92a6655f56ee9bef37615a889c371e2e137d8ac5c1176de811", + "runtimeHostSourceSha256": "94a73749267f5a80b5df19413a65b4b9263c00a9bf7e98fda00f00b8f9fd32ec", + "runtimeAdapterSourceSha256": "e4c764c2c3c7dd655a965b39472bd1ad8d577d1e7258b25594b171c9429752b8", + "runtimeSandboxSourceSha256": "92dfa53f5a3ed0062913ae417eeccec243b430d444495efde2a8e9c4c02e78ea", + "cancellationSourceSha256": "a07d1a9e982dec0ae130c1ad1efa5bc75c6a39e013b730ba3ee45256f09ceeca", + "agentFilesSourceSha256": "fb32f6badc13dfa22f231479509b631b2f0eba0c99b1ab9f3cd5c0c601aa03ba", + "nativeDistributionSourceSha256": "aed327c389e957483a503cb1bb8b247ee7cd7ff4b528fb4567c39bcb4577690b" + } +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 30d8792a64d..3ebaa15f773 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -313,6 +313,8 @@ vi.mock("./current-wake-comments.js", () => ({ })); vi.mock("../activity-log.js", () => ({ + logActivity: vi.fn(async () => undefined), + publishPluginDomainEvent: vi.fn(async () => undefined), persistActivity: state.persistActivity, publishActivity: state.publishActivity, })); @@ -884,6 +886,83 @@ describe("remote runner launch fingerprint compatibility", () => { }); describe("remote runner process supervision", () => { + it.each(["staging", "dispatch trace"])("retains cancellation during %s without a launch RPC", async (boundary) => { + let enter!: () => void; + let release!: () => void; + const entered = new Promise(resolve => { enter = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const pause = async () => { enter(); await gate; }; + const execute = vi.fn(); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, + remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + ...(boundary === "staging" ? { ensureArtifact: pause } : { trace: { record: pause } as never }), + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + try { + await entered; + expect(handle.child.kill("SIGTERM")).toBe(true); + expect(handle.isExitConfirmed?.()).toBe(false); + expect(execute).not.toHaveBeenCalled(); + release(); + await expect(handle.completion).resolves.toMatchObject({ code: null, signal: "SIGTERM" }); + expect(handle.isExitConfirmed?.()).toBe(true); + expect(execute).not.toHaveBeenCalled(); + } finally { + release(); + await handle.completion; + } + }); + + it("defers an in-flight launch signal until its exact identity is available", async () => { + let enter!: () => void; + let release!: () => void; + let releaseMonitor!: () => void; + const entered = new Promise(resolve => { enter = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + const monitor = new Promise(resolve => { releaseMonitor = resolve; }); + let nonce = ""; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { + nonce = input.args![4]!; + enter(); await gate; + return ok; + } + if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + if (label === "paperclip-runner-monitor") { await monitor; return { ...ok, exitCode: 45 }; } + if (label === "paperclip-runner-signal") return ok; + return { ...ok, exitCode: 1 }; + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, + remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + try { + await entered; + expect(handle.child.kill("SIGKILL")).toBe(true); + expect(handle.child.kill("SIGTERM")).toBe(true); + expect(execute.mock.calls.some(([input]) => input.args?.[2] === "paperclip-runner-signal")).toBe(false); + expect(handle.isExitConfirmed?.()).toBe(false); + release(); + await vi.waitFor(() => expect(execute.mock.calls.filter(([input]) => input.args?.[2] === "paperclip-runner-signal")).toHaveLength(1)); + const signal = execute.mock.calls.find(([input]) => input.args?.[2] === "paperclip-runner-signal")![0]; + expect(signal.args?.slice(3)).toEqual(["/runtime/identity", nonce, "runner-remote", "4321"]); + expect(signal.args?.[1]).toContain('kill -KILL "$expected_pid"'); + expect(handle.isExitConfirmed?.()).toBe(false); + releaseMonitor(); + await handle.completion; + expect(handle.isExitConfirmed?.()).toBe(true); + expect(execute.mock.calls.filter(([input]) => input.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); + } finally { + release(); releaseMonitor(); + await handle.completion; + } + }); it.each(["exit", "unreadable", "changed generation"] as const)("requires an exit receipt when process metadata changes during supervision (%s)", async failure => { const root = await mkdtemp(join(tmpdir(), "runner-exit-receipt-")); const marker = join(root, "identity"); @@ -14833,3 +14912,91 @@ it("keeps an ACPX missing runtime package failure actionable and stops automatic expect(code).toBe("native_provider_runtime_missing"); expect(nativeSessionFailureDisposition(1, new Date(), code)).toEqual({ phase: "terminal_failure", failureCode: code, nextAttemptAt: null }); }); +describe("HTTP Stop during remote runner staging", () => { + it("does not dispatch after Stop and bounded bootstrap cleanup", async () => { + const { createCapabilityRunnerdCodexTransport } = await import("@paperclipai/paperclip-runner/live"); + const { createNativeSessionBackend } = await import("@paperclipai/paperclip-runner"); + + vi.spyOn(runEventWriter, "appendHeartbeatRunEvent").mockResolvedValue({ row: { seq: 1 }, disposition: "committed", highestContiguousSourceSeq: 1 } as never); + const root = await mkdtemp(join(tmpdir(), "remote-startup-stop-")); + const retained = join(root, "retained.txt"); + const artifact = join(root, "unexecuted-artifact"); + await writeFile(retained, "preserve exact bytes\n"); + await writeFile(artifact, "fixture; launch RPC rejects before execution\n"); + let enter!: () => void, release!: () => void; + const entered = new Promise(resolve => { enter = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + let dispatches = 0; + const launch = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "controlled", leaseId: "controlled", remoteCwd: root }, + runner: { execute: async (input: { args?: string[] }) => { + if (input.args?.[2] === "paperclip-runner-launch") { dispatches++; } + return { exitCode: 1, timedOut: false, stdout: "", stderr: "controlled RPC failure; no process started", signal: null }; + } } as never, + remoteBinary: artifact, processIdentityPath: join(root, "identity"), stateDirectory: root, + diagnosticsDirectory: join(root, "diagnostics"), runnerInstanceId: "runner", + ensureArtifact: async () => { enter(); await gate; }, + }); + const bundle = createCapabilityRunnerdCodexTransport({ + stateDirectory: join(root, "transport"), runnerBinary: artifact, codexCommand: artifact, + provider: "codex", closeGraceMs: 100, sourceCodexHome: "", environment: { PATH: process.env.PATH }, + runnerProcessLauncher: launch, + controlPlaneRegistration: async authority => { await authority.start(); return { connectUrl: authority.connectUrl, release: async () => { await authority.stop(); } }; }, + }); + const dbModule = await import("@paperclipai/db"); + const { resolveExternalTestDatabaseUrl } = await import("../../__tests__/helpers/external-test-database.js"); + const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_TEST_DATABASE_URL"); + const temporary = externalUrl + ? { connectionString: externalUrl, cleanup: () => dbModule.closeRegisteredClients(externalUrl) } + : await dbModule.startEmbeddedPostgresTestDatabase("remote-staging-http-"); + const httpDb = dbModule.createDb(temporary.connectionString); + const localExecution = { ...execution, binding: { ...execution.binding, + companyId: randomUUID(), agentId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), executionWorkspaceId: randomUUID() }, + workspace: { ...execution.workspace, cwd: root } }; + const { companyId, agentId, issueId, runId } = localExecution.binding; + await httpDb.insert(dbModule.companies).values({ id: companyId, name: "Controlled Stop", issuePrefix: "STOP" }); + await httpDb.insert(dbModule.agents).values({ id: agentId, companyId, name: "Controlled agent" }); + await httpDb.insert(dbModule.issues).values({ id: issueId, companyId, title: "Controlled Stop", status: "in_progress", assigneeAgentId: agentId }); + await httpDb.insert(dbModule.heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, + invocationSource: "on_demand", status: "running", runtimeMode: "native" }); + await httpDb.insert(dbModule.nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed" }); + const originalIssueService = issueServiceModule.issueService; + vi.spyOn(issueServiceModule, "issueService").mockImplementation(db => db === httpDb ? originalIssueService(db) : ({ update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })) } as never)); + const { default: express } = await import("express"); + const { default: request } = await import("supertest"); + const { agentRoutes } = await import("../../routes/agents.js"); + const { errorHandler } = await import("../../middleware/error-handler.js"); + const app = express(); app.use(express.json()); + app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: "fixture-board", companyIds: [companyId] }; next(); }); + const httpErrors: string[] = []; + app.use("/api", agentRoutes(httpDb)); + app.use((error: Error, _req: unknown, _res: unknown, next: (error: Error) => void) => { httpErrors.push(error.stack ?? String(error)); next(error); }); + app.use(errorHandler); + const backend = createNativeSessionBackend(localExecution, { codexTransportFactory: () => bundle.transport, runnerInstanceId: "runner", workingDirectoryAuthority: "remote_runner", environment: { PAPERCLIP_WORKSPACE_CWD: root, PATH: process.env.PATH } }); + // Execute real startup and transport code. The execution lease/event sink + // are controlled ports; HTTP cancellation uses a fresh PostgreSQL database. + state.execute.mockReset().mockImplementationOnce(options => executeNativeSession({ ...options, timeoutMs: 1000, + controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { const sequence = "sourceSeq" in event ? event.sourceSeq : event.sequence; return { cursor: sequence, highestContiguousSourceSeq: sequence, disposition: "committed" as const }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {} }, + })); + const outcome = executePaperclipNativeSession({ db: leaseDb(localExecution), execution: localExecution, runnerInstanceId: "runner", backend, useRunnerd: false }).catch(error => error); + + let stopping: Promise | undefined; + try { + await Promise.race([entered, outcome.then(result => { throw new Error(`startup ended before staging: ${result instanceof Error ? result.stack : JSON.stringify(result)}`); })]); + stopping = request(app).post(`/api/heartbeat-runs/${runId}/cancel`).send({}).then(response => ({ status: response.status, body: response.body })).catch(error => error); + await new Promise(resolve => setTimeout(resolve, 1400)); + const close = await bundle.transport.close("join already requested bootstrap cleanup").catch(error => error); + expect(close).toBeInstanceOf(Error); + release(); + const result = await outcome; + const stop = await stopping; + await new Promise(resolve => setTimeout(resolve, 50)); + expect(result).toBeInstanceOf(Error); + expect(httpErrors).toEqual([]); + expect(await readFile(retained, "utf8")).toBe("preserve exact bytes\n"); + await expect(access(join(root, "identity"))).rejects.toThrow(); + expect(stop).toMatchObject({ status: 200, body: { status: "cancelled" } }); + expect(dispatches).toBe(0); + } finally { release(); await outcome; await stopping; await bundle.transport.detachControllerForRestart?.().catch(() => {}); await temporary.cleanup(); await rm(root, { recursive: true, force: true }); } + }, 120000); +}); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 15f54f4a5bb..d01ed4e2431 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11399,6 +11399,7 @@ export function createRemoteRunnerProcessLauncher(input: { const runner = input.runner; return (spec) => { let exitConfirmed = false; + let pendingSignal: NodeJS.Signals | null = null; let launchedIdentity: { nonce: string; pid: number; @@ -11409,12 +11410,20 @@ export function createRemoteRunnerProcessLauncher(input: { exitCode: null, signalCode: null, kill: (requestedSignal) => { + const requested = requestedSignal === "SIGKILL" || requestedSignal === 9 + ? "SIGKILL" + : requestedSignal === "SIGINT" || requestedSignal === 2 + ? "SIGINT" + : "SIGTERM"; + // Keep Stop through staging and launch RPC completion. Acceptance is + // not exit evidence, and a later TERM must not undo an earlier KILL. + pendingSignal = pendingSignal === "SIGKILL" ? pendingSignal : requested; const identity = launchedIdentity; - if (!identity) return false; + if (!identity) return true; const signal = - requestedSignal === "SIGKILL" || requestedSignal === 9 + pendingSignal === "SIGKILL" ? "KILL" - : requestedSignal === "SIGINT" || requestedSignal === 2 + : pendingSignal === "SIGINT" ? "INT" : "TERM"; // The durable marker, nonce, exact pid, and runner-id command line are @@ -11447,6 +11456,16 @@ export function createRemoteRunnerProcessLauncher(input: { return true; }, }; + const cancelledBeforeDispatch = () => { + exitConfirmed = true; + child.signalCode = pendingSignal; + return { + code: null, + signal: pendingSignal, + stdout: "", + stderr: "runner_process_launch_cancelled", + }; + }; const completion = (async () => { if (input.ensureArtifact) { if (input.trace) { @@ -11459,13 +11478,13 @@ export function createRemoteRunnerProcessLauncher(input: { await input.ensureArtifact(); } } + if (pendingSignal) return cancelledBeforeDispatch(); const launchStartedAtMs = Date.now(); // The provider's onSpawn callback is optional and some sandbox command // runners cannot report a remote pid until after the command has begun // streaming. Signal as soon as staging is complete and the launch RPC is // dispatched; this is late enough to avoid preview retries during staging // and early enough to avoid a callback-dependent deadlock. - input.onRunnerProcessSpawned?.(); await input.trace?.record({ name: "runner.process.dispatch", parentName: "runner.session.startup", @@ -11473,6 +11492,10 @@ export function createRemoteRunnerProcessLauncher(input: { endedAtMs: Date.now(), attributes: { target: "remote" }, }); + // Trace publication can yield after staging; recheck at dispatch. + if (pendingSignal) return cancelledBeforeDispatch(); + input.onRunnerProcessSpawned?.(); + if (pendingSignal) return cancelledBeforeDispatch(); const identityNonce = randomUUID(); const remoteArgs = [...spec.args]; const diagnosticsArgumentIndex = remoteArgs.indexOf( @@ -11549,6 +11572,7 @@ export function createRemoteRunnerProcessLauncher(input: { } launchedIdentity = { nonce: identityNonce, ...identity }; child.pid = identity.pid; + if (pendingSignal) child.kill(pendingSignal); await input.onSpawn?.({ pid: identity.pid, processGroupId: null,