diff --git a/server/scripts/benchmark-assigned-mcp-tools-memory.ts b/server/scripts/benchmark-assigned-mcp-tools-memory.ts new file mode 100644 index 00000000000..ae9694ec0bd --- /dev/null +++ b/server/scripts/benchmark-assigned-mcp-tools-memory.ts @@ -0,0 +1,70 @@ +import type { ToolGatewayDescriptor, ToolGatewayService } from "../src/services/tool-gateway.js"; +import { createAssignedMcpTools } from "../src/services/native-runtime/assigned-mcp-tools.js"; + +function option(name: string, fallback: number) { + const index = process.argv.indexOf(name); + const value = Number(index < 0 ? fallback : process.argv[index + 1]); + if (!Number.isInteger(value) || value < 1) throw new Error(`Expected a positive integer for ${name}`); + return value; +} +const toolCount = option("--tools", 80); +const schemaBytes = option("--schema-bytes", 2048); +const sessions = option("--sessions", 200); +const turns = option("--turns", 20); +const gc = (globalThis as { gc?: () => void }).gc; +if (!gc) throw new Error("Run with node --expose-gc"); + +function listing(): ToolGatewayDescriptor[] { + return Array.from({ length: toolCount }, (_, index) => { + const properties: Record = {}; + for (let field = 0; JSON.stringify(properties).length < schemaBytes; field += 1) { + properties[`field_${field}`] = { type: "string", description: `Field ${field} of tool ${index} with representative documentation text.` }; + } + return { + name: `fixture_app:tool_${index}`, displayName: `Tool ${index}`, description: `Fixture tool ${index}`, + parametersSchema: { type: "object", properties, required: ["field_0"] }, + pluginId: "fixture", providerType: "mcp_remote_http", risk: "read", + } as ToolGatewayDescriptor; + }); +} + +const gateway = { listToolsForNamedGateway: async () => listing() } as unknown as ToolGatewayService; +const create = () => createAssignedMcpTools({ gateway, gatewayPublicId: "benchmark", bearerToken: "benchmark" }); + +function heap() { + gc!(); gc!(); + return process.memoryUsage().heapUsed; +} + +function schemaIdentities(definitions: Array>) { + return definitions.map(definition => definition.inputSchema); +} + +async function measureSessions(baseline: number) { + const held: Array>> = []; + for (let index = 0; index < sessions; index += 1) held.push(await create()); + const retainedBytesPerSession = Math.round((heap() - baseline) / sessions); + const turnResults: Array>> = []; + const beforeTurns = heap(); + for (let index = 0; index < turns; index += 1) turnResults.push(held[0]!.definitions()); + const retainedBytesPerTurn = Math.round((heap() - beforeTurns) / turns); + const first = schemaIdentities(turnResults[0]!); + const schemaObjectsAfterTurns = new Set(turnResults.flatMap(schemaIdentities)).size; + const sharedAcrossTurns = turnResults.every(result => schemaIdentities(result).every((schema, index) => schema === first[index])); + return { retainedBytesPerSession, retainedBytesPerTurn, schemaObjectsAfterTurns, sharedAcrossTurns }; +} + +async function main() { + await create(); + const baseline = heap(); + const measured = await measureSessions(baseline); + const released = heap(); + console.log(JSON.stringify({ + tools: toolCount, schemaBytes, sessions, turns, + ...measured, + schemaObjectsPerTurn: measured.schemaObjectsAfterTurns / turns, + bytesStillHeldAfterRelease: Math.max(0, released - baseline), + }, null, 2)); +} + +await main(); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 87c514fd738..898df3a20e6 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -12513,6 +12513,52 @@ describe("native warm session supervision", () => { } }); + it("releases a retired warm predecessor's tool authority when the follow-up fails before admitting its own", async () => { + const { base } = fixture("authority-failed-follow-up"); + const next = { ...base, binding: { ...base.binding, runId: "authority-failed-follow-up-next" } }; + const root = await mkdtemp(join(tmpdir(), "native-authority-failed-follow-up-")); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", root); + const close = vi.fn(async () => undefined); + const issueService = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ + update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })), + } as unknown as ReturnType); + const assigned = { PAPERCLIP_NATIVE_MCP_NAME: "assigned", PAPERCLIP_NATIVE_MCP_URL: "http://127.0.0.1/mcp/gateways/assigned-follow-up", + PAPERCLIP_NATIVE_MCP_TOKEN: "follow-up-fixture" }; + state.createAssignedMcpTools.mockResolvedValueOnce({ definitions: () => [], has: () => false, execute: vi.fn() }) + .mockRejectedValueOnce(new Error("assigned tools unavailable")); + state.execute.mockReset().mockImplementationOnce(async options => { + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const factory = state.createTransport.mock.calls.at(-1)![0]; + const identity = { ...factory.prpIdentity, normalizedSessionId: base.session.normalizedSessionId }; + await mkdir(join(factory.stateDirectory!, "control-plane"), { recursive: true }); + await mkdir(join(factory.stateDirectory!, "runner"), { recursive: true }); + await writeFile(join(factory.stateDirectory!, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity))); + await writeFile(join(factory.stateDirectory!, "runner", "runner-state.json"), JSON.stringify(durableRunnerState(identity, "ready"))); + await options.onSession?.({ close }); return result; + }); + try { + const before = sessionToolAuthorityCount(); + await executePaperclipNativeSession({ db: leaseDb(base), execution: base, runnerInstanceId: "runner", useRunnerd: true, + runnerEnvironment: assigned }); + expect(sessionToolAuthorityCount()).toBe(before + 1); + const current = leaseDb(next); + const nextDb = { ...current, select: (columns?: Record) => columns?.status && columns?.runnerProfileJson + ? { from: () => ({ where: () => ({ limit: () => Promise.resolve([{ status: "succeeded", runnerProfileJson: { nativeExecutionInput: base } }]) }) }) } + : current.select(columns as never) } as unknown as Db; + const failure = await executePaperclipNativeSession({ db: nextDb, execution: next, runnerInstanceId: "runner", useRunnerd: true, + runnerEnvironment: { ...assigned, PAPERCLIP_AGENT_KEY_ID: "sha256:follow-up-key" } }).catch((error: Error) => error); + expect(failure).toMatchObject({ message: "assigned tools unavailable" }); + expect(state.createAssignedMcpTools).toHaveBeenCalledTimes(2); + expect(close).toHaveBeenCalledWith({ reason: "warm native session configuration changed" }); + expect(state.execute).toHaveBeenCalledOnce(); + expect(sessionToolAuthorityCount()).toBe(before); + } finally { + issueService.mockRestore(); + vi.unstubAllEnvs(); + await rm(root, { recursive: true, force: true }); + } + }); + it("awaits prior idle ownership retirement before launching the accepted-plan session", async () => { const { base, next, run } = fixture("identity-handoff"); let finishClose!: () => void; @@ -23099,3 +23145,68 @@ describe("isTransientRemoteCheckpointReadFailure", () => { await expect(retryTransientRemoteCheckpointRead(async () => { if (drops-- > 0) throw reset(); return "settled"; }, vi.fn(), 0)).resolves.toBe("settled"); }); }); + +describe("released native session tool authority", () => { + const assignedTool = (label: string) => ({ name: `app_released_${label}`, description: `Released ${label}`, inputSchema: { type: "object" } }); + const assignedTools = (label: string) => { + const tool = assignedTool(label); + return { definitions: () => [tool], has: (name: string) => name === tool.name, + execute: vi.fn(async (_call: unknown) => ({ content: [{ type: "text", text: label }] })) }; + }; + const scoped = (runId: string) => ({ ...execution, + binding: { ...execution.binding, runId, issueId: "released-authority-issue" }, + session: { ...execution.session, normalizedSessionId: "released-authority-session", + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } } } as NativeExecutionInputV1); + const environment = { PAPERCLIP_NATIVE_MCP_NAME: "assigned", PAPERCLIP_NATIVE_MCP_URL: "http://localhost/mcp/gateways/assigned-released", + PAPERCLIP_NATIVE_MCP_TOKEN: "released-token" }; + const call = (tool: string) => ({ tool, callId: `call-${tool}`, arguments: {} }) as never; + + beforeEach(() => { + state.toolAuthorityDefinitions.mockImplementation(async binding => + ((binding.assignedMcpTools as ReturnType | undefined)?.definitions() ?? []) as never); + state.toolAuthorityExecute.mockReset().mockImplementation(async (binding: Record, value: unknown) => { + const tools = binding.assignedMcpTools as ReturnType; + const { tool, arguments: args } = value as { tool: string; arguments: unknown }; + if (!tools.has(tool)) throw new Error("unknown tool"); + return tools.execute({ tool, arguments: args }); + }); + }); + afterEach(() => { + state.toolAuthorityDefinitions.mockImplementation(async () => []); + state.toolAuthorityExecute.mockReset(); + }); + + it("refuses a released session's late tool call and completion feedback and admits the next run with only its own assignments", async () => { + const first = assignedTools("first"), second = assignedTools("second"); + const current = scoped("released-authority-first"); + state.createAssignedMcpTools.mockResolvedValueOnce(first); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close: async () => {} }); + return { result: { summary: "done" }, terminal: { runTerminalState: "succeeded" }, turnId: "turn", + normalizedSessionId: current.session.normalizedSessionId, providerSessionId: null, driverKind: "test", driverVersion: "1", + nativeEventCount: 0, highestContiguousSourceSeq: 0 }; + }); + await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: "runner", useRunnerd: true, + runnerEnvironment: environment }); + const released = state.createBackend.mock.calls.at(-1)![1]; + await expect(Promise.resolve().then(() => released.dynamicToolHandler!(call(assignedTool("first").name)))) + .rejects.toThrow("native_session_tool_authority_unavailable"); + await expect(released.completionFeedback!({} as never)).rejects.toThrow("native_session_tool_authority_unavailable"); + expect(first.execute).not.toHaveBeenCalled(); + const next = scoped("released-authority-second"); + state.createAssignedMcpTools.mockResolvedValueOnce(second); + const directory = await mkdtemp(join(tmpdir(), "native-authority-readmission-")); + vi.stubEnv("PAPERCLIP_RUNNER_STATE_DIR", directory); + try { + await createRunnerdBackend({ db: leaseDb(next), execution: next, runnerInstanceId: "runner", runnerEnvironment: environment }); + } finally { + vi.unstubAllEnvs(); + await rm(directory, { recursive: true, force: true }); + } + const admitted = state.createBackend.mock.calls.at(-1)![1]; + await expect(admitted.dynamicToolHandler!(call(assignedTool("second").name))).resolves.toEqual({ content: [{ type: "text", text: "second" }] }); + await expect(admitted.dynamicToolHandler!(call(assignedTool("first").name))).rejects.toThrow("unknown tool"); + expect(first.execute).not.toHaveBeenCalled(); + expect(state.createAssignedMcpTools).toHaveBeenCalledTimes(2); + }); +});