add random port (#18504)

* add random port

Author: guru4elephant

python/paddle/fluid/incubate/fleet/base/role_maker.py

@@ -234,6 +234,7 @@ def __init__(self):
         super(MPISymetricRoleMaker, self).__init__()
         self._node_type = None
         self._proc_per_node = 2
+        self._pserver_rand_port = 0
 
     def _check_role_generation(self):
         if not self._role_is_generated:
@@ -248,6 +249,20 @@ def is_first_worker(self):
             return self.is_worker() and 0 == self.worker_index()
         return False
 
+    def get_pserver_endpoints(self):
+        if self._pserver_rand_port <= 0:
+            import random
+            random.seed(self._server_num())
+            # port will be randomly generated from 60001 to 63999
+            # random seed is server num so that all nodes will get
+            # the same port
+            self._pserver_rand_port = random.randint(60001, 64000)
+        endpoints = [
+            x + ":" + str(self._pserver_rand_port)
+            for x in self._server_endpoints
+        ]
+        return endpoints
+
     def worker_num(self):
         return self._worker_num()
 
@@ -273,33 +288,38 @@ def _worker_num(self):
         """
         if self._check_role_generation():
             if self.is_worker():
-                return self._get_size() / 2
+                return self._get_size() / self._proc_per_node
         return 0
 
     def _server_num(self):
         """
         return the current number of server
         """
         if self._check_role_generation():
-            if self.is_server():
-                return self._get_size() / 2
-        return 0
+            return self._get_size() / self._proc_per_node
+        else:
+            self.generate_role()
+            return self._get_size() / self._proc_per_node
 
     def worker_index(self):
         """
         return the index of worker
         """
         if self._check_role_generation():
             return self._rank / self._proc_per_node
-        return 0
+        else:
+            self.generate_role()
+            return self._get_size() / 2
 
     def server_index(self):
         """
         return the index of server
         """
         if self._check_role_generation():
             return self._rank / self._proc_per_node
-        return 0
+        else:
+            self.generate_role()
+            return self._get_size() / self._proc_per_node
 
     def _barrier_worker(self):
         """
@@ -308,6 +328,8 @@ def _barrier_worker(self):
         if self._check_role_generation():
             if self.is_worker():
                 self._node_type_comm.barrier()
+        else:
+            raise Exception("You should check role generation first")
 
     def _barrier_server(self):
         """
@@ -316,6 +338,8 @@ def _barrier_server(self):
         if self._check_role_generation():
             if self.is_server():
                 self._node_type_comm.barrier()
+        else:
+            raise Exception("You should check role generation first")
 
     def generate_role(self):
         """
@@ -332,6 +356,8 @@ def generate_role(self):
                 self._node_type = 1
             self._node_type_comm = self._comm.Split(self._node_type)
             self._role_is_generated = True
+        else:
+            raise Exception("You should check role generation first")
 
 
 class PaddleCloudRoleMaker(RoleMakerBase):