api: Use Random Exponential Backoff in do_api_query method.

Fixes #537.

Author: orientor

zulip/zulip/__init__.py

@@ -508,13 +508,10 @@ def do_api_query(self, orig_request, url, method="POST",
         query_state = {
             'had_error_retry': False,
             'request': request,
-            'failures': 0,
         }  # type: Dict[str, Any]
 
         def error_retry(error_string):
-            # type: (str) -> bool
-            if not self.retry_on_errors or query_state["failures"] >= 10:
-                return False
+            # type: (str) -> None
             if self.verbose:
                 if not query_state["had_error_retry"]:
                     sys.stdout.write("zulip API(%s): connection error%s -- retrying." %
@@ -524,9 +521,6 @@ def error_retry(error_string):
                     sys.stdout.write(".")
                 sys.stdout.flush()
             query_state["request"]["dont_block"] = json.dumps(True)
-            time.sleep(1)
-            query_state["failures"] += 1
-            return True
 
         def end_error_retry(succeeded):
             # type: (bool) -> None
@@ -536,7 +530,8 @@ def end_error_retry(succeeded):
                 else:
                     print("Failed!")
 
-        while True:
+        backoff = RandomExponentialBackoff(timeout_success_equivalent=300)
+        while backoff.keep_going():
             try:
                 if method == "GET":
                     kwarg = "params"
@@ -559,7 +554,9 @@ def end_error_retry(succeeded):
 
                 # On 50x errors, try again after a short sleep
                 if str(res.status_code).startswith('5'):
-                    if error_retry(" (server %s)" % (res.status_code,)):
+                    error_retry(" (server %s)" % (res.status_code,))
+                    backoff.fail()
+                    if backoff.keep_going():
                         continue
                     # Otherwise fall through and process the python-requests error normally
             except (requests.exceptions.Timeout, requests.exceptions.SSLError) as e:
@@ -585,7 +582,9 @@ def end_error_retry(succeeded):
                     # in an invalid site.
                     raise UnrecoverableNetworkError('cannot connect to server ' + self.base_url)
 
-                if error_retry(""):
+                error_retry("")
+                backoff.fail()
+                if backoff.keep_going():
                     continue
                 end_error_retry(False)
                 return {'msg': "Connection error:\n%s" % traceback.format_exc(),
@@ -609,6 +608,7 @@ def end_error_retry(succeeded):
             end_error_retry(False)
             return {'msg': "Unexpected error from the server", "result": "http-error",
                     "status_code": res.status_code}
+        return {'msg': "Unexpected error from the server", "result": "unexpected-error"}
 
     def call_endpoint(self, url=None, method="POST", request=None,
                       longpolling=False, files=None, timeout=None):