From 9ea5c13cc7dc56111fccdf5cefa2e73dfa693d41 Mon Sep 17 00:00:00 2001 From: Adam Mills Date: Mon, 3 May 2021 12:56:36 +1000 Subject: [PATCH 1/6] retry --- .gitignore | 1 + airtable/__init__.py | 8 +++++++- requirements-test.txt | 1 + test_airtable.py | 28 +++++++++++++++++++++++++++- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 56fc682..0a6e766 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ venv /AUTHORS /ChangeLog /.eggs +/.vscode \ No newline at end of file diff --git a/airtable/__init__.py b/airtable/__init__.py index ab7a974..c64ac63 100644 --- a/airtable/__init__.py +++ b/airtable/__init__.py @@ -5,6 +5,7 @@ import warnings import requests +from requests.packages.urllib3.util.retry import Retry import six API_URL = 'https://api.airtable.com/v%s/' @@ -91,10 +92,15 @@ def __init__(self, base_id, api_key, dict_class=OrderedDict): self.headers = {'Authorization': 'Bearer %s' % api_key} self._dict_class = dict_class + + def __request(self, method, url, params=None, payload=None): if method in ['POST', 'PUT', 'PATCH']: self.headers.update({'Content-type': 'application/json'}) - response = requests.request( + retries = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) + session = requests.Session() + session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries)) + response = session.request( method, posixpath.join(self.base_url, url), params=params, diff --git a/requirements-test.txt b/requirements-test.txt index b03a90a..868281f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ pylint flake8 requests-mock +httpretty diff --git a/test_airtable.py b/test_airtable.py index a7469b7..4f063aa 100644 --- a/test_airtable.py +++ b/test_airtable.py @@ -2,8 +2,9 @@ import unittest import requests_mock - +import httpretty import airtable +import re FAKE_TABLE_NAME = 'TableName' FAKE_BASE_ID = 'app12345' @@ -33,6 +34,31 @@ def test_build_headers(self): self.assertEqual(self.airtable.headers['Authorization'], 'Bearer fake_api_key') + @httpretty.activate + def test_retries(self): + httpretty.register_uri( + httpretty.GET, + re.compile(r'https://.*'), + responses=[ + httpretty.Response( + body='{}', + status=429, + ), + httpretty.Response( + body='{}', + status=500, + ), + httpretty.Response( + body='{}', + status=200, + ), + ] + ) + + self.get() + self.assertEqual(3, len(httpretty.latest_requests())) + + @requests_mock.mock() def test_get_all(self, mock_requests): mock_requests.get('https://api.airtable.com/v0/app12345/TableName', json={ From 6969dc7e6973b7ec277d1b9733843198c65cb630 Mon Sep 17 00:00:00 2001 From: Adam Mills Date: Wed, 5 May 2021 17:13:09 +1000 Subject: [PATCH 2/6] moduke import --- airtable/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airtable/__init__.py b/airtable/__init__.py index c64ac63..3babcae 100644 --- a/airtable/__init__.py +++ b/airtable/__init__.py @@ -5,7 +5,7 @@ import warnings import requests -from requests.packages.urllib3.util.retry import Retry +from requests.packages.urllib3.util import retry import six API_URL = 'https://api.airtable.com/v%s/' @@ -91,13 +91,11 @@ def __init__(self, base_id, api_key, dict_class=OrderedDict): self.base_url = posixpath.join(self.airtable_url, base_id) self.headers = {'Authorization': 'Bearer %s' % api_key} self._dict_class = dict_class - - def __request(self, method, url, params=None, payload=None): if method in ['POST', 'PUT', 'PATCH']: self.headers.update({'Content-type': 'application/json'}) - retries = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) + retries = retry.Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) session = requests.Session() session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries)) response = session.request( From f10172c5ca72c163e3c7995c04fdd9aee1311871 Mon Sep 17 00:00:00 2001 From: Adam Mills Date: Wed, 5 May 2021 17:15:18 +1000 Subject: [PATCH 3/6] gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0a6e766..e0dbc19 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,4 @@ venv /dist/* /AUTHORS /ChangeLog -/.eggs -/.vscode \ No newline at end of file +/.eggs \ No newline at end of file From a293d0739e02a6fc6e0b6fe5a1c72f223df4f98c Mon Sep 17 00:00:00 2001 From: Adam Mills Date: Wed, 5 May 2021 17:15:40 +1000 Subject: [PATCH 4/6] whitespace --- airtable/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtable/__init__.py b/airtable/__init__.py index 3babcae..5cfee28 100644 --- a/airtable/__init__.py +++ b/airtable/__init__.py @@ -91,7 +91,7 @@ def __init__(self, base_id, api_key, dict_class=OrderedDict): self.base_url = posixpath.join(self.airtable_url, base_id) self.headers = {'Authorization': 'Bearer %s' % api_key} self._dict_class = dict_class - + def __request(self, method, url, params=None, payload=None): if method in ['POST', 'PUT', 'PATCH']: self.headers.update({'Content-type': 'application/json'}) From 46c878600da78746624bd45434f109bced88c1c7 Mon Sep 17 00:00:00 2001 From: Adam Mills Date: Wed, 5 May 2021 17:23:54 +1000 Subject: [PATCH 5/6] adding body to test --- test_airtable.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test_airtable.py b/test_airtable.py index 4f063aa..b7f3a90 100644 --- a/test_airtable.py +++ b/test_airtable.py @@ -1,5 +1,6 @@ from typing import Any, Dict import unittest +import json import requests_mock import httpretty @@ -49,15 +50,27 @@ def test_retries(self): status=500, ), httpretty.Response( - body='{}', + body=json.dumps({ + 'records': [ + { + 'id': 'reccA6yaHKzw5Zlp0', + 'fields': { + 'Name': 'John', + 'Number': '(987) 654-3210' + } + } + ], + 'offset': 'reccg3Kke0QvTDW0H' + }), status=200, ), ] ) - self.get() + res = self.get() self.assertEqual(3, len(httpretty.latest_requests())) - + self.assertEqual(len(res['records']), 1) + self.assertEqual(res['offset'], 'reccg3Kke0QvTDW0H') @requests_mock.mock() def test_get_all(self, mock_requests): From 3f742e04dd2f77dffbb1fcea24403e2d1a64649e Mon Sep 17 00:00:00 2001 From: adam Date: Thu, 13 May 2021 17:34:11 +1000 Subject: [PATCH 6/6] upped retries to 10, probably should be a config option --- airtable/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airtable/__init__.py b/airtable/__init__.py index 5cfee28..05e3e53 100644 --- a/airtable/__init__.py +++ b/airtable/__init__.py @@ -95,7 +95,7 @@ def __init__(self, base_id, api_key, dict_class=OrderedDict): def __request(self, method, url, params=None, payload=None): if method in ['POST', 'PUT', 'PATCH']: self.headers.update({'Content-type': 'application/json'}) - retries = retry.Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) + retries = retry.Retry(total=10, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) session = requests.Session() session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries)) response = session.request(