diff --git a/src/meshapi/tests/test_slack_notification.py b/src/meshapi/tests/test_slack_notification.py index ce6f9eec..fa2fc690 100644 --- a/src/meshapi/tests/test_slack_notification.py +++ b/src/meshapi/tests/test_slack_notification.py @@ -269,6 +269,7 @@ def test_slack_notification_for_name_change_no_env_var(self, requests_mocker): @requests_mock.Mocker() @patch("meshapi.util.admin_notifications.SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL", "https://mock-slack-url") + @patch("meshapi.util.admin_notifications.SLACK_ADMIN_NOTIFICATIONS_RETRY_COUNT", 2) def test_slack_notification_for_name_change_slack_failure(self, requests_mocker): member = Member( name="Stacy Maidenname", @@ -293,6 +294,7 @@ def test_slack_notification_for_name_change_slack_failure(self, requests_mocker) ) self.assertEqual(requests_mocker.request_history[0].url, "https://mock-slack-url/") + self.assertEqual(len(requests_mocker.request_history), 3) @requests_mock.Mocker() @patch("meshapi.util.admin_notifications.SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL", "https://mock-slack-url") diff --git a/src/meshapi/util/admin_notifications.py b/src/meshapi/util/admin_notifications.py index 4676676c..f041eff7 100644 --- a/src/meshapi/util/admin_notifications.py +++ b/src/meshapi/util/admin_notifications.py @@ -1,6 +1,7 @@ import json import logging import os +import time from typing import Optional, Sequence, Type import requests @@ -13,6 +14,8 @@ SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL = os.environ.get("SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL") SITE_BASE_URL = os.environ.get("SITE_BASE_URL") +SLACK_ADMIN_NOTIFICATIONS_RETRY_COUNT = int(os.environ.get("SLACK_ADMIN_NOTIFICATIONS_RETRY_COUNT", 3)) + def escape_slack_text(text: str) -> str: return text.replace("↔", "<->").replace("&", "&").replace("<", "<").replace(">", ">") @@ -69,13 +72,21 @@ def notify_admins( ) return - response = requests.post(SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL, json=slack_message) - - if raise_exception_on_failure: - response.raise_for_status() - elif response.status_code != 200: - logging.error( - f"Got HTTP {response.status_code} while sending slack notification to slack admin. " - f"HTTP response was {response.text}. Unable to notify admins of " - f"the following message: {slack_message}" - ) + for i in range(0, SLACK_ADMIN_NOTIFICATIONS_RETRY_COUNT + 1): + response = requests.post(SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL, json=slack_message) + try: + response.raise_for_status() + return + except requests.exceptions.RequestException as e: + if i == SLACK_ADMIN_NOTIFICATIONS_RETRY_COUNT: + if raise_exception_on_failure: + raise e + else: + logging.error( + f"Got HTTP {response.status_code} while sending slack notification to slack admin. " + f"HTTP response was {response.text}. Unable to notify admins of " + f"the following message: {slack_message}" + ) + else: + # Exponential backoff to try to avoid triggering Slack's rate limiting logic + time.sleep(2**i)