From 0cb73f207a0aecbbf6f23fc709665b3e8372341c Mon Sep 17 00:00:00 2001 From: Andrew Dickinson Date: Tue, 6 May 2025 00:30:57 -0400 Subject: [PATCH 1/5] Keep Stripe metadata in sync with post_save hook --- .env.sample | 55 ++--- pyproject.toml | 1 + src/meshapi/tests/sample_data.py | 1 + .../tests/test_install_create_signals.py | 196 ++++++++++++++++++ src/meshapi/tests/test_stripe_helpers.py | 155 ++++++++++++++ src/meshapi/util/events/__init__.py | 1 + .../util/events/update_stripe_subscription.py | 156 ++++++++++++++ src/meshdb/settings.py | 1 + 8 files changed, 541 insertions(+), 25 deletions(-) create mode 100644 src/meshapi/tests/test_stripe_helpers.py create mode 100644 src/meshapi/util/events/update_stripe_subscription.py diff --git a/.env.sample b/.env.sample index e02667eb..47ec2c14 100644 --- a/.env.sample +++ b/.env.sample @@ -8,24 +8,9 @@ DB_PASSWORD_RO=secret # DB_HOST= DB_PORT=5432 - -# For password reset emails -SMTP_HOST= -SMTP_PORT= -SMTP_USER= -SMTP_PASSWORD= - # For local testing with Minio. Don't include unless you have minio configured # S3_ENDPOINT="http://127.0.0.1:9000" -# Backups -AWS_ACCESS_KEY_ID=sampleaccesskey -AWS_SECRET_ACCESS_KEY=samplesecretkey - -# For replaying Join Records -JOIN_RECORD_BUCKET_NAME="meshdb-join-form-log" -JOIN_RECORD_PREFIX="join-form-submissions-dev" - # Change to 'redis' when using meshdb in docker-compose. # Defaults to redis://localhost:6379/0 # CELERY_BROKER= @@ -42,9 +27,6 @@ QUERY_PSK=localdev SITE_BASE_URL=http://localhost:8000 -# Integ Testing Credentials -INTEG_TEST_MESHDB_API_TOKEN= - # Comment this out to enter prod mode DEBUG=True DISABLE_PROFILING=False # Set to True to disable profiling. Profiling also requires DEBUG=True @@ -52,24 +34,45 @@ DISABLE_PROFILING=False # Set to True to disable profiling. Profiling also requi # Comment this out to allow edits to the panoramas in the admin panel DISALLOW_PANO_EDITS=True -# https://github.com/settings/tokens -PANO_GITHUB_TOKEN= - # Docker compose environment variables # Set this to true in prod. Use false in dev COMPOSE_EXTERNAL_NETWORK=false # Set this to traefik-net in prod. Use api in dev COMPOSE_NETWORK_NAME=api -UISP_URL=https://uisp.mesh.nycmesh.net/nms -UISP_USER=nycmesh_readonly -UISP_PASS= - ADMIN_MAP_BASE_URL=http://adminmap.devdb.nycmesh.net MAP_BASE_URL=https://map.nycmesh.net LOS_URL=https://los.devdb.nycmesh.net FORMS_URL=https://forms.devdb.nycmesh.net +####################################################################################################### +# Everything below this point is not needed for most devs, don't sweat it if you don't have it +####################################################################################################### + +# Integ Testing Credentials +INTEG_TEST_MESHDB_API_TOKEN= + +# For password reset emails +SMTP_HOST= +SMTP_PORT= +SMTP_USER= +SMTP_PASSWORD= + +# Backups +AWS_ACCESS_KEY_ID=sampleaccesskey +AWS_SECRET_ACCESS_KEY=samplesecretkey + +# For replaying Join Records +JOIN_RECORD_BUCKET_NAME="meshdb-join-form-log" +JOIN_RECORD_PREFIX="join-form-submissions-dev" + +# https://github.com/settings/tokens +PANO_GITHUB_TOKEN= + +UISP_URL=https://uisp.mesh.nycmesh.net/nms +UISP_USER=nycmesh_readonly +UISP_PASS= + OSTICKET_URL=https://support.nycmesh.net OSTICKET_API_TOKEN= OSTICKET_NEW_TICKET_ENDPOINT=https://devsupport.nycmesh.net/api/http.php/tickets.json @@ -81,3 +84,5 @@ RECAPTCHA_DISABLE_VALIDATION=True # Set this to false in production! RECAPTCHA_SERVER_SECRET_KEY_V2= RECAPTCHA_SERVER_SECRET_KEY_V3= RECAPTCHA_INVISIBLE_TOKEN_SCORE_THRESHOLD=0.5 + +STRIPE_API_TOKEN= diff --git a/pyproject.toml b/pyproject.toml index a9147979..9792e1df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "django-import-export==4.0.*", "boto3==1.34.*", "six==1.16.0", + "stripe==12.1.*", "django-flags==5.0.*", "django-sql-explorer==5.2.*", "django-simple-history==3.7.*", diff --git a/src/meshapi/tests/sample_data.py b/src/meshapi/tests/sample_data.py index 086ae1c3..b9852930 100644 --- a/src/meshapi/tests/sample_data.py +++ b/src/meshapi/tests/sample_data.py @@ -40,6 +40,7 @@ "unit": "3", "roof_access": True, "notes": "Referral: Read about it on the internet", + "stripe_subscription_id": "sub_NotARealIDValue", } sample_address_response = { diff --git a/src/meshapi/tests/test_install_create_signals.py b/src/meshapi/tests/test_install_create_signals.py index 70492dd5..6bb8c75f 100644 --- a/src/meshapi/tests/test_install_create_signals.py +++ b/src/meshapi/tests/test_install_create_signals.py @@ -1,4 +1,5 @@ import json +from unittest import mock from unittest.mock import patch import requests_mock @@ -6,6 +7,7 @@ from flags.state import disable_flag, enable_flag from meshapi.models import Building, Install, Member, Node +from meshapi.serializers import InstallSerializer from meshapi.tests.sample_data import sample_building, sample_install, sample_member @@ -22,12 +24,29 @@ def setUp(self): self.maxDiff = None + self.mock_add_install_to_stripe_subscription_patch = mock.patch( + "meshapi.util.events.update_stripe_subscription.add_install_to_subscription" + ) + self.mock_add_install_to_stripe_subscription = self.mock_add_install_to_stripe_subscription_patch.start() + self.mock_remove_install_from_stripe_subscription_patch = mock.patch( + "meshapi.util.events.update_stripe_subscription.remove_install_from_subscription" + ) + self.mock_remove_install_from_stripe_subscription = ( + self.mock_remove_install_from_stripe_subscription_patch.start() + ) + + def tearDown(self): + self.mock_add_install_to_stripe_subscription_patch.stop() + self.mock_remove_install_from_stripe_subscription_patch.stop() + @requests_mock.Mocker() def test_no_events_happen_by_default(self, request_mocker): install = Install(**self.sample_install_copy) install.save() self.assertEqual(len(request_mocker.request_history), 0) + self.mock_add_install_to_stripe_subscription.assert_not_called() + self.mock_remove_install_from_stripe_subscription.assert_not_called() @patch( "meshapi.util.events.join_requests_slack_channel.SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL", @@ -324,3 +343,180 @@ def test_many_retry_no_crash_on_integration_404(self, request_mocker): ), 4, ) + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + def test_constructing_install_calls_stripe_install_number_add(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install = Install(**self.sample_install_copy) + install.save() + + self.mock_add_install_to_stripe_subscription.assert_called_with( + install.install_number, install.stripe_subscription_id + ) + self.mock_remove_install_from_stripe_subscription.assert_not_called() + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + def test_constructing_install_no_subscription_id_doesnt_call_stripe_install_number_add(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install = Install(**self.sample_install_copy) + install.stripe_subscription_id = None + install.save() + + self.mock_add_install_to_stripe_subscription.assert_not_called() + self.mock_remove_install_from_stripe_subscription.assert_not_called() + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + def test_adding_a_subscription_id_calls_stripe_install_number_remove(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + disable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install = Install(**self.sample_install_copy) + install.stripe_subscription_id = None + install.save() + + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install.stripe_subscription_id = "sub_foobar" + install.save() + + self.mock_add_install_to_stripe_subscription.assert_called_with(install.install_number, "sub_foobar") + self.mock_remove_install_from_stripe_subscription.assert_not_called() + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + def test_removing_a_subscription_id_calls_stripe_install_number_remove(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + disable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install = Install(**self.sample_install_copy) + original_stripe_subscription_id = install.stripe_subscription_id + install.save() + + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install.stripe_subscription_id = None + install.save() + + self.mock_remove_install_from_stripe_subscription.assert_called_with( + install.install_number, original_stripe_subscription_id + ) + self.mock_add_install_to_stripe_subscription.assert_not_called() + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + def test_modifying_a_subscription_id_calls_stripe_install_number_add(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + disable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install = Install(**self.sample_install_copy) + original_stripe_subscription_id = install.stripe_subscription_id + install.save() + + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install.stripe_subscription_id = "sub_foobarbaz" + install.save() + + self.mock_remove_install_from_stripe_subscription.assert_called_with( + install.install_number, original_stripe_subscription_id + ) + self.mock_add_install_to_stripe_subscription.assert_called_with( + install.install_number, install.stripe_subscription_id + ) + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + def test_modifying_an_install_without_changing_subscription_id_no_stripe_calls(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + disable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install1 = Install(**self.sample_install_copy) + install1.save() + + self.assertIsNotNone(install1.stripe_subscription_id) + + install2 = Install(**self.sample_install_copy) + install2.stripe_subscription_id = None + install2.save() + + self.assertIsNone(install2.stripe_subscription_id) + + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + install1.notes = "Some unrelated change" + install1.save() + + install2.notes = "Some unrelated change" + install2.save() + + self.assertIsNotNone(install1.stripe_subscription_id) + self.assertIsNone(install2.stripe_subscription_id) + + self.mock_remove_install_from_stripe_subscription.assert_not_called() + self.mock_add_install_to_stripe_subscription.assert_not_called() + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + None, + ) + def test_constructing_install_no_stripe_token_doesnt_fail_create(self): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + self.mock_add_install_to_stripe_subscription.side_effect = RuntimeError() + + install = Install(**self.sample_install_copy) + install.save() + + self.mock_remove_install_from_stripe_subscription.assert_not_called() + self.mock_add_install_to_stripe_subscription.assert_not_called() + + @patch( + "meshapi.util.events.update_stripe_subscription.STRIPE_API_TOKEN", + "mock-token", + ) + @patch("meshapi.util.events.update_stripe_subscription.notify_administrators_of_data_issue") + def test_constructing_install_stripe_exception_sends_slack_admin_alert( + self, mock_notify_administrators_of_data_issue + ): + disable_flag("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") + disable_flag("INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS") + enable_flag("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") + + self.mock_add_install_to_stripe_subscription.side_effect = RuntimeError() + + install = Install(**self.sample_install_copy) + install.save() + + mock_notify_administrators_of_data_issue.assert_called_with( + [install], + InstallSerializer, + "Fatal exception (after retries) when trying to update the Stripe subscription(s): " + "[None, 'sub_NotARealIDValue']", + ) diff --git a/src/meshapi/tests/test_stripe_helpers.py b/src/meshapi/tests/test_stripe_helpers.py new file mode 100644 index 00000000..6d0a9b1a --- /dev/null +++ b/src/meshapi/tests/test_stripe_helpers.py @@ -0,0 +1,155 @@ +from unittest.mock import patch, MagicMock + +import pytest +import stripe.error +from django.test import TestCase + +from meshapi.util.events.update_stripe_subscription import ( + add_install_to_subscription, + remove_install_from_subscription, + fetch_existing_installs, +) + + +class TestStripeHelpers(TestCase): + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_fetch_single(self, mock_stripe): + mock_subscription = MagicMock() + mock_subscription.metadata = {"installs": "1234"} + + mock_stripe.Subscription.retrieve.return_value = mock_subscription + + self.assertEqual(fetch_existing_installs("sub_mockid"), [1234]) + mock_stripe.Subscription.retrieve.assert_called_with("sub_mockid") + + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_fetch_many(self, mock_stripe): + mock_subscription = MagicMock() + mock_subscription.metadata = {"installs": "1234,567,9810"} + + mock_stripe.Subscription.retrieve.return_value = mock_subscription + + self.assertEqual(fetch_existing_installs("sub_mockid"), [1234, 567, 9810]) + mock_stripe.Subscription.retrieve.assert_called_with("sub_mockid") + + @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") + def test_fetch_non_existent(self, mock_stripe_Subscription): + mock_stripe_Subscription.retrieve.side_effect = stripe.error.InvalidRequestError( + "foo_message", "foo_param", http_status=404 + ) + + self.assertEqual(fetch_existing_installs("sub_mockid"), None) + mock_stripe_Subscription.retrieve.assert_called_with("sub_mockid") + + @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") + def test_fetch_other_invalid(self, mock_stripe_Subscription): + mock_stripe_Subscription.retrieve.side_effect = stripe.error.InvalidRequestError("foo_message", "foo_param") + + with pytest.raises(RuntimeError): + fetch_existing_installs("sub_mockid") + + mock_stripe_Subscription.retrieve.assert_called_with("sub_mockid") + self.assertEqual(mock_stripe_Subscription.retrieve.call_count, 4) + + @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") + def test_fetch_other_error(self, mock_stripe_Subscription): + mock_stripe_Subscription.retrieve.side_effect = stripe.error.PermissionError + + with pytest.raises(RuntimeError): + fetch_existing_installs("sub_mockid") + + mock_stripe_Subscription.retrieve.assert_called_with("sub_mockid") + self.assertEqual(mock_stripe_Subscription.retrieve.call_count, 4) + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_add_clean_slate(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [] + + add_install_to_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_called_with("sub_mockid", metadata={"installs": "1234"}) + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_add_to_existing(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [5678] + + add_install_to_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_called_with("sub_mockid", metadata={"installs": "1234,5678"}) + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_add_dont_duplicate(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [1234, 5678] + + add_install_to_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_not_called() + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_add_to_non_existent_subscription(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = None + + add_install_to_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_not_called() + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") + def test_add_throws_exception(self, mock_stripe_Subscription, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [] + + mock_stripe_Subscription.modify.side_effect = stripe.error.PermissionError + + with pytest.raises(RuntimeError): + add_install_to_subscription(1234, "sub_mockid") + + mock_stripe_Subscription.modify.assert_called_with("sub_mockid", metadata={"installs": "1234"}) + self.assertEqual(mock_stripe_Subscription.modify.call_count, 4) + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_remove_non_existing(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [] + remove_install_from_subscription(1234, "sub_mockid") + + mock_fetch_existing_installs.return_value = [5678] + remove_install_from_subscription(1234, "sub_mockid") + + mock_stripe.Subscription.modify.assert_not_called() + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_remove_install(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [1234] + + remove_install_from_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_called_with("sub_mockid", metadata={"installs": ""}) + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_remove_only_targeted_install(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [1234, 5678] + + remove_install_from_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_called_with("sub_mockid", metadata={"installs": "5678"}) + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe") + def test_remove_from_non_existent_subscription(self, mock_stripe, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = None + + remove_install_from_subscription(1234, "sub_mockid") + mock_stripe.Subscription.modify.assert_not_called() + + @patch("meshapi.util.events.update_stripe_subscription.fetch_existing_installs") + @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") + def test_remove_throws_exception(self, mock_stripe_Subscription, mock_fetch_existing_installs): + mock_fetch_existing_installs.return_value = [1234] + + mock_stripe_Subscription.modify.side_effect = stripe.error.PermissionError + + with pytest.raises(RuntimeError): + remove_install_from_subscription(1234, "sub_mockid") + + mock_stripe_Subscription.modify.assert_called_with("sub_mockid", metadata={"installs": ""}) + self.assertEqual(mock_stripe_Subscription.modify.call_count, 4) diff --git a/src/meshapi/util/events/__init__.py b/src/meshapi/util/events/__init__.py index 055b6a34..5aeb23dc 100644 --- a/src/meshapi/util/events/__init__.py +++ b/src/meshapi/util/events/__init__.py @@ -1,2 +1,3 @@ from .join_requests_slack_channel import send_join_request_slack_message from .osticket_creation import create_os_ticket_for_install +from .update_stripe_subscription import update_stripe_subscription_on_install_update diff --git a/src/meshapi/util/events/update_stripe_subscription.py b/src/meshapi/util/events/update_stripe_subscription.py new file mode 100644 index 00000000..061d5c26 --- /dev/null +++ b/src/meshapi/util/events/update_stripe_subscription.py @@ -0,0 +1,156 @@ +import logging +import os +import time +from typing import List, Optional + +import stripe +from django.db.models.base import ModelBase +from django.db.models.signals import post_save +from django.dispatch import receiver + +from meshapi.models import Install, Node +from meshapi.serializers import InstallSerializer +from meshapi.util.admin_notifications import notify_administrators_of_data_issue +from meshapi.util.django_flag_decorator import skip_if_flag_disabled +from meshapi.util.django_pglocks import advisory_lock + +STRIPE_API_TOKEN = os.environ.get("STRIPE_API_TOKEN") + + +def fetch_existing_installs(subscription_id: str) -> Optional[List[int]]: + attempts = 0 + stripe_error = None + while attempts < 4: + attempts += 1 + + try: + subscription = stripe.Subscription.retrieve(subscription_id) + return [int(install_num) for install_num in subscription.metadata.get("installs", "").split(",")] + except stripe.error.InvalidRequestError as e: + if e.http_status == 404: + return None + stripe_error = e + except stripe.error.StripeError as e: + stripe_error = e + + time.sleep(1) + + raise RuntimeError(f"Unable to query Stripe for subscription_id {subscription_id}") from stripe_error + + +def remove_install_from_subscription(install_number: int, subscription_id: str) -> None: + logging.info(f"Removing install number {install_number} from Stripe subscription_id {subscription_id}...") + existing_installs = fetch_existing_installs(subscription_id) + + if existing_installs is None: + logging.warning(f"No Stripe subscription found with id {subscription_id}. Taking no action") + return + + if install_number not in existing_installs: + logging.info( + f"Stripe subscription_id {subscription_id} does not have install {install_number}, taking no action" + ) + return + + attempts = 0 + stripe_error = None + while attempts < 4: + attempts += 1 + + try: + stripe.Subscription.modify( + subscription_id, + metadata={ + "installs": ",".join( + str(install) for install in sorted(existing_installs) if install != install_number + ) + }, + ) + return + except stripe.error.StripeError as e: + stripe_error = e + + time.sleep(1) + + raise RuntimeError( + f"Unable to remove install {install_number} from Stripe subscription_id {subscription_id}" + ) from stripe_error + + +def add_install_to_subscription(install_number: int, subscription_id: str): + logging.info(f"Adding install number {install_number} to Stripe subscription_id {subscription_id}...") + existing_installs = fetch_existing_installs(subscription_id) + + if existing_installs is None: + logging.warning(f"No Stripe subscription found with id {subscription_id}. Taking no action") + return + + if install_number in existing_installs: + logging.info(f"Stripe subscription_id {subscription_id} already has install {install_number}, taking no action") + return + + existing_installs.append(install_number) + + attempts = 0 + stripe_error = None + while attempts < 4: + attempts += 1 + + try: + stripe.Subscription.modify( + subscription_id, metadata={"installs": ",".join(str(install) for install in sorted(existing_installs))} + ) + return + except stripe.error.StripeError as e: + stripe_error = e + + time.sleep(1) + + raise RuntimeError( + f"Unable to add install {install_number} to Stripe subscription_id {subscription_id}" + ) from stripe_error + + +@receiver(post_save, sender=Install, dispatch_uid="update_stripe_subscription") +@skip_if_flag_disabled("INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS") +@advisory_lock("update_stripe_subscription") # TODO: Test this race condition +def update_stripe_subscription_on_install_update( + sender: ModelBase, instance: Install, created: bool, **kwargs: dict +) -> None: + install: Install = instance + install.refresh_from_db() + + if not STRIPE_API_TOKEN: + logging.error( + f"Unable to contact Stripe for update to install {str(install)}, did you set the STRIPE_API_TOKEN env var?" + ) + return + + stripe.api_key = STRIPE_API_TOKEN + + current_stripe_subscription_id = install.stripe_subscription_id or "" + install_history = install.history.all() + if len(install_history) > 1: + former_stripe_subscription_id = install_history[1].stripe_subscription_id + else: + former_stripe_subscription_id = None + + logging.info( + f"After an update to install number {install.install_number}, old stripe subscription ID is " + f"{former_stripe_subscription_id}, new one is {current_stripe_subscription_id}" + ) + + try: + if former_stripe_subscription_id and current_stripe_subscription_id != former_stripe_subscription_id: + remove_install_from_subscription(install.install_number, former_stripe_subscription_id) + + if current_stripe_subscription_id and current_stripe_subscription_id != former_stripe_subscription_id: + add_install_to_subscription(install.install_number, current_stripe_subscription_id) + except RuntimeError: + error_summary = ( + f"Fatal exception (after retries) when trying to update the Stripe subscription(s): " + + f"{[former_stripe_subscription_id, current_stripe_subscription_id]}" + ) + + notify_administrators_of_data_issue([install], InstallSerializer, error_summary) + logging.exception(f"{error_summary} during an update to install number {install.install_number}") diff --git a/src/meshdb/settings.py b/src/meshdb/settings.py index 53f73fb7..9ef12643 100644 --- a/src/meshdb/settings.py +++ b/src/meshdb/settings.py @@ -52,6 +52,7 @@ "JOIN_FORM_FAIL_ALL_INVISIBLE_RECAPTCHAS": [], "INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES": [], "INTEGRATION_ENABLED_CREATE_OSTICKET_TICKETS": [], + "INTEGRATION_ENABLED_UPDATE_STRIPE_SUBSCRIPTIONS": [], "INTEGRATION_OSTICKET_INCLUDE_EXISTING_NETWORK_NUMBER": [], "TASK_ENABLED_RUN_DATABASE_BACKUP": [], "TASK_ENABLED_RESET_DEV_DATABASE": [], From 5f976d4f936bfca2e399405e0566056c4e8bca15 Mon Sep 17 00:00:00 2001 From: Andrew Dickinson Date: Tue, 6 May 2025 00:31:31 -0400 Subject: [PATCH 2/5] Update reconciliation script with slack data --- scripts/reconcile_stripe_subscriptions.py | 89 ++++++++++++++++++++++- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/scripts/reconcile_stripe_subscriptions.py b/scripts/reconcile_stripe_subscriptions.py index 787115d8..be971d21 100644 --- a/scripts/reconcile_stripe_subscriptions.py +++ b/scripts/reconcile_stripe_subscriptions.py @@ -1,11 +1,15 @@ import csv +import json import os +import re import sys from csv import DictReader -from typing import List, Optional import requests +import stripe +from typing import List +stripe.api_key = os.environ["STRIPE_API_TOKEN"] MESHDB_API_TOKEN = os.environ["MESHDB_API_TOKEN"] @@ -34,11 +38,87 @@ def find_install_numbers_by_stripe_email(stripe_email: str) -> List[str]: return install_numbers +def get_subscription_ids_for_charge_id(charge_id: str) -> List[str]: + try: + # Retrieve the charge + charge = stripe.Charge.retrieve(charge_id) + + # Extract the customer ID from the charge + customer_id = charge.get("customer") + if not customer_id: + raise ValueError(f"Charge {charge_id} does not have a customer associated with it.") + + # Retrieve all subscriptions for the customer + subscriptions = stripe.Subscription.list(customer=customer_id) + + return [sub.id for sub in subscriptions.auto_paging_iter()] + except stripe.error.StripeError as e: + print(f"Stripe API error: {e.user_message or str(e)}") + return [] + except Exception as e: + print(f"Unexpected error: {str(e)}") + return [] + + def main(): - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} ") + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ") sys.exit(1) + slack_charge_mapping = {} + slack_export_file = sys.argv[2] + with open(slack_export_file, "r") as f: + slack_data = json.load(f) + for message in slack_data["messages"]: + if message.get("slackdump_thread_replies") and message.get("attachments"): + text_description = message["attachments"][0]["text"] + if "https://manage.stripe.com/payments" in text_description and "charged" in text_description: + stripe_id_match = re.search(r"https://manage\.stripe\.com/payments/(ch_.*)\|", text_description) + if stripe_id_match: + stripe_charge_id = stripe_id_match.group(1) + # print(message["attachments"][0]["text"]) + # print(stripe_id) + replies = [reply["text"] for reply in message["slackdump_thread_replies"] if reply.get("text")] + # print(replies) + replies_joined = "\n".join(replies) + if "thank you" in replies_joined.lower(): + continue + + install_number_pound_match = re.search(r"#(\d{3,5})", replies_joined) + if install_number_pound_match: + slack_charge_mapping[stripe_charge_id] = install_number_pound_match.group(1) + continue + + install_number_match = re.search( + r"(? Date: Tue, 6 May 2025 00:34:50 -0400 Subject: [PATCH 3/5] Formatting & types --- scripts/reconcile_stripe_subscriptions.py | 2 +- src/meshapi/tests/test_stripe_helpers.py | 16 +++++++-------- .../util/events/update_stripe_subscription.py | 20 +++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/reconcile_stripe_subscriptions.py b/scripts/reconcile_stripe_subscriptions.py index be971d21..fabd4939 100644 --- a/scripts/reconcile_stripe_subscriptions.py +++ b/scripts/reconcile_stripe_subscriptions.py @@ -4,10 +4,10 @@ import re import sys from csv import DictReader +from typing import List import requests import stripe -from typing import List stripe.api_key = os.environ["STRIPE_API_TOKEN"] MESHDB_API_TOKEN = os.environ["MESHDB_API_TOKEN"] diff --git a/src/meshapi/tests/test_stripe_helpers.py b/src/meshapi/tests/test_stripe_helpers.py index 6d0a9b1a..9851e641 100644 --- a/src/meshapi/tests/test_stripe_helpers.py +++ b/src/meshapi/tests/test_stripe_helpers.py @@ -1,13 +1,13 @@ -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch import pytest -import stripe.error +import stripe from django.test import TestCase from meshapi.util.events.update_stripe_subscription import ( add_install_to_subscription, - remove_install_from_subscription, fetch_existing_installs, + remove_install_from_subscription, ) @@ -34,7 +34,7 @@ def test_fetch_many(self, mock_stripe): @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") def test_fetch_non_existent(self, mock_stripe_Subscription): - mock_stripe_Subscription.retrieve.side_effect = stripe.error.InvalidRequestError( + mock_stripe_Subscription.retrieve.side_effect = stripe.InvalidRequestError( "foo_message", "foo_param", http_status=404 ) @@ -43,7 +43,7 @@ def test_fetch_non_existent(self, mock_stripe_Subscription): @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") def test_fetch_other_invalid(self, mock_stripe_Subscription): - mock_stripe_Subscription.retrieve.side_effect = stripe.error.InvalidRequestError("foo_message", "foo_param") + mock_stripe_Subscription.retrieve.side_effect = stripe.InvalidRequestError("foo_message", "foo_param") with pytest.raises(RuntimeError): fetch_existing_installs("sub_mockid") @@ -53,7 +53,7 @@ def test_fetch_other_invalid(self, mock_stripe_Subscription): @patch("meshapi.util.events.update_stripe_subscription.stripe.Subscription") def test_fetch_other_error(self, mock_stripe_Subscription): - mock_stripe_Subscription.retrieve.side_effect = stripe.error.PermissionError + mock_stripe_Subscription.retrieve.side_effect = stripe.PermissionError with pytest.raises(RuntimeError): fetch_existing_installs("sub_mockid") @@ -98,7 +98,7 @@ def test_add_to_non_existent_subscription(self, mock_stripe, mock_fetch_existing def test_add_throws_exception(self, mock_stripe_Subscription, mock_fetch_existing_installs): mock_fetch_existing_installs.return_value = [] - mock_stripe_Subscription.modify.side_effect = stripe.error.PermissionError + mock_stripe_Subscription.modify.side_effect = stripe.PermissionError with pytest.raises(RuntimeError): add_install_to_subscription(1234, "sub_mockid") @@ -146,7 +146,7 @@ def test_remove_from_non_existent_subscription(self, mock_stripe, mock_fetch_exi def test_remove_throws_exception(self, mock_stripe_Subscription, mock_fetch_existing_installs): mock_fetch_existing_installs.return_value = [1234] - mock_stripe_Subscription.modify.side_effect = stripe.error.PermissionError + mock_stripe_Subscription.modify.side_effect = stripe.PermissionError with pytest.raises(RuntimeError): remove_install_from_subscription(1234, "sub_mockid") diff --git a/src/meshapi/util/events/update_stripe_subscription.py b/src/meshapi/util/events/update_stripe_subscription.py index 061d5c26..d7753dcd 100644 --- a/src/meshapi/util/events/update_stripe_subscription.py +++ b/src/meshapi/util/events/update_stripe_subscription.py @@ -8,7 +8,7 @@ from django.db.models.signals import post_save from django.dispatch import receiver -from meshapi.models import Install, Node +from meshapi.models import Install from meshapi.serializers import InstallSerializer from meshapi.util.admin_notifications import notify_administrators_of_data_issue from meshapi.util.django_flag_decorator import skip_if_flag_disabled @@ -19,18 +19,18 @@ def fetch_existing_installs(subscription_id: str) -> Optional[List[int]]: attempts = 0 - stripe_error = None + stripe_error: Optional[Exception] = None while attempts < 4: attempts += 1 try: subscription = stripe.Subscription.retrieve(subscription_id) return [int(install_num) for install_num in subscription.metadata.get("installs", "").split(",")] - except stripe.error.InvalidRequestError as e: + except stripe.InvalidRequestError as e: if e.http_status == 404: return None stripe_error = e - except stripe.error.StripeError as e: + except stripe.StripeError as e: stripe_error = e time.sleep(1) @@ -53,7 +53,7 @@ def remove_install_from_subscription(install_number: int, subscription_id: str) return attempts = 0 - stripe_error = None + stripe_error: Optional[Exception] = None while attempts < 4: attempts += 1 @@ -67,7 +67,7 @@ def remove_install_from_subscription(install_number: int, subscription_id: str) }, ) return - except stripe.error.StripeError as e: + except stripe.StripeError as e: stripe_error = e time.sleep(1) @@ -77,7 +77,7 @@ def remove_install_from_subscription(install_number: int, subscription_id: str) ) from stripe_error -def add_install_to_subscription(install_number: int, subscription_id: str): +def add_install_to_subscription(install_number: int, subscription_id: str) -> None: logging.info(f"Adding install number {install_number} to Stripe subscription_id {subscription_id}...") existing_installs = fetch_existing_installs(subscription_id) @@ -92,7 +92,7 @@ def add_install_to_subscription(install_number: int, subscription_id: str): existing_installs.append(install_number) attempts = 0 - stripe_error = None + stripe_error: Optional[Exception] = None while attempts < 4: attempts += 1 @@ -101,7 +101,7 @@ def add_install_to_subscription(install_number: int, subscription_id: str): subscription_id, metadata={"installs": ",".join(str(install) for install in sorted(existing_installs))} ) return - except stripe.error.StripeError as e: + except stripe.StripeError as e: stripe_error = e time.sleep(1) @@ -148,7 +148,7 @@ def update_stripe_subscription_on_install_update( add_install_to_subscription(install.install_number, current_stripe_subscription_id) except RuntimeError: error_summary = ( - f"Fatal exception (after retries) when trying to update the Stripe subscription(s): " + "Fatal exception (after retries) when trying to update the Stripe subscription(s): " + f"{[former_stripe_subscription_id, current_stripe_subscription_id]}" ) From fe585c0c5a89085db5f44d72f481cfd08b93d61e Mon Sep 17 00:00:00 2001 From: Andrew Dickinson Date: Tue, 6 May 2025 00:48:48 -0400 Subject: [PATCH 4/5] Fix: sample ID is too short --- src/meshapi/tests/sample_data.py | 2 +- src/meshapi/tests/test_install_create_signals.py | 2 +- src/meshapi/tests/test_serializers.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/meshapi/tests/sample_data.py b/src/meshapi/tests/sample_data.py index b9852930..420d43d4 100644 --- a/src/meshapi/tests/sample_data.py +++ b/src/meshapi/tests/sample_data.py @@ -40,7 +40,7 @@ "unit": "3", "roof_access": True, "notes": "Referral: Read about it on the internet", - "stripe_subscription_id": "sub_NotARealIDValue", + "stripe_subscription_id": "sub_NotARealSubscriptionIDValue", } sample_address_response = { diff --git a/src/meshapi/tests/test_install_create_signals.py b/src/meshapi/tests/test_install_create_signals.py index 6bb8c75f..f35e8fb1 100644 --- a/src/meshapi/tests/test_install_create_signals.py +++ b/src/meshapi/tests/test_install_create_signals.py @@ -518,5 +518,5 @@ def test_constructing_install_stripe_exception_sends_slack_admin_alert( [install], InstallSerializer, "Fatal exception (after retries) when trying to update the Stripe subscription(s): " - "[None, 'sub_NotARealIDValue']", + "[None, 'sub_NotARealSubscriptionIDValue']", ) diff --git a/src/meshapi/tests/test_serializers.py b/src/meshapi/tests/test_serializers.py index cc1c3e15..c0eaaa3c 100644 --- a/src/meshapi/tests/test_serializers.py +++ b/src/meshapi/tests/test_serializers.py @@ -183,7 +183,7 @@ def test_views_get_install(self): "request_date": "2022-02-27T00:00:00Z", "roof_access": True, "status": "Active", - "stripe_subscription_id": None, + "stripe_subscription_id": "sub_NotARealSubscriptionIDValue", "ticket_number": "69", "unit": "3", } From c66118299195dd1cc26c42594d0963a17f6c821e Mon Sep 17 00:00:00 2001 From: Andrew Dickinson Date: Tue, 6 May 2025 22:57:28 -0400 Subject: [PATCH 5/5] Update reconciliation script --- scripts/reconcile_stripe_subscriptions.py | 66 ++++++++++++++++------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/scripts/reconcile_stripe_subscriptions.py b/scripts/reconcile_stripe_subscriptions.py index fabd4939..09f36c25 100644 --- a/scripts/reconcile_stripe_subscriptions.py +++ b/scripts/reconcile_stripe_subscriptions.py @@ -22,7 +22,9 @@ def find_install_numbers_by_stripe_email(stripe_email: str) -> List[str]: member_data = member_response.json() - install_numbers = [] + stripe_email_explicit_installs = [] + active_installs = [] + other_installs = [] for member in member_data["results"]: for install in member["installs"]: install_detail_response = requests.get( @@ -31,11 +33,16 @@ def find_install_numbers_by_stripe_email(stripe_email: str) -> List[str]: ) install_detail_response.raise_for_status() install_detail = install_detail_response.json() + if member["stripe_email_address"] == stripe_email: + stripe_email_explicit_installs.append(install_detail) + elif install_detail["status"] == "Active": + active_installs.append(install_detail) + else: + other_installs.append(install_detail) - if install_detail["status"] == "Active": - install_numbers.append(str(install["install_number"])) - - return install_numbers + return [ + str(install["install_number"]) for install in stripe_email_explicit_installs + active_installs + other_installs + ] def get_subscription_ids_for_charge_id(charge_id: str) -> List[str]: @@ -43,10 +50,10 @@ def get_subscription_ids_for_charge_id(charge_id: str) -> List[str]: # Retrieve the charge charge = stripe.Charge.retrieve(charge_id) - # Extract the customer ID from the charge + # Extract the customer ID from the charge (if present) customer_id = charge.get("customer") if not customer_id: - raise ValueError(f"Charge {charge_id} does not have a customer associated with it.") + return [] # Retrieve all subscriptions for the customer subscriptions = stripe.Subscription.list(customer=customer_id) @@ -65,6 +72,7 @@ def main(): print(f"Usage: {sys.argv[0]} ") sys.exit(1) + print("Parsing slack export...") slack_charge_mapping = {} slack_export_file = sys.argv[2] with open(slack_export_file, "r") as f: @@ -111,29 +119,47 @@ def main(): slack_charge_mapping[stripe_charge_id] = install_number continue + print("Querying stripe to convert payment IDs to subscription IDs...") slack_subscription_mapping = {} - for stripe_charge_id, install_number in slack_charge_mapping.items(): + for i, (stripe_charge_id, install_number) in enumerate(slack_charge_mapping.items()): + if i % 100 == 0: + print(f"{i} / {len(slack_charge_mapping)}") + stripe_subscription_ids = get_subscription_ids_for_charge_id(stripe_charge_id) for stripe_subscription_id in stripe_subscription_ids: if stripe_subscription_id not in slack_subscription_mapping: slack_subscription_mapping[stripe_subscription_id] = [] slack_subscription_mapping[stripe_subscription_id].append(install_number) + print("Loading stripe CSV export...") stripe_csv_filename = sys.argv[1] + stripe_csv_data = [] with open(stripe_csv_filename, "r") as csv_file: reader = DictReader(csv_file) - - with open("output.csv", "w") as output_file: - writer = csv.DictWriter(output_file, fieldnames=list(reader.fieldnames) + ["install_nums"]) - - for row in reader: - stripe_email = row["Customer Email"] - if stripe_email: # Some rows are blank - print(row["Customer Email"]) - install_numbers = find_install_numbers_by_stripe_email(stripe_email) - row["install_nums_email"] = ",".join(install_numbers) - row["install_num_slack"] = ",".join(slack_subscription_mapping.get(row["id"]) or []) - writer.writerow(row) + fieldnames = list(reader.fieldnames) + ["install_nums_email", "install_nums_slack"] + + for row in reader: + stripe_csv_data.append(row) + + print("Querying MeshDB to do email-based lookups...") + output_data = [] + for i, row in enumerate(stripe_csv_data): + if i % 100 == 0: + print(f"{i} / {len(stripe_csv_data)}") + + stripe_email = row["Customer Email"] + if stripe_email: # Some rows are blank + install_numbers = find_install_numbers_by_stripe_email(stripe_email) + row["install_nums_email"] = ",".join(install_numbers) + row["install_nums_slack"] = ",".join(slack_subscription_mapping.get(row["id"]) or []) + output_data.append(row) + + print("Writing output file...") + with open("output.csv", "w") as output_file: + writer = csv.DictWriter(output_file, fieldnames=fieldnames) + writer.writeheader() + for row in output_data: + writer.writerow(row) if __name__ == "__main__":