Add a post_save hook to keep Stripe metadata synced to MeshDB

.env.sample

@@ -8,24 +8,9 @@ DB_PASSWORD_RO=secret
 # DB_HOST=
 DB_PORT=5432
 
-
-# For password reset emails
-SMTP_HOST=
-SMTP_PORT=
-SMTP_USER=
-SMTP_PASSWORD=
-
 # For local testing with Minio. Don't include unless you have minio configured
 # S3_ENDPOINT="http://127.0.0.1:9000"
 
-# Backups
-AWS_ACCESS_KEY_ID=sampleaccesskey
-AWS_SECRET_ACCESS_KEY=samplesecretkey
-
-# For replaying Join Records
-JOIN_RECORD_BUCKET_NAME="meshdb-join-form-log"
-JOIN_RECORD_PREFIX="join-form-submissions-dev"
-
 # Change to 'redis' when using meshdb in docker-compose.
 # Defaults to redis://localhost:6379/0
 # CELERY_BROKER=
@@ -42,34 +27,52 @@ QUERY_PSK=localdev
 
 SITE_BASE_URL=http://localhost:8000
 
-# Integ Testing Credentials
-INTEG_TEST_MESHDB_API_TOKEN=
-
 # Comment this out to enter prod mode
 DEBUG=True
 DISABLE_PROFILING=False # Set to True to disable profiling. Profiling also requires DEBUG=True
 
 # Comment this out to allow edits to the panoramas in the admin panel
 DISALLOW_PANO_EDITS=True
 
-# https://github.com/settings/tokens
-PANO_GITHUB_TOKEN=
-
 # Docker compose environment variables
 # Set this to true in prod. Use false in dev
 COMPOSE_EXTERNAL_NETWORK=false
 # Set this to traefik-net in prod. Use api in dev
 COMPOSE_NETWORK_NAME=api
 
-UISP_URL=https://uisp.mesh.nycmesh.net/nms
-UISP_USER=nycmesh_readonly
-UISP_PASS=
-
 ADMIN_MAP_BASE_URL=http://adminmap.devdb.nycmesh.net
 MAP_BASE_URL=https://map.nycmesh.net
 LOS_URL=https://los.devdb.nycmesh.net
 FORMS_URL=https://forms.devdb.nycmesh.net
 
+#######################################################################################################
+# Everything below this point is not needed for most devs, don't sweat it if you don't have it
+#######################################################################################################
+
+# Integ Testing Credentials
+INTEG_TEST_MESHDB_API_TOKEN=
+
+# For password reset emails
+SMTP_HOST=
+SMTP_PORT=
+SMTP_USER=
+SMTP_PASSWORD=
+
+# Backups
+AWS_ACCESS_KEY_ID=sampleaccesskey
+AWS_SECRET_ACCESS_KEY=samplesecretkey
+
+# For replaying Join Records
+JOIN_RECORD_BUCKET_NAME="meshdb-join-form-log"
+JOIN_RECORD_PREFIX="join-form-submissions-dev"
+
+# https://github.com/settings/tokens
+PANO_GITHUB_TOKEN=
+
+UISP_URL=https://uisp.mesh.nycmesh.net/nms
+UISP_USER=nycmesh_readonly
+UISP_PASS=
+
 OSTICKET_URL=https://support.nycmesh.net
 OSTICKET_API_TOKEN=
 OSTICKET_NEW_TICKET_ENDPOINT=https://devsupport.nycmesh.net/api/http.php/tickets.json
@@ -81,3 +84,5 @@ RECAPTCHA_DISABLE_VALIDATION=True  # Set this to false in production!
 RECAPTCHA_SERVER_SECRET_KEY_V2=
 RECAPTCHA_SERVER_SECRET_KEY_V3=
 RECAPTCHA_INVISIBLE_TOKEN_SCORE_THRESHOLD=0.5
+
+STRIPE_API_TOKEN=

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
     "django-import-export==4.0.*",
     "boto3==1.34.*",
     "six==1.16.0",
+    "stripe==12.1.*",
     "django-flags==5.0.*",
     "django-sql-explorer==5.2.*",
     "django-simple-history==3.7.*",

scripts/reconcile_stripe_subscriptions.py

@@ -1,11 +1,15 @@
 import csv
+import json
 import os
+import re
 import sys
 from csv import DictReader
-from typing import List, Optional
+from typing import List
 
 import requests
+import stripe
 
+stripe.api_key = os.environ["STRIPE_API_TOKEN"]
 MESHDB_API_TOKEN = os.environ["MESHDB_API_TOKEN"]
 
 
@@ -18,7 +22,9 @@ def find_install_numbers_by_stripe_email(stripe_email: str) -> List[str]:
 
     member_data = member_response.json()
 
-    install_numbers = []
+    stripe_email_explicit_installs = []
+    active_installs = []
+    other_installs = []
     for member in member_data["results"]:
         for install in member["installs"]:
             install_detail_response = requests.get(
@@ -27,32 +33,133 @@ def find_install_numbers_by_stripe_email(stripe_email: str) -> List[str]:
             )
             install_detail_response.raise_for_status()
             install_detail = install_detail_response.json()
+            if member["stripe_email_address"] == stripe_email:
+                stripe_email_explicit_installs.append(install_detail)
+            elif install_detail["status"] == "Active":
+                active_installs.append(install_detail)
+            else:
+                other_installs.append(install_detail)
 
-            if install_detail["status"] == "Active":
-                install_numbers.append(str(install["install_number"]))
+    return [
+        str(install["install_number"]) for install in stripe_email_explicit_installs + active_installs + other_installs
+    ]
 
-    return install_numbers
+
+def get_subscription_ids_for_charge_id(charge_id: str) -> List[str]:
+    try:
+        # Retrieve the charge
+        charge = stripe.Charge.retrieve(charge_id)
+
+        # Extract the customer ID from the charge (if present)
+        customer_id = charge.get("customer")
+        if not customer_id:
+            return []
+
+        # Retrieve all subscriptions for the customer
+        subscriptions = stripe.Subscription.list(customer=customer_id)
+
+        return [sub.id for sub in subscriptions.auto_paging_iter()]
+    except stripe.error.StripeError as e:
+        print(f"Stripe API error: {e.user_message or str(e)}")
+        return []
+    except Exception as e:
+        print(f"Unexpected error: {str(e)}")
+        return []
 
 
 def main():
-    if len(sys.argv) != 2:
-        print(f"Usage: {sys.argv[0]} <stripe-csv-file>")
+    if len(sys.argv) != 3:
+        print(f"Usage: {sys.argv[0]} <stripe-csv-file> <slack-export-file>")
         sys.exit(1)
 
+    print("Parsing slack export...")
+    slack_charge_mapping = {}
+    slack_export_file = sys.argv[2]
+    with open(slack_export_file, "r") as f:
+        slack_data = json.load(f)
+        for message in slack_data["messages"]:
+            if message.get("slackdump_thread_replies") and message.get("attachments"):
+                text_description = message["attachments"][0]["text"]
+                if "https://manage.stripe.com/payments" in text_description and "charged" in text_description:
+                    stripe_id_match = re.search(r"https://manage\.stripe\.com/payments/(ch_.*)\|", text_description)
+                    if stripe_id_match:
+                        stripe_charge_id = stripe_id_match.group(1)
+                        # print(message["attachments"][0]["text"])
+                        # print(stripe_id)
+                        replies = [reply["text"] for reply in message["slackdump_thread_replies"] if reply.get("text")]
+                        # print(replies)
+                        replies_joined = "\n".join(replies)
+                        if "thank you" in replies_joined.lower():
+                            continue
+
+                        install_number_pound_match = re.search(r"#(\d{3,5})", replies_joined)
+                        if install_number_pound_match:
+                            slack_charge_mapping[stripe_charge_id] = install_number_pound_match.group(1)
+                            continue
+
+                        install_number_match = re.search(
+                            r"(?<!NY\s)(?<!NY,\s)(?<!Address:\s)(?<!Address:\s\s)\b(\d{4,5})\b", replies_joined
+                        )
+                        if install_number_match:
+                            install_number = install_number_match.group(1)
+                            if install_number not in [
+                                "1932",
+                                "1933",
+                                "1934",
+                                "1936",
+                                "2019",
+                                "2020",
+                                "2021",
+                                "2022",
+                                "2023",
+                                "2024",
+                                "2025",
+                                "10002",
+                            ]:
+                                slack_charge_mapping[stripe_charge_id] = install_number
+                                continue
+
+    print("Querying stripe to convert payment IDs to subscription IDs...")
+    slack_subscription_mapping = {}
+    for i, (stripe_charge_id, install_number) in enumerate(slack_charge_mapping.items()):
+        if i % 100 == 0:
+            print(f"{i} / {len(slack_charge_mapping)}")
+
+        stripe_subscription_ids = get_subscription_ids_for_charge_id(stripe_charge_id)
+        for stripe_subscription_id in stripe_subscription_ids:
+            if stripe_subscription_id not in slack_subscription_mapping:
+                slack_subscription_mapping[stripe_subscription_id] = []
+            slack_subscription_mapping[stripe_subscription_id].append(install_number)
+
+    print("Loading stripe CSV export...")
     stripe_csv_filename = sys.argv[1]
+    stripe_csv_data = []
     with open(stripe_csv_filename, "r") as csv_file:
         reader = DictReader(csv_file)
+        fieldnames = list(reader.fieldnames) + ["install_nums_email", "install_nums_slack"]
+
+        for row in reader:
+            stripe_csv_data.append(row)
+
+    print("Querying MeshDB to do email-based lookups...")
+    output_data = []
+    for i, row in enumerate(stripe_csv_data):
+        if i % 100 == 0:
+            print(f"{i} / {len(stripe_csv_data)}")
 
-        with open("output.csv", "w") as output_file:
-            writer = csv.DictWriter(output_file, fieldnames=list(reader.fieldnames) + ["install_nums"])
+        stripe_email = row["Customer Email"]
+        if stripe_email:  # Some rows are blank
+            install_numbers = find_install_numbers_by_stripe_email(stripe_email)
+            row["install_nums_email"] = ",".join(install_numbers)
+            row["install_nums_slack"] = ",".join(slack_subscription_mapping.get(row["id"]) or [])
+            output_data.append(row)
 
-            for row in reader:
-                stripe_email = row["Customer Email"]
-                if stripe_email:  # Some rows are blank
-                    print(row["Customer Email"])
-                    install_numbers = find_install_numbers_by_stripe_email(stripe_email)
-                    row["install_nums"] = ",".join(install_numbers)
-                    writer.writerow(row)
+    print("Writing output file...")
+    with open("output.csv", "w") as output_file:
+        writer = csv.DictWriter(output_file, fieldnames=fieldnames)
+        writer.writeheader()
+        for row in output_data:
+            writer.writerow(row)
 
 
 if __name__ == "__main__":

src/meshapi/tests/sample_data.py

@@ -40,6 +40,7 @@
     "unit": "3",
     "roof_access": True,
     "notes": "Referral: Read about it on the internet",
+    "stripe_subscription_id": "sub_NotARealSubscriptionIDValue",
 }
 
 sample_address_response = {