Add retries to the the email sending logic

When sending email using Django's default email
backend (SMTP), and if there is a network issue, the backend
can throw SMTPServerDisconnected.

This commit adds a retry logic which retries sending the
email two times when SMTPServerDisconnected is thrown.

CHANGELOG.md

@@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file.
 - Add support for script's exit status in ping URLs (#429)
 - Improve phone number sanitization: remove spaces and hyphens
 - Change the "Test Integration" behavior for webhooks: don't retry failed requests
+- Add retries to the the email sending logic
 
 ## v1.17.0 - 2020-10-14
 

hc/lib/emails.py

@@ -1,3 +1,4 @@
+import smtplib
 from threading import Thread
 
 from django.conf import settings
@@ -6,6 +7,8 @@ from django.template.loader import render_to_string as render
 
 
 class EmailThread(Thread):
+    MAX_TRIES = 3
+
     def __init__(self, subject, text, html, to, headers):
         Thread.__init__(self)
         self.subject = subject
@@ -15,12 +18,22 @@ class EmailThread(Thread):
         self.headers = headers
 
     def run(self):
-        msg = EmailMultiAlternatives(
-            self.subject, self.text, to=(self.to,), headers=self.headers
-        )
+        for attempt in range(0, self.MAX_TRIES):
+            try:
+                msg = EmailMultiAlternatives(
+                    self.subject, self.text, to=(self.to,), headers=self.headers
+                )
 
-        msg.attach_alternative(self.html, "text/html")
-        msg.send()
+                msg.attach_alternative(self.html, "text/html")
+                msg.send()
+            except smtplib.SMTPServerDisconnected as e:
+                if attempt + 1 == self.MAX_TRIES:
+                    # This was the last attempt and it failed:
+                    # re-raise the exception
+                    raise e
+            else:
+                # There was no exception, break out of the retry loop
+                break
 
 
 def send(name, to, ctx, headers={}):