Merge pull request #433 from brettp/notification_threads_master

Added notification threading.

AUTHORS.rst

@@ -25,3 +25,4 @@ answer newbie questions, and generally made taiga that much better:
 - Julien Palard
 - Ricky Posner <e@eposner.com>
 - Yamila Moreno <yamila.moreno@kaleidos.net>
+- Brett Profitt <brett.profitt@gmail.com>

CHANGELOG.md

@@ -15,6 +15,7 @@
 - Project can be starred or unstarred and the fans list can be obtained.
 - Now users can watch public issues, tasks and user stories.
 - Add endpoints to show the watchers list for issues, tasks and user stories.
+- Add headers to allow threading for notification emails about changes to issues, tasks, user stories, and wiki pages. (thanks to [@brett](https://github.com/brettp)).
 - i18n.
   - Add polish (pl) translation.
   - Add portuguese (Brazil) (pt_BR) translation.

taiga/projects/notifications/services.py

@@ -14,6 +14,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
+
 from functools import partial
 
 from django.apps import apps
@@ -256,7 +258,7 @@ def send_sync_notifications(notification_id):
     """
 
     notification = HistoryChangeNotification.objects.select_for_update().get(pk=notification_id)
-    # If the las modification is too recent we ignore it
+    # If the last modification is too recent we ignore it
     now = timezone.now()
     time_diff = now - notification.updated_datetime
     if time_diff.seconds < settings.CHANGE_NOTIFICATIONS_MIN_INTERVAL:
@@ -275,11 +277,34 @@ def send_sync_notifications(notification_id):
     model = get_model_from_key(notification.key)
     template_name = _resolve_template_name(model, change_type=notification.history_type)
     email = _make_template_mail(template_name)
+    domain = settings.SITES["api"]["domain"].split(":")[0] or settings.SITES["api"]["domain"]
+
+    if "ref" in obj.snapshot:
+        msg_id = obj.snapshot["ref"]
+    elif "slug" in obj.snapshot:
+        msg_id = obj.snapshot["slug"]
+    else:
+        msg_id = 'taiga-system'
+
+    now = datetime.datetime.now()
+    format_args = {"project_slug": notification.project.slug,
+                   "project_name": notification.project.name,
+                   "msg_id": msg_id,
+                   "time": int(now.timestamp()),
+                   "domain": domain}
+
+    headers = {"Message-ID": "<{project_slug}/{msg_id}/{time}@{domain}>".format(**format_args),
+               "In-Reply-To": "<{project_slug}/{msg_id}@{domain}>".format(**format_args),
+               "References": "<{project_slug}/{msg_id}@{domain}>".format(**format_args),
+
+               "List-ID": 'Taiga/{project_name} <taiga.{project_slug}@{domain}>'.format(**format_args),
+
+               "Thread-Index": make_ms_thread_index("<{project_slug}/{msg_id}@{domain}>".format(**format_args), now)}
 
     for user in notification.notify_users.distinct():
         context["user"] = user
         context["lang"] = user.lang or settings.LANGUAGE_CODE
-        email.send(user.email, context)
+        email.send(user.email, context, headers=headers)
 
     notification.delete()
 
@@ -367,3 +392,33 @@ def set_notify_policy(notify_policy, notify_level):
 
     notify_policy.notify_level = notify_level
     notify_policy.save()
+
+
+def make_ms_thread_index(msg_id, dt):
+    """
+    Create the 22-byte base of the thread-index string in the format:
+
+    6 bytes = First 6 significant bytes of the FILETIME stamp
+    16 bytes = GUID (we're using a md5 hash of the message id)
+
+    See http://www.meridiandiscovery.com/how-to/e-mail-conversation-index-metadata-computer-forensics/
+    """
+
+    import base64
+    import hashlib
+    import struct
+
+    # Convert to FILETIME epoch (microseconds since 1601)
+    delta = datetime.date(1970, 1, 1) - datetime.date(1601, 1, 1)
+    filetime = int(dt.timestamp() + delta.total_seconds()) * 10000000
+
+    # only want the first 6 bytes
+    thread_bin = struct.pack(">Q", filetime)[:6]
+
+    # Make a guid. This is usually generated by Outlook.
+    # The format is usually >IHHQ, but we don't care since it's just a hash of the id
+    md5 = hashlib.md5(msg_id.encode('utf-8'))
+    thread_bin += md5.digest()
+
+    # base64 encode
+    return base64.b64encode(thread_bin)

tests/integration/test_notifications.py

@@ -17,6 +17,13 @@
 
 import pytest
 import time
+import math
+import base64
+import datetime
+import hashlib
+import binascii
+import struct
+
 from unittest.mock import MagicMock, patch
 
 from django.core.urlresolvers import reverse
@@ -409,6 +416,63 @@ def test_send_notifications_using_services_method(settings, mail):
     services.process_sync_notifications()
     assert len(mail.outbox) == 12
 
+    # test headers
+    events = [issue, us, task, wiki]
+    domain = settings.SITES["api"]["domain"].split(":")[0] or settings.SITES["api"]["domain"]
+    i = 0
+    for msg in mail.outbox:
+        # each event has 3 msgs
+        event = events[math.floor(i / 3)]
+
+        # each set of 3 should have the same headers
+        if i % 3 == 0:
+            if hasattr(event, 'ref'):
+                e_slug = event.ref
+            elif hasattr(event, 'slug'):
+                e_slug = event.slug
+            else:
+                e_slug = 'taiga-system'
+
+            m_id = "{project_slug}/{msg_id}".format(
+                project_slug=project.slug,
+                msg_id=e_slug
+            )
+
+            message_id = "<{m_id}/".format(m_id=m_id)
+            message_id_domain = "@{domain}>".format(domain=domain)
+            in_reply_to = "<{m_id}@{domain}>".format(m_id=m_id, domain=domain)
+            list_id = "Taiga/{p_name} <taiga.{p_slug}@{domain}>" \
+                .format(p_name=project.name, p_slug=project.slug, domain=domain)
+
+        assert msg.extra_headers
+        headers = msg.extra_headers
+
+        # can't test the time part because it's set when sending
+        # check what we can
+        assert 'Message-ID' in headers
+        assert message_id in headers.get('Message-ID')
+        assert message_id_domain in headers.get('Message-ID')
+
+        assert 'In-Reply-To' in headers
+        assert in_reply_to == headers.get('In-Reply-To')
+        assert 'References' in headers
+        assert in_reply_to == headers.get('References')
+
+        assert 'List-ID' in headers
+        assert list_id == headers.get('List-ID')
+
+        assert 'Thread-Index' in headers
+        # always is b64 encoded 22 bytes
+        assert len(base64.b64decode(headers.get('Thread-Index'))) == 22
+
+        # hashes should match for identical ids and times
+        # we check the actual method in test_ms_thread_id()
+        msg_time = headers.get('Message-ID').split('/')[2].split('@')[0]
+        msg_ts = datetime.datetime.fromtimestamp(int(msg_time))
+        assert services.make_ms_thread_index(in_reply_to, msg_ts) == headers.get('Thread-Index')
+
+        i += 1
+
 
 def test_resource_notification_test(client, settings, mail):
     settings.CHANGE_NOTIFICATIONS_MIN_INTERVAL = 1
@@ -613,3 +677,37 @@ def test_retrieve_notify_policies_by_anonymous_user(client):
     response = client.get(url, content_type="application/json")
     assert response.status_code == 404, response.status_code
     assert response.data["_error_message"] == "No NotifyPolicy matches the given query.", str(response.content)
+
+
+def test_ms_thread_id():
+    id = '<test/message@localhost>'
+    now = datetime.datetime.now()
+
+    index = services.make_ms_thread_index(id, now)
+    parsed = parse_ms_thread_index(index)
+
+    assert parsed[0] == hashlib.md5(id.encode('utf-8')).hexdigest()
+    # always only one time
+    assert (now - parsed[1][0]).seconds <= 2
+
+
+# see http://stackoverflow.com/questions/27374077/parsing-thread-index-mail-header-with-python
+def parse_ms_thread_index(index):
+    s = base64.b64decode(index)
+
+    # ours are always md5 digests
+    guid = binascii.hexlify(s[6:22]).decode('utf-8')
+
+    # if we had real guids, we'd do something like
+    # guid = struct.unpack('>IHHQ', s[6:22])
+    # guid = '%08X-%04X-%04X-%04X-%12X' % (guid[0], guid[1], guid[2], (guid[3] >> 48) & 0xFFFF, guid[3] & 0xFFFFFFFFFFFF)
+
+    f = struct.unpack('>Q', s[:6] + b'\0\0')[0]
+    ts = [datetime.datetime(1601, 1, 1) + datetime.timedelta(microseconds=f//10)]
+
+    # for the 5 byte appendixes that we won't use
+    for n in range(22, len(s), 5):
+        f = struct.unpack('>I', s[n:n+4])[0]
+        ts.append(ts[-1] + datetime.timedelta(microseconds=(f << 18)//10))
+
+    return guid, ts