diff --git a/AUTHORS.rst b/AUTHORS.rst index 95154a8a..e7b493df 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -25,3 +25,4 @@ answer newbie questions, and generally made taiga that much better: - Julien Palard - Ricky Posner - Yamila Moreno +- Brett Profitt diff --git a/CHANGELOG.md b/CHANGELOG.md index 05daf163..55f931f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Project can be starred or unstarred and the fans list can be obtained. - Now users can watch public issues, tasks and user stories. - Add endpoints to show the watchers list for issues, tasks and user stories. +- Add headers to allow threading for notification emails about changes to issues, tasks, user stories, and wiki pages. (thanks to [@brett](https://github.com/brettp)). - i18n. - Add polish (pl) translation. - Add portuguese (Brazil) (pt_BR) translation. diff --git a/taiga/projects/notifications/services.py b/taiga/projects/notifications/services.py index 778e2770..3951622a 100644 --- a/taiga/projects/notifications/services.py +++ b/taiga/projects/notifications/services.py @@ -14,6 +14,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import datetime + from functools import partial from django.apps import apps @@ -256,7 +258,7 @@ def send_sync_notifications(notification_id): """ notification = HistoryChangeNotification.objects.select_for_update().get(pk=notification_id) - # If the las modification is too recent we ignore it + # If the last modification is too recent we ignore it now = timezone.now() time_diff = now - notification.updated_datetime if time_diff.seconds < settings.CHANGE_NOTIFICATIONS_MIN_INTERVAL: @@ -275,11 +277,34 @@ def send_sync_notifications(notification_id): model = get_model_from_key(notification.key) template_name = _resolve_template_name(model, change_type=notification.history_type) email = _make_template_mail(template_name) + domain = settings.SITES["api"]["domain"].split(":")[0] or settings.SITES["api"]["domain"] + + if "ref" in obj.snapshot: + msg_id = obj.snapshot["ref"] + elif "slug" in obj.snapshot: + msg_id = obj.snapshot["slug"] + else: + msg_id = 'taiga-system' + + now = datetime.datetime.now() + format_args = {"project_slug": notification.project.slug, + "project_name": notification.project.name, + "msg_id": msg_id, + "time": int(now.timestamp()), + "domain": domain} + + headers = {"Message-ID": "<{project_slug}/{msg_id}/{time}@{domain}>".format(**format_args), + "In-Reply-To": "<{project_slug}/{msg_id}@{domain}>".format(**format_args), + "References": "<{project_slug}/{msg_id}@{domain}>".format(**format_args), + + "List-ID": 'Taiga/{project_name} '.format(**format_args), + + "Thread-Index": make_ms_thread_index("<{project_slug}/{msg_id}@{domain}>".format(**format_args), now)} for user in notification.notify_users.distinct(): context["user"] = user context["lang"] = user.lang or settings.LANGUAGE_CODE - email.send(user.email, context) + email.send(user.email, context, headers=headers) notification.delete() @@ -367,3 +392,33 @@ def set_notify_policy(notify_policy, notify_level): notify_policy.notify_level = notify_level notify_policy.save() + + +def make_ms_thread_index(msg_id, dt): + """ + Create the 22-byte base of the thread-index string in the format: + + 6 bytes = First 6 significant bytes of the FILETIME stamp + 16 bytes = GUID (we're using a md5 hash of the message id) + + See http://www.meridiandiscovery.com/how-to/e-mail-conversation-index-metadata-computer-forensics/ + """ + + import base64 + import hashlib + import struct + + # Convert to FILETIME epoch (microseconds since 1601) + delta = datetime.date(1970, 1, 1) - datetime.date(1601, 1, 1) + filetime = int(dt.timestamp() + delta.total_seconds()) * 10000000 + + # only want the first 6 bytes + thread_bin = struct.pack(">Q", filetime)[:6] + + # Make a guid. This is usually generated by Outlook. + # The format is usually >IHHQ, but we don't care since it's just a hash of the id + md5 = hashlib.md5(msg_id.encode('utf-8')) + thread_bin += md5.digest() + + # base64 encode + return base64.b64encode(thread_bin) diff --git a/tests/integration/test_notifications.py b/tests/integration/test_notifications.py index 87ea400d..1a1b98b0 100644 --- a/tests/integration/test_notifications.py +++ b/tests/integration/test_notifications.py @@ -17,6 +17,13 @@ import pytest import time +import math +import base64 +import datetime +import hashlib +import binascii +import struct + from unittest.mock import MagicMock, patch from django.core.urlresolvers import reverse @@ -409,6 +416,63 @@ def test_send_notifications_using_services_method(settings, mail): services.process_sync_notifications() assert len(mail.outbox) == 12 + # test headers + events = [issue, us, task, wiki] + domain = settings.SITES["api"]["domain"].split(":")[0] or settings.SITES["api"]["domain"] + i = 0 + for msg in mail.outbox: + # each event has 3 msgs + event = events[math.floor(i / 3)] + + # each set of 3 should have the same headers + if i % 3 == 0: + if hasattr(event, 'ref'): + e_slug = event.ref + elif hasattr(event, 'slug'): + e_slug = event.slug + else: + e_slug = 'taiga-system' + + m_id = "{project_slug}/{msg_id}".format( + project_slug=project.slug, + msg_id=e_slug + ) + + message_id = "<{m_id}/".format(m_id=m_id) + message_id_domain = "@{domain}>".format(domain=domain) + in_reply_to = "<{m_id}@{domain}>".format(m_id=m_id, domain=domain) + list_id = "Taiga/{p_name} " \ + .format(p_name=project.name, p_slug=project.slug, domain=domain) + + assert msg.extra_headers + headers = msg.extra_headers + + # can't test the time part because it's set when sending + # check what we can + assert 'Message-ID' in headers + assert message_id in headers.get('Message-ID') + assert message_id_domain in headers.get('Message-ID') + + assert 'In-Reply-To' in headers + assert in_reply_to == headers.get('In-Reply-To') + assert 'References' in headers + assert in_reply_to == headers.get('References') + + assert 'List-ID' in headers + assert list_id == headers.get('List-ID') + + assert 'Thread-Index' in headers + # always is b64 encoded 22 bytes + assert len(base64.b64decode(headers.get('Thread-Index'))) == 22 + + # hashes should match for identical ids and times + # we check the actual method in test_ms_thread_id() + msg_time = headers.get('Message-ID').split('/')[2].split('@')[0] + msg_ts = datetime.datetime.fromtimestamp(int(msg_time)) + assert services.make_ms_thread_index(in_reply_to, msg_ts) == headers.get('Thread-Index') + + i += 1 + def test_resource_notification_test(client, settings, mail): settings.CHANGE_NOTIFICATIONS_MIN_INTERVAL = 1 @@ -613,3 +677,37 @@ def test_retrieve_notify_policies_by_anonymous_user(client): response = client.get(url, content_type="application/json") assert response.status_code == 404, response.status_code assert response.data["_error_message"] == "No NotifyPolicy matches the given query.", str(response.content) + + +def test_ms_thread_id(): + id = '' + now = datetime.datetime.now() + + index = services.make_ms_thread_index(id, now) + parsed = parse_ms_thread_index(index) + + assert parsed[0] == hashlib.md5(id.encode('utf-8')).hexdigest() + # always only one time + assert (now - parsed[1][0]).seconds <= 2 + + +# see http://stackoverflow.com/questions/27374077/parsing-thread-index-mail-header-with-python +def parse_ms_thread_index(index): + s = base64.b64decode(index) + + # ours are always md5 digests + guid = binascii.hexlify(s[6:22]).decode('utf-8') + + # if we had real guids, we'd do something like + # guid = struct.unpack('>IHHQ', s[6:22]) + # guid = '%08X-%04X-%04X-%04X-%12X' % (guid[0], guid[1], guid[2], (guid[3] >> 48) & 0xFFFF, guid[3] & 0xFFFFFFFFFFFF) + + f = struct.unpack('>Q', s[:6] + b'\0\0')[0] + ts = [datetime.datetime(1601, 1, 1) + datetime.timedelta(microseconds=f//10)] + + # for the 5 byte appendixes that we won't use + for n in range(22, len(s), 5): + f = struct.unpack('>I', s[n:n+4])[0] + ts.append(ts[-1] + datetime.timedelta(microseconds=(f << 18)//10)) + + return guid, ts