Merge pull request #382 from taigaio/issue/2981/bitbucket-webhooks-fix

Issue/2981/bitbucket webhooks fix
remotes/origin/enhancement/email-actions
David Barragán Merino 2015-06-30 17:54:16 +02:00
commit 485799918e
5 changed files with 352 additions and 76 deletions

View File

@ -7,6 +7,7 @@
- Add a "field type" property for custom fields: 'text' and 'multiline text' right now (thanks to [@artlepool](https://github.com/artlepool))
- Allow multiple actions in the commit messages.
- Now every user that coments USs, Issues or Tasks will be involved in it (add author to the watchers list).
- Fix the compatibility with BitBucket webhooks and add issues and issues comments integration.
### Misc
- API: Mixin fields 'users', 'members' and 'memberships' in ProjectDetailSerializer

View File

@ -29,18 +29,11 @@ from ipware.ip import get_ip
class BitBucketViewSet(BaseWebhookApiViewSet):
event_hook_classes = {
"push": event_hooks.PushEventHook,
"repo:push": event_hooks.PushEventHook,
"issue:created": event_hooks.IssuesEventHook,
"issue:comment_created": event_hooks.IssueCommentEventHook,
}
def _get_payload(self, request):
try:
body = parse_qs(request.body.decode("utf-8"), strict_parsing=True)
payload = body["payload"]
except (ValueError, KeyError):
raise exc.BadRequest(_("The payload is not a valid application/x-www-form-urlencoded"))
return payload
def _validate_signature(self, project, request):
secret_key = request.GET.get("key", None)
@ -75,4 +68,4 @@ class BitBucketViewSet(BaseWebhookApiViewSet):
return None
def _get_event_name(self, request):
return "push"
return request.META.get('HTTP_X_EVENT_KEY', None)

View File

@ -37,16 +37,9 @@ class PushEventHook(BaseEventHook):
if self.payload is None:
return
# In bitbucket the payload is a list! :(
for payload_element_text in self.payload:
try:
payload_element = json.loads(payload_element_text)
except ValueError:
raise exc.BadRequest(_("The payload is not valid"))
commits = payload_element.get("commits", [])
for commit in commits:
message = commit.get("message", None)
changes = self.payload.get("push", {}).get('changes', [])
for change in changes:
message = change.get("new", {}).get("target", {}).get("message", None)
self._process_message(message, None)
def _process_message(self, message, bitbucket_user):
@ -98,3 +91,98 @@ class PushEventHook(BaseEventHook):
def replace_bitbucket_references(project_url, wiki_text):
template = "\g<1>[BitBucket#\g<2>]({}/issues/\g<2>)\g<3>".format(project_url)
return re.sub(r"(\s|^)#(\d+)(\s|$)", template, wiki_text, 0, re.M)
class IssuesEventHook(BaseEventHook):
def process_event(self):
number = self.payload.get('issue', {}).get('id', None)
subject = self.payload.get('issue', {}).get('title', None)
bitbucket_url = self.payload.get('issue', {}).get('links', {}).get('html', {}).get('href', None)
bitbucket_user_id = self.payload.get('actor', {}).get('user', {}).get('uuid', None)
bitbucket_user_name = self.payload.get('actor', {}).get('user', {}).get('username', None)
bitbucket_user_url = self.payload.get('actor', {}).get('user', {}).get('links', {}).get('html', {}).get('href')
project_url = self.payload.get('repository', {}).get('links', {}).get('html', {}).get('href', None)
description = self.payload.get('issue', {}).get('content', {}).get('raw', '')
description = replace_bitbucket_references(project_url, description)
user = get_bitbucket_user(bitbucket_user_id)
if not all([subject, bitbucket_url, project_url]):
raise ActionSyntaxException(_("Invalid issue information"))
issue = Issue.objects.create(
project=self.project,
subject=subject,
description=description,
status=self.project.default_issue_status,
type=self.project.default_issue_type,
severity=self.project.default_severity,
priority=self.project.default_priority,
external_reference=['bitbucket', bitbucket_url],
owner=user
)
take_snapshot(issue, user=user)
if number and subject and bitbucket_user_name and bitbucket_user_url:
comment = _("Issue created by [@{bitbucket_user_name}]({bitbucket_user_url} "
"\"See @{bitbucket_user_name}'s BitBucket profile\") "
"from BitBucket.\nOrigin BitBucket issue: [gh#{number} - {subject}]({bitbucket_url} "
"\"Go to 'gh#{number} - {subject}'\"):\n\n"
"{description}").format(bitbucket_user_name=bitbucket_user_name,
bitbucket_user_url=bitbucket_user_url,
number=number,
subject=subject,
bitbucket_url=bitbucket_url,
description=description)
else:
comment = _("Issue created from BitBucket.")
snapshot = take_snapshot(issue, comment=comment, user=user)
send_notifications(issue, history=snapshot)
class IssueCommentEventHook(BaseEventHook):
def process_event(self):
number = self.payload.get('issue', {}).get('id', None)
subject = self.payload.get('issue', {}).get('title', None)
bitbucket_url = self.payload.get('issue', {}).get('links', {}).get('html', {}).get('href', None)
bitbucket_user_id = self.payload.get('actor', {}).get('user', {}).get('uuid', None)
bitbucket_user_name = self.payload.get('actor', {}).get('user', {}).get('username', None)
bitbucket_user_url = self.payload.get('actor', {}).get('user', {}).get('links', {}).get('html', {}).get('href')
project_url = self.payload.get('repository', {}).get('links', {}).get('html', {}).get('href', None)
comment_message = self.payload.get('comment', {}).get('content', {}).get('raw', '')
comment_message = replace_bitbucket_references(project_url, comment_message)
user = get_bitbucket_user(bitbucket_user_id)
if not all([comment_message, bitbucket_url, project_url]):
raise ActionSyntaxException(_("Invalid issue comment information"))
issues = Issue.objects.filter(external_reference=["bitbucket", bitbucket_url])
tasks = Task.objects.filter(external_reference=["bitbucket", bitbucket_url])
uss = UserStory.objects.filter(external_reference=["bitbucket", bitbucket_url])
for item in list(issues) + list(tasks) + list(uss):
if number and subject and bitbucket_user_name and bitbucket_user_url:
comment = _("Comment by [@{bitbucket_user_name}]({bitbucket_user_url} "
"\"See @{bitbucket_user_name}'s BitBucket profile\") "
"from BitBucket.\nOrigin BitBucket issue: [gh#{number} - {subject}]({bitbucket_url} "
"\"Go to 'gh#{number} - {subject}'\")\n\n"
"{message}").format(bitbucket_user_name=bitbucket_user_name,
bitbucket_user_url=bitbucket_user_url,
number=number,
subject=subject,
bitbucket_url=bitbucket_url,
message=comment_message)
else:
comment = _("Comment From BitBucket:\n\n{message}").format(message=comment_message)
snapshot = take_snapshot(item, comment=comment, user=user)
send_notifications(item, history=snapshot)

View File

@ -40,16 +40,5 @@ def get_or_generate_config(project):
return g_config
def get_bitbucket_user(user_email):
user = None
if user_email:
try:
user = User.objects.get(email=user_email)
except User.DoesNotExist:
pass
if user is None:
user = User.objects.get(is_system=True, username__startswith="bitbucket")
return user
def get_bitbucket_user(user_id):
return User.objects.get(is_system=True, username__startswith="bitbucket")

View File

@ -14,6 +14,10 @@ from taiga.hooks.exceptions import ActionSyntaxException
from taiga.projects.issues.models import Issue
from taiga.projects.tasks.models import Task
from taiga.projects.userstories.models import UserStory
from taiga.projects.models import Membership
from taiga.projects.history.services import get_history_queryset_by_model_instance, take_snapshot
from taiga.projects.notifications.choices import NotifyLevel
from taiga.projects.notifications.models import NotifyPolicy
from taiga.projects import services
from .. import factories as f
@ -30,8 +34,9 @@ def test_bad_signature(client):
url = reverse("bitbucket-hook-list")
url = "{}?project={}&key={}".format(url, project.id, "badbadbad")
data = {}
response = client.post(url, urllib.parse.urlencode(data, True), content_type="application/x-www-form-urlencoded")
data = "{}"
response = client.post(url, data, content_type="application/json", HTTP_X_EVENT_KEY="repo:push")
response_content = response.data
assert response.status_code == 400
assert "Bad signature" in response_content["_error_message"]
@ -47,10 +52,11 @@ def test_ok_signature(client):
url = reverse("bitbucket-hook-list")
url = "{}?project={}&key={}".format(url, project.id, "tpnIwJDz4e")
data = {'payload': ['{"commits": []}']}
data = json.dumps({"push": {"changes": [{"new": {"target": { "message": "test message"}}}]}})
response = client.post(url,
urllib.parse.urlencode(data, True),
content_type="application/x-www-form-urlencoded",
data,
content_type="application/json",
HTTP_X_EVENT_KEY="repo:push",
REMOTE_ADDR=settings.BITBUCKET_VALID_ORIGIN_IPS[0])
assert response.status_code == 204
@ -65,10 +71,11 @@ def test_invalid_ip(client):
url = reverse("bitbucket-hook-list")
url = "{}?project={}&key={}".format(url, project.id, "tpnIwJDz4e")
data = {'payload': ['{"commits": []}']}
data = json.dumps({"push": {"changes": [{"new": {"target": { "message": "test message"}}}]}})
response = client.post(url,
urllib.parse.urlencode(data, True),
content_type="application/x-www-form-urlencoded",
data,
content_type="application/json",
HTTP_X_EVENT_KEY="repo:push",
REMOTE_ADDR="111.111.111.112")
assert response.status_code == 400
@ -84,10 +91,11 @@ def test_valid_local_network_ip(client):
url = reverse("bitbucket-hook-list")
url = "{}?project={}&key={}".format(url, project.id, "tpnIwJDz4e")
data = {'payload': ['{"commits": []}']}
data = json.dumps({"push": {"changes": [{"new": {"target": { "message": "test message"}}}]}})
response = client.post(url,
urllib.parse.urlencode(data, True),
content_type="application/x-www-form-urlencoded",
data,
content_type="application/json",
HTTP_X_EVENT_KEY="repo:push",
REMOTE_ADDR="192.168.1.1")
assert response.status_code == 204
@ -103,10 +111,11 @@ def test_not_ip_filter(client):
url = reverse("bitbucket-hook-list")
url = "{}?project={}&key={}".format(url, project.id, "tpnIwJDz4e")
data = {'payload': ['{"commits": []}']}
data = json.dumps({"push": {"changes": [{"new": {"target": { "message": "test message"}}}]}})
response = client.post(url,
urllib.parse.urlencode(data, True),
content_type="application/x-www-form-urlencoded",
data,
content_type="application/json",
HTTP_X_EVENT_KEY="repo:push",
REMOTE_ADDR="111.111.111.112")
assert response.status_code == 204
@ -115,13 +124,14 @@ def test_push_event_detected(client):
project = f.ProjectFactory()
url = reverse("bitbucket-hook-list")
url = "%s?project=%s" % (url, project.id)
data = {'payload': ['{"commits": [{"message": "test message"}]}']}
data = json.dumps({"push": {"changes": [{"new": {"target": { "message": "test message"}}}]}})
BitBucketViewSet._validate_signature = mock.Mock(return_value=True)
with mock.patch.object(event_hooks.PushEventHook, "process_event") as process_event_mock:
response = client.post(url, urllib.parse.urlencode(data, True),
content_type="application/x-www-form-urlencoded")
response = client.post(url, data,
HTTP_X_EVENT_KEY="repo:push",
content_type="application/json")
assert process_event_mock.call_count == 1
@ -134,9 +144,7 @@ def test_push_event_issue_processing(client):
f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
new_status = f.IssueStatusFactory(project=creation_status.project)
issue = f.IssueFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
payload = [
'{"commits": [{"message": "test message test TG-%s #%s ok bye!"}]}' % (issue.ref, new_status.slug)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #%s ok bye!" % (issue.ref, new_status.slug)}}}]}}
mail.outbox = []
ev_hook = event_hooks.PushEventHook(issue.project, payload)
ev_hook.process_event()
@ -151,9 +159,7 @@ def test_push_event_task_processing(client):
f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
new_status = f.TaskStatusFactory(project=creation_status.project)
task = f.TaskFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
payload = [
'{"commits": [{"message": "test message test TG-%s #%s ok bye!"}]}' % (task.ref, new_status.slug)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #%s ok bye!" % (task.ref, new_status.slug)}}}]}}
mail.outbox = []
ev_hook = event_hooks.PushEventHook(task.project, payload)
ev_hook.process_event()
@ -168,9 +174,7 @@ def test_push_event_user_story_processing(client):
f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
new_status = f.UserStoryStatusFactory(project=creation_status.project)
user_story = f.UserStoryFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
payload = [
'{"commits": [{"message": "test message test TG-%s #%s ok bye!"}]}' % (user_story.ref, new_status.slug)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #%s ok bye!" % (user_story.ref, new_status.slug)}}}]}}
mail.outbox = []
ev_hook = event_hooks.PushEventHook(user_story.project, payload)
ev_hook.process_event()
@ -186,9 +190,7 @@ def test_push_event_multiple_actions(client):
new_status = f.IssueStatusFactory(project=creation_status.project)
issue1 = f.IssueFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
issue2 = f.IssueFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
payload = [
'{"commits": [{"message": "test message test TG-%s #%s ok test TG-%s #%s ok bye!"}]}' % (issue1.ref, new_status.slug, issue2.ref, new_status.slug)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #%s ok test TG-%s #%s ok bye!" % (issue1.ref, new_status.slug, issue2.ref, new_status.slug)}}}]}}
mail.outbox = []
ev_hook1 = event_hooks.PushEventHook(issue1.project, payload)
ev_hook1.process_event()
@ -205,9 +207,7 @@ def test_push_event_processing_case_insensitive(client):
f.MembershipFactory(project=creation_status.project, role=role, user=creation_status.project.owner)
new_status = f.TaskStatusFactory(project=creation_status.project)
task = f.TaskFactory.create(status=creation_status, project=creation_status.project, owner=creation_status.project.owner)
payload = [
'{"commits": [{"message": "test message test tg-%s #%s ok bye!"}]}' % (task.ref, new_status.slug.upper())
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #%s ok bye!" % (task.ref, new_status.slug)}}}]}}
mail.outbox = []
ev_hook = event_hooks.PushEventHook(task.project, payload)
ev_hook.process_event()
@ -218,9 +218,7 @@ def test_push_event_processing_case_insensitive(client):
def test_push_event_task_bad_processing_non_existing_ref(client):
issue_status = f.IssueStatusFactory()
payload = [
'{"commits": [{"message": "test message test TG-6666666 #%s ok bye!"}]}' % (issue_status.slug)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-6666666 #%s ok bye!" % (issue_status.slug)}}}]}}
mail.outbox = []
ev_hook = event_hooks.PushEventHook(issue_status.project, payload)
@ -233,9 +231,7 @@ def test_push_event_task_bad_processing_non_existing_ref(client):
def test_push_event_us_bad_processing_non_existing_status(client):
user_story = f.UserStoryFactory.create()
payload = [
'{"commits": [{"message": "test message test TG-%s #non-existing-slug ok bye!"}]}' % (user_story.ref)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #non-existing-slug ok bye!" % (user_story.ref)}}}]}}
mail.outbox = []
@ -249,9 +245,7 @@ def test_push_event_us_bad_processing_non_existing_status(client):
def test_push_event_bad_processing_non_existing_status(client):
issue = f.IssueFactory.create()
payload = [
'{"commits": [{"message": "test message test TG-%s #non-existing-slug ok bye!"}]}' % (issue.ref)
]
payload = {"push": {"changes": [{"new": {"target": { "message": "test message test TG-%s #non-existing-slug ok bye!" % (issue.ref)}}}]}}
mail.outbox = []
ev_hook = event_hooks.PushEventHook(issue.project, payload)
@ -262,6 +256,217 @@ def test_push_event_bad_processing_non_existing_status(client):
assert len(mail.outbox) == 0
def test_issues_event_opened_issue(client):
issue = f.IssueFactory.create()
issue.project.default_issue_status = issue.status
issue.project.default_issue_type = issue.type
issue.project.default_severity = issue.severity
issue.project.default_priority = issue.priority
issue.project.save()
Membership.objects.create(user=issue.owner, project=issue.project, role=f.RoleFactory.create(project=issue.project), is_owner=True)
notify_policy = NotifyPolicy.objects.get(user=issue.owner, project=issue.project)
notify_policy.notify_level = NotifyLevel.watch
notify_policy.save()
payload = {
"actor": {
"user": {
"uuid": "{ce1054cd-3f43-49dc-8aea-d3085ee7ec9b}",
"username": "test-user",
"links": {"html": {"href": "http://bitbucket.com/test-user"}}
}
},
"issue": {
"id": "10",
"title": "test-title",
"links": {"html": {"href": "http://bitbucket.com/site/master/issue/10"}},
"content": {"raw": "test-content"}
},
"repository": {
"links": {"html": {"href": "http://bitbucket.com/test-user/test-project"}}
}
}
mail.outbox = []
ev_hook = event_hooks.IssuesEventHook(issue.project, payload)
ev_hook.process_event()
assert Issue.objects.count() == 2
assert len(mail.outbox) == 1
def test_issues_event_bad_issue(client):
issue = f.IssueFactory.create()
issue.project.default_issue_status = issue.status
issue.project.default_issue_type = issue.type
issue.project.default_severity = issue.severity
issue.project.default_priority = issue.priority
issue.project.save()
payload = {
"actor": {
},
"issue": {
},
"repository": {
}
}
mail.outbox = []
ev_hook = event_hooks.IssuesEventHook(issue.project, payload)
with pytest.raises(ActionSyntaxException) as excinfo:
ev_hook.process_event()
assert str(excinfo.value) == "Invalid issue information"
assert Issue.objects.count() == 1
assert len(mail.outbox) == 0
def test_issue_comment_event_on_existing_issue_task_and_us(client):
project = f.ProjectFactory()
role = f.RoleFactory(project=project, permissions=["view_tasks", "view_issues", "view_us"])
f.MembershipFactory(project=project, role=role, user=project.owner)
user = f.UserFactory()
issue = f.IssueFactory.create(external_reference=["bitbucket", "http://bitbucket.com/site/master/issue/11"], owner=project.owner, project=project)
take_snapshot(issue, user=user)
task = f.TaskFactory.create(external_reference=["bitbucket", "http://bitbucket.com/site/master/issue/11"], owner=project.owner, project=project)
take_snapshot(task, user=user)
us = f.UserStoryFactory.create(external_reference=["bitbucket", "http://bitbucket.com/site/master/issue/11"], owner=project.owner, project=project)
take_snapshot(us, user=user)
payload = {
"actor": {
"user": {
"uuid": "{ce1054cd-3f43-49dc-8aea-d3085ee7ec9b}",
"username": "test-user",
"links": {"html": {"href": "http://bitbucket.com/test-user"}}
}
},
"issue": {
"id": "11",
"title": "test-title",
"links": {"html": {"href": "http://bitbucket.com/site/master/issue/11"}},
"content": {"raw": "test-content"}
},
"comment": {
"content": {"raw": "Test body"},
},
"repository": {
"links": {"html": {"href": "http://bitbucket.com/test-user/test-project"}}
}
}
mail.outbox = []
assert get_history_queryset_by_model_instance(issue).count() == 0
assert get_history_queryset_by_model_instance(task).count() == 0
assert get_history_queryset_by_model_instance(us).count() == 0
ev_hook = event_hooks.IssueCommentEventHook(issue.project, payload)
ev_hook.process_event()
issue_history = get_history_queryset_by_model_instance(issue)
assert issue_history.count() == 1
assert "Test body" in issue_history[0].comment
task_history = get_history_queryset_by_model_instance(task)
assert task_history.count() == 1
assert "Test body" in issue_history[0].comment
us_history = get_history_queryset_by_model_instance(us)
assert us_history.count() == 1
assert "Test body" in issue_history[0].comment
assert len(mail.outbox) == 3
def test_issue_comment_event_on_not_existing_issue_task_and_us(client):
issue = f.IssueFactory.create(external_reference=["bitbucket", "10"])
take_snapshot(issue, user=issue.owner)
task = f.TaskFactory.create(project=issue.project, external_reference=["bitbucket", "10"])
take_snapshot(task, user=task.owner)
us = f.UserStoryFactory.create(project=issue.project, external_reference=["bitbucket", "10"])
take_snapshot(us, user=us.owner)
payload = {
"actor": {
"user": {
"uuid": "{ce1054cd-3f43-49dc-8aea-d3085ee7ec9b}",
"username": "test-user",
"links": {"html": {"href": "http://bitbucket.com/test-user"}}
}
},
"issue": {
"id": "10",
"title": "test-title",
"links": {"html": {"href": "http://bitbucket.com/site/master/issue/10"}},
"content": {"raw": "test-content"}
},
"comment": {
"content": {"raw": "Test body"},
},
"repository": {
"links": {"html": {"href": "http://bitbucket.com/test-user/test-project"}}
}
}
mail.outbox = []
assert get_history_queryset_by_model_instance(issue).count() == 0
assert get_history_queryset_by_model_instance(task).count() == 0
assert get_history_queryset_by_model_instance(us).count() == 0
ev_hook = event_hooks.IssueCommentEventHook(issue.project, payload)
ev_hook.process_event()
assert get_history_queryset_by_model_instance(issue).count() == 0
assert get_history_queryset_by_model_instance(task).count() == 0
assert get_history_queryset_by_model_instance(us).count() == 0
assert len(mail.outbox) == 0
def test_issues_event_bad_comment(client):
issue = f.IssueFactory.create(external_reference=["bitbucket", "10"])
take_snapshot(issue, user=issue.owner)
payload = {
"actor": {
"user": {
"uuid": "{ce1054cd-3f43-49dc-8aea-d3085ee7ec9b}",
"username": "test-user",
"links": {"html": {"href": "http://bitbucket.com/test-user"}}
}
},
"issue": {
"id": "10",
"title": "test-title",
"links": {"html": {"href": "http://bitbucket.com/site/master/issue/10"}},
"content": {"raw": "test-content"}
},
"comment": {
},
"repository": {
"links": {"html": {"href": "http://bitbucket.com/test-user/test-project"}}
}
}
ev_hook = event_hooks.IssueCommentEventHook(issue.project, payload)
mail.outbox = []
with pytest.raises(ActionSyntaxException) as excinfo:
ev_hook.process_event()
assert str(excinfo.value) == "Invalid issue comment information"
assert Issue.objects.count() == 1
assert len(mail.outbox) == 0
def test_api_get_project_modules(client):
project = f.create_project()
f.MembershipFactory(project=project, user=project.owner, is_owner=True)