From f7595b65cc7c2aa7dc147ca2b61e612a1903dee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 09:25:39 +0100 Subject: [PATCH 01/12] Add trello importer --- requirements.txt | 4 + settings/common.py | 4 + taiga/front/urls.py | 2 + taiga/importers/api.py | 39 ++ taiga/importers/management/__init__.py | 0 .../importers/management/commands/__init__.py | 0 .../management/commands/import_from_trello.py | 89 +++ taiga/importers/permissions.py | 32 ++ taiga/importers/trello/api.py | 147 +++++ taiga/importers/trello/importer.py | 537 ++++++++++++++++++ taiga/importers/trello/tasks.py | 56 ++ .../0005_epic_external_reference.py | 21 + taiga/projects/epics/models.py | 3 + taiga/projects/references/models.py | 13 + taiga/projects/userstories/api.py | 1 + taiga/routers.py | 5 + .../integration/test_importers_trello_api.py | 244 ++++++++ 17 files changed, 1197 insertions(+) create mode 100644 taiga/importers/api.py create mode 100644 taiga/importers/management/__init__.py create mode 100644 taiga/importers/management/commands/__init__.py create mode 100644 taiga/importers/management/commands/import_from_trello.py create mode 100644 taiga/importers/permissions.py create mode 100644 taiga/importers/trello/api.py create mode 100644 taiga/importers/trello/importer.py create mode 100644 taiga/importers/trello/tasks.py create mode 100644 taiga/projects/epics/migrations/0005_epic_external_reference.py create mode 100644 tests/integration/test_importers_trello_api.py diff --git a/requirements.txt b/requirements.txt index 7ff142ab..11efe393 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,8 @@ Markdown==2.6.7 fn==0.4.3 diff-match-patch==20121119 requests==2.12.4 +requests-oauthlib==0.6.2 +webcolors==1.5 django-sr==0.0.4 easy-thumbnails==2.3 celery==3.1.24 @@ -35,3 +37,5 @@ netaddr==0.7.18 serpy==0.1.1 psd-tools==1.4 CairoSVG==2.0.1 +cryptography==1.7.1 +PyJWT==1.4.2 diff --git a/settings/common.py b/settings/common.py index b4c9e002..2e0cd3ad 100644 --- a/settings/common.py +++ b/settings/common.py @@ -318,6 +318,7 @@ INSTALLED_APPS = [ "taiga.hooks.bitbucket", "taiga.hooks.gogs", "taiga.webhooks", + "taiga.importers", "djmail", "django_jinja", @@ -561,6 +562,9 @@ MAX_PENDING_MEMBERSHIPS = 30 # Max number of unconfirmed memberships in a projec from .sr import * +TRELLO_API_KEY = "" +TRELLO_SECRET_KEY = "" + # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE TEST_RUNNER="django.test.runner.DiscoverRunner" diff --git a/taiga/front/urls.py b/taiga/front/urls.py index 77d53dab..5190d2f8 100644 --- a/taiga/front/urls.py +++ b/taiga/front/urls.py @@ -23,6 +23,8 @@ urls = { "login": "/login", "register": "/register", "forgot-password": "/forgot-password", + "new-project": "/project/new", + "new-project-import": "/project/new/import/{0}", "change-password": "/change-password/{0}", # user.token "change-email": "/change-email/{0}", # user.email_token diff --git a/taiga/importers/api.py b/taiga/importers/api.py new file mode 100644 index 00000000..c1e774cc --- /dev/null +++ b/taiga/importers/api.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from taiga.base.api import viewsets +from taiga.base.decorators import list_route + + +class BaseImporterViewSet(viewsets.ViewSet): + @list_route(methods=["GET"]) + def list_users(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["GET"]) + def list_projects(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + raise NotImplementedError + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + raise NotImplementedError diff --git a/taiga/importers/management/__init__.py b/taiga/importers/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taiga/importers/management/commands/__init__.py b/taiga/importers/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/taiga/importers/management/commands/import_from_trello.py b/taiga/importers/management/commands/import_from_trello.py new file mode 100644 index 00000000..c04537fe --- /dev/null +++ b/taiga/importers/management/commands/import_from_trello.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.db.models import Q + +from taiga.importers.trello.importer import TrelloImporter +from taiga.users.models import User +from taiga.projects.services import projects as service + +import unittest.mock +import timeit + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--template', dest='template', default="kanban", + help='template to use: scrum or kanban (default kanban)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--closed-data', dest='closed_data', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + if options.get('token', None): + token = options.get('token') + else: + (oauth_token, oauth_token_secret, url) = TrelloImporter.get_auth_url() + print("Go to here and come with your token: {}".format(url)) + oauth_verifier = input("Code: ") + access_data = TrelloImporter.get_access_token(oauth_token, oauth_token_secret, oauth_verifier) + token = access_data['oauth_token'] + print("Access token: {}".format(token)) + importer = TrelloImporter(admin, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['id'], project['name'])) + project_id = input("Project id: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next trello users:") + for user in importer.list_users(project_id): + while True: + username_or_email = input("{}: ".format(user['fullName'])) + if username_or_email == "": + break + try: + users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "import_closed_data": options.get("closed_data", False), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference') + } + importer.import_project(project_id, options) diff --git a/taiga/importers/permissions.py b/taiga/importers/permissions.py new file mode 100644 index 00000000..1532989e --- /dev/null +++ b/taiga/importers/permissions.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from taiga.base.api.permissions import TaigaResourcePermission, AllowAny, IsAuthenticated +from taiga.base.api.permissions import IsSuperUser, HasProjectPerm, IsProjectAdmin + +from taiga.permissions.permissions import CommentAndOrUpdatePerm + + +class ImporterPermission(TaigaResourcePermission): + enought_perms = IsAuthenticated() + global_perms = None + auth_url_perms = IsAuthenticated() + authorize_perms = IsAuthenticated() + list_users_perms = IsAuthenticated() + list_projects_perms = IsAuthenticated() + import_project_perms = IsAuthenticated() diff --git a/taiga/importers/trello/api.py b/taiga/importers/trello/api.py new file mode 100644 index 00000000..33b9fe40 --- /dev/null +++ b/taiga/importers/trello/api.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import uuid + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id +from taiga.projects.serializers import ProjectSerializer + +from .importer import TrelloImporter +from taiga.importers import permissions +from . import tasks + + +class TrelloImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + importer = TrelloImporter(request.user, token) + users = importer.list_users(project_id) + for user in users: + user['user'] = None + if not user['email']: + continue + + try: + taiga_user = User.objects.get(email=user['email']) + except User.DoesNotExist: + continue + + user['user'] = { + 'id': taiga_user.id, + 'full_name': taiga_user.get_full_name(), + 'gravatar_id': get_user_gravatar_id(taiga_user), + 'photo': get_user_photo_url(taiga_user), + } + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + token = request.DATA.get('token', None) + importer = TrelloImporter(request.user, token) + projects = importer.list_projects() + return response.Ok(projects) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + options = { + "name": request.DATA.get('name', None), + "description": request.DATA.get('description', None), + "template": request.DATA.get('template', "kanban"), + "users_bindings": request.DATA.get("users_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, token, project_id, options) + return response.Accepted({"task_id": task.id}) + + importer = TrelloImporter(request.user, token) + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + + (oauth_token, oauth_secret, url) = TrelloImporter.get_auth_url() + + (auth_data, created) = AuthData.objects.get_or_create( + user=request.user, + key="trello-oauth", + defaults={ + "value": uuid.uuid4().hex, + "extra": {}, + } + ) + auth_data.extra = { + "oauth_token": oauth_token, + "oauth_secret": oauth_secret, + } + auth_data.save() + + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + try: + oauth_data = request.user.auth_data.get(key="trello-oauth") + oauth_token = oauth_data.extra['oauth_token'] + oauth_secret = oauth_data.extra['oauth_secret'] + oauth_verifier = request.DATA.get('code') + oauth_data.delete() + trello_token = TrelloImporter.get_access_token(oauth_token, oauth_secret, oauth_verifier)['oauth_token'] + except Exception as e: + raise exc.WrongArguments(_("Invalid or expired auth token")) + + return response.Ok({ + "token": trello_token + }) diff --git a/taiga/importers/trello/importer.py b/taiga/importers/trello/importer.py new file mode 100644 index 00000000..e41063b8 --- /dev/null +++ b/taiga/importers/trello/importer.py @@ -0,0 +1,537 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ + +from requests_oauthlib import OAuth1Session, OAuth1 +from django.conf import settings +from django.core.files.base import ContentFile +from django.contrib.contenttypes.models import ContentType +import requests +import webcolors + +from django.template.defaultfilters import slugify +from taiga.base import exceptions as exc +from taiga.projects.services import projects as projects_service +from taiga.projects.models import Project, ProjectTemplate, Membership +from taiga.projects.userstories.models import UserStory +from taiga.projects.tasks.models import Task +from taiga.projects.attachments.models import Attachment +from taiga.projects.history.services import (make_diff_from_dicts, + make_diff_values, + make_key_from_model_object, + get_typename_for_model_class, + FrozenDiff) +from taiga.projects.history.models import HistoryEntry +from taiga.projects.history.choices import HistoryType +from taiga.projects.custom_attributes.models import UserStoryCustomAttribute +from taiga.mdrender.service import render as mdrender +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from taiga.front.templatetags.functions import resolve as resolve_front_url + +from taiga.base import exceptions + + +class TrelloClient: + def __init__(self, api_key, api_secret, token): + self.api_key = api_key + self.api_secret = api_secret + self.token = token + if self.token: + self.oauth = OAuth1( + client_key=self.api_key, + client_secret=self.api_secret, + resource_owner_key=self.token + ) + else: + self.oauth = None + + def get(self, uri_path, query_params=None): + headers = {'Accept': 'application/json'} + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = 'https://api.trello.com/1/%s' % uri_path + + response = requests.get(url, params=query_params, headers=headers, auth=self.oauth) + + if response.status_code == 400: + raise exc.WrongArguments(_("Invalid Request: %s at %s") % (response.text, url)) + if response.status_code == 401: + raise exc.AuthenticationFailed(_("Unauthorized: %s at %s") % (response.text, url)) + if response.status_code == 403: + raise exc.PermissionDenied(_("Unauthorized: %s at %s") % (response.text, url)) + if response.status_code == 404: + raise exc.NotFound(_("Resource Unavailable: %s at %s") % (response.text, url)) + if response.status_code != 200: + raise exc.WrongArguments(_("Resource Unavailable: %s at %s") % (response.text, url)) + + return response.json() + + +class TrelloImporter: + def __init__(self, user, token): + self._user = user + self._cached_orgs = {} + self._client = TrelloClient( + api_key=settings.TRELLO_API_KEY, + api_secret=settings.TRELLO_SECRET_KEY, + token=token, + ) + + def list_projects(self): + projects_data = self._client.get("/members/me/boards", { + "fields": "id,name,desc,prefs,idOrganization", + "organization": "true", + "organization_fields": "prefs", + }) + projects = [] + for project in projects_data: + is_private = False + if project['prefs']['permissionLevel'] == "private": + is_private = True + + if project['prefs']['permissionLevel'] == "org": + if 'organization' not in project: + is_private = True + elif project['organization']['prefs']['permissionLevel'] == "private": + is_private = True + + projects.append({ + "id": project['id'], + "name": project['name'], + "description": project['desc'], + "is_private": is_private, + }) + return projects + + def list_users(self, project_id): + members = [] + for member in self._client.get("/board/{}/members/all".format(project_id), {"fields": "id"}): + user = self._client.get("/member/{}".format(member['id']), {"fields": "id,fullName,email"}) + members.append({ + "id": user['id'], + "full_name": user['fullName'], + "email": user['email'], + }) + return members + + def import_project(self, project_id, options): + data = self._client.get( + "/board/{}".format(project_id), + { + "fields": "name,desc", + "cards": "all", + "card_fields": "closed,labels,idList,desc,due,name,pos,dateLastActivity,idChecklists,idMembers,url", + "card_attachments": "true", + "labels": "all", + "labels_limit": "1000", + "lists": "all", + "list_fields": "closed,name,pos", + "members": "none", + "checklists": "all", + "checklist_fields": "name", + "organization": "true", + "organization_fields": "logoHash", + } + ) + + project = self._import_project_data(data, options) + self._import_user_stories_data(data, project, options) + self._cleanup(project, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + return project + + def _import_project_data(self, data, options): + board = data + labels = board['labels'] + statuses = board['lists'] + project_template = ProjectTemplate.objects.get(slug=options.get('template', "kanban")) + project_template.us_statuses = [] + counter = 0 + for us_status in statuses: + if counter == 0: + project_template.default_options["us_status"] = us_status['name'] + + counter += 1 + if us_status['name'] not in [s['name'] for s in project_template.us_statuses]: + project_template.us_statuses.append({ + "name": us_status['name'], + "slug": slugify(us_status['name']), + "is_closed": False, + "is_archived": True if us_status['closed'] else False, + "color": "#999999", + "wip_limit": None, + "order": us_status['pos'], + }) + + project_template.task_statuses = [] + project_template.task_statuses.append({ + "name": "Incomplete", + "slug": "incomplete", + "is_closed": False, + "color": "#ff8a84", + "order": 1, + }) + project_template.task_statuses.append({ + "name": "Complete", + "slug": "complete", + "is_closed": True, + "color": "#669900", + "order": 2, + }) + project_template.default_options["task_status"] = "Incomplete" + project_template.roles.append({ + "name": "Trello", + "slug": "trello", + "computable": False, + "permissions": project_template.roles[0]['permissions'], + "order": 70, + }) + + tags_colors = [] + for label in labels: + name = label['name'] + if not name: + name = label['color'] + name = name.lower() + color = self._ensure_hex_color(label['color']) + tags_colors.append([name, color]) + + project = Project( + name=options.get('name', None) or board['name'], + description=options.get('description', None) or board['desc'], + owner=self._user, + tags_colors=tags_colors, + creation_template=project_template, + is_private=options.get('is_private', False), + ) + (can_create, error_message) = projects_service.check_if_project_can_be_created_or_updated(project) + if not can_create: + raise exceptions.NotEnoughSlotsForProject(project.is_private, 1, error_message) + project.save() + + if board.get('organization', None): + trello_avatar_template = "https://trello-logos.s3.amazonaws.com/{}/170.png" + project_logo_url = trello_avatar_template.format(board['organization']['logoHash']) + data = requests.get(project_logo_url) + project.logo.save("logo.png", ContentFile(data.content), save=True) + + UserStoryCustomAttribute.objects.create( + name="Due", + description="Due date", + type="date", + order=1, + project=project + ) + for user in options.get('users_bindings', {}).values(): + Membership.objects.create( + user=user, + project=project, + role=project.get_roles().get(slug="trello"), + is_admin=False, + invited_by=self._user, + ) + return project + + def _import_user_stories_data(self, data, project, options): + users_bindings = options.get('users_bindings', {}) + statuses = {s['id']: s for s in data['lists']} + cards = data['cards'] + due_date_field = project.userstorycustomattributes.first() + + for card in cards: + if card['closed'] and not options.get("import_closed_data", False): + continue + if statuses[card['idList']]['closed'] and not options.get("import_closed_data", False): + continue + + tags = [] + for tag in card['labels']: + name = tag['name'] + if not name: + name = tag['color'] + name = name.lower() + tags.append(name) + + assigned_to = None + if len(card['idMembers']) > 0: + assigned_to = users_bindings.get(card['idMembers'][0], None) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["trello", card['url']] + + us = UserStory.objects.create( + project=project, + owner=self._user, + assigned_to=assigned_to, + status=project.us_statuses.get(name=statuses[card['idList']]['name']), + kanban_order=card['pos'], + sprint_order=card['pos'], + backlog_order=card['pos'], + subject=card['name'], + description=card['desc'], + tags=tags, + external_reference=external_reference + ) + + if len(card['idMembers']) > 1: + for watcher in card['idMembers'][1:]: + watcher_user = users_bindings.get(watcher, None) + if watcher_user: + us.add_watcher(watcher_user) + + if card['due']: + us.custom_attributes_values.attributes_values = {due_date_field.id: card['due']} + us.custom_attributes_values.save() + + UserStory.objects.filter(id=us.id).update( + modified_date=card['dateLastActivity'], + created_date=card['dateLastActivity'] + ) + self._import_attachments(us, card, options) + self._import_tasks(data, us, card) + self._import_actions(us, card, statuses, options) + + def _import_tasks(self, data, us, card): + checklists_by_id = {c['id']: c for c in data['checklists']} + for checklist_id in card['idChecklists']: + for item in checklists_by_id.get(checklist_id, {}).get('checkItems', []): + Task.objects.create( + subject=item['name'], + status=us.project.task_statuses.get(slug=item['state']), + project=us.project, + user_story=us + ) + + def _import_attachments(self, us, card, options): + users_bindings = options.get('users_bindings', {}) + for attachment in card['attachments']: + if attachment['bytes'] is None: + continue + data = requests.get(attachment['url']) + att = Attachment( + owner=users_bindings.get(attachment['idMember'], self._user), + project=us.project, + content_type=ContentType.objects.get_for_model(UserStory), + object_id=us.id, + name=attachment['name'], + size=attachment['bytes'], + created_date=attachment['date'], + is_deprecated=False, + ) + att.attached_file.save(attachment['name'], ContentFile(data.content), save=True) + + UserStory.objects.filter(id=us.id, created_date__gt=attachment['date']).update( + created_date=attachment['date'] + ) + + def _import_actions(self, us, card, statuses, options): + included_actions = [ + "addAttachmentToCard", "addMemberToCard", "commentCard", + "convertToCardFromCheckItem", "copyCommentCard", "createCard", + "deleteAttachmentFromCard", "deleteCard", "removeMemberFromCard", + "updateCard", + ] + + actions = self._client.get( + "/card/{}/actions".format(card['id']), + { + "filter": ",".join(included_actions), + "limit": "1000", + "memberCreator": "true", + "memberCreator_fields": "fullName", + } + ) + + while actions: + for action in actions: + self._import_action(us, action, statuses, options) + actions = self._client.get( + "/card/{}/actions".format(card['id']), + { + "filter": ",".join(included_actions), + "limit": "1000", + "since": "lastView", + "before": action['date'], + "memberCreator": "true", + "memberCreator_fields": "fullName", + } + ) + + def _import_action(self, us, action, statuses, options): + key = make_key_from_model_object(us) + typename = get_typename_for_model_class(UserStory) + action_data = self._transform_action_data(us, action, statuses, options) + if action_data is None: + return + + change_old = action_data['change_old'] + change_new = action_data['change_new'] + hist_type = action_data['hist_type'] + comment = action_data['comment'] + user = action_data['user'] + + diff = make_diff_from_dicts(change_old, change_new) + fdiff = FrozenDiff(key, diff, {}) + + entry = HistoryEntry.objects.create( + user=user, + project_id=us.project.id, + key=key, + type=hist_type, + snapshot=None, + diff=fdiff.diff, + values=make_diff_values(typename, fdiff), + comment=comment, + comment_html=mdrender(us.project, comment), + is_hidden=False, + is_snapshot=False, + ) + HistoryEntry.objects.filter(id=entry.id).update(created_at=action['date']) + return HistoryEntry.objects.get(id=entry.id) + + def _transform_action_data(self, us, action, statuses, options): + users_bindings = options.get('users_bindings', {}) + due_date_field = us.project.userstorycustomattributes.first() + + ignored_actions = ["addAttachmentToCard", "addMemberToCard", + "deleteAttachmentFromCard", "deleteCard", + "removeMemberFromCard"] + + if action['type'] in ignored_actions: + return None + + user = {"pk": None, "name": action.get('memberCreator', {}).get('fullName', None)} + taiga_user = users_bindings.get(action.get('memberCreator', {}).get('id', None), None) + if taiga_user: + user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()} + + result = { + "change_old": {}, + "change_new": {}, + "hist_type": HistoryType.change, + "comment": "", + "user": user + } + + if action['type'] == "commentCard": + result['comment'] = str(action['data']['text']) + elif action['type'] == "convertToCardFromCheckItem": + UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update( + created_date=action['date'], + owner=users_bindings.get(action["idMemberCreator"], self._user) + ) + result['hist_type'] = HistoryType.create + elif action['type'] == "copyCommentCard": + UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update( + created_date=action['date'], + owner=users_bindings.get(action["idMemberCreator"], self._user) + ) + result['hist_type'] = HistoryType.create + elif action['type'] == "createCard": + UserStory.objects.filter(id=us.id, created_date__gt=action['date']).update( + created_date=action['date'], + owner=users_bindings.get(action["idMemberCreator"], self._user) + ) + result['hist_type'] = HistoryType.create + elif action['type'] == "updateCard": + if 'desc' in action['data']['old']: + result['change_old']["description"] = str(action['data']['old'].get('desc', '')) + result['change_new']["description"] = str(action['data']['card'].get('desc', '')) + result['change_old']["description_html"] = mdrender(us.project, str(action['data']['old'].get('desc', ''))) + result['change_new']["description_html"] = mdrender(us.project, str(action['data']['card'].get('desc', ''))) + if 'idList' in action['data']['old']: + old_status_name = statuses[action['data']['old']['idList']]['name'] + result['change_old']["status"] = us.project.us_statuses.get(name=old_status_name).id + new_status_name = statuses[action['data']['card']['idList']]['name'] + result['change_new']["status"] = us.project.us_statuses.get(name=new_status_name).id + if 'name' in action['data']['old']: + result['change_old']["subject"] = action['data']['old']['name'] + result['change_new']["subject"] = action['data']['card']['name'] + if 'due' in action['data']['old']: + result['change_old']["custom_attributes"] = [{ + "name": "Due", + "value": action['data']['old']['due'], + "id": due_date_field.id + }] + result['change_new']["custom_attributes"] = [{ + "name": "Due", + "value": action['data']['card']['due'], + "id": due_date_field.id + }] + + if result['change_old'] == {}: + return None + return result + + @classmethod + def get_auth_url(cls): + request_token_url = 'https://trello.com/1/OAuthGetRequestToken' + authorize_url = 'https://trello.com/1/OAuthAuthorizeToken' + return_url = resolve_front_url("new-project-import", "trello") + expiration = "1day" + scope = "read,write,account" + trello_key = settings.TRELLO_API_KEY + trello_secret = settings.TRELLO_SECRET_KEY + name = "Taiga" + + session = OAuth1Session(client_key=trello_key, client_secret=trello_secret) + response = session.fetch_request_token(request_token_url) + oauth_token, oauth_token_secret = response.get('oauth_token'), response.get('oauth_token_secret') + + return ( + oauth_token, + oauth_token_secret, + "{authorize_url}?oauth_token={oauth_token}&scope={scope}&expiration={expiration}&name={name}&return_url={return_url}".format( + authorize_url=authorize_url, + oauth_token=oauth_token, + expiration=expiration, + scope=scope, + name=name, + return_url=return_url, + ) + ) + + @classmethod + def get_access_token(cls, oauth_token, oauth_token_secret, oauth_verifier): + api_key = settings.TRELLO_API_KEY + api_secret = settings.TRELLO_SECRET_KEY + access_token_url = 'https://trello.com/1/OAuthGetAccessToken' + session = OAuth1Session(client_key=api_key, client_secret=api_secret, + resource_owner_key=oauth_token, resource_owner_secret=oauth_token_secret, + verifier=oauth_verifier) + access_token = session.fetch_access_token(access_token_url) + return access_token + + def _ensure_hex_color(self, color): + if color is None: + return None + try: + return webcolors.name_to_hex(color) + except ValueError: + return color + + def _cleanup(self, project, options): + if not options.get("import_closed_data", False): + project.us_statuses.filter(is_archived=True).delete() diff --git a/taiga/importers/trello/tasks.py b/taiga/importers/trello/tasks.py new file mode 100644 index 00000000..76f7f197 --- /dev/null +++ b/taiga/importers/trello/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.users.models import User +from taiga.celery import app +from .importer import TrelloImporter + +logger = logging.getLogger('taiga.importers.trello') + + +@app.task(bind=True) +def import_project(self, user_id, token, project_id, options): + user = User.object.get(id=user_id) + importer = TrelloImporter(user, token) + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing trello project"), + "error_message": _("Error importing trello project"), + "project": project_id, + "exception": e + } + email = mail_builder.trello_import_error(admin, ctx) + email.send() + logger.error('Error importing trello project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.trello_import_success(user, ctx) + email.send() diff --git a/taiga/projects/epics/migrations/0005_epic_external_reference.py b/taiga/projects/epics/migrations/0005_epic_external_reference.py new file mode 100644 index 00000000..a1d10880 --- /dev/null +++ b/taiga/projects/epics/migrations/0005_epic_external_reference.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.2 on 2016-11-08 11:19 +from __future__ import unicode_literals + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('epics', '0004_auto_20160928_0540'), + ] + + operations = [ + migrations.AddField( + model_name='epic', + name='external_reference', + field=django.contrib.postgres.fields.ArrayField(base_field=models.TextField(), blank=True, default=None, null=True, size=None, verbose_name='external reference'), + ), + ] diff --git a/taiga/projects/epics/models.py b/taiga/projects/epics/models.py index da0e4a3e..906e5f8d 100644 --- a/taiga/projects/epics/models.py +++ b/taiga/projects/epics/models.py @@ -18,6 +18,7 @@ from django.db import models from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.postgres.fields import ArrayField from django.conf import settings from django.utils.translation import ugettext_lazy as _ from django.utils import timezone @@ -67,6 +68,8 @@ class Epic(OCCModelMixin, WatchedModelMixin, BlockedMixin, TaggedMixin, models.M user_stories = models.ManyToManyField("userstories.UserStory", related_name="epics", through='RelatedUserStory', verbose_name=_("user stories")) + external_reference = ArrayField(models.TextField(null=False, blank=False), + null=True, blank=True, default=None, verbose_name=_("external reference")) attachments = GenericRelation("attachments.Attachment") diff --git a/taiga/projects/references/models.py b/taiga/projects/references/models.py index 61097ecb..d6ad400b 100644 --- a/taiga/projects/references/models.py +++ b/taiga/projects/references/models.py @@ -68,6 +68,19 @@ def make_reference(instance, project, create=False): return refval, refinstance +def recalc_reference_counter(project): + seqname = make_sequence_name(project) + max_ref_us = project.user_stories.all().aggregate(max=models.Max('ref')) + max_ref_task = project.tasks.all().aggregate(max=models.Max('ref')) + max_ref_issue = project.issues.all().aggregate(max=models.Max('ref')) + max_references = list(filter(lambda x: x is not None, [max_ref_us['max'], max_ref_task['max'], max_ref_issue['max']])) + + max_value = 0 + if len(max_references) > 0: + max_value = max(max_references) + seq.set_max(seqname, max_value) + + def create_sequence(sender, instance, created, **kwargs): if not created: return diff --git a/taiga/projects/userstories/api.py b/taiga/projects/userstories/api.py index 3df5ddc9..c21ea7ae 100644 --- a/taiga/projects/userstories/api.py +++ b/taiga/projects/userstories/api.py @@ -106,6 +106,7 @@ class UserStoryViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixi include_attachments = "include_attachments" in self.request.QUERY_PARAMS include_tasks = "include_tasks" in self.request.QUERY_PARAMS + epic_id = self.request.QUERY_PARAMS.get("epic", None) # We can be filtering by more than one epic so epic_id can consist # of different ids separete by comma. In that situation we will use diff --git a/taiga/routers.py b/taiga/routers.py index f59bea7e..528f7696 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -283,6 +283,11 @@ from taiga.external_apps.api import Application, ApplicationToken router.register(r"applications", Application, base_name="applications") router.register(r"application-tokens", ApplicationToken, base_name="application-tokens") +# Third party importers +from taiga.importers.trello.api import TrelloImporterViewSet + +router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") + # Stats # - see taiga.stats.routers and taiga.stats.apps diff --git a/tests/integration/test_importers_trello_api.py b/tests/integration/test_importers_trello_api.py new file mode 100644 index 00000000..c83f5c57 --- /dev/null +++ b/tests/integration/test_importers_trello_api.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import json + +from unittest import mock + +from django.core.urlresolvers import reverse + +from .. import factories as f +from taiga.base.utils import json +from taiga.base import exceptions as exc + + +pytestmark = pytest.mark.django_db + + +def test_auth_url(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-auth-url") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + OAuth1SessionMock.return_value = session + + response = client.get(url, content_type="application/json") + + assert response.status_code == 200 + assert 'url' in response.data + assert response.data['url'] == "https://trello.com/1/OAuthAuthorizeToken?oauth_token=token&scope=read,write,account&expiration=1day&name=Taiga&return_url=http://localhost:9001/project/new/import/trello" + + +def test_authorize(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-auth-url") + authorize_url = reverse("importers-trello-authorize") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + session.fetch_access_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + OAuth1SessionMock.return_value = session + + client.get(url, content_type="application/json") + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"})) + + assert response.status_code == 200 + assert 'token' in response.data + assert response.data['token'] == "token" + +def test_authorize_without_token_and_secret(client): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-trello-authorize") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_access_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + OAuth1SessionMock.return_value = session + + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"})) + + assert response.status_code == 400 + assert 'token' not in response.data + + +def test_authorize_with_bad_verify(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-auth-url") + authorize_url = reverse("importers-trello-authorize") + + with mock.patch('taiga.importers.trello.importer.OAuth1Session') as OAuth1SessionMock: + session = mock.Mock() + session.fetch_request_token.return_value = {"oauth_token": "token", "oauth_token_secret": "token"} + session.fetch_access_token.side_effect = Exception("Bad Token") + OAuth1SessionMock.return_value = session + + client.get(url, content_type="application/json") + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"oauth_verifier": "token"})) + + assert response.status_code == 400 + assert 'token' not in response.data + + +def test_import_trello_list_users(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-users") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "fullName": "user1", "email": None}, + {"id": 2, "fullName": "user2", "email": None} + ] + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert response.data[0]["id"] == 1 + assert response.data[1]["id"] == 2 + + +def test_import_trello_list_users_without_project(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-users") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "fullName": "user1", "email": None}, + {"id": 2, "fullName": "user2", "email": None} + ] + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_trello_list_users_with_problem_on_request(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-users") + + with mock.patch('taiga.importers.trello.importer.TrelloClient') as TrelloClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + TrelloClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 400 + + +def test_import_trello_list_projects(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-projects") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.list_projects.return_value = ["project1", "project2"] + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 200 + assert response.data[0] == "project1" + assert response.data[1] == "project2" + + +def test_import_trello_list_projects_with_problem_on_request(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-list-projects") + + with mock.patch('taiga.importers.trello.importer.TrelloClient') as TrelloClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + TrelloClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_trello_project_without_project_id(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-trello-import-project") + + with mock.patch('taiga.importers.trello.tasks.TrelloImporter') as TrelloImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_trello_project_with_celery_enabled(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="async-imported-project") + client.login(user) + + url = reverse("importers-trello-import-project") + + with mock.patch('taiga.importers.trello.tasks.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 202 + assert "task_id" in response.data + + +def test_import_trello_project_with_celery_disabled(client, settings): + settings.CELERY_ENABLED = False + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="imported-project") + client.login(user) + + url = reverse("importers-trello-import-project") + + with mock.patch('taiga.importers.trello.api.TrelloImporter') as TrelloImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + TrelloImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert "slug" in response.data + assert response.data['slug'] == "imported-project" From 723034d37343df87f0b16861e43f8ba2834de2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 09:39:10 +0100 Subject: [PATCH 02/12] Add jira importer --- settings/common.py | 4 + taiga/importers/jira/agile.py | 357 +++++++++ taiga/importers/jira/api.py | 229 ++++++ taiga/importers/jira/common.py | 748 ++++++++++++++++++ taiga/importers/jira/normal.py | 439 ++++++++++ taiga/importers/jira/tasks.py | 61 ++ .../management/commands/import_from_jira.py | 146 ++++ taiga/routers.py | 2 + tests/integration/test_importers_jira_api.py | 259 ++++++ 9 files changed, 2245 insertions(+) create mode 100644 taiga/importers/jira/agile.py create mode 100644 taiga/importers/jira/api.py create mode 100644 taiga/importers/jira/common.py create mode 100644 taiga/importers/jira/normal.py create mode 100644 taiga/importers/jira/tasks.py create mode 100644 taiga/importers/management/commands/import_from_jira.py create mode 100644 tests/integration/test_importers_jira_api.py diff --git a/settings/common.py b/settings/common.py index 2e0cd3ad..7a894969 100644 --- a/settings/common.py +++ b/settings/common.py @@ -565,6 +565,10 @@ from .sr import * TRELLO_API_KEY = "" TRELLO_SECRET_KEY = "" +JIRA_CONSUMER_KEY = "" +JIRA_CERT = "" +JIRA_PUB_CERT = "" + # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE TEST_RUNNER="django.test.runner.DiscoverRunner" diff --git a/taiga/importers/jira/agile.py b/taiga/importers/jira/agile.py new file mode 100644 index 00000000..0f4d70cf --- /dev/null +++ b/taiga/importers/jira/agile.py @@ -0,0 +1,357 @@ +import datetime + +from django.template.defaultfilters import slugify +from taiga.projects.references.models import recalc_reference_counter +from taiga.projects.models import Project, ProjectTemplate, Membership, Points +from taiga.projects.userstories.models import UserStory, RolePoints +from taiga.projects.tasks.models import Task +from taiga.projects.milestones.models import Milestone +from taiga.projects.epics.models import Epic, RelatedUserStory +from taiga.projects.history.services import take_snapshot +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from .common import JiraImporterCommon + + +class JiraAgileImporter(JiraImporterCommon): + def list_projects(self): + return [{"id": board['id'], + "name": board['name'], + "description": "", + "is_private": True, + "importer_type": "agile"} for board in self._client.get_agile('/board')['values']] + + def list_issue_types(self, project_id): + board_project = self._client.get_agile("/board/{}/project".format(project_id))['values'][0] + statuses = self._client.get("/project/{}/statuses".format(board_project['id'])) + return statuses + + def import_project(self, project_id, options=None): + self.resolve_user_bindings(options) + project = self._import_project_data(project_id, options) + self._import_epics_data(project_id, project, options) + self._import_user_stories_data(project_id, project, options) + self._cleanup(project, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + recalc_reference_counter(project) + return project + + def _import_project_data(self, project_id, options): + project = self._client.get_agile("/board/{}".format(project_id)) + project_config = self._client.get_agile("/board/{}/configuration".format(project_id)) + if project['type'] == "scrum": + project_template = ProjectTemplate.objects.get(slug="scrum") + options['type'] = "scrum" + elif project['type'] == "kanban": + project_template = ProjectTemplate.objects.get(slug="kanban") + options['type'] = "kanban" + + project_template.is_epics_activated = True + project_template.epic_statuses = [] + project_template.us_statuses = [] + project_template.task_statuses = [] + project_template.issue_statuses = [] + + counter = 0 + for column in project_config['columnConfig']['columns']: + project_template.epic_statuses.append({ + "name": column['name'], + "slug": slugify(column['name']), + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": counter, + }) + project_template.us_statuses.append({ + "name": column['name'], + "slug": slugify(column['name']), + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": counter, + }) + project_template.task_statuses.append({ + "name": column['name'], + "slug": slugify(column['name']), + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": counter, + }) + project_template.issue_statuses.append({ + "name": column['name'], + "slug": slugify(column['name']), + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": counter, + }) + counter += 1 + + project_template.default_options["epic_status"] = project_template.epic_statuses[0]['name'] + project_template.default_options["us_status"] = project_template.us_statuses[0]['name'] + project_template.default_options["task_status"] = project_template.task_statuses[0]['name'] + project_template.default_options["issue_status"] = project_template.issue_statuses[0]['name'] + + project_template.points = [{ + "value": None, + "name": "?", + "order": 0, + }] + + main_permissions = project_template.roles[0]['permissions'] + project_template.roles = [{ + "name": "Main", + "slug": "main", + "computable": True, + "permissions": main_permissions, + "order": 70, + }] + + project = Project.objects.create( + name=options.get('name', None) or project['name'], + description=options.get('description', None) or project.get('description', ''), + owner=self._user, + creation_template=project_template, + is_private=options.get('is_private', False), + ) + + self._create_custom_fields(project) + + for user in options.get('users_bindings', {}).values(): + if user != self._user: + Membership.objects.get_or_create( + user=user, + project=project, + role=project.get_roles().get(slug="main"), + is_admin=False, + ) + + if project_template.slug == "scrum": + for sprint in self._client.get_agile("/board/{}/sprint".format(project_id))['values']: + start_datetime = sprint.get('startDate', None) + end_datetime = sprint.get('startDate', None) + start_date = datetime.date.today() + if start_datetime: + start_date = start_datetime[:10] + end_date = datetime.date.today() + if end_datetime: + end_date = end_datetime[:10] + + milestone = Milestone.objects.create( + name=sprint['name'], + slug=slugify(sprint['name']), + owner=self._user, + project=project, + estimated_start=start_date, + estimated_finish=end_date, + ) + Milestone.objects.filter(id=milestone.id).update( + created_date=start_datetime or datetime.datetime.now(), + modified_date=start_datetime or datetime.datetime.now(), + ) + return project + + def _import_user_stories_data(self, project_id, project, options): + users_bindings = options.get('users_bindings', {}) + project_conf = self._client.get_agile("/board/{}/configuration".format(project_id)) + if options['type'] == "scrum": + estimation_field = project_conf['estimation']['field']['fieldId'] + + counter = 0 + offset = 0 + while True: + issues = self._client.get_agile("/board/{}/issue".format(project_id), { + "startAt": offset, + "expand": "changelog", + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", self._client.get_issue_url(issue['key'])] + + try: + milestone = project.milestones.get(name=issue['fields'].get('sprint', {}).get('name', '')) + except Milestone.DoesNotExist: + milestone = None + + us = UserStory.objects.create( + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.us_statuses.get(name=issue['fields']['status']['name']), + kanban_order=counter, + sprint_order=counter, + backlog_order=counter, + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + tags=issue['fields']['labels'], + external_reference=external_reference, + milestone=milestone, + ) + + try: + epic = project.epics.get(ref=int(issue['fields'].get("epic", {}).get("key", "FAKE-0").split("-")[1])) + RelatedUserStory.objects.create( + user_story=us, + epic=epic, + order=1 + ) + except Epic.DoesNotExist: + pass + + if options['type'] == "scrum": + estimation = None + if issue['fields'].get(estimation_field, None): + estimation = float(issue['fields'].get(estimation_field)) + + (points, _) = Points.objects.get_or_create( + project=project, + value=estimation, + defaults={ + "name": str(estimation), + "order": estimation, + } + ) + RolePoints.objects.filter(user_story=us, role__slug="main").update(points_id=points.id) + + self._import_to_custom_fields(us, issue, options) + + us.ref = issue['key'].split("-")[1] + UserStory.objects.filter(id=us.id).update( + ref=us.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + take_snapshot(us, comment="", user=None, delete=False) + self._import_subtasks(project_id, project, us, issue, options) + self._import_comments(us, issue, options) + self._import_attachments(us, issue, options) + self._import_changelog(project, us, issue, options) + counter += 1 + + if len(issues['issues']) < issues['maxResults']: + break + + def _import_subtasks(self, project_id, project, us, issue, options): + users_bindings = options.get('users_bindings', {}) + + if len(issue['fields']['subtasks']) == 0: + return + + counter = 0 + offset = 0 + while True: + issues = self._client.get_agile("/board/{}/issue".format(project_id), { + "jql": "parent={}".format(issue['key']), + "startAt": offset, + "expand": "changelog", + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", self._client.get_issue_url(issue['key'])] + + task = Task.objects.create( + user_story=us, + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.task_statuses.get(name=issue['fields']['status']['name']), + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + tags=issue['fields']['labels'], + external_reference=external_reference, + milestone=us.milestone, + ) + + self._import_to_custom_fields(task, issue, options) + + task.ref = issue['key'].split("-")[1] + Task.objects.filter(id=task.id).update( + ref=task.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + take_snapshot(task, comment="", user=None, delete=False) + for subtask in issue['fields']['subtasks']: + print("WARNING: Ignoring subtask {} because parent isn't a User Story".format(subtask['key'])) + self._import_comments(task, issue, options) + self._import_attachments(task, issue, options) + self._import_changelog(project, task, issue, options) + counter += 1 + if len(issues['issues']) < issues['maxResults']: + break + + def _import_epics_data(self, project_id, project, options): + users_bindings = options.get('users_bindings', {}) + + counter = 0 + offset = 0 + while True: + issues = self._client.get_agile("/board/{}/epic".format(project_id), { + "startAt": offset, + }) + offset += issues['maxResults'] + + for epic in issues['values']: + issue = self._client.get_agile("/issue/{}".format(epic['key'])) + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", self._client.get_issue_url(issue['key'])] + + epic = Epic.objects.create( + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.epic_statuses.get(name=issue['fields']['status']['name']), + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + epics_order=counter, + tags=issue['fields']['labels'], + external_reference=external_reference, + ) + + self._import_to_custom_fields(epic, issue, options) + + epic.ref = issue['key'].split("-")[1] + Epic.objects.filter(id=epic.id).update( + ref=epic.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + + take_snapshot(epic, comment="", user=None, delete=False) + for subtask in issue['fields']['subtasks']: + print("WARNING: Ignoring subtask {} because parent isn't a User Story".format(subtask['key'])) + self._import_comments(epic, issue, options) + self._import_attachments(epic, issue, options) + issue_with_changelog = self._client.get("/issue/{}".format(issue['key']), { + "expand": "changelog" + }) + self._import_changelog(project, epic, issue_with_changelog, options) + counter += 1 + + if len(issues['values']) < issues['maxResults']: + break diff --git a/taiga/importers/jira/api.py b/taiga/importers/jira/api.py new file mode 100644 index 00000000..47bb6cda --- /dev/null +++ b/taiga/importers/jira/api.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id + +from taiga.importers import permissions +from .normal import JiraNormalImporter +from .agile import JiraAgileImporter +from . import tasks + + +class JiraImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + def _get_token(self, request): + token_data = request.DATA.get('token', "").split(".") + + token = { + "access_token": token_data[0], + "access_token_secret": token_data[1], + "key_cert": settings.JIRA_CERT, + "consumer_key": settings.JIRA_CONSUMER_KEY + } + return token + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + url = request.DATA.get('url', None) + token = self._get_token(request) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + if not url: + raise exc.WrongArguments(_("The url param is needed")) + + importer = JiraNormalImporter(request.user, url, token) + users = importer.list_users() + for user in users: + user['user'] = None + if not user['email']: + continue + + try: + taiga_user = User.objects.get(email=user['email']) + except User.DoesNotExist: + continue + + user['user'] = { + 'id': taiga_user.id, + 'full_name': taiga_user.get_full_name(), + 'gravatar_id': get_user_gravatar_id(taiga_user), + 'photo': get_user_photo_url(taiga_user), + } + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + url = request.DATA.get('url', None) + if not url: + raise exc.WrongArguments(_("The url param is needed")) + + token = self._get_token(request) + importer = JiraNormalImporter(request.user, url, token) + agile_importer = JiraAgileImporter(request.user, url, token) + projects = importer.list_projects() + boards = agile_importer.list_projects() + return response.Ok(sorted(projects + boards, key=lambda x: x['name'])) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + url = request.DATA.get('url', None) + token = self._get_token(request) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + if not url: + raise exc.WrongArguments(_("The url param is needed")) + + options = { + "name": request.DATA.get('name', None), + "description": request.DATA.get('description', None), + "users_bindings": request.DATA.get("user_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + importer_type = request.DATA.get('importer_type', "normal") + if importer_type == "agile": + importer = JiraAgileImporter(request.user, url, token) + else: + project_type = request.DATA.get("project_type", "scrum") + if project_type == "kanban": + options['template'] = "kanban" + else: + options['template'] = "scrum" + + importer = JiraNormalImporter(request.user, url, token) + + types_bindings = { + "epic": [], + "us": [], + "task": [], + "issue": [], + } + for issue_type in importer.list_issue_types(project_id): + if project_type in ['scrum', 'kanban']: + # Set the type bindings + if issue_type['subtask']: + types_bindings['task'].append(issue_type) + elif issue_type['name'].upper() == "EPIC": + types_bindings["epic"].append(issue_type) + elif issue_type['name'].upper() in ["US", "USERSTORY", "USER STORY"]: + types_bindings["us"].append(issue_type) + elif issue_type['name'].upper() in ["ISSUE", "BUG", "ENHANCEMENT"]: + types_bindings["issue"].append(issue_type) + else: + types_bindings["us"].append(issue_type) + elif project_type == "issues": + # Set the type bindings + if issue_type['subtask']: + continue + types_bindings["issue"].append(issue_type) + elif project_type == "issues-with-subissues": + types_bindings["issue"].append(issue_type) + else: + raise exc.WrongArguments(_("Invalid project_type {}").format(project_type)) + + options["types_bindings"] = types_bindings + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, url, token, project_id, options, importer_type) + return response.Accepted({"task_id": task.id}) + + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + jira_url = request.QUERY_PARAMS.get('url', None) + + if not jira_url: + raise exc.WrongArguments(_("The url param is needed")) + + (oauth_token, oauth_secret, url) = JiraNormalImporter.get_auth_url( + jira_url, + settings.JIRA_CONSUMER_KEY, + settings.JIRA_CERT, + True + ) + + (auth_data, created) = AuthData.objects.get_or_create( + user=request.user, + key="jira-oauth", + defaults={ + "value": "", + "extra": {}, + } + ) + auth_data.extra = { + "oauth_token": oauth_token, + "oauth_secret": oauth_secret, + "url": jira_url, + } + auth_data.save() + + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + try: + oauth_data = request.user.auth_data.get(key="jira-oauth") + oauth_token = oauth_data.extra['oauth_token'] + oauth_secret = oauth_data.extra['oauth_secret'] + server_url = oauth_data.extra['url'] + oauth_data.delete() + + jira_token = JiraNormalImporter.get_access_token( + server_url, + settings.JIRA_CONSUMER_KEY, + settings.JIRA_CERT, + oauth_token, + oauth_secret, + True + ) + except Exception as e: + raise exc.WrongArguments(_("Invalid or expired auth token")) + + return response.Ok({ + "token": jira_token['access_token'] + "." + jira_token['access_token_secret'], + "url": server_url + }) diff --git a/taiga/importers/jira/common.py b/taiga/importers/jira/common.py new file mode 100644 index 00000000..c38d1993 --- /dev/null +++ b/taiga/importers/jira/common.py @@ -0,0 +1,748 @@ +import requests +from urllib.parse import parse_qsl +from oauthlib.oauth1 import SIGNATURE_RSA + +from requests_oauthlib import OAuth1 +from django.core.files.base import ContentFile +from django.contrib.contenttypes.models import ContentType + +from taiga.users.models import User +from taiga.projects.models import Project, ProjectTemplate, Membership, Points +from taiga.projects.userstories.models import UserStory +from taiga.projects.tasks.models import Task +from taiga.projects.issues.models import Issue +from taiga.projects.milestones.models import Milestone +from taiga.projects.epics.models import Epic +from taiga.projects.attachments.models import Attachment +from taiga.projects.history.services import take_snapshot +from taiga.projects.history.services import (make_diff_from_dicts, + make_diff_values, + make_key_from_model_object, + get_typename_for_model_class, + FrozenDiff) +from taiga.projects.custom_attributes.models import (UserStoryCustomAttribute, + TaskCustomAttribute, + IssueCustomAttribute, + EpicCustomAttribute) +from taiga.projects.history.models import HistoryEntry +from taiga.projects.history.choices import HistoryType +from taiga.mdrender.service import render as mdrender + +EPIC_COLORS = { + "ghx-label-0": "#ffffff", + "ghx-label-1": "#815b3a", + "ghx-label-2": "#f79232", + "ghx-label-3": "#d39c3f", + "ghx-label-4": "#3b7fc4", + "ghx-label-5": "#4a6785", + "ghx-label-6": "#8eb021", + "ghx-label-7": "#ac707a", + "ghx-label-8": "#654982", + "ghx-label-9": "#f15c75", +} + + +def links_to_richtext(importer, issue, links): + richtext = "" + importing_project_key = issue['key'].split("-")[0] + for link in links: + if "inwardIssue" in link: + (project_key, issue_key) = link['inwardIssue']['key'].split("-") + action = link['type']['inward'] + elif "outwardIssue" in link: + (project_key, issue_key) = link['outwardIssue']['key'].split("-") + action = link['type']['outward'] + else: + continue + + if importing_project_key == project_key: + richtext += " * This item {} #{}\n".format(action, issue_key) + else: + url = importer._client.server + "/projects/{}/issues/{}-{}".format( + project_key, + project_key, + issue_key + ) + richtext += " * This item {} [{}-{}]({})\n".format(action, project_key, issue_key, url) + + for link in links: + if "object" in link: + richtext += " * [{}]({})\n".format( + link['object']['title'] or link['object']['url'], + link['object']['url'], + ) + + return richtext + + +class JiraClient: + def __init__(self, server, oauth): + self.server = server + self.api_url = server + "/rest/agile/1.0/{}" + self.main_api_url = server + "/rest/api/2/{}" + if oauth: + self.oauth = OAuth1( + oauth['consumer_key'], + signature_method=SIGNATURE_RSA, + rsa_key=oauth['key_cert'], + resource_owner_key=oauth['access_token'], + resource_owner_secret=oauth['access_token_secret'] + ) + else: + self.oauth = None + + def get(self, uri_path, query_params=None): + headers = { + 'Content-Type': "application/json" + } + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = self.main_api_url.format(uri_path) + + response = requests.get(url, params=query_params, headers=headers, auth=self.oauth) + + if response.status_code == 401: + raise Exception("Unauthorized: %s at %s" % (response.text, url), response) + if response.status_code != 200: + raise Exception("Resource Unavailable: %s at %s" % (response.text, url), response) + + return response.json() + + def get_agile(self, uri_path, query_params=None): + headers = { + 'Content-Type': "application/json" + } + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = self.api_url.format(uri_path) + + response = requests.get(url, params=query_params, headers=headers, auth=self.oauth) + + if response.status_code == 401: + raise Exception("Unauthorized: %s at %s" % (response.text, url), response) + if response.status_code != 200: + raise Exception("Resource Unavailable: %s at %s" % (response.text, url), response) + + return response.json() + + def raw_get(self, absolute_uri, query_params=None): + if query_params is None: + query_params = {} + + response = requests.get(absolute_uri, params=query_params, auth=self.oauth) + + if response.status_code == 401: + raise Exception("Unauthorized: %s at %s" % (response.text, absolute_uri), response) + if response.status_code != 200: + raise Exception("Resource Unavailable: %s at %s" % (response.text, absolute_uri), response) + + return response.content + + def get_issue_url(self, key): + (project_key, issue_key) = key.split("-") + return self.server + "/projects/{}/issues/{}".format(project_key, key) + + +class JiraImporterCommon: + def __init__(self, user, server, oauth): + self._user = user + self._client = JiraClient(server=server, oauth=oauth) + + def resolve_user_bindings(self, options): + for option in list(options['users_bindings'].keys()): + try: + user = User.objects.get(id=options['users_bindings'][option]) + options['users_bindings'][option] = user + except User.DoesNotExist: + del(options['users_bindings'][option]) + + def list_users(self): + result = [] + users = self._client.get("/user/picker", { + "query": "@", + "maxResults": 1000, + }) + for user in users['users']: + user_data = self._client.get("/user", { + "key": user['key'] + }) + result.append({ + "id": user_data['key'], + "full_name": user_data['displayName'], + "email": user_data['emailAddress'], + }) + return result + + def _import_comments(self, obj, issue, options): + users_bindings = options.get('users_bindings', {}) + offset = 0 + while True: + comments = self._client.get("/issue/{}/comment".format(issue['key']), {"startAt": offset}) + for comment in comments['comments']: + snapshot = take_snapshot( + obj, + comment=comment['body'], + user=users_bindings.get( + comment['author']['name'], + User(full_name=comment['author']['displayName']) + ), + delete=False + ) + HistoryEntry.objects.filter(id=snapshot.id).update(created_at=comment['created']) + + offset += len(comments['comments']) + if len(comments['comments']) <= comments['maxResults']: + break + + def _create_custom_fields(self, project): + custom_fields = [] + for model in [UserStoryCustomAttribute, TaskCustomAttribute, IssueCustomAttribute, EpicCustomAttribute]: + model.objects.create( + name="Due date", + description="Due date", + type="date", + order=1, + project=project + ) + model.objects.create( + name="Priority", + description="Priority", + type="text", + order=1, + project=project + ) + model.objects.create( + name="Resolution", + description="Resolution", + type="text", + order=1, + project=project + ) + model.objects.create( + name="Resolution date", + description="Resolution date", + type="date", + order=1, + project=project + ) + model.objects.create( + name="Environment", + description="Environment", + type="text", + order=1, + project=project + ) + model.objects.create( + name="Components", + description="Components", + type="text", + order=1, + project=project + ) + model.objects.create( + name="Affects Version/s", + description="Affects Version/s", + type="text", + order=1, + project=project + ) + model.objects.create( + name="Fix Version/s", + description="Fix Version/s", + type="text", + order=1, + project=project + ) + model.objects.create( + name="Links", + description="Links", + type="richtext", + order=1, + project=project + ) + custom_fields.append({ + "history_name": "duedate", + "jira_field_name": "duedate", + "taiga_field_name": "Due date", + }) + custom_fields.append({ + "history_name": "priority", + "jira_field_name": "priority", + "taiga_field_name": "Priority", + "transform": lambda issue, obj: obj.get('name', None) + }) + custom_fields.append({ + "history_name": "resolution", + "jira_field_name": "resolution", + "taiga_field_name": "Resolution", + "transform": lambda issue, obj: obj.get('name', None) + }) + custom_fields.append({ + "history_name": "Resolution date", + "jira_field_name": "resolutiondate", + "taiga_field_name": "Resolution date", + }) + custom_fields.append({ + "history_name": "environment", + "jira_field_name": "environment", + "taiga_field_name": "Environment", + }) + custom_fields.append({ + "history_name": "Component", + "jira_field_name": "components", + "taiga_field_name": "Components", + "transform": lambda issue, obj: ", ".join([c.get('name', None) for c in obj]) + }) + custom_fields.append({ + "history_name": "Version", + "jira_field_name": "versions", + "taiga_field_name": "Affects Version/s", + "transform": lambda issue, obj: ", ".join([c.get('name', None) for c in obj]) + }) + custom_fields.append({ + "history_name": "Fix Version", + "jira_field_name": "fixVersions", + "taiga_field_name": "Fix Version/s", + "transform": lambda issue, obj: ", ".join([c.get('name', None) for c in obj]) + }) + custom_fields.append({ + "history_name": "Link", + "jira_field_name": "issuelinks", + "taiga_field_name": "Links", + "transform": lambda issue, obj: links_to_richtext(self, issue, obj) + }) + + greenhopper_fields = {} + for custom_field in self._client.get("/field"): + if custom_field['custom']: + if custom_field['schema']['custom'] == "com.pyxis.greenhopper.jira:gh-sprint": + greenhopper_fields["sprint"] = custom_field['id'] + elif custom_field['schema']['custom'] == "com.pyxis.greenhopper.jira:gh-epic-link": + greenhopper_fields["link"] = custom_field['id'] + elif custom_field['schema']['custom'] == "com.pyxis.greenhopper.jira:gh-epic-status": + greenhopper_fields["status"] = custom_field['id'] + elif custom_field['schema']['custom'] == "com.pyxis.greenhopper.jira:gh-epic-label": + greenhopper_fields["label"] = custom_field['id'] + elif custom_field['schema']['custom'] == "com.pyxis.greenhopper.jira:gh-epic-color": + greenhopper_fields["color"] = custom_field['id'] + elif custom_field['schema']['custom'] == "com.pyxis.greenhopper.jira:gh-lexo-rank": + greenhopper_fields["rank"] = custom_field['id'] + elif ( + custom_field['name'] == "Story Points" and + custom_field['schema']['custom'] == 'com.atlassian.jira.plugin.system.customfieldtypes:float' + ): + greenhopper_fields["points"] = custom_field['id'] + else: + multiline_types = [ + "com.atlassian.jira.plugin.system.customfieldtypes:textarea" + ] + date_types = [ + "com.atlassian.jira.plugin.system.customfieldtypes:datepicker" + "com.atlassian.jira.plugin.system.customfieldtypes:datetime" + ] + if custom_field['schema']['custom'] in multiline_types: + field_type = "multiline" + elif custom_field['schema']['custom'] in date_types: + field_type = "date" + else: + field_type = "text" + + custom_field_data = { + "name": custom_field['name'][:64], + "description": custom_field['name'], + "type": field_type, + "order": 1, + "project": project + } + + UserStoryCustomAttribute.objects.get_or_create(**custom_field_data) + TaskCustomAttribute.objects.get_or_create(**custom_field_data) + IssueCustomAttribute.objects.get_or_create(**custom_field_data) + EpicCustomAttribute.objects.get_or_create(**custom_field_data) + + custom_fields.append({ + "history_name": custom_field['name'], + "jira_field_name": custom_field['id'], + "taiga_field_name": custom_field['name'][:64], + }) + + self.greenhopper_fields = greenhopper_fields + self.custom_fields = custom_fields + + def _import_to_custom_fields(self, obj, issue, options): + if isinstance(obj, Epic): + custom_att_manager = obj.project.epiccustomattributes + elif isinstance(obj, UserStory): + custom_att_manager = obj.project.userstorycustomattributes + elif isinstance(obj, Task): + custom_att_manager = obj.project.taskcustomattributes + elif isinstance(obj, Issue): + custom_att_manager = obj.project.issuecustomattributes + else: + raise NotImplementedError("Not implemented custom attributes for this object ({})".format(obj)) + + custom_attributes_values = {} + for custom_field in self.custom_fields: + if issue['key'] == "PS-10": + import pprint; pprint.pprint(issue['fields']) + data = issue['fields'].get(custom_field['jira_field_name'], None) + if data and "transform" in custom_field: + data = custom_field['transform'](issue, data) + + if data: + taiga_field = custom_att_manager.get(name=custom_field['taiga_field_name']) + custom_attributes_values[taiga_field.id] = data + + if custom_attributes_values != {}: + obj.custom_attributes_values.attributes_values = custom_attributes_values + obj.custom_attributes_values.save() + + def _import_attachments(self, obj, issue, options): + users_bindings = options.get('users_bindings', {}) + + for attachment in issue['fields']['attachment']: + try: + data = self._client.raw_get(attachment['content']) + att = Attachment( + owner=users_bindings.get(attachment['author']['name'], self._user), + project=obj.project, + content_type=ContentType.objects.get_for_model(obj), + object_id=obj.id, + name=attachment['filename'], + size=attachment['size'], + created_date=attachment['created'], + is_deprecated=False, + ) + att.attached_file.save(attachment['filename'], ContentFile(data), save=True) + except Exception: + print("ERROR getting attachment url {}".format(attachment['content'])) + + + def _import_changelog(self, project, obj, issue, options): + obj.cummulative_attachments = [] + for history in sorted(issue['changelog']['histories'], key=lambda h: h['created']): + self._import_history(project, obj, history, options) + + def _import_history(self, project, obj, history, options): + key = make_key_from_model_object(obj) + typename = get_typename_for_model_class(obj.__class__) + history_data = self._transform_history_data(project, obj, history, options) + if history_data is None: + return + + change_old = history_data['change_old'] + change_new = history_data['change_new'] + hist_type = history_data['hist_type'] + comment = history_data['comment'] + user = history_data['user'] + + diff = make_diff_from_dicts(change_old, change_new) + fdiff = FrozenDiff(key, diff, {}) + + values = make_diff_values(typename, fdiff) + values.update(history_data['update_values']) + + entry = HistoryEntry.objects.create( + user=user, + project_id=obj.project.id, + key=key, + type=hist_type, + snapshot=None, + diff=fdiff.diff, + values=values, + comment=comment, + comment_html=mdrender(obj.project, comment), + is_hidden=False, + is_snapshot=False, + ) + HistoryEntry.objects.filter(id=entry.id).update(created_at=history['created']) + return HistoryEntry.objects.get(id=entry.id) + + def _transform_history_data(self, project, obj, history, options): + users_bindings = options.get('users_bindings', {}) + + user = {"pk": None, "name": history.get('author', {}).get('displayName', None)} + taiga_user = users_bindings.get(history.get('author', {}).get('key', None), None) + if taiga_user: + user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()} + + result = { + "change_old": {}, + "change_new": {}, + "update_values": {}, + "hist_type": HistoryType.change, + "comment": "", + "user": user + } + custom_fields_by_names = {f["history_name"]: f for f in self.custom_fields} + has_data = False + for history_item in history['items']: + if history_item['field'] == "Attachment": + result['change_old']["attachments"] = [] + for att in obj.cummulative_attachments: + result['change_old']["attachments"].append({ + "id": 0, + "filename": att + }) + + if history_item['from'] is not None: + try: + idx = obj.cummulative_attachments.index(history_item['fromString']) + obj.cummulative_attachments.pop(idx) + except ValueError: + print("ERROR: Removing attachment that doesn't exist in the history ({})".format(history_item['fromString'])) + if history_item['to'] is not None: + obj.cummulative_attachments.append(history_item['toString']) + + result['change_new']["attachments"] = [] + for att in obj.cummulative_attachments: + result['change_new']["attachments"].append({ + "id": 0, + "filename": att + }) + has_data = True + elif history_item['field'] == "description": + result['change_old']["description"] = history_item['fromString'] + result['change_new']["description"] = history_item['toString'] + result['change_old']["description_html"] = mdrender(obj.project, history_item['fromString'] or "") + result['change_new']["description_html"] = mdrender(obj.project, history_item['toString'] or "") + has_data = True + elif history_item['field'] == "Epic Link": + pass + elif history_item['field'] == "Workflow": + pass + elif history_item['field'] == "Link": + pass + elif history_item['field'] == "labels": + result['change_old']["tags"] = history_item['fromString'].split() + result['change_new']["tags"] = history_item['toString'].split() + has_data = True + elif history_item['field'] == "Rank": + pass + elif history_item['field'] == "RemoteIssueLink": + pass + elif history_item['field'] == "Sprint": + old_milestone = None + if history_item['fromString']: + try: + old_milestone = obj.project.milestones.get(name=history_item['fromString']).id + except Milestone.DoesNotExist: + old_milestone = -1 + + new_milestone = None + if history_item['toString']: + try: + new_milestone = obj.project.milestones.get(name=history_item['toString']).id + except Milestone.DoesNotExist: + new_milestone = -2 + + result['change_old']["milestone"] = old_milestone + result['change_new']["milestone"] = new_milestone + + if old_milestone == -1 or new_milestone == -2: + result['update_values']["milestone"] = {} + + if old_milestone == -1: + result['update_values']["milestone"]["-1"] = history_item['fromString'] + if new_milestone == -2: + result['update_values']["milestone"]["-2"] = history_item['toString'] + has_data = True + elif history_item['field'] == "status": + if isinstance(obj, Task): + try: + old_status = obj.project.task_statuses.get(name=history_item['fromString']).id + except Exception: + old_status = -1 + try: + new_status = obj.project.task_statuses.get(name=history_item['toString']).id + except Exception: + new_status = -2 + elif isinstance(obj, UserStory): + try: + old_status = obj.project.us_statuses.get(name=history_item['fromString']).id + except Exception: + old_status = -1 + try: + new_status = obj.project.us_statuses.get(name=history_item['toString']).id + except Exception: + new_status = -2 + elif isinstance(obj, Issue): + try: + old_status = obj.project.issue_statuses.get(name=history_item['fromString']).id + except Exception: + old_status = -1 + try: + new_status = obj.project.us_statuses.get(name=history_item['toString']).id + except Exception: + new_status = -2 + elif isinstance(obj, Epic): + try: + old_status = obj.project.epic_statuses.get(name=history_item['fromString']).id + except Exception: + old_status = -1 + try: + new_status = obj.project.epic_statuses.get(name=history_item['toString']).id + except Exception: + new_status = -2 + + if old_status == -1 or new_status == -2: + result['update_values']["status"] = {} + + if old_status == -1: + result['update_values']["status"]["-1"] = history_item['fromString'] + if new_status == -2: + result['update_values']["status"]["-2"] = history_item['toString'] + + result['change_old']["status"] = old_status + result['change_new']["status"] = new_status + has_data = True + elif history_item['field'] == "Story Points": + old_points = None + if history_item['fromString']: + estimation = float(history_item['fromString']) + (old_points, _) = Points.objects.get_or_create( + project=project, + value=estimation, + defaults={ + "name": str(estimation), + "order": estimation, + } + ) + old_points = old_points.id + new_points = None + if history_item['toString']: + estimation = float(history_item['toString']) + (new_points, _) = Points.objects.get_or_create( + project=project, + value=estimation, + defaults={ + "name": str(estimation), + "order": estimation, + } + ) + new_points = new_points.id + result['change_old']["points"] = {project.roles.get(slug="main").id: old_points} + result['change_new']["points"] = {project.roles.get(slug="main").id: new_points} + has_data = True + elif history_item['field'] == "summary": + result['change_old']["subject"] = history_item['fromString'] + result['change_new']["subject"] = history_item['toString'] + has_data = True + elif history_item['field'] == "Epic Color": + if isinstance(obj, Epic): + result['change_old']["color"] = EPIC_COLORS.get(history_item['fromString'], None) + result['change_new']["color"] = EPIC_COLORS.get(history_item['toString'], None) + Epic.objects.filter(id=obj.id).update( + color=EPIC_COLORS.get(history_item['toString'], "#999999") + ) + has_data = True + elif history_item['field'] == "assignee": + old_assigned_to = None + if history_item['from'] is not None: + old_assigned_to = users_bindings.get(history_item['from'], -1) + if old_assigned_to != -1: + old_assigned_to = old_assigned_to.id + + new_assigned_to = None + if history_item['to'] is not None: + new_assigned_to = users_bindings.get(history_item['to'], -2) + if new_assigned_to != -2: + new_assigned_to = new_assigned_to.id + + result['change_old']["assigned_to"] = old_assigned_to + result['change_new']["assigned_to"] = new_assigned_to + + if old_assigned_to == -1 or new_assigned_to == -2: + result['update_values']["users"] = {} + + if old_assigned_to == -1: + result['update_values']["users"]["-1"] = history_item['fromString'] + if new_assigned_to == -2: + result['update_values']["users"]["-2"] = history_item['toString'] + has_data = True + elif history_item['field'] in custom_fields_by_names: + custom_field = custom_fields_by_names[history_item['field']] + if isinstance(obj, Task): + field_obj = obj.project.taskcustomattributes.get(name=custom_field['taiga_field_name']) + elif isinstance(obj, UserStory): + field_obj = obj.project.userstorycustomattributes.get(name=custom_field['taiga_field_name']) + elif isinstance(obj, Issue): + field_obj = obj.project.issuecustomattributes.get(name=custom_field['taiga_field_name']) + elif isinstance(obj, Epic): + field_obj = obj.project.epiccustomattributes.get(name=custom_field['taiga_field_name']) + + result['change_old']["custom_attributes"] = [{ + "name": custom_field['taiga_field_name'], + "value": history_item['fromString'], + "id": field_obj.id + }] + result['change_new']["custom_attributes"] = [{ + "name": custom_field['taiga_field_name'], + "value": history_item['toString'], + "id": field_obj.id + }] + has_data = True + else: + import pprint; pprint.pprint(history_item) + + if not has_data: + return None + + return result + + def _cleanup(self, project, options): + for epic_custom_field in project.epiccustomattributes.all(): + if project.epics.filter(custom_attributes_values__attributes_values__has_key=str(epic_custom_field.id)).count() == 0: + epic_custom_field.delete() + for us_custom_field in project.userstorycustomattributes.all(): + if project.user_stories.filter(custom_attributes_values__attributes_values__has_key=str(us_custom_field.id)).count() == 0: + us_custom_field.delete() + for task_custom_field in project.taskcustomattributes.all(): + if project.tasks.filter(custom_attributes_values__attributes_values__has_key=str(task_custom_field.id)).count() == 0: + task_custom_field.delete() + for issue_custom_field in project.issuecustomattributes.all(): + if project.issues.filter(custom_attributes_values__attributes_values__has_key=str(issue_custom_field.id)).count() == 0: + issue_custom_field.delete() + + @classmethod + def get_auth_url(cls, server, consumer_key, key_cert_data, verify=None): + if verify is None: + verify = server.startswith('https') + + oauth = OAuth1(consumer_key, signature_method=SIGNATURE_RSA, rsa_key=key_cert_data) + r = requests.post( + server + '/plugins/servlet/oauth/request-token', verify=verify, auth=oauth) + request = dict(parse_qsl(r.text)) + request_token = request['oauth_token'] + request_token_secret = request['oauth_token_secret'] + + return ( + request_token, + request_token_secret, + '{}/plugins/servlet/oauth/authorize?oauth_token={}'.format(server, request_token) + ) + + @classmethod + def get_access_token(cls, server, consumer_key, key_cert_data, request_token, request_token_secret, verify=False): + oauth = OAuth1( + consumer_key, + signature_method=SIGNATURE_RSA, + rsa_key=key_cert_data, + resource_owner_key=request_token, + resource_owner_secret=request_token_secret + ) + r = requests.post(server + '/plugins/servlet/oauth/access-token', verify=verify, auth=oauth) + access = dict(parse_qsl(r.text)) + + return { + 'access_token': access['oauth_token'], + 'access_token_secret': access['oauth_token_secret'], + 'consumer_key': consumer_key, + 'key_cert': key_cert_data + } diff --git a/taiga/importers/jira/normal.py b/taiga/importers/jira/normal.py new file mode 100644 index 00000000..45584545 --- /dev/null +++ b/taiga/importers/jira/normal.py @@ -0,0 +1,439 @@ +from collections import OrderedDict + +from django.template.defaultfilters import slugify +from taiga.projects.references.models import recalc_reference_counter +from taiga.projects.models import Project, ProjectTemplate, Membership, Points +from taiga.projects.userstories.models import UserStory, RolePoints +from taiga.projects.tasks.models import Task +from taiga.projects.issues.models import Issue +from taiga.projects.epics.models import Epic, RelatedUserStory +from taiga.projects.history.services import take_snapshot +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from .common import JiraImporterCommon + + +class JiraNormalImporter(JiraImporterCommon): + def list_projects(self): + return [{"id": project['id'], + "name": project['name'], + "description": project['description'], + "is_private": True, + "importer_type": "normal"} for project in self._client.get('/project', {"expand": "description"})] + + def list_issue_types(self, project_id): + statuses = self._client.get("/project/{}/statuses".format(project_id)) + return statuses + + def import_project(self, project_id, options): + self.resolve_user_bindings(options) + project = self._import_project_data(project_id, options) + self._import_user_stories_data(project_id, project, options) + self._import_epics_data(project_id, project, options) + self._link_epics_with_user_stories(project_id, project, options) + self._import_issues_data(project_id, project, options) + self._cleanup(project, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + recalc_reference_counter(project) + return project + + def _import_project_data(self, project_id, options): + project = self._client.get("/project/{}".format(project_id)) + project_template = ProjectTemplate.objects.get(slug=options['template']) + + epic_statuses = OrderedDict() + for issue_type in options.get('types_bindings', {}).get("epic", []): + for status in issue_type['statuses']: + epic_statuses[status['name']] = status + + us_statuses = OrderedDict() + for issue_type in options.get('types_bindings', {}).get("us", []): + for status in issue_type['statuses']: + us_statuses[status['name']] = status + + task_statuses = OrderedDict() + for issue_type in options.get('types_bindings', {}).get("task", []): + for status in issue_type['statuses']: + task_statuses[status['name']] = status + + issue_statuses = OrderedDict() + for issue_type in options.get('types_bindings', {}).get("issue", []): + for status in issue_type['statuses']: + issue_statuses[status['name']] = status + + counter = 0 + if epic_statuses: + project_template.epic_statuses = [] + project_template.is_epics_activated = True + for epic_status in epic_statuses.values(): + project_template.epic_statuses.append({ + "name": epic_status['name'], + "slug": slugify(epic_status['name']), + "is_closed": False, + "color": "#999999", + "order": counter, + }) + counter += 1 + if epic_statuses: + project_template.default_options["epic_status"] = list(epic_statuses.values())[0]['name'] + + project_template.points = [{ + "value": None, + "name": "?", + "order": 0, + }] + + main_permissions = project_template.roles[0]['permissions'] + project_template.roles = [{ + "name": "Main", + "slug": "main", + "computable": True, + "permissions": main_permissions, + "order": 70, + }] + + counter = 0 + if us_statuses: + project_template.us_statuses = [] + for us_status in us_statuses.values(): + project_template.us_statuses.append({ + "name": us_status['name'], + "slug": slugify(us_status['name']), + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": counter, + }) + counter += 1 + if us_statuses: + project_template.default_options["us_status"] = list(us_statuses.values())[0]['name'] + + counter = 0 + if task_statuses: + project_template.task_statuses = [] + for task_status in task_statuses.values(): + project_template.task_statuses.append({ + "name": task_status['name'], + "slug": slugify(task_status['name']), + "is_closed": False, + "color": "#999999", + "order": counter, + }) + counter += 1 + if task_statuses: + project_template.default_options["task_status"] = list(task_statuses.values())[0]['name'] + + counter = 0 + if issue_statuses: + project_template.issue_statuses = [] + for issue_status in issue_statuses.values(): + project_template.issue_statuses.append({ + "name": issue_status['name'], + "slug": slugify(issue_status['name']), + "is_closed": False, + "color": "#999999", + "order": counter, + }) + counter += 1 + if issue_statuses: + project_template.default_options["issue_status"] = list(issue_statuses.values())[0]['name'] + + + main_permissions = project_template.roles[0]['permissions'] + project_template.roles = [{ + "name": "Main", + "slug": "main", + "computable": True, + "permissions": main_permissions, + "order": 70, + }] + + project = Project.objects.create( + name=options.get('name', None) or project['name'], + description=options.get('description', None) or project.get('description', ''), + owner=self._user, + creation_template=project_template, + is_private=options.get('is_private', False), + ) + + self._create_custom_fields(project) + + for user in options.get('users_bindings', {}).values(): + if user != self._user: + Membership.objects.get_or_create( + user=user, + project=project, + role=project.get_roles().get(slug="main"), + is_admin=False, + ) + return project + + def _import_user_stories_data(self, project_id, project, options): + users_bindings = options.get('users_bindings', {}) + + types = options.get('types_bindings', {}).get("us", []) + for issue_type in types: + counter = 0 + offset = 0 + while True: + issues = self._client.get("/search", { + "jql": "project={} AND issuetype={}".format(project_id, issue_type['id']), + "startAt": offset, + "fields": "*all", + "expand": "changelog,attachment", + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", issue['fields']['url']] + + + us = UserStory.objects.create( + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.us_statuses.get(name=issue['fields']['status']['name']), + kanban_order=counter, + sprint_order=counter, + backlog_order=counter, + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + tags=issue['fields']['labels'], + external_reference=external_reference, + ) + + points_value = issue['fields'].get(self.greenhopper_fields.get('points', None), None) + if points_value: + (points, _) = Points.objects.get_or_create( + project=project, + value=points_value, + defaults={ + "name": str(points_value), + "order": points_value, + } + ) + RolePoints.objects.filter(user_story=us, role__slug="main").update(points_id=points.id) + else: + points = Points.objects.get(project=project, value__isnull=True) + RolePoints.objects.filter(user_story=us, role__slug="main").update(points_id=points.id) + + self._import_to_custom_fields(us, issue, options) + + us.ref = issue['key'].split("-")[1] + UserStory.objects.filter(id=us.id).update( + ref=us.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + take_snapshot(us, comment="", user=None, delete=False) + self._import_subtasks(project_id, project, us, issue, options) + self._import_comments(us, issue, options) + self._import_attachments(us, issue, options) + self._import_changelog(project, us, issue, options) + counter += 1 + + if len(issues['issues']) < issues['maxResults']: + break + + def _import_subtasks(self, project_id, project, us, issue, options): + users_bindings = options.get('users_bindings', {}) + + if len(issue['fields']['subtasks']) == 0: + return + + counter = 0 + offset = 0 + while True: + issues = self._client.get("/search", { + "jql": "parent={}".format(issue['key']), + "startAt": offset, + "fields": "*all", + "expand": "changelog,attachment", + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", issue['fields']['url']] + + task = Task.objects.create( + user_story=us, + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.task_statuses.get(name=issue['fields']['status']['name']), + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + tags=issue['fields']['labels'], + external_reference=external_reference, + ) + + self._import_to_custom_fields(task, issue, options) + + task.ref = issue['key'].split("-")[1] + Task.objects.filter(id=task.id).update( + ref=task.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + take_snapshot(task, comment="", user=None, delete=False) + for subtask in issue['fields']['subtasks']: + print("WARNING: Ignoring subtask {} because parent isn't a User Story".format(subtask['key'])) + self._import_comments(task, issue, options) + self._import_attachments(task, issue, options) + self._import_changelog(project, task, issue, options) + counter += 1 + if len(issues['issues']) < issues['maxResults']: + break + + def _import_issues_data(self, project_id, project, options): + users_bindings = options.get('users_bindings', {}) + + types = options.get('types_bindings', {}).get("issue", []) + for issue_type in types: + counter = 0 + offset = 0 + while True: + issues = self._client.get("/search", { + "jql": "project={} AND issuetype={}".format(project_id, issue_type['id']), + "startAt": offset, + "fields": "*all", + "expand": "changelog,attachment", + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", issue['fields']['url']] + + taiga_issue = Issue.objects.create( + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.issue_statuses.get(name=issue['fields']['status']['name']), + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + tags=issue['fields']['labels'], + external_reference=external_reference, + ) + + self._import_to_custom_fields(taiga_issue, issue, options) + + taiga_issue.ref = issue['key'].split("-")[1] + Issue.objects.filter(id=taiga_issue.id).update( + ref=taiga_issue.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + take_snapshot(taiga_issue, comment="", user=None, delete=False) + for subtask in issue['fields']['subtasks']: + print("WARNING: Ignoring subtask {} because parent isn't a User Story".format(subtask['key'])) + self._import_comments(taiga_issue, issue, options) + self._import_attachments(taiga_issue, issue, options) + self._import_changelog(project, taiga_issue, issue, options) + counter += 1 + + if len(issues['issues']) < issues['maxResults']: + break + + def _import_epics_data(self, project_id, project, options): + users_bindings = options.get('users_bindings', {}) + + types = options.get('types_bindings', {}).get("epic", []) + for issue_type in types: + counter = 0 + offset = 0 + while True: + issues = self._client.get("/search", { + "jql": "project={} AND issuetype={}".format(project_id, issue_type['id']), + "startAt": offset, + "fields": "*all", + "expand": "changelog,attachment", + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + issue['fields']['issuelinks'] += self._client.get("/issue/{}/remotelink".format(issue['key'])) + assigned_to = users_bindings.get(issue['fields']['assignee']['key'] if issue['fields']['assignee'] else None, None) + owner = users_bindings.get(issue['fields']['creator']['key'] if issue['fields']['creator'] else None, self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["jira", issue['fields']['url']] + + epic = Epic.objects.create( + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.epic_statuses.get(name=issue['fields']['status']['name']), + subject=issue['fields']['summary'], + description=issue['fields']['description'] or '', + epics_order=counter, + tags=issue['fields']['labels'], + external_reference=external_reference, + ) + + self._import_to_custom_fields(epic, issue, options) + + epic.ref = issue['key'].split("-")[1] + Epic.objects.filter(id=epic.id).update( + ref=epic.ref, + modified_date=issue['fields']['updated'], + created_date=issue['fields']['created'] + ) + take_snapshot(epic, comment="", user=None, delete=False) + for subtask in issue['fields']['subtasks']: + print("WARNING: Ignoring subtask {} because parent isn't a User Story".format(subtask['key'])) + self._import_comments(epic, issue, options) + self._import_attachments(epic, issue, options) + issue_with_changelog = self._client.get("/issue/{}".format(issue['key']), { + "expand": "changelog" + }) + self._import_changelog(project, epic, issue_with_changelog, options) + counter += 1 + + if len(issues['issues']) < issues['maxResults']: + break + + def _link_epics_with_user_stories(self, project_id, project, options): + types = options.get('types_bindings', {}).get("us", []) + for issue_type in types: + offset = 0 + while True: + issues = self._client.get("/search", { + "jql": "project={} AND issuetype={}".format(project_id, issue_type['id']), + "startAt": offset + }) + offset += issues['maxResults'] + + for issue in issues['issues']: + epic_key = issue['fields'][self.greenhopper_fields['link']] + if epic_key: + epic = project.epics.get(ref=int(epic_key.split("-")[1])) + us = project.user_stories.get(ref=int(issue['key'].split("-")[1])) + RelatedUserStory.objects.create( + user_story=us, + epic=epic, + order=1 + ) + + if len(issues['issues']) < issues['maxResults']: + break diff --git a/taiga/importers/jira/tasks.py b/taiga/importers/jira/tasks.py new file mode 100644 index 00000000..a1551f1b --- /dev/null +++ b/taiga/importers/jira/tasks.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.users.models import User +from taiga.celery import app +from .normal import JiraNormalImporter + +logger = logging.getLogger('taiga.importers.jira') + + +@app.task(bind=True) +def import_project(self, user_id, url, token, project_id, options, importer_type): + user = User.object.get(id=user_id) + + if importer_type == "agile": + importer = JiraAgileImporter(user, url, token) + else: + importer = JiraNormalImporter(user, url, token) + + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing jira project"), + "error_message": _("Error importing jira project"), + "project": project_id, + "exception": e + } + email = mail_builder.jira_import_error(admin, ctx) + email.send() + logger.error('Error importing jira project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.jira_import_success(user, ctx) + email.send() diff --git a/taiga/importers/management/commands/import_from_jira.py b/taiga/importers/management/commands/import_from_jira.py new file mode 100644 index 00000000..972e6ba6 --- /dev/null +++ b/taiga/importers/management/commands/import_from_jira.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.db.models import Q +from django.conf import settings + +from taiga.importers.jira.agile import JiraAgileImporter +from taiga.importers.jira.normal import JiraNormalImporter +from taiga.users.models import User +from taiga.projects.services import projects as service + +import unittest.mock +import timeit +import json + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--server', dest="server", type=str, + help='Server address (default: https://jira.atlassian.com)', + default="https://jira.atlassian.com") + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--project-type', dest="project_type", type=str, + help='Project type in jira: project or board') + parser.add_argument('--template', dest='template', default="scrum", + help='template to use: scrum or scrum (default scrum)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--closed-data', dest='closed_data', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + server = options.get("server") + + if options.get('token', None) == "anon": + token = None + elif options.get('token', None): + token = json.loads(options.get('token')) + else: + (rtoken, rtoken_secret, url) = JiraNormalImporter.get_auth_url(server, settings.JIRA_CONSUMER_KEY, settings.JIRA_CERT, True) + print(url) + code = input("Go to the url and get back the code") + token = JiraNormalImporter.get_access_token(server, settings.JIRA_CONSUMER_KEY, settings.JIRA_CERT, rtoken, rtoken_secret, True) + print("Auth token: {}".format(json.dumps(token))) + + + if options.get('project_type', None) is None: + print("Select the type of project to import (project or board): ") + project_type = input("Project type: ") + else: + project_type = options.get('project_type') + + if project_type not in ["project", "board"]: + print("ERROR: Bad project type.") + return + + if project_type == "project": + importer = JiraNormalImporter(admin, server, token) + else: + importer = JiraAgileImporter(admin, server, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['id'], project['name'])) + project_id = input("Project id or key: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next jira users:") + for user in importer.list_users(): + try: + users_bindings[user['key']] = User.objects.get(Q(email=user['email'])) + break + except User.DoesNotExist: + pass + + while True: + username_or_email = input("{}: ".format(user['full_name'])) + if username_or_email == "": + break + try: + users_bindings[user['key']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "import_closed_data": options.get("closed_data", False), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference'), + } + + if project_type == "project": + print("Bind jira issue types to (epic, us, issue)") + types_bindings = { + "epic": [], + "us": [], + "task": [], + "issue": [], + } + + for issue_type in importer.list_issue_types(project_id): + while True: + if issue_type['subtask']: + types_bindings['task'].append(issue_type) + break + + taiga_type = input("{}: ".format(issue_type['name'])) + if taiga_type not in ['epic', 'us', 'issue']: + print("use a valid taiga type (epic, us, issue)") + continue + + types_bindings[taiga_type].append(issue_type) + break + options["types_bindings"] = types_bindings + + importer.import_project(project_id, options) diff --git a/taiga/routers.py b/taiga/routers.py index 528f7696..1546ef8b 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -285,8 +285,10 @@ router.register(r"application-tokens", ApplicationToken, base_name="application- # Third party importers from taiga.importers.trello.api import TrelloImporterViewSet +from taiga.importers.jira.api import JiraImporterViewSet router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") +router.register(r"importers/jira", JiraImporterViewSet, base_name="importers-jira") # Stats diff --git a/tests/integration/test_importers_jira_api.py b/tests/integration/test_importers_jira_api.py new file mode 100644 index 00000000..ebb96a11 --- /dev/null +++ b/tests/integration/test_importers_jira_api.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import json + +from unittest import mock + +from django.core.urlresolvers import reverse + +from .. import factories as f +from taiga.base.utils import json +from taiga.base import exceptions as exc +from taiga.users.models import AuthData + + +pytestmark = pytest.mark.django_db + + +fake_token = "access.secret" + + +def test_auth_url(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-auth-url")+"?url=http://jiraserver" + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as JiraNormalImporter: + JiraNormalImporter.get_auth_url.return_value = ("test_oauth_token", "test_oauth_secret", "http://jira-server-url") + response = client.get(url, content_type="application/json") + + auth_data = user.auth_data.get(key="jira-oauth") + assert auth_data.extra['oauth_token'] == "test_oauth_token" + assert auth_data.extra['oauth_secret'] == "test_oauth_secret" + assert auth_data.extra['url'] == "http://jiraserver" + + assert response.status_code == 200 + assert 'url' in response.data + assert response.data['url'] == "http://jira-server-url" + + +def test_authorize(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-auth-url") + authorize_url = reverse("importers-jira-authorize") + + AuthData.objects.get_or_create( + user=user, + key="jira-oauth", + value="", + extra={ + "oauth_token": "test-oauth-token", + "oauth_secret": "test-oauth-secret", + "url": "http://jiraserver", + } + ) + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as JiraNormalImporter: + JiraNormalImporter.get_access_token.return_value = { + "access_token": "test-access-token", + "access_token_secret": "test-access-token-secret" + } + response = client.post(authorize_url, content_type="application/json", data={}) + + assert response.status_code == 200 + assert 'token' in response.data + assert response.data['token'] == "test-access-token.test-access-token-secret" + assert 'url' in response.data + assert response.data['url'] == "http://jiraserver" + + +def test_authorize_without_token_and_secret(client): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-jira-authorize") + AuthData.objects.filter(user=user, key="jira-oauth").delete() + + response = client.post(authorize_url, content_type="application/json", data={}) + + assert response.status_code == 400 + assert 'token' not in response.data + + +def test_import_jira_list_users(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-list-users") + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as JiraNormalImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "fullName": "user1", "email": None}, + {"id": 2, "fullName": "user2", "email": None} + ] + JiraNormalImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver", "project": 1})) + + assert response.status_code == 200 + assert response.data[0]["id"] == 1 + assert response.data[1]["id"] == 2 + + +def test_import_jira_list_users_without_project(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-list-users") + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as JiraNormalImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "fullName": "user1", "email": None}, + {"id": 2, "fullName": "user2", "email": None} + ] + JiraNormalImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver"})) + + assert response.status_code == 400 + + +def test_import_jira_list_users_with_problem_on_request(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-list-users") + + with mock.patch('taiga.importers.jira.common.JiraClient') as JiraClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + JiraClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver", "project": 1})) + + assert response.status_code == 400 + + +def test_import_jira_list_projects(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-list-projects") + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as JiraNormalImporterMock: + with mock.patch('taiga.importers.jira.api.JiraAgileImporter') as JiraAgileImporterMock: + instance = mock.Mock() + instance.list_projects.return_value = [{"name": "project1"}, {"name": "project2"}] + JiraNormalImporterMock.return_value = instance + instance_agile = mock.Mock() + instance_agile.list_projects.return_value = [{"name": "agile1"}, {"name": "agile2"}] + JiraAgileImporterMock.return_value = instance_agile + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver"})) + + assert response.status_code == 200 + assert response.data[0] == {"name": "agile1"} + assert response.data[1] == {"name": "agile2"} + assert response.data[2] == {"name": "project1"} + assert response.data[3] == {"name": "project2"} + + +def test_import_jira_list_projects_with_problem_on_request(client, settings): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-list-projects") + + with mock.patch('taiga.importers.jira.common.JiraClient') as JiraClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + JiraClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver"})) + + assert response.status_code == 400 + + +def test_import_jira_project_without_project_id(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-import-project") + + with mock.patch('taiga.importers.jira.tasks.JiraNormalImporter') as JiraNormalImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver"})) + + assert response.status_code == 400 + + +def test_import_jira_project_without_url(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-jira-import-project") + + with mock.patch('taiga.importers.jira.tasks.JiraNormalImporter') as JiraNormalImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "project_id": 1})) + + assert response.status_code == 400 + + +def test_import_jira_project_with_celery_enabled(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="async-imported-project") + client.login(user) + + url = reverse("importers-jira-import-project") + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as ApiJiraNormalImporterMock: + with mock.patch('taiga.importers.jira.tasks.JiraNormalImporter') as TasksJiraNormalImporterMock: + TasksJiraNormalImporterMock.return_value.import_project.return_value = project + ApiJiraNormalImporterMock.return_value.list_issue_types.return_value = [] + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver", "project": 1})) + + assert response.status_code == 202 + assert "task_id" in response.data + + +def test_import_jira_project_with_celery_disabled(client, settings): + settings.CELERY_ENABLED = False + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="imported-project") + client.login(user) + + url = reverse("importers-jira-import-project") + + with mock.patch('taiga.importers.jira.api.JiraNormalImporter') as JiraNormalImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + instance.list_issue_types.return_value = [] + JiraNormalImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "access.secret", "url": "http://jiraserver", "project": 1})) + + assert response.status_code == 200 + assert "slug" in response.data + assert response.data['slug'] == "imported-project" From cca421b14d22d0bb11c3a5e2d4adcfaa44aca01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 09:40:05 +0100 Subject: [PATCH 03/12] Add github importer --- settings/common.py | 2 + taiga/importers/exceptions.py | 5 + taiga/importers/github/api.py | 131 ++++ taiga/importers/github/importer.py | 604 ++++++++++++++++++ taiga/importers/github/tasks.py | 56 ++ .../management/commands/import_from_github.py | 100 +++ taiga/routers.py | 2 + .../integration/test_importers_github_api.py | 228 +++++++ 8 files changed, 1128 insertions(+) create mode 100644 taiga/importers/exceptions.py create mode 100644 taiga/importers/github/api.py create mode 100644 taiga/importers/github/importer.py create mode 100644 taiga/importers/github/tasks.py create mode 100644 taiga/importers/management/commands/import_from_github.py create mode 100644 tests/integration/test_importers_github_api.py diff --git a/settings/common.py b/settings/common.py index 7a894969..9ce0e415 100644 --- a/settings/common.py +++ b/settings/common.py @@ -561,6 +561,8 @@ MAX_PENDING_MEMBERSHIPS = 30 # Max number of unconfirmed memberships in a projec from .sr import * +GITHUB_API_CLIENT_ID = "" +GITHUB_API_CLIENT_SECRET = "" TRELLO_API_KEY = "" TRELLO_SECRET_KEY = "" diff --git a/taiga/importers/exceptions.py b/taiga/importers/exceptions.py new file mode 100644 index 00000000..da430079 --- /dev/null +++ b/taiga/importers/exceptions.py @@ -0,0 +1,5 @@ +class InvalidAuthResult(Exception): + pass + +class FailedRequest(Exception): + pass diff --git a/taiga/importers/github/api.py b/taiga/importers/github/api.py new file mode 100644 index 00000000..e8991c0d --- /dev/null +++ b/taiga/importers/github/api.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id +from taiga.projects.serializers import ProjectSerializer + +from taiga.importers import permissions +from taiga.importers import exceptions +from .importer import GithubImporter +from . import tasks + + +class GithubImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + importer = GithubImporter(request.user, token) + users = importer.list_users(project_id) + for user in users: + if user['detected_user']: + user['user'] = { + 'id': user['detected_user'].id, + 'full_name': user['detected_user'].get_full_name(), + 'gravatar_id': get_user_gravatar_id(user['detected_user']), + 'photo': get_user_photo_url(user['detected_user']), + } + del(user['detected_user']) + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + token = request.DATA.get('token', None) + importer = GithubImporter(request.user, token) + projects = importer.list_projects() + return response.Ok(projects) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + template = request.DATA.get('template', "scrum") + items_type = "user_stories" + if template == "issues": + items_type = "issues" + template = "scrum" + + options = { + "name": request.DATA.get('name', None), + "description": request.DATA.get('description', None), + "template": template, + "type": items_type, + "users_bindings": request.DATA.get("users_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, token, project_id, options) + return response.Accepted({"task_id": task.id}) + + importer = GithubImporter(request.user, token) + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + callback_uri = request.QUERY_PARAMS.get('uri') + url = GithubImporter.get_auth_url(settings.GITHUB_API_CLIENT_ID, callback_uri) + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + code = request.DATA.get('code', None) + if code is None: + raise exc.BadRequest(_("Code param needed")) + + try: + token = GithubImporter.get_access_token(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, code) + return response.Ok({ + "token": token + }) + except exceptions.InvalidAuthResult: + raise exc.BadRequest(_("Invalid auth data")) + except exceptions.FailedRequest: + raise exc.BadRequest(_("Third party service failing")) diff --git a/taiga/importers/github/importer.py b/taiga/importers/github/importer.py new file mode 100644 index 00000000..b042feef --- /dev/null +++ b/taiga/importers/github/importer.py @@ -0,0 +1,604 @@ +import requests +from urllib.parse import parse_qsl +from django.core.files.base import ContentFile + +from taiga.projects.models import Project, ProjectTemplate, Membership +from taiga.projects.references.models import recalc_reference_counter +from taiga.projects.userstories.models import UserStory +from taiga.projects.issues.models import Issue +from taiga.projects.milestones.models import Milestone +from taiga.projects.history.services import take_snapshot +from taiga.projects.history.services import (make_diff_from_dicts, + make_diff_values, + make_key_from_model_object, + get_typename_for_model_class, + FrozenDiff) +from taiga.projects.history.models import HistoryEntry +from taiga.projects.history.choices import HistoryType +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from taiga.users.models import User, AuthData + + +class GithubClient: + def __init__(self, token): + self.api_url = "https://api.github.com/{}" + self.token = token + + def get(self, uri_path, query_params=None): + headers = { + "Content-Type": "application/json", + "X-GitHub-Media-Type": "github.v3" + } + if self.token: + headers['Authorization'] = 'token {}'.format(self.token) + + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = self.api_url.format(uri_path) + + response = requests.get(url, params=query_params, headers=headers) + + if response.status_code == 401: + raise Exception("Unauthorized: %s at %s" % (response.text, url), response) + if response.status_code != 200: + raise Exception("Resource Unavailable: %s at %s" % (response.text, url), response) + + return response.json() + + +class GithubImporter: + def __init__(self, user, token, import_closed_data=False): + self._import_closed_data = import_closed_data + self._user = user + self._client = GithubClient(token) + self._me = self._client.get("/user") + + def list_projects(self): + projects = [] + page = 1 + while True: + repos = self._client.get("/user/repos", { + "sort": "full_name", + "page": page, + "per_page": 100 + }) + page += 1 + + for repo in repos: + projects.append({ + "id": repo['full_name'], + "name": repo['full_name'], + "description": repo['description'], + "is_private": repo['private'], + }) + + if len(repos) < 100: + break + return projects + + def list_users(self, project_full_name): + collaborators = self._client.get("/repos/{}/collaborators".format(project_full_name)) + collaborators = [self._client.get("/users/{}".format(u['login'])) for u in collaborators] + return [{"id": u['id'], + "username": u['login'], + "full_name": u.get('name', u['login']), + "detected_user": self._get_user(u) } for u in collaborators] + + def _get_user(self, user, default=None): + if not user: + return default + + try: + return AuthData.objects.get(key="github", value=user['id']).user + except AuthData.DoesNotExist: + pass + + try: + return User.objects.get(email=user.get('email', "not-valid")) + except User.DoesNotExist: + pass + + return default + + def import_project(self, project_full_name, options={"keep_external_reference": False, "template": "kanban", "type": "user_stories"}): + repo = self._client.get('/repos/{}'.format(project_full_name)) + project = self._import_project_data(repo, options) + if options.get('type', None) == "user_stories": + self._import_user_stories_data(project, repo, options) + elif options.get('type', None) == "issues": + self._import_issues_data(project, repo, options) + self._import_comments(project, repo, options) + self._import_history(project, repo, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + recalc_reference_counter(project) + return project + + def _import_project_data(self, repo, options): + users_bindings = options.get('users_bindings', {}) + project_template = ProjectTemplate.objects.get(slug=options['template']) + + if options['type'] == "user_stories": + project_template.us_statuses = [] + project_template.us_statuses.append({ + "name": "Open", + "slug": "open", + "is_closed": False, + "is_archived": False, + "color": "#ff8a84", + "wip_limit": None, + "order": 1, + }) + project_template.us_statuses.append({ + "name": "Closed", + "slug": "closed", + "is_closed": True, + "is_archived": False, + "color": "#669900", + "wip_limit": None, + "order": 2, + }) + project_template.default_options["us_status"] = "Open" + elif options['type'] == "issues": + project_template.issue_statuses = [] + project_template.issue_statuses.append({ + "name": "Open", + "slug": "open", + "is_closed": False, + "color": "#ff8a84", + "order": 1, + }) + project_template.issue_statuses.append({ + "name": "Closed", + "slug": "closed", + "is_closed": True, + "color": "#669900", + "order": 2, + }) + project_template.default_options["issue_status"] = "Open" + + project_template.roles.append({ + "name": "Github", + "slug": "github", + "computable": False, + "permissions": project_template.roles[0]['permissions'], + "order": 70, + }) + + tags_colors = [] + for label in self._client.get("/repos/{}/labels".format(repo['full_name'])): + name = label['name'].lower() + color = "#{}".format(label['color']) + tags_colors.append([name, color]) + + project = Project.objects.create( + name=options.get('name', None) or repo['full_name'], + description=options.get('description', None) or repo['description'], + owner=self._user, + tags_colors=tags_colors, + creation_template=project_template, + is_private=options.get('is_private', False), + ) + + if 'organization' in repo and repo['organization'].get('avatar_url', None): + data = requests.get(repo['organization']['avatar_url']) + project.logo.save("logo.png", ContentFile(data.content), save=True) + + for user in self._client.get("/repos/{}/collaborators".format(repo['full_name'])): + taiga_user = users_bindings.get(user['id'], None) + if taiga_user is None or taiga_user == self._user: + continue + + Membership.objects.create( + user=taiga_user, + project=project, + role=project.get_roles().get(slug="github"), + is_admin=False, + invited_by=self._user, + ) + + for milestone in self._client.get("/repos/{}/milestones".format(repo['full_name'])): + taiga_milestone = Milestone.objects.create( + name=milestone['title'], + owner=users_bindings.get(milestone.get('creator', {}).get('id', None), self._user), + project=project, + estimated_start=milestone['created_at'][:10], + estimated_finish=milestone['due_on'][:10], + ) + Milestone.objects.filter(id=taiga_milestone.id).update( + created_date=milestone['created_at'], + modified_date=milestone['updated_at'], + ) + return project + + def _import_user_stories_data(self, project, repo, options): + users_bindings = options.get('users_bindings', {}) + + page = 1 + while True: + issues = self._client.get("/repos/{}/issues".format(repo['full_name']), { + "state": "all", + "sort": "created", + "direction": "asc", + "page": page, + "per_page": 100 + }) + page += 1 + for issue in issues: + tags = [] + for label in issue['labels']: + tags.append(label['name'].lower()) + + assigned_to = users_bindings.get(issue['assignee']['id'] if issue['assignee'] else None, None) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["github", issue['html_url']] + + us = UserStory.objects.create( + ref=issue['number'], + project=project, + owner=users_bindings.get(issue['user']['id'], self._user), + milestone=project.milestones.get(name=issue['milestone']['title']) if issue['milestone'] else None, + assigned_to=assigned_to, + status=project.us_statuses.get(slug=issue['state']), + kanban_order=issue['number'], + sprint_order=issue['number'], + backlog_order=issue['number'], + subject=issue['title'], + description=issue.get("body", "") or "", + tags=tags, + external_reference=external_reference, + modified_date=issue['updated_at'], + created_date=issue['created_at'], + ) + + assignees = issue.get('assignees', []) + if len(assignees) > 1: + for assignee in assignees: + if assignee['id'] != issue.get('assignee', {}).get('id', None): + assignee_user = users_bindings.get(assignee['id'], None) + if assignee_user is not None: + us.add_watcher(assignee_user) + + UserStory.objects.filter(id=us.id).update( + ref=issue['number'], + modified_date=issue['updated_at'], + created_date=issue['created_at'] + ) + + take_snapshot(us, comment="", user=None, delete=False) + + if len(issues) < 100: + break + + def _import_issues_data(self, project, repo, options): + users_bindings = options.get('users_bindings', {}) + + page = 1 + while True: + issues = self._client.get("/repos/{}/issues".format(repo['full_name']), { + "state": "all", + "sort": "created", + "direction": "asc", + "page": page, + "per_page": 100 + }) + page += 1 + for issue in issues: + tags = [] + for label in issue['labels']: + tags.append(label['name'].lower()) + + assigned_to = users_bindings.get(issue['assignee']['id'] if issue['assignee'] else None, None) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["github", issue['html_url']] + + taiga_issue = Issue.objects.create( + ref=issue['number'], + project=project, + owner=users_bindings.get(issue['user']['id'], self._user), + assigned_to=assigned_to, + status=project.issue_statuses.get(slug=issue['state']), + subject=issue['title'], + description=issue.get('body', "") or "", + tags=tags, + external_reference=external_reference, + modified_date=issue['updated_at'], + created_date=issue['created_at'], + ) + + assignees = issue.get('assignees', []) + if len(assignees) > 1: + for assignee in assignees: + if assignee['id'] != issue.get('assignee', {}).get('id', None): + assignee_user = users_bindings.get(assignee['id'], None) + if assignee_user is not None: + taiga_issue.add_watcher(assignee_user) + + Issue.objects.filter(id=taiga_issue.id).update( + ref=issue['number'], + modified_date=issue['updated_at'], + created_date=issue['created_at'] + ) + + take_snapshot(taiga_issue, comment="", user=None, delete=False) + + if len(issues) < 100: + break + + def _import_comments(self, project, repo, options): + users_bindings = options.get('users_bindings', {}) + + page = 1 + while True: + comments = self._client.get("/repos/{}/issues/comments".format(repo['full_name']), { + "page": page, + "per_page": 100 + }) + page += 1 + + for comment in comments: + issue_id = comment['issue_url'].split("/")[-1] + if options.get('type', None) == "user_stories": + obj = UserStory.objects.get(project=project, ref=issue_id) + elif options.get('type', None) == "issues": + obj = Issue.objects.get(project=project, ref=issue_id) + + snapshot = take_snapshot( + obj, + comment=comment['body'], + user=users_bindings.get(comment['user']['id'], User(full_name=comment['user'].get('name', None) or comment['user']['login'])), + delete=False + ) + HistoryEntry.objects.filter(id=snapshot.id).update(created_at=comment['created_at']) + + if len(comments) < 100: + break + + def _import_history(self, project, repo, options): + cumulative_data = {} + page = 1 + all_events = [] + while True: + events = self._client.get("/repos/{}/issues/events".format(repo['full_name']), { + "page": page, + "per_page": 100 + }) + page += 1 + all_events = all_events + events + + if len(events) < 100: + break + + for event in sorted(all_events, key=lambda x: x['id']): + if options.get('type', None) == "user_stories": + obj = UserStory.objects.get(project=project, ref=event['issue']['number']) + elif options.get('type', None) == "issues": + obj = Issue.objects.get(project=project, ref=event['issue']['number']) + + if event['issue']['number'] in cumulative_data: + obj_cumulative_data = cumulative_data[event['issue']['number']] + else: + obj_cumulative_data = { + "tags": set(), + "assigned_to": None, + "assigned_to_github_id": None, + "assigned_to_name": None, + "milestone": None, + } + cumulative_data[event['issue']['number']] = obj_cumulative_data + self._import_event(obj, event, options, obj_cumulative_data) + + def _import_event(self, obj, event, options, cumulative_data): + typename = get_typename_for_model_class(type(obj)) + key = make_key_from_model_object(obj) + event_data = self._transform_event_data(obj, event, options, cumulative_data) + if event_data is None: + return + + change_old = event_data['change_old'] + change_new = event_data['change_new'] + user = event_data['user'] + + diff = make_diff_from_dicts(change_old, change_new) + fdiff = FrozenDiff(key, diff, {}) + values = make_diff_values(typename, fdiff) + values.update(event_data['update_values']) + entry = HistoryEntry.objects.create( + user=user, + project_id=obj.project.id, + key=key, + type=HistoryType.change, + snapshot=None, + diff=fdiff.diff, + values=values, + comment="", + comment_html="", + is_hidden=False, + is_snapshot=False, + ) + HistoryEntry.objects.filter(id=entry.id).update(created_at=event['created_at']) + return HistoryEntry.objects.get(id=entry.id) + + def _transform_event_data(self, obj, event, options, cumulative_data): + users_bindings = options.get('users_bindings', {}) + + ignored_events = ["committed", "cross-referenced", "head_ref_deleted", + "head_ref_restored", "locked", "unlocked", "merged", + "referenced", "mentioned", "subscribed", + "unsubscribed"] + + if event['event'] in ignored_events: + return None + + user = {"pk": None, "name": event['actor'].get('name', event['actor']['login'])} + taiga_user = users_bindings.get(event['actor']['id'], None) if event['actor'] else None + if taiga_user: + user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()} + + result = { + "change_old": {}, + "change_new": {}, + "user": user, + "update_values": {}, + } + + if event['event'] == "renamed": + result['change_old']["subject"] = event['rename']['from'] + result['change_new']["subject"] = event['rename']['to'] + elif event['event'] == "reopened": + if isinstance(obj, Issue): + result['change_old']["status"] = obj.project.issue_statuses.get(name='Closed').id + result['change_new']["status"] = obj.project.issue_statuses.get(name='Open').id + elif isinstance(obj, UserStory): + result['change_old']["status"] = obj.project.us_statuses.get(name='Closed').id + result['change_new']["status"] = obj.project.us_statuses.get(name='Open').id + elif event['event'] == "closed": + if isinstance(obj, Issue): + result['change_old']["status"] = obj.project.issue_statuses.get(name='Open').id + result['change_new']["status"] = obj.project.issue_statuses.get(name='Closed').id + elif isinstance(obj, UserStory): + result['change_old']["status"] = obj.project.us_statuses.get(name='Open').id + result['change_new']["status"] = obj.project.us_statuses.get(name='Closed').id + elif event['event'] == "assigned": + AssignedEventHandler(result, cumulative_data, users_bindings).handle(event) + elif event['event'] == "unassigned": + UnassignedEventHandler(result, cumulative_data, users_bindings).handle(event) + elif event['event'] == "demilestoned": + if isinstance(obj, UserStory): + try: + result['change_old']["milestone"] = obj.project.milestones.get(name=event['milestone']['title']).id + except Milestone.DoesNotExist: + result['change_old']["milestone"] = 0 + result['update_values'] = {"milestone": {"0": event['milestone']['title']}} + result['change_new']["milestone"] = None + cumulative_data['milestone'] = None + elif event['event'] == "milestoned": + if isinstance(obj, UserStory): + result['update_values']["milestone"] = {} + if cumulative_data['milestone'] is not None: + result['update_values']['milestone'][str(cumulative_data['milestone'])] = cumulative_data['milestone_name'] + result['change_old']["milestone"] = cumulative_data['milestone'] + try: + taiga_milestone = obj.project.milestones.get(name=event['milestone']['title']) + cumulative_data["milestone"] = taiga_milestone.id + cumulative_data['milestone_name'] = taiga_milestone.name + except Milestone.DoesNotExist: + if cumulative_data['milestone'] == 0: + cumulative_data['milestone'] = -1 + else: + cumulative_data['milestone'] = 0 + cumulative_data['milestone_name'] = event['milestone']['title'] + result['change_new']["milestone"] = cumulative_data['milestone'] + result['update_values']['milestone'][str(cumulative_data['milestone'])] = cumulative_data['milestone_name'] + elif event['event'] == "labeled": + result['change_old']["tags"] = list(cumulative_data['tags']) + cumulative_data['tags'].add(event['label']['name'].lower()) + result['change_new']["tags"] = list(cumulative_data['tags']) + elif event['event'] == "unlabeled": + result['change_old']["tags"] = list(cumulative_data['tags']) + if event['label']['name'].lower() in cumulative_data['tags']: + cumulative_data['tags'].remove(event['label']['name'].lower()) + result['change_new']["tags"] = list(cumulative_data['tags']) + + return result + + @classmethod + def get_auth_url(cls, client_id, callback_uri=None): + if callback_uri is None: + return "https://github.com/login/oauth/authorize?client_id={}&scope=user,repo".format(client_id) + return "https://github.com/login/oauth/authorize?client_id={}&scope=user,repo&redirect_uri={}".format(client_id, callback_uri) + + @classmethod + def get_access_token(cls, client_id, client_secret, code): + try: + result = requests.post("https://github.com/login/oauth/access_token", { + "client_id": client_id, + "client_secret": client_secret, + "code": code, + }) + except Exception: + raise FailedRequest() + + if result.status_code > 299: + raise InvalidAuthResult() + else: + return dict(parse_qsl(result.content))[b'access_token'].decode('utf-8') + + +class AssignedEventHandler: + def __init__(self, result, cumulative_data, users_bindings): + self.result = result + self.cumulative_data = cumulative_data + self.users_bindings = users_bindings + + def handle(self, event): + if self.cumulative_data['assigned_to_github_id'] is None: + self.result['update_values']["users"] = {} + self.generate_change_old(event) + self.generate_update_values_from_cumulative_data(event) + user = self.users_bindings.get(event['assignee']['id'], None) + self.generate_change_new(event, user) + self.update_cumulative_data(event, user) + self.generate_update_values_from_cumulative_data(event) + + def generate_change_old(self, event): + self.result['change_old']["assigned_to"] = self.cumulative_data['assigned_to'] + + def generate_update_values_from_cumulative_data(self, event): + if self.cumulative_data['assigned_to_name'] is not None: + self.result['update_values']["users"][str(self.cumulative_data['assigned_to'])] = self.cumulative_data['assigned_to_name'] + + def generate_change_new(self, event, user): + if user is None: + self.result['change_new']["assigned_to"] = 0 + else: + self.result['change_new']["assigned_to"] = user.id + + def update_cumulative_data(self, event, user): + self.cumulative_data['assigned_to_github_id'] = event['assignee']['id'] + if user is None: + self.cumulative_data['assigned_to'] = 0 + self.cumulative_data['assigned_to_name'] = event['assignee']['login'] + else: + self.cumulative_data['assigned_to'] = user.id + self.cumulative_data['assigned_to_name'] = user.get_full_name() + + +class UnassignedEventHandler: + def __init__(self, result, cumulative_data, users_bindings): + self.result = result + self.cumulative_data = cumulative_data + self.users_bindings = users_bindings + + def handle(self, event): + if self.cumulative_data['assigned_to_github_id'] == event['assignee']['id']: + self.result['update_values']["users"] = {} + + self.generate_change_old(event) + self.generate_update_values_from_cumulative_data(event) + self.generate_change_new(event) + self.update_cumulative_data(event) + self.generate_update_values_from_cumulative_data(event) + + def generate_change_old(self, event): + self.result['change_old']["assigned_to"] = self.cumulative_data['assigned_to'] + + def generate_update_values_from_cumulative_data(self, event): + if self.cumulative_data['assigned_to_name'] is not None: + self.result['update_values']["users"][str(self.cumulative_data['assigned_to'])] = self.cumulative_data['assigned_to_name'] + + def generate_change_new(self, event): + self.result['change_new']["assigned_to"] = None + + def update_cumulative_data(self, event): + self.cumulative_data['assigned_to_github_id'] = None + self.cumulative_data['assigned_to'] = None + self.cumulative_data['assigned_to_name'] = None diff --git a/taiga/importers/github/tasks.py b/taiga/importers/github/tasks.py new file mode 100644 index 00000000..f1069e9e --- /dev/null +++ b/taiga/importers/github/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.celery import app +from taiga.users.models import User +from .importer import GithubImporter + +logger = logging.getLogger('taiga.importers.github') + + +@app.task(bind=True) +def import_project(self, user_id, token, project_id, options): + user = User.object.get(id=user_id) + importer = GithubImporter(user, token) + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing github project"), + "error_message": _("Error importing github project"), + "project": project_id, + "exception": e + } + email = mail_builder.github_import_error(admin, ctx) + email.send() + logger.error('Error importing github project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.github_import_success(user, ctx) + email.send() diff --git a/taiga/importers/management/commands/import_from_github.py b/taiga/importers/management/commands/import_from_github.py new file mode 100644 index 00000000..9c93a527 --- /dev/null +++ b/taiga/importers/management/commands/import_from_github.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.db.models import Q + +from taiga.importers.github.importer import GithubImporter +from taiga.users.models import User, AuthData +from taiga.projects.services import projects as service + +import unittest.mock +import timeit + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--template', dest='template', default="kanban", + help='template to use: scrum or kanban (default kanban)') + parser.add_argument('--type', dest='type', default="user_stories", + help='type of object to use: user_stories or issues (default user_stories)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + + if options.get('token', None): + token = options.get('token') + else: + url = GithubImporter.get_auth_url(settings.GITHUB_API_CLIENT_ID) + print("Go to here and come with your code (in the redirected url): {}".format(url)) + code = input("Code: ") + access_data = GithubImporter.get_access_token(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, code) + token = access_data + + importer = GithubImporter(admin, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['id'], project['name'])) + project_id = input("Project id: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next github users:") + + for user in importer.list_users(project_id): + while True: + if user['detected_user'] is not None: + print("User automatically detected: {} as {}".format(user['full_name'], user['detected_user'])) + users_bindings[user['id']] = user['detected_user'] + break + + if not options.get('ask_for_users', False): + break + + username_or_email = input("{}: ".format(user['full_name'] or user['username'])) + if username_or_email == "": + break + try: + users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "type": options.get('type'), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference') + } + + importer.import_project(project_id, options) diff --git a/taiga/routers.py b/taiga/routers.py index 1546ef8b..961b7d19 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -286,9 +286,11 @@ router.register(r"application-tokens", ApplicationToken, base_name="application- # Third party importers from taiga.importers.trello.api import TrelloImporterViewSet from taiga.importers.jira.api import JiraImporterViewSet +from taiga.importers.github.api import GithubImporterViewSet router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") router.register(r"importers/jira", JiraImporterViewSet, base_name="importers-jira") +router.register(r"importers/github", GithubImporterViewSet, base_name="importers-github") # Stats diff --git a/tests/integration/test_importers_github_api.py b/tests/integration/test_importers_github_api.py new file mode 100644 index 00000000..bda1c928 --- /dev/null +++ b/tests/integration/test_importers_github_api.py @@ -0,0 +1,228 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import json + +from unittest import mock + +from django.core.urlresolvers import reverse + +from .. import factories as f +from taiga.importers import exceptions +from taiga.base.utils import json +from taiga.base import exceptions as exc + + +pytestmark = pytest.mark.django_db + + +def test_auth_url(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-auth-url")+"?uri=http://localhost:9001/project/new?from=github" + + response = client.get(url, content_type="application/json") + + assert response.status_code == 200 + assert 'url' in response.data + assert response.data['url'] == "https://github.com/login/oauth/authorize?client_id=&scope=user,repo&redirect_uri=http://localhost:9001/project/new?from=github" + +def test_authorize(client, settings): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-github-authorize") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + GithubImporterMock.get_access_token.return_value = "token" + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "code"})) + assert GithubImporterMock.get_access_token.calledWith(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, "code") + + assert response.status_code == 200 + assert 'token' in response.data + assert response.data['token'] == "token" + +def test_authorize_without_code(client): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-github-authorize") + + response = client.post(authorize_url, content_type="application/json", data=json.dumps({})) + + assert response.status_code == 400 + assert 'token' not in response.data + assert '_error_message' in response.data + assert response.data['_error_message'] == "Code param needed" + + +def test_authorize_with_bad_verify(client, settings): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-github-authorize") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + GithubImporterMock.get_access_token.side_effect = exceptions.InvalidAuthResult() + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "bad"})) + assert GithubImporterMock.get_access_token.calledWith(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, "bad") + + assert response.status_code == 400 + assert 'token' not in response.data + assert '_error_message' in response.data + assert response.data['_error_message'] == "Invalid auth data" + + +def test_import_github_list_users(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-users") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "username": "user1", "full_name": "user1", "detected_user": None}, + {"id": 2, "username": "user2", "full_name": "user2", "detected_user": None} + ] + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert response.data[0]["id"] == 1 + assert response.data[1]["id"] == 2 + + +def test_import_github_list_users_without_project(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-users") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "username": "user1", "full_name": "user1", "detected_user": None}, + {"id": 2, "username": "user2", "full_name": "user2", "detected_user": None} + ] + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_github_list_users_with_problem_on_request(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-users") + + with mock.patch('taiga.importers.github.importer.GithubClient') as GithubClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + GithubClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 400 + + +def test_import_github_list_projects(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-projects") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.list_projects.return_value = ["project1", "project2"] + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 200 + assert response.data[0] == "project1" + assert response.data[1] == "project2" + + +def test_import_github_list_projects_with_problem_on_request(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-list-projects") + + with mock.patch('taiga.importers.github.importer.GithubClient') as GithubClientMock: + instance = mock.Mock() + instance.get.side_effect = exc.WrongArguments("Invalid Request") + GithubClientMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_github_project_without_project_id(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-github-import-project") + + with mock.patch('taiga.importers.github.tasks.GithubImporter') as GithubImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_github_project_with_celery_enabled(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="async-imported-project") + client.login(user) + + url = reverse("importers-github-import-project") + + with mock.patch('taiga.importers.github.tasks.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 202 + assert "task_id" in response.data + + +def test_import_github_project_with_celery_disabled(client, settings): + settings.CELERY_ENABLED = False + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="imported-project") + client.login(user) + + url = reverse("importers-github-import-project") + + with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + GithubImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert "slug" in response.data + assert response.data['slug'] == "imported-project" From 477d9647700d0a65d95f8639f6d037e680ffa181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 09:40:37 +0100 Subject: [PATCH 04/12] Add asana importer --- requirements.txt | 1 + settings/common.py | 4 + taiga/importers/asana/api.py | 135 +++++++ taiga/importers/asana/importer.py | 351 ++++++++++++++++++ taiga/importers/asana/tasks.py | 56 +++ taiga/importers/exceptions.py | 3 + .../management/commands/import_from_asana.py | 101 +++++ taiga/routers.py | 2 + tests/integration/test_importers_asana_api.py | 236 ++++++++++++ 9 files changed, 889 insertions(+) create mode 100644 taiga/importers/asana/api.py create mode 100644 taiga/importers/asana/importer.py create mode 100644 taiga/importers/asana/tasks.py create mode 100644 taiga/importers/management/commands/import_from_asana.py create mode 100644 tests/integration/test_importers_asana_api.py diff --git a/requirements.txt b/requirements.txt index 11efe393..8f633141 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,3 +39,4 @@ psd-tools==1.4 CairoSVG==2.0.1 cryptography==1.7.1 PyJWT==1.4.2 +asana==0.6.2 diff --git a/settings/common.py b/settings/common.py index 9ce0e415..a0156181 100644 --- a/settings/common.py +++ b/settings/common.py @@ -567,6 +567,10 @@ GITHUB_API_CLIENT_SECRET = "" TRELLO_API_KEY = "" TRELLO_SECRET_KEY = "" +ASANA_APP_CALLBACK_URL = "" +ASANA_APP_ID = "" +ASANA_APP_SECRET = "" + JIRA_CONSUMER_KEY = "" JIRA_CERT = "" JIRA_PUB_CERT = "" diff --git a/taiga/importers/asana/api.py b/taiga/importers/asana/api.py new file mode 100644 index 00000000..3982a958 --- /dev/null +++ b/taiga/importers/asana/api.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id +from taiga.projects.serializers import ProjectSerializer + +from taiga.importers import permissions, exceptions +from .importer import AsanaImporter +from . import tasks + + +class AsanaImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + importer = AsanaImporter(request.user, token) + + try: + users = importer.list_users(project_id) + except exceptions.InvalidRequest: + raise exc.BadRequest(_('Invalid asana api request')) + except exceptions.FailedRequest: + raise exc.BadRequest(_('Failed to make the request to asana api')) + + for user in users: + if user['detected_user']: + user['user'] = { + 'id': user['detected_user'].id, + 'full_name': user['detected_user'].get_full_name(), + 'gravatar_id': get_user_gravatar_id(user['detected_user']), + 'photo': get_user_photo_url(user['detected_user']), + } + del(user['detected_user']) + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + token = request.DATA.get('token', None) + importer = AsanaImporter(request.user, token) + try: + projects = importer.list_projects() + except exceptions.InvalidRequest: + raise exc.BadRequest(_('Invalid asana api request')) + except exceptions.FailedRequest: + raise exc.BadRequest(_('Failed to make the request to asana api')) + return response.Ok(projects) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + options = { + "name": request.DATA.get('name', None), + "description": request.DATA.get('description', None), + "template": request.DATA.get('template', "scrum"), + "users_bindings": request.DATA.get("users_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, token, project_id, options) + return response.Accepted({"task_id": task.id}) + + importer = AsanaImporter(request.user, token) + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + + url = AsanaImporter.get_auth_url(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + code = request.DATA.get('code', None) + if code is None: + raise exc.BadRequest(_("Code param needed")) + + try: + asana_token = AsanaImporter.get_access_token(code, settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + except exceptions.InvalidRequest: + raise exc.BadRequest(_('Invalid asana api request')) + except exceptions.FailedRequest: + raise exc.BadRequest(_('Failed to make the request to asana api')) + + return response.Ok({"token": asana_token}) diff --git a/taiga/importers/asana/importer.py b/taiga/importers/asana/importer.py new file mode 100644 index 00000000..6c3c675a --- /dev/null +++ b/taiga/importers/asana/importer.py @@ -0,0 +1,351 @@ +import requests +import asana +import json +from django.core.files.base import ContentFile +from django.contrib.contenttypes.models import ContentType + +from taiga.projects.models import Project, ProjectTemplate, Membership +from taiga.projects.userstories.models import UserStory +from taiga.projects.tasks.models import Task +from taiga.projects.attachments.models import Attachment +from taiga.projects.history.services import take_snapshot +from taiga.projects.history.models import HistoryEntry +from taiga.projects.custom_attributes.models import UserStoryCustomAttribute, TaskCustomAttribute +from taiga.users.models import User +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline +from taiga.importers import exceptions + + +class AsanaClient(asana.Client): + def request(self, method, path, **options): + try: + return super().request(method, path, **options) + except asana.error.AsanaError: + raise exceptions.InvalidRequest() + except Exception as e: + raise exceptions.FailedRequest() + + +class AsanaImporter: + def __init__(self, user, token, import_closed_data=False): + self._import_closed_data = import_closed_data + self._user = user + self._client = AsanaClient.oauth(token=token) + + def list_projects(self): + projects = [] + for ws in self._client.workspaces.find_all(): + for project in self._client.projects.find_all(workspace=ws['id']): + project = self._client.projects.find_by_id(project['id']) + projects.append({ + "id": project['id'], + "name": "{}/{}".format(ws['name'], project['name']), + "description": project['notes'], + "is_private": True, + }) + return projects + + def list_users(self, project_id): + users = [] + for ws in self._client.workspaces.find_all(): + for user in self._client.users.find_by_workspace(ws['id'], fields=["id", "name", "email"]): + users.append({ + "id": user["id"], + "full_name": user['name'], + "detected_user": self._get_user(user) + }) + return users + + def _get_user(self, user, default=None): + if not user: + return default + + try: + return User.objects.get(email=user['email']) + except User.DoesNotExist: + pass + + return default + + def import_project(self, project_id, options): + project = self._client.projects.find_by_id(project_id) + taiga_project = self._import_project_data(project, options) + self._import_user_stories_data(taiga_project, project, options) + Timeline.objects.filter(project=taiga_project).delete() + rebuild_timeline(None, None, taiga_project.id) + return taiga_project + + def _import_project_data(self, project, options): + users_bindings = options.get('users_bindings', {}) + project_template = ProjectTemplate.objects.get(slug=options.get('template', 'scrum')) + + project_template.us_statuses = [] + project_template.us_statuses.append({ + "name": "Open", + "slug": "open", + "is_closed": False, + "is_archived": False, + "color": "#ff8a84", + "wip_limit": None, + "order": 1, + }) + project_template.us_statuses.append({ + "name": "Closed", + "slug": "closed", + "is_closed": True, + "is_archived": False, + "color": "#669900", + "wip_limit": None, + "order": 2, + }) + project_template.default_options["us_status"] = "Open" + + project_template.task_statuses = [] + project_template.task_statuses.append({ + "name": "Open", + "slug": "open", + "is_closed": False, + "color": "#ff8a84", + "order": 1, + }) + project_template.task_statuses.append({ + "name": "Closed", + "slug": "closed", + "is_closed": True, + "color": "#669900", + "order": 2, + }) + project_template.default_options["task_status"] = "Open" + + project_template.roles.append({ + "name": "Asana", + "slug": "asana", + "computable": False, + "permissions": project_template.roles[0]['permissions'], + "order": 70, + }) + + tags_colors = [] + for tag in self._client.tags.find_by_workspace(project['workspace']['id'], fields=["name", "color"]): + name = tag['name'].lower() + color = tag['color'] + tags_colors.append([name, color]) + + taiga_project = Project.objects.create( + name=options.get('name', None) or project['name'], + description=options.get('description', None) or project['notes'], + owner=self._user, + tags_colors=tags_colors, + creation_template=project_template, + is_private=options.get('is_private', False) + ) + + for user in self._client.users.find_by_workspace(project['workspace']['id']): + taiga_user = users_bindings.get(user['id'], None) + if taiga_user is None or taiga_user == self._user: + continue + + Membership.objects.create( + user=taiga_user, + project=taiga_project, + role=taiga_project.get_roles().get(slug="asana"), + is_admin=False, + invited_by=self._user, + ) + + UserStoryCustomAttribute.objects.create( + name="Due date", + description="Due date", + type="date", + order=1, + project=taiga_project + ) + + TaskCustomAttribute.objects.create( + name="Due date", + description="Due date", + type="date", + order=1, + project=taiga_project + ) + + return taiga_project + + def _import_user_stories_data(self, taiga_project, project, options): + users_bindings = options.get('users_bindings', {}) + tasks = self._client.tasks.find_by_project( + project['id'], + fields=["parent", "tags", "name", "notes", "tags.name", + "completed", "followers", "modified_at", "created_at", + "project", "due_on"] + ) + due_date_field = taiga_project.userstorycustomattributes.first() + + for task in tasks: + if task['parent']: + continue + + tags = [] + for tag in task['tags']: + tags.append(tag['name'].lower()) + + assigned_to = users_bindings.get(task.get('assignee', {}).get('id', None)) or None + + external_reference = None + if options.get('keep_external_reference', False): + external_url = "https://app.asana.com/0/{}/{}".format( + project['id'], + task['id'], + ) + external_reference = ["asana", external_url] + + us = UserStory.objects.create( + project=taiga_project, + owner=self._user, + assigned_to=assigned_to, + status=taiga_project.us_statuses.get(slug="closed" if task['completed'] else "open"), + kanban_order=task['id'], + sprint_order=task['id'], + backlog_order=task['id'], + subject=task['name'], + description=task.get('notes', ""), + tags=tags, + external_reference=external_reference + ) + + if task['due_on']: + us.custom_attributes_values.attributes_values = {due_date_field.id: task['due_on']} + us.custom_attributes_values.save() + + for follower in task['followers']: + follower_user = users_bindings.get(follower['id'], None) + if follower_user is not None: + us.add_watcher(follower_user) + + UserStory.objects.filter(id=us.id).update( + modified_date=task['modified_at'], + created_date=task['created_at'] + ) + + subtasks = self._client.tasks.subtasks( + task['id'], + fields=["parent", "tags", "name", "notes", "tags.name", + "completed", "followers", "modified_at", "created_at", + "due_on"] + ) + for subtask in subtasks: + self._import_task_data(taiga_project, us, project, subtask, options) + + take_snapshot(us, comment="", user=None, delete=False) + self._import_history(us, task, options) + self._import_attachments(us, task, options) + + def _import_task_data(self, taiga_project, us, assana_project, task, options): + users_bindings = options.get('users_bindings', {}) + tags = [] + for tag in task['tags']: + tags.append(tag['name'].lower()) + due_date_field = taiga_project.taskcustomattributes.first() + + assigned_to = users_bindings.get(task.get('assignee', {}).get('id', None)) or None + + external_reference = None + if options.get('keep_external_reference', False): + external_url = "https://app.asana.com/0/{}/{}".format( + assana_project['id'], + task['id'], + ) + external_reference = ["asana", external_url] + + taiga_task = Task.objects.create( + project=taiga_project, + user_story=us, + owner=self._user, + assigned_to=assigned_to, + status=taiga_project.task_statuses.get(slug="closed" if task['completed'] else "open"), + us_order=task['id'], + taskboard_order=task['id'], + subject=task['name'], + description=task.get('notes', ""), + tags=tags, + external_reference=external_reference + ) + + if task['due_on']: + taiga_task.custom_attributes_values.attributes_values = {due_date_field.id: task['due_on']} + taiga_task.custom_attributes_values.save() + + for follower in task['followers']: + follower_user = users_bindings.get(follower['id'], None) + if follower_user is not None: + taiga_task.add_watcher(follower_user) + + Task.objects.filter(id=taiga_task.id).update( + modified_date=task['modified_at'], + created_date=task['created_at'] + ) + + subtasks = self._client.tasks.subtasks( + task['id'], + fields=["parent", "tags", "name", "notes", "tags.name", + "completed", "followers", "modified_at", "created_at", + "due_on"] + ) + for subtask in subtasks: + self._import_task_data(taiga_project, us, assana_project, subtask, options) + + take_snapshot(taiga_task, comment="", user=None, delete=False) + self._import_history(taiga_task, task, options) + self._import_attachments(taiga_task, task, options) + + def _import_history(self, obj, task, options): + users_bindings = options.get('users_bindings', {}) + stories = self._client.stories.find_by_task(task['id']) + for story in stories: + if story['type'] == "comment": + snapshot = take_snapshot( + obj, + comment=story['text'], + user=users_bindings.get(story['created_by']['id'], User(full_name=story['created_by']['name'])), + delete=False + ) + HistoryEntry.objects.filter(id=snapshot.id).update(created_at=story['created_at']) + + def _import_attachments(self, obj, task, options): + attachments = self._client.attachments.find_by_task( + task['id'], + fields=['name', 'download_url', 'created_at'] + ) + for attachment in attachments: + data = requests.get(attachment['download_url']) + att = Attachment( + owner=self._user, + project=obj.project, + content_type=ContentType.objects.get_for_model(obj), + object_id=obj.id, + name=attachment['name'], + size=len(data.content), + created_date=attachment['created_at'], + is_deprecated=False, + ) + att.attached_file.save(attachment['name'], ContentFile(data.content), save=True) + + @classmethod + def get_auth_url(cls, client_id, client_secret, callback_url=None): + client = AsanaClient.oauth( + client_id=client_id, + client_secret=client_secret, + redirect_uri=callback_url + ) + (url, state) = client.session.authorization_url() + return url + + @classmethod + def get_access_token(cls, code, client_id, client_secret, callback_url=None): + client = AsanaClient.oauth( + client_id=client_id, + client_secret=client_secret, + redirect_uri=callback_url + ) + return client.session.fetch_token(code=code) diff --git a/taiga/importers/asana/tasks.py b/taiga/importers/asana/tasks.py new file mode 100644 index 00000000..5d36bed9 --- /dev/null +++ b/taiga/importers/asana/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.users.models import User +from taiga.celery import app +from .importer import AsanaImporter + +logger = logging.getLogger('taiga.importers.asana') + + +@app.task(bind=True) +def import_project(self, user_id, token, project_id, options): + user = User.object.get(id=user_id) + importer = AsanaImporter(user, token) + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing asana project"), + "error_message": _("Error importing asana project"), + "project": project_id, + "exception": e + } + email = mail_builder.asana_import_error(admin, ctx) + email.send() + logger.error('Error importing asana project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.asana_import_success(user, ctx) + email.send() diff --git a/taiga/importers/exceptions.py b/taiga/importers/exceptions.py index da430079..ebb8de86 100644 --- a/taiga/importers/exceptions.py +++ b/taiga/importers/exceptions.py @@ -1,3 +1,6 @@ +class InvalidRequest(Exception): + pass + class InvalidAuthResult(Exception): pass diff --git a/taiga/importers/management/commands/import_from_asana.py b/taiga/importers/management/commands/import_from_asana.py new file mode 100644 index 00000000..d7d7dd21 --- /dev/null +++ b/taiga/importers/management/commands/import_from_asana.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.conf import settings +from django.db.models import Q + +from taiga.importers.asana.importer import AsanaImporter +from taiga.users.models import User, AuthData +from taiga.projects.services import projects as service + +import unittest.mock +import timeit +import json + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--template', dest='template', default="kanban", + help='template to use: scrum or kanban (default kanban)') + parser.add_argument('--type', dest='type', default="user_stories", + help='type of object to use: user_stories or issues (default user_stories)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + + if options.get('token', None): + token = json.loads(options.get('token')) + else: + url = AsanaImporter.get_auth_url(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + print("Go to here and come with your code (in the redirected url): {}".format(url)) + code = input("Code: ") + access_data = AsanaImporter.get_access_token(code, settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + token = access_data + + importer = AsanaImporter(admin, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['id'], project['name'])) + project_id = input("Project id: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next asana users:") + + for user in importer.list_users(project_id): + while True: + if user['detected_user'] is not None: + print("User automatically detected: {} as {}".format(user['full_name'], user['detected_user'])) + users_bindings[user['id']] = user['detected_user'] + break + + if not options.get('ask_for_users', False): + break + + username_or_email = input("{}: ".format(user['full_name'] or user['username'])) + if username_or_email == "": + break + try: + users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "type": options.get('type'), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference') + } + + importer.import_project(project_id, options) diff --git a/taiga/routers.py b/taiga/routers.py index 961b7d19..382ede82 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -287,10 +287,12 @@ router.register(r"application-tokens", ApplicationToken, base_name="application- from taiga.importers.trello.api import TrelloImporterViewSet from taiga.importers.jira.api import JiraImporterViewSet from taiga.importers.github.api import GithubImporterViewSet +from taiga.importers.asana.api import AsanaImporterViewSet router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") router.register(r"importers/jira", JiraImporterViewSet, base_name="importers-jira") router.register(r"importers/github", GithubImporterViewSet, base_name="importers-github") +router.register(r"importers/asana", AsanaImporterViewSet, base_name="importers-asana") # Stats diff --git a/tests/integration/test_importers_asana_api.py b/tests/integration/test_importers_asana_api.py new file mode 100644 index 00000000..bc158694 --- /dev/null +++ b/tests/integration/test_importers_asana_api.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import json + +from unittest import mock + +from django.core.urlresolvers import reverse + +from .. import factories as f +from taiga.importers import exceptions +from taiga.base.utils import json +from taiga.base import exceptions as exc + + +pytestmark = pytest.mark.django_db + + +def test_auth_url(client, settings): + user = f.UserFactory.create() + client.login(user) + settings.ASANA_APP_CALLBACK_URL = "http://testserver/url" + settings.ASANA_APP_ID = "test-id" + settings.ASANA_APP_SECRET = "test-secret" + + url = reverse("importers-asana-auth-url") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + AsanaImporterMock.get_auth_url.return_value = "https://auth_url" + response = client.get(url, content_type="application/json") + assert AsanaImporterMock.get_auth_url.calledWith(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + + assert response.status_code == 200 + assert 'url' in response.data + assert response.data['url'] == "https://auth_url" + + +def test_authorize(client, settings): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-asana-authorize") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + AsanaImporterMock.get_access_token.return_value = "token" + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "code"})) + assert AsanaImporterMock.get_access_token.calledWith(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, "code") + + assert response.status_code == 200 + assert 'token' in response.data + assert response.data['token'] == "token" + + +def test_authorize_without_code(client): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-asana-authorize") + + response = client.post(authorize_url, content_type="application/json", data=json.dumps({})) + + assert response.status_code == 400 + assert 'token' not in response.data + assert '_error_message' in response.data + assert response.data['_error_message'] == "Code param needed" + + +def test_authorize_with_bad_verify(client, settings): + user = f.UserFactory.create() + client.login(user) + + authorize_url = reverse("importers-asana-authorize") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + AsanaImporterMock.get_access_token.side_effect = exceptions.InvalidRequest() + response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "bad"})) + assert AsanaImporterMock.get_access_token.calledWith(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, "bad") + + assert response.status_code == 400 + assert 'token' not in response.data + assert '_error_message' in response.data + assert response.data['_error_message'] == "Invalid asana api request" + + +def test_import_asana_list_users(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-asana-list-users") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "username": "user1", "full_name": "user1", "detected_user": None}, + {"id": 2, "username": "user2", "full_name": "user2", "detected_user": None} + ] + AsanaImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert response.data[0]["id"] == 1 + assert response.data[1]["id"] == 2 + + +def test_import_asana_list_users_without_project(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-asana-list-users") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + instance = mock.Mock() + instance.list_users.return_value = [ + {"id": 1, "username": "user1", "full_name": "user1", "detected_user": None}, + {"id": 2, "username": "user2", "full_name": "user2", "detected_user": None} + ] + AsanaImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_asana_list_users_with_problem_on_request(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-asana-list-users") + + with mock.patch('taiga.importers.asana.importer.AsanaClient') as AsanaClientMock: + instance = mock.Mock() + instance.workspaces.find_all.side_effect = exceptions.InvalidRequest() + AsanaClientMock.oauth.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 400 + + +def test_import_asana_list_projects(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-asana-list-projects") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + instance = mock.Mock() + instance.list_projects.return_value = ["project1", "project2"] + AsanaImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 200 + assert response.data[0] == "project1" + assert response.data[1] == "project2" + + +def test_import_asana_list_projects_with_problem_on_request(client): + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-asana-list-projects") + + with mock.patch('taiga.importers.asana.importer.AsanaClient') as AsanaClientMock: + instance = mock.Mock() + instance.workspaces.find_all.side_effect = exc.WrongArguments("Invalid Request") + AsanaClientMock.oauth.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_asana_project_without_project_id(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + client.login(user) + + url = reverse("importers-asana-import-project") + + with mock.patch('taiga.importers.asana.tasks.AsanaImporter') as AsanaImporterMock: + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token"})) + + assert response.status_code == 400 + + +def test_import_asana_project_with_celery_enabled(client, settings): + settings.CELERY_ENABLED = True + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="async-imported-project") + client.login(user) + + url = reverse("importers-asana-import-project") + + with mock.patch('taiga.importers.asana.tasks.AsanaImporter') as AsanaImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + AsanaImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 202 + assert "task_id" in response.data + + +def test_import_asana_project_with_celery_disabled(client, settings): + settings.CELERY_ENABLED = False + + user = f.UserFactory.create() + project = f.ProjectFactory.create(slug="imported-project") + client.login(user) + + url = reverse("importers-asana-import-project") + + with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: + instance = mock.Mock() + instance.import_project.return_value = project + AsanaImporterMock.return_value = instance + response = client.post(url, content_type="application/json", data=json.dumps({"token": "token", "project": 1})) + + assert response.status_code == 200 + assert "slug" in response.data + assert response.data['slug'] == "imported-project" From 15c2868d24e6755224a3deeb7f0cd4ae66436240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 09:42:34 +0100 Subject: [PATCH 05/12] Add pivotal importer prototype --- .../commands/import_from_pivotal.py | 93 +++ taiga/importers/pivotal/api.py | 143 ++++ taiga/importers/pivotal/importer.py | 702 ++++++++++++++++++ taiga/importers/pivotal/tasks.py | 56 ++ 4 files changed, 994 insertions(+) create mode 100644 taiga/importers/management/commands/import_from_pivotal.py create mode 100644 taiga/importers/pivotal/api.py create mode 100644 taiga/importers/pivotal/importer.py create mode 100644 taiga/importers/pivotal/tasks.py diff --git a/taiga/importers/management/commands/import_from_pivotal.py b/taiga/importers/management/commands/import_from_pivotal.py new file mode 100644 index 00000000..f3430c69 --- /dev/null +++ b/taiga/importers/management/commands/import_from_pivotal.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.core.management.base import BaseCommand +from django.db.models import Q + +from taiga.importers.pivotal import PivotalImporter +from taiga.users.models import User +from taiga.projects.services import projects as service + +import unittest.mock +import timeit + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--token', dest="token", type=str, + help='Auth token') + parser.add_argument('--project-id', dest="project_id", type=str, + help='Project ID or full name (ex: taigaio/taiga-back)') + parser.add_argument('--template', dest='template', default="scrum", + help='template to use: scrum or scrum (default scrum)') + parser.add_argument('--ask-for-users', dest='ask_for_users', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--closed-data', dest='closed_data', const=True, + action="store_const", default=False, + help='Import closed data') + parser.add_argument('--keep-external-reference', dest='keep_external_reference', const=True, + action="store_const", default=False, + help='Store external reference of imported data') + + def handle(self, *args, **options): + admin = User.objects.get(username="admin") + + if options.get('token', None): + token = options.get('token') + else: + print("You need a user token") + return + + importer = PivotalImporter(admin, token) + + if options.get('project_id', None): + project_id = options.get('project_id') + else: + print("Select the project to import:") + for project in importer.list_projects(): + print("- {}: {}".format(project['project_id'], project['project_name'])) + project_id = input("Project id: ") + + users_bindings = {} + if options.get('ask_for_users', None): + print("Add the username or email for next pivotal users:") + for user in importer.list_users(project_id): + try: + users_bindings[user['id']] = User.objects.get(Q(email=user['person'].get('email', "not-valid"))) + break + except User.DoesNotExist: + pass + + while True: + username_or_email = input("{}: ".format(user['person']['name'])) + if username_or_email == "": + break + try: + users_bindings[user['id']] = User.objects.get(Q(username=username_or_email) | Q(email=username_or_email)) + break + except User.DoesNotExist: + print("ERROR: Invalid username or email") + + options = { + "template": options.get('template'), + "import_closed_data": options.get("closed_data", False), + "users_bindings": users_bindings, + "keep_external_reference": options.get('keep_external_reference') + } + importer.import_project(project_id, options) diff --git a/taiga/importers/pivotal/api.py b/taiga/importers/pivotal/api.py new file mode 100644 index 00000000..1176f9aa --- /dev/null +++ b/taiga/importers/pivotal/api.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Taiga Agile LLC +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from django.utils.translation import ugettext as _ +from django.conf import settings + +from taiga.base.api import viewsets +from taiga.base import response +from taiga.base import exceptions as exc +from taiga.base.decorators import list_route +from taiga.users.models import AuthData, User +from taiga.users.services import get_user_photo_url +from taiga.users.gravatar import get_user_gravatar_id +from taiga.projects.serializers import ProjectSerializer + +from taiga.importers import permissions +from .importer import PivotalImporter +from . import tasks + + +class PivotalImporterViewSet(viewsets.ViewSet): + permission_classes = (permissions.ImporterPermission,) + + @list_route(methods=["POST"]) + def list_users(self, request, *args, **kwargs): + self.check_permissions(request, "list_users", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + importer = PivotalImporter(request.user, token) + users = importer.list_users(project_id) + for user in users: + user['user'] = None + if not user['email']: + continue + + try: + taiga_user = User.objects.get(email=user['email']) + except User.DoesNotExist: + continue + + user['user'] = { + 'id': taiga_user.id, + 'full_name': taiga_user.get_full_name(), + 'gravatar_id': get_user_gravatar_id(taiga_user), + 'photo': get_user_photo_url(taiga_user), + } + return response.Ok(users) + + @list_route(methods=["POST"]) + def list_projects(self, request, *args, **kwargs): + self.check_permissions(request, "list_projects", None) + token = request.DATA.get('token', None) + importer = PivotalImporter(request.user, token) + projects = importer.list_projects() + return response.Ok(projects) + + @list_route(methods=["POST"]) + def import_project(self, request, *args, **kwargs): + self.check_permissions(request, "import_project", None) + + token = request.DATA.get('token', None) + project_id = request.DATA.get('project', None) + if not project_id: + raise exc.WrongArguments(_("The project param is needed")) + + options = { + "template": request.DATA.get('template', "kanban"), + "users_bindings": request.DATA.get("users_bindings", {}), + "keep_external_reference": request.DATA.get("keep_external_reference", False), + "is_private": request.DATA.get("is_private", False), + } + + if settings.CELERY_ENABLED: + task = tasks.import_project.delay(request.user.id, token, project_id, options) + return response.Accepted({"pivotal_import_id": task.id}) + + importer = PivotalImporter(request.user, token) + project = importer.import_project(project_id, options) + project_data = { + "slug": project.slug, + "my_permissions": ["view_us"], + "is_backlog_activated": project.is_backlog_activated, + "is_kanban_activated": project.is_kanban_activated, + } + + return response.Ok(project_data) + + @list_route(methods=["GET"]) + def auth_url(self, request, *args, **kwargs): + self.check_permissions(request, "auth_url", None) + + (oauth_token, oauth_secret, url) = PivotalImporter.get_auth_url() + + (auth_data, created) = AuthData.objects.get_or_create( + user=request.user, + key="pivotal-oauth", + defaults={ + "value": "", + "extra": {}, + } + ) + auth_data.extra = { + "oauth_token": oauth_token, + "oauth_secret": oauth_secret, + } + auth_data.save() + + return response.Ok({"url": url}) + + @list_route(methods=["POST"]) + def authorize(self, request, *args, **kwargs): + self.check_permissions(request, "authorize", None) + + try: + oauth_data = request.user.auth_data.get(key="pivotal-oauth") + oauth_token = oauth_data.extra['oauth_token'] + oauth_secret = oauth_data.extra['oauth_secret'] + oauth_verifier = request.DATA.get('code') + oauth_data.delete() + pivotal_token = PivotalImporter.get_access_token(oauth_token, oauth_secret, oauth_verifier)['oauth_token'] + except Exception as e: + raise exc.WrongArguments(_("Invalid or expired auth token")) + + return response.Ok({ + "token": pivotal_token + }) diff --git a/taiga/importers/pivotal/importer.py b/taiga/importers/pivotal/importer.py new file mode 100644 index 00000000..6f1456ca --- /dev/null +++ b/taiga/importers/pivotal/importer.py @@ -0,0 +1,702 @@ +from django.core.files.base import ContentFile +from django.contrib.contenttypes.models import ContentType +import requests + +from taiga.users.models import User +from taiga.projects.references.models import recalc_reference_counter +from taiga.projects.models import Project, ProjectTemplate, Membership, Points +from taiga.projects.userstories.models import UserStory, RolePoints +from taiga.projects.tasks.models import Task +from taiga.projects.milestones.models import Milestone +from taiga.projects.epics.models import Epic, RelatedUserStory +from taiga.projects.attachments.models import Attachment +from taiga.projects.history.services import take_snapshot +from taiga.projects.history.services import (make_diff_from_dicts, + make_diff_values, + make_key_from_model_object, + get_typename_for_model_class, + FrozenDiff) +from taiga.projects.history.models import HistoryEntry +from taiga.projects.history.choices import HistoryType +from taiga.projects.custom_attributes.models import UserStoryCustomAttribute +from taiga.mdrender.service import render as mdrender +from taiga.timeline.rebuilder import rebuild_timeline +from taiga.timeline.models import Timeline + + +class PivotalClient: + def __init__(self, token): + self.api_url = "https://www.pivotaltracker.com/services/v5/{}" + self.token = token + self.me = self.get('/me') + + def get(self, uri_path, query_params=None): + headers = { + 'X-TrackerToken': self.token + } + if query_params is None: + query_params = {} + + if uri_path[0] == '/': + uri_path = uri_path[1:] + url = self.api_url.format(uri_path) + + response = requests.get(url, params=query_params, headers=headers) + + if response.status_code == 401: + raise Exception("Unauthorized: %s at %s" % (response.text, url), response) + if response.status_code != 200: + raise Exception("Resource Unavailable: %s at %s" % (response.text, url), response) + + return response.json() + + def get_attachment(self, attachment_id): + headers = { + 'X-TrackerToken': self.token + } + url = "https://www.pivotaltracker.com/file_attachments/{}/download".format(attachment_id) + response = requests.get(url, headers=headers) + return response.content + + +class PivotalImporter: + def __init__(self, user, token): + self._user = user + self._client = PivotalClient(token=token) + + def list_projects(self): + return self._client.me['projects'] + + def list_users(self, project_id): + return self._client.get("/projects/{}/memberships".format(project_id)) + + def import_project(self, project_id, options={"template": "scrum", "users_bindings": {}, "keep_external_reference": False}): + (project, project_data) = self._import_project_data(project_id, options) + self._import_epics_data(project_data, project, options) + self._import_user_stories_data(project_data, project, options) + Timeline.objects.filter(project=project).delete() + rebuild_timeline(None, None, project.id) + recalc_reference_counter(project) + + def _import_project_data(self, project_id, options): + project_data = self._client.get( + "/projects/{}".format(project_id), + { + "fields": ",".join([ + "point_scale", + "name", + "description", + "labels(name)", + ]) + } + ) + project_data['iterations'] = self._client.get( + "/projects/{}/iterations".format(project_id), + { + "fields": ",".join([ + "number", + "start", + "finish", + "stories", + ]) + } + ) + project_data['epics'] = self._client.get( + "/projects/{}/epics".format(project_data['id']), + { + "fields": ",".join([ + "name", + "label", + "description", + "comments(text,file_attachments,google_attachments,person,created_at)", + "follower_ids", + "created_at", + "updated_at", + "url", + ]) + } + ) + + project_template = ProjectTemplate.objects.get(slug=options['template']) + project_template.is_epics_activated = True + project_template.us_statuses = [] + project_template.points = [{ + "value": None, + "name": "?", + "order": 1, + }] + + counter = 2 + for points in project_data['point_scale'].split(","): + project_template.points.append({ + "value": int(points), + "name": points, + "order": counter + }) + counter += 1 + + project_template.us_statuses.append({ + "name": "Unscheduled", + "slug": "unscheduled", + "is_closed": True, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": 1, + }) + project_template.us_statuses.append({ + "name": "Unstarted", + "slug": "unstarted", + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": 2, + }) + project_template.us_statuses.append({ + "name": "Planned", + "slug": "planned", + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": 3, + }) + project_template.us_statuses.append({ + "name": "Started", + "slug": "started", + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": 4, + }) + project_template.us_statuses.append({ + "name": "Finished", + "slug": "finished", + "is_closed": False, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": 5, + }) + project_template.us_statuses.append({ + "name": "Delivered", + "slug": "delivered", + "is_closed": True, + "is_archived": False, + "color": "#999999", + "wip_limit": None, + "order": 6, + }) + project_template.us_statuses.append({ + "name": "Rejected", + "slug": "rejected", + "is_closed": True, + "is_archived": True, + "color": "#999999", + "wip_limit": None, + "order": 7, + }) + project_template.us_statuses.append({ + "name": "Accepted", + "slug": "accepted", + "is_closed": True, + "is_archived": True, + "color": "#999999", + "wip_limit": None, + "order": 8, + }) + project_template.default_options["us_status"] = "Unscheduled" + + project_template.task_statuses = [] + project_template.task_statuses.append({ + "name": "Incomplete", + "slug": "incomplete", + "is_closed": False, + "color": "#ff8a84", + "order": 1, + }) + project_template.task_statuses.append({ + "name": "Complete", + "slug": "complete", + "is_closed": True, + "color": "#669900", + "order": 2, + }) + project_template.default_options["task_status"] = "Incomplete" + + main_permissions = project_template.roles[0]['permissions'] + project_template.roles = [{ + "name": "Main", + "slug": "main", + "computable": True, + "permissions": main_permissions, + "order": 70, + }] + + tags_colors = [] + for label in project_data['labels']: + name = label['name'].lower() + tags_colors.append([name, None]) + + project = Project.objects.create( + name=project_data['name'], + description=project_data.get('description', ''), + owner=self._user, + tags_colors=tags_colors, + creation_template=project_template + ) + + UserStoryCustomAttribute.objects.create( + name="Due date", + description="Due date", + type="date", + order=1, + project=project + ) + UserStoryCustomAttribute.objects.create( + name="Type", + description="Story type", + type="text", + order=2, + project=project + ) + for user in options.get('users_bindings', {}).values(): + if user != self._user: + Membership.objects.get_or_create( + user=user, + project=project, + role=project.get_roles().get(slug="main"), + is_admin=False, + ) + + for iteration in project_data['iterations']: + milestone = Milestone.objects.create( + name="Sprint {}".format(iteration['number']), + slug="sprint-{}".format(iteration['number']), + owner=self._user, + project=project, + estimated_start=iteration['start'][:10], + estimated_finish=iteration['finish'][:10], + ) + Milestone.objects.filter(id=milestone.id).update( + created_date=iteration['start'], + modified_date=iteration['start'], + ) + return (project, project_data) + + def _import_user_stories_data(self, project_data, project, options): + users_bindings = options.get('users_bindings', {}) + epics = {e['label']['id']: e for e in project_data['epics']} + due_date_field = project.userstorycustomattributes.get(name="Due date") + story_type_field = project.userstorycustomattributes.get(name="Type") + story_milestone_binding = {} + for iteration in project_data['iterations']: + for story in iteration['stories']: + story_milestone_binding[story['id']] = Milestone.objects.get( + project=project, + slug="sprint-{}".format(iteration['number']) + ) + + counter = 0 + offset = 0 + while True: + stories = self._client.get("/projects/{}/stories".format(project_data['id']), { + "envelope": "true", + "limit": 300, + "offset": offset, + "fields": ",".join([ + "name", + "description", + "estimate", + "story_type", + "current_state", + "deadline", + "requested_by_id", + "owner_ids", + "labels(id,name)", + "comments(text,file_attachments,google_attachments,person,created_at)", + "tasks(id,description,position,complete,created_at,updated_at)", + "follower_ids", + "created_at", + "updated_at", + "url", + ])}) + offset += 300 + for story in stories['data']: + tags = [] + for label in story['labels']: + tags.append(label['name']) + + assigned_to = None + if len(story['owner_ids']) > 0: + assigned_to = users_bindings.get(story['owner_ids'][0], None) + + owner = users_bindings.get(story['requested_by_id'], self._user) + + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["pivotal", story['url']] + + us = UserStory.objects.create( + project=project, + owner=owner, + assigned_to=assigned_to, + status=project.us_statuses.get(slug=story['current_state']), + kanban_order=counter, + sprint_order=counter, + backlog_order=counter, + subject=story['name'], + description=story.get('description', ''), + tags=tags, + external_reference=external_reference, + milestone=story_milestone_binding.get(story['id'], None) + ) + + points = Points.objects.get(project=project, value=story.get('estimate', None)) + RolePoints.objects.filter(user_story=us, role__slug="main").update(points_id=points.id) + + if len(story['owner_ids']) > 1: + watchers = list(set(story['owner_ids'][1:] + story['follower_ids'])) + else: + watchers = story['follower_ids'] + + for watcher in watchers: + watcher_user = users_bindings.get(watcher, None) + if watcher_user: + us.add_watcher(watcher_user) + + if story.get('deadline', None): + us.custom_attributes_values.attributes_values = {due_date_field.id: story['deadline']} + us.custom_attributes_values.save() + if story.get('story_type', None): + us.custom_attributes_values.attributes_values = {story_type_field.id: story['story_type']} + us.custom_attributes_values.save() + + UserStory.objects.filter(id=us.id).update( + ref=story['id'], + modified_date=story['updated_at'], + created_date=story['created_at'] + ) + take_snapshot(us, comment="", user=None, delete=False) + + for label in story['labels']: + if epics.get(label['id'], None): + RelatedUserStory.objects.create( + epic=Epic.objects.get(project=project, ref=epics.get(label['id'])['id']), + user_story=us, + order=us.backlog_order + ) + self._import_tasks(project_data, us, story) + self._import_user_story_activity(project_data, us, story, options) + self._import_comments(project_data, us, story, options) + counter += 1 + + if len(stories['data']) < 300: + break + + def _import_epics_data(self, project_data, project, options): + users_bindings = options.get('users_bindings', {}) + counter = 0 + + for epic in project_data['epics']: + external_reference = None + if options.get('keep_external_reference', False): + external_reference = ["pivotal", epic['url']] + + taiga_epic = Epic.objects.create( + project=project, + owner=self._user, + status=project.epic_statuses.get(slug="new"), + epics_order=counter, + subject=epic['name'], + description=epic.get('description', ''), + tags=[], + external_reference=external_reference + ) + + Epic.objects.filter(id=taiga_epic.id).update( + ref=epic['id'], + modified_date=epic['updated_at'], + created_date=epic['created_at'] + ) + + for watcher in epic['follower_ids']: + watcher_user = users_bindings.get(watcher, None) + if watcher_user: + taiga_epic.add_watcher(watcher_user) + + take_snapshot(taiga_epic, comment="", user=None, delete=False) + self._import_comments(project_data, taiga_epic, epic, options) + self._import_epic_activity(project_data, taiga_epic, epic, options) + counter += 1 + + def _import_tasks(self, project_data, us, story): + for task in story['tasks']: + taiga_task = Task.objects.create( + subject=task['description'], + status=us.project.task_statuses.get(slug="complete" if task['complete'] else "incomplete"), + project=us.project, + us_order=task['position'], + taskboard_order=task['position'], + user_story=us + ) + + Task.objects.filter(id=taiga_task.id).update( + ref=task['id'], + modified_date=task['updated_at'], + created_date=task['created_at'] + ) + take_snapshot(taiga_task, comment="", user=None, delete=False) + + def _import_attachment(self, obj, attachment_id, attachment_name, created_at, person_id, options): + users_bindings = options.get('users_bindings', {}) + + data = self._client.get_attachment(attachment_id) + att = Attachment( + owner=users_bindings.get(person_id, self._user), + project=obj.project, + content_type=ContentType.objects.get_for_model(obj), + object_id=obj.id, + name=attachment_name, + size=len(data), + created_date=created_at, + is_deprecated=False, + ) + att.attached_file.save(attachment_name, ContentFile(data), save=True) + + def _import_comments(self, project_data, obj, story, options): + users_bindings = options.get('users_bindings', {}) + + for comment in story['comments']: + if 'text' in comment: + snapshot = take_snapshot( + obj, + comment=comment['text'], + user=users_bindings.get(comment['person']['id'], User(full_name=comment['person']['name'])), + delete=False + ) + HistoryEntry.objects.filter(id=snapshot.id).update(created_at=comment['created_at']) + for attachment in comment['file_attachments']: + self._import_attachment( + obj, + attachment['id'], + attachment['filename'], + comment['created_at'], + comment['person']['id'], + options + ) + + def _import_user_story_activity(self, project_data, us, story, options): + offset = 0 + while True: + activities = self._client.get( + "/projects/{}/stories/{}/activity".format( + project_data['id'], + story['id'], + ), + {"envelope": "true", "limit": 300, "offset": offset} + ) + offset += 300 + for activity in activities['data']: + self._import_activity(us, activity, options) + + if len(activities['data']) < 300: + break + + def _import_epic_activity(self, project_data, taiga_epic, epic, options): + offset = 0 + while True: + activities = self._client.get( + "/projects/{}/epics/{}/activity".format( + project_data['id'], + epic['id'], + ), + {"envelope": "true", "limit": 300, "offset": offset} + ) + offset += 300 + for activity in activities['data']: + self._import_activity(taiga_epic, activity, options) + + if len(activities['data']) < 300: + break + + def _import_activity(self, obj, activity, options): + activity_data = self._transform_activity_data(obj, activity, options) + if activity_data is None: + return + + change_old = activity_data['change_old'] + change_new = activity_data['change_new'] + hist_type = activity_data['hist_type'] + comment = activity_data['comment'] + user = activity_data['user'] + + key = make_key_from_model_object(activity_data['obj']) + typename = get_typename_for_model_class(type(activity_data['obj'])) + + diff = make_diff_from_dicts(change_old, change_new) + fdiff = FrozenDiff(key, diff, {}) + + entry = HistoryEntry.objects.create( + user=user, + project_id=obj.project.id, + key=key, + type=hist_type, + snapshot=None, + diff=fdiff.diff, + values=make_diff_values(typename, fdiff), + comment=comment, + comment_html=mdrender(obj.project, comment), + is_hidden=False, + is_snapshot=False, + ) + HistoryEntry.objects.filter(id=entry.id).update(created_at=activity['occurred_at']) + return HistoryEntry.objects.get(id=entry.id) + + def _transform_activity_data(self, obj, activity, options): + users_bindings = options.get('users_bindings', {}) + due_date_field = obj.project.userstorycustomattributes.get(name="Due date") + story_type_field = obj.project.userstorycustomattributes.get(name="Type") + + user = {"pk": None, "name": activity.get('performed_by', {}).get('name', None)} + taiga_user = users_bindings.get(activity.get('performed_by', {}).get('id', None), None) + if taiga_user: + user = {"pk": taiga_user.id, "name": taiga_user.get_full_name()} + + result = { + "change_old": {}, + "change_new": {}, + "hist_type": HistoryType.change, + "comment": "", + "user": user, + "obj": obj + } + + if activity['kind'] == "story_create_activity": + UserStory.objects.filter(id=obj.id, created_date__gt=activity['occurred_at']).update( + created_date=activity['occurred_at'], + owner=users_bindings.get(activity["performed_by"]["id"], self._user) + ) + return None + elif activity['kind'] == "epic_create_activity": + Epic.objects.filter(id=obj.id, created_date__gt=activity['occurred_at']).update( + created_date=activity['occurred_at'], + owner=users_bindings.get(activity["performed_by"]["id"], self._user) + ) + return None + elif activity['kind'] in ["story_update_activity", "epic_update_activity"]: + for change in activity['changes']: + if change['change_type'] != "update" or change['kind'] not in ["story", "epic"]: + continue + + if 'description' in change['new_values']: + result['change_old']["description"] = str(change['original_values']['description']) + result['change_new']["description"] = str(change['new_values']['description']) + result['change_old']["description_html"] = mdrender(obj.project, str(change['original_values']['description'])) + result['change_new']["description_html"] = mdrender(obj.project, str(change['new_values']['description'])) + + if 'estimate' in change['new_values']: + old_points = None + if change['original_values']['estimate']: + estimation = change['original_values']['estimate'] + (old_points, _) = Points.objects.get_or_create( + project=obj.project, + value=estimation, + defaults={ + "name": str(estimation), + "order": estimation, + } + ) + old_points = old_points.id + new_points = None + if change['new_values']['estimate']: + estimation = change['new_values']['estimate'] + (new_points, _) = Points.objects.get_or_create( + project=obj.project, + value=estimation, + defaults={ + "name": str(estimation), + "order": estimation, + } + ) + new_points = new_points.id + result['change_old']["points"] = {obj.project.roles.get(slug="main").id: old_points} + result['change_new']["points"] = {obj.project.roles.get(slug="main").id: new_points} + + if 'name' in change['new_values']: + result['change_old']["subject"] = change['original_values']['name'] + result['change_new']["subject"] = change['new_values']['name'] + + if 'labels' in change['new_values']: + result['change_old']["tags"] = [l.lower() for l in change['original_values']['labels']] + result['change_new']["tags"] = [l.lower() for l in change['new_values']['labels']] + + if 'current_state' in change['new_values']: + result['change_old']["status"] = obj.project.us_statuses.get(slug=change['original_values']['current_state']).id + result['change_new']["status"] = obj.project.us_statuses.get(slug=change['new_values']['current_state']).id + + if 'story_type' in change['new_values']: + if "custom_attributes" not in result['change_old']: + result['change_old']["custom_attributes"] = [] + if "custom_attributes" not in result['change_new']: + result['change_new']["custom_attributes"] = [] + + result['change_old']["custom_attributes"].append({ + "name": "Type", + "value": change['original_values']['story_type'], + "id": story_type_field.id + }) + result['change_new']["custom_attributes"].append({ + "name": "Type", + "value": change['new_values']['story_type'], + "id": story_type_field.id + }) + + if 'deadline' in change['new_values']: + if "custom_attributes" not in result['change_old']: + result['change_old']["custom_attributes"] = [] + if "custom_attributes" not in result['change_new']: + result['change_new']["custom_attributes"] = [] + + result['change_old']["custom_attributes"].append({ + "name": "Due date", + "value": change['original_values']['deadline'], + "id": due_date_field.id + }) + result['change_new']["custom_attributes"].append({ + "name": "Due date", + "value": change['new_values']['deadline'], + "id": due_date_field.id + }) + + # TODO: Process owners_ids + + elif activity['kind'] == "task_create_activity": + return None + elif activity['kind'] == "task_update_activity": + for change in activity['changes']: + if change['change_type'] != "update" or change['kind'] != "task": + continue + + try: + task = Task.objects.get(project=obj.project, ref=change['id']) + if 'description' in change['new_values']: + result['change_old']["subject"] = change['original_values']['description'] + result['change_new']["subject"] = change['new_values']['description'] + result['obj'] = task + if 'complete' in change['new_values']: + result['change_old']["status"] = obj.project.task_statuses.get(slug="complete" if change['original_values']['complete'] else "incomplete").id + result['change_new']["status"] = obj.project.task_statuses.get(slug="complete" if change['new_values']['complete'] else "incomplete").id + result['obj'] = task + except Task.DoesNotExist: + return None + + elif activity['kind'] == "comment_create_activity": + return None + elif activity['kind'] == "comment_update_activity": + return None + elif activity['kind'] == "story_move_activity": + return None + return result diff --git a/taiga/importers/pivotal/tasks.py b/taiga/importers/pivotal/tasks.py new file mode 100644 index 00000000..2e8f54ea --- /dev/null +++ b/taiga/importers/pivotal/tasks.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 2014-2016 Andrey Antukh +# Copyright (C) 2014-2016 Jesús Espino +# Copyright (C) 2014-2016 David Barragán +# Copyright (C) 2014-2016 Alejandro Alonso +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import sys + +from django.utils.translation import ugettext as _ + +from taiga.base.mails import mail_builder +from taiga.users.models import User +from taiga.celery import app +from .importer import PivotalImporter + +logger = logging.getLogger('taiga.importers.pivotal') + + +@app.task(bind=True) +def import_project(self, user_id, token, project_id, options): + user = User.object.get(id=user_id) + importer = PivotalImporter(user, token) + try: + project = importer.import_project(project_id, options) + except Exception as e: + # Error + ctx = { + "user": user, + "error_subject": _("Error importing pivotal project"), + "error_message": _("Error importing pivotal project"), + "project": project_id, + "exception": e + } + email = mail_builder.pivotal_import_error(admin, ctx) + email.send() + logger.error('Error importing pivotal project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + else: + ctx = { + "project": project, + "user": user, + } + email = mail_builder.pivotal_import_success(user, ctx) + email.send() From 475092f4c2677e0445fef39eb77ee4f643b622f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Jan 2017 10:12:48 +0100 Subject: [PATCH 06/12] Add importers changelog entries --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbc26535..d38f260e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ - Add japanese (ja) translation. - Add korean (ko) translation. - Add chinese simplified (zh-Hans) translation. +- Third party services project importers: + - Trello + - Jira 7 + - Github + - Asana + - Pivotal Tracker ### Misc - API: From c2b0ac84e6065a12fd05dd09384bf35cfe5ed556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 25 Jan 2017 17:40:47 +0100 Subject: [PATCH 07/12] [i18n] Fix some messages and update locales --- taiga/importers/asana/api.py | 12 +- taiga/importers/asana/tasks.py | 6 +- taiga/importers/github/tasks.py | 6 +- taiga/importers/jira/tasks.py | 6 +- taiga/importers/pivotal/tasks.py | 6 +- taiga/importers/trello/tasks.py | 6 +- taiga/locale/en/LC_MESSAGES/django.po | 137 ++++++++++++++---- tests/integration/test_importers_asana_api.py | 2 +- 8 files changed, 130 insertions(+), 51 deletions(-) diff --git a/taiga/importers/asana/api.py b/taiga/importers/asana/api.py index 3982a958..611eda8b 100644 --- a/taiga/importers/asana/api.py +++ b/taiga/importers/asana/api.py @@ -48,9 +48,9 @@ class AsanaImporterViewSet(viewsets.ViewSet): try: users = importer.list_users(project_id) except exceptions.InvalidRequest: - raise exc.BadRequest(_('Invalid asana api request')) + raise exc.BadRequest(_('Invalid Asana API request')) except exceptions.FailedRequest: - raise exc.BadRequest(_('Failed to make the request to asana api')) + raise exc.BadRequest(_('Failed to make the request to Asana API')) for user in users: if user['detected_user']: @@ -71,9 +71,9 @@ class AsanaImporterViewSet(viewsets.ViewSet): try: projects = importer.list_projects() except exceptions.InvalidRequest: - raise exc.BadRequest(_('Invalid asana api request')) + raise exc.BadRequest(_('Invalid Asana API request')) except exceptions.FailedRequest: - raise exc.BadRequest(_('Failed to make the request to asana api')) + raise exc.BadRequest(_('Failed to make the request to Asana API')) return response.Ok(projects) @list_route(methods=["POST"]) @@ -128,8 +128,8 @@ class AsanaImporterViewSet(viewsets.ViewSet): try: asana_token = AsanaImporter.get_access_token(code, settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) except exceptions.InvalidRequest: - raise exc.BadRequest(_('Invalid asana api request')) + raise exc.BadRequest(_('Invalid Asana API request')) except exceptions.FailedRequest: - raise exc.BadRequest(_('Failed to make the request to asana api')) + raise exc.BadRequest(_('Failed to make the request to Asana API')) return response.Ok({"token": asana_token}) diff --git a/taiga/importers/asana/tasks.py b/taiga/importers/asana/tasks.py index 5d36bed9..4dcef793 100644 --- a/taiga/importers/asana/tasks.py +++ b/taiga/importers/asana/tasks.py @@ -39,14 +39,14 @@ def import_project(self, user_id, token, project_id, options): # Error ctx = { "user": user, - "error_subject": _("Error importing asana project"), - "error_message": _("Error importing asana project"), + "error_subject": _("Error importing Asana project"), + "error_message": _("Error importing Asana project"), "project": project_id, "exception": e } email = mail_builder.asana_import_error(admin, ctx) email.send() - logger.error('Error importing asana project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + logger.error('Error importing Asana project %s (by %s)', project_id, user, exc_info=sys.exc_info()) else: ctx = { "project": project, diff --git a/taiga/importers/github/tasks.py b/taiga/importers/github/tasks.py index f1069e9e..c3cf16f8 100644 --- a/taiga/importers/github/tasks.py +++ b/taiga/importers/github/tasks.py @@ -39,14 +39,14 @@ def import_project(self, user_id, token, project_id, options): # Error ctx = { "user": user, - "error_subject": _("Error importing github project"), - "error_message": _("Error importing github project"), + "error_subject": _("Error importing GitHub project"), + "error_message": _("Error importing GitHub project"), "project": project_id, "exception": e } email = mail_builder.github_import_error(admin, ctx) email.send() - logger.error('Error importing github project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + logger.error('Error importing GitHub project %s (by %s)', project_id, user, exc_info=sys.exc_info()) else: ctx = { "project": project, diff --git a/taiga/importers/jira/tasks.py b/taiga/importers/jira/tasks.py index a1551f1b..874aca49 100644 --- a/taiga/importers/jira/tasks.py +++ b/taiga/importers/jira/tasks.py @@ -44,14 +44,14 @@ def import_project(self, user_id, url, token, project_id, options, importer_type # Error ctx = { "user": user, - "error_subject": _("Error importing jira project"), - "error_message": _("Error importing jira project"), + "error_subject": _("Error importing Jira project"), + "error_message": _("Error importing Jira project"), "project": project_id, "exception": e } email = mail_builder.jira_import_error(admin, ctx) email.send() - logger.error('Error importing jira project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + logger.error('Error importing Jira project %s (by %s)', project_id, user, exc_info=sys.exc_info()) else: ctx = { "project": project, diff --git a/taiga/importers/pivotal/tasks.py b/taiga/importers/pivotal/tasks.py index 2e8f54ea..b9954201 100644 --- a/taiga/importers/pivotal/tasks.py +++ b/taiga/importers/pivotal/tasks.py @@ -39,14 +39,14 @@ def import_project(self, user_id, token, project_id, options): # Error ctx = { "user": user, - "error_subject": _("Error importing pivotal project"), - "error_message": _("Error importing pivotal project"), + "error_subject": _("Error importing PivotalTracker project"), + "error_message": _("Error importing PivotalTracker project"), "project": project_id, "exception": e } email = mail_builder.pivotal_import_error(admin, ctx) email.send() - logger.error('Error importing pivotal project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + logger.error('Error importing PivotalTracker project %s (by %s)', project_id, user, exc_info=sys.exc_info()) else: ctx = { "project": project, diff --git a/taiga/importers/trello/tasks.py b/taiga/importers/trello/tasks.py index 76f7f197..1cb2fe32 100644 --- a/taiga/importers/trello/tasks.py +++ b/taiga/importers/trello/tasks.py @@ -39,14 +39,14 @@ def import_project(self, user_id, token, project_id, options): # Error ctx = { "user": user, - "error_subject": _("Error importing trello project"), - "error_message": _("Error importing trello project"), + "error_subject": _("Error importing Trello project"), + "error_message": _("Error importing Trello project"), "project": project_id, "exception": e } email = mail_builder.trello_import_error(admin, ctx) email.send() - logger.error('Error importing trello project %s (by %s)', project_id, user, exc_info=sys.exc_info()) + logger.error('Error importing Trello project %s (by %s)', project_id, user, exc_info=sys.exc_info()) else: ctx = { "project": project, diff --git a/taiga/locale/en/LC_MESSAGES/django.po b/taiga/locale/en/LC_MESSAGES/django.po index 757246c6..53faaa9f 100644 --- a/taiga/locale/en/LC_MESSAGES/django.po +++ b/taiga/locale/en/LC_MESSAGES/django.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: taiga-back\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-02-09 09:12+0100\n" +"POT-Creation-Date: 2017-02-15 12:07+0100\n" "PO-Revision-Date: 2015-03-25 20:09+0100\n" "Last-Translator: Taiga Dev Team \n" "Language-Team: Taiga Dev Team \n" @@ -184,8 +184,8 @@ msgstr "" #: taiga/projects/api.py:754 taiga/projects/epics/api.py:200 #: taiga/projects/epics/api.py:284 taiga/projects/issues/api.py:224 #: taiga/projects/mixins/ordering.py:59 taiga/projects/tasks/api.py:247 -#: taiga/projects/tasks/api.py:272 taiga/projects/userstories/api.py:322 -#: taiga/projects/userstories/api.py:374 taiga/webhooks/api.py:71 +#: taiga/projects/tasks/api.py:272 taiga/projects/userstories/api.py:323 +#: taiga/projects/userstories/api.py:375 taiga/webhooks/api.py:71 msgid "Blocked element" msgstr "" @@ -764,7 +764,7 @@ msgstr "" #: taiga/external_apps/models.py:39 taiga/projects/attachments/models.py:62 #: taiga/projects/custom_attributes/models.py:37 -#: taiga/projects/epics/models.py:55 +#: taiga/projects/epics/models.py:56 #: taiga/projects/history/templatetags/functions.py:25 #: taiga/projects/issues/models.py:60 taiga/projects/models.py:152 #: taiga/projects/models.py:739 taiga/projects/tasks/models.py:62 @@ -801,7 +801,7 @@ msgstr "" #: taiga/feedback/models.py:31 taiga/projects/attachments/models.py:48 #: taiga/projects/contact/models.py:34 #: taiga/projects/custom_attributes/models.py:46 -#: taiga/projects/epics/models.py:48 taiga/projects/issues/models.py:52 +#: taiga/projects/epics/models.py:49 taiga/projects/issues/models.py:52 #: taiga/projects/likes/models.py:33 taiga/projects/milestones/models.py:48 #: taiga/projects/models.py:159 taiga/projects/models.py:743 #: taiga/projects/notifications/models.py:89 taiga/projects/tasks/models.py:48 @@ -861,7 +861,7 @@ msgstr "" #: taiga/hooks/api.py:63 taiga/projects/epics/api.py:154 #: taiga/projects/issues/api.py:139 taiga/projects/tasks/api.py:201 -#: taiga/projects/userstories/api.py:276 +#: taiga/projects/userstories/api.py:277 msgid "The project doesn't exist" msgstr "" @@ -949,6 +949,85 @@ msgstr "" msgid "The status doesn't exist" msgstr "" +#: taiga/importers/asana/api.py:44 taiga/importers/asana/api.py:86 +#: taiga/importers/github/api.py:45 taiga/importers/github/api.py:75 +#: taiga/importers/jira/api.py:56 taiga/importers/jira/api.py:102 +#: taiga/importers/pivotal/api.py:44 taiga/importers/pivotal/api.py:81 +#: taiga/importers/trello/api.py:46 taiga/importers/trello/api.py:83 +msgid "The project param is needed" +msgstr "" + +#: taiga/importers/asana/api.py:51 taiga/importers/asana/api.py:74 +#: taiga/importers/asana/api.py:131 +msgid "Invalid Asana API request" +msgstr "" + +#: taiga/importers/asana/api.py:53 taiga/importers/asana/api.py:76 +#: taiga/importers/asana/api.py:133 +msgid "Failed to make the request to Asana API" +msgstr "" + +#: taiga/importers/asana/api.py:126 taiga/importers/github/api.py:121 +msgid "Code param needed" +msgstr "" + +#: taiga/importers/asana/tasks.py:42 taiga/importers/asana/tasks.py:43 +msgid "Error importing Asana project" +msgstr "" + +#: taiga/importers/github/api.py:129 +msgid "Invalid auth data" +msgstr "" + +#: taiga/importers/github/api.py:131 +msgid "Third party service failing" +msgstr "" + +#: taiga/importers/github/tasks.py:42 taiga/importers/github/tasks.py:43 +msgid "Error importing GitHub project" +msgstr "" + +#: taiga/importers/jira/api.py:58 taiga/importers/jira/api.py:85 +#: taiga/importers/jira/api.py:105 taiga/importers/jira/api.py:178 +msgid "The url param is needed" +msgstr "" + +#: taiga/importers/jira/api.py:154 +msgid "Invalid project_type {}" +msgstr "" + +#: taiga/importers/jira/api.py:224 taiga/importers/pivotal/api.py:139 +#: taiga/importers/trello/api.py:143 +msgid "Invalid or expired auth token" +msgstr "" + +#: taiga/importers/jira/tasks.py:47 taiga/importers/jira/tasks.py:48 +msgid "Error importing Jira project" +msgstr "" + +#: taiga/importers/pivotal/tasks.py:42 taiga/importers/pivotal/tasks.py:43 +msgid "Error importing PivotalTracker project" +msgstr "" + +#: taiga/importers/trello/importer.py:77 +#, python-format +msgid "Invalid Request: %s at %s" +msgstr "" + +#: taiga/importers/trello/importer.py:79 taiga/importers/trello/importer.py:81 +#, python-format +msgid "Unauthorized: %s at %s" +msgstr "" + +#: taiga/importers/trello/importer.py:83 taiga/importers/trello/importer.py:85 +#, python-format +msgid "Resource Unavailable: %s at %s" +msgstr "" + +#: taiga/importers/trello/tasks.py:42 taiga/importers/trello/tasks.py:43 +msgid "Error importing Trello project" +msgstr "" + #: taiga/permissions/choices.py:23 taiga/permissions/choices.py:34 msgid "View project" msgstr "" @@ -1134,7 +1213,7 @@ msgid "Fans" msgstr "" #: taiga/projects/admin.py:144 taiga/projects/attachments/models.py:39 -#: taiga/projects/epics/models.py:39 taiga/projects/issues/models.py:37 +#: taiga/projects/epics/models.py:40 taiga/projects/issues/models.py:37 #: taiga/projects/milestones/models.py:42 taiga/projects/models.py:164 #: taiga/projects/notifications/models.py:62 taiga/projects/tasks/models.py:39 #: taiga/projects/userstories/models.py:69 taiga/projects/wiki/models.py:40 @@ -1217,7 +1296,7 @@ msgstr "" #: taiga/projects/attachments/models.py:41 taiga/projects/contact/models.py:29 #: taiga/projects/custom_attributes/models.py:43 -#: taiga/projects/epics/models.py:37 taiga/projects/issues/models.py:50 +#: taiga/projects/epics/models.py:38 taiga/projects/issues/models.py:50 #: taiga/projects/milestones/models.py:44 taiga/projects/models.py:506 #: taiga/projects/models.py:528 taiga/projects/models.py:565 #: taiga/projects/models.py:593 taiga/projects/models.py:619 @@ -1240,7 +1319,7 @@ msgstr "" #: taiga/projects/attachments/models.py:51 #: taiga/projects/custom_attributes/models.py:48 -#: taiga/projects/epics/models.py:51 taiga/projects/issues/models.py:55 +#: taiga/projects/epics/models.py:52 taiga/projects/issues/models.py:55 #: taiga/projects/milestones/models.py:51 taiga/projects/models.py:162 #: taiga/projects/models.py:746 taiga/projects/tasks/models.py:51 #: taiga/projects/userstories/models.py:90 taiga/projects/wiki/models.py:47 @@ -1266,7 +1345,7 @@ msgstr "" #: taiga/projects/attachments/models.py:63 #: taiga/projects/custom_attributes/models.py:41 -#: taiga/projects/epics/models.py:101 taiga/projects/milestones/models.py:57 +#: taiga/projects/epics/models.py:104 taiga/projects/milestones/models.py:57 #: taiga/projects/models.py:522 taiga/projects/models.py:555 #: taiga/projects/models.py:589 taiga/projects/models.py:613 #: taiga/projects/models.py:645 taiga/projects/models.py:665 @@ -1409,26 +1488,26 @@ msgstr "" msgid "You don't have permissions to set this status to this epic." msgstr "" -#: taiga/projects/epics/models.py:35 taiga/projects/issues/models.py:35 +#: taiga/projects/epics/models.py:36 taiga/projects/issues/models.py:35 #: taiga/projects/tasks/models.py:37 taiga/projects/userstories/models.py:62 msgid "ref" msgstr "" -#: taiga/projects/epics/models.py:42 taiga/projects/issues/models.py:39 +#: taiga/projects/epics/models.py:43 taiga/projects/issues/models.py:39 #: taiga/projects/tasks/models.py:41 taiga/projects/userstories/models.py:72 msgid "status" msgstr "" -#: taiga/projects/epics/models.py:45 +#: taiga/projects/epics/models.py:46 msgid "epics order" msgstr "" -#: taiga/projects/epics/models.py:54 taiga/projects/issues/models.py:59 +#: taiga/projects/epics/models.py:55 taiga/projects/issues/models.py:59 #: taiga/projects/tasks/models.py:55 taiga/projects/userstories/models.py:94 msgid "subject" msgstr "" -#: taiga/projects/epics/models.py:58 taiga/projects/models.py:526 +#: taiga/projects/epics/models.py:59 taiga/projects/models.py:526 #: taiga/projects/models.py:561 taiga/projects/models.py:617 #: taiga/projects/models.py:647 taiga/projects/models.py:667 #: taiga/projects/models.py:691 taiga/projects/models.py:719 @@ -1436,23 +1515,28 @@ msgstr "" msgid "color" msgstr "" -#: taiga/projects/epics/models.py:61 taiga/projects/issues/models.py:63 +#: taiga/projects/epics/models.py:62 taiga/projects/issues/models.py:63 #: taiga/projects/tasks/models.py:65 taiga/projects/userstories/models.py:98 msgid "assigned to" msgstr "" -#: taiga/projects/epics/models.py:63 taiga/projects/userstories/models.py:100 +#: taiga/projects/epics/models.py:64 taiga/projects/userstories/models.py:100 msgid "is client requirement" msgstr "" -#: taiga/projects/epics/models.py:65 taiga/projects/userstories/models.py:102 +#: taiga/projects/epics/models.py:66 taiga/projects/userstories/models.py:102 msgid "is team requirement" msgstr "" -#: taiga/projects/epics/models.py:69 +#: taiga/projects/epics/models.py:70 msgid "user stories" msgstr "" +#: taiga/projects/epics/models.py:72 taiga/projects/issues/models.py:66 +#: taiga/projects/tasks/models.py:70 taiga/projects/userstories/models.py:109 +msgid "external reference" +msgstr "" + #: taiga/projects/epics/validators.py:37 msgid "There's no epic with that id" msgstr "" @@ -1635,11 +1719,6 @@ msgstr "" msgid "finished date" msgstr "" -#: taiga/projects/issues/models.py:66 taiga/projects/tasks/models.py:70 -#: taiga/projects/userstories/models.py:109 -msgid "external reference" -msgstr "" - #: taiga/projects/likes/models.py:36 msgid "Like" msgstr "" @@ -3307,25 +3386,25 @@ msgstr "" msgid "Stakeholder" msgstr "" -#: taiga/projects/userstories/api.py:127 +#: taiga/projects/userstories/api.py:128 msgid "You don't have permissions to set this sprint to this user story." msgstr "" -#: taiga/projects/userstories/api.py:131 +#: taiga/projects/userstories/api.py:132 msgid "You don't have permissions to set this status to this user story." msgstr "" -#: taiga/projects/userstories/api.py:221 +#: taiga/projects/userstories/api.py:222 #, python-brace-format msgid "Invalid role id '{role_id}'" msgstr "" -#: taiga/projects/userstories/api.py:228 +#: taiga/projects/userstories/api.py:229 #, python-brace-format msgid "Invalid points id '{points_id}'" msgstr "" -#: taiga/projects/userstories/api.py:243 +#: taiga/projects/userstories/api.py:244 #, python-brace-format msgid "Generating the user story #{ref} - {subject}" msgstr "" diff --git a/tests/integration/test_importers_asana_api.py b/tests/integration/test_importers_asana_api.py index bc158694..aab4d143 100644 --- a/tests/integration/test_importers_asana_api.py +++ b/tests/integration/test_importers_asana_api.py @@ -95,7 +95,7 @@ def test_authorize_with_bad_verify(client, settings): assert response.status_code == 400 assert 'token' not in response.data assert '_error_message' in response.data - assert response.data['_error_message'] == "Invalid asana api request" + assert response.data['_error_message'] == "Invalid Asana API request" def test_import_asana_list_users(client): From ea7187b30d2f05d2196ba6c042c3d045df3bb460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Tue, 14 Feb 2017 15:57:37 +0100 Subject: [PATCH 08/12] Capture any error processing authorization on github and convert it to a Http400 --- taiga/importers/github/importer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/taiga/importers/github/importer.py b/taiga/importers/github/importer.py index b042feef..daee40c8 100644 --- a/taiga/importers/github/importer.py +++ b/taiga/importers/github/importer.py @@ -19,6 +19,8 @@ from taiga.timeline.rebuilder import rebuild_timeline from taiga.timeline.models import Timeline from taiga.users.models import User, AuthData +from taiga.importers.exceptions import InvalidAuthResult + class GithubClient: def __init__(self, token): @@ -530,7 +532,10 @@ class GithubImporter: if result.status_code > 299: raise InvalidAuthResult() else: - return dict(parse_qsl(result.content))[b'access_token'].decode('utf-8') + try: + return dict(parse_qsl(result.content))[b'access_token'].decode('utf-8') + except: + raise InvalidAuthResult() class AssignedEventHandler: From a952d200fc3711900abc7fa4ad56820405c61837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Tue, 14 Feb 2017 17:31:45 +0100 Subject: [PATCH 09/12] Improved users_bindings --- taiga/importers/asana/api.py | 3 ++- taiga/importers/github/api.py | 3 ++- taiga/importers/jira/api.py | 3 ++- taiga/importers/services.py | 16 ++++++++++++++++ taiga/importers/trello/api.py | 3 ++- 5 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 taiga/importers/services.py diff --git a/taiga/importers/asana/api.py b/taiga/importers/asana/api.py index 611eda8b..d92dda55 100644 --- a/taiga/importers/asana/api.py +++ b/taiga/importers/asana/api.py @@ -26,6 +26,7 @@ from taiga.users.gravatar import get_user_gravatar_id from taiga.projects.serializers import ProjectSerializer from taiga.importers import permissions, exceptions +from taiga.importers.services import resolve_users_bindings from .importer import AsanaImporter from . import tasks @@ -89,7 +90,7 @@ class AsanaImporterViewSet(viewsets.ViewSet): "name": request.DATA.get('name', None), "description": request.DATA.get('description', None), "template": request.DATA.get('template', "scrum"), - "users_bindings": request.DATA.get("users_bindings", {}), + "users_bindings": resolve_users_bindings(request.DATA.get("users_bindings", {})), "keep_external_reference": request.DATA.get("keep_external_reference", False), "is_private": request.DATA.get("is_private", False), } diff --git a/taiga/importers/github/api.py b/taiga/importers/github/api.py index e8991c0d..675d7a90 100644 --- a/taiga/importers/github/api.py +++ b/taiga/importers/github/api.py @@ -27,6 +27,7 @@ from taiga.projects.serializers import ProjectSerializer from taiga.importers import permissions from taiga.importers import exceptions +from taiga.importers.services import resolve_users_bindings from .importer import GithubImporter from . import tasks @@ -85,7 +86,7 @@ class GithubImporterViewSet(viewsets.ViewSet): "description": request.DATA.get('description', None), "template": template, "type": items_type, - "users_bindings": request.DATA.get("users_bindings", {}), + "users_bindings": resolve_users_bindings(request.DATA.get("users_bindings", {})), "keep_external_reference": request.DATA.get("keep_external_reference", False), "is_private": request.DATA.get("is_private", False), } diff --git a/taiga/importers/jira/api.py b/taiga/importers/jira/api.py index 47bb6cda..5e0d1de8 100644 --- a/taiga/importers/jira/api.py +++ b/taiga/importers/jira/api.py @@ -25,6 +25,7 @@ from taiga.users.services import get_user_photo_url from taiga.users.gravatar import get_user_gravatar_id from taiga.importers import permissions +from taiga.importers.services import resolve_users_bindings from .normal import JiraNormalImporter from .agile import JiraAgileImporter from . import tasks @@ -107,7 +108,7 @@ class JiraImporterViewSet(viewsets.ViewSet): options = { "name": request.DATA.get('name', None), "description": request.DATA.get('description', None), - "users_bindings": request.DATA.get("user_bindings", {}), + "users_bindings": resolve_users_bindings(request.DATA.get("users_bindings", {})), "keep_external_reference": request.DATA.get("keep_external_reference", False), "is_private": request.DATA.get("is_private", False), } diff --git a/taiga/importers/services.py b/taiga/importers/services.py new file mode 100644 index 00000000..4817c049 --- /dev/null +++ b/taiga/importers/services.py @@ -0,0 +1,16 @@ +from taiga.users.models import User + + +def resolve_users_bindings(users_bindings): + new_users_bindings = {} + for key,value in users_bindings.items(): + if isinstance(value, str): + try: + new_users_bindings[int(key)] = User.objects.get(email_iexact=value) + except User.MultipleObjectsReturned: + new_users_bindings[int(key)] = User.objects.get(email=value) + except User.DoesNotExists: + new_users_bindings[int(key)] = None + else: + new_users_bindings[int(key)] = User.objects.get(id=value) + return new_users_bindings diff --git a/taiga/importers/trello/api.py b/taiga/importers/trello/api.py index 33b9fe40..f616d9b8 100644 --- a/taiga/importers/trello/api.py +++ b/taiga/importers/trello/api.py @@ -29,6 +29,7 @@ from taiga.projects.serializers import ProjectSerializer from .importer import TrelloImporter from taiga.importers import permissions +from taiga.importers.services import resolve_users_bindings from . import tasks @@ -86,7 +87,7 @@ class TrelloImporterViewSet(viewsets.ViewSet): "name": request.DATA.get('name', None), "description": request.DATA.get('description', None), "template": request.DATA.get('template', "kanban"), - "users_bindings": request.DATA.get("users_bindings", {}), + "users_bindings": resolve_users_bindings(request.DATA.get("users_bindings", {})), "keep_external_reference": request.DATA.get("keep_external_reference", False), "is_private": request.DATA.get("is_private", False), } From aaedfab43927a67fa108ed812810226999fe0045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Tue, 14 Feb 2017 22:48:57 +0100 Subject: [PATCH 10/12] Change settings format for importers --- settings/common.py | 37 ++++++++++++------- settings/testing.py | 6 +++ taiga/importers/asana/api.py | 13 ++++++- taiga/importers/github/api.py | 11 +++++- taiga/importers/jira/api.py | 12 +++--- .../management/commands/import_from_asana.py | 13 ++++++- .../management/commands/import_from_github.py | 10 ++++- .../management/commands/import_from_jira.py | 16 +++++++- taiga/importers/trello/importer.py | 12 +++--- taiga/routers.py | 23 ++++++++---- tests/integration/test_importers_asana_api.py | 24 +++++++++--- .../integration/test_importers_github_api.py | 12 +++++- 12 files changed, 138 insertions(+), 51 deletions(-) diff --git a/settings/common.py b/settings/common.py index a0156181..5bc1605c 100644 --- a/settings/common.py +++ b/settings/common.py @@ -561,19 +561,30 @@ MAX_PENDING_MEMBERSHIPS = 30 # Max number of unconfirmed memberships in a projec from .sr import * -GITHUB_API_CLIENT_ID = "" -GITHUB_API_CLIENT_SECRET = "" - -TRELLO_API_KEY = "" -TRELLO_SECRET_KEY = "" - -ASANA_APP_CALLBACK_URL = "" -ASANA_APP_ID = "" -ASANA_APP_SECRET = "" - -JIRA_CONSUMER_KEY = "" -JIRA_CERT = "" -JIRA_PUB_CERT = "" +IMPORTERS = { + "github": { + "active": False, + "client_id": "", + "client_secret": "", + }, + "trello": { + "active": False, + "api_key": "", + "secret_key": "", + }, + "jira": { + "active": False, + "consumer_key": "", + "cert": "", + "pub_cert": "", + }, + "asana": { + "active": False, + "callback_url": "", + "app_id": "", + "app_secret": "", + } +} # NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE TEST_RUNNER="django.test.runner.DiscoverRunner" diff --git a/settings/testing.py b/settings/testing.py index 12532b9b..ffabbe12 100644 --- a/settings/testing.py +++ b/settings/testing.py @@ -38,3 +38,9 @@ REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] = { "register-success": None, "user-detail": None, } + + +IMPORTERS['github']['active'] = True +IMPORTERS['jira']['active'] = True +IMPORTERS['asana']['active'] = True +IMPORTERS['trello']['active'] = True diff --git a/taiga/importers/asana/api.py b/taiga/importers/asana/api.py index d92dda55..b13914bf 100644 --- a/taiga/importers/asana/api.py +++ b/taiga/importers/asana/api.py @@ -114,7 +114,11 @@ class AsanaImporterViewSet(viewsets.ViewSet): def auth_url(self, request, *args, **kwargs): self.check_permissions(request, "auth_url", None) - url = AsanaImporter.get_auth_url(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + url = AsanaImporter.get_auth_url( + settings.IMPORTERS.get('asana', {}).get('app_id', None), + settings.IMPORTERS.get('asana', {}).get('app_secret', None), + settings.IMPORTERS.get('asana', {}).get('callback_url', None) + ) return response.Ok({"url": url}) @@ -127,7 +131,12 @@ class AsanaImporterViewSet(viewsets.ViewSet): raise exc.BadRequest(_("Code param needed")) try: - asana_token = AsanaImporter.get_access_token(code, settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + asana_token = AsanaImporter.get_access_token( + code, + settings.IMPORTERS.get('asana', {}).get('app_id', None), + settings.IMPORTERS.get('asana', {}).get('app_secret', None), + settings.IMPORTERS.get('asana', {}).get('callback_url', None) + ) except exceptions.InvalidRequest: raise exc.BadRequest(_('Invalid Asana API request')) except exceptions.FailedRequest: diff --git a/taiga/importers/github/api.py b/taiga/importers/github/api.py index 675d7a90..0c7af495 100644 --- a/taiga/importers/github/api.py +++ b/taiga/importers/github/api.py @@ -110,7 +110,10 @@ class GithubImporterViewSet(viewsets.ViewSet): def auth_url(self, request, *args, **kwargs): self.check_permissions(request, "auth_url", None) callback_uri = request.QUERY_PARAMS.get('uri') - url = GithubImporter.get_auth_url(settings.GITHUB_API_CLIENT_ID, callback_uri) + url = GithubImporter.get_auth_url( + settings.IMPORTERS.get('github', {}).get('client_id', None), + callback_uri + ) return response.Ok({"url": url}) @list_route(methods=["POST"]) @@ -122,7 +125,11 @@ class GithubImporterViewSet(viewsets.ViewSet): raise exc.BadRequest(_("Code param needed")) try: - token = GithubImporter.get_access_token(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, code) + token = GithubImporter.get_access_token( + settings.IMPORTERS.get('github', {}).get('client_id', None), + settings.IMPORTERS.get('github', {}).get('client_secret', None), + code + ) return response.Ok({ "token": token }) diff --git a/taiga/importers/jira/api.py b/taiga/importers/jira/api.py index 5e0d1de8..c704a2fd 100644 --- a/taiga/importers/jira/api.py +++ b/taiga/importers/jira/api.py @@ -40,8 +40,8 @@ class JiraImporterViewSet(viewsets.ViewSet): token = { "access_token": token_data[0], "access_token_secret": token_data[1], - "key_cert": settings.JIRA_CERT, - "consumer_key": settings.JIRA_CONSUMER_KEY + "key_cert": settings.IMPORTERS.get('jira', {}).get('cert', None), + "consumer_key": settings.IMPORTERS.get('jira', {}).get('consumer_key', None) } return token @@ -180,8 +180,8 @@ class JiraImporterViewSet(viewsets.ViewSet): (oauth_token, oauth_secret, url) = JiraNormalImporter.get_auth_url( jira_url, - settings.JIRA_CONSUMER_KEY, - settings.JIRA_CERT, + settings.IMPORTERS.get('jira', {}).get('consumer_key', None), + settings.IMPORTERS.get('jira', {}).get('cert', None), True ) @@ -215,8 +215,8 @@ class JiraImporterViewSet(viewsets.ViewSet): jira_token = JiraNormalImporter.get_access_token( server_url, - settings.JIRA_CONSUMER_KEY, - settings.JIRA_CERT, + settings.IMPORTERS.get('jira', {}).get('consumer_key', None), + settings.IMPORTERS.get('jira', {}).get('cert', None), oauth_token, oauth_secret, True diff --git a/taiga/importers/management/commands/import_from_asana.py b/taiga/importers/management/commands/import_from_asana.py index d7d7dd21..9d954422 100644 --- a/taiga/importers/management/commands/import_from_asana.py +++ b/taiga/importers/management/commands/import_from_asana.py @@ -52,10 +52,19 @@ class Command(BaseCommand): if options.get('token', None): token = json.loads(options.get('token')) else: - url = AsanaImporter.get_auth_url(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + url = AsanaImporter.get_auth_url( + settings.IMPORTERS.get('asana', {}).get('app_id', None), + settings.IMPORTERS.get('asana', {}).get('app_secret', None), + settings.IMPORTERS.get('asana', {}).get('callback_url', None) + ) print("Go to here and come with your code (in the redirected url): {}".format(url)) code = input("Code: ") - access_data = AsanaImporter.get_access_token(code, settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + access_data = AsanaImporter.get_access_token( + code, + settings.IMPORTERS.get('asana', {}).get('app_id', None), + settings.IMPORTERS.get('asana', {}).get('app_secret', None), + settings.IMPORTERS.get('asana', {}).get('callback_url', None) + ) token = access_data importer = AsanaImporter(admin, token) diff --git a/taiga/importers/management/commands/import_from_github.py b/taiga/importers/management/commands/import_from_github.py index 9c93a527..e2ad7b96 100644 --- a/taiga/importers/management/commands/import_from_github.py +++ b/taiga/importers/management/commands/import_from_github.py @@ -51,10 +51,16 @@ class Command(BaseCommand): if options.get('token', None): token = options.get('token') else: - url = GithubImporter.get_auth_url(settings.GITHUB_API_CLIENT_ID) + url = GithubImporter.get_auth_url( + settings.IMPORTERS.get('github', {}).get('client_id', None) + ) print("Go to here and come with your code (in the redirected url): {}".format(url)) code = input("Code: ") - access_data = GithubImporter.get_access_token(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, code) + access_data = GithubImporter.get_access_token( + settings.IMPORTERS.get('github', {}).get('client_id', None) + settings.IMPORTERS.get('github', {}).get('client_secret', None) + code + ) token = access_data importer = GithubImporter(admin, token) diff --git a/taiga/importers/management/commands/import_from_jira.py b/taiga/importers/management/commands/import_from_jira.py index 972e6ba6..30c723d6 100644 --- a/taiga/importers/management/commands/import_from_jira.py +++ b/taiga/importers/management/commands/import_from_jira.py @@ -62,10 +62,22 @@ class Command(BaseCommand): elif options.get('token', None): token = json.loads(options.get('token')) else: - (rtoken, rtoken_secret, url) = JiraNormalImporter.get_auth_url(server, settings.JIRA_CONSUMER_KEY, settings.JIRA_CERT, True) + (rtoken, rtoken_secret, url) = JiraNormalImporter.get_auth_url( + server, + settings.IMPORTERS.get('jira', {}).get('consumer_key', None), + settings.IMPORTERS.get('jira', {}).get('cert', None), + True + ) print(url) code = input("Go to the url and get back the code") - token = JiraNormalImporter.get_access_token(server, settings.JIRA_CONSUMER_KEY, settings.JIRA_CERT, rtoken, rtoken_secret, True) + token = JiraNormalImporter.get_access_token( + server, + settings.IMPORTERS.get('jira', {}).get('consumer_key', None), + settings.IMPORTERS.get('jira', {}).get('cert', None), + rtoken, + rtoken_secret, + True + ) print("Auth token: {}".format(json.dumps(token))) diff --git a/taiga/importers/trello/importer.py b/taiga/importers/trello/importer.py index e41063b8..c36e3086 100644 --- a/taiga/importers/trello/importer.py +++ b/taiga/importers/trello/importer.py @@ -92,8 +92,8 @@ class TrelloImporter: self._user = user self._cached_orgs = {} self._client = TrelloClient( - api_key=settings.TRELLO_API_KEY, - api_secret=settings.TRELLO_SECRET_KEY, + api_key=settings.IMPORTERS.get('trello', {}).get('api_key', None), + api_secret=settings.IMPORTERS.get('trello', {}).get('secret_key', None), token=token, ) @@ -492,8 +492,8 @@ class TrelloImporter: return_url = resolve_front_url("new-project-import", "trello") expiration = "1day" scope = "read,write,account" - trello_key = settings.TRELLO_API_KEY - trello_secret = settings.TRELLO_SECRET_KEY + trello_key = settings.IMPORTERS.get('trello', {}).get('api_key', None) + trello_secret = settings.IMPORTERS.get('trello', {}).get('secret_key', None) name = "Taiga" session = OAuth1Session(client_key=trello_key, client_secret=trello_secret) @@ -515,8 +515,8 @@ class TrelloImporter: @classmethod def get_access_token(cls, oauth_token, oauth_token_secret, oauth_verifier): - api_key = settings.TRELLO_API_KEY - api_secret = settings.TRELLO_SECRET_KEY + api_key = settings.IMPORTERS.get('trello', {}).get('api_key', None) + api_secret = settings.IMPORTERS.get('trello', {}).get('secret_key', None) access_token_url = 'https://trello.com/1/OAuthGetAccessToken' session = OAuth1Session(client_key=api_key, client_secret=api_secret, resource_owner_key=oauth_token, resource_owner_secret=oauth_token_secret, diff --git a/taiga/routers.py b/taiga/routers.py index 382ede82..013e7453 100644 --- a/taiga/routers.py +++ b/taiga/routers.py @@ -17,6 +17,7 @@ # along with this program. If not, see . from taiga.base import routers +from django.conf import settings router = routers.DefaultRouter(trailing_slash=False) @@ -284,15 +285,21 @@ router.register(r"applications", Application, base_name="applications") router.register(r"application-tokens", ApplicationToken, base_name="application-tokens") # Third party importers -from taiga.importers.trello.api import TrelloImporterViewSet -from taiga.importers.jira.api import JiraImporterViewSet -from taiga.importers.github.api import GithubImporterViewSet -from taiga.importers.asana.api import AsanaImporterViewSet +if settings.IMPORTERS.get('trello', {}).get('active', False): + from taiga.importers.trello.api import TrelloImporterViewSet + router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") -router.register(r"importers/trello", TrelloImporterViewSet, base_name="importers-trello") -router.register(r"importers/jira", JiraImporterViewSet, base_name="importers-jira") -router.register(r"importers/github", GithubImporterViewSet, base_name="importers-github") -router.register(r"importers/asana", AsanaImporterViewSet, base_name="importers-asana") +if settings.IMPORTERS.get('jira', {}).get('active', False): + from taiga.importers.jira.api import JiraImporterViewSet + router.register(r"importers/jira", JiraImporterViewSet, base_name="importers-jira") + +if settings.IMPORTERS.get('github', {}).get('active', False): + from taiga.importers.github.api import GithubImporterViewSet + router.register(r"importers/github", GithubImporterViewSet, base_name="importers-github") + +if settings.IMPORTERS.get('asana', {}).get('active', False): + from taiga.importers.asana.api import AsanaImporterViewSet + router.register(r"importers/asana", AsanaImporterViewSet, base_name="importers-asana") # Stats diff --git a/tests/integration/test_importers_asana_api.py b/tests/integration/test_importers_asana_api.py index aab4d143..d44a7232 100644 --- a/tests/integration/test_importers_asana_api.py +++ b/tests/integration/test_importers_asana_api.py @@ -35,16 +35,20 @@ pytestmark = pytest.mark.django_db def test_auth_url(client, settings): user = f.UserFactory.create() client.login(user) - settings.ASANA_APP_CALLBACK_URL = "http://testserver/url" - settings.ASANA_APP_ID = "test-id" - settings.ASANA_APP_SECRET = "test-secret" + settings.IMPORTERS['asana']['callback_url'] = "http://testserver/url" + settings.IMPORTERS['asana']['app_id'] = "test-id" + settings.IMPORTERS['asana']['app_secret'] = "test-secret" url = reverse("importers-asana-auth-url") with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: AsanaImporterMock.get_auth_url.return_value = "https://auth_url" response = client.get(url, content_type="application/json") - assert AsanaImporterMock.get_auth_url.calledWith(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, settings.ASANA_APP_CALLBACK_URL) + assert AsanaImporterMock.get_auth_url.calledWith( + settings.IMPORTERS['asana']['app_id'], + settings.IMPORTERS['asana']['app_secret'], + settings.IMPORTERS['asana']['callback_url'] + ) assert response.status_code == 200 assert 'url' in response.data @@ -60,7 +64,11 @@ def test_authorize(client, settings): with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: AsanaImporterMock.get_access_token.return_value = "token" response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "code"})) - assert AsanaImporterMock.get_access_token.calledWith(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, "code") + assert AsanaImporterMock.get_access_token.calledWith( + settings.IMPORTERS['asana']['app_id'], + settings.IMPORTERS['asana']['app_secret'], + "code" + ) assert response.status_code == 200 assert 'token' in response.data @@ -90,7 +98,11 @@ def test_authorize_with_bad_verify(client, settings): with mock.patch('taiga.importers.asana.api.AsanaImporter') as AsanaImporterMock: AsanaImporterMock.get_access_token.side_effect = exceptions.InvalidRequest() response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "bad"})) - assert AsanaImporterMock.get_access_token.calledWith(settings.ASANA_APP_ID, settings.ASANA_APP_SECRET, "bad") + assert AsanaImporterMock.get_access_token.calledWith( + settings.IMPORTERS['asana']['app_id'], + settings.IMPORTERS['asana']['app_secret'], + "bad" + ) assert response.status_code == 400 assert 'token' not in response.data diff --git a/tests/integration/test_importers_github_api.py b/tests/integration/test_importers_github_api.py index bda1c928..4d30b8ff 100644 --- a/tests/integration/test_importers_github_api.py +++ b/tests/integration/test_importers_github_api.py @@ -53,7 +53,11 @@ def test_authorize(client, settings): with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: GithubImporterMock.get_access_token.return_value = "token" response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "code"})) - assert GithubImporterMock.get_access_token.calledWith(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, "code") + assert GithubImporterMock.get_access_token.calledWith( + settings.IMPORTERS['github']['client_id'], + settings.IMPORTERS['github']['client_secret'], + "code" + ) assert response.status_code == 200 assert 'token' in response.data @@ -82,7 +86,11 @@ def test_authorize_with_bad_verify(client, settings): with mock.patch('taiga.importers.github.api.GithubImporter') as GithubImporterMock: GithubImporterMock.get_access_token.side_effect = exceptions.InvalidAuthResult() response = client.post(authorize_url, content_type="application/json", data=json.dumps({"code": "bad"})) - assert GithubImporterMock.get_access_token.calledWith(settings.GITHUB_API_CLIENT_ID, settings.GITHUB_API_CLIENT_SECRET, "bad") + assert GithubImporterMock.get_access_token.calledWith( + settings.IMPORTERS['github']['client_id'], + settings.IMPORTERS['github']['client_secret'], + "bad" + ) assert response.status_code == 400 assert 'token' not in response.data From 176e2fc2c547a0f7b06003ae997d7fa4552fd821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 15 Feb 2017 09:16:57 +0100 Subject: [PATCH 11/12] Adding avatar to list_user function in importers --- taiga/importers/asana/importer.py | 5 +++-- taiga/importers/github/importer.py | 1 + taiga/importers/jira/common.py | 1 + taiga/importers/trello/importer.py | 9 ++++++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/taiga/importers/asana/importer.py b/taiga/importers/asana/importer.py index 6c3c675a..2899ace9 100644 --- a/taiga/importers/asana/importer.py +++ b/taiga/importers/asana/importer.py @@ -49,11 +49,12 @@ class AsanaImporter: def list_users(self, project_id): users = [] for ws in self._client.workspaces.find_all(): - for user in self._client.users.find_by_workspace(ws['id'], fields=["id", "name", "email"]): + for user in self._client.users.find_by_workspace(ws['id'], fields=["id", "name", "email", "photo"]): users.append({ "id": user["id"], "full_name": user['name'], - "detected_user": self._get_user(user) + "detected_user": self._get_user(user), + "avatar": user.get('photo', None) and user['photo'].get('image_60x60', None) }) return users diff --git a/taiga/importers/github/importer.py b/taiga/importers/github/importer.py index daee40c8..18c5cbce 100644 --- a/taiga/importers/github/importer.py +++ b/taiga/importers/github/importer.py @@ -88,6 +88,7 @@ class GithubImporter: return [{"id": u['id'], "username": u['login'], "full_name": u.get('name', u['login']), + "avatar": u.get('avatar_url', None), "detected_user": self._get_user(u) } for u in collaborators] def _get_user(self, user, default=None): diff --git a/taiga/importers/jira/common.py b/taiga/importers/jira/common.py index c38d1993..1448deeb 100644 --- a/taiga/importers/jira/common.py +++ b/taiga/importers/jira/common.py @@ -176,6 +176,7 @@ class JiraImporterCommon: "id": user_data['key'], "full_name": user_data['displayName'], "email": user_data['emailAddress'], + "avatar": user_data.get('avatarUrls', None) and user_data['avatarUrls'].get('48x48', None), }) return result diff --git a/taiga/importers/trello/importer.py b/taiga/importers/trello/importer.py index c36e3086..227339e4 100644 --- a/taiga/importers/trello/importer.py +++ b/taiga/importers/trello/importer.py @@ -126,11 +126,18 @@ class TrelloImporter: def list_users(self, project_id): members = [] for member in self._client.get("/board/{}/members/all".format(project_id), {"fields": "id"}): - user = self._client.get("/member/{}".format(member['id']), {"fields": "id,fullName,email"}) + user = self._client.get("/member/{}".format(member['id']), {"fields": "id,fullName,email,avatarSource,avatarHash,gravatarHash"}) + print(user) + if user['avatarSource'] == "gravatar": + avatar = 'https://www.gravatar.com/avatar/' + user['gravatarHash'] + '.jpg?s=50' + else: + avatar = 'https://trello-avatars.s3.amazonaws.com/' + user['avatarHash'] + '/50.png' + members.append({ "id": user['id'], "full_name": user['fullName'], "email": user['email'], + "avatar": avatar }) return members From 0438a389b08598b1dd3ae3378411f93dd20d4fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 15 Feb 2017 09:51:01 +0100 Subject: [PATCH 12/12] Add sample settings in local.py.example --- settings/local.py.example | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/settings/local.py.example b/settings/local.py.example index 7defff37..c5f6491e 100644 --- a/settings/local.py.example +++ b/settings/local.py.example @@ -153,3 +153,43 @@ DATABASES = { # To use celery in memory #CELERY_ENABLED = True #CELERY_ALWAYS_EAGER = True + + +######################################### +## IMPORTERS +######################################### + +# Configuration for the GitHub importer +# Remember to enable it in the front client too. +#IMPORTERS["github"] = { +# "active": True, # Enable or disable the importer +# "client_id": "XXXXXX_get_a_valid_client_id_from_github_XXXXXX", +# "client_secret": "XXXXXX_get_a_valid_client_secret_from_github_XXXXXX" +#} + +# Configuration for the Trello importer +# Remember to enable it in the front client too. +#IMPORTERS["trello"] = { +# "active": True, # Enable or disable the importer +# "api_key": "XXXXXX_get_a_valid_api_key_from_trello_XXXXXX", +# "secret_key": "XXXXXX_get_a_valid_secret_key_from_trello_XXXXXX" +#} + +# Configuration for the Jira importer +# Remember to enable it in the front client too. +#IMPORTERS["jira"] = { +# "active": True, # Enable or disable the importer +# "consumer_key": "XXXXXX_get_a_valid_consumer_key_from_jira_XXXXXX", +# "cert": "XXXXXX_get_a_valid_cert_from_jira_XXXXXX", +# "pub_cert": "XXXXXX_get_a_valid_pub_cert_from_jira_XXXXXX" +#} + +# Configuration for the Asane importer +# Remember to enable it in the front client too. +#IMPORTERS["asana"] = { +# "active": True, # Enable or disable the importer +# "callback_url": "{}://{}/project/new/import/asana".format(SITES["front"]["scheme"], +# SITES["front"]["domain"]), +# "app_id": "XXXXXX_get_a_valid_app_id_from_asana_XXXXXX", +# "app_secret": "XXXXXX_get_a_valid_app_secret_from_asana_XXXXXX" +#}