Merge pull request #204 from taigaio/us/1233/expose-import-export

Us/1233/expose import export
remotes/origin/enhancement/email-actions
Alejandro 2015-01-13 14:03:50 +01:00
commit 97fa6bca56
27 changed files with 512 additions and 15 deletions

View File

@ -21,7 +21,7 @@ diff-match-patch==20121119
requests==2.4.1
easy-thumbnails==2.1
celery==3.1.12
celery==3.1.17
redis==2.10.3
Unidecode==0.04.16
raven==5.1.1

View File

@ -201,6 +201,7 @@ INSTALLED_APPS = [
"rest_framework",
"djmail",
"django_jinja",
"django_jinja.contrib._humanize",
"easy_thumbnails",
"raven.contrib.django.raven_compat",
]
@ -300,7 +301,8 @@ REST_FRAMEWORK = {
"DEFAULT_THROTTLE_RATES": {
"anon": None,
"user": None,
"import-mode": None
"import-mode": None,
"import-dump-mode": "1/minute",
},
"FILTER_BACKEND": "taiga.base.filters.FilterBackend",
"EXCEPTION_HANDLER": "taiga.base.exceptions.exception_handler",
@ -362,6 +364,9 @@ PROJECT_MODULES_CONFIGURATORS = {
BITBUCKET_VALID_ORIGIN_IPS = ["131.103.20.165", "131.103.20.166"]
GITLAB_VALID_ORIGIN_IPS = []
EXPORTS_TTL = 60 * 60 * 24 # 24 hours
CELERY_ENABLED = False
# NOTE: DON'T INSERT MORE SETTINGS AFTER THIS LINE
TEST_RUNNER="django.test.runner.DiscoverRunner"

View File

@ -28,5 +28,6 @@ INSTALLED_APPS = INSTALLED_APPS + ["tests"]
REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"] = {
"anon": None,
"user": None,
"import-mode": None
"import-mode": None,
"import-dump-mode": None,
}

View File

@ -14,7 +14,10 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from djmail.template_mail import MagicMailBuilder, InlineCSSTemplateMail
@ -76,6 +79,32 @@ class Command(BaseCommand):
email = mbuilder.change_email(test_email, context)
email.send()
# Export/Import emails
context = {
"user": User.objects.all().order_by("?").first(),
"error_subject": "Error generating project dump",
"error_message": "Error generating project dump",
}
email = mbuilder.export_import_error(test_email, context)
email.send()
deletion_date = timezone.now() + datetime.timedelta(seconds=60*60*24)
context = {
"url": "http://dummyurl.com",
"user": User.objects.all().order_by("?").first(),
"project": Project.objects.all().order_by("?").first(),
"deletion_date": deletion_date,
}
email = mbuilder.dump_project(test_email, context)
email.send()
context = {
"user": User.objects.all().order_by("?").first(),
"project": Project.objects.all().order_by("?").first(),
}
email = mbuilder.load_dump(test_email, context)
email.send()
# Notification emails
notification_emails = [
"issues/issue-change",

View File

@ -14,17 +14,24 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import codecs
from rest_framework.exceptions import APIException
from rest_framework.response import Response
from rest_framework.decorators import throttle_classes
from rest_framework import status
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext_lazy as _
from django.db.transaction import atomic
from django.db.models import signals
from django.conf import settings
from taiga.base.api.mixins import CreateModelMixin
from taiga.base.api.viewsets import GenericViewSet
from taiga.base.decorators import detail_route
from taiga.base.decorators import detail_route, list_route
from taiga.base import exceptions as exc
from taiga.projects.models import Project, Membership
from taiga.projects.issues.models import Issue
@ -32,15 +39,46 @@ from . import mixins
from . import serializers
from . import service
from . import permissions
from . import tasks
from . import dump_service
from . import throttling
from taiga.base.api.utils import get_object_or_404
class Http400(APIException):
status_code = 400
class ProjectExporterViewSet(mixins.ImportThrottlingPolicyMixin, GenericViewSet):
model = Project
permission_classes = (permissions.ImportExportPermission, )
def retrieve(self, request, pk, *args, **kwargs):
throttle = throttling.ImportDumpModeRateThrottle()
if not throttle.allow_request(request, self):
self.throttled(request, throttle.wait())
project = get_object_or_404(self.get_queryset(), pk=pk)
self.check_permissions(request, 'export_project', project)
if settings.CELERY_ENABLED:
task = tasks.dump_project.delay(request.user, project)
tasks.delete_project_dump.apply_async((project.pk,), countdown=settings.EXPORTS_TTL)
return Response({"export-id": task.id}, status=status.HTTP_202_ACCEPTED)
return Response(
service.project_to_dict(project),
status=status.HTTP_200_OK,
headers={
"Content-Disposition": "attachment; filename={}.json".format(project.slug)
}
)
class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixin, GenericViewSet):
model = Project
permission_classes = (permissions.ImportPermission, )
permission_classes = (permissions.ImportExportPermission, )
@method_decorator(atomic)
def create(self, request, *args, **kwargs):
@ -113,6 +151,39 @@ class ProjectImporterViewSet(mixins.ImportThrottlingPolicyMixin, CreateModelMixi
headers = self.get_success_headers(response_data)
return Response(response_data, status=status.HTTP_201_CREATED, headers=headers)
@list_route(methods=["POST"])
@method_decorator(atomic)
def load_dump(self, request):
throttle = throttling.ImportDumpModeRateThrottle()
if not throttle.allow_request(request, self):
self.throttled(request, throttle.wait())
self.check_permissions(request, "load_dump", None)
dump = request.FILES.get('dump', None)
if not dump:
raise exc.WrongArguments(_("Needed dump file"))
reader = codecs.getreader("utf-8")
try:
dump = json.load(reader(dump))
except Exception:
raise exc.WrongArguments(_("Invalid dump format"))
if Project.objects.filter(slug=dump['slug']).exists():
del dump['slug']
if settings.CELERY_ENABLED:
task = tasks.load_project_dump.delay(request.user, dump)
return Response({"import-id": task.id}, status=status.HTTP_202_ACCEPTED)
dump_service.dict_to_project(dump, request.user.email)
return Response(None, status=status.HTTP_204_NO_CONTENT)
@detail_route(methods=['post'])
@method_decorator(atomic)
def issue(self, request, *args, **kwargs):

View File

@ -72,6 +72,12 @@ def store_issues(project, data):
return issues
def store_tags_colors(project, data):
project.tags_colors = data.get("tags_colors", [])
project.save()
return None
def dict_to_project(data, owner=None):
if owner:
data['owner'] = owner
@ -148,3 +154,7 @@ def dict_to_project(data, owner=None):
if service.get_errors(clear=False):
raise TaigaImportError('error importing issues')
store_tags_colors(proj, data)
return proj

View File

@ -19,6 +19,8 @@ from taiga.base.api.permissions import (TaigaResourcePermission,
IsProjectOwner, IsAuthenticated)
class ImportPermission(TaigaResourcePermission):
class ImportExportPermission(TaigaResourcePermission):
import_project_perms = IsAuthenticated()
import_item_perms = IsProjectOwner()
export_project_perms = IsProjectOwner()
load_dump_perms = IsAuthenticated()

View File

@ -46,8 +46,10 @@ class AttachedFileField(serializers.WritableField):
if not obj:
return None
data = base64.b64encode(obj.read()).decode('utf-8')
return OrderedDict([
("data", base64.b64encode(obj.read()).decode('utf-8')),
("data", data),
("name", os.path.basename(obj.name)),
])
@ -120,7 +122,7 @@ class ProjectRelatedField(serializers.RelatedField):
class HistoryUserField(JsonField):
def to_native(self, obj):
if obj is None:
if obj is None or obj == {}:
return []
try:
user = users_models.User.objects.get(pk=obj['pk'])
@ -190,7 +192,7 @@ class HistoryExportSerializer(serializers.ModelSerializer):
class Meta:
model = history_models.HistoryEntry
exclude = ("id", "comment_html")
exclude = ("id", "comment_html", "key")
class HistoryExportSerializerMixin(serializers.ModelSerializer):

View File

@ -84,7 +84,7 @@ def store_choice(project, data, field, serializer):
def store_choices(project, data, field, serializer):
result = []
for choice_data in data[field]:
for choice_data in data.get(field, []):
result.append(store_choice(project, choice_data, field, serializer))
return result
@ -102,7 +102,7 @@ def store_role(project, role):
def store_roles(project, data):
results = []
for role in data['roles']:
for role in data.get('roles', []):
results.append(store_role(project, role))
return results

View File

@ -0,0 +1,82 @@
# Copyright (C) 2014 Andrey Antukh <niwi@niwi.be>
# Copyright (C) 2014 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014 David Barragán <bameda@dbarragan.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import datetime
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.utils import timezone
from django.conf import settings
from djmail.template_mail import MagicMailBuilder
from taiga.celery import app
from .service import project_to_dict
from .dump_service import dict_to_project
from .renderers import ExportRenderer
@app.task(bind=True)
def dump_project(self, user, project):
mbuilder = MagicMailBuilder()
path = "exports/{}/{}.json".format(project.pk, self.request.id)
try:
content = ContentFile(ExportRenderer().render(project_to_dict(project), renderer_context={"indent": 4}).decode('utf-8'))
default_storage.save(path, content)
url = default_storage.url(path)
except Exception:
email = mbuilder.export_import_error(
user.email,
{
"user": user,
"error_subject": "Error generating project dump",
"error_message": "Error generating project dump",
}
)
email.send()
return
deletion_date = timezone.now() + datetime.timedelta(seconds=settings.EXPORTS_TTL)
email = mbuilder.dump_project(user.email, {"url": url, "project": project, "user": user, "deletion_date": deletion_date})
email.send()
@app.task
def delete_project_dump(project_id, task_id):
default_storage.delete("exports/{}/{}.json".format(project_id, task_id))
@app.task
def load_project_dump(user, dump):
mbuilder = MagicMailBuilder()
try:
project = dict_to_project(dump, user.email)
except Exception:
email = mbuilder.export_import_error(
user.email,
{
"user": user,
"error_subject": "Error loading project dump",
"error_message": "Error loading project dump",
}
)
email.send()
return
email = mbuilder.load_dump(user.email, {"user": user, "project": project})
email.send()

View File

@ -0,0 +1,28 @@
{% extends "emails/base.jinja" %}
{% block body %}
<table border="0" width="100%" cellpadding="0" cellspacing="0" class="table-body">
<tr>
<td>
<h1>Project dump generated</h1>
<p>Hello {{ user.get_full_name() }},</p>
<h3>Your project dump has been correctly generated.</h3>
<p>You can download it from here: <a style="color: #669900;" href="{{ url }}">{{ url }}</a></p>
<p>This file will be deleted on {{ deletion_date|date("r") }}.</p>
<p>The Taiga Team</p>
</td>
</tr>
</table>
{% endblock %}
{% block footer %}
<em>Copyright © 2014 Taiga Agile, LLC, All rights reserved.</em>
<br>
<strong>Contact us:</strong>
<br>
<strong>Support:</strong>
<a href="mailto:support@taiga.io" title="Taiga Support">support@taiga.io</a>
<br>
<strong>Our mailing address is:</strong>
<a href="https://groups.google.com/forum/#!forum/taigaio" title="Taiga mailing list">https://groups.google.com/forum/#!forum/taigaio</a>
{% endblock %}

View File

@ -0,0 +1,9 @@
Hello {{ user.get_full_name() }},
Your project dump has been correctly generated. You can download it from here:
{{ url }}
This file will be deleted on {{ deletion_date|date("r") }}.
The Taiga Team

View File

@ -0,0 +1 @@
[Taiga] Your project dump has been generated

View File

@ -0,0 +1,14 @@
{% extends "emails/base.jinja" %}
{% block body %}
<table border="0" width="100%" cellpadding="0" cellspacing="0" class="table-body">
<tr>
<td>
<h1>{{ error_message }}</h1>
<p>Hello {{ user.get_full_name() }},</p>
<p>Please, contact with the support team at <a style="color: #669900;" href="mailto:support@taiga.io">support@taiga.io</a></p>
<p>The Taiga Team</p>
</td>
</tr>
</table>
{% endblock %}

View File

@ -0,0 +1,7 @@
Hello {{ user.get_full_name() }},
{{ error_message }}
Please, contact with the support team at support@taiga.io
The Taiga Team

View File

@ -0,0 +1 @@
[Taiga] {{ error_subject }}

View File

@ -0,0 +1,29 @@
{% extends "emails/base.jinja" %}
{% set final_url = resolve_front_url("project", project.slug) %}
{% block body %}
<table border="0" width="100%" cellpadding="0" cellspacing="0" class="table-body">
<tr>
<td>
<h1>Project dump imported</h1>
<p>Hello {{ user.get_full_name() }},</p>
<h3>Your project dump has been correctly imported.</h3>
<p>You can see the project here: <a style="color: #669900;" href="{{ final_url }}">{{ final_url }}</a></p>
<p>The Taiga Team</p>
</td>
</tr>
</table>
{% endblock %}
{% block footer %}
<em>Copyright © 2014 Taiga Agile, LLC, All rights reserved.</em>
<br>
<strong>Contact us:</strong>
<br>
<strong>Support:</strong>
<a href="mailto:support@taiga.io" title="Taiga Support">support@taiga.io</a>
<br>
<strong>Our mailing address is:</strong>
<a href="https://groups.google.com/forum/#!forum/taigaio" title="Taiga mailing list">https://groups.google.com/forum/#!forum/taigaio</a>
{% endblock %}

View File

@ -0,0 +1,7 @@
Hello {{ user.get_full_name() }},
Your project dump has been correctly imported. You can see the project here:
{{ resolve_front_url('project', project.slug) }}
The Taiga Team

View File

@ -0,0 +1 @@
[Taiga] Your project dump has been imported

View File

@ -19,3 +19,6 @@ from taiga.base import throttling
class ImportModeRateThrottle(throttling.UserRateThrottle):
scope = "import-mode"
class ImportDumpModeRateThrottle(throttling.UserRateThrottle):
scope = "import-dump-mode"

View File

@ -80,7 +80,7 @@ class Attachment(models.Model):
class Meta:
verbose_name = "attachment"
verbose_name_plural = "attachments"
ordering = ["project", "created_date"]
ordering = ["project", "created_date", "id"]
permissions = (
("view_attachment", "Can view attachment"),
)

View File

@ -69,7 +69,7 @@ class Issue(OCCModelMixin, WatchedModelMixin, BlockedMixin, TaggedMixin, models.
class Meta:
verbose_name = "issue"
verbose_name_plural = "issues"
ordering = ["project", "-created_date"]
ordering = ["project", "-id"]
permissions = (
("view_issue", "Can view issue"),
)

View File

@ -70,7 +70,7 @@ class Task(OCCModelMixin, WatchedModelMixin, BlockedMixin, TaggedMixin, models.M
class Meta:
verbose_name = "task"
verbose_name_plural = "tasks"
ordering = ["project", "created_date"]
ordering = ["project", "created_date", "ref"]
# unique_together = ("ref", "project")
permissions = (
("view_task", "Can view task"),

View File

@ -35,6 +35,7 @@ def cached_prev_us(sender, instance, **kwargs):
def update_role_points_when_create_or_edit_us(sender, instance, **kwargs):
if instance._importing:
return
instance.project.update_role_points(user_stories=[instance])
@ -52,15 +53,24 @@ def update_milestone_of_tasks_when_edit_us(sender, instance, created, **kwargs):
####################################
def try_to_close_or_open_us_and_milestone_when_create_or_edit_us(sender, instance, created, **kwargs):
if instance._importing:
return
_try_to_close_or_open_us_when_create_or_edit_us(instance)
_try_to_close_or_open_milestone_when_create_or_edit_us(instance)
def try_to_close_milestone_when_delete_us(sender, instance, **kwargs):
if instance._importing:
return
_try_to_close_milestone_when_delete_us(instance)
# US
def _try_to_close_or_open_us_when_create_or_edit_us(instance):
if instance._importing:
return
from . import services as us_service
if us_service.calculate_userstory_is_closed(instance):
@ -71,6 +81,9 @@ def _try_to_close_or_open_us_when_create_or_edit_us(instance):
# Milestone
def _try_to_close_or_open_milestone_when_create_or_edit_us(instance):
if instance._importing:
return
from taiga.projects.milestones import services as milestone_service
if instance.milestone_id:
@ -87,6 +100,9 @@ def _try_to_close_or_open_milestone_when_create_or_edit_us(instance):
def _try_to_close_milestone_when_delete_us(instance):
if instance._importing:
return
from taiga.projects.milestones import services as milestone_service
with suppress(ObjectDoesNotExist):

View File

@ -45,9 +45,10 @@ router.register(r"search", SearchViewSet, base_name="search")
# Importer
from taiga.export_import.api import ProjectImporterViewSet
from taiga.export_import.api import ProjectImporterViewSet, ProjectExporterViewSet
router.register(r"importer", ProjectImporterViewSet, base_name="importer")
router.register(r"exporter", ProjectExporterViewSet, base_name="exporter")
# Projects & Types

View File

@ -0,0 +1,84 @@
# Copyright (C) 2014 Andrey Antukh <niwi@niwi.be>
# Copyright (C) 2014 Jesús Espino <jespinog@gmail.com>
# Copyright (C) 2014 David Barragán <bameda@dbarragan.com>
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import pytest
import json
from django.core.urlresolvers import reverse
from .. import factories as f
pytestmark = pytest.mark.django_db
def test_invalid_project_export(client):
user = f.UserFactory.create()
client.login(user)
url = reverse("exporter-detail", args=[1000000])
response = client.get(url, content_type="application/json")
assert response.status_code == 404
def test_valid_project_export_with_celery_disabled(client, settings):
settings.CELERY_ENABLED = False
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
f.MembershipFactory(project=project, user=user, is_owner=True)
client.login(user)
url = reverse("exporter-detail", args=[project.pk])
response = client.get(url, content_type="application/json")
assert response.status_code == 200
response_data = json.loads(response.content.decode("utf-8"))
assert response_data["slug"] == project.slug
def test_valid_project_export_with_celery_enabled(client, settings):
settings.CELERY_ENABLED = True
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
f.MembershipFactory(project=project, user=user, is_owner=True)
client.login(user)
url = reverse("exporter-detail", args=[project.pk])
response = client.get(url, content_type="application/json")
assert response.status_code == 202
response_data = json.loads(response.content.decode("utf-8"))
assert "export-id" in response_data
def test_valid_project_with_throttling(client, settings):
settings.CELERY_ENABLED = False
settings.REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"]["import-dump-mode"] = "1/minute"
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
f.MembershipFactory(project=project, user=user, is_owner=True)
client.login(user)
url = reverse("exporter-detail", args=[project.pk])
response = client.get(url, content_type="application/json")
assert response.status_code == 200
response = client.get(url, content_type="application/json")
assert response.status_code == 429

View File

@ -20,6 +20,7 @@ import base64
import datetime
from django.core.urlresolvers import reverse
from django.core.files.base import ContentFile
from .. import factories as f
@ -703,3 +704,96 @@ def test_milestone_import_duplicated_milestone(client):
assert response.status_code == 400
response_data = json.loads(response.content.decode("utf-8"))
assert response_data["milestones"][0]["name"][0] == "Name duplicated for the project"
def test_invalid_dump_import(client):
user = f.UserFactory.create()
client.login(user)
url = reverse("importer-load-dump")
data = ContentFile(b"test")
data.name = "test"
response = client.post(url, {'dump': data})
assert response.status_code == 400
response_data = json.loads(response.content.decode("utf-8"))
assert response_data["_error_message"] == "Invalid dump format"
def test_valid_dump_import_with_celery_disabled(client, settings):
settings.CELERY_ENABLED = False
user = f.UserFactory.create()
client.login(user)
url = reverse("importer-load-dump")
data = ContentFile(bytes(json.dumps({
"slug": "valid-project",
"name": "Valid project",
"description": "Valid project desc"
}), "utf-8"))
data.name = "test"
response = client.post(url, {'dump': data})
assert response.status_code == 204
def test_valid_dump_import_with_celery_enabled(client, settings):
settings.CELERY_ENABLED = True
user = f.UserFactory.create()
client.login(user)
url = reverse("importer-load-dump")
data = ContentFile(bytes(json.dumps({
"slug": "valid-project",
"name": "Valid project",
"description": "Valid project desc"
}), "utf-8"))
data.name = "test"
response = client.post(url, {'dump': data})
assert response.status_code == 202
response_data = json.loads(response.content.decode("utf-8"))
assert "import-id" in response_data
def test_dump_import_duplicated_project(client):
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
client.login(user)
url = reverse("importer-load-dump")
data = ContentFile(bytes(json.dumps({
"slug": project.slug,
"name": "Test import",
"description": "Valid project desc"
}), "utf-8"))
data.name = "test"
response = client.post(url, {'dump': data})
assert response.status_code == 204
new_project = Project.objects.all().order_by("-id").first()
assert new_project.name == "Test import"
assert new_project.slug == "{}-test-import".format(user.username)
def test_dump_import_throttling(client, settings):
settings.REST_FRAMEWORK["DEFAULT_THROTTLE_RATES"]["import-dump-mode"] = "1/minute"
user = f.UserFactory.create()
project = f.ProjectFactory.create(owner=user)
client.login(user)
url = reverse("importer-load-dump")
data = ContentFile(bytes(json.dumps({
"slug": project.slug,
"name": "Test import",
"description": "Valid project desc"
}), "utf-8"))
data.name = "test"
response = client.post(url, {'dump': data})
assert response.status_code == 204
response = client.post(url, {'dump': data})
assert response.status_code == 429