Merge pull request #539 from taigaio/Improving-API-performance

Improving API performance

taiga/base/api/pagination.py

@@ -14,7 +14,13 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from django.core.paginator import Paginator, InvalidPage
+from django.core.paginator import (
+    EmptyPage,
+    Page,
+    PageNotAnInteger,
+    Paginator,
+    InvalidPage,
+)
 from django.http import Http404
 from django.utils.translation import ugettext as _
 
@@ -36,6 +42,90 @@ def strict_positive_int(integer_string, cutoff=None):
     return ret
 
 
+class CustomPage(Page):
+    """Handle different number of items on the first page."""
+
+    def start_index(self):
+        """Return the 1-based index of the first item on this page."""
+        paginator = self.paginator
+        # Special case, return zero if no items.
+        if paginator.count == 0:
+            return 0
+        elif self.number == 1:
+            return 1
+        return (
+            (self.number - 2) * paginator.per_page + paginator.first_page + 1)
+
+    def end_index(self):
+        """Return the 1-based index of the last item on this page."""
+        paginator = self.paginator
+        # Special case for the last page because there can be orphans.
+        if self.number == paginator.num_pages:
+            return paginator.count
+        return (self.number - 1) * paginator.per_page + paginator.first_page
+
+
+class LazyPaginator(Paginator):
+    """Implement lazy pagination."""
+
+    def __init__(self, object_list, per_page, **kwargs):
+        if 'first_page' in kwargs:
+            self.first_page = kwargs.pop('first_page')
+        else:
+            self.first_page = per_page
+        super(LazyPaginator, self).__init__(object_list, per_page, **kwargs)
+
+    def get_current_per_page(self, number):
+        return self.first_page if number == 1 else self.per_page
+
+    def validate_number(self, number):
+        try:
+            number = int(number)
+        except ValueError:
+            raise PageNotAnInteger('That page number is not an integer')
+        if number < 1:
+            raise EmptyPage('That page number is less than 1')
+        return number
+
+    def page(self, number):
+        number = self.validate_number(number)
+        current_per_page = self.get_current_per_page(number)
+        if number == 1:
+            bottom = 0
+        else:
+            bottom = ((number - 2) * self.per_page + self.first_page)
+        top = bottom + current_per_page
+        # Retrieve more objects to check if there is a next page.
+        objects = list(self.object_list[bottom:top + self.orphans + 1])
+        objects_count = len(objects)
+        if objects_count > (current_per_page + self.orphans):
+            # If another page is found, increase the total number of pages.
+            self._num_pages = number + 1
+            # In any case,  return only objects for this page.
+            objects = objects[:current_per_page]
+        elif (number != 1) and (objects_count <= self.orphans):
+            raise EmptyPage('That page contains no results')
+        else:
+            # This is the last page.
+            self._num_pages = number
+        return Page(objects, number, self)
+
+    def _get_count(self):
+        raise NotImplementedError
+
+    count = property(_get_count)
+
+    def _get_num_pages(self):
+        return self._num_pages
+
+    num_pages = property(_get_num_pages)
+
+    def _get_page_range(self):
+        raise NotImplementedError
+
+    page_range = property(_get_page_range)
+
+
 class PaginationMixin(object):
     # Pagination settings
     paginate_by = api_settings.PAGINATE_BY
@@ -77,6 +167,12 @@ class PaginationMixin(object):
         Paginate a queryset if required, either returning a page object,
         or `None` if pagination is not configured for this view.
         """
+        if "HTTP_X_DISABLE_PAGINATION" in self.request.META:
+            return None
+
+        if "HTTP_X_LAZY_PAGINATION" in self.request.META:
+            self.paginator_class = LazyPaginator
+
         deprecated_style = False
         if page_size is not None:
             warnings.warn('The `page_size` parameter to `paginate_queryset()` '
@@ -103,6 +199,7 @@ class PaginationMixin(object):
 
         paginator = self.paginator_class(queryset, page_size,
                                          allow_empty_first_page=self.allow_empty)
+
         page_kwarg = self.kwargs.get(self.page_kwarg)
         page_query_param = self.request.QUERY_PARAMS.get(self.page_kwarg)
         page = page_kwarg or page_query_param or 1
@@ -124,7 +221,9 @@ class PaginationMixin(object):
         if page is None:
             return page
 
-        self.headers["x-pagination-count"] = page.paginator.count
+        if not "HTTP_X_LAZY_PAGINATION" in self.request.META:
+            self.headers["x-pagination-count"] = page.paginator.count
+
         self.headers["x-paginated"] = "true"
         self.headers["x-paginated-by"] = page.paginator.per_page
         self.headers["x-pagination-current"] = page.number

taiga/permissions/service.py

@@ -24,10 +24,8 @@ def _get_user_project_membership(user, project):
     if user.is_anonymous():
         return None
 
-    try:
+    return user.cached_membership_for_project(project)
-        return Membership.objects.get(user=user, project=project)
+
-    except Membership.DoesNotExist:
-        return None
 
 def _get_object_project(obj):
     project = None
@@ -48,6 +46,8 @@ def is_project_owner(user, obj):
         return True
 
     project = _get_object_project(obj)
+    if project is None:
+        return False
 
     membership = _get_user_project_membership(user, project)
     if membership and membership.is_owner:

taiga/projects/issues/api.py

@@ -67,6 +67,7 @@ class IssueViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixin, W
 
     filter_fields = ("project",
                      "status__is_closed")
+
     order_by_fields = ("type",
                        "status",
                        "severity",
@@ -143,7 +144,8 @@ class IssueViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixin, W
 
     def get_queryset(self):
         qs = super().get_queryset()
-        qs = qs.prefetch_related("attachments")
+        qs = qs.prefetch_related("attachments", "generated_user_stories")
+        qs = qs.select_related("owner", "assigned_to", "status", "project")
         qs = self.attach_votes_attrs_to_queryset(qs)
         return self.attach_watchers_attrs_to_queryset(qs)
 

taiga/projects/issues/serializers.py

@@ -52,7 +52,11 @@ class IssueSerializer(WatchersValidator, VoteResourceSerializerMixin, EditableWa
         return ""
 
     def get_generated_user_stories(self, obj):
-        return obj.generated_user_stories.values("id", "ref", "subject")
+        return [{
+            "id": us.id,
+            "ref": us.ref,
+            "subject": us.subject,
+        } for us in obj.generated_user_stories.all()]
 
     def get_blocked_note_html(self, obj):
         return mdrender(obj.project, obj.blocked_note)

taiga/projects/issues/services.py

@@ -133,18 +133,22 @@ def _get_issues_statuses(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-      SELECT "projects_issuestatus"."id",
+        WITH counters AS (
+        	SELECT status_id, count(status_id) count
+        		FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
+        		WHERE {where}
+        		GROUP BY status_id
+               )
+        SELECT "projects_issuestatus"."id",
              "projects_issuestatus"."name",
              "projects_issuestatus"."color",
              "projects_issuestatus"."order",
-             (SELECT count(*)
+             COALESCE(counters.count, 0)
-                FROM "issues_issue"
-                     INNER JOIN "projects_project" ON
-                                ("issues_issue"."project_id" = "projects_project"."id")
-               WHERE {where} AND "issues_issue"."status_id" = "projects_issuestatus"."id")
         FROM "projects_issuestatus"
-       WHERE "projects_issuestatus"."project_id" = %s
+        LEFT OUTER JOIN counters ON counters.status_id = projects_issuestatus.id
-    ORDER BY "projects_issuestatus"."order";
+        WHERE "projects_issuestatus"."project_id" = %s
+        ORDER BY "projects_issuestatus"."order";
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:
@@ -170,18 +174,22 @@ def _get_issues_types(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-      SELECT "projects_issuetype"."id",
+        WITH counters AS (
+        	SELECT type_id, count(type_id) count
+        		FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
+        		WHERE {where}
+        		GROUP BY type_id
+               )
+        SELECT "projects_issuetype"."id",
              "projects_issuetype"."name",
              "projects_issuetype"."color",
              "projects_issuetype"."order",
-             (SELECT count(*)
+             COALESCE(counters.count, 0)
-                FROM "issues_issue"
-                     INNER JOIN "projects_project" ON
-                                ("issues_issue"."project_id" = "projects_project"."id")
-               WHERE {where} AND "issues_issue"."type_id" = "projects_issuetype"."id")
         FROM "projects_issuetype"
-       WHERE "projects_issuetype"."project_id" = %s
+        LEFT OUTER JOIN counters ON counters.type_id = projects_issuetype.id
-    ORDER BY "projects_issuetype"."order";
+        WHERE "projects_issuetype"."project_id" = %s
+        ORDER BY "projects_issuetype"."order";
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:
@@ -207,18 +215,22 @@ def _get_issues_priorities(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-      SELECT "projects_priority"."id",
+        WITH counters AS (
+        	SELECT priority_id, count(priority_id) count
+        		FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
+        		WHERE {where}
+        		GROUP BY priority_id
+               )
+        SELECT "projects_priority"."id",
              "projects_priority"."name",
              "projects_priority"."color",
              "projects_priority"."order",
-             (SELECT count(*)
+             COALESCE(counters.count, 0)
-                FROM "issues_issue"
-                     INNER JOIN "projects_project" ON
-                                ("issues_issue"."project_id" = "projects_project"."id")
-               WHERE {where} AND "issues_issue"."priority_id" = "projects_priority"."id")
         FROM "projects_priority"
-       WHERE "projects_priority"."project_id" = %s
+        LEFT OUTER JOIN counters ON counters.priority_id = projects_priority.id
-    ORDER BY "projects_priority"."order";
+        WHERE "projects_priority"."project_id" = %s
+        ORDER BY "projects_priority"."order";
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:
@@ -244,18 +256,22 @@ def _get_issues_severities(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-      SELECT "projects_severity"."id",
+        WITH counters AS (
+        	SELECT severity_id, count(severity_id) count
+        		FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
+        		WHERE {where}
+        		GROUP BY severity_id
+               )
+        SELECT "projects_severity"."id",
              "projects_severity"."name",
              "projects_severity"."color",
              "projects_severity"."order",
-             (SELECT count(*)
+             COALESCE(counters.count, 0)
-                FROM "issues_issue"
-                     INNER JOIN "projects_project" ON
-                                ("issues_issue"."project_id" = "projects_project"."id")
-               WHERE {where} AND "issues_issue"."severity_id" = "projects_severity"."id")
         FROM "projects_severity"
-       WHERE "projects_severity"."project_id" = %s
+        LEFT OUTER JOIN counters ON counters.severity_id = projects_severity.id
-    ORDER BY "projects_severity"."order";
+        WHERE "projects_severity"."project_id" = %s
+        ORDER BY "projects_severity"."order";
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:
@@ -281,37 +297,55 @@ def _get_issues_assigned_to(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-          SELECT NULL,
+        WITH counters AS (
-                 NULL,
+        	SELECT assigned_to_id,  count(assigned_to_id) count
-                 (SELECT count(*)
+        		FROM "issues_issue"
-                    FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
-                         INNER JOIN "projects_project" ON
+        		WHERE {where} AND "issues_issue"."assigned_to_id" IS NOT NULL
-                                    ("issues_issue"."project_id" = "projects_project"."id" )
+        		GROUP BY assigned_to_id
-                   WHERE {where} AND "issues_issue"."assigned_to_id" IS NULL)
+               )
-    UNION SELECT "users_user"."id",
+        SELECT
-                 "users_user"."full_name",
+        	"projects_membership"."user_id" user_id,
-                 (SELECT count(*)
+        	"users_user"."full_name",
-                    FROM "issues_issue"
+        	COALESCE("counters".count, 0) count
-                         INNER JOIN "projects_project" ON
+        FROM projects_membership
-                                    ("issues_issue"."project_id" = "projects_project"."id" )
+        LEFT OUTER JOIN counters ON ("projects_membership"."user_id" = "counters"."assigned_to_id")
-                   WHERE {where} AND "issues_issue"."assigned_to_id" = "projects_membership"."user_id")
+        INNER JOIN "users_user" ON ("projects_membership"."user_id" = "users_user"."id")
-            FROM "projects_membership"
+        WHERE "projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL
-                 INNER JOIN "users_user" ON
+
-                            ("projects_membership"."user_id" = "users_user"."id")
+        -- unassigned issues
-           WHERE "projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL;
+        UNION
+        SELECT NULL user_id, NULL,  count(coalesce(assigned_to_id, -1)) count
+        		FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
+        		WHERE {where} AND "issues_issue"."assigned_to_id" IS NULL
+        		GROUP BY assigned_to_id
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:
-        cursor.execute(extra_sql, where_params + where_params + [project.id])
+        cursor.execute(extra_sql, where_params + [project.id] + where_params)
         rows = cursor.fetchall()
 
     result = []
+    none_valued_added = False
     for id, full_name, count in rows:
         result.append({
             "id": id,
             "full_name": full_name or "",
             "count": count,
         })
+
+        if id is None:
+            none_valued_added = True
+
+    # If there was no issue with null assigned_to we manually add it
+    if not none_valued_added:
+        result.append({
+            "id": None,
+            "full_name": "",
+            "count": 0,
+        })
+
     return sorted(result, key=itemgetter("full_name"))
 
 
@@ -322,18 +356,31 @@ def _get_issues_owners(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-       SELECT "users_user"."id",
+        WITH counters AS (
-              "users_user"."full_name",
+        	SELECT "issues_issue"."owner_id" owner_id,  count("issues_issue"."owner_id") count
-              (SELECT count(*)
+        		FROM "issues_issue"
-                FROM "issues_issue"
+        		INNER JOIN "projects_project" ON ("issues_issue"."project_id" = "projects_project"."id")
-                      INNER JOIN "projects_project" ON
+			    WHERE {where}
-                                 ("issues_issue"."project_id" = "projects_project"."id")
+        		GROUP BY "issues_issue"."owner_id"
-               WHERE {where} and "issues_issue"."owner_id" = "projects_membership"."user_id")
+               )
-        FROM "projects_membership"
+        SELECT
-             RIGHT OUTER JOIN "users_user" ON
+        	"projects_membership"."user_id" id,
-                              ("projects_membership"."user_id" = "users_user"."id")
+        	"users_user"."full_name",
-       WHERE ("projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL)
+        	COALESCE("counters".count, 0) count
-             OR ("users_user"."is_system" IS TRUE);
+        FROM projects_membership
+        LEFT OUTER JOIN counters ON ("projects_membership"."user_id" = "counters"."owner_id")
+        INNER JOIN "users_user" ON ("projects_membership"."user_id" = "users_user"."id")
+        WHERE ("projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL)
+
+        -- System users
+        UNION
+    		SELECT
+    			"users_user"."id" user_id,
+    			"users_user"."full_name" full_name,
+    			COALESCE("counters".count, 0) count
+    		FROM users_user
+    		LEFT OUTER JOIN counters ON ("users_user"."id" = "counters"."owner_id")
+    		WHERE ("users_user"."is_system" IS TRUE)
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:

taiga/projects/milestones/api.py

@@ -62,14 +62,16 @@ class MilestoneViewSet(HistoryResourceMixin, WatchedResourceMixin, ModelCrudView
 
     def get_queryset(self):
         qs = super().get_queryset()
-        qs = self.attach_watchers_attrs_to_queryset(qs)
         qs = qs.prefetch_related("user_stories",
                                  "user_stories__role_points",
                                  "user_stories__role_points__points",
-                                 "user_stories__role_points__role",
+                                 "user_stories__role_points__role")
-                                 "user_stories__generated_from_issue",
+
-                                 "user_stories__project")
+        qs = qs.select_related("project",
-        qs = qs.select_related("project")
+                               "owner")
+
+        qs = self.attach_watchers_attrs_to_queryset(qs)
+
         qs = qs.order_by("-estimated_start")
         return qs
 

taiga/projects/milestones/serializers.py

@@ -21,16 +21,14 @@ from taiga.base.utils import json
 from taiga.projects.notifications.mixins import WatchedResourceModelSerializer
 from taiga.projects.notifications.validators import WatchersValidator
 
-from ..userstories.serializers import UserStorySerializer
+from ..userstories.serializers import MilestoneUserStorySerializer
 from . import models
 
 
 class MilestoneSerializer(WatchersValidator, WatchedResourceModelSerializer, serializers.ModelSerializer):
-    user_stories = UserStorySerializer(many=True, required=False, read_only=True)
+    user_stories = MilestoneUserStorySerializer(many=True, required=False, read_only=True)
     total_points = serializers.SerializerMethodField("get_total_points")
     closed_points = serializers.SerializerMethodField("get_closed_points")
-    client_increment_points = serializers.SerializerMethodField("get_client_increment_points")
-    team_increment_points = serializers.SerializerMethodField("get_team_increment_points")
 
     class Meta:
         model = models.Milestone
@@ -42,12 +40,6 @@ class MilestoneSerializer(WatchersValidator, WatchedResourceModelSerializer, ser
     def get_closed_points(self, obj):
         return sum(obj.closed_points.values())
 
-    def get_client_increment_points(self, obj):
-        return sum(obj.client_increment_points.values())
-
-    def get_team_increment_points(self, obj):
-        return sum(obj.team_increment_points.values())
-
     def validate_name(self, attrs, source):
         """
         Check the milestone name is not duplicated in the project on creation

taiga/projects/tasks/api.py

@@ -88,6 +88,13 @@ class TaskViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixin, Wa
     def get_queryset(self):
         qs = super().get_queryset()
         qs = self.attach_votes_attrs_to_queryset(qs)
+        qs = qs.select_related(
+            "milestone",
+            "owner",
+            "assigned_to",
+            "status",
+            "project")
+        
         return self.attach_watchers_attrs_to_queryset(qs)
 
     def pre_save(self, obj):

taiga/projects/userstories/api.py

@@ -116,7 +116,12 @@ class UserStoryViewSet(OCCResourceMixin, VotedResourceMixin, HistoryResourceMixi
         qs = qs.prefetch_related("role_points",
                                  "role_points__points",
                                  "role_points__role")
-        qs = qs.select_related("milestone", "project")
+        qs = qs.select_related("milestone",
+                               "project",
+                               "status",
+                               "owner",
+                               "assigned_to",
+                               "generated_from_issue")
         qs = self.attach_votes_attrs_to_queryset(qs)
         return self.attach_watchers_attrs_to_queryset(qs)
 

taiga/projects/userstories/serializers.py

@@ -45,6 +45,18 @@ class RolePointsField(serializers.WritableField):
         return json.loads(obj)
 
 
+class MilestoneUserStorySerializer(serializers.ModelSerializer):
+    total_points = serializers.SerializerMethodField("get_total_points")
+
+    class Meta:
+        model = models.UserStory
+        depth = 0
+        fields = ("id", "ref", "subject", "is_closed", "is_blocked", "total_points")
+
+    def get_total_points(self, obj):
+        return obj.get_total_points()
+
+
 class UserStorySerializer(WatchersValidator, VoteResourceSerializerMixin, EditableWatchedResourceModelSerializer, serializers.ModelSerializer):
     tags = TagsField(default=[], required=False)
     external_reference = PgArrayField(required=False)

taiga/projects/userstories/services.py

@@ -236,37 +236,55 @@ def _get_userstories_assigned_to(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-          SELECT NULL,
+        WITH counters AS (
-                 NULL,
+        	SELECT assigned_to_id,  count(assigned_to_id) count
-                 (SELECT count(*)
+        		FROM "userstories_userstory"
-                    FROM "userstories_userstory"
+        		INNER JOIN "projects_project" ON ("userstories_userstory"."project_id" = "projects_project"."id")
-                         INNER JOIN "projects_project" ON
+        		WHERE {where} AND "userstories_userstory"."assigned_to_id" IS NOT NULL
-                                    ("userstories_userstory"."project_id" = "projects_project"."id" )
+        		GROUP BY assigned_to_id
-                   WHERE {where} AND "userstories_userstory"."assigned_to_id" IS NULL)
+               )
-    UNION SELECT "users_user"."id",
+        SELECT
-                 "users_user"."full_name",
+        	"projects_membership"."user_id" user_id,
-                 (SELECT count(*)
+        	"users_user"."full_name",
-                    FROM "userstories_userstory"
+        	COALESCE("counters".count, 0) count
-                         INNER JOIN "projects_project" ON
+        FROM projects_membership
-                                    ("userstories_userstory"."project_id" = "projects_project"."id" )
+        LEFT OUTER JOIN counters ON ("projects_membership"."user_id" = "counters"."assigned_to_id")
-                   WHERE {where} AND "userstories_userstory"."assigned_to_id" = "projects_membership"."user_id")
+        INNER JOIN "users_user" ON ("projects_membership"."user_id" = "users_user"."id")
-            FROM "projects_membership"
+        WHERE "projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL
-                 INNER JOIN "users_user" ON
+
-                            ("projects_membership"."user_id" = "users_user"."id")
+        -- unassigned userstories
-           WHERE "projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL;
+        UNION
+        SELECT NULL user_id, NULL,  count(coalesce(assigned_to_id, -1)) count
+        		FROM "userstories_userstory"
+        		INNER JOIN "projects_project" ON ("userstories_userstory"."project_id" = "projects_project"."id")
+        		WHERE {where} AND "userstories_userstory"."assigned_to_id" IS NULL
+        		GROUP BY assigned_to_id
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:
-        cursor.execute(extra_sql, where_params + where_params + [project.id])
+        cursor.execute(extra_sql, where_params + [project.id] + where_params)
         rows = cursor.fetchall()
 
     result = []
+    none_valued_added = False
     for id, full_name, count in rows:
         result.append({
             "id": id,
             "full_name": full_name or "",
             "count": count,
         })
+        
+        if id is None:
+            none_valued_added = True
+
+    # If there was no userstory with null assigned_to we manually add it
+    if not none_valued_added:
+        result.append({
+            "id": None,
+            "full_name": "",
+            "count": 0,
+        })
+
     return sorted(result, key=itemgetter("full_name"))
 
 
@@ -277,18 +295,31 @@ def _get_userstories_owners(project, queryset):
     where_params = queryset_where_tuple[1]
 
     extra_sql = """
-       SELECT "users_user"."id",
+        WITH counters AS (
-              "users_user"."full_name",
+        	SELECT "userstories_userstory"."owner_id" owner_id,  count(coalesce("userstories_userstory"."owner_id", -1)) count
-              (SELECT count(*)
+        		FROM "userstories_userstory"
-                 FROM "userstories_userstory"
+        		INNER JOIN "projects_project" ON ("userstories_userstory"."project_id" = "projects_project"."id")
-                      INNER JOIN "projects_project" ON
+			    WHERE {where}
-                                 ("userstories_userstory"."project_id" = "projects_project"."id")
+        		GROUP BY "userstories_userstory"."owner_id"
-                WHERE {where} AND "userstories_userstory"."owner_id" = "projects_membership"."user_id")
+               )
-        FROM "projects_membership"
+        SELECT
-             RIGHT OUTER JOIN "users_user" ON
+        	"projects_membership"."user_id" id,
-                              ("projects_membership"."user_id" = "users_user"."id")
+        	"users_user"."full_name",
-       WHERE (("projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL)
+        	COALESCE("counters".count, 0) count
-              OR ("users_user"."is_system" IS TRUE));
+        FROM projects_membership
+        LEFT OUTER JOIN counters ON ("projects_membership"."user_id" = "counters"."owner_id")
+        INNER JOIN "users_user" ON ("projects_membership"."user_id" = "users_user"."id")
+        WHERE ("projects_membership"."project_id" = %s AND "projects_membership"."user_id" IS NOT NULL)
+
+        -- System users
+        UNION
+    		SELECT
+    			"users_user"."id" user_id,
+    			"users_user"."full_name" full_name,
+    			COALESCE("counters".count, 0) count
+    		FROM users_user
+    		LEFT OUTER JOIN counters ON ("users_user"."id" = "counters"."owner_id")
+    		WHERE ("users_user"."is_system" IS TRUE)
     """.format(where=where)
 
     with closing(connection.cursor()) as cursor:

taiga/timeline/api.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from django.contrib.contenttypes.models import ContentType
+from django.apps import apps
 
 from taiga.base import response
 from taiga.base.api.utils import get_object_or_404
@@ -46,6 +47,14 @@ class TimelineViewSet(ReadOnlyListViewSet):
         # Switch between paginated or standard style responses
         page = self.paginate_queryset(queryset)
         if page is not None:
+            user_ids = list(set([obj.data.get("user", {}).get("id", None) for obj in page.object_list]))
+            User = apps.get_model("users", "User")
+            users = {u.id: u for u in User.objects.filter(id__in=user_ids)}
+
+            for obj in page.object_list:
+                user_id = obj.data.get("user", {}).get("id", None)
+                obj._prefetched_user = users.get(user_id, None)
+
             serializer = self.get_pagination_serializer(page)
         else:
             serializer = self.get_serializer(queryset, many=True)

taiga/timeline/serializers.py

@@ -24,39 +24,34 @@ from taiga.users.services import get_photo_or_gravatar_url, get_big_photo_or_gra
 from . import models
 from . import service
 
-class TimelineDataJsonField(serializers.WritableField):
-    """
-    Timeline Json objects serializer.
-    """
-    widget = widgets.Textarea
-
-    def to_native(self, obj):
-        #Updates the data user info saved if the user exists
-        User = apps.get_model("users", "User")
-        userData = obj.get("user", None)
-        if userData:
-            try:
-                user = User.objects.get(id=userData["id"])
-                obj["user"] = {
-                    "id": user.pk,
-                    "name": user.get_full_name(),
-                    "photo": get_photo_or_gravatar_url(user),
-                    "big_photo": get_big_photo_or_gravatar_url(user),
-                    "username": user.username,
-                    "is_profile_visible": user.is_active and not user.is_system,
-                    "date_joined": user.date_joined
-                }
-            except User.DoesNotExist:
-                pass
-
-        return obj
-
-    def from_native(self, data):
-        return data
-
 
 class TimelineSerializer(serializers.ModelSerializer):
-    data = TimelineDataJsonField()
+    data = serializers.SerializerMethodField("get_data")
 
     class Meta:
         model = models.Timeline
+
+    def get_data(self, obj):
+        #Updates the data user info saved if the user exists
+        if hasattr(obj, "_prefetched_user"):
+            user = obj._prefetched_user
+        else:
+            User = apps.get_model("users", "User")
+            userData = obj.data.get("user", None)
+            try:
+                user = User.objects.get(id=userData["id"])
+            except User.DoesNotExist:
+                user = None
+
+        if user is not None:
+            obj.data["user"] = {
+                "id": user.pk,
+                "name": user.get_full_name(),
+                "photo": get_photo_or_gravatar_url(user),
+                "big_photo": get_big_photo_or_gravatar_url(user),
+                "username": user.username,
+                "is_profile_visible": user.is_active and not user.is_system,
+                "date_joined": user.date_joined
+            }
+
+        return obj.data

taiga/timeline/service.py

@@ -96,6 +96,7 @@ def get_timeline(obj, namespace=None):
     if namespace is not None:
         timeline = timeline.filter(namespace=namespace)
 
+    timeline = timeline.select_related("project")
     timeline = timeline.order_by("-created", "-id")
     return timeline
 
@@ -128,9 +129,7 @@ def filter_timeline_for_user(timeline, user):
 
     # Filtering private projects where user is member
     if not user.is_anonymous():
-        membership_model = apps.get_model('projects', 'Membership')
+        for membership in user.cached_memberships:
-        memberships_qs = membership_model.objects.filter(user=user)
-        for membership in memberships_qs:
             for content_type_key, content_type in content_types.items():
                 if content_type_key in membership.role.permissions or membership.is_owner:
                     tl_filter |= Q(project=membership.project, data_content_type=content_type)

taiga/users/models.py

@@ -23,6 +23,8 @@ import uuid
 
 from unidecode import unidecode
 
+from django.apps import apps
+from django.contrib.contenttypes.models import ContentType
 from django.db import models
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
@@ -39,6 +41,7 @@ from taiga.auth.tokens import get_token_for_user
 from taiga.base.utils.slug import slugify_uniquely
 from taiga.base.utils.iterators import split_by_n
 from taiga.permissions.permissions import MEMBERS_PERMISSIONS
+from taiga.projects.notifications.choices import NotifyLevel
 
 from easy_thumbnails.files import get_thumbnailer
 
@@ -135,6 +138,10 @@ class User(AbstractBaseUser, PermissionsMixin):
     new_email = models.EmailField(_('new email address'), null=True, blank=True)
 
     is_system = models.BooleanField(null=False, blank=False, default=False)
+    _cached_memberships = None
+    _cached_liked_ids = None
+    _cached_watched_ids = None
+    _cached_notify_levels = None
 
     USERNAME_FIELD = 'username'
     REQUIRED_FIELDS = ['email']
@@ -152,6 +159,63 @@ class User(AbstractBaseUser, PermissionsMixin):
     def __str__(self):
         return self.get_full_name()
 
+    def _fill_cached_memberships(self):
+        self._cached_memberships = {}
+        qs = self.memberships.prefetch_related("user", "project", "role")
+        for membership in qs.all():
+            self._cached_memberships[membership.project.id] = membership
+
+    @property
+    def cached_memberships(self):
+        if self._cached_memberships is None:
+            self._fill_cached_memberships()
+
+        return self._cached_memberships.values()
+
+    def cached_membership_for_project(self, project):
+        if self._cached_memberships is None:
+            self._fill_cached_memberships()
+
+        return self._cached_memberships.get(project.id, None)
+
+    def is_fan(self, obj):
+        if self._cached_liked_ids is None:
+            self._cached_liked_ids = set()
+            for like in self.likes.select_related("content_type").all():
+                like_id = "{}-{}".format(like.content_type.id, like.object_id)
+                self._cached_liked_ids.add(like_id)
+
+        obj_type = ContentType.objects.get_for_model(obj)
+        obj_id = "{}-{}".format(obj_type.id, obj.id)
+        return obj_id in self._cached_liked_ids
+
+    def is_watcher(self, obj):
+        if self._cached_watched_ids is None:
+            self._cached_watched_ids = set()
+            for watched in self.watched.select_related("content_type").all():
+                watched_id = "{}-{}".format(watched.content_type.id, watched.object_id)
+                self._cached_watched_ids.add(watched_id)
+
+            notify_policies = self.notify_policies.select_related("project")\
+                .exclude(notify_level=NotifyLevel.none)
+
+            for notify_policy in notify_policies:
+                obj_type = ContentType.objects.get_for_model(notify_policy.project)
+                watched_id = "{}-{}".format(obj_type.id, notify_policy.project.id)
+                self._cached_watched_ids.add(watched_id)
+
+        obj_type = ContentType.objects.get_for_model(obj)
+        obj_id = "{}-{}".format(obj_type.id, obj.id)
+        return obj_id in self._cached_watched_ids
+
+    def get_notify_level(self, project):
+        if self._cached_notify_levels is None:
+            self._cached_notify_levels = {}
+            for notify_policy in self.notify_policies.select_related("project"):
+                self._cached_notify_levels[notify_policy.project.id] = notify_policy.notify_level
+
+        return self._cached_notify_levels.get(project.id, None)
+
     def get_short_name(self):
         "Returns the short name for the user."
         return self.username