From a7287df03943b32ca7f5ba717eb9f3be90762a5c Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Tue, 29 Nov 2016 10:50:09 +0100 Subject: [PATCH] Issue 4530: some searches without results --- CHANGELOG.md | 5 +-- taiga/base/filters.py | 4 +-- taiga/projects/filters.py | 8 ++--- .../migrations/0057_auto_20161129_0945.py | 35 +++++++++++++++++++ .../migrations/0005_auto_20161201_1628.py | 20 +++++++++++ taiga/projects/wiki/models.py | 2 +- taiga/projects/wiki/validators.py | 3 ++ taiga/searches/services.py | 14 ++++---- taiga/users/filters.py | 4 +-- taiga/users/services.py | 6 ++-- 10 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 taiga/projects/migrations/0057_auto_20161129_0945.py create mode 100644 taiga/projects/wiki/migrations/0005_auto_20161201_1628.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b5a5fc6..344e5b8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,15 +5,16 @@ ### Features - Contact with the project: if the projects have this module enabled Taiga users can contact them. -- Memberships API endpoints now allows using usernames and emails instead of using only emails. -- Contacts API search by free text: consulting the username, full name and email. - Ability to create rich text custom fields in Epics, User Stories, Tasks and Isues. +- Full text search now use simple as tolenizer so search with non-english text are allowed. - i18n: - Add japanese (ja) translation. - Add chinese simplified (zh-Hans) translation. ### Misc - API: + - Memberships API endpoints now allows using usernames and emails instead of using only emails. + - Contacts API allow full text search (by the username, full name or email). - Filter milestones, user stories and tasks by estimated_start and estimated_finish dates. - Add project_extra_info to epics, tasks, milestones, issues and wiki pages endpoints. - Lots of small and not so small bugfixes. diff --git a/taiga/base/filters.py b/taiga/base/filters.py index 80c7e280..4c330178 100644 --- a/taiga/base/filters.py +++ b/taiga/base/filters.py @@ -552,11 +552,11 @@ class QFilter(FilterBackend): if q: table = queryset.model._meta.db_table where_clause = (""" - to_tsvector('english_nostop', + to_tsvector('simple', coalesce({table}.subject, '') || ' ' || coalesce(array_to_string({table}.tags, ' '), '') || ' ' || coalesce({table}.ref) || ' ' || - coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s) + coalesce({table}.description, '')) @@ to_tsquery('simple', %s) """.format(table=table)) queryset = queryset.extra(where=[where_clause], params=[to_tsquery(q)]) diff --git a/taiga/projects/filters.py b/taiga/projects/filters.py index fe720f97..c4e4f90b 100644 --- a/taiga/projects/filters.py +++ b/taiga/projects/filters.py @@ -94,14 +94,14 @@ class QFilterBackend(FilterBackend): # NOTE: See migtration 0033_text_search_indexes q = request.QUERY_PARAMS.get('q', None) if q: - tsquery = "to_tsquery('english_nostop', %s)" + tsquery = "to_tsquery('simple', %s)" tsquery_params = [to_tsquery(q)] tsvector = """ - setweight(to_tsvector('english_nostop', + setweight(to_tsvector('simple', coalesce(projects_project.name, '')), 'A') || - setweight(to_tsvector('english_nostop', + setweight(to_tsvector('simple', coalesce(inmutable_array_to_string(projects_project.tags), '')), 'B') || - setweight(to_tsvector('english_nostop', + setweight(to_tsvector('simple', coalesce(projects_project.description, '')), 'C') """ diff --git a/taiga/projects/migrations/0057_auto_20161129_0945.py b/taiga/projects/migrations/0057_auto_20161129_0945.py new file mode 100644 index 00000000..0e161a4f --- /dev/null +++ b/taiga/projects/migrations/0057_auto_20161129_0945.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.3 on 2016-11-29 09:45 +from __future__ import unicode_literals + +from django.db import migrations + + +DROP_INDEX = """ + DROP INDEX IF EXISTS projects_project_textquery_idx; +""" + + +# NOTE: This index is needed by taiga.projects.filters.QFilter +CREATE_INDEX = """ + CREATE INDEX projects_project_textquery_idx + ON projects_project + USING gin((setweight(to_tsvector('simple', + coalesce(projects_project.name, '')), 'A') || + setweight(to_tsvector('simple', + coalesce(inmutable_array_to_string(projects_project.tags), '')), 'B') || + setweight(to_tsvector('simple', + coalesce(projects_project.description, '')), 'C'))); +""" + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0056_auto_20161110_1518'), + ] + + operations = [ + migrations.RunSQL([DROP_INDEX, CREATE_INDEX], + [DROP_INDEX]), + ] diff --git a/taiga/projects/wiki/migrations/0005_auto_20161201_1628.py b/taiga/projects/wiki/migrations/0005_auto_20161201_1628.py new file mode 100644 index 00000000..87723645 --- /dev/null +++ b/taiga/projects/wiki/migrations/0005_auto_20161201_1628.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10.3 on 2016-12-01 16:28 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('wiki', '0004_auto_20160928_0540'), + ] + + operations = [ + migrations.AlterField( + model_name='wikipage', + name='slug', + field=models.SlugField(allow_unicode=True, max_length=500, verbose_name='slug'), + ), + ] diff --git a/taiga/projects/wiki/models.py b/taiga/projects/wiki/models.py index 1c51fff0..7b913107 100644 --- a/taiga/projects/wiki/models.py +++ b/taiga/projects/wiki/models.py @@ -33,7 +33,7 @@ class WikiPage(OCCModelMixin, WatchedModelMixin, models.Model): project = models.ForeignKey("projects.Project", null=False, blank=False, related_name="wiki_pages", verbose_name=_("project")) slug = models.SlugField(max_length=500, db_index=True, null=False, blank=False, - verbose_name=_("slug")) + verbose_name=_("slug"), allow_unicode=True) content = models.TextField(null=False, blank=True, verbose_name=_("content")) owner = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True, diff --git a/taiga/projects/wiki/validators.py b/taiga/projects/wiki/validators.py index 033fac1b..9c5601f5 100644 --- a/taiga/projects/wiki/validators.py +++ b/taiga/projects/wiki/validators.py @@ -17,12 +17,15 @@ # along with this program. If not, see . from taiga.base.api import validators +from taiga.base.api import serializers from taiga.projects.notifications.validators import WatchersValidator from . import models class WikiPageValidator(WatchersValidator, validators.ModelValidator): + slug = serializers.CharField() + class Meta: model = models.WikiPage read_only_fields = ('modified_date', 'created_date', 'owner') diff --git a/taiga/searches/services.py b/taiga/searches/services.py index adda60bb..5bc6159b 100644 --- a/taiga/searches/services.py +++ b/taiga/searches/services.py @@ -55,23 +55,23 @@ def search_issues(project, text): def search_wiki_pages(project, text): model = apps.get_model("wiki", "WikiPage") queryset = model.objects.filter(project_id=project.pk) - tsquery = "to_tsquery('english_nostop', %s)" + tsquery = "to_tsquery('simple', %s)" tsvector = """ - setweight(to_tsvector('english_nostop', coalesce(wiki_wikipage.slug)), 'A') || - setweight(to_tsvector('english_nostop', coalesce(wiki_wikipage.content)), 'B') + setweight(to_tsvector('simple', coalesce(wiki_wikipage.slug)), 'A') || + setweight(to_tsvector('simple', coalesce(wiki_wikipage.content)), 'B') """ return _search_by_query(queryset, tsquery, tsvector, text) def _search_items(queryset, table, text): - tsquery = "to_tsquery('english_nostop', %s)" + tsquery = "to_tsquery('simple', %s)" tsvector = """ - setweight(to_tsvector('english_nostop', + setweight(to_tsvector('simple', coalesce({table}.subject) || ' ' || coalesce({table}.ref)), 'A') || - setweight(to_tsvector('english_nostop', coalesce(inmutable_array_to_string({table}.tags))), 'B') || - setweight(to_tsvector('english_nostop', coalesce({table}.description)), 'C') + setweight(to_tsvector('simple', coalesce(inmutable_array_to_string({table}.tags))), 'B') || + setweight(to_tsvector('simple', coalesce({table}.description)), 'C') """.format(table=table) return _search_by_query(queryset, tsquery, tsvector, text) diff --git a/taiga/users/filters.py b/taiga/users/filters.py index 03f7b10e..2c7e47f9 100644 --- a/taiga/users/filters.py +++ b/taiga/users/filters.py @@ -29,10 +29,10 @@ class ContactsFilterBackend(PermissionBasedFilterBackend): if q: table = qs.model._meta.db_table where_clause = (""" - to_tsvector('english_nostop', + to_tsvector('simple', coalesce({table}.username, '') || ' ' || coalesce({table}.full_name) || ' ' || - coalesce({table}.email, '')) @@ to_tsquery('english_nostop', %s) + coalesce({table}.email, '')) @@ to_tsquery('simple', %s) """.format(table=table)) qs = qs.extra(where=[where_clause], params=[to_tsquery(q)]) diff --git a/taiga/users/services.py b/taiga/users/services.py index 4b49353e..14a3802a 100644 --- a/taiga/users/services.py +++ b/taiga/users/services.py @@ -312,7 +312,7 @@ def get_watched_list(for_user, from_user, type=None, q=None): if q: filters_sql += """ AND ( - to_tsvector('english_nostop', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('english_nostop', %(q)s) + to_tsvector('simple', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('simple', %(q)s) ) """ @@ -412,7 +412,7 @@ def get_liked_list(for_user, from_user, type=None, q=None): if q: filters_sql += """ AND ( - to_tsvector('english_nostop', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('english_nostop', %(q)s) + to_tsvector('simple', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('simple', %(q)s) ) """ @@ -495,7 +495,7 @@ def get_voted_list(for_user, from_user, type=None, q=None): if q: filters_sql += """ AND ( - to_tsvector('english_nostop', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('english_nostop', %(q)s) + to_tsvector('simple', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('simple', %(q)s) ) """