Issue 4530: some searches without results

CHANGELOG.md

@@ -5,15 +5,16 @@
 
 ### Features
 - Contact with the project: if the projects have this module enabled Taiga users can contact them.
-- Memberships API endpoints now allows using usernames and emails instead of using only emails.
-- Contacts API search by free text: consulting the username, full name and email.
 - Ability to create rich text custom fields in Epics, User Stories, Tasks and Isues.
+- Full text search now use simple as tolenizer so search with non-english text are allowed.
 - i18n:
   - Add japanese (ja) translation.
   - Add chinese simplified (zh-Hans) translation.
 
 ### Misc
 - API:
+    - Memberships API endpoints now allows using usernames and emails instead of using only emails.
+    - Contacts API allow full text search (by the username, full name or email).
     - Filter milestones, user stories and tasks by estimated_start and estimated_finish dates.
     - Add project_extra_info to epics, tasks, milestones, issues and wiki pages endpoints.
 - Lots of small and not so small bugfixes.

taiga/base/filters.py

@@ -552,11 +552,11 @@ class QFilter(FilterBackend):
         if q:
             table = queryset.model._meta.db_table
             where_clause = ("""
-                to_tsvector('english_nostop',
+                to_tsvector('simple',
                             coalesce({table}.subject, '') || ' ' ||
                             coalesce(array_to_string({table}.tags, ' '), '') || ' ' ||
                             coalesce({table}.ref) || ' ' ||
-                            coalesce({table}.description, '')) @@ to_tsquery('english_nostop', %s)
+                            coalesce({table}.description, '')) @@ to_tsquery('simple', %s)
             """.format(table=table))
 
             queryset = queryset.extra(where=[where_clause], params=[to_tsquery(q)])

taiga/projects/filters.py

@@ -94,14 +94,14 @@ class QFilterBackend(FilterBackend):
         # NOTE: See migtration 0033_text_search_indexes
         q = request.QUERY_PARAMS.get('q', None)
         if q:
-            tsquery = "to_tsquery('english_nostop', %s)"
+            tsquery = "to_tsquery('simple', %s)"
             tsquery_params = [to_tsquery(q)]
             tsvector = """
-             setweight(to_tsvector('english_nostop',
+             setweight(to_tsvector('simple',
                                    coalesce(projects_project.name, '')), 'A') ||
-             setweight(to_tsvector('english_nostop',
+             setweight(to_tsvector('simple',
                                    coalesce(inmutable_array_to_string(projects_project.tags), '')), 'B') ||
-             setweight(to_tsvector('english_nostop',
+             setweight(to_tsvector('simple',
                                    coalesce(projects_project.description, '')), 'C')
             """
 

taiga/projects/migrations/0057_auto_20161129_0945.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.3 on 2016-11-29 09:45
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+DROP_INDEX = """
+    DROP INDEX IF EXISTS projects_project_textquery_idx;
+"""
+
+
+# NOTE: This index is needed by taiga.projects.filters.QFilter
+CREATE_INDEX = """
+    CREATE INDEX projects_project_textquery_idx
+              ON projects_project
+           USING gin((setweight(to_tsvector('simple',
+                                coalesce(projects_project.name, '')), 'A') ||
+                      setweight(to_tsvector('simple',
+                                 coalesce(inmutable_array_to_string(projects_project.tags), '')), 'B') ||
+                      setweight(to_tsvector('simple',
+                                coalesce(projects_project.description, '')), 'C')));
+"""
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0056_auto_20161110_1518'),
+    ]
+
+    operations = [
+        migrations.RunSQL([DROP_INDEX, CREATE_INDEX],
+                          [DROP_INDEX]),
+    ]

taiga/projects/wiki/migrations/0005_auto_20161201_1628.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.3 on 2016-12-01 16:28
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('wiki', '0004_auto_20160928_0540'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='wikipage',
+            name='slug',
+            field=models.SlugField(allow_unicode=True, max_length=500, verbose_name='slug'),
+        ),
+    ]

taiga/projects/wiki/models.py

@@ -33,7 +33,7 @@ class WikiPage(OCCModelMixin, WatchedModelMixin, models.Model):
     project = models.ForeignKey("projects.Project", null=False, blank=False,
                                 related_name="wiki_pages", verbose_name=_("project"))
     slug = models.SlugField(max_length=500, db_index=True, null=False, blank=False,
-                            verbose_name=_("slug"))
+                            verbose_name=_("slug"), allow_unicode=True)
     content = models.TextField(null=False, blank=True,
                                verbose_name=_("content"))
     owner = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True,

taiga/projects/wiki/validators.py

@@ -17,12 +17,15 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from taiga.base.api import validators
+from taiga.base.api import serializers
 from taiga.projects.notifications.validators import WatchersValidator
 
 from . import models
 
 
 class WikiPageValidator(WatchersValidator, validators.ModelValidator):
+    slug = serializers.CharField()
+
     class Meta:
         model = models.WikiPage
         read_only_fields = ('modified_date', 'created_date', 'owner')

taiga/searches/services.py

@@ -55,23 +55,23 @@ def search_issues(project, text):
 def search_wiki_pages(project, text):
     model = apps.get_model("wiki", "WikiPage")
     queryset = model.objects.filter(project_id=project.pk)
-    tsquery = "to_tsquery('english_nostop', %s)"
+    tsquery = "to_tsquery('simple', %s)"
     tsvector = """
-        setweight(to_tsvector('english_nostop', coalesce(wiki_wikipage.slug)), 'A') ||
+        setweight(to_tsvector('simple', coalesce(wiki_wikipage.slug)), 'A') ||
-        setweight(to_tsvector('english_nostop', coalesce(wiki_wikipage.content)), 'B')
+        setweight(to_tsvector('simple', coalesce(wiki_wikipage.content)), 'B')
     """
 
     return _search_by_query(queryset, tsquery, tsvector, text)
 
 
 def _search_items(queryset, table, text):
-    tsquery = "to_tsquery('english_nostop', %s)"
+    tsquery = "to_tsquery('simple', %s)"
     tsvector = """
-        setweight(to_tsvector('english_nostop',
+        setweight(to_tsvector('simple',
                               coalesce({table}.subject) || ' ' ||
                               coalesce({table}.ref)), 'A') ||
-        setweight(to_tsvector('english_nostop', coalesce(inmutable_array_to_string({table}.tags))), 'B') ||
+        setweight(to_tsvector('simple', coalesce(inmutable_array_to_string({table}.tags))), 'B') ||
-        setweight(to_tsvector('english_nostop', coalesce({table}.description)), 'C')
+        setweight(to_tsvector('simple', coalesce({table}.description)), 'C')
     """.format(table=table)
     return _search_by_query(queryset, tsquery, tsvector, text)
 

taiga/users/filters.py

@@ -29,10 +29,10 @@ class ContactsFilterBackend(PermissionBasedFilterBackend):
         if q:
             table = qs.model._meta.db_table
             where_clause = ("""
-                to_tsvector('english_nostop',
+                to_tsvector('simple',
                             coalesce({table}.username, '') || ' ' ||
                             coalesce({table}.full_name) || ' ' ||
-                            coalesce({table}.email, '')) @@ to_tsquery('english_nostop', %s)
+                            coalesce({table}.email, '')) @@ to_tsquery('simple', %s)
             """.format(table=table))
 
             qs = qs.extra(where=[where_clause], params=[to_tsquery(q)])

taiga/users/services.py

@@ -312,7 +312,7 @@ def get_watched_list(for_user, from_user, type=None, q=None):
 
     if q:
         filters_sql += """ AND (
-            to_tsvector('english_nostop', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('english_nostop', %(q)s)
+            to_tsvector('simple', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('simple', %(q)s)
         )
         """
 
@@ -412,7 +412,7 @@ def get_liked_list(for_user, from_user, type=None, q=None):
 
     if q:
         filters_sql += """ AND (
-            to_tsvector('english_nostop', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('english_nostop', %(q)s)
+            to_tsvector('simple', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('simple', %(q)s)
         )
         """
 
@@ -495,7 +495,7 @@ def get_voted_list(for_user, from_user, type=None, q=None):
 
     if q:
         filters_sql += """ AND (
-            to_tsvector('english_nostop', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('english_nostop', %(q)s)
+            to_tsvector('simple', coalesce(subject,'') || ' ' ||coalesce(entities.name,'') || ' ' ||coalesce(to_char(ref, '999'),'')) @@ to_tsquery('simple', %(q)s)
         )
         """