Commit 663c6238 authored by Eliot Berriot's avatar Eliot Berriot
Browse files

Merge branch '344-query-language' into 'develop'

Resolve "Smarter query language in search bar"

Closes #344

See merge request funkwhale/funkwhale!301
parents daf8a229 4428d740
import django_filters
from django.db import models
from funkwhale_api.music import utils
from . import search
PRIVACY_LEVEL_CHOICES = [
("me", "Only me"),
......@@ -34,5 +34,17 @@ class SearchFilter(django_filters.CharFilter):
def filter(self, qs, value):
if not value:
return qs
query = utils.get_query(value, self.search_fields)
query = search.get_query(value, self.search_fields)
return qs.filter(query)
class SmartSearchFilter(django_filters.CharFilter):
def __init__(self, *args, **kwargs):
self.config = kwargs.pop("config")
super().__init__(*args, **kwargs)
def filter(self, qs, value):
if not value:
return qs
cleaned = self.config.clean(value)
return search.apply(qs, cleaned)
import re
from django.db.models import Q
QUERY_REGEX = re.compile('(((?P<key>\w+):)?(?P<value>"[^"]+"|[\S]+))')
def parse_query(query):
"""
Given a search query such as "hello is:issue status:opened",
returns a list of dictionnaries discribing each query token
"""
matches = [m.groupdict() for m in QUERY_REGEX.finditer(query.lower())]
for m in matches:
if m["value"].startswith('"') and m["value"].endswith('"'):
m["value"] = m["value"][1:-1]
return matches
def normalize_query(
query_string,
findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
normspace=re.compile(r"\s{2,}").sub,
):
""" Splits the query string in invidual keywords, getting rid of unecessary spaces
and grouping quoted words together.
Example:
>>> normalize_query(' some random words "with quotes " and spaces')
['some', 'random', 'words', 'with quotes', 'and', 'spaces']
"""
return [normspace(" ", (t[0] or t[1]).strip()) for t in findterms(query_string)]
def get_query(query_string, search_fields):
""" Returns a query, that is a combination of Q objects. That combination
aims to search keywords within a model by testing the given search fields.
"""
query = None # Query to search for every search term
terms = normalize_query(query_string)
for term in terms:
or_query = None # Query to search for a given term in each field
for field_name in search_fields:
q = Q(**{"%s__icontains" % field_name: term})
if or_query is None:
or_query = q
else:
or_query = or_query | q
if query is None:
query = or_query
else:
query = query & or_query
return query
def filter_tokens(tokens, valid):
return [t for t in tokens if t["key"] in valid]
def apply(qs, config_data):
for k in ["filter_query", "search_query"]:
q = config_data.get(k)
if q:
qs = qs.filter(q)
return qs
class SearchConfig:
def __init__(self, search_fields={}, filter_fields={}, types=[]):
self.filter_fields = filter_fields
self.search_fields = search_fields
self.types = types
def clean(self, query):
tokens = parse_query(query)
cleaned_data = {}
cleaned_data["types"] = self.clean_types(filter_tokens(tokens, ["is"]))
cleaned_data["search_query"] = self.clean_search_query(
filter_tokens(tokens, [None, "in"])
)
unhandled_tokens = [t for t in tokens if t["key"] not in [None, "is", "in"]]
cleaned_data["filter_query"] = self.clean_filter_query(unhandled_tokens)
return cleaned_data
def clean_search_query(self, tokens):
if not self.search_fields or not tokens:
return
fields_subset = {
f for t in filter_tokens(tokens, ["in"]) for f in t["value"].split(",")
} or set(self.search_fields.keys())
fields_subset = set(self.search_fields.keys()) & fields_subset
to_fields = [self.search_fields[k]["to"] for k in fields_subset]
query_string = " ".join([t["value"] for t in filter_tokens(tokens, [None])])
return get_query(query_string, sorted(to_fields))
def clean_filter_query(self, tokens):
if not self.filter_fields or not tokens:
return
matching = [t for t in tokens if t["key"] in self.filter_fields]
queries = [
Q(**{self.filter_fields[t["key"]]["to"]: t["value"]}) for t in matching
]
query = None
for q in queries:
if not query:
query = q
else:
query = query & q
return query
def clean_types(self, tokens):
if not self.types:
return []
if not tokens:
# no filtering on type, we return all types
return [t for key, t in self.types]
types = []
for token in tokens:
for key, t in self.types:
if key.lower() == token["value"]:
types.append(t)
return types
import django_filters
from funkwhale_api.common import fields
from funkwhale_api.common import search
from . import models
......@@ -23,8 +24,21 @@ class LibraryFilter(django_filters.FilterSet):
class LibraryTrackFilter(django_filters.FilterSet):
library = django_filters.CharFilter("library__uuid")
status = django_filters.CharFilter(method="filter_status")
q = fields.SearchFilter(
search_fields=["artist_name", "title", "album_title", "library__actor__domain"]
q = fields.SmartSearchFilter(
config=search.SearchConfig(
search_fields={
"domain": {"to": "library__actor__domain"},
"artist": {"to": "artist_name"},
"album": {"to": "album_title"},
"title": {"to": "title"},
},
filter_fields={
"domain": {"to": "library__actor__domain"},
"artist": {"to": "artist_name__iexact"},
"album": {"to": "album_title__iexact"},
"title": {"to": "title__iexact"},
},
)
)
def filter_status(self, queryset, field_name, value):
......
import mimetypes
import re
import magic
import mutagen
from django.db.models import Q
def normalize_query(
query_string,
findterms=re.compile(r'"([^"]+)"|(\S+)').findall,
normspace=re.compile(r"\s{2,}").sub,
):
""" Splits the query string in invidual keywords, getting rid of unecessary spaces
and grouping quoted words together.
Example:
>>> normalize_query(' some random words "with quotes " and spaces')
['some', 'random', 'words', 'with quotes', 'and', 'spaces']
"""
return [normspace(" ", (t[0] or t[1]).strip()) for t in findterms(query_string)]
def get_query(query_string, search_fields):
""" Returns a query, that is a combination of Q objects. That combination
aims to search keywords within a model by testing the given search fields.
"""
query = None # Query to search for every search term
terms = normalize_query(query_string)
for term in terms:
or_query = None # Query to search for a given term in each field
for field_name in search_fields:
q = Q(**{"%s__icontains" % field_name: term})
if or_query is None:
or_query = q
else:
or_query = or_query | q
if query is None:
query = or_query
else:
query = query & or_query
return query
from funkwhale_api.common.search import normalize_query, get_query # noqa
def guess_mimetype(f):
......
import pytest
from django.db.models import Q
from funkwhale_api.common import search
from funkwhale_api.music import models as music_models
@pytest.mark.parametrize(
"query,expected",
[
("", [music_models.Album, music_models.Artist]),
("is:album", [music_models.Album]),
("is:artist is:album", [music_models.Artist, music_models.Album]),
],
)
def test_search_config_is(query, expected):
s = search.SearchConfig(
types=[("album", music_models.Album), ("artist", music_models.Artist)]
)
cleaned = s.clean(query)
assert cleaned["types"] == expected
@pytest.mark.parametrize(
"query,expected",
[
("", None),
("hello world", search.get_query("hello world", ["f1", "f2", "f3"])),
("hello in:field2", search.get_query("hello", ["f2"])),
("hello in:field1,field2", search.get_query("hello", ["f1", "f2"])),
],
)
def test_search_config_query(query, expected):
s = search.SearchConfig(
search_fields={
"field1": {"to": "f1"},
"field2": {"to": "f2"},
"field3": {"to": "f3"},
}
)
cleaned = s.clean(query)
assert cleaned["search_query"] == expected
@pytest.mark.parametrize(
"query,expected",
[
("", None),
("status:pending", Q(status="pending")),
('user:"silent bob"', Q(user__username__iexact="silent bob")),
(
"user:me status:pending",
Q(user__username__iexact="me") & Q(status="pending"),
),
],
)
def test_search_config_filter(query, expected):
s = search.SearchConfig(
filter_fields={
"user": {"to": "user__username__iexact"},
"status": {"to": "status"},
}
)
cleaned = s.clean(query)
assert cleaned["filter_query"] == expected
def test_apply():
cleaned = {
"filter_query": Q(batch__submitted_by__username__iexact="me"),
"search_query": Q(source="test"),
}
result = search.apply(music_models.ImportJob.objects.all(), cleaned)
assert str(result.query) == str(
music_models.ImportJob.objects.filter(
Q(batch__submitted_by__username__iexact="me"), Q(source="test")
).query
)
Implemented a basic but functionnal Github-like search on federated tracks list (#344)
Improved search on federated tracks list
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Having a powerful but easy-to-use search is important but difficult to achieve, especially
if you do not want to have a real complex search interface.
Github does a pretty good job with that, using a structured but simple query system
(See https://help.github.com/articles/searching-issues-and-pull-requests/#search-only-issues-or-pull-requests).
This release implements a limited but working subset of this query system. You can use it only on the federated
tracks list (/manage/federation/tracks) at the moment, but depending on feedback it will be rolled-out on other pages as well.
This is the type of query you can run:
- ``hello world``: search for "hello" and "world" in all the available fields
- ``hello in:artist`` search for results where artist name is "hello"
- ``spring in:artist,album`` search for results where artist name or album title contain "spring"
- ``artist:hello`` search for results where artist name equals "hello"
- ``artist:"System of a Down" domain:instance.funkwhale`` search for results where artist name equals "System of a Down" and inside "instance.funkwhale" library
FROM node:9
# needed to compile translations
RUN curl -L -o /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x /usr/local/bin/jq
EXPOSE 8080
WORKDIR /app/
ADD package.json .
......
......@@ -236,6 +236,7 @@ html, body {
.discrete.link {
color: rgba(0, 0, 0, 0.87);
cursor: pointer;
}
.floated.buttons .button ~ .dropdown {
......
......@@ -2,7 +2,7 @@
<div>
<div class="ui inline form">
<div class="fields">
<div class="ui field">
<div class="ui six wide field">
<label><translate>Search</translate></label>
<input type="text" v-model="search" :placeholder="labels.searchPlaceholder" />
</div>
......@@ -56,16 +56,16 @@
<span :title="scope.obj.title">{{ scope.obj.title|truncate(30) }}</span>
</td>
<td>
<span :title="scope.obj.artist_name">{{ scope.obj.artist_name|truncate(30) }}</span>
<span class="discrete link" @click="updateSearch({key: 'artist', value: scope.obj.artist_name})" :title="scope.obj.artist_name">{{ scope.obj.artist_name|truncate(30) }}</span>
</td>
<td>
<span :title="scope.obj.album_title">{{ scope.obj.album_title|truncate(20) }}</span>
<span class="discrete link" @click="updateSearch({key: 'album', value: scope.obj.album_title})" :title="scope.obj.album_title">{{ scope.obj.album_title|truncate(20) }}</span>
</td>
<td>
<human-date :date="scope.obj.published_date"></human-date>
</td>
<td v-if="showLibrary">
{{ scope.obj.library.actor.domain }}
<span class="discrete link" @click="updateSearch({key: 'domain', value: scope.obj.library.actor.domain})">{{ scope.obj.library.actor.domain }}</span>
</td>
</template>
</action-table>
......@@ -120,6 +120,12 @@ export default {
this.fetchData()
},
methods: {
updateSearch ({key, value}) {
if (value.indexOf(' ') > -1) {
value = `"${value}"`
}
this.search = `${key}:${value}`
},
fetchData () {
let params = _.merge({
'page': this.page,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment