server/users: add searching prototype

server/szurubooru/api/base_api.py

@@ -5,7 +5,8 @@ import types
 def _bind_method(target, desired_method_name):
     actual_method = getattr(target, desired_method_name)
     def _wrapper_method(self, request, response, *args, **kwargs):
-        request.context.result = actual_method(request.context, *args, **kwargs)
+        request.context.result = actual_method(
+            request, request.context, *args, **kwargs)
     return types.MethodType(_wrapper_method, target)
 
 class BaseApi(object):

server/szurubooru/api/user_api.py

@@ -1,8 +1,9 @@
 ''' Users public API. '''
 
 import sqlalchemy
-from szurubooru.errors import IntegrityError, ValidationError
 from szurubooru.api.base_api import BaseApi
+from szurubooru.errors import IntegrityError, ValidationError
+from szurubooru.services.search import UserSearchConfig, SearchExecutor
 
 def _serialize_user(authenticated_user, user):
     ret = {
@@ -23,13 +24,23 @@ class UserListApi(BaseApi):
         super().__init__()
         self._auth_service = auth_service
         self._user_service = user_service
+        self._search_executor = SearchExecutor(UserSearchConfig())
 
-    def get(self, context):
+    def get(self, request, context):
         ''' Retrieves a list of users. '''
         self._auth_service.verify_privilege(context.user, 'users:list')
-        return {'message': 'Searching for users'}
+        query = request.get_param_as_string('query')
+        page = request.get_param_as_int('page', 1)
+        count, users = self._search_executor.execute(context.session, query, page)
+        return {
+            'query': query,
+            'page': page,
+            'page_size': self._search_executor.page_size,
+            'total': count,
+            'users': [_serialize_user(context.user, user) for user in users],
+        }
 
-    def post(self, context):
+    def post(self, request, context):
         ''' Creates a new user. '''
         self._auth_service.verify_privilege(context.user, 'users:create')
 
@@ -55,13 +66,13 @@ class UserDetailApi(BaseApi):
         self._auth_service = auth_service
         self._user_service = user_service
 
-    def get(self, context, user_name):
+    def get(self, request, context, user_name):
         ''' Retrieves an user. '''
         self._auth_service.verify_privilege(context.user, 'users:view')
         user = self._user_service.get_by_name(context.session, user_name)
         return {'user': _serialize_user(context.user, user)}
 
-    def put(self, context, user_name):
+    def put(self, request, context, user_name):
         ''' Updates an existing user. '''
         self._auth_service.verify_privilege(context.user, 'users:edit')
         return {'message': 'Updating user ' + user_name}

server/szurubooru/app.py

@@ -8,18 +8,35 @@ import szurubooru.api
 import szurubooru.config
 import szurubooru.middleware
 import szurubooru.services
+import szurubooru.services.search
 import szurubooru.util
 from szurubooru.errors import *
 
 class _CustomRequest(falcon.Request):
     context_type = szurubooru.util.dotdict
 
+    def get_param_as_string(self, name, required=False, store=None, default=None):
+        params = self._params
+        if name in params:
+            param = params[name]
+            if isinstance(param, list):
+                param = ','.join(param)
+            if store is not None:
+                store[name] = param
+            return param
+        if not required:
+            return default
+        raise falcon.HTTPMissingParam(name)
+
 def _on_auth_error(ex, request, response, params):
     raise falcon.HTTPForbidden('Authentication error', str(ex))
 
 def _on_validation_error(ex, request, response, params):
     raise falcon.HTTPBadRequest('Validation error', str(ex))
 
+def _on_search_error(ex, request, response, params):
+    raise falcon.HTTPBadRequest('Search error', str(ex))
+
 def _on_integrity_error(ex, request, response, params):
     raise falcon.HTTPConflict('Integrity violation', ex.args[0])
 
@@ -60,6 +77,7 @@ def create_app():
     app.add_error_handler(szurubooru.errors.AuthError, _on_auth_error)
     app.add_error_handler(szurubooru.errors.IntegrityError, _on_integrity_error)
     app.add_error_handler(szurubooru.errors.ValidationError, _on_validation_error)
+    app.add_error_handler(szurubooru.errors.SearchError, _on_search_error)
 
     app.add_route('/users/', user_list)
     app.add_route('/user/{user_name}', user)

server/szurubooru/errors.py

@@ -8,3 +8,6 @@ class IntegrityError(RuntimeError):
 
 class ValidationError(RuntimeError):
     ''' Validation error (e.g. trying to create user with invalid name) '''
+
+class SearchError(RuntimeError):
+    ''' Search error (e.g. trying to use special: where it doesn't make sense) '''

server/szurubooru/services/search/__init__.py

@@ -0,0 +1,2 @@
+from szurubooru.services.search.search_executor import SearchExecutor
+from szurubooru.services.search.user_search_config import UserSearchConfig

server/szurubooru/services/search/base_search_config.py

@@ -0,0 +1,98 @@
+''' Exports BaseSearchConfig. '''
+
+import sqlalchemy
+from szurubooru.errors import SearchError
+from szurubooru.services.search.criteria import *
+from szurubooru.util import parse_time_range
+
+def _apply_criterion_to_column(
+    column, query, criterion, allow_composite=True, allow_ranged=True):
+    ''' Decorates SQLAlchemy filter on given column using supplied criterion. '''
+    if isinstance(criterion, StringSearchCriterion):
+        filter = column == criterion.value
+        if criterion.negated:
+            filter = ~filter
+        return query.filter(filter)
+    elif isinstance(criterion, ArraySearchCriterion):
+        if not allow_composite:
+            raise SearchError(
+                'Composite token %r is invalid in this context.' % (criterion,))
+        filter = column.in_(criterion.values)
+        if criterion.negated:
+            filter = ~filter
+        return query.filter(filter)
+    elif isinstance(criterion, RangedSearchCriterion):
+        if not allow_ranged:
+            raise SearchError(
+                'Ranged token %r is invalid in this context.' % (criterion,))
+        filter = column.between(criterion.min_value, criterion.max_value)
+        if criterion.negated:
+            filter = ~filter
+        return query.filter(filter)
+    else:
+        raise RuntimeError('Invalid search type: %r.' % (criterion,))
+
+def _apply_date_criterion_to_column(column, query, criterion):
+    '''
+    Decorates SQLAlchemy filter on given column using supplied criterion.
+    Parses the datetime inside the criterion.
+    '''
+    if isinstance(criterion, StringSearchCriterion):
+        min_date, max_date = parse_time_range(criterion.value)
+        filter = column.between(min_date, max_date)
+        if criterion.negated:
+            filter = ~filter
+        return query.filter(filter)
+    elif isinstance(criterion, ArraySearchCriterion):
+        result = query
+        filter = sqlalchemy.sql.false()
+        for value in criterion.values:
+            min_date, max_date = parse_time_range(value)
+            filter = filter | column.between(min_date, max_date)
+        if criterion.negated:
+            filter = ~filter
+        return query.filter(filter)
+    elif isinstance(criterion, RangedSearchCriterion):
+        assert criterion.min_value or criterion.max_value
+        if criterion.min_value and criterion.max_value:
+            min_date = parse_time_range(criterion.min_value)[0]
+            max_date = parse_time_range(criterion.max_value)[1]
+            filter = column.between(min_date, max_date)
+        elif criterion.min_value:
+            min_date = parse_time_range(criterion.min_value)[0]
+            filter = column >= min_date
+        elif criterion.max_value:
+            max_date = parse_time_range(criterion.max_value)[1]
+            filter = column <= max_date
+        if criterion.negated:
+            filter = ~filter
+        return query.filter(filter)
+
+class BaseSearchConfig(object):
+    def create_query(self, session):
+        raise NotImplementedError()
+
+    @property
+    def anonymous_filter(self):
+        raise NotImplementedError()
+
+    @property
+    def special_filters(self):
+        raise NotImplementedError()
+
+    @property
+    def named_filters(self):
+        raise NotImplementedError()
+
+    @property
+    def order_columns(self):
+        raise NotImplementedError()
+
+    def _create_basic_filter(
+            self, column, allow_composite=True, allow_ranged=True):
+        return lambda query, criterion: _apply_criterion_to_column(
+            column, query, criterion, allow_composite, allow_ranged)
+
+    def _create_date_filter(self, column):
+        return lambda query, criterion: _apply_date_criterion_to_column(
+            column, query, criterion)

server/szurubooru/services/search/criteria.py

@@ -0,0 +1,23 @@
+class BaseSearchCriterion(object):
+    def __init__(self, original_text, negated):
+        self.original_text = original_text
+        self.negated = negated
+
+    def __repr__(self):
+        return self.original_text
+
+class RangedSearchCriterion(BaseSearchCriterion):
+    def __init__(self, original_text, negated, min_value, max_value):
+        super().__init__(original_text, negated)
+        self.min_value = min_value
+        self.max_value = max_value
+
+class StringSearchCriterion(BaseSearchCriterion):
+    def __init__(self, original_text, negated, value):
+        super().__init__(original_text, negated)
+        self.value = value
+
+class ArraySearchCriterion(BaseSearchCriterion):
+    def __init__(self, original_text, negated, values):
+        super().__init__(original_text, negated)
+        self.values = values

server/szurubooru/services/search/search_executor.py

@@ -0,0 +1,120 @@
+''' Exports SearchExecutor. '''
+
+import re
+import sqlalchemy
+from szurubooru.errors import SearchError
+from szurubooru.services.search.criteria import *
+
+class SearchExecutor(object):
+    ORDER_DESC = 1
+    ORDER_ASC = 2
+
+    '''
+    Class for search parsing and execution. Handles plaintext parsing and
+    delegates sqlalchemy filter decoration to SearchConfig instances.
+    '''
+
+    def __init__(self, search_config):
+        self.page_size = 100
+        self._search_config = search_config
+
+    def execute(self, session, query_text, page):
+        '''
+        Parse input and return tuple containing total record count and filtered
+        entities.
+        '''
+        filter_query = self._prepare(session, query_text)
+        entities = filter_query \
+            .offset((page - 1) * self.page_size).limit(self.page_size).all()
+        count_query = filter_query.statement \
+            .with_only_columns([sqlalchemy.func.count()]).order_by(None)
+        count = filter_query.session.execute(count_query).scalar()
+        return (count, entities)
+
+    def _prepare(self, session, query_text):
+        ''' Parse input and return SQLAlchemy query. '''
+        query = self._search_config.create_query(session)
+        for token in re.split(r'\s+', (query_text or '').lower()):
+            if not token:
+                continue
+            negated = False
+            while token[0] == '-':
+                token = token[1:]
+                negated = not negated
+
+            if ':' in token:
+                key, value = token.split(':', 2)
+                query = self._handle_key_value(query, key, value, negated)
+            else:
+                query = self._handle_anonymous(
+                    query, self._create_criterion(token, negated))
+
+        return query
+
+    def _handle_key_value(self, query, key, value, negated):
+        if key == 'order':
+            if value.count(',') == 0:
+                order = self.ORDER_ASC
+            elif value.count(',') == 1:
+                value, order_str = value.split(',')
+                if order_str == 'asc':
+                    order = self.ORDER_ASC
+                elif order_str == 'desc':
+                    order = self.ORDER_DESC
+                else:
+                    raise SearchError('Unknown search direction: %r.' % order_str)
+            else:
+                raise SearchError('Too many commas in order search token.')
+            if negated:
+                if order == self.ORDER_DESC:
+                    order = self.ORDER_ASC
+                else:
+                    order = self.ORDER_DESC
+            return self._handle_order(query, value, order)
+        elif key == 'special':
+            return self._handle_special(query, value, negated)
+        else:
+            return self._handle_named(
+                query, key, self._create_criterion(value, negated))
+
+    def _handle_anonymous(self, query, criterion):
+        if not self._search_config.anonymous_filter:
+            raise SearchError(
+                'Anonymous tokens are not valid in this context.')
+        return self._search_config.anonymous_filter(query, criterion)
+
+    def _handle_named(self, query, key, criterion):
+        if key in self._search_config.named_filters:
+            return self._search_config.named_filters[key](query, criterion)
+        raise SearchError(
+            'Unknown named token: %r. Available named tokens: %r.' % (
+                key, list(self._search_config.named_filters.keys())))
+
+    def _handle_special(self, query, value, negated):
+        if value in self._search_config.special_filters:
+            return self._search_config.special_filters[value](query, criterion)
+        raise SearchError(
+            'Unknown special token: %r. Available special tokens: %r.' % (
+                value, list(self._search_config.special_filters.keys())))
+
+    def _handle_order(self, query, value, order):
+        if value in self._search_config.order_columns:
+            column = self._search_config.order_columns[value]
+            if order == self.ORDER_ASC:
+                column = column.asc()
+            else:
+                column = column.desc()
+            return query.order_by(column)
+        raise SearchError(
+            'Unknown search order: %r. Available search orders: %r.' % (
+                value, list(self._search_config.order_columns.keys())))
+
+    def _create_criterion(self, value, negated):
+        if '..' in value:
+            low, high = value.split('..')
+            if not low and not high:
+                raise SearchError('Empty ranged value')
+            return RangedSearchCriterion(value, negated, low, high)
+        if ',' in value:
+            return ArraySearchCriterion(value, negated, value.split(','))
+        return StringSearchCriterion(value, negated, value)

server/szurubooru/services/search/user_search_config.py

@@ -0,0 +1,45 @@
+''' Exports UserSearchConfig. '''
+
+from sqlalchemy.sql.expression import func
+from szurubooru.errors import SearchError
+from szurubooru.model import User
+from szurubooru.services.search.base_search_config import BaseSearchConfig
+
+class UserSearchConfig(BaseSearchConfig):
+    ''' Executes searches related to the users. '''
+
+    def create_query(self, session):
+        return session.query(User)
+
+    @property
+    def anonymous_filter(self):
+        return self._create_basic_filter(User.name, allow_ranged=False)
+
+    @property
+    def special_filters(self):
+        return {}
+
+    @property
+    def named_filters(self):
+        return {
+            'name': self._create_basic_filter(User.name, allow_ranged=False),
+            'creation_date': self._create_date_filter(User.creation_time),
+            'creation_time': self._create_date_filter(User.creation_time),
+            'last_login_date': self._create_date_filter(User.last_login_time),
+            'last_login_time': self._create_date_filter(User.last_login_time),
+            'login_date': self._create_date_filter(User.last_login_time),
+            'login_time': self._create_date_filter(User.last_login_time),
+        }
+
+    @property
+    def order_columns(self):
+        return {
+            'random': func.random(),
+            'name': User.name,
+            'creation_date': User.creation_time,
+            'creation_time': User.creation_time,
+            'last_login_date': User.last_login_time,
+            'last_login_time': User.last_login_time,
+            'login_date': User.last_login_time,
+            'login_time': User.last_login_time,
+        }

server/szurubooru/tests/database_test_case.py

@@ -0,0 +1,11 @@
+import unittest
+import sqlalchemy
+from szurubooru.model import Base
+
+class DatabaseTestCase(unittest.TestCase):
+    def setUp(self):
+        engine = sqlalchemy.create_engine('sqlite:///:memory:')
+        session_maker = sqlalchemy.orm.sessionmaker(bind=engine)
+        self.session = sqlalchemy.orm.scoped_session(session_maker)
+        Base.query = self.session.query_property()
+        Base.metadata.create_all(bind=engine)

server/szurubooru/tests/services/search/test_user_search_config.py

@@ -0,0 +1,164 @@
+from datetime import datetime
+from szurubooru.errors import SearchError
+from szurubooru.model.user import User
+from szurubooru.services.search.search_executor import SearchExecutor
+from szurubooru.services.search.user_search_config import UserSearchConfig
+from szurubooru.tests.database_test_case import DatabaseTestCase
+
+class TestUserSearchExecutor(DatabaseTestCase):
+    def setUp(self):
+        super().setUp()
+        self.search_config = UserSearchConfig()
+        self.executor = SearchExecutor(self.search_config)
+
+    def _create_user(self, name):
+        user = User()
+        user.name = name
+        user.password = 'dummy'
+        user.password_salt = 'dummy'
+        user.password_hash = 'dummy'
+        user.email = 'dummy'
+        user.access_rank = 'dummy'
+        user.creation_time = datetime.now()
+        user.avatar_style = User.AVATAR_GRAVATAR
+        return user
+
+    def _test(self, query, page, expected_count, expected_user_names):
+        count, users = self.executor.execute(self.session, query, page)
+        self.assertEqual(count, expected_count)
+        self.assertEqual([u.name for u in users], expected_user_names)
+
+    def test_filter_by_creation_time(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2])
+        for alias in ['creation_time', 'creation_date']:
+            self._test('%s:2014' % alias, 1, 1, ['u1'])
+
+    def test_filter_by_negated_creation_time(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2])
+        for alias in ['creation_time', 'creation_date']:
+            self._test('-%s:2014' % alias, 1, 1, ['u2'])
+
+    def test_filter_by_ranged_creation_time(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user3 = self._create_user('u3')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2014, 6, 1)
+        user3.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2, user3])
+        for alias in ['creation_time', 'creation_date']:
+            self._test('%s:2014..2014-06' % alias, 1, 2, ['u1', 'u2'])
+            self._test('%s:2014-06..2015-01-01' % alias, 1, 2, ['u2', 'u3'])
+            self._test('%s:2014-06..' % alias, 1, 2, ['u2', 'u3'])
+            self._test('%s:..2014-06' % alias, 1, 2, ['u1', 'u2'])
+            self.assertRaises(
+                SearchError, self.executor.execute, self.session, '%s:..', 1)
+
+    def test_filter_by_negated_ranged_creation_time(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user3 = self._create_user('u3')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2014, 6, 1)
+        user3.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2, user3])
+        for alias in ['creation_time', 'creation_date']:
+            self._test('-%s:2014..2014-06' % alias, 1, 1, ['u3'])
+            self._test('-%s:2014-06..2015-01-01' % alias, 1, 1, ['u1'])
+
+    def test_filter_by_composite_creation_time(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user3 = self._create_user('u3')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2014, 6, 1)
+        user3.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2, user3])
+        for alias in ['creation_time', 'creation_date']:
+            self._test('%s:2014-01,2015' % alias, 1, 2, ['u1', 'u3'])
+
+    def test_filter_by_negated_composite_creation_time(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user3 = self._create_user('u3')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2014, 6, 1)
+        user3.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2, user3])
+        for alias in ['creation_time', 'creation_date']:
+            self._test('-%s:2014-01,2015' % alias, 1, 1, ['u2'])
+
+    def test_filter_by_name(self):
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self._test('name:u1', 1, 1, ['u1'])
+        self._test('name:u2', 1, 1, ['u2'])
+
+    def test_filter_by_negated_name(self):
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self._test('-name:u1', 1, 1, ['u2'])
+        self._test('-name:u2', 1, 1, ['u1'])
+
+    def test_filter_by_composite_name(self):
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self.session.add(self._create_user('u3'))
+        self._test('name:u1,u2', 1, 2, ['u1', 'u2'])
+
+    def test_filter_by_ranged_name(self):
+        self.assertRaises(
+            SearchError, self.executor.execute, self.session, 'name:u1..u2', 1)
+
+    def test_paging(self):
+        self.executor.page_size = 1
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self._test('', 1, 2, ['u1'])
+        self._test('', 2, 2, ['u2'])
+
+    def test_order_by_name(self):
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self._test('order:name', 1, 2, ['u1', 'u2'])
+        self._test('-order:name', 1, 2, ['u2', 'u1'])
+        self._test('order:name,asc', 1, 2, ['u1', 'u2'])
+        self._test('order:name,desc', 1, 2, ['u2', 'u1'])
+        self._test('-order:name,asc', 1, 2, ['u2', 'u1'])
+        self._test('-order:name,desc', 1, 2, ['u1', 'u2'])
+
+    def test_anonymous(self):
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self._test('u1', 1, 1, ['u1'])
+        self._test('u2', 1, 1, ['u2'])
+
+    def test_negated_anonymous(self):
+        self.session.add(self._create_user('u1'))
+        self.session.add(self._create_user('u2'))
+        self._test('-u1', 1, 1, ['u2'])
+        self._test('-u2', 1, 1, ['u1'])
+
+    def test_combining(self):
+        user1 = self._create_user('u1')
+        user2 = self._create_user('u2')
+        user3 = self._create_user('u3')
+        user1.creation_time = datetime(2014, 1, 1)
+        user2.creation_time = datetime(2014, 6, 1)
+        user3.creation_time = datetime(2015, 1, 1)
+        self.session.add_all([user1, user2, user3])
+        self._test('creation_time:2014 u1', 1, 1, ['u1'])
+        self._test('creation_time:2014 u2', 1, 1, ['u2'])
+        self._test('creation_time:2016 u2', 1, 0, [])
+
+    def test_special(self):
+        self.assertRaises(
+            SearchError, self.executor.execute, self.session, 'special:-', 1)