add full support for regular expressions

Author: Bodyhealer

README.rst

@@ -133,6 +133,7 @@ parenthesis. DjangoQL is case-sensitive.
 - comparison operators: ``=``, ``!=``, ``<``, ``<=``, ``>``, ``>=``
   - work as you expect. ``~`` and ``!~`` - test whether or not a string contains
   a substring (translated into ``__icontains``);
+- ``regex`` - find based on regex (translated into ``__iregex``);
 - test a value vs. list: ``in``, ``not in``. Example:
   ``pk in (2, 3)``.
 

djangoql/lexer.py

@@ -53,7 +53,7 @@ def find_column(self, t):
 
     re_escaped_char = r'\\[\"\\/bfnrt]'
     re_escaped_unicode = r'\\u[0-9A-Fa-f]{4}'
-    re_string_char = r'[^\"\\' + re_line_terminators + u']'
+    re_string_char = r'[^\"' + re_line_terminators + u']'
 
     re_int_value = r'(-?0|-?[1-9][0-9]*)'
     re_fraction_part = r'\.[0-9]+'
@@ -82,6 +82,7 @@ def find_column(self, t):
         'LESS_EQUAL',
         'CONTAINS',
         'NOT_CONTAINS',
+        'REGEX',
     ]
 
     t_COMMA = ','
@@ -135,6 +136,10 @@ def t_NOT(self, t):
     def t_IN(self, t):
         return t
 
+    @TOKEN('regex' + not_followed_by_name)
+    def t_REGEX(self, t):
+        return t
+
     @TOKEN('True' + not_followed_by_name)
     def t_TRUE(self, t):
         return t

djangoql/parser.py

@@ -93,6 +93,7 @@ def p_comparison_string(self, p):
         comparison_string : comparison_equality
                           | comparison_greater_less
                           | comparison_contains
+                          | comparison_regex
         """
         p[0] = p[1]
 
@@ -119,6 +120,12 @@ def p_comparison_contains(self, p):
         """
         p[0] = Comparison(operator=p[1])
 
+    def p_comparison_regex(self, p):
+        """
+        comparison_regex : REGEX
+        """
+        p[0] = Comparison(operator=p[1])
+
     def p_comparison_in_list(self, p):
         """
         comparison_in_list : IN

djangoql/schema.py

@@ -101,6 +101,7 @@ def get_operator(self, operator):
             '<': '__lt',
             '<=': '__lte',
             '~': '__icontains',
+            'regex': '__iregex',
             'in': '__in',
         }.get(operator)
         if op is not None:
@@ -124,11 +125,12 @@ def get_lookup(self, path, operator, value):
             be ['author', 'groups']. 'name' is not included, because it's the
             current field instance itself.
         :param operator: a string with comparison operator. It could be one of
-            the following: '=', '!=', '>', '>=', '<', '<=', '~', '!~', 'in',
-            'not in'. Depending on the field type, some operators may be
-            excluded. '~' and '!~' can be applied to StrField only and aren't
-            allowed for any other fields. BoolField can't be used with less or
-            greater operators, '>', '>=', '<' and '<=' are excluded for it.
+            the following: '=', '!=', '>', '>=', '<', '<=', '~', '!~', 'regex',
+            'in', 'not in'. Depending on the field type, some operators
+            may be excluded. '~' and '!~' can be applied to StrField only and
+            aren't allowed for any other fields. BoolField can't be used with
+            less or greater operators, '>', '>=', '<' and '<=' are excluded
+            for it.
         :param value: value passed for comparison
         :return: Q-object
         """
@@ -277,7 +279,7 @@ def get_lookup(self, path, operator, value):
         # and resulting comparison would look like
         #       'created LIKE %2017-01-30 00:00:00%'
         # which is not what we want for this case.
-        val = value if operator in ('~', '!~') else self.get_lookup_value(value)
+        val = value if operator in ('~', '!~', 'regex') else self.get_lookup_value(value)
 
         q = models.Q(**{'%s%s' % (search, op): val})
         return ~q if invert else q

djangoql/static/djangoql/js/completion.js

@@ -87,6 +87,9 @@
   lexer.addRule(new RegExp('in' + reNotFollowedByName), function (l) {
     return token('IN', l);
   });
+  lexer.addRule(new RegExp('regex' + reNotFollowedByName), function (l) {
+    return token('REGEX', l);
+  });
   lexer.addRule(new RegExp('True' + reNotFollowedByName), function (l) {
     return token('TRUE', l);
   });
@@ -871,8 +874,9 @@
         }
       } else if (lastToken && whitespace &&
           nextToLastToken && nextToLastToken.name === 'NAME' &&
-          ['EQUALS', 'NOT_EQUALS', 'CONTAINS', 'NOT_CONTAINS', 'GREATER_EQUAL',
-            'GREATER', 'LESS_EQUAL', 'LESS'].indexOf(lastToken.name) >= 0) {
+          ['EQUALS', 'NOT_EQUALS', 'CONTAINS', 'NOT_CONTAINS', 'REGEX',
+            'GREATER_EQUAL', 'GREATER', 'LESS_EQUAL', 'LESS'
+          ].indexOf(lastToken.name) >= 0) {
         resolvedName = this.resolveName(nextToLastToken.value);
         if (resolvedName.model) {
           scope = 'value';
@@ -1090,6 +1094,7 @@
             if (['str', 'date', 'datetime'].indexOf(field.type) >= 0) {
               suggestions.push(['~', 'contains']);
               suggestions.push(['!~', 'does not contain']);
+              suggestions.push(['regex', 'regex']);
               snippetAfter = ' "|"';
             } else if (field.options) {
               snippetAfter = ' "|"';

djangoql/templates/djangoql/syntax_help.html

@@ -242,6 +242,11 @@ <h2 id="comparison-operators">Comparison operators</h2>
             <td>does not contain a substring</td>
             <td>username !~ "test"</td>
           </tr>
+          <tr>
+            <td>regex</td>
+            <td>find based on regex</td>
+            <td>email regex "\w+@\w+\.com"</td>
+          </tr>
           <tr>
             <td>&gt;</td>
             <td>greater</td>

js_tests/tests.js

@@ -137,7 +137,8 @@ describe('DjangoQL completion', function () {
         token('LESS', '<'),
         token('LESS_EQUAL', '<='),
         token('CONTAINS', '~'),
-        token('NOT_CONTAINS', '!~')
+        token('NOT_CONTAINS', '!~'),
+        token('REGEX', 'regex')
       ];
       djangoQL.lexer.setInput('() ., = != >\t >= < <= ~ !~');
       tokens.forEach(function (t) {
@@ -155,7 +156,7 @@ describe('DjangoQL completion', function () {
     });
 
     it('should recognize reserved words', function () {
-      var words = ['True', 'False', 'None', 'or', 'and', 'in'];
+      var words = ['True', 'False', 'None', 'or', 'and', 'in', 'regex'];
       djangoQL.lexer.setInput(words.join(' '));
       words.forEach(function (word) {
         expect(djangoQL.lexer.lex()).to.eql(token(word.toUpperCase(), word));

test_project/core/tests/test_lexer.py

@@ -54,7 +54,7 @@ def test_entity_props(self):
             pass
 
     def test_reserved_words(self):
-        reserved = ('True', 'False', 'None', 'or', 'and', 'in')
+        reserved = ('True', 'False', 'None', 'or', 'and', 'in', 'regex')
         for word in reserved:
             self.assert_output(self.lexer.input(word), [(word.upper(), word)])
         # A word made of reserved words should be treated as a name

test_project/core/tests/test_schema.py

@@ -142,6 +142,9 @@ def test_invalid_config(self):
     def test_validation_pass(self):
         samples = [
             'first_name = "Lolita"',
+            'first_name regex "^Lol"',
+            'first_name regex "ita$"',
+            r'first_name regex "\w+i\w+a"',
             'groups.id < 42',
             'groups = None',  # that's ok to compare a model to None
             'groups != None',