glob support * / ? / [] in {} (#416)

* glob support `*` / `?` / `[]` in `{}`

* add tests for new fnmatch

* make lint happy

* make lint happy 2

* fix: fix static check (#417)

---------

Co-authored-by: liyang <[email protected]>
Co-authored-by: Hongyang Peng <[email protected]>

Author: 3 people

megfile/interfaces.py

@@ -194,7 +194,9 @@ def truncate(self, size: Optional[int] = None) -> int:
     def write(self, data: AnyStr) -> int:
         raise OSError("not writable")
 
-    def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
+    def writelines(  # pyre-ignore[14]  # pytype: disable=signature-mismatch
+        self, lines: Iterable[AnyStr]
+    ) -> None:
         raise OSError("not writable")
 
 
@@ -210,7 +212,9 @@ def write(self, data: AnyStr) -> int:
         Return the number of bytes or string written.
         """
 
-    def writelines(self, lines: Iterable[AnyStr]) -> None:  # pyre-ignore[14]
+    def writelines(  # pyre-ignore[14]  # pytype: disable=signature-mismatch
+        self, lines: Iterable[AnyStr]
+    ) -> None:
         """Write `lines` to the file.
 
         Note that newlines are not added.

megfile/lib/fnmatch.py

@@ -13,6 +13,7 @@
 """Compared with the standard library, syntax '{seq1,seq2}' is supported"""
 
 import functools
+import io
 import os
 import re
 from typing import Callable, List, Match, Optional
@@ -71,14 +72,9 @@ def _compat(res: str) -> str:
     return r"(?s:%s)\Z" % res
 
 
-def translate(pat: str) -> str:
-    """Translate a shell PATTERN to a regular expression.
-
-    There is no way to quote meta-characters.
-    """
-
+def _translate(pat: str, match_curly: bool) -> str:
     i, n = 0, len(pat)
-    res = ""
+    buf = io.StringIO()
     while i < n:
         c = pat[i]
         i = i + 1
@@ -90,14 +86,14 @@ def translate(pat: str) -> str:
                 if (j < n and pat[j] == "/") and (i <= 1 or pat[i - 2] == "/"):
                     # hit /**/ instead of /seq**/
                     j = j + 1
-                    res = res + r"(.*/)?"
+                    buf.write(r"(.*/)?")
                 else:
-                    res = res + r".*"
+                    buf.write(r".*")
             else:
-                res = res + r"[^/]*"
+                buf.write(r"[^/]*")
             i = j
         elif c == "?":
-            res = res + r"."
+            buf.write(r".")
         elif c == "[":
             j = i
             if j < n and pat[j] == "!":
@@ -107,28 +103,37 @@ def translate(pat: str) -> str:
             while j < n and pat[j] != "]":
                 j = j + 1
             if j >= n:
-                res = res + r"\["
+                buf.write(r"\[")
             else:
                 stuff = pat[i:j].replace("\\", r"\\")
                 i = j + 1
                 if stuff[0] == "!":
                     stuff = r"^" + stuff[1:]
                 elif stuff[0] == "^":
                     stuff = "\\" + stuff
-                res = r"%s[%s]" % (res, stuff)
-        elif c == "{":
+                buf.write(r"[%s]" % stuff)
+        elif match_curly and c == "{":
             j = i
             if j < n and pat[j] == "}":
                 j = j + 1
             while j < n and pat[j] != "}":
                 j = j + 1
             if j >= n:
-                res = res + r"\{"
+                buf.write(r"\{")
             else:
                 stuff = pat[i:j].replace("\\", r"\\")
-                stuff = r"|".join(map(re.escape, stuff.split(",")))  # pyre-ignore[6]
-                res = r"%s(%s)" % (res, stuff)
+                stuff = r"|".join(_translate(part, False) for part in stuff.split(","))
+                buf.write(r"(%s)" % stuff)
                 i = j + 1
         else:
-            res = res + re.escape(c)
-    return _compat(res)
+            buf.write(re.escape(c))
+    return buf.getvalue()
+
+
+def translate(pat: str) -> str:
+    """Translate a shell PATTERN to a regular expression.
+
+    There is no way to quote meta-characters.
+    """
+
+    return _compat(_translate(pat, True))

megfile/lib/shadow_handler.py

@@ -11,7 +11,9 @@ class BaseShadowHandler(RawIOBase):
     """ShadowHandler using RawIOBase's interface. (avoid type checking error)"""
 
 
-class ShadowHandler(Readable, Seekable, Writable, BaseShadowHandler):
+class ShadowHandler(  # pytype: disable=signature-mismatch
+    Readable, Seekable, Writable, BaseShadowHandler
+):
     """Create a File-Like Object, maintaining file pointer,
     to avoid misunderstanding the position when read / write / seek.
 

tests/lib/test_fnmatch.py

@@ -41,9 +41,9 @@ def test_translate():
 
     # weirdos
     assert fnmatch.translate("(a|b)") == fnmatch._compat(r"\(a\|b\)")
-    assert fnmatch.translate("{*,d}") == fnmatch._compat(r"(\*|d)")
-    assert fnmatch.translate("{**,d}") == fnmatch._compat(r"(\*\*|d)")
-    assert fnmatch.translate("{[abc],d}") == fnmatch._compat(r"(\[abc\]|d)")
+    assert fnmatch.translate("{*,d}") == fnmatch._compat(r"([^/]*|d)")
+    assert fnmatch.translate("{**,d}") == fnmatch._compat(r"(.*|d)")
+    assert fnmatch.translate("{[abc],d}") == fnmatch._compat(r"([abc]|d)")
     if sys.version_info > (3, 7):
         assert fnmatch.translate("{{a,b},d}") == fnmatch._compat(r"(\{a|b),d\}")
     else:
@@ -174,9 +174,60 @@ def test_filter():
 
     # weirdos
     assert fnmatch.filter(file_list, "(a|b)") == ["(a|b)"]
-    assert fnmatch.filter(file_list, "{*,d}") == ["d", "*"]
-    assert fnmatch.filter(file_list, "{**,d}") == ["d", "**"]
-    assert fnmatch.filter(file_list, "{[abc],d}") == ["d", "[abc]"]
+    assert fnmatch.filter(file_list, "{*,d}") == [
+        "a",
+        "b",
+        "d",
+        "*",
+        "**",
+        "[abc]",
+        "(a|b)",
+        "{a",
+        "b}",
+        "{a,d}",
+        "[",
+        "]",
+        "[]",
+        "[!]",
+        "{",
+        "}",
+        "{}",
+        "{,}",
+        "^",
+        "!",
+        "?",
+        ",",
+    ]
+    assert fnmatch.filter(file_list, "{**,d}") == [
+        "a",
+        "b",
+        "c/d",
+        "d",
+        "*",
+        "**",
+        "[abc]",
+        "(a|b)",
+        "{a",
+        "b}",
+        "{a,d}",
+        "[",
+        "]",
+        "[]",
+        "[!]",
+        "{",
+        "}",
+        "{}",
+        "{,}",
+        "^",
+        "!",
+        "?",
+        ",",
+    ]
+    assert fnmatch.filter(file_list, "{[abc],d}") == [
+        "a",
+        "b",
+        "d",
+    ]
     assert fnmatch.filter(file_list, "{{a,b},d}") == ["{a,d}"]
 
 

tests/lib/test_glob.py

@@ -324,13 +324,35 @@ def _glob_with_dironly():
     )
 
 
+def _glob_with_curly():
+    """
+    scenario: pathname with the curly braces('{}')
+    expectation: returns only contains pathname of files
+    """
+    assert_glob(
+        "/bucketForGlobTest/{1,2}/", ["/bucketForGlobTest/1/", "/bucketForGlobTest/2/"]
+    )
+
+    assert_glob("/bucketForGlobTest/{[2-4],[4-9]}/", ["/bucketForGlobTest/2/"])
+
+    assert_glob(
+        "/bucketForGlobTest/1/**/*.{json,msg}",
+        [
+            "/bucketForGlobTest/1/a/b/1.json",
+            "/bucketForGlobTest/1/a/b/c/1.json",
+            "/bucketForGlobTest/1/a/b/c/A.msg",
+        ],
+    )
+
+
 def test_glob(fs_setup):
     _glob_with_common_wildcard()
     _glob_with_recursive_pathname()
     _glob_with_same_file_and_folder()
     _glob_with_nested_pathname()
     _glob_with_not_exists_dir()
     _glob_with_dironly()
+    _glob_with_curly()
 
 
 def test_escape():