Merge pull request #5 from 1kbgz/tkp/reg

Add python protocol handler

Author: timkpaine

fsspec_python/__init__.py

@@ -1,3 +1,4 @@
+from .fs import *
 from .importer import *
 from .open import *
 

fsspec_python/fs.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import inspect
+
+from fsspec import AbstractFileSystem, filesystem
+
+from .importer import install_importer, uninstall_importer
+
+__all__ = ("PythonFileSystem",)
+
+
+class PythonFileSystem(AbstractFileSystem):
+    """Python import filesystem"""
+
+    def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs):
+        """
+        Args:
+            target_protocol: str (optional) Target filesystem protocol. Provide either this or ``fs``.
+            target_options: dict or None Passed to the instantiation of the FS, if fs is None.
+            fs: filesystem instance The target filesystem to run against. Provide this or ``protocol``.
+        """
+        super().__init__(**kwargs)
+        if fs is None and target_protocol is None:
+            raise ValueError("Please provide filesystem instance(fs) or target_protocol")
+        if not (fs is None) ^ (target_protocol is None):
+            raise ValueError("Both filesystems (fs) and target_protocol may not be both given.")
+
+        target_options = target_options or {}
+        self.target_protocol = (
+            target_protocol if isinstance(target_protocol, str) else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
+        )
+        self.fs = fs if fs is not None else filesystem(target_protocol, **target_options)
+
+        if target_protocol and kwargs.get("fo"):
+            install_importer(f"{self.target_protocol}://{kwargs['fo']}", **target_options)
+        else:
+            install_importer(self.fs, **target_options, **kwargs)
+
+    def close(self):
+        uninstall_importer(self.target_protocol)
+        self.fs.close()
+        super().close()
+
+    def __getattribute__(self, item):
+        if item in {
+            "__init__",
+            "__getattribute__",
+            "__reduce__",
+            "_make_local_details",
+            "open",
+            "cat",
+            "cat_file",
+            "_cat_file",
+            "cat_ranges",
+            "_cat_ranges",
+            "get",
+            "read_block",
+            "tail",
+            "head",
+            "info",
+            "ls",
+            "exists",
+            "isfile",
+            "isdir",
+            "_check_file",
+            "_check_cache",
+            "_mkcache",
+            "clear_cache",
+            "clear_expired_cache",
+            "pop_from_cache",
+            "local_file",
+            "_paths_from_path",
+            "get_mapper",
+            "open_many",
+            "commit_many",
+            "hash_name",
+            "__hash__",
+            "__eq__",
+            "to_json",
+            "to_dict",
+            "cache_size",
+            "pipe_file",
+            "pipe",
+            "start_transaction",
+            "end_transaction",
+        }:
+            return object.__getattribute__(self, item)
+        d = object.__getattribute__(self, "__dict__")
+        fs = d.get("fs", None)  # fs is not immediately defined
+        if item in d:
+            return d[item]
+        if fs is not None:
+            if item in fs.__dict__:
+                # attribute of instance
+                return fs.__dict__[item]
+            # attributed belonging to the target filesystem
+            cls = type(fs)
+            m = getattr(cls, item)
+            if (inspect.isfunction(m) or inspect.isdatadescriptor(m)) and (not hasattr(m, "__self__") or m.__self__ is None):
+                # instance method
+                return m.__get__(fs, cls)
+            return m  # class method or attribute
+        # attributes of the superclass, while target is being set up
+        return super().__getattribute__(item)

fsspec_python/importer.py

@@ -5,7 +5,7 @@
 from importlib.machinery import SOURCE_SUFFIXES, ModuleSpec
 from os.path import join
 from types import ModuleType
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict
 
 from fsspec import url_to_fs
 from fsspec.implementations.local import AbstractFileSystem
@@ -26,7 +26,11 @@ class FSSpecImportFinder(MetaPathFinder):
     def __init__(self, fsspec: str, **fsspec_args: str) -> None:
         self.fsspec_fs: AbstractFileSystem
         self.root: str
-        self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args)
+        if isinstance(fsspec, AbstractFileSystem):
+            self.fsspec_fs = fsspec
+            self.root = fsspec_args.get("fo", fsspec.root_marker)
+        else:
+            self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args)
         self.remote_modules: dict[str, str] = {}
 
     def find_spec(self, fullname: str, path: Sequence[str | bytes] | None, target: ModuleType | None = None) -> ModuleSpec | None:
@@ -49,7 +53,7 @@ def unload(self) -> None:
 
 
 # Singleton for use elsewhere
-_finder: FSSpecImportFinder = None
+_finders: Dict[str, FSSpecImportFinder] = {}
 
 
 class FSSpecImportLoader(SourceLoader):
@@ -77,18 +81,30 @@ def install_importer(fsspec: str, **fsspec_args: str) -> FSSpecImportFinder:
         fsspec: fsspec filesystem string
     Returns: The finder instance that was installed.
     """
-    global _finder
-    if _finder is None:
-        _finder = FSSpecImportFinder(fsspec, **fsspec_args)
+    if isinstance(fsspec, AbstractFileSystem):
+        # Reassemble fsspec and args
+        fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec.root_marker}"
+        fsspec_args = fsspec_args or {}
 
-    sys.meta_path.insert(0, _finder)
-    return _finder
+    global _finders
+    if fsspec in _finders:
+        return _finders[fsspec]
+    _finders[fsspec] = FSSpecImportFinder(fsspec, **fsspec_args)
+    sys.meta_path.insert(0, _finders[fsspec])
+    return _finders[fsspec]
 
 
-def uninstall_importer() -> None:
+def uninstall_importer(fsspec: str = "") -> None:
     """Uninstall the fsspec importer."""
-    global _finder
-    if _finder is not None and _finder in sys.meta_path:
-        _finder.unload()
-        sys.meta_path.remove(_finder)
-        _finder = None
+    global _finders
+    if not fsspec:
+        # clear last
+        if not _finders:
+            return
+        fsspec = list(_finders.keys())[-1]
+    if fsspec in _finders:
+        finder = _finders[fsspec]
+        del _finders[fsspec]
+        if finder in sys.meta_path:
+            finder.unload()
+            sys.meta_path.remove(finder)

fsspec_python/open.py

@@ -18,12 +18,8 @@ def install_open_hook(fsspec: str, **fsspec_args):
     root: str
     fsspec_fs, root = url_to_fs(fsspec, **fsspec_args)
 
-    print(root)
-
     def open_from_fsspec(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None):
         filename = join(root, file)
-        print(filename)
-
         if ("w" in mode or "a" in mode or "x" in mode) or fsspec_fs.exists(filename):
             return fsspec_fs.open(filename, mode=mode, encoding=encoding, errors=errors, newline=newline)
         return _original_open(file, mode, buffering, encoding, errors, newline, closefd, opener)

fsspec_python/tests/conftest.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 
 import pytest
+from fsspec import open
 
 from fsspec_python import install_importer, uninstall_importer
 
@@ -29,3 +30,10 @@ def open_hook():
     install_open_hook(f"file://{Path(__file__).parent}/dump/")
     yield
     uninstall_open_hook()
+
+
+@pytest.fixture()
+def fs_importer():
+    fs = open(f"python::file://{Path(__file__).parent}/local2")
+    yield fs
+    fs.close()

fsspec_python/tests/local2/my_local_file2.py

@@ -0,0 +1,2 @@
+def baz():
+    return "This is a local file."

fsspec_python/tests/test_fs.py

@@ -0,0 +1,8 @@
+class TestFs:
+    def test_fs(self, fs_importer):
+        fs = fs_importer
+        import my_local_file2
+
+        assert my_local_file2.baz() == "This is a local file."
+
+        fs.close()

pyproject.toml

@@ -30,6 +30,9 @@ dependencies = [
     "fsspec",
 ]
 
+[project.entry-points."fsspec.specs"]
+python = "fsspec_python.fs.PythonFileSystem"
+
 [project.optional-dependencies]
 develop = [
     "build",