diff --git a/fsspec_python/__init__.py b/fsspec_python/__init__.py index a6d47e5..bf98e41 100644 --- a/fsspec_python/__init__.py +++ b/fsspec_python/__init__.py @@ -1,3 +1,4 @@ +from .fs import * from .importer import * from .open import * diff --git a/fsspec_python/fs.py b/fsspec_python/fs.py new file mode 100644 index 0000000..2968e05 --- /dev/null +++ b/fsspec_python/fs.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import inspect + +from fsspec import AbstractFileSystem, filesystem + +from .importer import install_importer, uninstall_importer + +__all__ = ("PythonFileSystem",) + + +class PythonFileSystem(AbstractFileSystem): + """Python import filesystem""" + + def __init__(self, target_protocol=None, target_options=None, fs=None, **kwargs): + """ + Args: + target_protocol: str (optional) Target filesystem protocol. Provide either this or ``fs``. + target_options: dict or None Passed to the instantiation of the FS, if fs is None. + fs: filesystem instance The target filesystem to run against. Provide this or ``protocol``. + """ + super().__init__(**kwargs) + if fs is None and target_protocol is None: + raise ValueError("Please provide filesystem instance(fs) or target_protocol") + if not (fs is None) ^ (target_protocol is None): + raise ValueError("Both filesystems (fs) and target_protocol may not be both given.") + + target_options = target_options or {} + self.target_protocol = ( + target_protocol if isinstance(target_protocol, str) else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]) + ) + self.fs = fs if fs is not None else filesystem(target_protocol, **target_options) + + if target_protocol and kwargs.get("fo"): + install_importer(f"{self.target_protocol}://{kwargs['fo']}", **target_options) + else: + install_importer(self.fs, **target_options, **kwargs) + + def close(self): + uninstall_importer(self.target_protocol) + self.fs.close() + super().close() + + def __getattribute__(self, item): + if item in { + "__init__", + "__getattribute__", + "__reduce__", + "_make_local_details", + "open", + "cat", + "cat_file", + "_cat_file", + "cat_ranges", + "_cat_ranges", + "get", + "read_block", + "tail", + "head", + "info", + "ls", + "exists", + "isfile", + "isdir", + "_check_file", + "_check_cache", + "_mkcache", + "clear_cache", + "clear_expired_cache", + "pop_from_cache", + "local_file", + "_paths_from_path", + "get_mapper", + "open_many", + "commit_many", + "hash_name", + "__hash__", + "__eq__", + "to_json", + "to_dict", + "cache_size", + "pipe_file", + "pipe", + "start_transaction", + "end_transaction", + }: + return object.__getattribute__(self, item) + d = object.__getattribute__(self, "__dict__") + fs = d.get("fs", None) # fs is not immediately defined + if item in d: + return d[item] + if fs is not None: + if item in fs.__dict__: + # attribute of instance + return fs.__dict__[item] + # attributed belonging to the target filesystem + cls = type(fs) + m = getattr(cls, item) + if (inspect.isfunction(m) or inspect.isdatadescriptor(m)) and (not hasattr(m, "__self__") or m.__self__ is None): + # instance method + return m.__get__(fs, cls) + return m # class method or attribute + # attributes of the superclass, while target is being set up + return super().__getattribute__(item) diff --git a/fsspec_python/importer.py b/fsspec_python/importer.py index 4ff9200..724fbad 100644 --- a/fsspec_python/importer.py +++ b/fsspec_python/importer.py @@ -5,7 +5,7 @@ from importlib.machinery import SOURCE_SUFFIXES, ModuleSpec from os.path import join from types import ModuleType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict from fsspec import url_to_fs from fsspec.implementations.local import AbstractFileSystem @@ -26,7 +26,11 @@ class FSSpecImportFinder(MetaPathFinder): def __init__(self, fsspec: str, **fsspec_args: str) -> None: self.fsspec_fs: AbstractFileSystem self.root: str - self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args) + if isinstance(fsspec, AbstractFileSystem): + self.fsspec_fs = fsspec + self.root = fsspec_args.get("fo", fsspec.root_marker) + else: + self.fsspec_fs, self.root = url_to_fs(fsspec, **fsspec_args) self.remote_modules: dict[str, str] = {} def find_spec(self, fullname: str, path: Sequence[str | bytes] | None, target: ModuleType | None = None) -> ModuleSpec | None: @@ -49,7 +53,7 @@ def unload(self) -> None: # Singleton for use elsewhere -_finder: FSSpecImportFinder = None +_finders: Dict[str, FSSpecImportFinder] = {} class FSSpecImportLoader(SourceLoader): @@ -77,18 +81,30 @@ def install_importer(fsspec: str, **fsspec_args: str) -> FSSpecImportFinder: fsspec: fsspec filesystem string Returns: The finder instance that was installed. """ - global _finder - if _finder is None: - _finder = FSSpecImportFinder(fsspec, **fsspec_args) + if isinstance(fsspec, AbstractFileSystem): + # Reassemble fsspec and args + fsspec = f"{fsspec.protocol if isinstance(fsspec.protocol, str) else fsspec.protocol[0]}://{fsspec.root_marker}" + fsspec_args = fsspec_args or {} - sys.meta_path.insert(0, _finder) - return _finder + global _finders + if fsspec in _finders: + return _finders[fsspec] + _finders[fsspec] = FSSpecImportFinder(fsspec, **fsspec_args) + sys.meta_path.insert(0, _finders[fsspec]) + return _finders[fsspec] -def uninstall_importer() -> None: +def uninstall_importer(fsspec: str = "") -> None: """Uninstall the fsspec importer.""" - global _finder - if _finder is not None and _finder in sys.meta_path: - _finder.unload() - sys.meta_path.remove(_finder) - _finder = None + global _finders + if not fsspec: + # clear last + if not _finders: + return + fsspec = list(_finders.keys())[-1] + if fsspec in _finders: + finder = _finders[fsspec] + del _finders[fsspec] + if finder in sys.meta_path: + finder.unload() + sys.meta_path.remove(finder) diff --git a/fsspec_python/open.py b/fsspec_python/open.py index 0689400..26993ac 100644 --- a/fsspec_python/open.py +++ b/fsspec_python/open.py @@ -18,12 +18,8 @@ def install_open_hook(fsspec: str, **fsspec_args): root: str fsspec_fs, root = url_to_fs(fsspec, **fsspec_args) - print(root) - def open_from_fsspec(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): filename = join(root, file) - print(filename) - if ("w" in mode or "a" in mode or "x" in mode) or fsspec_fs.exists(filename): return fsspec_fs.open(filename, mode=mode, encoding=encoding, errors=errors, newline=newline) return _original_open(file, mode, buffering, encoding, errors, newline, closefd, opener) diff --git a/fsspec_python/tests/conftest.py b/fsspec_python/tests/conftest.py index 0a9677e..bdd0f23 100644 --- a/fsspec_python/tests/conftest.py +++ b/fsspec_python/tests/conftest.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +from fsspec import open from fsspec_python import install_importer, uninstall_importer @@ -29,3 +30,10 @@ def open_hook(): install_open_hook(f"file://{Path(__file__).parent}/dump/") yield uninstall_open_hook() + + +@pytest.fixture() +def fs_importer(): + fs = open(f"python::file://{Path(__file__).parent}/local2") + yield fs + fs.close() diff --git a/fsspec_python/tests/local2/my_local_file2.py b/fsspec_python/tests/local2/my_local_file2.py new file mode 100644 index 0000000..9b172c7 --- /dev/null +++ b/fsspec_python/tests/local2/my_local_file2.py @@ -0,0 +1,2 @@ +def baz(): + return "This is a local file." diff --git a/fsspec_python/tests/test_fs.py b/fsspec_python/tests/test_fs.py new file mode 100644 index 0000000..52759c5 --- /dev/null +++ b/fsspec_python/tests/test_fs.py @@ -0,0 +1,8 @@ +class TestFs: + def test_fs(self, fs_importer): + fs = fs_importer + import my_local_file2 + + assert my_local_file2.baz() == "This is a local file." + + fs.close() diff --git a/pyproject.toml b/pyproject.toml index 3c8cb7f..3767d34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,9 @@ dependencies = [ "fsspec", ] +[project.entry-points."fsspec.specs"] +python = "fsspec_python.fs.PythonFileSystem" + [project.optional-dependencies] develop = [ "build",