adding support to install from local container (#602)

* adding support to install from local container
* add --keep-path to not relocate container

this means that a user can select a registry entry
(container.yaml) to use for metadata, and apply to
a local container. This still requires choice of
an explicit registry entry for metadata selection.

Signed-off-by: vsoch <[email protected]>

Author: vsoch

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/singularityhub/singularity-hpc/tree/main) (0.0.x)
+ - support for install using registry recipe and local image (0.1.15)
  - fix views .view_module modulefile and loading (0.1.14)
  - support for system modules, depends on, in views and editor envars (0.1.13)
  - Wrappers now supported for shell/exec/run container commands (0.1.12)

docs/getting_started/user-guide.rst

@@ -1158,13 +1158,47 @@ the container technology.
 If you don't have any module software on your system, you can now test interacting
 with the module via the :ref:`getting_started-development` instructions.
 
+.. _getting_started-commands-install-private:
+
 
 Install Private Images
 ----------------------
 
 What about private containers on Docker Hub? If you have a private image, you can
 simply use `Singularity remote login <https://github.com/sylabs/singularity-userdocs/blob/master/singularity_and_docker.rst#singularity-cli-remote-command>`_ before attempting the install and everything should work.
 
+.. _getting_started-commands-install-local:
+
+
+Install Local Image
+-------------------
+
+The concept of installing a local image means that you are selecting a container.yaml recipe from an existing registry,
+however instead of pulling it, you are pairing it was a particular URI of a local image. As an example, let's say we have pulled a local
+samtools container:
+
+.. code-block:: console
+
+    $ singularity pull docker://quay.io/biocontainers/samtools:1.10--h2e538c0_3
+
+We might then want to install it to the samtools namespace and using the same metadata (e.g., aliases, environment, etc.):
+
+.. code-block:: console
+
+    $ shpc install quay.io/biocontainers/samtools:1.10--h2e538c0_3 samtools_1.2--0.sif
+
+This is similar to an ``shpc add``, however instead of needing to write a container.yaml in a local
+filesystem, you are using an existing one. The use case or assumption here is that you have a local
+directory of containers that can be matched to existing shpc recipes. Finally to request using the
+container path "as is" without copying anything into your container folder, add ``--keep-path``:
+
+
+.. code-block:: console
+    $ shpc install quay.io/biocontainers/samtools:1.10--h2e538c0_3 samtools_1.2--0.sif --keep-path
+
+This feature is supported for shpc versions 0.1.15 and up.
+
+
 .. _getting_started-commands-namespace:
 
 

shpc/client/__init__.py

@@ -101,6 +101,20 @@ def get_parser():
         "install_recipe",
         help="recipe to install\nshpc install python\nshpc install python:3.9.5-alpine",
     )
+
+    install.add_argument(
+        "container_image",
+        help="path to an existing container image for this software",
+        nargs="?",
+    )
+
+    install.add_argument(
+        "--keep-path",
+        help="if installing a local container, do not copy the container - use the provided path.",
+        default=False,
+        action="store_true",
+    )
+
     install.add_argument(
         "--no-view",
         dest="no_view",

shpc/client/install.py

@@ -25,8 +25,16 @@ def main(args, parser, extra, subparser):
     cli.settings.update_params(args.config_params)
 
     # And do the install
-    cli.install(args.install_recipe, force=args.force)
+    cli.install(
+        args.install_recipe,
+        force=args.force,
+        container_image=args.container_image,
+        keep_path=args.keep_path,
+    )
     if cli.settings.default_view and not args.no_view:
         cli.view_install(
-            cli.settings.default_view, args.install_recipe, force=args.force
+            cli.settings.default_view,
+            args.install_recipe,
+            force=args.force,
+            container_image=args.container_image,
         )

shpc/main/modules/base.py

@@ -346,7 +346,9 @@ def check(self, module_name):
 
         return module.check()
 
-    def new_module(self, name, tag=None, tag_exists=True):
+    def new_module(
+        self, name, tag=None, tag_exists=True, container_image=None, keep_path=False
+    ):
         """
         Create a new module
         """
@@ -366,9 +368,21 @@ def new_module(self, name, tag=None, tag_exists=True):
         # Pass on container and settings
         module.container = self.container
         module.settings = self.settings
+
+        # Do we want to use a container from the local filesystem?
+        if container_image:
+            module.add_local_container(container_image, keep_path=keep_path)
         return module
 
-    def install(self, name, tag=None, force=False, **kwargs):
+    def install(
+        self,
+        name,
+        tag=None,
+        force=False,
+        container_image=None,
+        keep_path=False,
+        **kwargs
+    ):
         """
         Given a unique resource identifier, install a recipe.
 
@@ -378,7 +392,13 @@ def install(self, name, tag=None, force=False, **kwargs):
         "force" is currently not used.
         """
         # Create a new module
-        module = self.new_module(name, tag=tag, tag_exists=True)
+        module = self.new_module(
+            name,
+            tag=tag,
+            tag_exists=True,
+            container_image=container_image,
+            keep_path=keep_path,
+        )
 
         # We always load overrides for an install
         module.load_override_file()
@@ -411,12 +431,16 @@ def install(self, name, tag=None, force=False, **kwargs):
         logger.info("Module %s was created." % module.tagged_name)
         return module.container_path
 
-    def view_install(self, view_name, name, tag=None, force=False):
+    def view_install(
+        self, view_name, name, tag=None, force=False, container_image=None
+    ):
         """
         Install a module in a view. The module must already be installed.
         Set "force" to True to allow overwriting existing symlinks.
         """
-        module = self.new_module(name, tag=tag, tag_exists=True)
+        module = self.new_module(
+            name, tag=tag, tag_exists=True, container_image=container_image
+        )
 
         # A view is a symlink under views_base/$view/$module
         if view_name not in self.views:

shpc/main/modules/module.py

@@ -5,6 +5,7 @@
 import os
 import shutil
 
+import shpc.utils as utils
 from shpc.logger import logger
 
 
@@ -63,32 +64,20 @@ def container_dir(self):
             self._container_dir = self.container.container_dir(self.module_basepath)
         return self._container_dir
 
-    @property
-    def container_path(self):
+    def add_container(self, container_image=None):
         """
-        Derive the container path, if possible.
+        Ensure a container is pulled (or provided)
+
+        This should be called after self.config is set in new_module.
         """
-        if self._container_path:
-            return self._container_path
+        # First preference goes to provided image (actual file)
+        # This is only allowed for Singularity containers
+        if container_image and os.path.exists(container_image):
+            self.add_local_container(container_image)
 
         # If we have a sif URI provided by path, the container needs to exist
-        if self.config.path:
-            self._container_path = os.path.join(
-                self.config.entry.dirname, self.config.path
-            )
-            if not os.path.exists(self._container_path):
-                logger.exit(
-                    "Expected container defined by path %s not found in %s."
-                    % (self.config.path, self.config.entry.dirname)
-                )
-            container_dest = os.path.join(self.container_dir, self.config.path)
-
-            # Note that here we are *duplicating* the container, assuming we
-            # cannot use a link, and the registry won't be deleted but the
-            # module container might!
-            if not os.path.exists(container_dest):
-                shutil.copyfile(self._container_path, container_dest)
-            self._container_path = container_dest
+        elif self.config.path:
+            self._add_config_container()
 
         # For Singularity this is a path, podman is a uri. If None is returned
         # there was an error and we cleanup
@@ -98,6 +87,55 @@ def container_path(self):
             )
         return self._container_path
 
+    def add_local_container(self, container_image, keep_path=False):
+        """
+        Set the module container to be a local container (copied over)
+        """
+        basename = os.path.basename(container_image)
+
+        # The user has requested to use their own image, we don't copy
+        if keep_path:
+            self._container_path = os.path.abspath(container_image)
+            return
+
+        container_dest = os.path.join(self.container_dir, basename)
+        if not os.path.exists(self.container_dir):
+            utils.mkdir_p(self.container_dir)
+        if not os.path.exists(container_dest):
+            shutil.copyfile(container_image, container_dest)
+        self._container_path = container_dest
+
+    def _add_config_container(self):
+        """
+        Set the module container to be the one defined by the config.path
+        """
+        self._container_path = os.path.join(self.config.entry.dirname, self.config.path)
+
+        if not os.path.exists(self._container_path):
+            logger.exit(
+                "Expected container defined by path %s not found in %s."
+                % (self.config.path, self.config.entry.dirname)
+            )
+        container_dest = os.path.join(self.container_dir, self.config.path)
+
+        # Note that here we are *duplicating* the container, assuming we
+        # cannot use a link, and the registry won't be deleted but the
+        # module container might!
+        if not os.path.exists(container_dest):
+            shutil.copyfile(self._container_path, container_dest)
+        self._container_path = container_dest
+
+    @property
+    def container_path(self):
+        """
+        Derive the container path, if possible.
+        """
+        if self._container_path:
+            return self._container_path
+
+        # Default to pulling container based on config
+        return self.add_container()
+
     @property
     def tag(self):
         """

shpc/tests/test_client.py

@@ -305,6 +305,32 @@ def test_check(tmp_path, module_sys, container_tech, remote):
     client.check("vanessa/salad:latest")
 
 
+@pytest.mark.parametrize(
+    "module_sys,remote",
+    [("lmod", True), ("tcl", True), ("lmod", False), ("tcl", False)],
+)
+def test_install_local(tmp_path, module_sys, remote):
+    """
+    Test adding a custom container associated with an existing recipe
+    """
+    client = init_client(str(tmp_path), module_sys, "singularity", remote=remote)
+
+    # Create a copy of the latest image to add
+    container = os.path.join(str(tmp_path), "salad_latest.sif")
+    shutil.copyfile(os.path.join(here, "testdata", "salad_latest.sif"), container)
+
+    # It still needs to be a known tag!
+    with pytest.raises(SystemExit):
+        client.install(
+            "quay.io/biocontainers/samtools:1.2--0", container_image=container
+        )
+
+    # This should install our custom image using samtools metadata
+    container_image = "quay.io/biocontainers/samtools:1.10--h2e538c0_3"
+    client.install(container_image, container_image=container)
+    assert os.path.basename(client.get(container_image)) == os.path.basename(container)
+
+
 @pytest.mark.parametrize(
     "module_sys,remote",
     [("lmod", True), ("tcl", True), ("lmod", False), ("tcl", False)],

shpc/version.py

@@ -2,7 +2,7 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
-__version__ = "0.1.14"
+__version__ = "0.1.15"
 AUTHOR = "Vanessa Sochat"
 EMAIL = "[email protected]"
 NAME = "singularity-hpc"