[virtual-vlnv] Add CLI option to add virtual providers

Add --add-vlnv to enable adding user-requested VLNVs that are not under
the "system" or top core's explicit dependency tree. The intention of
this argument is to allow users to specify cores that provide the
implementations of virtual VLNVs in these "side" trees. The CLI option
allows users to provide these without requiring someone to write a core
file with the explicit deps, so targets may be reused across multiple
different configurations, and users don't need to have duplicate core
files that only differ by virtual providers.

Signed-off-by: Alexander Williams <[email protected]>

Author: a-will

fusesoc/coremanager.py

@@ -110,8 +110,8 @@ def _hash_flags_dict(self, flags):
             h ^= hash(pair)
         return h
 
-    def solve(self, top_core, flags):
-        return self._solve(top_core, flags)
+    def solve(self, top_core, flags, added_cores):
+        return self._solve(top_core, flags, added_cores)
 
     def _get_conflict_map(self):
         """Return a map of cores to their conflicts
@@ -142,7 +142,7 @@ def _get_conflict_map(self):
             conflict_set.remove(real_pkg)
         return conflict_map
 
-    def _solve(self, top_core, flags={}, only_matching_vlnv=False):
+    def _solve(self, top_core, flags={}, added_cores=(), only_matching_vlnv=False):
         def eq_vln(this, that):
             return (
                 this.vendor == that.vendor
@@ -151,7 +151,12 @@ def eq_vln(this, that):
             )
 
         # Try to return a cached result
-        solver_cache_key = (top_core, self._hash_flags_dict(flags), only_matching_vlnv)
+        requested_cores = frozenset(added_cores + (top_core,))
+        solver_cache_key = (
+            requested_cores,
+            self._hash_flags_dict(flags),
+            only_matching_vlnv,
+        )
         cached_solution = self._solver_cache_lookup(solver_cache_key)
         if cached_solution:
             return cached_solution
@@ -206,12 +211,13 @@ def eq_vln(this, that):
             repo.add_package(package)
 
         request = Request()
-        _top_dep = "{} {} {}".format(
-            self._package_name(top_core),
-            top_core.relation,
-            self._package_version(top_core),
-        )
-        request.install(Requirement._from_string(_top_dep))
+        for requested_core in requested_cores:
+            dep = "{} {} {}".format(
+                self._package_name(requested_core),
+                requested_core.relation,
+                self._package_version(requested_core),
+            )
+            request.install(Requirement._from_string(dep))
 
         installed_repository = Repository()
         pool = Pool([repo])
@@ -249,11 +255,12 @@ def eq_vln(this, that):
                 ]
         # Print a warning for all virtual selections that has no concrete requirement selection
         for virtual in virtual_selection.values():
-            logger.warning(
-                "Non-deterministic selection of virtual core {} selected {}".format(
-                    virtual[1], virtual[0]
+            if virtual[0] not in requested_cores:
+                logger.warning(
+                    "Non-deterministic selection of virtual core {} selected {}".format(
+                        virtual[1], virtual[0]
+                    )
                 )
-            )
 
         result = [op.package.core for op in transaction.operations]
 
@@ -402,13 +409,16 @@ def get_libraries(self):
         """Get all registered libraries"""
         return self._lm.get_libraries()
 
-    def get_depends(self, core, flags):
+    def get_depends(self, core, flags, added_cores):
         """Get an ordered list of all dependencies of a core
 
         All direct and indirect dependencies are resolved into a dependency
         tree, the tree is flattened, and an ordered list of dependencies is
         created.
 
+        The list is augmented by any added_cores required by the call, which can
+        deterministically satisfy virtual VLNV dependencies.
+
         The first element in the list is a leaf dependency, the last element
         is the core at the root of the dependency tree.
         """
@@ -417,8 +427,9 @@ def get_depends(self, core, flags):
                 core.relation, str(core), str(flags)
             )
         )
+        logger.debug(" User added cores to request: " + ", ".join([str(c) for c in added_cores]))
         resolved_core = self.db.find(core)
-        deps = self.db.solve(resolved_core.name, flags)
+        deps = self.db.solve(resolved_core.name, flags, added_cores)
         logger.debug(" Resolved core to {}".format(str(resolved_core.name)))
         logger.debug(" with dependencies " + ", ".join([str(c.name) for c in deps]))
         return deps

fusesoc/edalizer.py

@@ -49,12 +49,14 @@ def __init__(
         export_root=None,
         system_name=None,
         resolve_env_vars=False,
+        added_core_reqs=(),
     ):
         logger.debug("Building EDAM structure")
 
         self.toplevel = toplevel
         self.flags = flags
         self.core_manager = core_manager
+        self.added_core_reqs = added_core_reqs
         self.work_root = work_root
         self.export_root = export_root
         self.system_name = system_name
@@ -74,7 +76,11 @@ def cores(self):
     @property
     def resolved_cores(self):
         """Get a list of all "used" cores after the dependency resolution"""
-        return self.core_manager.get_depends(self.toplevel, self.flags)
+        return self.core_manager.get_depends(
+            self.toplevel,
+            self.flags,
+            self.added_core_reqs
+        )
 
     @property
     def discovered_cores(self):

fusesoc/fusesoc.py

@@ -115,7 +115,7 @@ def get_work_root(self, core, flags):
 
         return work_root
 
-    def get_backend(self, core, flags, backendargs=[]):
+    def get_backend(self, core, flags, added_core_reqs=(), backendargs=[]):
 
         work_root = self.get_work_root(core, flags)
 
@@ -156,6 +156,7 @@ def get_backend(self, core, flags, backendargs=[]):
             export_root=export_root,
             system_name=self.config.system_name,
             resolve_env_vars=self.config.resolve_env_vars_early,
+            added_core_reqs=added_core_reqs,
         )
 
         try:

fusesoc/main.py

@@ -29,6 +29,7 @@
 from fusesoc.coremanager import DependencyError
 from fusesoc.fusesoc import Fusesoc
 from fusesoc.librarymanager import Library
+from fusesoc.vlnv import Vlnv
 
 logger = logging.getLogger(__name__)
 
@@ -303,6 +304,11 @@ def run(fs, args):
         else:
             flags[flag] = True
 
+    added_core_reqs = []
+    for vlnv in args.add_vlnv:
+        added_core_reqs.append(Vlnv(vlnv))
+    added_core_reqs = tuple(added_core_reqs)
+
     core = _get_core(fs, args.system)
 
     try:
@@ -326,7 +332,9 @@ def run(fs, args):
     # Frontend/backend separation
 
     try:
-        edam_file, backend = fs.get_backend(core, flags, args.backendargs)
+        edam_file, backend = fs.get_backend(
+            core, flags, added_core_reqs, args.backendargs
+        )
 
     except RuntimeError as e:
         logger.error(str(e))
@@ -592,6 +600,13 @@ def get_parser():
     parser_run.add_argument("--run", action="store_true", help="Execute run stage")
     parser_run.add_argument("--target", help="Override default target")
     parser_run.add_argument("--tool", help="Override default tool for target")
+    parser_run.add_argument(
+        "--add-vlnv",
+        help="Add a VLNV to the build as a dependency of the 'system'. Intended for explicit virtual providers. Multiple uses allowed.",
+        action="append",
+        default=[],
+        metavar="VLNV",
+    )
     parser_run.add_argument(
         "--flag",
         help="Set custom use flags. Can be specified multiple times",

tests/test_coremanager.py

@@ -88,7 +88,7 @@ def test_deptree(tmp_path):
     edam = edalizer.run()
 
     # Check dependency tree (after running all generators)
-    deps = cm.get_depends(root_core.name, {})
+    deps = cm.get_depends(root_core.name, {}, ())
     deps_names = [str(c) for c in deps]
 
     all_core_names = set()
@@ -274,7 +274,7 @@ def test_virtual():
         )
         edalizer.run()
 
-        deps = cm.get_depends(root_core.name, {})
+        deps = cm.get_depends(root_core.name, {}, ())
         deps_names = [str(c) for c in deps]
 
         assert deps_names == expected_deps
@@ -355,7 +355,7 @@ def test_virtual_non_deterministic_virtual(caplog):
         edalizer.run()
     assert "Non-deterministic selection of virtual core" in caplog.text
 
-    deps = cm.get_depends(root_core.name, {})
+    deps = cm.get_depends(root_core.name, {}, ())
     deps_names = [str(c) for c in deps]
 
     for dependency in deps_names:
@@ -365,3 +365,55 @@ def test_virtual_non_deterministic_virtual(caplog):
             "::user:0",
             "::top_non_deterministic:0",
         ]
+
+
+def test_virtual_explicit_providers(caplog):
+    """
+    Test virtual core selection when there are explicit selected implementations on the side.
+    This shall NOT result in a warning that the virtual core selection is non-deteministic.
+    """
+    import logging
+    import os
+    import tempfile
+
+    from fusesoc.config import Config
+    from fusesoc.coremanager import CoreManager
+    from fusesoc.edalizer import Edalizer
+    from fusesoc.librarymanager import Library
+    from fusesoc.vlnv import Vlnv
+
+    flags = {"tool": "icarus"}
+
+    build_root = tempfile.mkdtemp(prefix="export_")
+    work_root = os.path.join(build_root, "work")
+
+    core_dir = os.path.join(os.path.dirname(__file__), "capi2_cores", "virtual")
+
+    cm = CoreManager(Config())
+    cm.add_library(Library("virtual", core_dir), [])
+
+    root_core = cm.get_core(Vlnv("::top_non_deterministic"))
+
+    added_core_options = ((Vlnv("::impl1:0"),), (Vlnv("::impl2:0"),))
+
+    for added_cores in added_core_options:
+        edalizer = Edalizer(
+            toplevel=root_core.name,
+            flags=flags,
+            core_manager=cm,
+            work_root=work_root,
+            added_core_reqs=added_cores,
+        )
+
+        with caplog.at_level(logging.WARNING):
+            edalizer.run()
+        assert "Non-deterministic selection of virtual core" not in caplog.text
+
+        deps = cm.get_depends(root_core.name, {}, added_cores)
+        deps_names = [str(c) for c in deps]
+
+        for dependency in deps_names:
+            assert dependency in [
+                "::user:0",
+                "::top_non_deterministic:0",
+            ] + [str(c) for c in added_cores]