Merge pull request #136 from getgauge/134_multiple_stepImplDir

#134 Add support for multiple step implementation directories

Author: NivedhaSenthil

getgauge/impl_loader.py

@@ -11,7 +11,7 @@
 from getgauge.util import *
 
 project_root = get_project_root()
-impl_dir = get_step_impl_dir()
+impl_dirs = get_step_impl_dirs()
 env_dir = os.path.join(project_root, 'env', 'default')
 requirements_file = os.path.join(project_root, 'requirements.txt')
 sys.path.append(project_root)
@@ -21,22 +21,23 @@
 SKEL = 'skel'
 
 
-def load_impls(step_impl_dir=impl_dir):
+def load_impls(step_impl_dirs=impl_dirs):
     os.chdir(project_root)
-    if not os.path.isdir(step_impl_dir):
-        logging.error('Cannot import step implementations. Error: {} does not exist.'.format(step_impl_dir))
-        logging.error('Make sure `STEP_IMPL_DIR` env var is set to a valid directory path.')
-        return
-    _import_impl(step_impl_dir)
+    for impl_dir in step_impl_dirs:
+        if not os.path.isdir(impl_dir):
+            logging.error('Cannot import step implementations. Error: {} does not exist.'.format(step_impl_dirs))
+            logging.error('Make sure `STEP_IMPL_DIR` env var is set to a valid directory path.')
+            return
+        _import_impl(impl_dir)
 
 
 def copy_skel_files():
     try:
         logging.info('Initialising Gauge Python project')
         logging.info('create  {}'.format(env_dir))
         os.makedirs(env_dir)
-        logging.info('create  {}'.format(impl_dir))
-        shutil.copytree(os.path.join(SKEL, STEP_IMPL_DIR_NAME), impl_dir)
+        logging.info('create  {}'.format(impl_dirs[0]))
+        shutil.copytree(os.path.join(SKEL, STEP_IMPL_DIR_NAMES[0]), impl_dirs[0])
         logging.info('create  {}'.format(os.path.join(env_dir, PYTHON_PROPERTIES)))
         shutil.copy(os.path.join(SKEL, PYTHON_PROPERTIES), env_dir)
         open(requirements_file, 'w').write('getgauge==' + _get_version())
@@ -51,7 +52,7 @@ def _import_impl(step_impl_dir):
         if f.endswith('.py'):
             _import_file(file_path)
         elif path.isdir(file_path):
-            load_impls(file_path)
+            load_impls([file_path])
 
 
 def _import_file(file_path):

getgauge/lsp_server.py

@@ -6,7 +6,7 @@
     ImplementationFileGlobPatternResponse, StepNamesResponse, \
     ImplementationFileListResponse
 from getgauge.registry import registry
-from getgauge.util import get_impl_files, get_step_impl_dir
+from getgauge.util import get_impl_files, get_step_impl_dirs
 
 
 class LspServerHandler(lsp_pb2_grpc.lspServiceServicer):
@@ -60,7 +60,8 @@ def GetStepName(self, request, context):
 
     def GetGlobPatterns(self, request, context):
         res = ImplementationFileGlobPatternResponse()
-        res.globPatterns.extend(["{}/**/*.py".format(get_step_impl_dir())])
+        globPatterns = [["{}/**/*.py".format(d)] for d in get_step_impl_dirs()]
+        res.globPatterns.extend([item for sublist in globPatterns for item in sublist])
         return res
 
     def KillProcess(self, request, context):

getgauge/processor.py

@@ -15,7 +15,7 @@
 from getgauge.refactor import refactor_step
 from getgauge.registry import registry, MessagesStore, ScreenshotsStore
 from getgauge.static_loader import reload_steps
-from getgauge.util import get_step_impl_dir, get_impl_files, read_file_contents, get_file_name
+from getgauge.util import get_step_impl_dirs, get_impl_files, read_file_contents, get_file_name
 from getgauge.validator import validate_step
 
 ATTACH_DEBUGGER_EVENT = 'Runner Ready for Debugging'
@@ -82,7 +82,7 @@ def handle_detached():
 def _execute_before_suite_hook(request, response, _socket, clear=True):
     if clear:
         registry.clear()
-        load_impls(get_step_impl_dir())
+        load_impls(get_step_impl_dirs())
     if environ.get('DEBUGGING'):
         ptvsd.enable_attach(address=(
             '127.0.0.1', int(environ.get('DEBUG_PORT'))))
@@ -278,8 +278,8 @@ def stub_impl_response(codes, file_name, response):
 
 
 def _glob_pattern(_request, response, _socket):
-    patterns = ["{}/**/*.py".format(get_step_impl_dir())]
-    return response.implementationFileGlobPatternResponse.globPatterns.extend(patterns)
+    patterns = [["{}/**/*.py".format(d)] for d in get_step_impl_dirs()]
+    return response.implementationFileGlobPatternResponse.globPatterns.extend([item for sublist in patterns for item in sublist])
 
 
 processors = {Message.ExecutionStarting: _execute_before_suite_hook,

getgauge/static_loader.py

@@ -16,11 +16,11 @@ def reload_steps(file_path, content=None):
         load_steps(pf)
 
 
-def load_files(step_impl_dir):
-    for dirpath, dirs, files in os.walk(step_impl_dir):
-        py_files = (os.path.join(dirpath, f)
-                    for f in files if f.endswith('.py'))
-        for file_path in py_files:
-            pf = PythonFile.parse(file_path)
-            if pf:
-                load_steps(pf)
+def load_files(step_impl_dirs):
+    for step_impl_dir in step_impl_dirs: 
+        for dirpath, _, files in os.walk(step_impl_dir):
+                py_files = (os.path.join(dirpath, f) for f in files if f.endswith('.py'))
+                for file_path in py_files:
+                        pf = PythonFile.parse(file_path)
+                        if pf:
+                                load_steps(pf)

getgauge/util.py

@@ -2,7 +2,6 @@
 
 PROJECT_ROOT_ENV = 'GAUGE_PROJECT_ROOT'
 STEP_IMPL_DIR_ENV = 'STEP_IMPL_DIR'
-STEP_IMPL_DIR_NAME = os.getenv(STEP_IMPL_DIR_ENV) or 'step_impl'
 
 
 def get_project_root():
@@ -12,17 +11,19 @@ def get_project_root():
         return ""
 
 
-def get_step_impl_dir():
-    return os.path.join(get_project_root(), STEP_IMPL_DIR_NAME)
+def get_step_impl_dirs():
+    STEP_IMPL_DIR_NAMES = map(str.strip, os.getenv(STEP_IMPL_DIR_ENV).split(',')) if os.getenv(STEP_IMPL_DIR_ENV) else ['step_impl']
+    return [os.path.join(get_project_root(), name) for name in STEP_IMPL_DIR_NAMES]
 
 
 def get_impl_files():
-    step_impl_dir = get_step_impl_dir()
+    step_impl_dirs = get_step_impl_dirs()
     file_list = []
-    for root, _, files in os.walk(step_impl_dir):
-        for file in files:
-            if file.endswith('.py') and '__init__.py' != os.path.basename(file):
-                file_list.append(os.path.join(root, file))
+    for step_impl_dir in step_impl_dirs:
+        for root, _, files in os.walk(step_impl_dir):
+            for file in files:
+                if file.endswith('.py') and '__init__.py' != os.path.basename(file):
+                    file_list.append(os.path.join(root, file))
     return file_list
 
 
@@ -37,7 +38,7 @@ def read_file_contents(file_name):
 
 def get_file_name(prefix='', counter=0):
     name = 'step_implementation{}.py'.format(prefix)
-    file_name = os.path.join(get_step_impl_dir(), name)
+    file_name = os.path.join(get_step_impl_dirs()[0], name)
     if not os.path.exists(file_name):
         return file_name
     else:

start.py

@@ -11,7 +11,7 @@
 from getgauge.impl_loader import copy_skel_files
 from getgauge.messages import lsp_pb2_grpc
 from getgauge.static_loader import load_files
-from getgauge.util import get_step_impl_dir
+from getgauge.util import get_step_impl_dirs
 
 PLUGIN_JSON = 'python.json'
 VERSION = 'version'
@@ -28,12 +28,12 @@ def main():
 
 
 def load_implementations():
-    d = get_step_impl_dir()
-    if path.exists(d):
-        load_files(d)
-    else:
-        logging.error(
-            'can not load implementations from {}. {} does not exist.'.format(d, d))
+    d = get_step_impl_dirs()
+    for impl_dir in d:
+        if not path.exists(impl_dir):
+            logging.error('can not load implementations from {}. {} does not exist.'.format(impl_dir, impl_dir))
+    load_files(d)
+        
 
 
 def start():

tests/test_lsp_server.py

@@ -9,7 +9,7 @@
 from getgauge.messages.messages_pb2 import *
 from getgauge.messages.spec_pb2 import ProtoStepValue
 from getgauge.registry import registry
-from getgauge.util import get_step_impl_dir
+from getgauge.util import get_step_impl_dirs
 from getgauge.parser import PythonFile
 
 
@@ -35,7 +35,7 @@ def test_LspServerHandler_glob_pattern(self):
     def test_LspServerHandler_file_list(self):
         handler = LspServerHandler(None)
         req = ImplementationFileListRequest()
-        self.fs.create_file(os.path.join(get_step_impl_dir(), 'foo.py'))
+        self.fs.create_file(os.path.join(get_step_impl_dirs()[0], 'foo.py'))
 
         res = handler.GetImplementationFiles(req, None)
 
@@ -118,8 +118,8 @@ def foo(vowels):
             print(vowels)
         ''')
         self.fs.create_file(os.path.join(
-            get_step_impl_dir(), 'foo.py'), contents=content)
-        loader.load_files(get_step_impl_dir())
+            get_step_impl_dirs()[0], 'foo.py'), contents=content)
+        loader.load_files(get_step_impl_dirs())
 
         request = RefactorRequest()
         request.saveChanges = False

tests/test_utils.py

@@ -0,0 +1,20 @@
+from unittest import main, TestCase
+from getgauge.util import get_step_impl_dirs
+import os
+
+
+class UtilTests(TestCase):
+
+    def test_get_step_impl_gives_default_if_no_env_Set(self):
+        dirs = get_step_impl_dirs()
+        expected = ['step_impl']
+        self.assertEquals(dirs,expected)
+
+    def test_get_step_impl_returns_array_of_impl_dirs(self):
+        os.environ["STEP_IMPL_DIR"] = "step_impl, step_impl1"
+        dirs = get_step_impl_dirs()
+        expected = ['step_impl','step_impl1']
+        self.assertEquals(dirs,expected)
+
+if __name__ == '__main__':
+    main()

tests/test_validate.py

@@ -16,3 +16,6 @@ def test_is_valid(self):
 
     def test_format_params(self):
         self.assertEqual("a, b, arg3", _format_params(["a", "b", "2"]))
+
+if __name__ == '__main__':
+    unittest.main()