Merge pull request #89 from LUMC/release_1.2.2

Release 1.2.2

Author: rhpvorderman

.travis.yml

@@ -9,15 +9,17 @@ matrix:
   include:
     # TEST DOCS AND LINTING
     # Use default python3 version here.
-    - env: TOX_ENV=py3-docs
-    - env: TOX_ENV=py3-lint
+    - env: TOX_ENV=docs
+    - env: TOX_ENV=lint
 
     # UNIT TESTS
     # On most recent versions of python.
     - python: 3.6
       env: TOX_ENV=py36
     - python: 3.7
       env: TOX_ENV=py37
+    - python: 3.8
+      env: TOX_ENV=py38
     - python: 3.5
       env: TOX_ENV=py35
       install:
@@ -37,11 +39,11 @@ matrix:
         - codecov -v  # -v to make sure coverage upload works.
 
     # FUNCTIONAL TESTS
-    - python: 3.5
-      env: TOX_ENV=py3-snakemake
     - python: 3.6
-      env: TOX_ENV=py3-snakemake
-    - env: TOX_ENV=py3-cromwell
+      env: TOX_ENV=snakemake
+    - python: 3.7
+      env: TOX_ENV=snakemake
+    - env: TOX_ENV=cromwell
       before_install:
         # Install conda
         - export MINICONDA=${HOME}/miniconda

HISTORY.rst

@@ -4,9 +4,25 @@ Changelog
 
 .. Newest changes should be on top.
 
-.. NOTE: This document is user facing. Please word the changes in such a way
+.. This document is user facing. Please word the changes in such a way
 .. that users understand how the changes affect the new version.
 
+version 1.2.2
+---------------------------
++ Test against python3.8
++ Do not test on python3.5 snakemake as it crashes. Added test for python3.7
+  snakemake.
++ Fix a typo in the documentation.
++ Add tags 'wdl', 'cromwell' and 'snakemake' to the package to increase
+  discoverability.
++ Remove pylint from the lint procedure as it was very strict and got stricter
+  with every update, causing tests that previously succeeded to fail on a
+  regular basis.
++ Make sure pytest-workflow crashes when multiple workflows have the same name,
+  even when they are in different files.
++ Added setup.cfg to include license in source distributions on PyPI for
+  future versions
+
 version 1.2.1
 ---------------------------
 + Since pytest 4.5.0 unknown markers give a warning. ``@pytest.mark.workflow``

docs/installation.rst

@@ -22,9 +22,9 @@ On Ubuntu or Debian
     ``python3 -m pip install --user pytest-workflow``
 - ``pytest`` can now be run with ``python3 -m pytest``.
 
-.. container:: note
+.. note::
 
-    NOTE: Running plain ``pytest`` on Ubuntu or Debian outside of a virtual
+    Running plain ``pytest`` on Ubuntu or Debian outside of a virtual
     environment will not work with ``pytest-workflow`` because this will start
     the python2 version of ``pytest``. This is because python2 is the default
     python on any distribution released before January 1st 2020.

docs/running_pytest_workflow.rst

@@ -21,9 +21,9 @@ workflows that have succeeded.
 If you wish to change the temporary directory in which the workflows are run
 use ``--basetemp <dir>`` to change pytest's base temp directory.
 
-.. container:: warning
+.. warning::
 
-  WARNING: When a directory is passed to ``--basetemp`` some of the directory
+  When a directory is passed to ``--basetemp`` some of the directory
   contents will be deleted. For example: if your workflow is named
   ``"my workflow"`` then any file or directory named ``my_workflow`` will be
   deleted. This makes sure you start with a clean slate if you run pytest

docs/writing_tests.rst

@@ -60,7 +60,8 @@ Test options
     files:
       - path: "fail.log"               # Multiple files can be tested for each workflow
       - path: "TomCruise.txt.gz"       # Gzipped files can also be searched, provided their extension is '.gz'
-        contains: "starring"
+        contains: 
+          - "starring"
     stderr:                            # Options for testing stderr (optional)
       contains:                        # A list of strings which should be in stderr (optional)
         - "BSOD error, please contact the IT crowd"
@@ -70,6 +71,10 @@ Test options
 
 The above YAML file contains all the possible options for a workflow test.
 
+.. note::
+    Workflow names must be unique. Pytest workflow will crash when multiple
+    workflows have the same name, even if they are in different files.
+
 Writing custom tests
 --------------------
 
@@ -102,7 +107,7 @@ containing numbers' has run.
 points to the folder where the named workflow was executed. This allows writing of
 advanced python tests for each file produced by the workflow.
 
-.. container:: note
+.. note::
 
-    NOTE: stdout and stderr are available as files in the root of the
+    stdout and stderr are available as files in the root of the
     ``workflow_dir`` as ``log.out`` and ``log.err`` respectively.

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers = 
+    functional: This marks functional tests for pytest-workflow. These are the tests for pytest-workflow that are executed using different workflow engines.

setup.cfg

@@ -0,0 +1,3 @@
+[metadata]
+license_file=LICENSE
+

setup.py

@@ -21,15 +21,15 @@
 
 setup(
     name="pytest-workflow",
-    version="1.2.1",
+    version="1.2.2",
     description="A pytest plugin for configuring workflow/pipeline tests "
                 "using YAML files",
     author="Leiden University Medical Center",
     author_email="[email protected]",  # A placeholder for now
     long_description=LONG_DESCRIPTION,
     long_description_content_type="text/x-rst",
     license="AGPL-3.0-or-later",
-    keywords="pytest workflow pipeline yaml yml",
+    keywords="pytest workflow pipeline yaml yml wdl cromwell snakemake",
     zip_safe=False,
     packages=find_packages('src'),
     package_dir={'': 'src'},
@@ -43,6 +43,7 @@
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: "
         "GNU Affero General Public License v3 or later (AGPLv3+)",

src/pytest_workflow/plugin.py

@@ -19,7 +19,7 @@
 import tempfile
 import warnings
 from pathlib import Path
-from typing import List, Optional  # noqa: F401 needed for typing.
+from typing import Dict, List, Optional  # noqa: F401 needed for typing.
 
 from _pytest.config import Config as PytestConfig
 from _pytest.config.argparsing import Parser as PytestParser
@@ -115,7 +115,7 @@ def pytest_configure(config: PytestConfig):
     setattr(config, "workflow_queue", workflow_queue)
 
     # Save which workflows are run and which are not.
-    executed_workflows = []  # type: List[str]
+    executed_workflows = {}  # type: Dict[str, str]
     setattr(config, "executed_workflows", executed_workflows)
 
     # Save workflow for cleanup in this var.
@@ -191,7 +191,7 @@ def pytest_collection_modifyitems(config: PytestConfig,
             raise TypeError("A workflow name should be defined in the "
                             "workflow marker of {0}".format(item.nodeid))
 
-        if workflow_name not in config.executed_workflows:
+        if workflow_name not in config.executed_workflows.keys():
             skip_marker = pytest.mark.skip(
                 reason="'{0}' has not run.".format(workflow_name))
             item.add_marker(skip_marker)
@@ -204,6 +204,26 @@ def pytest_runtestloop(session: pytest.Session):
     )
 
 
+def pytest_collectstart(collector: pytest.Collector):
+    """This runs before the collector runs its collect attribute"""
+
+    if isinstance(collector, WorkflowTestsCollector):
+        name = collector.workflow_test.name  # type: str
+
+        # Executed workflows contains workflow name as key and nodeid as value.
+        executed_workflows = collector.config.executed_workflows  # type: Dict[str,str]  # noqa: E501
+
+        if name in executed_workflows.keys():
+            raise ValueError(
+                "Workflow name '{this}' used more than once. Conflicting "
+                "tests: {this_test}, {existing_test}. ".format(
+                    this=name,
+                    this_test=collector.nodeid,
+                    existing_test=executed_workflows[name]
+                )
+            )
+
+
 def pytest_sessionfinish(session: pytest.Session, exitstatus: int):
     # No cleanup needed if we keep workflow directories
     # Or if there are no directories to cleanup. (I.e. pytest-workflow plugin
@@ -340,7 +360,10 @@ def collect(self):
             return []
         else:
             # If we run the workflow, save this for reference later.
-            self.config.executed_workflows.append(self.workflow_test.name)
+            # Save the nodeid because it also contains the originating file.
+            # This is useful for error messages later.
+            self.config.executed_workflows[self.workflow_test.name] = (
+                self.nodeid)
 
         # This creates a workflow that is queued for processing after the
         # collection phase.

tests/test_miscellaneous_crashes.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2018 Leiden University Medical Center
+# This file is part of pytest-workflow
+#
+# pytest-workflow is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# pytest-workflow is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with pytest-workflow.  If not, see <https://www.gnu.org/licenses/
+
+from .test_success_messages import SIMPLE_ECHO
+
+
+def test_same_name_different_files(testdir):
+    testdir.makefile(".yml", test_a=SIMPLE_ECHO)
+    testdir.makefile(".yml", test_b=SIMPLE_ECHO)
+    result = testdir.runpytest()
+    assert result.ret != 0
+    assert ("Workflow name 'simple echo' used more than once"
+            in result.stdout.str())
+    conflicting_message = (
+        "Conflicting tests: {0}, {1}.".format(
+            "test_b.yml::" + "simple echo",
+            "test_a.yml::" + "simple echo")
+    )
+    assert conflicting_message in result.stdout.str()