Fix recrusive glob bug

tovrstra · tovrstra · commit 22d85851159c · 2025-02-06T16:21:48.000+01:00
diff --git a/.github/scripts/extract-notes.sh b/.github/scripts/extract-notes.sh
@@ -9,18 +9,18 @@ GITREF=${2}
 if [[ "${GITREF}" == refs/tags/* ]]; then
     TAG="${GITREF#refs/tags/}"
     VERSION="${TAG#v}"
-    MAJMIN=${VERSION%.*}
+    MACRO_MESO=$(echo "${VERSION}" | cut -d. -f1,2)
 else
     TAG="unreleased"
     VERSION="Unreleased"
-    MAJMIN="dev"
+    MACRO_MESO="dev"
 fi
 
 # Extract the release notes from the changelog
 sed -n "/## \[${VERSION}\]/, /## /{ /##/!p }" docs/changelog.md > notes.md
 
 # Add a link to the release notes
-URL="https://${OWNER}.github.io/${SLUG}/${MAJMIN}/changelog/#${TAG}"
+URL="https://${OWNER}.github.io/${SLUG}/${MACRO_MESO}/changelog/#${TAG}"
 echo "See [docs/changelog/#${TAG}](${URL}) for more details." >> notes.md
 
 # Remove leading and trailing empty lines
diff --git a/.github/workflows/mkdocs.yaml b/.github/workflows/mkdocs.yaml
@@ -4,27 +4,27 @@ name: mkdocs
 on:
   push:
     branches:
-    # Run on the main branch ...
-    - main
+      # Run on the main branch ...
+      - main
     tags-ignore:
-    # ... but not for tags (avoids duplicate work) ...
-    - '**'
+      # ... and on releas tags ...
+      - 'v*'
     paths:
-    # ... and only if relevant files have changed.
-    - stepup/**
-    - docs/**
-    - mkdocs.yaml
-    - pyproject.toml
-    - .github/workflows/mkdocs.yaml
+      # ... and only if relevant files have changed.
+      - stepup/**
+      - docs/**
+      - mkdocs.yaml
+      - pyproject.toml
+      - .github/workflows/mkdocs.yaml
   pull_request:
   # Run tests on pull requests ...
     paths:
-    # ... only if relevant files have changed.
-    - stepup/**
-    - docs/**
-    - mkdocs.yaml
-    - pyproject.toml
-    - .github/workflows/mkdocs.yaml
+      # ... only if relevant files have changed.
+      - stepup/**
+      - docs/**
+      - mkdocs.yaml
+      - pyproject.toml
+      - .github/workflows/mkdocs.yaml
 
 permissions:
   contents: write
@@ -71,6 +71,12 @@ jobs:
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
       - name: Build mkdocs
         run: mike deploy dev
-      - name: Push gh-pages branch to GitHub
+      - name: Push gh-pages branch with development version of docs
         if: github.ref == 'refs/heads/main'
         run: git push origin gh-pages
+      - name: Deploy gh-pages branch with stable version of docs
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          VERSION="${GITHUB_REF#refs/tags/v}"
+          MACRO_MESO=$(echo "${VERSION}" | cut -d. -f1,2)
+          mike deploy $(MACRO_MESO) -u stable -p
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -2,25 +2,25 @@ name: pytest
 on:
   push:
     branches:
-    # Run tests for change on the main branch ...
-    - main
+      # Run tests for change on the main branch ...
+      - main
     tags-ignore:
-    # ... but not for tags (avoids duplicate work) ...
-    - '**'
+      # ... but not for tags (avoids duplicate work) ...
+      - '**'
     paths:
-    # ... and only if relevant files have changed.
-    - stepup/**
-    - tests/**
-    - pyproject.toml
-    - .github/workflows/pytest.yaml
+      # ... and only if relevant files have changed.
+      - stepup/**
+      - tests/**
+      - pyproject.toml
+      - .github/workflows/pytest.yaml
   pull_request:
-  # Run tests on pull requests ...
+    # Run tests on pull requests ...
     paths:
-    # ... only if relevant files have changed.
-    - stepup/**
-    - tests/**
-    - pyproject.toml
-    - .github/workflows/pytest.yaml
+      # ... only if relevant files have changed.
+      - stepup/**
+      - tests/**
+      - pyproject.toml
+      - .github/workflows/pytest.yaml
 
 jobs:
   tests:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -51,7 +51,7 @@ jobs:
 
   publish-to-pypi:
     name: Publish Python distribution to PyPI
-    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    if: startsWith(github.ref, 'refs/tags/v')  # only publish to PyPI on tag pushes
     needs:
     - build
     runs-on: ubuntu-latest
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -10,10 +10,18 @@ and this project adheres to [Effort-based Versioning](https://jacobtomlinson.dev
 
 ## [Unreleased][]
 
+(nothing yet)
+
+## [2.0.7][] - 2025-02-06 {: #v2.0.7 }
+
+This release fixes two recursive glob issues.
+
 ### Fixed
 
 - Fixed issues with directories matching `glob("...", _defer=True)`,
   which are later used as parent directories in various scenarios.
+- Fix bug in recursive glob to match `data/inp.txt` with the pattern `data/**/inp.txt`
+
 
 ## [2.0.6][] - 2025-02-05 {: #v2.0.6 }
 
@@ -350,6 +358,7 @@ This release fixes several bugs.
 Initial release
 
 [Unreleased]: https://github.com/reproducible-reporting/stepup-core
+[2.0.7]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v2.0.7
 [2.0.6]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v2.0.6
 [2.0.5]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v2.0.5
 [2.0.4]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v2.0.4
diff --git a/stepup/core/nglob.py b/stepup/core/nglob.py
@@ -92,9 +92,9 @@
 
 RE_WILD_PARTS = [
     r"^[*][*]$",  # recursive ** wildcard, full string
-    r"^[*][*](?=/)",  # recursive ** wildcard, leading
+    r"^[*][*]/",  # recursive ** wildcard, leading
     r"(?<=/)[*][*]$",  # recursive ** wildcard, trailing
-    r"(?<=/)[*][*](?=/)",  # recursive ** wildcard, middle
+    r"(?<=/)[*][*]/",  # recursive ** wildcard, middle
     r"\[.*?]",  # anonymous [abc] wildcard
     r"[*]",  # anonymous * wildcard
     r"[?]",  # anonymous ? wildcard
@@ -732,6 +732,11 @@ def convert_nglob_to_regex(
                     regex = r".*"
                     if last in ["*"]:
                         replace = True
+            elif part == "**/":
+                if last != "**/":
+                    regex = r"(?:.*/|)"
+                    if last in ["*", "**"]:
+                        replace = True
             elif part.startswith("[") and part.endswith("]"):
                 regex = rf"[^{part[2:-1]}]" if part[1] == "!" else rf"[{part[1:-1]}]"
             elif part.startswith("${*") and part.endswith("}"):
@@ -819,6 +824,11 @@ def convert_nglob_to_glob(pattern: str, subs: dict[str, str] | None = None) -> s
                 texts[-1] = "**"
             elif texts[-1] != "**":
                 texts.append("**")
+        elif part == "**/":
+            if texts[-1] in ["*"] or texts[-1] == "**":
+                texts[-1] = "**/"
+            elif texts[-1] != "**/":
+                texts.append("**/")
         else:
             texts.append(part)
     return "".join(texts)
diff --git a/tests/examples/deferred_dir_glob/expected_graph.txt b/tests/examples/deferred_dir_glob/expected_graph.txt
@@ -25,7 +25,6 @@ step:./plan.py
             consumes   file:./
             consumes   file:plan.py
              creates   dg:data/**/
-             creates   file:data/
              creates   file:data/part1/bar.inp
              creates   file:data/part1/bar.out
              creates   file:data/part1/foo.inp
@@ -36,16 +35,12 @@ step:./plan.py
              creates   file:data/part2/foo.out
              creates   file:data/part2/sub/other.inp
              creates   file:data/part2/sub/other.out
-
-file:data/
-               state = STATIC
-          created by   step:./plan.py
-            consumes   file:./
-            supplies   file:data/part1/
-            supplies   file:data/part2/
+             creates   file:data/top.inp
+             creates   file:data/top.out
 
 dg:data/**/
           created by   step:./plan.py
+             creates   file:data/
              creates   file:data/part1/
              creates   file:data/part2/
              creates   file:data/part2/sub/
@@ -64,6 +59,15 @@ file:data/part1/
             supplies   file:data/part1/foo.inp
             supplies   file:data/part1/foo.out
 
+file:data/
+               state = STATIC
+          created by   dg:data/**/
+            consumes   file:./
+            supplies   file:data/part1/
+            supplies   file:data/part2/
+            supplies   file:data/top.inp
+            supplies   file:data/top.out
+
 file:data/part1/foo.inp
                state = STATIC
           created by   step:./plan.py
@@ -101,6 +105,11 @@ file:data/part2/sub/
             supplies   file:data/part2/sub/other.inp
             supplies   file:data/part2/sub/other.out
 
+file:data/top.inp
+               state = STATIC
+          created by   step:./plan.py
+            consumes   file:data/
+
 file:data/part1/bar.out
                state = STATIC
           created by   step:./plan.py
@@ -125,3 +134,8 @@ file:data/part2/sub/other.out
                state = STATIC
           created by   step:./plan.py
             consumes   file:data/part2/sub/
+
+file:data/top.out
+               state = STATIC
+          created by   step:./plan.py
+            consumes   file:data/
diff --git a/tests/examples/deferred_dir_glob/expected_inp.json b/tests/examples/deferred_dir_glob/expected_inp.json
@@ -3,5 +3,6 @@
   "data/part1/foo.inp",
   "data/part2/bar.inp",
   "data/part2/foo.inp",
-  "data/part2/sub/other.inp"
+  "data/part2/sub/other.inp",
+  "data/top.inp"
 ]
diff --git a/tests/examples/deferred_dir_glob/expected_out.json b/tests/examples/deferred_dir_glob/expected_out.json
@@ -3,5 +3,6 @@
   "data/part1/foo.out",
   "data/part2/bar.out",
   "data/part2/foo.out",
-  "data/part2/sub/other.out"
+  "data/part2/sub/other.out",
+  "data/top.out"
 ]
diff --git a/tests/examples/deferred_dir_glob/main.sh b/tests/examples/deferred_dir_glob/main.sh
@@ -25,3 +25,7 @@ wait
 
 # Check files that are expected to be present and/or missing.
 [[ -f plan.py ]] || exit 1
+grep 'inp"' current_inp.json
+grep 'out"' current_out.json
+[[ $(wc -l current_inp.json | cut -d' ' -f1) -eq 8 ]] || exit 1
+[[ $(wc -l current_out.json | cut -d' ' -f1) -eq 8 ]] || exit 1
diff --git a/tests/examples/deferred_dir_glob/plan.py b/tests/examples/deferred_dir_glob/plan.py
@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
 import json
 
-from stepup.core.api import glob, static
+from stepup.core.api import glob
 
 # Make data and subdirectories static when they are needed.
 # In this example, they become static when files in them are declared static.
-static("data/")
 glob("data/**/", _defer=True)
 
 # Make all inp and out files in data static and get them as separate lists.
diff --git a/tests/test_nglob.py b/tests/test_nglob.py
@@ -40,7 +40,20 @@
 
 
 @pytest.mark.parametrize(
-    "pattern", ["bar_${*foo}", "foo*", "*", "?", "**", "ls/ff**f/", "num[0-9]"]
+    "pattern",
+    [
+        "bar_${*foo}",
+        "foo*",
+        "*",
+        "?",
+        "**",
+        "ls/ff**f/",
+        "**/*.txt",
+        "data/**",
+        "data/**/*",
+        "data/**/out.txt",
+        "num[0-9]",
+    ],
 )
 def test_has_wildcards_true(pattern):
     assert has_wildcards(pattern)
@@ -354,10 +367,11 @@ def test_nglob_multi_single_anonymous():
         ("foo[*]", ["[*]"]),
         ("foo[${*ab}]", ["[${*ab}]"]),
         ("foo[[]a]", ["[[]"]),
-        ("**/", ["**"]),
+        ("**/", ["**/"]),
         ("/**", ["**"]),
         ("**", ["**"]),
-        ("./**/*.txt", ["**", "*"]),
+        ("./**/*.txt", ["**/", "*"]),
+        ("data/**/*.txt", ["**/", "*"]),
     ],
 )
 def test_nglob_wild(string, matches):
@@ -390,6 +404,8 @@ def test_nglob_wild(string, matches):
         ("${*sub}/**", "*/**"),
         ("${*sub}/**/", "*/**/"),
         ("data**", "data*"),
+        ("data/**/", "data/**/"),
+        ("data/**/*.txt", "data/**/*.txt"),
     ],
 )
 def test_nglob_to_glob(pattern, normal):
@@ -446,8 +462,10 @@ def test_nglob_to_glob_subs(pattern, subs, normal):
             r"(?P<generic>[^/]*)/ch(?P<foo>[^/]*)/[^/]*\.md",
         ),
         ("generic/ch${*foo}/${*md}", r"generic/ch(?P<foo>[^/]*)/(?P<md>[^/]*)"),
-        ("generic/${*md}${*ch}/${*md}", "generic/(?P<md>[^/]*)(?P<ch>[^/]*)/(?P=md)"),
-        ("data**", "data[^/]*/?"),
+        ("generic/${*md}${*ch}/${*md}", r"generic/(?P<md>[^/]*)(?P<ch>[^/]*)/(?P=md)"),
+        ("data**", r"data[^/]*/?"),
+        ("data/**/", r"data/(?:.*/|)"),
+        ("data/**/*.txt", r"data/(?:.*/|)[^/]*\.txt"),
     ],
 )
 def test_nglob_to_regex(pattern, regex):
@@ -723,6 +741,17 @@ def test_recursive6(tmpdir):
     )
 
 
+def test_recursive7(tmpdir):
+    _check_ngm_multi(
+        tmpdir,
+        patterns=["data/**/*.txt"],
+        subs={},
+        paths=["data.txt", "data/sub/part1.txt", "data/part2.txt"],
+        used_names=(),
+        results={(): [{"data/sub/part1.txt", "data/part2.txt"}]},
+    )
+
+
 def test_hidden(tmpdir):
     _check_ngm_multi(
         tmpdir,
diff --git a/tests/test_workflow.py b/tests/test_workflow.py
@@ -1858,10 +1858,10 @@ def test_deferred_glob_amend_out(wfp: Workflow):
 
 def test_deferred_glob_recursive_dirs(wfp: Workflow):
     plan = wfp.find("step", "./plan.py")
-    declare_static(wfp, plan, ["data/"])
     wfp.defer_glob(plan, "data/**/")
     to_check = wfp.declare_missing(plan, ["data/foo/a/bar.txt", "data/foo/b/egg.txt"])
     assert to_check == [
+        ("data/", FileHash.unknown()),
         ("data/foo/", FileHash.unknown()),
         ("data/foo/a/", FileHash.unknown()),
         ("data/foo/a/bar.txt", FileHash.unknown()),
@@ -2328,12 +2328,11 @@ def test_recurse_deferred_inputs3(wfp: Workflow):
     plan = wfp.find("step", "./plan.py")
     wfp.define_step(plan, "prog", inp_paths=["data/sub/other/deep.txt"])
     prog = wfp.find("step", "prog")
-    wfp.declare_missing(plan, ["data/"])
     wfp.defer_glob(plan, "data/**/")
     wfp.declare_missing(plan, ["data/sub/other/deep.txt"])
     rows = wfp.con.execute(RECURSE_DEFERRED_INPUTS, (prog.i,)).fetchall()
-    assert len(rows) == 2
-    assert {row[1] for row in rows} == {"data/sub/", "data/sub/other/"}
+    assert len(rows) == 3
+    assert {row[1] for row in rows} == {"data/", "data/sub/", "data/sub/other/"}
 
 
 def test_recreate_step_to_check(wfp: Workflow):

Original file line number	Diff line number	Diff line change
`@@ -3,5 +3,6 @@`
`3`	`3`	`"data/part1/foo.inp",`
`4`	`4`	`"data/part2/bar.inp",`
`5`	`5`	`"data/part2/foo.inp",`
`6`		`- "data/part2/sub/other.inp"`
	`6`	`+ "data/part2/sub/other.inp",`
	`7`	`+ "data/top.inp"`
`7`	`8`	`]`
Original file line number	Diff line number	Diff line change
`@@ -3,5 +3,6 @@`
`3`	`3`	`"data/part1/foo.out",`
`4`	`4`	`"data/part2/bar.out",`
`5`	`5`	`"data/part2/foo.out",`
`6`		`- "data/part2/sub/other.out"`
	`6`	`+ "data/part2/sub/other.out",`
	`7`	`+ "data/top.out"`
`7`	`8`	`]`