Merge pull request #29 from openscm/resolution-strategies

Add tests of different resolution strategies

Author: znicholls

.github/workflows/ci.yaml

@@ -104,6 +104,53 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  tests-resolution-strategies:
+    strategy:
+      fail-fast: false
+      matrix:
+        # Only test on ubuntu here for now.
+        # We could consider doing this on different platforms too,
+        # although that probably belongs better with the PyPI tests.
+        os: [ "ubuntu-latest" ]
+        # Tests with lowest direct resolution.
+        # We don't do lowest because that is essentially testing
+        # whether downstream dependencies
+        # have set their minimum support dependencies correctly,
+        # which isn't our problem to solve.
+        resolution-strategy: [ "lowest-direct" ]
+        # Only test against the oldest supported python version
+        # because python is itself a direct dependency
+        # (so we're testing against the lowest direct python too).
+        python-version: [ "3.9" ]
+    runs-on: "${{ matrix.os }}"
+    defaults:
+      run:
+        # This might be needed for Windows
+        # and doesn't seem to affect unix-based systems so we include it.
+        # If you have better proof of whether this is needed or not,
+        # feel free to update.
+        shell: bash
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Setup uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.5.11"
+          python-version: ${{ matrix.python-version }}
+      - name: Create venv
+        run: |
+          uv venv --seed
+      - name: Install dependencies
+        run: |
+          uv pip install --requirements requirements-only-tests-locked.txt
+          uv pip compile --python ${{ matrix.python-version }} --resolution ${{ matrix.resolution-strategy }} --all-extras pyproject.toml -o requirements-tmp.txt
+          uv pip install -r requirements-tmp.txt .
+      - name: Run tests
+        run: |
+          uv run --no-sync pytest tests -r a -v
+
   tests-without-extras:
     # Run the tests without installing extras.
     # This is just a test to make sure to avoid
@@ -135,7 +182,7 @@ jobs:
         run: |
           pip install --upgrade pip wheel
           pip install .
-          pip install -r requirements-only-tests-locked.txt
+          pip install -r requirements-only-tests-min-locked.txt
       - name: Run tests
         run: |
           pytest tests -r a -vv tests

.github/workflows/install-pypi.yaml

@@ -22,7 +22,8 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: [ "3.9", "3.10", "3.11" ]
+        # Test against all security and bugfix versions: https://devguide.python.org/versions/
+        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
         # Check both 'library' install and the 'application' (i.e. locked) install
         install-target: ["continuous-timeseries", "continuous-timeseries[locked]"]
     runs-on: "${{ matrix.os }}"
@@ -67,7 +68,7 @@ jobs:
         python scripts/test-install.py
     - name: Install min test dependencies
       run: |
-        pip install pytest pytest-regressions
+        pip install -r requirements-only-tests-min-locked.txt
     - name: Run tests
       run: |
         # Can't run doctests here because the paths are different.

.github/workflows/test-upstream-latest.yaml

@@ -0,0 +1,57 @@
+name: Test against upstream latest
+
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    # This means At 03:00 on Wednesday.
+    # see https://crontab.guru/#0_0_*_*_3
+    - cron:  '0 3 * * 3'
+
+jobs:
+  tests-upstream-latest:
+    strategy:
+      fail-fast: false
+      matrix:
+        # Only test on ubuntu here for now.
+        # We could consider doing this on different platforms too,
+        # but this is mainly a warning for us of what is coming,
+        # rather than a super robust dive.
+        os: [ "ubuntu-latest" ]
+        # Test against all bugfix versions: https://devguide.python.org/versions/
+        # as they are latest and ones most likely to support new features
+        python-version: [ "3.12", "3.13" ]
+    runs-on: "${{ matrix.os }}"
+    defaults:
+      run:
+        # This might be needed for Windows
+        # and doesn't seem to affect unix-based systems so we include it.
+        # If you have better proof of whether this is needed or not,
+        # feel free to update.
+        shell: bash
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Setup uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "0.5.11"
+          python-version: ${{ matrix.python-version }}
+      - name: Setup compilation dependencies
+        run: |
+          echo "python${{ matrix.python-version }}-dev"
+          sudo add-apt-repository ppa:deadsnakes/ppa -y
+          sudo apt update
+          sudo apt install -y libopenblas-dev "python${{ matrix.python-version }}-dev"
+      - name: Create venv
+        run: |
+          uv venv --seed
+      - name: Install dependencies
+        run: |
+          uv pip install --requirements requirements-only-tests-locked.txt --requirements requirements-only-tests-min-locked.txt
+          uv pip install --all-extras pyproject.toml
+          uv pip install -r requirements-upstream-dev.txt
+      - name: Run tests
+        run: |
+          uv run --no-sync pytest tests -r a -v

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ ci:
   autoupdate_schedule: quarterly
   autoupdate_branch: pre-commit-autoupdate
   # Currently network access isn't supported in the pre-commit CI product.
-  skip: [uv-lock, uv-export, pdm-lock-check]
+  skip: [uv-sync, uv-lock, uv-export, pdm-lock-check]
 
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
@@ -39,9 +39,9 @@ repos:
         args: [ --fix, --exit-non-zero-on-fix ]
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    # uv version.
-    rev: 0.5.11
+    rev: 0.5.21
     hooks:
+      - id: uv-sync
       - id: uv-lock
         name: uv-lock-check
         args: ["--check"]
@@ -55,11 +55,12 @@ repos:
       - id: uv-export
         name: export-requirements-docs
         args: ["-o", "requirements-docs-locked.txt", "--no-hashes", "--no-dev", "--no-emit-project", "--all-extras", "--group", "docs"]
+      - id: uv-export
+        name: export-requirements-only-tests-min
+        args: ["-o", "requirements-only-tests-min-locked.txt", "--no-hashes", "--no-dev", "--no-emit-project", "--only-group", "tests-min"]
       - id: uv-export
         name: export-requirements-only-tests
         args: ["-o", "requirements-only-tests-locked.txt", "--no-hashes", "--no-dev", "--no-emit-project", "--only-group", "tests"]
-      # # Not released yet
-      # - id: uv-sync
   - repo: https://github.com/pdm-project/pdm
     rev: 2.22.1
     hooks:

changelog/29.docs.md

@@ -0,0 +1 @@
+Added documentation on our dependency pinning and testing strategy.

changelog/29.fix.md

@@ -0,0 +1 @@
+Fixed the minimum versions of our requirements (also tested that installation with minimum versions works).

changelog/29.trivial.md

@@ -0,0 +1 @@
+Added testing of our lowest direct dependencies.

docs/NAVIGATION.md

@@ -17,6 +17,7 @@ See https://oprypin.github.io/mkdocs-literate-nav/
     - [Why this API?](further-background/why-this-api.py)
     - [Discrete to continuous conversions](further-background/discrete_to_continuous_conversions.py)
     - [Representations](further-background/representations.py)
+    - [Dependency pinning and testing strategy](further-background/dependency-pinning-and-testing.md)
 - [Development](development.md)
 - [Pandas accessors](pandas-accessors.md)
 - [API reference](api/continuous_timeseries/)

docs/further-background/dependency-pinning-and-testing.md

@@ -0,0 +1,139 @@
+# Dependency pinning and associated testing strategy
+
+<!--
+    This text comes from the copier template.
+    If you find you need to update your testing strategy,
+    you probably want to update this too.
+-->
+Here we explain our dependency pinning and associated testing strategy.
+This will help you, as a user, to know what to expect
+and what your options are.
+As a developer, these docs can also be helpful to understand
+the overall philosophy and thinking.
+
+## Dependency pinning
+
+We use lower-bound pinning.
+In other words, we pin the lowest supported version of the packages on which we depend.
+As a user, this helps you get a working install
+while giving you freedom to use newer versions, should you wish.
+
+We don't use upper-bound pins.
+The reason is that we have had bad experiences with upper-bound pinning.
+In the majority of cases, new releases do not cause issues
+so pinning simply forces users to workaround overly strict pins[^1]
+(which can be done, see
+[working around incorrectly set pins][working-around-incorrectly-set-pins]).
+The tradeoff with this approach is
+This does run the risk that,
+if a dependency releases a breaking change,
+the function provided by our package may break too.
+
+[^1]:
+    Yes, if the entire world followed semantic versioning perfectly,
+    we could use upper-bound pins for the next major version with more confidence
+    but that isn't the current state of the ecosystem.
+    Even if it were, we still think this would result in unnecessary pins
+    in many cases because many major releases are still compatible
+    because most packages don't use the entire API of their dependencies.
+
+### Working around incorrectly set pins
+
+Despite our best efforts, it is possible that we will set our pins incorrectly.
+Part of this is because we simply cannot test all possible combinations of package installs
+(see [testing strategy][testing-strategy]),
+so we might miss valid/invalid combinations.
+
+If we set our pins incorrectly and you need to effectively overwrite them,
+unfortunately there is currently no universal solution.
+There has been quite some discussion,
+see e.g. [this issue](https://github.com/pypa/pip/issues/8076),
+but no universal resolution.
+
+However, for some environment managers, there is a solution.
+This comes in the form of dependency overrides,
+which allow you to override a package's stated dependencies
+(essentially fixing them on the fly,
+rather than having to fix them upstream).
+Here are the docs for the package managers that we know support this:
+
+- [uv dependency overrides](https://docs.astral.sh/uv/concepts/resolution/#dependency-overrides).
+- [pdm dependency overrides](https://pdm-project.org/latest/usage/dependency/#dependency-overrides).
+
+We do not know if this strategy can be used for packaging.
+For example, you are building package A.
+This depends on version 2 of package B and version 1 of package C.
+However, version 1 of package C (incorrectly) says
+that it is only compatible with version 1 of package B.
+We are not sure if the dependency overrides
+can be used to release a version of package A
+that can be relased to and installed from PyPI.
+If this is the situation you are in and you would like a resolution,
+please comment on [this issue](https://gitlab.com/openscm/copier-core-python-repository/-/issues/4).
+
+## Testing strategy
+
+We test against multiple python versions in our CI.
+These tests run with the latest compatible versions of our dependencies
+and a 'full' installation, i.e. with all optional dependencies too.
+This gives us the best possible coverage of our code base
+against the latest compatible version of all our possible dependencies.
+
+In an attempt to anticipate changes to the API's of our key dependencies,
+we also test against the latest unreleased version of our key dependencies once a week.
+As a user, this probably won't matter too much,
+except that it should reduce the chance
+that a new release of one of our dependencies breaks our package
+without us knowing in advance and being able to set a pin in anticipation.
+As a developer, this is important to be aware of,
+so we can anticipate changes as early as possible.
+
+We additionally test with the lowest/oldest compatible versions of our direct dependencies.
+This includes Python, i.e. these tests are only run
+with the lowest/oldest version of Python compatible with our project.
+This is because Python is itself a dependency of our project
+and newer versions of Python tend to not work
+with the lowest/oldest versions of our direct dependencies.
+These tests ensure that our minimum supported versions are actually supported
+(if they are all installed simultaneously,
+see the next paragraph for why this caveat matters).
+As a note for developers,
+the key trick to making this work is to use `uv pip compile`
+rather than `uv run` (or similar) in the CI.
+The reason is that `uv pip compile`
+allows you to install dependencies for a very specific combination of things,
+which is different to `uv`'s normal 'all-at-once' environment handling
+(for more details, see [here](https://github.com/astral-sh/uv/issues/10774#issuecomment-2601925564)).
+
+We do not test the combinations in between lowest-supported and latest,
+e.g. the oldest compatible version of package A
+with the newest compatiable version of package B.
+The reason for this is simply combinatorics,
+it is generally not feasible
+for us to test all possible combinations of our dependencies' versions.
+
+We also don't test with the oldest versions of our dependencies' dependencies.
+We don't do this because, in practice,
+all that such tests actually test is
+whether our dependencies have set their minimum support dependencies correctly,
+which isn't our problem to solve.
+
+Once a week, we also test what happens when a user installs from PyPI on the 'happy path'.
+In other words, they do `pip install continuous-timeseries`.
+We check that such an install passes all the tests that don't require extras
+(for developers, this is why we have `tests-min` and `tests-full` dev dependency groups,
+they allow us to test a truly minimal testing environment,
+separate from any extras we install to get full coverage).
+Finally, we also check the installation of the locked versions of the package,
+i.e. installation with `pip install 'continuous-timeseries[locked]'`.
+These tests give us the greatest coverage of Python versions and operating systems
+and help alert us to places where users may face issues.
+Having said that, these tests do require 30 separate CI runs,
+which is why we don't run them in CI.
+
+Through this combination of CI testing and installation testing,
+we get a pretty good coverage of the different ways in which our package can be used.
+It is not perfect, largely because the combinatorics become unfriendly.
+If we find a particular, key, use case failing often,
+then we would happily discuss whether this should be included in the CI too,
+to catch issues earlier than at user time.

docs/further-background/representations.py

@@ -30,6 +30,10 @@
 #   (see https://github.com/hgrecco/pint/blob/74b708661577623c0c288933d8ed6271f32a4b8b/pint/util.py#L1004)
 #
 # In short, we try and have as nice an experience for developers as possible.
+#
+# (As one other note/trick for representation of objects,
+# you can control how numpy represents its objects using
+# [numpy.set_printoptions](https://numpy.org/doc/stable/reference/generated/numpy.set_printoptions.html)).
 
 # %% [markdown]
 # ## Imports