Add STEPUP_STEP_INP_DIGEST env var

Author: tovrstra

docs/changelog.md

@@ -12,6 +12,23 @@ and this project adheres to [Effort-based Versioning](https://jacobtomlinson.dev
 
 (no changes yet)
 
+## [3.0.1][] - 2025-05-13 {: #v3.0.1 }
+
+Minor tweaks, improved progress format and `STEPUP_STEP_INP_DIGEST` environment variable.
+
+### Added
+
+- The `STEPUP_STEP_INP_DIGEST` environment variable is set in the worker processes to
+  the hex-formatted digest of the inputs of the step.
+
+### Changed
+
+- Improved timer format of running steps in progress bar.
+
+### Fixed
+
+- Minor documentation and configuration fixes.
+
 ## [3.0.0][] - 2025-05-11 {: #v3.0.0 }
 
 Major release with breaking changes.
@@ -540,6 +557,7 @@ This release fixes several bugs.
 Initial release
 
 [Unreleased]: https://github.com/reproducible-reporting/stepup-core
+[3.0.1]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v3.0.1
 [3.0.0]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v3.0.0
 [2.1.7]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v2.1.7
 [2.1.6]: https://github.com/reproducible-reporting/stepup-core/releases/tag/v2.1.6

docs/reference/environment_variables.md

@@ -1,5 +1,7 @@
 # Environment Variables
 
+## Configuration of StepUp
+
 The following environment variables can be used to configure StepUp.
 
 - `STEPUP_DEBUG`: Set to `"1"` to enable debug output.
@@ -45,3 +47,17 @@ export STEPUP_ROOT=${PWD}
 
 When you change to the project directory (or any of its subdirectories) in your shell,
 the environment variables will be set automatically.
+
+## Environment Variables in Worker Processes
+
+The following environment variables are set when a worker runs a step:
+
+- `HERE` and `ROOT`, as documented in the tutorial on [`HERE` and `ROOT` variables](../advanced_topics/here_and_root.md)
+- `STEPUP_STEP_I` is a unique integer index for the current step.
+  This is mainly relevant for StepUp and has little significance for users implementing workflows.
+- `STEPUP_STEP_INP_DIGEST` is a hex-formatted digest of all the inputs to the step
+  (including environment variables used).
+  This is useful in special cases.
+  For example, it can be used to decide if cached results from a previously interrupted run of the step
+  are still valid.
+  It can also be useful when action submit jobs to a scheduler, to decide if a running job is still valid.

stepup/core/api.py

@@ -698,7 +698,8 @@ def getenv(
     path = path or back or multi
     value = os.getenv(name, default)
     # Do not amend environment variables set by the worker.
-    if name not in ["HERE", "ROOT"]:
+    # See stepup.core.worker.WorkerHandler.run
+    if name not in ["HERE", "ROOT", "STEPUP_STEP_I", "STEPUP_STEP_INP_DIGEST"]:
         amend(env=name)
     if multi:
         if value is None:

stepup/core/worker.py

@@ -475,6 +475,15 @@ class WorkerStep:
     inp_messages: list = attrs.field(init=False, factory=list)
     """Messages related to input validation issues: unexpected changes and deleted inputs."""
 
+    inp_digest: bytes = attrs.field(init=False, default=b"")
+    """The input digest, which can be useful for some actions.
+
+    They may use this to decide if cached results from a previously interrupted run
+    of the same stepo are still valid.
+    This can also be useful when action submit jobs to a scheduler,
+    to decide if a running job is still valid.
+    """
+
     out_missing: list = attrs.field(init=False, factory=list)
     """List of expected output files that were not created."""
 
@@ -794,7 +803,11 @@ def compute_inp_step_hash(
         label = self.step.action
         if self.step.workdir != "./":
             label += f"  # wd={self.step.workdir}"
-        return StepHash.from_inp(f"{label}", self.explain_rerun, all_inp_hashes, env_var_values), []
+        result = StepHash.from_inp(f"{label}", self.explain_rerun, all_inp_hashes, env_var_values)
+
+        # Copy the inp_digest, because it can be useful for some actions.
+        self.step.inp_digest = result.inp_digest
+        return result, []
 
     @allow_rpc
     def compute_out_step_hash(
@@ -897,11 +910,14 @@ async def run(self):
         await self.reporter("START", self.step.description)
         await self.reporter.start_step(self.step.description, self.step.i)
 
-        # For internal use only:
+        # For internal use in actions:
         os.environ["STEPUP_STEP_I"] = str(self.step.i)
         # Client code may use the following:
+        os.environ["STEPUP_STEP_INP_DIGEST"] = self.step.inp_digest.hex()
         os.environ["ROOT"] = str(Path.cwd().relpath(self.step.workdir))
         os.environ["HERE"] = str(self.step.workdir.relpath())
+        # Note: the variables defined here should be listed in stepup.core.api.getenv
+
         # Create IO redirection for stdout and stderr
         step_err = io.StringIO()
         step_out = io.StringIO()
@@ -922,6 +938,12 @@ async def run(self):
         self.step.stdout = step_out.getvalue()
         self.step.stderr = step_err.getvalue()
 
+        # Clean up environment variables (to avoid potential confusion)
+        del os.environ["STEPUP_STEP_I"]
+        del os.environ["STEPUP_STEP_INP_DIGEST"]
+        del os.environ["ROOT"]
+        del os.environ["HERE"]
+
         # Process results of the step.
         if self.show_perf:
             ru_final = resource.getrusage(resource.RUSAGE_CHILDREN)

tests/examples/inp_digest/.gitignore

@@ -0,0 +1,2 @@
+.stepup
+current_*

tests/examples/inp_digest/README.txt

@@ -0,0 +1 @@
+Minimal example checking that the input digest is present in the environment variables.

tests/examples/inp_digest/expected_graph.txt

@@ -0,0 +1,22 @@
+root:
+             creates   file:./
+             creates   file:plan.py
+             creates   step:runpy ./plan.py
+
+file:./
+               state = STATIC
+          created by   root:
+            supplies   file:plan.py
+            supplies   step:runpy ./plan.py
+
+file:plan.py
+               state = STATIC
+          created by   root:
+            consumes   file:./
+            supplies   step:runpy ./plan.py
+
+step:runpy ./plan.py
+               state = SUCCEEDED
+          created by   root:
+            consumes   file:./
+            consumes   file:plan.py

tests/examples/inp_digest/expected_stdout.txt

@@ -0,0 +1,9 @@
+        0/0 |    STARTUP │ (Re)initialized boot script
+        0/0 |   DIRECTOR │ Launched worker 0
+        0/1 |      PHASE │ run
+        0/1 |      START │ runpy ./plan.py
+        1/1 |    SUCCESS │ runpy ./plan.py
+        1/1 |   DIRECTOR │ Trying to delete 0 outdated output(s)
+        1/1 |      PHASE │ watch
+        1/1 |   DIRECTOR │ Stopping workers
+        1/1 |   DIRECTOR │ See you!

tests/examples/inp_digest/main.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env -S bash -x
+# Exit on first error and clean up.
+set -e
+trap 'kill $(pgrep -g $$ | grep -v $$) > /dev/null 2> /dev/null || :' EXIT
+rm -rvf $(cat .gitignore)
+
+# Run the example
+stepup boot -n 1 -w & # > current_stdout.txt &
+
+# Get the graph after completion of the pending steps.
+stepup wait
+stepup graph current_graph
+stepup join
+
+# Wait for background processes, if any.
+wait
+
+# Check files that are expected to be present and/or missing.
+[[ -f plan.py ]] || exit 1

tests/examples/inp_digest/plan.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from stepup.core.api import getenv
+
+digest = getenv("STEPUP_STEP_INP_DIGEST")
+assert digest is not None