fix(taskfiles): Output unexpected errors in checksum tasks; Call gtar on macos; Add unit tests for checksum tasks (fixes #67).

README.md

@@ -12,6 +12,14 @@ Before you submit a pull request, ensure you follow the testing and linting inst
 * [Task] 3.40 or higher
 * [uv] 0.7.10 or higher
 
+### MacOS
+
+The exported tasks use GNU utilities that are not always pre-installed on MacOS. You may need to
+install the following brew packages:
+
+* [coreutils]
+* [gnu-tar]
+
 ## Testing
 
 To run all tests:
@@ -42,5 +50,7 @@ To clean up any generated files:
 task clean
 ```
 
+[coreutils]: https://formulae.brew.sh/formula/coreutils
+[gnu-tar]: https://formulae.brew.sh/formula/gnu-tar
 [Task]: https://taskfile.dev/
 [uv]: https://docs.astral.sh/uv

exports/taskfiles/utils/checksum.yaml

@@ -1,20 +1,26 @@
 version: "3"
 
-set: ["u", "pipefail"]
-shopt: ["globstar"]
-
 tasks:
+  # Compute the checksum of the given path include patterns saving the result to `CHECKSUM_FILE`. A
+  # calling task can set `FAIL` to false if they wish to continue after errors.
+  #
   # @param {string} CHECKSUM_FILE
   # @param {string[]} INCLUDE_PATTERNS Path wildcard patterns to compute the checksum for.
   # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns, relative to any `INCLUDE_PATTERNS`,
   # to exclude from the checksum.
+  # @param {string} [FAIL="true"] If set to "false" the task will not fail.
   compute:
-    desc: "Tries to compute a checksum for the given paths and output it to a file."
     internal: true
     label: "{{.TASK}}-{{.CHECKSUM_FILE}}"
     silent: true
+    vars:
+      FAIL: "{{if eq \"false\" .FAIL}}false{{else}}true{{end}}"
+      TAR: "{{if eq OS \"darwin\"}}gtar{{else}}tar{{end}}"
+      TMP_ERR_LOG: "{{.CHECKSUM_FILE}}.log.tmp"
     requires:
-      vars: ["CHECKSUM_FILE", "INCLUDE_PATTERNS"]
+      vars:
+        - "CHECKSUM_FILE"
+        - "INCLUDE_PATTERNS"
     cmds:
       # We explicitly set `--no-anchored` and `--wildcards` to make the inclusion behaviour match
       # the default exclusion behaviour.
@@ -24,42 +30,58 @@ tasks:
       # input patterns cannot be quoted since they're evaluated by the shell and the results are
       # passed to `tar` as arguments. If the input patterns are passed to `tar` with quotes, the
       # pattern won't be evaluated and will instead be treated literally.
-      - >-
-        tar
-        --create
-        --file -
-        --group 0
-        --mtime "UTC 1970-01-01"
-        --numeric-owner
-        --owner 0
-        --sort name
-        --no-anchored
-        --wildcards
-        {{- range .EXCLUDE_PATTERNS}}
-        --exclude="{{.}}"
-        {{- end}}
-        {{- range .INCLUDE_PATTERNS}}
-        {{.}}
-        {{- end}}
-        2> /dev/null
-        | md5sum > {{.CHECKSUM_FILE}}
-    # Ignore errors so that dependent tasks don't fail
-    ignore_error: true
+      - defer: "rm -f '{{.TMP_ERR_LOG}}'"
+      - |-
+        if ! \
+          {{.TAR}} \
+          --create \
+          --file - \
+          --group 0 \
+          --mtime "UTC 1970-01-01" \
+          --numeric-owner \
+          --owner 0 \
+          --sort name \
+          --no-anchored \
+          --wildcards \
+          {{- range .EXCLUDE_PATTERNS}}
+          --exclude="{{.}}" \
+          {{- end}}
+          {{- range .INCLUDE_PATTERNS}}
+          {{.}} \
+          {{- end}}
+          2> {{.TMP_ERR_LOG}} \
+            | md5sum > {{.CHECKSUM_FILE}} \
+        ; then
+          {{- if eq "true" .FAIL}}
+            echo "[{{.TASK}} error] tar failed with:\n$(cat {{.TMP_ERR_LOG}})"
+            exit 1
+          {{- else}}
+            exit 0
+          {{- end}}
+        fi
 
+  # Validates the checksum of the given path include patterns matches the checksum in the given
+  # file. If validation fails the checksum file is deleted so that a calling task with the checksum
+  # file in their `generates` field will be reran. A calling task can set `FAIL` to true if they
+  # wish to halt on failure.
+  #
   # @param {string} CHECKSUM_FILE
   # @param {string[]} INCLUDE_PATTERNS Path wildcard patterns to validate the checksum for.
   # @param {string[]} [EXCLUDE_PATTERNS] Path wildcard patterns, relative to any `INCLUDE_PATTERNS`,
   # to exclude from the checksum.
+  # @param {string} [FAIL="false"] If set to "true" the task fails.
   validate:
-    desc: "Validates the checksum of the given directory matches the checksum in the given file, or
-    deletes the checksum file otherwise."
     internal: true
     label: "{{.TASK}}-{{.CHECKSUM_FILE}}"
     silent: true
     vars:
+      FAIL: "{{if eq \"true\" .FAIL}}true{{else}}false{{end}}"
       TMP_CHECKSUM_FILE: "{{.CHECKSUM_FILE}}.tmp"
+      TMP_ERR_LOG: "{{.CHECKSUM_FILE}}.log.tmp"
     requires:
-      vars: ["CHECKSUM_FILE", "INCLUDE_PATTERNS"]
+      vars:
+        - "CHECKSUM_FILE"
+        - "INCLUDE_PATTERNS"
     cmds:
       - task: "compute"
         vars:
@@ -68,14 +90,29 @@ tasks:
           EXCLUDE_PATTERNS:
             ref: "default (list) .EXCLUDE_PATTERNS"
           CHECKSUM_FILE: "{{.TMP_CHECKSUM_FILE}}"
-      - defer: "rm -f '{{.TMP_CHECKSUM_FILE}}'"
-      # Check that all paths exist and the checksum matches; otherwise delete the checksum file.
+          FAIL: "false"
+      - defer: |-
+          rm -f "{{.TMP_CHECKSUM_FILE}}"
+          rm -f "{{.TMP_ERR_LOG}}"
       - |-
         (
           {{- range .INCLUDE_PATTERNS}}
           for path in {{.}}; do
-            test -e "$path"
+            test -e "$path" \
+              || (echo "Include path does not exist: $path" > "{{$.TMP_ERR_LOG}}"; exit 1)
           done
           {{- end}}
-          diff -q "{{.TMP_CHECKSUM_FILE}}" "{{.CHECKSUM_FILE}}" 2> /dev/null
-        ) || rm -f "{{.CHECKSUM_FILE}}"
+          cmp -s "{{.TMP_CHECKSUM_FILE}}" "{{.CHECKSUM_FILE}}" \
+            || (
+              echo "cmp failed for '{{.TMP_CHECKSUM_FILE}}' '{{.CHECKSUM_FILE}}'" \
+                > "{{.TMP_ERR_LOG}}"
+              exit 1
+            )
+        ) || ( \
+          {{- if eq "true" .FAIL}}
+            echo "[{{.TASK}} error] failed with:\n$(cat {{.TMP_ERR_LOG}})"
+            exit 1
+          {{- else}}
+            rm -f "{{.CHECKSUM_FILE}}"
+          {{- end}}
+          )

taskfiles/checksum/tests.yaml

@@ -0,0 +1,148 @@
+version: "3"
+
+includes:
+  checksum:
+    internal: true
+    taskfile: "../../exports/taskfiles/utils/checksum.yaml"
+
+tasks:
+  default:
+    cmds:
+      - task: "checksum-test-rerun"
+      - task: "checksum-test-skip"
+      - task: "checksum-test-update"
+
+  checksum-test-rerun:
+    vars:
+      OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}"
+      SRC_DIR: "{{.OUTPUT_DIR}}/src"
+
+      CHECKSUM_FILE: "{{.SRC_DIR}}.md5"
+      CHECKSUM_FILE_REF: "{{.CHECKSUM_FILE}}.ref"
+      FILE_0: "{{.SRC_DIR}}/0.txt"
+      FILE_1: "{{.SRC_DIR}}/1.txt"
+    cmds:
+      - task: "checksum-test-init"
+        vars:
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+      - task: "create-dir-with-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          FILE_PATH: "{{.FILE_0}}"
+      - "mv '{{.CHECKSUM_FILE}}' '{{.CHECKSUM_FILE_REF}}'"
+      - task: "create-dir-with-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          FILE_PATH: "{{.FILE_1}}"
+
+      # Test create-dir-with-checksum ran the second time and created a different checksum.
+      - "test ! -e '{{.FILE_0}}'"
+      - "test -e '{{.FILE_1}}'"
+      - |-
+        if ! cmp -s '{{.CHECKSUM_FILE}}' '{{.CHECKSUM_FILE_REF}}'; then
+          exit 0
+        fi
+        exit 1
+
+  checksum-test-skip:
+    vars:
+      OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}"
+      SRC_DIR: "{{.OUTPUT_DIR}}/src"
+
+      CHECKSUM_FILE: "{{.SRC_DIR}}.md5"
+      CHECKSUM_MOD_TS: "{{.CHECKSUM_FILE}}-mod-ts.txt"
+      FILE_0: "{{.SRC_DIR}}/0.txt"
+      FILE_1: "{{.SRC_DIR}}/1.txt"
+    cmds:
+      - task: "checksum-test-init"
+        vars:
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+      - task: "create-dir-with-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          FILE_PATH: "{{.FILE_0}}"
+      - "date -r '{{.CHECKSUM_FILE}}' > '{{.CHECKSUM_MOD_TS}}'"
+      - task: "create-dir-with-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          FILE_PATH: "{{.FILE_1}}"
+
+      # Test create-dir-with-checksum didn't run the second time and the checksum is unmodified.
+      - "test -e '{{.FILE_0}}'"
+      - "test ! -e '{{.FILE_1}}'"
+      - "cmp -s '{{.CHECKSUM_MOD_TS}}' <(date -r '{{.CHECKSUM_FILE}}')"
+
+  checksum-test-update:
+    vars:
+      OUTPUT_DIR: "{{.G_OUTPUT_DIR}}/{{.TASK | replace \":\" \"#\"}}"
+      SRC_DIR: "{{.OUTPUT_DIR}}/src"
+
+      CHECKSUM_FILE: "{{.SRC_DIR}}.md5"
+      CHECKSUM_FILE_REF0: "{{.CHECKSUM_FILE}}.ref0"
+      CHECKSUM_FILE_REF1: "{{.CHECKSUM_FILE}}.ref1"
+      FILE_0: "{{.SRC_DIR}}/0.txt"
+      FILE_1: "{{.SRC_DIR}}/1.txt"
+    cmds:
+      - task: "checksum-test-init"
+        vars:
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+      - task: "create-dir-with-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          FILE_PATH: "{{.FILE_0}}"
+      - "cp '{{.CHECKSUM_FILE}}' '{{.CHECKSUM_FILE_REF0}}'"
+
+      - "cat '{{.CHECKSUM_FILE}}' > '{{.FILE_0}}'"
+      - task: "checksum:compute"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.SRC_DIR}}"]
+      - "cp '{{.CHECKSUM_FILE}}' '{{.CHECKSUM_FILE_REF1}}'"
+
+      - task: "create-dir-with-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          FILE_PATH: "{{.FILE_1}}"
+
+      # Test create-dir-with-checksum didn't run the second time and the updated checksum is
+      # different from the original.
+      - "test -e '{{.FILE_0}}'"
+      - "test ! -e '{{.FILE_1}}'"
+      - "cmp -s '{{.FILE_0}}'  '{{.CHECKSUM_FILE_REF0}}'"
+      - "cmp -s '{{.CHECKSUM_FILE}}' '{{.CHECKSUM_FILE_REF1}}'"
+      - |-
+        if ! cmp -s '{{.CHECKSUM_FILE}}' '{{.CHECKSUM_FILE_REF0}}'; then
+          exit 0
+        fi
+        exit 1
+
+  checksum-test-init:
+    internal: true
+    requires:
+      vars: ["OUTPUT_DIR"]
+    cmds:
+      - "rm -rf '{{.OUTPUT_DIR}}'"
+      - "mkdir -p '{{.OUTPUT_DIR}}'"
+
+  create-dir-with-checksum:
+    internal: true
+    vars:
+      DIR: "{{dir .FILE_PATH}}"
+    requires:
+      vars: ["CHECKSUM_FILE", "FILE_PATH"]
+    sources: ["{{.TASKFILE}}"]
+    generates: ["{{.CHECKSUM_FILE}}"]
+    deps:
+      - task: "checksum:validate"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.DIR}}"]
+    cmds:
+      - |-
+        rm -rf "{{.DIR}}"
+        mkdir -p "{{.DIR}}"
+        touch "{{.FILE_PATH}}"
+      - task: "checksum:compute"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          INCLUDE_PATTERNS: ["{{.DIR}}"]

taskfiles/tests.yaml

@@ -2,14 +2,16 @@ version: "3"
 
 includes:
   boost: "boost/tests.yaml"
+  checksum: "checksum/tests.yaml"
   remote: "remote/tests.yaml"
   ystdlib-py: "ystdlib-py/tests.yaml"
 
 tasks:
   all:
     internal: true
     cmds:
-      - task: "boost"
+      # - task: "boost"
+      - task: "checksum"
       - task: "remote"
       - task: "ystdlib-py"