Add support for multiple test suites (#12)

* First pass tweaks

* Add setup to skip tests and provide formatted file name

* Fix format output file path/name

* Fix plugin.yml

* tweak syntax

* Fix var sub

* Fix sum command

* Tidy up code and add docs

* Code review changes

* Remove quotes

* Ensure DS_Store is ignored

* Revert code review change

* Add quotes to coverage/"${formatted_file_prefix}".*.json

* Remove quotes

* Fix readme

Author: timReynolds

.gitignore

@@ -0,0 +1,33 @@
+# Created by https://www.toptal.com/developers/gitignore/api/macos
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# End of https://www.toptal.com/developers/gitignore/api/macos

README.md

@@ -8,7 +8,7 @@ to report coverage with the Code Climate test reporter
 
 https://github.com/codeclimate/test-reporter
 
-Plugin can handle single, or multiple (parallel), tests.
+Plugin can handle single, [parallel](https://docs.codeclimate.com/docs/configuring-test-coverage#parallel-tests) and [multiple](https://docs.codeclimate.com/docs/configuring-test-coverage#multiple-test-suites) test suites.
 
 Also see: https://docs.codeclimate.com/docs/configuring-test-coverage
 
@@ -20,14 +20,16 @@ This plugin will download build artifact(s) generated by a previous step, compil
 steps:
   - label: ":codeclimate: Report coverage"
     plugins:
-      - jobready/codeclimate-test-reporter#v2.0:
+      - jobready/codeclimate-test-reporter#v2.2:
           artifact: "coverage/.resultset.json"
           input_type: simplecov
           prefix: /app
     env:
       CC_TEST_REPORTER_ID:
 ```
 
+### Parallel tests
+
 If you are running parallel builds you'll probably want to create uniquely
 named artifacts. That can be done by renaming the `.resultset.json` file
 after your tests have run:
@@ -36,6 +38,86 @@ after your tests have run:
 mv -v coverage/.resultset.json coverage/.resultset${BUILDKITE_PARALLEL_JOB}.json
 ```
 
+### Multiple test suites
+
+If your repository has more than one test suite you'll need to format each artifact (coverage file) before the result can be summed and reported to codeclimate.
+
+To do this you'll need to configure multiple steps persisting the formated codeclimate compatable configuration files as artifacts between steps.
+
+This is best shown via an example;
+
+```
+  - label: ":rails: Tests"
+    key: "rails-tests"
+    command: "rails db:test:prepare && rails test"
+    env:
+      RUBY_OPT: "-W:deprecated"
+    depends_on:
+      - "build-base"
+    plugins:
+      - docker-compose#v3.7.0:
+          run: base
+          config: .buildkite/docker-compose.yml
+          volumes:
+            - "./coverage:/app/coverage"
+    artifact_paths:
+      - "coverage/**/*"
+
+  - label: ":codeclimate: Format Rails Coverage"
+    key: "codeclimate-format-rails-report"
+    plugins:
+      - jobready/codeclimate-test-reporter#v2.2:
+          artifact: "coverage/coverage.json"
+          input_type: simplecov
+          prefix: /app
+          report: false
+          file_prefix: "codeclimate.rails"
+    depends_on:
+      - "rails-tests"
+    artifact_paths:
+      - "coverage/codeclimate.**"
+
+  - label: ":react: Tests"
+    key: "react-tests"
+    command: "yarn test"
+    depends_on:
+      - "build-base"
+    plugins:
+      - docker-compose#v3.7.0:
+          run: base
+          config: .buildkite/docker-compose.yml
+          volumes:
+            - "./coverage:/app/coverage"
+    artifact_paths:
+      - "coverage/**/*"
+
+  - label: ":codeclimate: Format React Coverage"
+    key: "codeclimate-format-react-report"
+    plugins:
+      - jobready/codeclimate-test-reporter#v2.2:
+          artifact: "coverage/lcov.info"
+          input_type: lcov
+          prefix: /app
+          report: false
+          file_prefix: "codeclimate.react"
+    depends_on:
+      - "react-tests"
+    artifact_paths:
+      - "coverage/codeclimate.**"
+
+  - label: ":codeclimate: Report coverage"
+    key: "codeclimate-coverage-report"
+    plugins:
+      - jobready/codeclimate-test-reporter#v2.2:
+          artifact: "coverage/codeclimate**"
+          format: false
+    env:
+      CC_TEST_REPORTER_ID:
+    depends_on:
+      - "codeclimate-format-rails-report"
+      - "codeclimate-format-react-report"
+```
+
 ## Configuration
 
 ### `artifact` (required)
@@ -86,11 +168,26 @@ Set to true to enable cc-test-reporter --debug flag
 
 Example: `true`
 
+### `format` (optional, default `true`)
+
+Controls if formatting of artifacts should be performed. Set to false when wanting to upload artifacts formatted in a previous step.
+
+Example: `false`
+
+### `report` (optional, default `true`)
+
+Controls if coverage reporting to codeclimate should be performed. Set to false when needing to parse multiple test suites for later upload.
+
+Example: `false`
+
+### `file_prefix` (optional, default `codeclimate`)
+
+Controls the name outputs files formatted by the codeclimate test reporter. Required to stop artifact names clashing when processing muiltple test suites.
+
 ### `CC_TEST_REPORTER_ID` (required)
 
 The `CC_TEST_REPORTER_ID` environment variable must be configured.
 
-
 ## Linting
 
 To run the [Buildkite Plugin Linter](https://github.com/buildkite-plugins/buildkite-plugin-linter), run

hooks/command

@@ -15,21 +15,26 @@ install_reporter() {
 }
 
 download_artifacts() {
-  printf -- "--- :buildkite: downloading artifact\\n"
+  printf -- "--- :buildkite: Downloading artifact\\n"
   buildkite-agent artifact download "${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_ARTIFACT}" ./
 }
 
 format_file() {
-  printf -- "Formatting file: %s\\n" "$1"
+  output_file="coverage/$formatted_file_prefix.$2.json"
+  printf -- "Formatting file: %s to %s\\n" "$1" "$output_file"
   ./cc-test-reporter format-coverage ${debug:+"-d"} \
     --input-type "${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_INPUT_TYPE}" \
     --prefix "${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_PREFIX}" \
-    --output "coverage/codeclimate.$2.json" \
+    --output "${output_file}" \
     "$1"
 }
 
-report_coverage() {
-  printf -- "--- :codeclimate: reporting coverage\\n"
+format_files() {
+  if [ "$format" = false ] ; then
+    return
+  fi
+
+  printf -- "--- :codeclimate: Formatting coverage\\n"
 
   count=1
   for f in ${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_ARTIFACT}; do
@@ -38,13 +43,22 @@ report_coverage() {
   done
 
   wait
+}
+
+report_coverage() {
+  if [ "$report" = false ] ; then
+    return
+  fi
+
+  printf -- "--- :codeclimate: Reporting coverage\\n"
 
   if [[ -v BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_PARTS ]] ; then
     SUM_PARTS="--parts ${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_PARTS}"
   else
     SUM_PARTS=""
   fi
-  ./cc-test-reporter sum-coverage ${debug:+"-d"} ${SUM_PARTS} coverage/codeclimate.*.json
+
+  ./cc-test-reporter sum-coverage ${debug:+"-d"} ${SUM_PARTS} coverage/${formatted_file_prefix}.*.json
 
   ./cc-test-reporter upload-coverage ${debug:+"-d"}
 }
@@ -56,6 +70,11 @@ fi
 debug=""
 [[ -v BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_DEBUG ]] && debug="yes"
 
+format=${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_FORMAT:-true}
+formatted_file_prefix=${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_FILE_PREFIX:-"codeclimate"} 
+report=${BUILDKITE_PLUGIN_CODECLIMATE_TEST_REPORTER_REPORT:-true}
+
 install_reporter
 download_artifacts
+format_files
 report_coverage

plugin.yml

@@ -16,4 +16,10 @@ configuration:
       type: integer
     debug:
       type: string
+    format:
+      type: boolean
+    report:
+      type: boolean
+    file_prefix:
+      type: string
   required: [artifact, input_type]