Merge pull request #105 from bitrise-io/bazel-fields

feat: ACI-3563 collect host cache config metadata

Author: zsolt-marta-bitrise

bitrise.yml

@@ -91,11 +91,6 @@ workflows:
         machine_type_id: g2-m1.8core
         stack: osx-xcode-edge
   test:
-    triggers:
-      push:
-      - branch: "main"
-      pull_request:
-      - source_branch: "*"
     steps:
     - activate-ssh-key@4:
         run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}'
@@ -266,11 +261,6 @@ workflows:
     envs:
     - TEST_APP_URL: [email protected]:duckduckgo/Android.git
     - BRANCH: develop
-    triggers:
-      push:
-      - branch: "main"
-      pull_request:
-      - source_branch: "*"
     steps:
     - bundle::feature-e2e-setup: {}
     - script:
@@ -284,27 +274,23 @@ workflows:
         inputs:
         - content: |-
             set -exo pipefail
-            (./gradlew debug 2>&1) | tee logs.txt
+            (./gradlew debug 2>&1) | tee "$BITRISE_DEPLOY_DIR/logs.txt"
     - script:
         title: Check for cache invocations
         inputs:
         - content: |-
             set -exo pipefail
             
-            ../scripts/check_pattern.sh "$PWD/logs.txt" \
+            ../scripts/check_pattern.sh "$BITRISE_DEPLOY_DIR/logs.txt" \
               '\[Bitrise Build Cache\].*🤖 Bitrise remote cache enabled' \
               '\[Bitrise Build Cache\].*Request metadata invocationId' \
               '\[Bitrise Analytics\].*🤖 Bitrise analytics enabled for tasks.*Invocation ID: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+    - deploy-to-bitrise-io@2: {}
 
   feature-e2e-gradle-bitwarden:
     envs:
     - TEST_APP_URL: [email protected]:bitwarden/android.git
     - BRANCH: main
-    triggers:
-      push:
-      - branch: "main"
-      pull_request:
-      - source_branch: "*"
     steps:
     - bundle::feature-e2e-setup: {}
     - script:
@@ -345,28 +331,27 @@ workflows:
         inputs:
         - content: |-
             set -exo pipefail
-            (./gradlew debug --configuration-cache 2>&1) | tee logs.txt
+            (./gradlew debug --configuration-cache 2>&1) | tee "$BITRISE_DEPLOY_DIR/logs.txt"
     - script:
         title: Check for cache invocations
         inputs:
         - content: |-
             set -exo pipefail
 
-            ../scripts/check_pattern.sh "$PWD/logs.txt" \
+            ../scripts/check_pattern.sh "$BITRISE_DEPLOY_DIR/logs.txt" \
               'Reusing configuration cache.' \
               '\[Bitrise Build Cache\].*🤖 Bitrise remote cache enabled' \
               '\[Bitrise Build Cache\].*Request metadata invocationId' \
               '\[Bitrise Analytics\].*🤖 Bitrise analytics enabled for tasks.*Invocation ID: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
+    - deploy-to-bitrise-io@2: {}
 
   feature-e2e-bazel:
+    meta:
+      bitrise.io:
+        stack: linux-docker-android-22.04
     envs:
     - TEST_APP_URL: [email protected]:bitrise-io/bazel.git
     - BRANCH: master
-    triggers:
-      push:
-      - branch: "main"
-      pull_request:
-      - source_branch: "*"
     steps:
     - bundle::feature-e2e-setup: {}
     - script@1:
@@ -387,17 +372,23 @@ workflows:
         inputs:
         - content: |-
             set -exo pipefail
-            (/usr/local/bin/bazel build //src:bazel-dev --announce_rc --copt='-w' 2>&1) | tee logs.txt
+            (/usr/local/bin/bazel build //src:bazel-dev --announce_rc --copt='-w' 2>&1) | tee "$BITRISE_DEPLOY_DIR/logs.txt"
     - script:
         title: Check for cache invocations
         inputs:
         - content: |-
             #!/bin/bash
             set -exo pipefail
             
-            ../scripts/check_pattern.sh "$PWD/logs.txt" \
+            ../scripts/check_pattern.sh "$BITRISE_DEPLOY_DIR/logs.txt" \
               '--remote_cache=grpcs:\/\/remote-build-cache\.services\.bitrise\.io' \
-              'Invocation ID: '
+              'Invocation ID: ' \
+              '--bes_header=x-os=Linux' \
+              '--bes_header=x-cpu-cores=[0-9]+' \
+              '--bes_header=x-mem-size=[0-9]+' \
+              '--bes_header=x-locale=\w+' \
+              '--bes_header=x-default-charset=\w+' \
+    - deploy-to-bitrise-io@2: {}
 
 step_bundles:
   generate_gradle_verification_reference:
@@ -559,6 +550,7 @@ step_bundles:
 pipelines:
   features-e2e:
     workflows:
+      test: {}
       feature-e2e-bazel: {}
       feature-e2e-gradle-duckduck: {}
       feature-e2e-gradle-bitwarden: {}

cmd/client.go

@@ -20,6 +20,7 @@ type CreateKVClientParams struct {
 	ClientName       string
 	AuthConfig       common.CacheAuthConfig
 	EnvProvider      common.EnvProviderFunc
+	CommandFunc      common.CommandFunc
 	Logger           log.Logger
 }
 
@@ -44,7 +45,7 @@ func createKVClient(ctx context.Context,
 		ClientName:          params.ClientName,
 		AuthConfig:          params.AuthConfig,
 		Logger:              params.Logger,
-		CacheConfigMetadata: common.NewCacheConfigMetadata(params.EnvProvider),
+		CacheConfigMetadata: common.NewCacheConfigMetadata(params.EnvProvider, params.CommandFunc, params.Logger),
 		CacheOperationID:    params.CacheOperationID,
 	})
 	if err != nil {

cmd/deleteXcodeDerivedData.go

@@ -1,12 +1,10 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
-
-	"github.com/spf13/cobra"
-
-	"context"
+	"os/exec"
 	"strings"
 	"time"
 
@@ -17,6 +15,7 @@ import (
 	"github.com/bitrise-io/bitrise-build-cache-cli/internal/xcode"
 	"github.com/bitrise-io/go-utils/v2/log"
 	"github.com/google/uuid"
+	"github.com/spf13/cobra"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -38,7 +37,13 @@ var deleteXcodeDerivedDataCmd = &cobra.Command{
 		cacheKey, _ := cmd.Flags().GetString("key")
 		empty, _ := cmd.Flags().GetBool("empty")
 
-		if err := deleteXcodeDerivedDataCmdFn(cmd.Context(), cacheKey, empty, logger, os.Getenv); err != nil {
+		if err := deleteXcodeDerivedDataCmdFn(cmd.Context(), cacheKey, empty, logger,
+			os.Getenv,
+			func(name string, v ...string) (string, error) {
+				output, err := exec.Command(name, v...).Output()
+
+				return string(output), err
+			}); err != nil {
 			return fmt.Errorf("delete Xcode DerivedData into Bitrise Build Cache: %w", err)
 		}
 
@@ -55,7 +60,12 @@ func init() {
 	deleteXcodeDerivedDataCmd.Flags().Bool("empty", false, "If true, upload an empty metadata")
 }
 
-func deleteXcodeDerivedDataCmdFn(ctx context.Context, providedCacheKey string, uploadEmpty bool, logger log.Logger, envProvider func(string) string) error {
+func deleteXcodeDerivedDataCmdFn(ctx context.Context,
+	providedCacheKey string,
+	uploadEmpty bool,
+	logger log.Logger,
+	envProvider func(string) string,
+	commandFunc func(string, ...string) (string, error)) error {
 	logger.Infof("(i) Check Auth Config")
 	authConfig, err := common.ReadAuthConfigFromEnvironments(envProvider)
 	if err != nil {
@@ -79,6 +89,7 @@ func deleteXcodeDerivedDataCmdFn(ctx context.Context, providedCacheKey string, u
 			ClientName:       ClientNameXcode,
 			AuthConfig:       authConfig,
 			EnvProvider:      envProvider,
+			CommandFunc:      commandFunc,
 			Logger:           logger,
 		})
 	if err != nil {

cmd/enableForBazel.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	bazelconfig "github.com/bitrise-io/bitrise-build-cache-cli/internal/config/bazel"
@@ -79,7 +80,13 @@ func enableForBazelCmdFn(logger log.Logger, homeDirPath string, envProvider func
 		}
 	}
 	// Metadata
-	cacheConfig := common.NewCacheConfigMetadata(os.Getenv)
+	cacheConfig := common.NewCacheConfigMetadata(os.Getenv,
+		func(name string, v ...string) (string, error) {
+			output, err := exec.Command(name, v...).Output()
+
+			return string(output), err
+		},
+		logger)
 	logger.Infof("(i) Cache Config: %+v", cacheConfig)
 
 	logger.Infof("(i) Check ~/.bazelrc")

cmd/enableForGradle.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	"github.com/bitrise-io/bitrise-build-cache-cli/internal/config/common"
@@ -125,7 +126,12 @@ func enableForGradleCmdFn(logger log.Logger, gradleHomePath string, envProvider
 		return errInvalidCacheLevel
 	}
 	// Metadata
-	cacheConfigMetadata := common.NewCacheConfigMetadata(os.Getenv)
+	cacheConfigMetadata := common.NewCacheConfigMetadata(os.Getenv,
+		func(name string, v ...string) (string, error) {
+			output, err := exec.Command(name, v...).Output()
+
+			return string(output), err
+		}, logger)
 	logger.Infof("(i) Cache Config Metadata: %+v", cacheConfigMetadata)
 
 	prefs := gradleconfig.Preferences{

cmd/gradleVerificationAddDeps.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"os/exec"
 
 	"github.com/bitrise-io/bitrise-build-cache-cli/internal/config/common"
 	gradleconfig "github.com/bitrise-io/bitrise-build-cache-cli/internal/config/gradle"
@@ -57,7 +58,11 @@ func addGradlePluginsFn(logger log.Logger, gradleHomePath string, envProvider fu
 	logger.Infof("(i) Debug mode and verbose logs: %t", isDebugLogMode)
 
 	// Metadata
-	cacheConfigMetadata := common.NewCacheConfigMetadata(os.Getenv)
+	cacheConfigMetadata := common.NewCacheConfigMetadata(os.Getenv, func(name string, v ...string) (string, error) {
+		output, err := exec.Command(name, v...).Output()
+
+		return string(output), err
+	}, logger)
 	logger.Infof("(i) Cache Config Metadata: %+v", cacheConfigMetadata)
 
 	authToken := "placeholder-token"

cmd/restoreGradleConfigurationCache.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"strings"
 
 	"github.com/bitrise-io/bitrise-build-cache-cli/internal/build_cache/kv"
@@ -44,7 +45,12 @@ var restoreGradleConfigCacheCmd = &cobra.Command{
 			authConfig,
 			cacheKey,
 			logger,
-			os.Getenv)
+			os.Getenv,
+			func(name string, v ...string) (string, error) {
+				output, err := exec.Command(name, v...).Output()
+
+				return string(output), err
+			})
 		if err != nil {
 			return fmt.Errorf("restore Gradle config cache from Bitrise Build Cache: %w", err)
 		}
@@ -65,13 +71,15 @@ func restoreGradleConfigCacheCmdFn(ctx context.Context,
 	authConfig common.CacheAuthConfig,
 	providedCacheKey string,
 	logger log.Logger,
-	envProvider func(string) string) error {
+	envProvider func(string) string,
+	commandFunc func(string, ...string) (string, error)) error {
 	kvClient, err := createKVClient(ctx,
 		CreateKVClientParams{
 			CacheOperationID: uuid.NewString(),
 			ClientName:       ClientNameGradleConfigCache,
 			AuthConfig:       authConfig,
 			EnvProvider:      envProvider,
+			CommandFunc:      commandFunc,
 			Logger:           logger,
 		})
 	if err != nil {

cmd/restoreXcodeDerivedDataFiles.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"os/user"
 	"strings"
 	"time"
@@ -60,8 +61,24 @@ var restoreXcodeDerivedDataFilesCmd = &cobra.Command{
 			return fmt.Errorf("read auth config from environments: %w", err)
 		}
 
-		op, cmdError := restoreXcodeDerivedDataFilesCmdFn(cmd.Context(), authConfig, XCodeCacheMetadataPath, projectRoot,
-			cacheKey, logger, tracker, startT, os.Getenv, isDebugLogMode, skipExisting, forceOverwrite, maxLoggedErrors)
+		op, cmdError := restoreXcodeDerivedDataFilesCmdFn(cmd.Context(),
+			authConfig,
+			XCodeCacheMetadataPath,
+			projectRoot,
+			cacheKey,
+			logger,
+			tracker,
+			startT,
+			os.Getenv,
+			func(name string, v ...string) (string, error) {
+				output, err := exec.Command(name, v...).Output()
+
+				return string(output), err
+			},
+			isDebugLogMode,
+			skipExisting,
+			forceOverwrite,
+			maxLoggedErrors)
 		if op != nil {
 			if cmdError != nil {
 				errStr := cmdError.Error()
@@ -97,15 +114,24 @@ func init() {
 	restoreXcodeDerivedDataFilesCmd.Flags().Int("max-logged-errors", 150, "The maximum number of errors logged to the console during restoring the cache.")
 }
 
-func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.CacheAuthConfig, cacheMetadataPath, projectRoot, providedCacheKey string, logger log.Logger,
-	tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string, isDebugLogMode, skipExisting, forceOverwrite bool, maxLoggedDownloadErrors int) (*xa.CacheOperation, error) {
+func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context,
+	authConfig common.CacheAuthConfig,
+	cacheMetadataPath, projectRoot, providedCacheKey string,
+	logger log.Logger,
+	tracker xcode.StepAnalyticsTracker,
+	startT time.Time,
+	envProvider func(string) string,
+	commandFunc func(string, ...string) (string, error),
+	isDebugLogMode, skipExisting, forceOverwrite bool,
+	maxLoggedDownloadErrors int) (*xa.CacheOperation, error) {
 	op := newCacheOperation(startT, xa.OperationTypeDownload, envProvider)
 	kvClient, err := createKVClient(ctx,
 		CreateKVClientParams{
 			CacheOperationID: op.OperationID,
 			ClientName:       ClientNameXcode,
 			AuthConfig:       authConfig,
 			EnvProvider:      envProvider,
+			CommandFunc:      commandFunc,
 			Logger:           logger,
 		})
 	if err != nil {