E2E with agent RCs

.gitlab-ci.yml

@@ -5,12 +5,13 @@ stages:
 variables:
   RUN_E2E_TEST:
     description: "set RUN_E2E_TEST to 'true' if you want to trigger the e2e test on your pipeline."
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 2e9af8ebdfcb
 
 noop:
   stage: test
   script: 
     - echo "No op job to get merge queue working"
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:95dca87f269a
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
   tags: ["arch:amd64"]
 
 e2e:
@@ -35,29 +36,54 @@ e2e:
     - if: $RUN_E2E_TEST == "true"
       when: manual
     - when: never
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:95dca87f269a
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
   tags: ["arch:amd64"]
   variables:
-    AWS_KEYPAIR_NAME: datadog-agent-ci
-    AWS_PRIVATE_KEY_FILE: $CI_PROJECT_DIR/ssh_key
-    KUBERNETES_CPU_REQUEST: 2
-    KUBERNETES_MEMORY_REQUEST: 4Gi
-    KUBERNETES_MEMORY_LIMIT: 12Gi
+    E2E_GCP_PUBLIC_KEY_PATH: /tmp/agent-qa-gcp-ssh-key.pub
+    E2E_GCP_PRIVATE_KEY_PATH: /tmp/agent-qa-gcp-ssh-key
+    E2E_KEY_PAIR_NAME: ci.helm-charts
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 16Gi
+  parallel:
+    matrix:
+      - E2E_AGENT_IMG_TAG: "latest"
+        E2E_BUILD_TAGS:
+          - e2e
+          - e2e_autopilot
+          - e2e_autopilot_systemprobe
+      - E2E_AGENT_IMG_TAG: "7-rc"
+        E2E_BUILD_TAGS:
+          - e2e_autopilot
+
   before_script:
-    - mkdir -p ~/.aws
-    - set +x
+    # Initial credentials setup with default build-stable AWS profile
+    - echo "Starting setup for E2E testing..."
+
+    # TODO: Retrieve with datadog-agent-qa AWS profile when secret is moved to SSM in that account
+    - E2E_GCP_PRIVATE_KEY_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_private_key_password --with-decryption --query "Parameter.Value" --out text)
+    - export E2E_GCP_PRIVATE_KEY_PASSWORD
+
     # Set GITHUB_TOKEN to avoid getting rate-limited when pulumi sdk downloads the kubernetes provider
     - export GITHUB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.helm-charts.github_token --with-decryption --query "Parameter.Value" --out text)
-    # Configure AWS EC2 ssh key needed for create pulumi EKS environment
-    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_key --with-decryption --query "Parameter.Value" --out text > $AWS_PRIVATE_KEY_FILE
-    - set -x
-    # Without the newline ssh silently fails and moves on to try other auth methods
-    - echo "" >> $AWS_PRIVATE_KEY_FILE
-    - chmod 600 $AWS_PRIVATE_KEY_FILE
-    # Configure AWS profile
-    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.e2e-agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
-    - set -x
+
+    # Use S3 backend to store stack status
+    - export PULUMI_CONFIG_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.helm-charts.pulumi_password --with-decryption --query "Parameter.Value" --out text)
+
+    # Configure and use datadog-agent-qa AWS profile
+    - mkdir -p ~/.aws
+    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
     - export AWS_PROFILE=agent-qa-ci
+
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
+
+    # SSH Key retrieval for GCP
+    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_public_key --with-decryption --query "Parameter.Value" --out text > $E2E_GCP_PUBLIC_KEY_PATH || exit $?
+    - touch $E2E_GCP_PRIVATE_KEY_PATH && chmod 600 $E2E_GCP_PRIVATE_KEY_PATH && aws ssm get-parameter --region us-east-1 --name ci.helm-charts.ssh_private_key --with-decryption --query "Parameter.Value" --out text > $E2E_GCP_PRIVATE_KEY_PATH || exit $?
+
+    # Setup GCP credentials. https://www.pulumi.com/registry/packages/gcp/installation-configuration/
+    # The service account is called `agent-e2e-tests`
+    - aws ssm get-parameter --region us-east-1 --name ci.helm-charts.e2e_tests_gcp_credentials --with-decryption --query "Parameter.Value" --out text  > ~/gcp-credentials.json || exit $?
+    - export GOOGLE_APPLICATION_CREDENTIALS=~/gcp-credentials.json
+
   script:
-    - make test-e2e
+    - E2E_AGENT_IMG_TAG=$E2E_AGENT_IMG_TAG E2E_BUILD_TAGS=$E2E_BUILD_TAGS E2E_PROFILE=ci make test-e2e

Makefile

@@ -4,12 +4,14 @@ GOTESTSUM_FORMAT?=standard-verbose
 
 # E2E environment variables
 E2E_CONFIG_PARAMS?=
+E2E_KEY_PAIR_NAME=ci.helm-charts
 DD_TEAM?=container-ecosystems
 DD_TAGS?=
+E2E_BUILD_TAGS?="e2e e2e_autopilot e2e_autopilot_systemprobe"
 
 ## Local profile
 E2E_PROFILE?=local
-export AWS_KEYPAIR_NAME?=${USER}
+export E2E_KEY_PAIR_NAME?=${USER}
 export E2E_API_KEY?=
 export E2E_APP_KEY?=
 export PULUMI_CONFIG_PASSPHRASE?=
@@ -25,6 +27,7 @@ override E2E_PROFILE=ci
 endif
 
 ifeq ($(E2E_PROFILE), ci)
+export E2E_KEY_PAIR_NAME
 export CI_ENV_NAMES
 export DD_TEAM
 export DD_TAGS
@@ -93,14 +96,4 @@ test-e2e: fmt vet e2e-test
 # aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test
 .PHONY: e2e-test
 e2e-test:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1
-
-# aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-preserve-stacks
-.PHONY: e2e-test-preserve-stacks
-e2e-test-preserve-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -preserveStacks=true
-
-# aws-vault exec sso-agent-sandbox-account-admin -- make e2e-test-cleanup-stacks
-.PHONY: e2e-test-cleanup-stacks
-e2e-test-cleanup-stacks:
-	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=e2e -v -vet=off -timeout 1h -count=1 -args -destroyStacks=true
+	E2E_CONFIG_PARAMS=$(E2E_CONFIG_PARAMS) E2E_PROFILE=$(E2E_PROFILE) go test -C test/e2e ./... --tags=$(E2E_BUILD_TAGS) -v -vet=off -timeout 1h -count=1

test/common/args.go

@@ -3,12 +3,8 @@ package common
 import "flag"
 
 var UpdateBaselines bool
-var PreserveStacks bool
-var DestroyStacks bool
 
 func ParseArgs() {
 	flag.BoolVar(&UpdateBaselines, "updateBaselines", false, "When set to true overwrites existing baselines with the rendered ones")
-	flag.BoolVar(&PreserveStacks, "preserveStacks", false, "When set to true, preserves newly-created or existing Pulumi end-to-end (E2E) stacks after completing tests. When set to false, destroys Pulumi E2E stacks upon test completion.")
-	flag.BoolVar(&DestroyStacks, "destroyStacks", false, "When set to true, destroys existing Pulumi end-to-end (E2E) stacks and performs cleanup. When set to false, `preserveStacks` value takes precedence.")
 	flag.Parse()
 }

test/common/common_e2e.go

@@ -1,90 +1,23 @@
 package common
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"strings"
 
-	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra"
-	"github.com/DataDog/test-infra-definitions/scenarios/aws/eks"
-
 	"github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner"
 	"github.com/pulumi/pulumi/sdk/v3/go/auto"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/remotecommand"
 )
 
 var defaultPulumiConfigs = runner.ConfigMap{
-	"ddinfra:aws/defaultKeyPairName": auto.ConfigValue{Value: os.Getenv("AWS_KEYPAIR_NAME")},
-}
-var defaultCIPulumiConfigs = runner.ConfigMap{
-	"aws:skipCredentialsValidation":     auto.ConfigValue{Value: "true"},
-	"aws:skipMetadataApiCheck":          auto.ConfigValue{Value: "false"},
-	"ddinfra:aws/defaultPrivateKeyPath": auto.ConfigValue{Value: os.Getenv("AWS_PRIVATE_KEY_FILE")},
-}
-
-type E2EEnv struct {
-	context     context.Context
-	name        string
-	StackOutput auto.UpResult
-}
-
-// Create new EKS pulumi stack
-// The latest datadog/datadog helm chart is installed by default via the stack config `ddagent:deploy`
-func NewEKStack(stackConfig runner.ConfigMap, destroyStacks bool) (*E2EEnv, error) {
-	eksE2eEnv := &E2EEnv{
-		context: context.Background(),
-		name:    "eks-e2e",
-	}
-
-	stackManager := infra.GetStackManager()
-
-	// Get or create stack if it doesn't exist
-	_, stackOutput, err := stackManager.GetStack(eksE2eEnv.context, eksE2eEnv.name, stackConfig, eks.Run, destroyStacks)
-	if err != nil {
-		return nil, err
-	}
-	eksE2eEnv.StackOutput = stackOutput
-	return eksE2eEnv, nil
-}
-
-func TeardownE2EStack(e2eEnv *E2EEnv, preserveStacks bool) error {
-	if !preserveStacks {
-		log.Println("Tearing down E2E stack. ")
-		if e2eEnv != nil {
-			err := infra.GetStackManager().DeleteStack(e2eEnv.context, e2eEnv.name)
-			if err != nil {
-				return fmt.Errorf("error tearing down E2E stack: %s", err)
-			}
-		} else {
-			return cleanupStacks()
-		}
-	} else {
-		log.Println("Preserving E2E stack. ")
-		return nil
-	}
-	return nil
+	"ddinfra:kubernetesVersion": auto.ConfigValue{Value: "1.32"},
 }
 
-func cleanupStacks() error {
-	log.Println("Cleaning up E2E stacks. ")
-	errs := infra.GetStackManager().Cleanup(context.Background())
-	for _, err := range errs {
-		log.Println(err.Error())
-	}
-	if errs != nil {
-		return fmt.Errorf("error cleaning up E2E stacks")
-	}
-	return nil
+var defaultCIPulumiConfigs = runner.ConfigMap{
+	"ddinfra:env":                           auto.ConfigValue{Value: "gcp/agent-qa"},
+	"ddinfra:gcp/defaultPrivateKeyPassword": auto.ConfigValue{Value: os.Getenv("E2E_GCP_PRIVATE_KEY_PASSWORD"), Secret: true},
+	"ddagent:fullImagePath":                 auto.ConfigValue{Value: fmt.Sprintf("gcr.io/datadoghq/agent:%s", os.Getenv("E2E_AGENT_IMG_TAG"))},
 }
 
 func parseE2EConfigParams() []string {
@@ -109,9 +42,9 @@ func SetupConfig() (runner.ConfigMap, error) {
 		// fast-fail if missing required env vars
 		_, e2eApiKeyBool := os.LookupEnv("E2E_API_KEY")
 		_, e2eAppKeyBool := os.LookupEnv("E2E_APP_KEY")
-		_, e2eAwsKeypairNameBool := os.LookupEnv("AWS_KEYPAIR_NAME")
+		_, e2eAwsKeypairNameBool := os.LookupEnv("E2E_KEY_PAIR_NAME")
 		if !e2eApiKeyBool || !e2eAppKeyBool || !e2eAwsKeypairNameBool {
-			return nil, fmt.Errorf("missing required environment variables. Must set `E2E_API_KEY`, `E2E_APP_KEY`, and `AWS_KEYPAIR_NAME` for the local E2E profile")
+			return nil, fmt.Errorf("missing required environment variables. Must set `E2E_API_KEY`, `E2E_APP_KEY`, and `E2E_KEY_PAIR_NAME` for the local E2E profile")
 		} else {
 			res.Merge(defaultPulumiConfigs)
 		}
@@ -120,105 +53,13 @@ func SetupConfig() (runner.ConfigMap, error) {
 	if len(configs) > 0 {
 		for _, config := range configs {
 			kv := strings.Split(config, "=")
-			_, exists := res[kv[0]]
-			if !exists {
-				res[kv[0]] = auto.ConfigValue{Value: kv[1]}
+			if _, exists := res[kv[0]]; !exists {
+				isSecret := strings.Contains(strings.ToLower(kv[0]), "password")
+				res[kv[0]] = auto.ConfigValue{Value: kv[1], Secret: isSecret}
 			} else {
-				log.Printf("Config param %s used more than once. Value: %s", kv[0], kv[1])
+				log.Printf("Config param %s used more than once.", kv[0])
 			}
 		}
 	}
-	log.Printf("Setting up Pulumi E2E stack with configs: %v", res)
 	return res, nil
 }
-
-func ListPods(namespace string, podLabelSelector string, client *kubernetes.Clientset) (*corev1.PodList, error) {
-	pods, err := client.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: podLabelSelector})
-	if err != nil {
-		log.Panicf("error getting pods: %v", err)
-		return nil, err
-	}
-	return pods, nil
-}
-
-func ListNodes(namespace string, client kubernetes.Interface) (*corev1.NodeList, error) {
-	nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
-
-	if err != nil {
-		log.Panicf("error getting nodes: %v", err)
-	}
-	return nodes, nil
-}
-
-func NewClientFromKubeconfig(kc map[string]interface{}) (clientcmd.ClientConfig, *rest.Config, *kubernetes.Clientset, error) {
-	kubeconfig, err := json.Marshal(kc)
-	if err != nil {
-		log.Printf("Error encoding kubeconfig json. %v", err)
-		return nil, nil, nil, err
-	}
-	clientConfig, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
-	if err != nil {
-		log.Printf("Error creating client config from kubeconfig. %v", err)
-		return nil, nil, nil, err
-	}
-	restConfig, err := clientConfig.ClientConfig()
-	if err != nil {
-		log.Printf("Error creating rest config. %v", err)
-		return nil, nil, nil, err
-	}
-
-	clientSet, err := kubernetes.NewForConfig(restConfig)
-	if err != nil {
-		log.Printf("Error creating clientset from rest config. %v", err)
-		return nil, nil, nil, err
-	}
-
-	return clientConfig, restConfig, clientSet, err
-}
-
-type K8sExec struct {
-	ClientSet  kubernetes.Interface
-	RestConfig *rest.Config
-}
-
-func (k8s *K8sExec) K8sExec(namespace string, podName string, containerName string, command []string) ([]byte, []byte, error) {
-	req := k8s.ClientSet.CoreV1().RESTClient().Post().
-		Resource("pods").
-		Name(podName).
-		Namespace(namespace).
-		SubResource("exec")
-
-	req.VersionedParams(&corev1.PodExecOptions{
-		Container: containerName,
-		Command:   command,
-		Stdin:     false,
-		Stdout:    true,
-		Stderr:    true,
-		TTY:       true,
-	}, scheme.ParameterCodec)
-
-	exec, err := remotecommand.NewSPDYExecutor(k8s.RestConfig, "POST", req.URL())
-	if err != nil {
-		log.Printf("Failed to exec:%v", err)
-		return []byte{}, []byte{}, err
-	}
-	var stdout, stderr bytes.Buffer
-	err = exec.Stream(remotecommand.StreamOptions{
-		Stdin:  nil,
-		Stdout: &stdout,
-		Stderr: &stderr,
-	})
-	if err != nil {
-		log.Printf("Failed to get result:%v", err)
-		return []byte{}, []byte{}, err
-	}
-	return stdout.Bytes(), stderr.Bytes(), nil
-}
-
-func NewK8sExec(clientSet *kubernetes.Clientset, restConfig *rest.Config) K8sExec {
-	k8sExec := K8sExec{
-		ClientSet:  clientSet,
-		RestConfig: restConfig,
-	}
-	return k8sExec
-}