kubeconfig_configure refactor

Author: kzadorozhny

README.md

@@ -103,7 +103,7 @@ When you run `bazel run ///helloworld:mynamespace.apply`, it applies this file i
 | ***objects***             | `[]`           | A list of other instances of `k8s_deploy` that this one depends on. See [Adding Dependencies](#adding-dependencies).
 | ***images***              | `{}`           | A dict of labels of Docker images. See [Injecting Docker Images](#injecting-docker-images).
 | ***image_digest_tag***    | `False`        | A flag for whether or not to tag the image with the container digest.
-| ***image_registry***      | `docker.io`    | The registry to push images to. 
+| ***image_registry***      | `docker.io`    | The registry to push images to.
 | ***image_repository***    | `None`         | The repository to push images to. By default, this is generated from the current package path.
 | ***image_repository_prefix*** | `None`     | Add a prefix to the image_repository. Can be used to upload the images in
 | ***release_branch_prefix*** | `master`     | A git branch name/prefix. Automatically run GitOps while building this branch. See [GitOps and Deployment](#gitops_and_deployment).
@@ -288,7 +288,7 @@ spec:
     - name: java_container
       image: registry.example.com/examples/image@sha256:c94d75d68f4c1b436f545729bbce82774fda07
 ```
-Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example, 
+Image substitutions for Custom Resource Definitions (CRD) resources could also use target references directly. Their digests are availabe through string substitution. For example,
 ```yaml
 apiVersion: v1
 kind: MyCrd
@@ -300,7 +300,7 @@ metadata:
 spec:
   image: "{{//example:my_image}}"
 ```
-would become 
+would become
 ```yaml
 apiVersion: v1
 kind: MyCrd
@@ -381,7 +381,7 @@ The `k8s_test_setup` rule produces a shell script which creates a temporary name
 
 The output of the `k8s_test_setup` rule (a shell script) is referenced in the `java_test` rule. It's listed under the `data` attribute, which declares the target as a dependency, and is included in the jvm flags in this clause: `$(location :service_it.setup)`. The "location" function is specific to Bazel: given a target, it returns the path to the file produced by that target. In this case, it returns the path to the shell script created by our `k8s_test_setup` rule.
 
-The test code launches the script to perform the test setup. The tes code should also monitor the script console output to listen to the pod readiness events.
+The test code launches the script to perform the test setup. The test code should also monitor the script console output to listen to the pod readiness events.
 
 
 ## Building & Testing

examples/WORKSPACE

@@ -36,9 +36,9 @@ go_image_repositories()
 #
 # Kubeconfig repository that will be used in k8s_test_setup
 #
-load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig")
+load("@com_adobe_rules_gitops//gitops:defs.bzl", "kubeconfig_configure")
 
-kubeconfig(
+kubeconfig_configure(
     name = "k8s_test",
     cluster = "kind-kind",
     user = "kind-kind",  # kind cluster user name

gitops/defs.bzl

@@ -16,9 +16,14 @@ load(
     "@com_adobe_rules_gitops//skylib:k8s.bzl",
     _k8s_deploy = "k8s_deploy",
     _k8s_test_setup = "k8s_test_setup",
+)
+load(
+    "@com_adobe_rules_gitops//skylib:kubeconfig.bzl",
     _kubeconfig = "kubeconfig",
+    _kubeconfig_configure = "kubeconfig_configure",
 )
 
 k8s_deploy = _k8s_deploy
 k8s_test_setup = _k8s_test_setup
 kubeconfig = _kubeconfig
+kubeconfig_configure = _kubeconfig_configure

skylib/k8s.bzl

@@ -20,6 +20,7 @@ load(
     "kustomize",
     kustomize_gitops = "gitops",
 )
+load("//skylib:kubeconfig.bzl", "KubeconfigInfo")
 load("//skylib:push.bzl", "k8s_container_push")
 
 def _runfiles(ctx, f):
@@ -275,219 +276,6 @@ def k8s_deploy(
             visibility = visibility,
         )
 
-KubeconfigInfo = provider(fields = [
-    "server",
-    "cluster",
-    "user",
-])
-
-def _kubeconfig_file_impl(ctx):
-    kubeconfig_file = ctx.actions.declare_file(ctx.label.name)
-    ctx.actions.symlink(output = kubeconfig_file, target_file = ctx.file.config)
-    files = depset(direct = [kubeconfig_file])
-    runfiles = ctx.runfiles(files = [kubeconfig_file])
-    return [
-        DefaultInfo(files = files, runfiles = runfiles),
-        KubeconfigInfo(
-            server = ctx.attr.server,
-            cluster = ctx.attr.cluster,
-            user = ctx.attr.user,
-        ),
-    ]
-
-kubeconfig_file = rule(
-    implementation = _kubeconfig_file_impl,
-    attrs = {
-        "config": attr.label(
-            doc = "Config file.",
-            allow_single_file = True,
-            mandatory = True,
-        ),
-        "server": attr.string(
-            doc = "Optional Kubernetes server url.",
-            mandatory = False,
-        ),
-        "cluster": attr.string(
-            doc = "Optional Kubernetes cluster name.",
-            mandatory = False,
-        ),
-        "user": attr.string(
-            doc = "Optional Kubernetes user name.",
-            mandatory = True,
-        ),
-    },
-    provides = [DefaultInfo, KubeconfigInfo],
-)
-
-_KUBECONFIG_BUILD_TEMPLATE = """# Generated by kubeconfig repostiory rule
-
-load("@com_adobe_rules_gitops//skylib:k8s.bzl", "kubeconfig_file")
-
-exports_files(["kubectl"])
-
-kubeconfig_file(
-    name = "kubeconfig",
-    config = ":config",
-    server = "{server}",
-    cluster = "{cluster}",
-    user = "{user}",
-    visibility = ["//visibility:public"],
-)
-"""
-
-# kubectl template
-def _kubectl_config(repository_ctx, args):
-    kubectl = repository_ctx.path("kubectl")
-    kubeconfig_yaml = repository_ctx.path("config")
-    exec_result = repository_ctx.execute(
-        [kubectl, "--kubeconfig", kubeconfig_yaml, "config"] + args,
-        environment = {
-            # prevent kubectl config to stumble on shared .kube/config.lock file
-            "HOME": str(repository_ctx.path(".")),
-        },
-        quiet = True,
-    )
-    if exec_result.return_code != 0:
-        fail("Error executing kubectl config %s" % " ".join(args))
-
-def _kubeconfig_impl(repository_ctx):
-    """Find local kubernetes certificates"""
-
-    # find and symlink kubectl
-    kubectl = repository_ctx.which("kubectl")
-    if not kubectl:
-        fail("Unable to find kubectl executable. PATH=%s" % repository_ctx.path)
-    repository_ctx.symlink(kubectl, "kubectl")
-
-    home = repository_ctx.path(repository_ctx.os.environ["HOME"])
-
-    # use provided user name or fall back to current os user name
-    if repository_ctx.attr.user:
-        user = repository_ctx.attr.user
-    elif "USER" in repository_ctx.os.environ:
-        user = repository_ctx.os.environ["USER"]
-    else:
-        exec_result = repository_ctx.execute(["whoami"])
-        if exec_result.return_code != 0:
-            fail("Error detecting current user")
-        user = exec_result.stdout.rstrip()
-
-    token = None
-    ca_crt = None
-    kubecert_cert = None
-    kubecert_key = None
-    server = repository_ctx.attr.server
-    config = None
-
-    # check service account first
-    serviceaccount = repository_ctx.path("/var/run/secrets/kubernetes.io/serviceaccount")
-    if serviceaccount.exists:
-        ca_crt = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-        token_file = serviceaccount.get_child("token")
-        if token_file.exists:
-            exec_result = repository_ctx.execute(["cat", token_file.realpath])
-            if exec_result.return_code != 0:
-                fail("Error reading user token")
-            token = exec_result.stdout.rstrip()
-
-        # use master url from the environemnt
-        if "KUBERNETES_SERVICE_HOST" in repository_ctx.os.environ:
-            server = "https://%s:%s" % (
-                repository_ctx.os.environ["KUBERNETES_SERVICE_HOST"],
-                repository_ctx.os.environ["KUBERNETES_SERVICE_PORT"],
-            )
-        else:
-            # fall back to the default
-            server = "https://kubernetes.default"
-    else:
-        # check kubectl config file
-        config = home.get_child(".kube").get_child("config")
-
-    if config and config.exists:
-        # symlink ~/.kube/config if this file exists
-        repository_ctx.symlink(config, repository_ctx.path("config"))
-    else:
-        # generate new config file service account token or certificates
-        certs = home.get_child(".kube").get_child("certs")
-        ca_crt = certs.get_child("ca.crt").realpath
-        kubecert_cert = certs.get_child("kubecert.cert")
-        kubecert_key = certs.get_child("kubecert.key")
-
-        # config set-cluster {cluster} \
-        #     --certificate-authority=... \
-        #     --server=https://dev3.k8s.tubemogul.info:443 \
-        #     --embed-certs",
-        _kubectl_config(repository_ctx, [
-            "set-cluster",
-            repository_ctx.attr.cluster,
-            "--server",
-            server,
-            "--certificate-authority",
-            ca_crt,
-        ])
-
-        # config set-credentials {user} --token=...",
-        if token:
-            _kubectl_config(repository_ctx, [
-                "set-credentials",
-                user,
-                "--token",
-                token,
-            ])
-
-        # config set-credentials {user} --client-certificate=...",
-        if kubecert_cert and kubecert_cert.exists:
-            _kubectl_config(repository_ctx, [
-                "set-credentials",
-                user,
-                "--client-certificate",
-                kubecert_cert.realpath,
-            ])
-
-        # config set-credentials {user} --client-key=...",
-        if kubecert_key and kubecert_key.exists:
-            _kubectl_config(repository_ctx, [
-                "set-credentials",
-                user,
-                "--client-key",
-                kubecert_key.realpath,
-            ])
-
-    # export repostory contents
-    repository_ctx.file("BUILD", _KUBECONFIG_BUILD_TEMPLATE.format(
-        cluster = repository_ctx.attr.cluster,
-        server = repository_ctx.attr.server,
-        user = user,
-    ), False)
-
-    return {
-        "cluster": repository_ctx.attr.cluster,
-        "server": repository_ctx.attr.server,
-        "user": user,
-    }
-
-kubeconfig = repository_rule(
-    attrs = {
-        "cluster": attr.string(
-            mandatory = True,
-        ),
-        "server": attr.string(
-            mandatory = False,
-        ),
-        "user": attr.string(
-            mandatory = False,
-        ),
-    },
-    environ = [
-        "HOME",
-        "USER",
-        "KUBERNETES_SERVICE_HOST",
-        "KUBERNETES_SERVICE_PORT",
-    ],
-    local = True,
-    implementation = _kubeconfig_impl,
-)
-
 def _stamp(ctx, string, output):
     stamps = [ctx.file._info_file]
     stamp_args = [

skylib/k8s_test_namespace.sh.tpl

@@ -10,6 +10,7 @@
 # governing permissions and limitations under the License.
 
 set -euo pipefail
+
 [ -o xtrace ] && env
 
 function guess_runfiles() {
@@ -39,8 +40,25 @@ echo "Cluster: ${CLUSTER}" >&2
 # use BUILD_USER by defalt
 USER=${USER:-$BUILD_USER}
 
+# create miniified self-contained kubectl configuration with the default context set to use newly created namespace
+mkdir -p $(dirname $KUBECONFIG_FILE)
+
+# create context partion of new kubeconfig file from scratch
+# use --kubeconfig parameter to prevent any merging
+# create
+rm -f $KUBECONFIG_FILE-context
+CONTEXT=$CLUSTER-$BUILD_USER
+kubectl --kubeconfig=$KUBECONFIG_FILE-context --cluster=$CLUSTER --server=$SERVER --user=$USER --namespace=$BUILD_USER config  set-context $CONTEXT >&2
+kubectl --kubeconfig=$KUBECONFIG_FILE-context config use-context $CONTEXT >&2
+
+# merge newly generated context with system kubeconfig, flatten and minify the result
+KUBECONFIG=$KUBECONFIG_FILE-context:$KUBECONFIG kubectl config view --merge=true --minify=true --flatten=true --raw >$KUBECONFIG_FILE
+
+# set generated kubeconfig for all following kubectl commands
+export KUBECONFIG=$KUBECONFIG_FILE
+
 # check if username from provided configuration exists
-KUBECONFIG_USER=$(${KUBECTL} --kubeconfig=${KUBECONFIG} config view -o jsonpath='{.users[?(@.name == '"\"${USER}\")].name}")
+KUBECONFIG_USER=$(${KUBECTL} config view -o jsonpath='{.users[?(@.name == '"\"${USER}\")].name}")
 if [ -z "${KUBECONFIG_USER}" ]; then
     echo "Unable to find user configuration ${USER} for cluster ${CLUSTER}" >&2
     exit 1
@@ -64,13 +82,18 @@ else
     COUNT="0"
     while true; do
         NAMESPACE=${BUILD_USER}-$(( (RANDOM) + 32767 ))
-        ${KUBECTL} --kubeconfig=${KUBECONFIG} --cluster=${CLUSTER} --server=${SERVER} --user=${USER} create namespace ${NAMESPACE} && break
+        ${KUBECTL} create namespace ${NAMESPACE} && break
         COUNT=$[$COUNT + 1]
         if [ $COUNT -ge 10 ]; then
             echo "Unable to create namespace in $COUNT attempts!" >&2
             exit 1
         fi
     done
+    # update context with created test namespace
+    kubectl --namespace=$NAMESPACE config set-context $CONTEXT >&2
+
+    # rename test context (Note: this is required for backward compatibiliy)
+    kubectl config rename-context $CONTEXT $CLUSTER-$NAMESPACE >&2
 fi
 echo "Namespace: ${NAMESPACE}" >&2
 set -e
@@ -79,21 +102,7 @@ set -e
 mkdir -p $(dirname $NAMESPACE_NAME_FILE)
 echo $NAMESPACE > $NAMESPACE_NAME_FILE
 
-# create miniified self-contained kubectl configuration with the default context set to use newly created namespace
-mkdir -p $(dirname $KUBECONFIG_FILE)
-
-# create context partion of new kubeconfig file from scratch
-# use --kubeconfig parameter to prevent any merging
-rm -f $KUBECONFIG_FILE-context
-CONTEXT=$CLUSTER-$NAMESPACE
-kubectl --kubeconfig=$KUBECONFIG_FILE-context --cluster=$CLUSTER --server=${SERVER} --user=$USER --namespace=$NAMESPACE config  set-context $CONTEXT >&2
-kubectl --kubeconfig=$KUBECONFIG_FILE-context config use-context $CONTEXT >&2
-
-# merge newly generated context with system kubeconfig, flatten and minify the result
-KUBECONFIG=$KUBECONFIG_FILE-context:$KUBECONFIG kubectl config view --merge=true --minify=true --flatten=true --raw >$KUBECONFIG_FILE
-
-# set generated kubeconfig for all following kubectl commands
-export KUBECONFIG=$KUBECONFIG_FILE
+[ -o xtrace ] && kubectl config view >&2
 
 # set runfiles for STMTS
 export PYTHON_RUNFILES=${RUNFILES}