From ded6ab0e2583281abb3a3fa7e37edaa73526cb0c Mon Sep 17 00:00:00 2001 From: ematysek Date: Wed, 16 Oct 2019 13:29:40 -0400 Subject: [PATCH 1/6] Logging test automation added with concurrent project creation --- docs/logging.md | 68 ++++++++++++++++ workloads/logging.yml | 92 +++++++++++++++++++++ workloads/templates/logtest-rc.json.j2 | 106 +++++++++++++++++++++++++ workloads/templates/logtest.yml.j2 | 23 ++++++ workloads/vars/logging.yml | 15 ++++ 5 files changed, 304 insertions(+) create mode 100644 docs/logging.md create mode 100644 workloads/logging.yml create mode 100644 workloads/templates/logtest-rc.json.j2 create mode 100644 workloads/templates/logtest.yml.j2 create mode 100644 workloads/vars/logging.yml diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 00000000..561a176b --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,68 @@ +# Logging Workload + +Stress test the cluster logging and ElasticSearch operators. + +## Requirements + +* Cluster logging operator is deployed +* kubeconfig present in $HOME/.kube/config on orchestration host +* podman and libselinux-python2 installed on orchestration host +* selinux set to permissive (otherwise cluster-loader will fail) + +## Run from CLI + +```shell script +cp workloads/inventory.example inventory +# Add orchestration host to inventory +# Define environmental variables or use default logtest parameters +ansible-playbook -v -i inventory workloads/logging.yml +``` + +## Implementation Notes +Currently I am templating parameters directly into logtest-rc.json because +I ran into issues trying to get cluster-loader to pass in the values from logtest.yml. +I left in the step where the variables are templated into logtest.yml in case anything changes. + +## Ansible Variables + +### LABAL_ALL_NODES +default: False +If False: Remove all placement=logtest labels from worker nodes then add it back to just 1 worker node. +If True: Apply placement=logtest label to all worker nodes + +## Environmental Variables + +### ORCHESTRATION_USER + +default: `root` +Remote user to connect as. + +### NUM_PROJECTS + +default: `1` +Number of logtest projects to create. + +### NUM_LINES + +default: `1800000` +Number of lines for logtest to generate per project. + +### LINE_LENGTH + +default: `1024` +Length of log lines in bytes. + +### RATE + +default: `60000` +Messages per second for logtest pod to generate. + +### PAUSE_OFFSET + +default: `5` +Time to wait (in minutes) after logtest is done before verifying all messages showed up in elasticsearch. + +### ORIGIN_TESTS_VERSION + +default: `latest` +Version of quay.io/openshift/origin-tests to pull. \ No newline at end of file diff --git a/workloads/logging.yml b/workloads/logging.yml new file mode 100644 index 00000000..8491c981 --- /dev/null +++ b/workloads/logging.yml @@ -0,0 +1,92 @@ +--- +# Assumptions: +# - KUBECONFIG env var is set on orchestration host (would have to be in bashrc) or kubeconfig is set in ~/.kube/config +# - cluster-logging operator deployed +# - podman installed +# - selinux set to permissive (otherwise cluster-loader will fail) +# - libselinux-python2 (or variation depending on OS) installed +- name: Run logging test + hosts: orchestration + remote_user: "{{orchestration_user}}" + vars_files: + - vars/logging.yml + tasks: + - name: Verify oc command is available + command: oc version + changed_when: False + - name: Verify openshift-logging exists + shell: oc get projects | grep "openshift-logging" + changed_when: False + - name: Verify ElasticSearch pods exist + shell: oc get pods -n openshift-logging | grep elasticsearch + changed_when: False + - name: Get ElasticSearch pod to query from + shell: oc get pods -n openshift-logging | grep elasticsearch | grep Running | head -n 1 | awk '{print $1}' + register: es_pod + changed_when: False + tags: + - delete_indices + - name: Delete existing logtest indices + shell: | + BASE_CMD="oc exec -n openshift-logging -c elasticsearch {{ es_pod.stdout }} -- curl --connect-timeout 2 -s -k --cert /etc/elasticsearch/secret/admin-cert --key /etc/elasticsearch/secret/admin-key" + ${BASE_CMD} -X DELETE https://localhost:9200/project.logtest*; + tags: + - delete_indices + - name: Get worker nodes + shell: oc get nodes | grep worker | awk '{print $1}' + register: worker_nodes + changed_when: False + - name: Label single worker node + block: + - name: Remove any existing "placement" labels from worker nodes + shell: "oc label node {{ item }} placement-" + loop: "{{ worker_nodes.stdout_lines }}" + register: result + changed_when: '"not labeled" not in result.stdout' + - name: Label one worker node with placement=logtest + shell: oc label node $(oc get nodes | grep worker | head -n 1 | awk '{print $1}') placement=logtest + when: not LABAL_ALL_NODES + tags: label_node + - name: Label all nodes + shell: "for i in $(oc get nodes | grep worker | awk '{print $1}'); do oc label node/$i placement=logtest --overwrite; done" + when: LABAL_ALL_NODES + tags: label_node + - name: Create workloads directory + file: + path: workloads + state: directory + mode: '0755' + - name: Template cluster-loader logtest logging config file + template: + src: templates/logtest.yml.j2 + dest: workloads/logtest.yml + - name: Template logtest-rc.json + template: + src: templates/logtest-rc.json.j2 + dest: workloads/logtest-rc.json + - name: Launch cluster-loader + shell: | + sudo podman run \ + -v {{ ansible_facts['env']['KUBECONFIG']|default(ansible_facts['env']['HOME'] + '/.kube/config') }}:/root/.kube/config:z \ + -v {{ansible_facts['env']['HOME']}}/workloads/:/root/workloads/:z \ + -i quay.io/openshift/origin-tests:{{ORIGIN_TESTS_VERSION}} \ + /bin/bash -c 'export KUBECONFIG=/root/.kube/config && \ + export VIPERCONFIG=/root/workloads/logtest.yml && \ + openshift-tests run-test "[Feature:Performance][Serial][Slow] Load cluster concurrently with templates [Suite:openshift]"' \ + |& tee workloads/cluster-loader.log + - name: Puase for time of test + PAUSE_OFFSET + pause: + minutes: "{{ (NUM_LINES|int / RATE|int + PAUSE_OFFSET|int) | round(0, 'ceil') | int }}" + - name: Get number of messages indexed + shell: x=0; for i in $(oc exec -n openshift-logging {{ es_pod.stdout }} -c elasticsearch -- es_util --query='_cat/indices'| grep logtest | awk '{print $7}'); do (( x += i )); done; echo "$x" + register: num_indexed + - debug: + var: num_indexed + - name: Assert number of messages is equal to NUM_LINES*NUM_PROJECTS + assert: + that: + - "{{ num_indexed.stdout }} == {{ (NUM_LINES|int) * (NUM_PROJECTS|int) }}" + - name: Clean up logtest projects + shell: oc delete project $(oc get projects | grep logtest | awk '{print $1}' | sed ':a;N;$!ba;s/\n/ /g') + tags: + - cleanup diff --git a/workloads/templates/logtest-rc.json.j2 b/workloads/templates/logtest-rc.json.j2 new file mode 100644 index 00000000..3e44633b --- /dev/null +++ b/workloads/templates/logtest-rc.json.j2 @@ -0,0 +1,106 @@ +{% raw %} +{ + "apiVersion": "template.openshift.io/v1", + "kind": "Template", + "metadata": { + "name": "centos-logtest-template" + }, + "objects": [ + { + "apiVersion": "v1", + "data": { + "ocp_logtest.cfg": "${INITIAL_FLAGS}" + }, + "kind": "ConfigMap", + "metadata": { + "name": "logtest-config" + } + }, + { + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "name": "centos-logtest", + "labels": { + "run": "centos-logtest", + "test": "centos-logtest" + } + }, + "spec": { + "replicas": "${{REPLICAS}}", + "template": { + "metadata": { + "generateName": "centos-logtest-", + "labels": { + "run": "centos-logtest", + "test": "centos-logtest" + } + }, + "spec": { + "nodeSelector": { + "placement": "${PLACEMENT}" + }, + "containers": [ + { + "env": [ + ], + "image": "${LOGTEST_IMAGE}", + "imagePullPolicy": "Always", + "name": "centos-logtest", + "resources": {}, + "volumeMounts": [ + { + "name": "config", + "mountPath": "/var/lib/svt" + } + ], + "terminationMessagePath": "/dev/termination-log" + } + ], + "volumes": [ + { + "name": "config", + "configMap": { + "name": "logtest-config" + } + } + ], + "imagePullSecrets": [ + { + "name": "default-dockercfg-ukomu" + } + ] + } + } + } + } + ], +{% endraw %} + "parameters": [ + { + "name": "LOGTEST_IMAGE", + "displayName": "logtest image", + "value": "quay.io/mffiedler/ocp-logtest:latest" + }, + { + "name": "INITIAL_FLAGS", + "description": "The initial flags to pass to ocp_logtest.py", + "value": "--num-lines {{ NUM_LINES }} --line-length {{ LINE_LENGTH }} --word-length 9 --rate {{ RATE }} --fixed-line\n" + }, + { + "name": "IDENTIFIER", + "displayName": "identifier", + "value": "1" + }, + { + "name": "REPLICAS", + "displayName": "Replicas", + "value": "1" + }, + { + "name": "PLACEMENT", + "displayName": "Placement", + "value": "logtest" + } + ] +} diff --git a/workloads/templates/logtest.yml.j2 b/workloads/templates/logtest.yml.j2 new file mode 100644 index 00000000..a15c5251 --- /dev/null +++ b/workloads/templates/logtest.yml.j2 @@ -0,0 +1,23 @@ +provider: local +ClusterLoader: + cleanup: false + projects: + - num: {{NUM_PROJECTS}} + basename: logtest- + ifexists: delete + tuning: default + templates: + - num: 1 + file: ./logtest-rc.json + parameters: + - REPLICAS: "1" + - INITIAL_FLAGS: "--num-lines {{NUM_LINES}} --line-length {{LINE_LENGTH}} --word-length 9 --rate {{RATE}} --fixed-line\n" + + tuningsets: + - name: default + pods: + stepping: + stepsize: 5 + pause: 0 + rate_limit: + delay: 0 diff --git a/workloads/vars/logging.yml b/workloads/vars/logging.yml new file mode 100644 index 00000000..2184219f --- /dev/null +++ b/workloads/vars/logging.yml @@ -0,0 +1,15 @@ +orchestration_user: "{{ lookup('env', 'ORCHESTRATION_USER')|default('root', true) }}" + +LABAL_ALL_NODES: False + +# Logtest parameters +NUM_PROJECTS: "{{ lookup('env', 'NUM_PROJECTS')|default(1, true) }}" +NUM_LINES: "{{ lookup('env', 'NUM_LINES')|default(1800000, true) }}" +LINE_LENGTH: "{{ lookup('env', 'LINE_LENGTH')|default(1024, true) }}" +# Messages per minute +RATE: "{{ lookup('env', 'RATE')|default(60000, true) }}" +# Time to wait (in minutes) after logtest is done before verifying all messages showed up in elasticsearch +PAUSE_OFFSET: "{{ lookup('env', 'PAUSE_OFFSET')|default(5, true) }}" + +# Version of quay.io/openshift/origin-tests to pull +ORIGIN_TESTS_VERSION: "{{ lookup('env', 'ORIGIN_TESTS_VERSION')|default('latest', true) }}" From a33ee69cf74eae60cdfaaf160cfbbd39e53b29ac Mon Sep 17 00:00:00 2001 From: ematysek Date: Thu, 7 May 2020 12:10:50 -0400 Subject: [PATCH 2/6] Make logging workload compatible with newer openshift-tests changes in cl.go as of https://github.com/openshift/origin/commit/62773f7649519e5d4a02fb5dbdd000bb007aecdd --- workloads/logging.yml | 2 +- workloads/vars/logging.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/workloads/logging.yml b/workloads/logging.yml index 8491c981..237d5a60 100644 --- a/workloads/logging.yml +++ b/workloads/logging.yml @@ -72,7 +72,7 @@ -i quay.io/openshift/origin-tests:{{ORIGIN_TESTS_VERSION}} \ /bin/bash -c 'export KUBECONFIG=/root/.kube/config && \ export VIPERCONFIG=/root/workloads/logtest.yml && \ - openshift-tests run-test "[Feature:Performance][Serial][Slow] Load cluster concurrently with templates [Suite:openshift]"' \ + openshift-tests run-test "{{ CLUSTER_LOADER_STRING }} concurrently with templates [Suite:openshift]"' \ |& tee workloads/cluster-loader.log - name: Puase for time of test + PAUSE_OFFSET pause: diff --git a/workloads/vars/logging.yml b/workloads/vars/logging.yml index 2184219f..34787c66 100644 --- a/workloads/vars/logging.yml +++ b/workloads/vars/logging.yml @@ -13,3 +13,8 @@ PAUSE_OFFSET: "{{ lookup('env', 'PAUSE_OFFSET')|default(5, true) }}" # Version of quay.io/openshift/origin-tests to pull ORIGIN_TESTS_VERSION: "{{ lookup('env', 'ORIGIN_TESTS_VERSION')|default('latest', true) }}" + +CLUSTER_LOADER_STRING: "{{ '[sig-scalability][Feature:Performance] Load cluster' + if ORIGIN_TESTS_VERSION == 'latest' + or ORIGIN_TESTS_VERSION is version('4.4', '>=') + else '[Feature:Performance][Serial][Slow] Load cluster' }}" \ No newline at end of file From c258a37ce09360bc55a49ec6f7a964fde48874f7 Mon Sep 17 00:00:00 2001 From: ematysek Date: Thu, 7 May 2020 13:03:40 -0400 Subject: [PATCH 3/6] quick fix --- workloads/logging.yml | 2 +- workloads/vars/logging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workloads/logging.yml b/workloads/logging.yml index 237d5a60..cf00e347 100644 --- a/workloads/logging.yml +++ b/workloads/logging.yml @@ -66,7 +66,7 @@ dest: workloads/logtest-rc.json - name: Launch cluster-loader shell: | - sudo podman run \ + podman run \ -v {{ ansible_facts['env']['KUBECONFIG']|default(ansible_facts['env']['HOME'] + '/.kube/config') }}:/root/.kube/config:z \ -v {{ansible_facts['env']['HOME']}}/workloads/:/root/workloads/:z \ -i quay.io/openshift/origin-tests:{{ORIGIN_TESTS_VERSION}} \ diff --git a/workloads/vars/logging.yml b/workloads/vars/logging.yml index 34787c66..8426c018 100644 --- a/workloads/vars/logging.yml +++ b/workloads/vars/logging.yml @@ -14,7 +14,7 @@ PAUSE_OFFSET: "{{ lookup('env', 'PAUSE_OFFSET')|default(5, true) }}" # Version of quay.io/openshift/origin-tests to pull ORIGIN_TESTS_VERSION: "{{ lookup('env', 'ORIGIN_TESTS_VERSION')|default('latest', true) }}" -CLUSTER_LOADER_STRING: "{{ '[sig-scalability][Feature:Performance] Load cluster' +CLUSTER_LOADER_STRING: "{{ '[sig-scalability][Feature:Performance][Serial][Slow] Load cluster' if ORIGIN_TESTS_VERSION == 'latest' or ORIGIN_TESTS_VERSION is version('4.4', '>=') else '[Feature:Performance][Serial][Slow] Load cluster' }}" \ No newline at end of file From 5696e588305b9f3091a488b3a5069c2b8f2be61b Mon Sep 17 00:00:00 2001 From: ematysek Date: Mon, 11 May 2020 14:57:44 -0400 Subject: [PATCH 4/6] Added podman pull step to make sure our podman image isn't stale. Add new vars and updated docs. Removed note in playbook that selinux needed to be permissive. --- docs/logging.md | 11 +++++++++++ workloads/logging.yml | 5 ++++- workloads/templates/logtest.yml.j2 | 2 +- workloads/vars/logging.yml | 3 ++- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index 561a176b..ff6d6455 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -37,6 +37,17 @@ If True: Apply placement=logtest label to all worker nodes default: `root` Remote user to connect as. +### LABAL_ALL_NODES + +default: `False` +If True, label all nodes with placement=logtest. +If False, remove placment label from all worker nodes and only add it back to 1. + +### PROJECT_BASENAME + +default: `logtest-` +Basename for project creation + ### NUM_PROJECTS default: `1` diff --git a/workloads/logging.yml b/workloads/logging.yml index cf00e347..907f2129 100644 --- a/workloads/logging.yml +++ b/workloads/logging.yml @@ -3,7 +3,6 @@ # - KUBECONFIG env var is set on orchestration host (would have to be in bashrc) or kubeconfig is set in ~/.kube/config # - cluster-logging operator deployed # - podman installed -# - selinux set to permissive (otherwise cluster-loader will fail) # - libselinux-python2 (or variation depending on OS) installed - name: Run logging test hosts: orchestration @@ -64,6 +63,10 @@ template: src: templates/logtest-rc.json.j2 dest: workloads/logtest-rc.json + - name: Podman pull cluster-loader image + podman_image: + name: quay.io/openshift/origin-tests + tag: "{{ORIGIN_TESTS_VERSION}}" - name: Launch cluster-loader shell: | podman run \ diff --git a/workloads/templates/logtest.yml.j2 b/workloads/templates/logtest.yml.j2 index a15c5251..cbaa2b54 100644 --- a/workloads/templates/logtest.yml.j2 +++ b/workloads/templates/logtest.yml.j2 @@ -3,7 +3,7 @@ ClusterLoader: cleanup: false projects: - num: {{NUM_PROJECTS}} - basename: logtest- + basename: {{PROJECT_BASENAME}} ifexists: delete tuning: default templates: diff --git a/workloads/vars/logging.yml b/workloads/vars/logging.yml index 8426c018..6d077b06 100644 --- a/workloads/vars/logging.yml +++ b/workloads/vars/logging.yml @@ -1,8 +1,9 @@ orchestration_user: "{{ lookup('env', 'ORCHESTRATION_USER')|default('root', true) }}" -LABAL_ALL_NODES: False +LABAL_ALL_NODES: "{{ lookup('env', 'LABAL_ALL_NODES')|default(False, true) }}" # Logtest parameters +PROJECT_BASENAME: "{{ lookup('env', 'PROJECT_BASENAME')|default('logtest-', true) }}" NUM_PROJECTS: "{{ lookup('env', 'NUM_PROJECTS')|default(1, true) }}" NUM_LINES: "{{ lookup('env', 'NUM_LINES')|default(1800000, true) }}" LINE_LENGTH: "{{ lookup('env', 'LINE_LENGTH')|default(1024, true) }}" From d00e0b039015f563a184e51cf0eb8543bed0cc48 Mon Sep 17 00:00:00 2001 From: ematysek Date: Wed, 13 May 2020 12:45:25 -0400 Subject: [PATCH 5/6] Fixes for BASENAME --- workloads/logging.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workloads/logging.yml b/workloads/logging.yml index 907f2129..39dbd177 100644 --- a/workloads/logging.yml +++ b/workloads/logging.yml @@ -28,7 +28,7 @@ - name: Delete existing logtest indices shell: | BASE_CMD="oc exec -n openshift-logging -c elasticsearch {{ es_pod.stdout }} -- curl --connect-timeout 2 -s -k --cert /etc/elasticsearch/secret/admin-cert --key /etc/elasticsearch/secret/admin-key" - ${BASE_CMD} -X DELETE https://localhost:9200/project.logtest*; + ${BASE_CMD} -X DELETE https://localhost:9200/project.{{PROJECT_BASENAME}}*; tags: - delete_indices - name: Get worker nodes @@ -81,7 +81,7 @@ pause: minutes: "{{ (NUM_LINES|int / RATE|int + PAUSE_OFFSET|int) | round(0, 'ceil') | int }}" - name: Get number of messages indexed - shell: x=0; for i in $(oc exec -n openshift-logging {{ es_pod.stdout }} -c elasticsearch -- es_util --query='_cat/indices'| grep logtest | awk '{print $7}'); do (( x += i )); done; echo "$x" + shell: x=0; for i in $(oc exec -n openshift-logging {{ es_pod.stdout }} -c elasticsearch -- es_util --query='_cat/indices'| grep '{{ PROJECT_BASENAME }}' | awk '{print $7}'); do (( x += i )); done; echo "$x" register: num_indexed - debug: var: num_indexed @@ -90,6 +90,6 @@ that: - "{{ num_indexed.stdout }} == {{ (NUM_LINES|int) * (NUM_PROJECTS|int) }}" - name: Clean up logtest projects - shell: oc delete project $(oc get projects | grep logtest | awk '{print $1}' | sed ':a;N;$!ba;s/\n/ /g') + shell: oc delete project $(oc get projects | grep '{{ PROJECT_BASENAME }}' | awk '{print $1}' | sed ':a;N;$!ba;s/\n/ /g') tags: - cleanup From cc584b8b36fb0128eb65c416a5c3b92144b6ddb5 Mon Sep 17 00:00:00 2001 From: ematysek Date: Tue, 19 May 2020 16:37:35 -0400 Subject: [PATCH 6/6] Add --rm flag to podman run --- workloads/logging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workloads/logging.yml b/workloads/logging.yml index 39dbd177..d9ae7d49 100644 --- a/workloads/logging.yml +++ b/workloads/logging.yml @@ -69,7 +69,7 @@ tag: "{{ORIGIN_TESTS_VERSION}}" - name: Launch cluster-loader shell: | - podman run \ + podman run --rm \ -v {{ ansible_facts['env']['KUBECONFIG']|default(ansible_facts['env']['HOME'] + '/.kube/config') }}:/root/.kube/config:z \ -v {{ansible_facts['env']['HOME']}}/workloads/:/root/workloads/:z \ -i quay.io/openshift/origin-tests:{{ORIGIN_TESTS_VERSION}} \