diff --git a/docs/logging.md b/docs/logging.md new file mode 100644 index 00000000..ff6d6455 --- /dev/null +++ b/docs/logging.md @@ -0,0 +1,79 @@ +# Logging Workload + +Stress test the cluster logging and ElasticSearch operators. + +## Requirements + +* Cluster logging operator is deployed +* kubeconfig present in $HOME/.kube/config on orchestration host +* podman and libselinux-python2 installed on orchestration host +* selinux set to permissive (otherwise cluster-loader will fail) + +## Run from CLI + +```shell script +cp workloads/inventory.example inventory +# Add orchestration host to inventory +# Define environmental variables or use default logtest parameters +ansible-playbook -v -i inventory workloads/logging.yml +``` + +## Implementation Notes +Currently I am templating parameters directly into logtest-rc.json because +I ran into issues trying to get cluster-loader to pass in the values from logtest.yml. +I left in the step where the variables are templated into logtest.yml in case anything changes. + +## Ansible Variables + +### LABAL_ALL_NODES +default: False +If False: Remove all placement=logtest labels from worker nodes then add it back to just 1 worker node. +If True: Apply placement=logtest label to all worker nodes + +## Environmental Variables + +### ORCHESTRATION_USER + +default: `root` +Remote user to connect as. + +### LABAL_ALL_NODES + +default: `False` +If True, label all nodes with placement=logtest. +If False, remove placment label from all worker nodes and only add it back to 1. + +### PROJECT_BASENAME + +default: `logtest-` +Basename for project creation + +### NUM_PROJECTS + +default: `1` +Number of logtest projects to create. + +### NUM_LINES + +default: `1800000` +Number of lines for logtest to generate per project. + +### LINE_LENGTH + +default: `1024` +Length of log lines in bytes. + +### RATE + +default: `60000` +Messages per second for logtest pod to generate. + +### PAUSE_OFFSET + +default: `5` +Time to wait (in minutes) after logtest is done before verifying all messages showed up in elasticsearch. + +### ORIGIN_TESTS_VERSION + +default: `latest` +Version of quay.io/openshift/origin-tests to pull. \ No newline at end of file diff --git a/workloads/logging.yml b/workloads/logging.yml new file mode 100644 index 00000000..d9ae7d49 --- /dev/null +++ b/workloads/logging.yml @@ -0,0 +1,95 @@ +--- +# Assumptions: +# - KUBECONFIG env var is set on orchestration host (would have to be in bashrc) or kubeconfig is set in ~/.kube/config +# - cluster-logging operator deployed +# - podman installed +# - libselinux-python2 (or variation depending on OS) installed +- name: Run logging test + hosts: orchestration + remote_user: "{{orchestration_user}}" + vars_files: + - vars/logging.yml + tasks: + - name: Verify oc command is available + command: oc version + changed_when: False + - name: Verify openshift-logging exists + shell: oc get projects | grep "openshift-logging" + changed_when: False + - name: Verify ElasticSearch pods exist + shell: oc get pods -n openshift-logging | grep elasticsearch + changed_when: False + - name: Get ElasticSearch pod to query from + shell: oc get pods -n openshift-logging | grep elasticsearch | grep Running | head -n 1 | awk '{print $1}' + register: es_pod + changed_when: False + tags: + - delete_indices + - name: Delete existing logtest indices + shell: | + BASE_CMD="oc exec -n openshift-logging -c elasticsearch {{ es_pod.stdout }} -- curl --connect-timeout 2 -s -k --cert /etc/elasticsearch/secret/admin-cert --key /etc/elasticsearch/secret/admin-key" + ${BASE_CMD} -X DELETE https://localhost:9200/project.{{PROJECT_BASENAME}}*; + tags: + - delete_indices + - name: Get worker nodes + shell: oc get nodes | grep worker | awk '{print $1}' + register: worker_nodes + changed_when: False + - name: Label single worker node + block: + - name: Remove any existing "placement" labels from worker nodes + shell: "oc label node {{ item }} placement-" + loop: "{{ worker_nodes.stdout_lines }}" + register: result + changed_when: '"not labeled" not in result.stdout' + - name: Label one worker node with placement=logtest + shell: oc label node $(oc get nodes | grep worker | head -n 1 | awk '{print $1}') placement=logtest + when: not LABAL_ALL_NODES + tags: label_node + - name: Label all nodes + shell: "for i in $(oc get nodes | grep worker | awk '{print $1}'); do oc label node/$i placement=logtest --overwrite; done" + when: LABAL_ALL_NODES + tags: label_node + - name: Create workloads directory + file: + path: workloads + state: directory + mode: '0755' + - name: Template cluster-loader logtest logging config file + template: + src: templates/logtest.yml.j2 + dest: workloads/logtest.yml + - name: Template logtest-rc.json + template: + src: templates/logtest-rc.json.j2 + dest: workloads/logtest-rc.json + - name: Podman pull cluster-loader image + podman_image: + name: quay.io/openshift/origin-tests + tag: "{{ORIGIN_TESTS_VERSION}}" + - name: Launch cluster-loader + shell: | + podman run --rm \ + -v {{ ansible_facts['env']['KUBECONFIG']|default(ansible_facts['env']['HOME'] + '/.kube/config') }}:/root/.kube/config:z \ + -v {{ansible_facts['env']['HOME']}}/workloads/:/root/workloads/:z \ + -i quay.io/openshift/origin-tests:{{ORIGIN_TESTS_VERSION}} \ + /bin/bash -c 'export KUBECONFIG=/root/.kube/config && \ + export VIPERCONFIG=/root/workloads/logtest.yml && \ + openshift-tests run-test "{{ CLUSTER_LOADER_STRING }} concurrently with templates [Suite:openshift]"' \ + |& tee workloads/cluster-loader.log + - name: Puase for time of test + PAUSE_OFFSET + pause: + minutes: "{{ (NUM_LINES|int / RATE|int + PAUSE_OFFSET|int) | round(0, 'ceil') | int }}" + - name: Get number of messages indexed + shell: x=0; for i in $(oc exec -n openshift-logging {{ es_pod.stdout }} -c elasticsearch -- es_util --query='_cat/indices'| grep '{{ PROJECT_BASENAME }}' | awk '{print $7}'); do (( x += i )); done; echo "$x" + register: num_indexed + - debug: + var: num_indexed + - name: Assert number of messages is equal to NUM_LINES*NUM_PROJECTS + assert: + that: + - "{{ num_indexed.stdout }} == {{ (NUM_LINES|int) * (NUM_PROJECTS|int) }}" + - name: Clean up logtest projects + shell: oc delete project $(oc get projects | grep '{{ PROJECT_BASENAME }}' | awk '{print $1}' | sed ':a;N;$!ba;s/\n/ /g') + tags: + - cleanup diff --git a/workloads/templates/logtest-rc.json.j2 b/workloads/templates/logtest-rc.json.j2 new file mode 100644 index 00000000..3e44633b --- /dev/null +++ b/workloads/templates/logtest-rc.json.j2 @@ -0,0 +1,106 @@ +{% raw %} +{ + "apiVersion": "template.openshift.io/v1", + "kind": "Template", + "metadata": { + "name": "centos-logtest-template" + }, + "objects": [ + { + "apiVersion": "v1", + "data": { + "ocp_logtest.cfg": "${INITIAL_FLAGS}" + }, + "kind": "ConfigMap", + "metadata": { + "name": "logtest-config" + } + }, + { + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "name": "centos-logtest", + "labels": { + "run": "centos-logtest", + "test": "centos-logtest" + } + }, + "spec": { + "replicas": "${{REPLICAS}}", + "template": { + "metadata": { + "generateName": "centos-logtest-", + "labels": { + "run": "centos-logtest", + "test": "centos-logtest" + } + }, + "spec": { + "nodeSelector": { + "placement": "${PLACEMENT}" + }, + "containers": [ + { + "env": [ + ], + "image": "${LOGTEST_IMAGE}", + "imagePullPolicy": "Always", + "name": "centos-logtest", + "resources": {}, + "volumeMounts": [ + { + "name": "config", + "mountPath": "/var/lib/svt" + } + ], + "terminationMessagePath": "/dev/termination-log" + } + ], + "volumes": [ + { + "name": "config", + "configMap": { + "name": "logtest-config" + } + } + ], + "imagePullSecrets": [ + { + "name": "default-dockercfg-ukomu" + } + ] + } + } + } + } + ], +{% endraw %} + "parameters": [ + { + "name": "LOGTEST_IMAGE", + "displayName": "logtest image", + "value": "quay.io/mffiedler/ocp-logtest:latest" + }, + { + "name": "INITIAL_FLAGS", + "description": "The initial flags to pass to ocp_logtest.py", + "value": "--num-lines {{ NUM_LINES }} --line-length {{ LINE_LENGTH }} --word-length 9 --rate {{ RATE }} --fixed-line\n" + }, + { + "name": "IDENTIFIER", + "displayName": "identifier", + "value": "1" + }, + { + "name": "REPLICAS", + "displayName": "Replicas", + "value": "1" + }, + { + "name": "PLACEMENT", + "displayName": "Placement", + "value": "logtest" + } + ] +} diff --git a/workloads/templates/logtest.yml.j2 b/workloads/templates/logtest.yml.j2 new file mode 100644 index 00000000..cbaa2b54 --- /dev/null +++ b/workloads/templates/logtest.yml.j2 @@ -0,0 +1,23 @@ +provider: local +ClusterLoader: + cleanup: false + projects: + - num: {{NUM_PROJECTS}} + basename: {{PROJECT_BASENAME}} + ifexists: delete + tuning: default + templates: + - num: 1 + file: ./logtest-rc.json + parameters: + - REPLICAS: "1" + - INITIAL_FLAGS: "--num-lines {{NUM_LINES}} --line-length {{LINE_LENGTH}} --word-length 9 --rate {{RATE}} --fixed-line\n" + + tuningsets: + - name: default + pods: + stepping: + stepsize: 5 + pause: 0 + rate_limit: + delay: 0 diff --git a/workloads/vars/logging.yml b/workloads/vars/logging.yml new file mode 100644 index 00000000..6d077b06 --- /dev/null +++ b/workloads/vars/logging.yml @@ -0,0 +1,21 @@ +orchestration_user: "{{ lookup('env', 'ORCHESTRATION_USER')|default('root', true) }}" + +LABAL_ALL_NODES: "{{ lookup('env', 'LABAL_ALL_NODES')|default(False, true) }}" + +# Logtest parameters +PROJECT_BASENAME: "{{ lookup('env', 'PROJECT_BASENAME')|default('logtest-', true) }}" +NUM_PROJECTS: "{{ lookup('env', 'NUM_PROJECTS')|default(1, true) }}" +NUM_LINES: "{{ lookup('env', 'NUM_LINES')|default(1800000, true) }}" +LINE_LENGTH: "{{ lookup('env', 'LINE_LENGTH')|default(1024, true) }}" +# Messages per minute +RATE: "{{ lookup('env', 'RATE')|default(60000, true) }}" +# Time to wait (in minutes) after logtest is done before verifying all messages showed up in elasticsearch +PAUSE_OFFSET: "{{ lookup('env', 'PAUSE_OFFSET')|default(5, true) }}" + +# Version of quay.io/openshift/origin-tests to pull +ORIGIN_TESTS_VERSION: "{{ lookup('env', 'ORIGIN_TESTS_VERSION')|default('latest', true) }}" + +CLUSTER_LOADER_STRING: "{{ '[sig-scalability][Feature:Performance][Serial][Slow] Load cluster' + if ORIGIN_TESTS_VERSION == 'latest' + or ORIGIN_TESTS_VERSION is version('4.4', '>=') + else '[Feature:Performance][Serial][Slow] Load cluster' }}" \ No newline at end of file