Logtest with concurrent project creation

docs/logging.md

@@ -0,0 +1,79 @@
+# Logging Workload
+
+Stress test the cluster logging and ElasticSearch operators.
+
+## Requirements
+
+* Cluster logging operator is deployed
+* kubeconfig present in $HOME/.kube/config on orchestration host
+* podman and libselinux-python2 installed on orchestration host
+* selinux set to permissive (otherwise cluster-loader will fail)
+
+## Run from CLI
+
+```shell script
+cp workloads/inventory.example inventory
+# Add orchestration host to inventory
+# Define environmental variables or use default logtest parameters
+ansible-playbook -v -i inventory workloads/logging.yml
+```
+
+## Implementation Notes
+Currently I am templating parameters directly into logtest-rc.json because 
+I ran into issues trying to get cluster-loader to pass in the values from logtest.yml. 
+I left in the step where the variables are templated into logtest.yml in case anything changes.
+
+## Ansible Variables
+
+### LABAL_ALL_NODES
+default: False  
+If False: Remove all placement=logtest labels from worker nodes then add it back to just 1 worker node.  
+If True: Apply placement=logtest label to all worker nodes
+
+## Environmental Variables
+
+### ORCHESTRATION_USER
+
+default: `root`  
+Remote user to connect as.
+
+### LABAL_ALL_NODES
+
+default: `False`  
+If True, label all nodes with placement=logtest.  
+If False, remove placment label from all worker nodes and only add it back to 1.
+
+### PROJECT_BASENAME
+
+default: `logtest-`  
+Basename for project creation
+
+### NUM_PROJECTS
+
+default: `1`  
+Number of logtest projects to create.
+
+### NUM_LINES
+
+default: `1800000`  
+Number of lines for logtest to generate per project.
+
+### LINE_LENGTH
+
+default: `1024`  
+Length of log lines in bytes.
+
+### RATE
+
+default: `60000`  
+Messages per second for logtest pod to generate.
+
+### PAUSE_OFFSET
+
+default: `5`  
+Time to wait (in minutes) after logtest is done before verifying all messages showed up in elasticsearch.
+
+### ORIGIN_TESTS_VERSION
+
+default: `latest`  
+Version of quay.io/openshift/origin-tests to pull.

workloads/logging.yml

@@ -0,0 +1,95 @@
+---
+# Assumptions:
+#  - KUBECONFIG env var is set on orchestration host (would have to be in bashrc) or kubeconfig is set in ~/.kube/config
+#  - cluster-logging operator deployed
+#  - podman installed
+#  - libselinux-python2 (or variation depending on OS) installed
+- name: Run logging test
+  hosts: orchestration
+  remote_user: "{{orchestration_user}}"
+  vars_files:
+    - vars/logging.yml
+  tasks:
+    - name: Verify oc command is available
+      command: oc version
+      changed_when: False
+    - name: Verify openshift-logging exists
+      shell: oc get projects | grep "openshift-logging"
+      changed_when: False
+    - name: Verify ElasticSearch pods exist
+      shell: oc get pods -n openshift-logging | grep elasticsearch
+      changed_when: False
+    - name: Get ElasticSearch pod to query from
+      shell: oc get pods -n openshift-logging | grep elasticsearch | grep Running | head -n 1 | awk '{print $1}'
+      register: es_pod
+      changed_when: False
+      tags:
+        - delete_indices
+    - name: Delete existing logtest indices
+      shell: |
+        BASE_CMD="oc exec -n openshift-logging -c elasticsearch {{ es_pod.stdout }} -- curl --connect-timeout 2 -s -k --cert /etc/elasticsearch/secret/admin-cert --key /etc/elasticsearch/secret/admin-key"
+        ${BASE_CMD} -X DELETE https://localhost:9200/project.{{PROJECT_BASENAME}}*;
+      tags:
+        - delete_indices
+    - name: Get worker nodes
+      shell: oc get nodes | grep worker | awk '{print $1}'
+      register: worker_nodes
+      changed_when: False
+    - name: Label single worker node
+      block:
+        - name: Remove any existing "placement" labels from worker nodes
+          shell: "oc label node {{ item }} placement-"
+          loop: "{{ worker_nodes.stdout_lines }}"
+          register: result
+          changed_when: '"not labeled" not in result.stdout'
+        - name: Label one worker node with placement=logtest
+          shell: oc label node $(oc get nodes | grep worker | head -n 1 | awk '{print $1}') placement=logtest
+      when: not LABAL_ALL_NODES
+      tags: label_node
+    - name: Label all nodes
+      shell: "for i in $(oc get nodes | grep worker | awk '{print $1}'); do oc label node/$i placement=logtest --overwrite; done"
+      when: LABAL_ALL_NODES
+      tags: label_node
+    - name: Create workloads directory
+      file:
+        path: workloads
+        state: directory
+        mode: '0755'
+    - name: Template cluster-loader logtest logging config file
+      template:
+        src: templates/logtest.yml.j2
+        dest: workloads/logtest.yml
+    - name: Template logtest-rc.json
+      template:
+        src: templates/logtest-rc.json.j2
+        dest: workloads/logtest-rc.json
+    - name: Podman pull cluster-loader image
+      podman_image:
+        name: quay.io/openshift/origin-tests
+        tag: "{{ORIGIN_TESTS_VERSION}}"
+    - name: Launch cluster-loader
+      shell: |
+        podman run --rm \
+        -v {{ ansible_facts['env']['KUBECONFIG']|default(ansible_facts['env']['HOME'] + '/.kube/config') }}:/root/.kube/config:z \
+        -v {{ansible_facts['env']['HOME']}}/workloads/:/root/workloads/:z \
+        -i quay.io/openshift/origin-tests:{{ORIGIN_TESTS_VERSION}} \
+        /bin/bash -c 'export KUBECONFIG=/root/.kube/config && \
+        export VIPERCONFIG=/root/workloads/logtest.yml && \
+        openshift-tests run-test "{{ CLUSTER_LOADER_STRING }} concurrently with templates [Suite:openshift]"' \
+        |& tee workloads/cluster-loader.log
+    - name: Puase for time of test + PAUSE_OFFSET
+      pause:
+        minutes: "{{ (NUM_LINES|int / RATE|int + PAUSE_OFFSET|int) | round(0, 'ceil') | int }}"
+    - name: Get number of messages indexed
+      shell: x=0; for i in $(oc exec -n openshift-logging {{ es_pod.stdout }} -c elasticsearch -- es_util --query='_cat/indices'| grep '{{ PROJECT_BASENAME }}' | awk '{print $7}'); do (( x += i )); done; echo "$x"
+      register: num_indexed
+    - debug:
+        var: num_indexed
+    - name: Assert number of messages is equal to NUM_LINES*NUM_PROJECTS
+      assert:
+        that:
+          - "{{ num_indexed.stdout }} == {{ (NUM_LINES|int) * (NUM_PROJECTS|int) }}"
+    - name: Clean up logtest projects
+      shell: oc delete project $(oc get projects | grep '{{ PROJECT_BASENAME }}' | awk '{print $1}' | sed ':a;N;$!ba;s/\n/ /g')
+      tags:
+        - cleanup

workloads/templates/logtest-rc.json.j2

@@ -0,0 +1,106 @@
+{% raw %}
+{
+  "apiVersion": "template.openshift.io/v1",
+  "kind": "Template",
+  "metadata": {
+    "name": "centos-logtest-template"
+  },
+  "objects": [
+    {
+      "apiVersion": "v1",
+      "data": {
+        "ocp_logtest.cfg": "${INITIAL_FLAGS}"
+      },
+      "kind": "ConfigMap",
+      "metadata": {
+        "name": "logtest-config"
+      }
+    },
+    {
+      "apiVersion": "v1",
+      "kind": "ReplicationController",
+      "metadata": {
+        "name": "centos-logtest",
+        "labels": {
+          "run": "centos-logtest",
+          "test": "centos-logtest"
+        }
+      },
+      "spec": {
+        "replicas": "${{REPLICAS}}",
+        "template": {
+          "metadata": {
+            "generateName": "centos-logtest-",
+            "labels": {
+              "run": "centos-logtest",
+              "test": "centos-logtest"
+            }
+          },
+          "spec": {
+            "nodeSelector": {
+              "placement": "${PLACEMENT}"
+            },
+            "containers": [
+              {
+                "env": [
+                ],
+                "image": "${LOGTEST_IMAGE}",
+                "imagePullPolicy": "Always",
+                "name": "centos-logtest",
+                "resources": {},
+                "volumeMounts": [
+                  {
+                    "name": "config",
+                    "mountPath": "/var/lib/svt"
+                  }
+                ],
+                "terminationMessagePath": "/dev/termination-log"
+              }
+            ],
+            "volumes": [
+              {
+                "name": "config",
+                "configMap": {
+                  "name": "logtest-config"
+                }
+              }
+            ],
+            "imagePullSecrets": [
+              {
+                "name": "default-dockercfg-ukomu"
+              }
+            ]
+          }
+        }
+      }
+    }
+  ],
+{% endraw %}
+  "parameters": [
+    {
+      "name": "LOGTEST_IMAGE",
+      "displayName": "logtest image",
+      "value": "quay.io/mffiedler/ocp-logtest:latest"
+    },
+    {
+      "name": "INITIAL_FLAGS",
+      "description": "The initial flags to pass to ocp_logtest.py",
+      "value": "--num-lines {{ NUM_LINES }} --line-length {{ LINE_LENGTH }} --word-length 9 --rate {{ RATE }} --fixed-line\n"
+    },
+    {
+      "name": "IDENTIFIER",
+      "displayName": "identifier",
+      "value": "1"
+    },
+    {
+      "name": "REPLICAS",
+      "displayName": "Replicas",
+      "value": "1"
+    },
+    {
+      "name": "PLACEMENT",
+      "displayName": "Placement",
+      "value": "logtest"
+    }
+  ]
+}

workloads/templates/logtest.yml.j2

@@ -0,0 +1,23 @@
+provider: local
+ClusterLoader:
+  cleanup: false
+  projects:
+    - num: {{NUM_PROJECTS}}
+      basename: {{PROJECT_BASENAME}}
+      ifexists: delete
+      tuning: default
+      templates:
+        - num: 1
+          file: ./logtest-rc.json
+          parameters:
+           - REPLICAS: "1"
+           - INITIAL_FLAGS: "--num-lines {{NUM_LINES}} --line-length {{LINE_LENGTH}} --word-length 9 --rate {{RATE}} --fixed-line\n"
+
+  tuningsets:
+    - name: default
+      pods:
+        stepping:
+          stepsize: 5
+          pause: 0
+        rate_limit:
+          delay: 0

workloads/vars/logging.yml

@@ -0,0 +1,21 @@
+orchestration_user: "{{ lookup('env', 'ORCHESTRATION_USER')|default('root', true) }}"
+
+LABAL_ALL_NODES: "{{ lookup('env', 'LABAL_ALL_NODES')|default(False, true) }}"
+
+# Logtest parameters
+PROJECT_BASENAME: "{{ lookup('env', 'PROJECT_BASENAME')|default('logtest-', true) }}"
+NUM_PROJECTS: "{{ lookup('env', 'NUM_PROJECTS')|default(1, true) }}"
+NUM_LINES: "{{ lookup('env', 'NUM_LINES')|default(1800000, true) }}"
+LINE_LENGTH: "{{ lookup('env', 'LINE_LENGTH')|default(1024, true) }}"
+# Messages per minute
+RATE: "{{ lookup('env', 'RATE')|default(60000, true) }}"
+# Time to wait (in minutes) after logtest is done before verifying all messages showed up in elasticsearch
+PAUSE_OFFSET: "{{ lookup('env', 'PAUSE_OFFSET')|default(5, true) }}"
+
+# Version of quay.io/openshift/origin-tests to pull
+ORIGIN_TESTS_VERSION: "{{ lookup('env', 'ORIGIN_TESTS_VERSION')|default('latest', true) }}"
+
+CLUSTER_LOADER_STRING: "{{ '[sig-scalability][Feature:Performance][Serial][Slow] Load cluster'
+                            if ORIGIN_TESTS_VERSION == 'latest'
+                            or ORIGIN_TESTS_VERSION is version('4.4', '>=')
+                            else '[Feature:Performance][Serial][Slow] Load cluster' }}"