Scale back up if exclude annotation is added while downscaled (#45)

* Scale back up if exclude annotation is added while downscaled

* Scales back up when namespace is excluded

* Adds log message before continue (so that it's tracked in test coverage)

Author: Jason Gwartz

kube_downscaler/scaler.py

@@ -35,16 +35,19 @@ def pods_force_uptime(api, namespace: str):
 
 def autoscale_resource(resource: pykube.objects.NamespacedAPIObject,
                        default_uptime: str, default_downtime: str, forced_uptime: bool, dry_run: bool,
-                       now: datetime.datetime, grace_period: int, downtime_replicas: int):
+                       now: datetime.datetime, grace_period: int, downtime_replicas: int, namespace_excluded=False):
     try:
         # any value different from "false" will ignore the resource (to be on the safe side)
-        exclude = resource.annotations.get(EXCLUDE_ANNOTATION, 'false').lower() != 'false'
-        if exclude:
+        exclude = namespace_excluded or (resource.annotations.get(EXCLUDE_ANNOTATION, 'false').lower() != 'false')
+        original_replicas = resource.annotations.get(ORIGINAL_REPLICAS_ANNOTATION)
+        downtime_replicas = resource.annotations.get(DOWNTIME_REPLICAS_ANNOTATION, downtime_replicas)
+
+        if exclude and not original_replicas:
             logger.debug('%s %s/%s was excluded', resource.kind, resource.namespace, resource.name)
         else:
             replicas = resource.replicas
 
-            if forced_uptime:
+            if forced_uptime or (exclude and original_replicas):
                 uptime = "forced"
                 downtime = "ignored"
                 is_uptime = True
@@ -53,8 +56,6 @@ def autoscale_resource(resource: pykube.objects.NamespacedAPIObject,
                 downtime = resource.annotations.get(DOWNTIME_ANNOTATION, default_downtime)
                 is_uptime = helper.matches_time_spec(now, uptime) and not helper.matches_time_spec(now, downtime)
 
-            original_replicas = resource.annotations.get(ORIGINAL_REPLICAS_ANNOTATION)
-            downtime_replicas = resource.annotations.get(DOWNTIME_REPLICAS_ANNOTATION, downtime_replicas)
             logger.debug('%s %s/%s has %s replicas (original: %s, uptime: %s)',
                          resource.kind, resource.namespace, resource.name, replicas, original_replicas, uptime)
             update_needed = False
@@ -94,21 +95,20 @@ def autoscale_resources(api, kind, namespace: str,
                         now: datetime.datetime, grace_period: int, downtime_replicas: int):
     for resource in kind.objects(api, namespace=(namespace or pykube.all)):
         if resource.namespace in exclude_namespaces or resource.name in exclude_names:
+            logger.debug('Resource %s was excluded (either resource itself or namespace %s are excluded)', resource.name, namespace)
             continue
 
         # Override defaults with (optional) annotations from Namespace
         namespace_obj = pykube.Namespace.objects(api).get_by_name(resource.namespace)
 
-        if namespace_obj.annotations.get(EXCLUDE_ANNOTATION, 'false').lower() != 'false':
-            logger.debug('Namespace %s was excluded (because of namespace annotation)', namespace)
-            continue
+        excluded = namespace_obj.annotations.get(EXCLUDE_ANNOTATION, 'false').lower() != 'false'
 
         default_uptime_for_namespace = namespace_obj.annotations.get(UPTIME_ANNOTATION, default_uptime)
         default_downtime_for_namespace = namespace_obj.annotations.get(DOWNTIME_ANNOTATION, default_downtime)
         forced_uptime_for_namespace = namespace_obj.annotations.get(FORCE_UPTIME_ANNOTATION, forced_uptime)
 
         autoscale_resource(resource, default_uptime_for_namespace, default_downtime_for_namespace,
-                           forced_uptime_for_namespace, dry_run, now, grace_period, downtime_replicas)
+                           forced_uptime_for_namespace, dry_run, now, grace_period, downtime_replicas, namespace_excluded=excluded)
 
 
 def scale(namespace: str, default_uptime: str, default_downtime: str, kinds: FrozenSet[str],

tests/test_scaler.py

@@ -1,7 +1,7 @@
 import json
 from unittest.mock import MagicMock
 
-from kube_downscaler.scaler import scale, ORIGINAL_REPLICAS_ANNOTATION, DOWNTIME_REPLICAS_ANNOTATION
+from kube_downscaler.scaler import scale, ORIGINAL_REPLICAS_ANNOTATION, EXCLUDE_ANNOTATION, DOWNTIME_REPLICAS_ANNOTATION
 
 
 def test_scaler_always_up(monkeypatch):
@@ -186,3 +186,84 @@ def get(url, version, **kwargs):
     assert api.patch.call_args[1]['url'] == 'deployments/deploy-1'
     assert json.loads(api.patch.call_args[1]['data'])['spec']['replicas'] == ORIGINAL
     assert not json.loads(api.patch.call_args[1]['data'])['metadata']['annotations'][ORIGINAL_REPLICAS_ANNOTATION]
+
+
+def test_scaler_upscale_on_exclude(monkeypatch):
+    api = MagicMock()
+    monkeypatch.setattr('kube_downscaler.scaler.helper.get_kube_api', MagicMock(return_value=api))
+    ORIGINAL_REPLICAS = 2
+
+    def get(url, version, **kwargs):
+        if url == 'pods':
+            data = {'items': []}
+        elif url == 'deployments':
+            data = {'items': [
+                {
+                    'metadata': {
+                        'name': 'deploy-1', 'namespace': 'default',
+                        'annotations': {
+                            EXCLUDE_ANNOTATION: 'true',
+                            ORIGINAL_REPLICAS_ANNOTATION: ORIGINAL_REPLICAS,
+                        }
+                    },
+                    'spec': {'replicas': 0}},
+                ]}
+        elif url == 'namespaces/default':
+            data = {'metadata': {}}
+        else:
+            raise Exception(f'unexpected call: {url}, {version}, {kwargs}')
+
+        response = MagicMock()
+        response.json.return_value = data
+        return response
+
+    api.get = get
+
+    kinds = frozenset(['deployment'])
+    scale(namespace=None, default_uptime='never', default_downtime='always', kinds=kinds,
+          exclude_namespaces=[], exclude_deployments=[], exclude_statefulsets=[], dry_run=False, grace_period=300, downtime_replicas=0)
+
+    assert api.patch.call_count == 1
+    assert api.patch.call_args[1]['url'] == 'deployments/deploy-1'
+    assert json.loads(api.patch.call_args[1]['data'])["spec"]["replicas"] == ORIGINAL_REPLICAS
+    assert not json.loads(api.patch.call_args[1]['data'])["metadata"]["annotations"][ORIGINAL_REPLICAS_ANNOTATION]
+
+
+def test_scaler_upscale_on_exclude_namespace(monkeypatch):
+    api = MagicMock()
+    monkeypatch.setattr('kube_downscaler.scaler.helper.get_kube_api', MagicMock(return_value=api))
+    ORIGINAL_REPLICAS = 2
+
+    def get(url, version, **kwargs):
+        if url == 'pods':
+            data = {'items': []}
+        elif url == 'deployments':
+            data = {'items': [
+                {
+                    'metadata': {
+                        'name': 'deploy-1', 'namespace': 'default',
+                        "annotations": {
+                            ORIGINAL_REPLICAS_ANNOTATION: ORIGINAL_REPLICAS,
+                        }
+                    },
+                    'spec': {'replicas': 0}},
+                ]}
+        elif url == 'namespaces/default':
+            data = {'metadata': {'annotations': {EXCLUDE_ANNOTATION: 'true'}}}
+        else:
+            raise Exception(f'unexpected call: {url}, {version}, {kwargs}')
+
+        response = MagicMock()
+        response.json.return_value = data
+        return response
+
+    api.get = get
+
+    kinds = frozenset(['deployment'])
+    scale(namespace=None, default_uptime='never', default_downtime='always', kinds=kinds,
+          exclude_namespaces=[], exclude_deployments=[], exclude_statefulsets=[], dry_run=False, grace_period=300, downtime_replicas=0)
+
+    assert api.patch.call_count == 1
+    assert api.patch.call_args[1]['url'] == 'deployments/deploy-1'
+    assert json.loads(api.patch.call_args[1]['data'])["spec"]["replicas"] == ORIGINAL_REPLICAS
+    assert not json.loads(api.patch.call_args[1]['data'])["metadata"]["annotations"][ORIGINAL_REPLICAS_ANNOTATION]