Introduce check for "out of CI credits" situation

When a Cirrus CI job started by 'cirrus-run' fails due lack of CI
credits, it would look exactly like a failure in the job itself, thus
the user would not be able to distinguish the problems.

If a 'taks' failed due to CI minute lack, the 'notifications' field of
given 'task' contains the following message:

 Failed to start an instance: FAILED_PRECONDITION: Monthly compute limit exceeded!

To detect this, receive also task data and their notifications when
waiting for a job and look for the message about CI credits.

In order to see this status from scripts this will also return a new
error code '3' if the out of CI credits situation is detected.

Additionally in order to allow quick adaptation when the above string
will be changed add '--cirrus-out-of-ci-credits-message' parameter which
can configure the string.

Signed-off-by: Peter Krempa <[email protected]>

Author: pipo

README.md

@@ -93,6 +93,7 @@ return values:
   - 0: The cirrus job was successful
   - 1: The job failed
   - 2: Error in 'cirrus-run'
+  - 3: The Cirrus CI job ran out of CI minutes
 ```
 
 

cirrus_run/cli.py

@@ -14,7 +14,7 @@
 
 from . import CirrusAPI
 from .throbber import ProgressBar
-from .queries import build_log, get_repo, create_build, wait_build, CirrusBuildError
+from .queries import build_log, get_repo, create_build, wait_build, CirrusBuildError, CirrusCreditsError
 
 log = logging.getLogger(__name__)
 
@@ -47,10 +47,12 @@ def run(args, retry_index=0):
     print('Build created: {}'.format(build_url))
     with ProgressBar('' if args.verbose else '.'):
         try:
-            wait_build(api, build_id, abort=args.timeout*60)
+            wait_build(api, build_id, abort=args.timeout*60, credits_error_message=args.cirrus_out_of_ci_credits_message)
             rc, status, message = 0, 'successful', ''
         except CirrusBuildError:
             rc, status, message = 1, 'failed', ''
+        except CirrusCreditsError:
+            rc, status, message = 3, 'error', 'Out of CI credits'
         except Exception as exc:
             rc, status, message = 2, 'error', '{exception}: {text}'.format(
                                         exception=exc.__class__.__name__,
@@ -224,6 +226,15 @@ def parse_args(*a, **ka):
             'the build is retried once more. Default: ${}'
         ).format(ENVIRONMENT['flaky_markers']),
     )
+    parser.add_argument(
+        '--cirrus-out-of-ci-credits-message',
+        default='Monthly compute limit exceeded',
+        help=(
+            'Error message passed via "notifications" from Cirrus CI reported'
+            'when the CI job failed due to lack of CI credits.'
+            'Default: "Monthly compute limit exceeded"'
+        ),
+    )
     args = parser.parse_args(*a, **ka)
 
     if not args.token:

cirrus_run/queries.py

@@ -22,6 +22,8 @@ class CirrusQueryError(ValueError):
 class CirrusBuildError(RuntimeError):
     '''Raised on build failures'''
 
+class CirrusCreditsError(RuntimeError):
+    '''Raised when build fails due to lack of CI credits'''
 
 class CirrusTimeoutError(RuntimeError):
     '''Raised when build takes too long'''
@@ -85,14 +87,19 @@ def create_build(api: CirrusAPI,
     return answer['createBuild']['build']['id']
 
 
-def wait_build(api, build_id: str, delay=3, abort=60*60):
+def wait_build(api, build_id: str, delay=3, abort=60*60, credits_error_message=None):
     '''Wait until build finishes'''
     ERROR_CONFIRM_TIMES = 3
 
     query = '''
         query GetBuild($build: ID!) {
             build(id: $build) {
                 status
+                tasks {
+                    notifications {
+                        message
+                    }
+                }
             }
         }
     '''
@@ -116,6 +123,12 @@ def wait_build(api, build_id: str, delay=3, abort=60*60):
                 sleep(2 * delay / (ERROR_CONFIRM_TIMES - 1))
                 continue
             else:
+                if credits_error_message is not None:
+                    for task in response['build']['tasks']:
+                        for notif in task['notifications']:
+                            if credits_error_message in notif['message']:
+                                raise CirrusCreditsError('build {} ran out of CI credits'.format(build_id))
+
                 raise CirrusBuildError('build {} was terminated: {}'.format(build_id, status))
         raise ValueError('build {} returned unknown status: {}'.format(build_id, status))
     raise CirrusTimeoutError('build {} timed out'.format(build_id))