CORE-1219 - upload latest ancillary version

bakery-src/scripts/check_feed.py

@@ -4,6 +4,7 @@
 from typing import NamedTuple, Any, Optional
 from datetime import datetime
 from operator import itemgetter
+from itertools import groupby
 
 import boto3
 import botocore
@@ -128,6 +129,18 @@ def unique(it, *, key=hash):
             yield entry
 
 
+def get_latest_code_version(api_root):
+    url = api_root.rstrip("/") + "/api/version/"
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.json()
+
+
+def get_is_latest_code_version(api_root, code_version):
+    version_json = get_latest_code_version(api_root)
+    return version_json["tag"] == code_version
+
+
 # replace the book feed from github with the accepted books from the ABL endpoint
 # somewhere in the pipeline code api_root has the url to the ABL endpoint
 # and the code version is passed as an argument
@@ -138,14 +151,34 @@ def unique(it, *, key=hash):
 def get_abl(api_root, code_version):
     url = api_root.rstrip("/") + "/api/abl/?code_version=" + code_version
     response = requests.get(url)
+    is_latest_code_version = get_is_latest_code_version(api_root, code_version)
     response.raise_for_status()
     abl_json = response.json()
-    entries = unique(abl_json, key=itemgetter("repository_name", "commit_sha"))
-    entries = sorted(entries, key=itemgetter("committed_at"))
-    results = [
-        {"repo": entry["repository_name"], "version": entry["commit_sha"]}
-        for entry in entries
-    ]
+    entries = list(
+        unique(abl_json, key=itemgetter("repository_name", "commit_sha"))
+    )
+    # Each edition of a book has a unique uuid
+    # This identity scopes to repo + edition
+    identity_getter = itemgetter("repository_name", "uuid")
+    version_getter = itemgetter("committed_at")
+    ranked = {
+        identity: list(sorted(group, key=version_getter, reverse=True))
+        for identity, group in groupby(entries, key=identity_getter)
+    }
+
+    results = []
+    for entry in entries:
+        latest = ranked[identity_getter(entry)][0]
+        book = {}
+        book["repo"] = entry["repository_name"]
+        book["version"] = entry["commit_sha"]
+        book["metadata"] = metadata = {}
+        metadata["is_latest"] = (
+            is_latest_code_version and
+            version_getter(latest) == version_getter(entry)
+        )
+        results.append(book)
+
     return results
 
 

bakery-src/tests/test_bakery_scripts.py

@@ -1172,16 +1172,29 @@ def test_check_feed(mocker):
 
     api_root = "https://mock.corgi"
 
-    class MockBookFeedResponse:
-        def __init__(self, book_feed):
-            self.book_feed = book_feed
+    class MockJsonResponse:
+        def __init__(self, json_value):
+            self.json_value = json_value
 
         def json(self):
-            return self.book_feed
+            return self.json_value
 
         def raise_for_status(self):
             pass
 
+
+    class MockApi:
+        def __init__(self, book_feed, version_json):
+            self.book_feed = book_feed
+            self.version_json = version_json
+
+        def get(self, url):
+            if "/api/abl" in url:
+                return self.book_feed
+            elif "/api/version" in url:
+                return self.version_json
+
+
     # We'll use the botocore stubber to play out a simple scenario to test the
     # script where we'll trigger multiple invocations to "build" all books
     # above. Documenting this in words just to help with readability and
@@ -1259,6 +1272,7 @@ def _stubber_delete_object(expected_key):
     book = {
         "repo": input_book_feed[0]["repository_name"],
         "version": input_book_feed[0]["commit_sha"],
+        "metadata": {"is_latest": True}
     }
 
     repo1 = input_book_feed[0]["repository_name"]
@@ -1327,8 +1341,17 @@ def _stubber_delete_object(expected_key):
     mock_send_slack_message = unittest.mock.Mock(wraps=send_slack_message)
     mock_datetime = unittest.mock.Mock()
     mock_datetime.now.return_value = datetime.fromtimestamp(0)
+    mock_version_json = MockJsonResponse({
+        "stack_name":"mock",
+        "tag": code_version,
+        "revision":"0",
+        "deployed_at":"0"
+    })
 
-    check_feed.requests.get = lambda url: MockBookFeedResponse(input_book_feed)
+    mock_book_feed = MockJsonResponse(input_book_feed)
+    api = MockApi(mock_book_feed, mock_version_json)
+
+    check_feed.requests.get = api.get
 
     with (
         unittest.mock.patch(
@@ -1395,9 +1418,18 @@ def _stubber_delete_object(expected_key):
 
     repo1 = second_book_feed[0]["repository_name"]
     vers1 = second_book_feed[0]["commit_sha"]
-    book = {"repo": repo1, "version": vers1}
-
-    check_feed.requests.get = lambda url: MockBookFeedResponse(second_book_feed)
+    book = {"repo": repo1, "version": vers1, "metadata": {"is_latest": False}}
+
+    mock_book_feed = MockJsonResponse(second_book_feed)
+    mock_version_json = MockJsonResponse({
+        "stack_name":"mock",
+        # Newer code version means this is not the latest version of this build
+        "tag": "20210101.00000002",
+        "revision":"0",
+        "deployed_at":"0"
+    })
+    api = MockApi(mock_book_feed, mock_version_json)
+    check_feed.requests.get = api.get
 
     # Book: Check for .complete file
     _stubber_add_head_object_404(

dockerfiles/docker-entrypoint.sh

@@ -327,6 +327,10 @@ function parse_book_dir() {
     ARG_GIT_REF="$(cat $IO_BOOK/version)"
     ARG_ENABLE_CORGI_UPLOAD=0
     ARG_ENABLE_SOURCEMAPS=0
+    ARG_IS_LATEST=0
+    if [[ -f $IO_BOOK/metadata ]]; then
+        ARG_IS_LATEST="$(jq -r 'if .is_latest then 1 else 0 end' $IO_BOOK/metadata)"
+    fi
     if [[ -f $IO_BOOK/job_id ]]; then
         ARG_ENABLE_CORGI_UPLOAD=1
         ARG_ENABLE_SOURCEMAPS=1
@@ -388,6 +392,8 @@ function do_step() {
                 echo -n "$arg_book_slug" > "$INPUT_SOURCE_DIR/slugs"
             fi
             echo "$version" > "$INPUT_SOURCE_DIR/version"
+            # NOTE: for now this is only here to help test stubbed uploads
+            echo '{ "is_latest": true }' > "$INPUT_SOURCE_DIR/metadata"
             # Dummy files
             echo '-123456' > "$INPUT_SOURCE_DIR/id" # job_id
             echo '{"content_server":{"name":"not_a_real_job_json_file"}}' > "$INPUT_SOURCE_DIR/job.json"
@@ -408,7 +414,9 @@ function do_step() {
             if [[ -f "$INPUT_SOURCE_DIR/slugs" ]]; then
                 cp "$INPUT_SOURCE_DIR/slugs" "$IO_BOOK/slugs"
             fi
-
+            if [[ -f "$INPUT_SOURCE_DIR/metadata" ]]; then
+                cp "$INPUT_SOURCE_DIR/metadata" "$IO_BOOK/metadata"
+            fi
             return
         ;;
     esac

dockerfiles/steps/git-dequeue-book.bash

@@ -8,5 +8,6 @@ fi
 
 echo -n "$(cat $book | jq -r '.repo')" >$IO_BOOK/repo
 echo -n "$(cat $book | jq -r '.version')" >$IO_BOOK/version
+echo -n "$(cat $book | jq -r '.metadata')" >$IO_BOOK/metadata
 
 # LCOV_EXCL_STOP

dockerfiles/steps/step-upload-book.bash

@@ -74,8 +74,7 @@ if [[ $ARG_ENABLE_CORGI_UPLOAD == 1 ]]; then
     jo -a "${book_slug_urls[@]}" > "$IO_ARTIFACTS/artifact_urls.json"
 
     echo "View web preview here: $rex_prod_url"
-else
-    # Do not do this in CORGI builds
+elif [[ "${ARG_IS_LATEST:-0}" -eq 1 ]]; then
     upload_ancillaries "$IO_ANCILLARY"
 fi
 

test/test-step-10.bash

@@ -8,10 +8,6 @@ if [[ $CI = '' && $VIRTUAL_ENV = '' ]]; then
    exit 1
 fi
 
-
-# More details: https://github.com/aws/aws-cli/issues/8036#issuecomment-1638544754
-# and: https://github.com/yaml/pyyaml/issues/601
-# PyYAML installed as dependency here [awscli](https://github.com/aws/aws-cli/blob/dbbf1ce01acec0116710968bbe5a96680e791c1b/setup.py#L30)
 pip install "../bakery-src/scripts/.[test]"
 flake8 --config ../bakery-src/.flake8 ../bakery-src/scripts