diff --git a/boot/bootutil/src/bootutil_misc.c b/boot/bootutil/src/bootutil_misc.c
index 5359e0d26..ddb69e27f 100644
--- a/boot/bootutil/src/bootutil_misc.c
+++ b/boot/bootutil/src/bootutil_misc.c
@@ -391,28 +391,47 @@ int
 boot_read_enc_key(const struct flash_area *fap, uint8_t slot, struct boot_status *bs)
 {
     uint32_t off;
-#if MCUBOOT_SWAP_SAVE_ENCTLV
     uint32_t i;
-#endif
     int rc;
+    uint8_t *read_dst;
+    uint32_t read_size;
 
-    off = boot_enc_key_off(fap, slot);
 #if MCUBOOT_SWAP_SAVE_ENCTLV
-    rc = flash_area_read(fap, off, bs->enctlv[slot], BOOT_ENC_TLV_ALIGN_SIZE);
+    /* In this case we have stored entire encryted TLV in swap-state and bs->enckey
+     * will be decrypted from the TLV.
+     */
+    BOOT_LOG_DBG("boot_read_enc_key: TLV");
+    read_dst = bs->enctlv[slot];
+    read_size = BOOT_ENC_TLV_ALIGN_SIZE;
+#else
+    BOOT_LOG_DBG("boot_read_enc_key: RAW key");
+    read_dst = bs->enckey[slot];
+    read_size = BOOT_ENC_KEY_ALIGN_SIZE;
+#endif
+
+    off = boot_enc_key_off(fap, slot);
+
+    rc = flash_area_read(fap, off, read_dst, read_size);
     if (rc == 0) {
-        for (i = 0; i < BOOT_ENC_TLV_ALIGN_SIZE; i++) {
-            if (bs->enctlv[slot][i] != 0xff) {
+        for (i = 0; i < read_size; i++) {
+            if (read_dst[i] != 0xff) {
                 break;
             }
         }
-        /* Only try to decrypt non-erased TLV metadata */
-        if (i != BOOT_ENC_TLV_ALIGN_SIZE) {
+
+        if (i == read_size) {
+            BOOT_LOG_ERR("boot_read_enc_key: No key, read all 0xFF");
+            rc = 1;
+        }
+#if MCUBOOT_SWAP_SAVE_ENCTLV
+        else {
+            /* read_dst is the same as bs->enctlv[slot], and serves as a source
+             * of the encrypted key.
+             */
             rc = boot_decrypt_key(bs->enctlv[slot], bs->enckey[slot]);
         }
-    }
-#else
-    rc = flash_area_read(fap, off, bs->enckey[slot], BOOT_ENC_KEY_ALIGN_SIZE);
 #endif
+    }
 
     return rc;
 }
diff --git a/boot/bootutil/src/loader.c b/boot/bootutil/src/loader.c
index aabb9069a..1726f076a 100644
--- a/boot/bootutil/src/loader.c
+++ b/boot/bootutil/src/loader.c
@@ -1588,7 +1588,6 @@ boot_swap_image(struct boot_loader_state *state, struct boot_status *bs)
     const struct flash_area *fap;
 #ifdef MCUBOOT_ENC_IMAGES
     uint8_t slot;
-    uint8_t i;
 #endif
     uint32_t size;
     uint32_t copy_size;
@@ -1676,15 +1675,10 @@ boot_swap_image(struct boot_loader_state *state, struct boot_status *bs)
             boot_enc_init(BOOT_CURR_ENC_SLOT(state, slot));
 
             rc = boot_read_enc_key(fap, slot, bs);
-            assert(rc == 0);
-
-            for (i = 0; i < BOOT_ENC_KEY_SIZE; i++) {
-                if (bs->enckey[slot][i] != 0xff) {
-                    break;
-                }
-            }
-
-            if (i != BOOT_ENC_KEY_SIZE) {
+            if (rc) {
+                BOOT_LOG_DBG("boot_swap_image: Failed loading key (%d, %d)",
+                              image_index, slot);
+            } else {
                 boot_enc_set_key(BOOT_CURR_ENC_SLOT(state, slot), bs->enckey[slot]);
             }
         }