feat: E-tag should be calculated based on blob's content #954

storage/src/main/java/com/epam/aidial/core/storage/data/ResourceUpload.java

@@ -1,6 +1,7 @@
 package com.epam.aidial.core.storage.data;
 
 import com.epam.aidial.core.storage.blobstore.BlobStorage;
+import com.epam.aidial.core.storage.util.EtagBuilder;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufInputStream;
 import lombok.Data;
@@ -21,25 +22,36 @@ public class ResourceUpload {
     private final String contentType;
     private final long updatedAt;
     private final long createdAt;
+    private final EtagBuilder etagBuilder;
     private long contentLength;
     private int chunkNumber = 0;
+    private String etag;
 
     public ResourceUpload(BlobStorage blobStorage, MultipartUpload mpu, String contentType, long createdAt, long updatedAt) {
         this.multipartUpload = mpu;
         this.blobStorage = blobStorage;
         this.contentType = contentType;
         this.createdAt = createdAt;
         this.updatedAt = updatedAt;
+        this.etagBuilder = new EtagBuilder();
     }
 
     public void addChunk(ByteBuf chunk) throws IOException {
         try (Payload payload = bufferToPayload(chunk.duplicate())) {
             MultipartPart part = blobStorage.storeMultipartPart(multipartUpload, ++chunkNumber, payload);
             parts.add(part);
         }
+        etagBuilder.append(chunk.nioBuffer());
         contentLength += chunk.readableBytes();
     }
 
+    public String calculateEtag() {
+        if (etag == null) {
+            etag = etagBuilder.build();
+        }
+        return etag;
+    }
+
     public void abort() {
         blobStorage.abortMultipartUpload(multipartUpload);
     }

storage/src/main/java/com/epam/aidial/core/storage/service/ResourceService.java

@@ -495,7 +495,15 @@ public FileMetadata finishFileUpload(ResourceDescriptor descriptor, ResourceUplo
             long updatedAt = resourceUpload.getUpdatedAt();
             Long createdAt = resourceUpload.getCreatedAt();
 
-            String etag = blobStore.completeMultipartUpload(multipartUpload, parts);
+            String etag = resourceUpload.calculateEtag();
+
+            // try to update user metadata with the computed e-tag.
+            // Note. That doesn't work in AWS S3. Instead of that we take e-tag to be provided by AWS S3.
+            // AWS S3 computes e-tag based blob content.
+            Map<String, String> userMetadata =  multipartUpload.blobMetadata().getUserMetadata();
+            userMetadata.put(ETAG_ATTRIBUTE, etag);
+
+            blobStore.completeMultipartUpload(multipartUpload, parts);
 
             ResourceEvent.Action action = metadata == null
                     ? ResourceEvent.Action.CREATE
@@ -901,6 +909,20 @@ static String decode(String val) {
         return new String(Base58.decode(decoded), StandardCharsets.UTF_8);
     }
 
+    /**
+     * Extracts e-tag from the blob metadata.
+     *
+     * <p>
+     *     There are two cases:
+     *     <ul>
+     *         <li>e-tag exists in user metadata. GCP and Azure support updating user metadata on completion of multipart upload request.</li>
+     *         <li>e-tag doesn't exists in user metadata. AWS S3 provides own etag instead because it doesn't support user metadata update</li>
+     *     </ul>
+     * </p>
+     *
+     * @param meta - blob metadata
+     * @return e-tag
+     */
     private static String extractEtag(BlobMetadata meta) {
         Map<String, String> attributes = meta.getUserMetadata();
         String etag = attributes.get(ETAG_ATTRIBUTE);