Tests for new retry logic (#26)

---------

Co-authored-by: Justin Funston <[email protected]>

Author: andrebsguedes

Project.toml

@@ -1,6 +1,6 @@
 name = "RustyObjectStore"
 uuid = "1b5eed3d-1f46-4baa-87f3-a4a892b23610"
-version = "0.3.0"
+version = "0.3.1"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
@@ -16,7 +16,7 @@ ReTestItems = "1"
 Sockets = "1"
 Test = "1"
 julia = "1.8"
-object_store_ffi_jll = "0.4"
+object_store_ffi_jll = "0.4.2"
 
 [extras]
 CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed"

test/aws_s3_exception_tests.jl

@@ -158,16 +158,97 @@ end # @testitem
     import Sockets
 
     max_retries = 2
-    retry_timeout_secs = 2
+    retry_timeout_secs = 10
+    request_timeout_secs = 1
+    region = "us-east-1"
+    container = "mybucket"
+    dummy_access_key_id = "qUwJPLlmEtlCDXJ1OUzF"
+    dummy_secret_access_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+
+    function test_tcp_error(method)
+        @assert method === :GET || method === :PUT
+        nrequests = Ref(0)
+
+        (port, tcp_server) = Sockets.listenany(8082)
+        @async begin
+            while true
+                sock = Sockets.accept(tcp_server)
+                _ = read(sock, 4)
+                close(sock)
+                nrequests[] += 1
+            end
+        end
+
+        baseurl = "http://127.0.0.1:$port"
+        conf = AWSConfig(;
+            region=region,
+            bucket_name=container,
+            access_key_id=dummy_access_key_id,
+            secret_access_key=dummy_secret_access_key,
+            host=baseurl,
+            opts=ClientOptions(;
+                max_retries=max_retries,
+                retry_timeout_secs=retry_timeout_secs
+            )
+        )
+
+        try
+            method === :GET && get_object!(zeros(UInt8, 5), "blob", conf)
+            method === :PUT && put_object(codeunits("a,b,c"), "blob", conf)
+            @test false # Should have thrown an error
+        catch e
+            method === :GET && @test e isa RustyObjectStore.GetException
+            method === :PUT && @test e isa RustyObjectStore.PutException
+            @test occursin("connection closed", e.msg)
+        finally
+            close(tcp_server)
+        end
+        return nrequests[]
+    end
+
+    function test_get_stream_error()
+        nrequests = Ref(0)
+
+        (port, tcp_server) = Sockets.listenany(8083)
+        http_server = HTTP.listen!(tcp_server) do http::HTTP.Stream
+            nrequests[] += 1
+            HTTP.setstatus(http, 200)
+            HTTP.setheader(http, "Content-Length" => "20")
+            HTTP.startwrite(http)
+            write(http, "not enough")
+            close(http.stream)
+        end
+
+        baseurl = "http://127.0.0.1:$port"
+        conf = AWSConfig(;
+            region=region,
+            bucket_name=container,
+            access_key_id=dummy_access_key_id,
+            secret_access_key=dummy_secret_access_key,
+            host=baseurl,
+            opts=ClientOptions(;
+                max_retries=max_retries,
+                retry_timeout_secs=retry_timeout_secs
+            )
+        )
+
+        try
+            get_object!(zeros(UInt8, 20), "blob", conf)
+            @test false # Should have thrown an error
+        catch e
+            @test e isa RustyObjectStore.GetException
+            @test occursin("end of file before message length reached", e.msg)
+        finally
+            close(http_server)
+        end
+        wait(http_server)
+        return nrequests[]
+    end
 
     function test_status(method, response_status, headers=nothing)
         @assert method === :GET || method === :PUT
         nrequests = Ref(0)
         response_body = "response body from the dummy server"
-        region = "us-east-1"
-        container = "mybucket"
-        dummy_access_key_id = "qUwJPLlmEtlCDXJ1OUzF"
-        dummy_secret_access_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 
         (port, tcp_server) = Sockets.listenany(8081)
         http_server = HTTP.serve!(tcp_server) do request::HTTP.Request
@@ -209,6 +290,53 @@ end # @testitem
         return nrequests[]
     end
 
+    function test_timeout(method, message, wait_secs::Int = 60)
+        @assert method === :GET || method === :PUT
+        nrequests = Ref(0)
+        response_body = "response body from the dummy server"
+
+        (port, tcp_server) = Sockets.listenany(8081)
+        http_server = HTTP.serve!(tcp_server) do request::HTTP.Request
+            if request.method == "GET" && request.target == "/$container/_this_file_does_not_exist"
+                # This is the exploratory ping from connect_and_test in lib.rs
+                return HTTP.Response(404, "Yup, still doesn't exist")
+            end
+            nrequests[] += 1
+            if wait_secs > 0
+                sleep(wait_secs)
+            end
+            return HTTP.Response(200, response_body)
+        end
+
+        baseurl = "http://127.0.0.1:$port"
+        conf = AWSConfig(;
+            region=region,
+            bucket_name=container,
+            access_key_id=dummy_access_key_id,
+            secret_access_key=dummy_secret_access_key,
+            host=baseurl,
+            opts=ClientOptions(;
+                max_retries=max_retries,
+                retry_timeout_secs=retry_timeout_secs,
+                request_timeout_secs
+            )
+        )
+
+        try
+            method === :GET && get_object!(zeros(UInt8, 5), "blob", conf)
+            method === :PUT && put_object(codeunits("a,b,c"), "blob", conf)
+            @test false # Should have thrown an error
+        catch e
+            method === :GET && @test e isa RustyObjectStore.GetException
+            method === :PUT && @test e isa RustyObjectStore.PutException
+            @test occursin(string(message), e.msg)
+        finally
+            close(http_server)
+        end
+        wait(http_server)
+        return nrequests[]
+    end
+
     @testset "400: Bad Request" begin
         # Returned when there's an error in the request URI, headers, or body. The response body
         # contains an error message explaining what the specific problem is.
@@ -326,4 +454,23 @@ end # @testitem
         nrequests = test_status(:PUT, 504)
         @test nrequests == 1 + max_retries
     end
+
+    @testset "Timeout" begin
+        nrequests = test_timeout(:GET, "timed out", 2)
+        @test nrequests == 1 + max_retries
+        nrequests = test_timeout(:PUT, "timed out", 2)
+        @test nrequests == 1 + max_retries
+    end
+
+    @testset "TCP Closed" begin
+        nrequests = test_tcp_error(:GET)
+        @test nrequests == 1 + max_retries
+        nrequests = test_tcp_error(:PUT)
+        @test nrequests == 1 + max_retries
+    end
+
+    @testset "Incomplete GET body" begin
+        nrequests = test_get_stream_error()
+        @test nrequests == 1 + max_retries
+    end
 end