Adds global panic hook (#24)

* Add rust thread panic hook
---------

Co-authored-by: Justin Funston <[email protected]>
Co-authored-by: Justin Funston <[email protected]>

Author: andrebsguedes

Project.toml

@@ -1,6 +1,6 @@
 name = "RustyObjectStore"
 uuid = "1b5eed3d-1f46-4baa-87f3-a4a892b23610"
-version = "0.2.0"
+version = "0.3.0"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"

README.md

@@ -95,6 +95,10 @@ In this way, the requests are asynchronous all the way up to Julia and the netwo
 For a GET request, Julia provides a buffer for the native library to write into.
 This requires Julia to know a suitable size before-hand and requires the native library to do an extra memory copy, but the upside is that Julia controls the lifetime of the memory.
 
+The library provides a way for Julia code to be notifed about a panic on a Rust thread through the `on_rust_panic` argument of `init_object_store`.
+The default behavior is to log the stack trace (if enabled through RUST_BACKTRACE) and exit the process.
+The general recommendation is to treat Rust panics as fatal because Julia tasks may hang due to not being notified.
+
 #### Threading Model
 
 Rust object_store uses the [tokio](https://docs.rs/tokio) async runtime.

src/RustyObjectStore.jl

@@ -85,9 +85,15 @@ struct InitException <: Exception
     return_code::Cint
 end
 
+function default_panic_hook()
+    println("Rust thread panicked, exiting the process")
+    exit(1)
+end
+
 """
     init_object_store()
     init_object_store(config::StaticConfig)
+    init_object_store(config::StaticConfig; on_rust_panic::Function)
 
 Initialise object store.
 
@@ -96,15 +102,33 @@ It must be called before sending a request e.g. with `get_object!` or `put_objec
 The runtime is only started once and cannot be re-initialised with a different config,
 subsequent `init_object_store` calls have no effect.
 
+An optional panic hook may be provided to react to panics on Rust's native threads.
+The default behavior is to log and exit the process.
+
 # Throws
 - `InitException`: if the runtime fails to start.
 """
-function init_object_store(config::StaticConfig=DEFAULT_CONFIG)
+function init_object_store(
+    config::StaticConfig=DEFAULT_CONFIG;
+    on_rust_panic::Function=default_panic_hook
+)
     @lock _INIT_LOCK begin
         if _OBJECT_STORE_STARTED[]
             return nothing
         end
-        res = @ccall rust_lib.start(config::StaticConfig)::Cint
+        cond = Base.AsyncCondition()
+        errormonitor(Threads.@spawn begin
+            while true
+                wait(cond)
+                try
+                    on_rust_panic()
+                catch e
+                    @error "Custom panic hook failed" exception=(e, catch_backtrace())
+                end
+            end
+        end)
+        panic_cond_handle = cond.handle
+        res = @ccall rust_lib.start(config::StaticConfig, panic_cond_handle::Ptr{Cvoid})::Cint
         if res != 0
             throw(InitException("Failed to initialise object store runtime.", res))
         end

test/rust_panic_test.jl

@@ -0,0 +1,31 @@
+@testitem "Handle object_store_ffi panic" begin
+    # This needs to be run on a spawned process to ensure proper initialization
+    julia_cmd_ffi_panic = Base.julia_cmd()
+    code = """
+    using Test
+    using RustyObjectStore
+
+    triggered = false
+
+    function on_panic()
+        global triggered
+        triggered = true
+    end
+
+    init_object_store(;on_rust_panic=on_panic)
+
+    @test !triggered
+
+    @ccall RustyObjectStore.rust_lib._trigger_panic()::Cint
+
+    @test timedwait(() -> triggered, 0.5) == :ok
+
+    triggered = false
+
+    @ccall RustyObjectStore.rust_lib._trigger_panic()::Cint
+
+    @test timedwait(() -> triggered, 0.5) == :ok
+    """
+    cmd = `$(julia_cmd_ffi_panic) --startup-file=no --project=. -e $code`
+    @test success(pipeline(cmd; stdout=stdout, stderr=stderr))
+end