Switch CI to DistributedNext and threads

Author: jpsamaroo

.buildkite/pipeline.yml

@@ -27,9 +27,12 @@ steps:
       - JuliaCI/julia#v1:
           version: "1.9"
       - JuliaCI/julia-test#v1:
-          julia_args: "--threads=1"
+          julia_args: "--threads=2"
       - JuliaCI/julia-coverage#v1:
           codecov: true
+    env:
+      JULIA_DAGGER_DISTRIBUTED_CHOICE: "DistributedNext"
+      JULIA_NUM_THREADS: "2"
 
   - label: Julia 1.10
     timeout_in_minutes: 90

src/Dagger.jl

@@ -204,6 +204,14 @@ function __init__()
     catch err
         @warn "Error parsing JULIA_DAGGER_DEBUG" exception=err
     end
+
+    # Enable Distributed/DistributedNext, if specified
+    if haskey(ENV, "JULIA_DAGGER_DISTRIBUTED_CHOICE")
+        choice = ENV["JULIA_DAGGER_DISTRIBUTED_CHOICE"]
+        if choice != @load_preference("distributed-package")
+            set_distributed_package!(choice)
+        end
+    end
 end
 
 end # module

test/runtests.jl

@@ -54,6 +54,11 @@ all_test_names = map(test -> replace(last(test), ".jl"=>""), tests)
 
 additional_workers::Int = 3
 
+if haskey(ENV, "JULIA_DAGGER_DISTRIBUTED_CHOICE")
+    choice = ENV["JULIA_DAGGER_DISTRIBUTED_CHOICE"]
+    @assert choice == @load_preference("distributed-package") "Please reload Julia to switch to $choice distributed backend"
+end
+
 if PROGRAM_FILE != "" && realpath(PROGRAM_FILE) == @__FILE__
     pushfirst!(LOAD_PATH, @__DIR__)
     pushfirst!(LOAD_PATH, joinpath(@__DIR__, ".."))