support keep/leave = :random in nonunique/unique/unique!

Author: sl-solution

src/abstractdataset/abstractdataset.jl

@@ -989,9 +989,10 @@ A row is a duplicate if there exists a prior row with all columns containing
 equal values (according to `isequal`).
 
 If `mapformats = true` the values are checked based on their formatted values.
-`leave = :first` means that everey occurance after the first one be marked as non-unique value, and
-`leave = :last` means that every occurance before the last one be marked as non-unique value.
-`leave = :none` means that no duplicated rows are marked as non-unique value.
+`leave = :first` means that everey occurance after the first one be marked as non-unique value,
+`leave = :last` means that every occurance before the last one be marked as non-unique value,
+`leave = :none` means that no duplicated rows are marked as non-unique value,
+`leave = :random` means that a random occurance of duplicated values be marked as non-unique value.
 
 See also [`unique`](@ref) and [`unique!`](@ref).
 
@@ -1052,13 +1053,16 @@ julia> nonunique(ds, 2)
 """
 function nonunique(ds::AbstractDataset, cols::MultiColumnIndex = :; mapformats = false, leave = :first)
     # :xor, :nor, :and, :or are undocumented
-    !(leave in (:first, :last, :none, :xor, :nand, :nor, :and, :or)) && throw(ArgumentError("`leave` must be either `:first`, `:last`, or `:none`"))
+    !(leave in (:first, :last, :none, :random, :xor, :nand, :nor, :and, :or)) && throw(ArgumentError("`leave` must be either `:first`, `:last`, `:none`, or `random`"))
     if ncol(ds) == 0
         throw(ArgumentError("finding duplicate rows in data set with no " *
                             "columns is not allowed"))
     end
 
     groups, gslots, ngroups = _gather_groups(ds, cols, nrow(ds) < typemax(Int32) ? Val(Int32) : Val(Int64), mapformats = mapformats, stable = false)
+    if leave === :random
+        return _nonunique_random_leave(groups, ngroups, nrow(ds))
+    end
     res = trues(nrow(ds))
     seen_groups = falses(ngroups)
     if leave === :first
@@ -1105,6 +1109,29 @@ function _nonunique_barrier!(res, groups, seen_groups; first = true)
     nothing
 end
 
+function _nonunique_random_leave_barrier!(counts, groups)
+    for i in 1:length(groups)
+        counts[groups[i]] += 1
+    end
+    map!(x->rand(1:x), counts, counts)
+end
+
+function _fill_nonunique_randomleave!(res, counts, groups)
+    for i in 1:length(res)
+        res[i] = counts[groups[i]] == 1
+        counts[groups[i]] -= 1
+    end
+end
+
+function _nonunique_random_leave(groups, ngroups, nrows)
+    counts = Vector{nrows < typemax(Int32) ? Int32 : Int64}(undef, ngroups)
+    fill!(counts, 0)
+    _nonunique_random_leave_barrier!(counts, groups)
+    res = falses(nrows)
+    _fill_nonunique_randomleave!(res, counts, groups)
+    .!res
+end
+
 """
     vcat(dss::AbstractDataset...;
          cols::Union{Symbol, AbstractVector{Symbol},

src/dataset/other.jl

@@ -622,35 +622,41 @@ function _unique_none_case(ds::Dataset, cols; mapformats = false)
 end
 
 # Modify Dataset
-function Base.unique!(ds::Dataset; mapformats = false, keep = :first)
-    !(keep in (:first, :last, :none, :only)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, or :none"))
+function Base.unique!(ds::AbstractDataset; mapformats = false, keep = :first)
+    !(keep in (:first, :last, :none, :only, :random)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, :none, or :random"))
     if keep == :none
         rowidx = nonunique(ds, mapformats = mapformats, leave = :or)
     elseif keep == :only
         rowidx = nonunique(ds, mapformats = mapformats, leave = :none)
+    elseif keep == :random
+        rowidx = nonunique(ds, mapformats = mapformats, leave = :random)
     else
         rowidx = nonunique(ds, mapformats = mapformats, leave = keep)
     end
     deleteat!(ds, rowidx)
 end
 # this is for fixing ambiguity
-function Base.unique!(ds::Dataset, cols::AbstractVector; mapformats = false, keep = :first)
-    !(keep in (:first, :last, :none, :only)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, or :none"))
+function Base.unique!(ds::AbstractDataset, cols::AbstractVector; mapformats = false, keep = :first)
+    !(keep in (:first, :last, :none, :only, :random)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, :none, or :random"))
     if keep == :none
         rowidx = nonunique(ds, cols, mapformats = mapformats, leave = :or)
     elseif keep == :only
         rowidx = nonunique(ds, cols, mapformats = mapformats, leave = :none)
+    elseif keep == :random
+        rowidx = nonunique(ds, cols, mapformats = mapformats, leave = :random)
     else
         rowidx = nonunique(ds, cols, mapformats = mapformats, leave = keep)
     end
     deleteat!(ds, rowidx)
 end
-function Base.unique!(ds::Dataset, cols; mapformats = false, keep = :first)
-    !(keep in (:first, :last, :none, :only)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, or :none"))
+function Base.unique!(ds::AbstractDataset, cols; mapformats = false, keep = :first)
+    !(keep in (:first, :last, :none, :only, :random)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, :none, or :random"))
     if keep == :none
         rowidx = nonunique(ds, cols, mapformats = mapformats, leave = :or)
     elseif keep == :only
         rowidx = nonunique(ds, cols, mapformats = mapformats, leave = :none)
+    elseif keep == :random
+        rowidx = nonunique(ds, cols, mapformats = mapformats, leave = :random)
     else
         rowidx = nonunique(ds, cols, mapformats = mapformats, leave = keep)
     end
@@ -660,23 +666,27 @@ end
 
 # Unique rows of an Dataset.
 @inline function Base.unique(ds::AbstractDataset; view::Bool=false, mapformats = false, keep = :first)
-    !(keep in (:first, :last, :none, :only)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, or :none"))
+    !(keep in (:first, :last, :none, :only, :random)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, :none, or :random"))
     if keep == :none
         rowidxs = nonunique(ds, mapformats = mapformats, leave = :none)
     elseif keep == :only
         rowidxs = nonunique(ds, mapformats = mapformats, leave = :or)
+    elseif keep == :random
+        rowidxs = .!nonunique(ds, mapformats = mapformats, leave = :random)
     else
         rowidxs = (!).(nonunique(ds, mapformats = mapformats, leave = keep))
     end
     return view ? Base.view(ds, rowidxs, :) : ds[rowidxs, :]
 end
 
 @inline function Base.unique(ds::AbstractDataset, cols; view::Bool=false, mapformats = false, keep = :first)
-    !(keep in (:first, :last, :none, :only)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, or :none"))
+    !(keep in (:first, :last, :none, :only, :random)) && throw(ArgumentError( "The `keep` keyword argument must be one of :first, :last, :only, :none, or :random"))
     if keep == :none
         rowidxs = nonunique(ds, cols, mapformats = mapformats, leave = :none)
     elseif keep == :only
         rowidxs = nonunique(ds, cols, mapformats = mapformats, leave = :or)
+    elseif keep == :random
+        rowidxs = .!nonunique(ds, cols, mapformats = mapformats, leave = :random)
     else
         rowidxs = (!).(nonunique(ds, cols, mapformats = mapformats, leave = keep))
     end
@@ -688,10 +698,10 @@ end
     unique(ds::AbstractDataset, cols = : ; [mapformats = false, keep = :first, view::Bool=false])
     unique!(ds::AbstractDataset, cols = : ; [mapformats = false, keep = :first, view::Bool=false])
 
-Return a data set containing only the unique occurrence of unique rows in `ds` where `keep` can be one of the following value: `:first`, `:last`, `:none`, or `:only`. The
+Return a data set containing only the unique occurrence of unique rows in `ds` where `keep` can be one of the following value: `:first`, `:last`, `:none`, `:only`, or `:random`. The
 `keep` keyword argument detemines which occurrence of the unique value should be kept, i.e. when `keep = :first` the
 first occurrence of the unique value will be kept and when `keep = :last` the last occurrence will be kept. When `keep` is set to `:none`
-all duplicates will be dropped from the result, and when `keep` is set to `:only` only duplicated rows are kept.
+all duplicates will be dropped from the result, when `keep` is set to `:only` only duplicated rows are kept, and when `keep` is set ot `:random` a random occurance of duplicates will be kept.
  When `cols` is specified, the unique occurrence is detemined by given combination of values
 in selected columns. `cols` can be any column selector.