Fix struct/const revision

docs/src/debugging.md

@@ -23,7 +23,7 @@
 
 Currently, the best way to turn on logging is within a running Julia session:
 
 ```jldoctest; setup=(using Revise)
 julia> rlogger = Revise.debug_logger()
 Revise.ReviseLogger(Revise.LogRecord[], Debug)
 ```
@@ -104,7 +104,7 @@
 ### The structure of the logs
 
 For those who want to do a little investigating on their own, it may be helpful to
-know that Revise's core decisions are captured in the group called "Action," and they come in three
+know that Revise's core changes are captured in the group called "Action," and they come in three
 flavors:
 
 - log entries with message `"Eval"` signify a call to `eval`; for these events,

src/loading.jl

@@ -110,3 +110,19 @@ function modulefiles(mod::Module)
     included_files = filter(mf->mf.id == id, includes)
     return keypath(parentfile), [keypath(mf.filename) for mf in included_files]
 end
+
+function modulefiles_basestlibs(id)
+    ret = Revise.pkg_fileinfo(id)
+    cachefile, includes = ret === nothing ? (nothing, nothing) : ret[1:2]
+    # `cachefile` will be nothing for Base and stdlibs that *haven't* been moved out
+    cachefile === nothing && return Iterators.drop(Base._included_files, 1)  # stepping through sysimg.jl rebuilds Base, omit it
+    # stdlibs that are packages
+    mod = Base.loaded_modules[id]
+    return map(includes) do inc
+        submod = mod
+        for sm in inc.modpath
+            submod = getfield(submod, Symbol(sm))
+        end
+        return (submod, inc.filename)
+    end
+end

src/logging.jl

@@ -44,6 +44,7 @@ CoreLogging.catch_exceptions(::ReviseLogger) = false
 
 function Base.show(io::IO, l::LogRecord; kwargs...)
     verbose = get(io, :verbose, false)::Bool
+    tmin = get(io, :time_min, nothing)::Union{Float64, Nothing}
     if !isempty(kwargs)
         Base.depwarn("Supplying keyword arguments to `show(io, l::Revise.LogRecord; verbose)` is deprecated, use `IOContext` instead.", :show)
         for (kw, val) in kwargs
@@ -57,6 +58,9 @@ function Base.show(io::IO, l::LogRecord; kwargs...)
         print(io, '(', l.level, ", ", l.message, ", ", l.group, ", ", l.id, ", \"", l.file, "\", ", l.line)
     else
         print(io, "Revise ", l.message)
+        if tmin !== nothing
+            print(io, ", time=", l.kwargs[:time] - tmin)
+        end
     end
     exc = nothing
     if !isempty(l.kwargs)
@@ -69,11 +73,11 @@ function Base.show(io::IO, l::LogRecord; kwargs...)
             elseif kw === :deltainfo
                 keepitem = nothing
                 for item in val
-                    if isa(item, DataType) || isa(item, MethodSummary) || (keepitem === nothing && isa(item, Union{RelocatableExpr,Expr}))
+                    if isa(item, DataType) || isa(item, Union{MethodSummary,Vector{MethodSummary}}) || (keepitem === nothing && isa(item, Union{RelocatableExpr,Expr}))
                         keepitem = item
                     end
                 end
-                if isa(keepitem, MethodSummary)
+                if isa(keepitem, Union{MethodSummary,Vector{MethodSummary}})
                     print(io, ": ", keepitem)
                 elseif isa(keepitem, Union{RelocatableExpr,Expr})
                     print(io, ": ", firstline(keepitem))
@@ -95,6 +99,15 @@ function Base.show(io::IO, l::LogRecord; kwargs...)
     end
 end
 
+function Base.show(io::IO, ::MIME"text/plain", rlogger::ReviseLogger)
+    print(io, "ReviseLogger with min_level=", rlogger.min_level)
+    if !isempty(rlogger.logs)
+        println(io, ":")
+        ioctx = IOContext(io, :time_min => first(rlogger.logs).kwargs[:time], :compact => true)
+        show(ioctx, MIME("text/plain"), rlogger.logs)
+    end
+end
+
 const _debug_logger = ReviseLogger()
 
 """
@@ -113,6 +126,8 @@ with the following relevant fields:
     examined for possible code changes. This is typically done on the basis of `mtime`,
     the modification time of the file, and does not necessarily indicate that there were
     any changes.
+  + "Bindings": "propagating" consequences of rebinding event(s), where dependent types
+    or methods need to be re-evaluated.
 - `message`: a string containing more information. Some examples:
   + For entries in the "Action" group, `message` can be `"Eval"` when modifying
     old methods or defining new ones, "DeleteMethod" when deleting a method,

src/lowered.jl

@@ -448,10 +448,31 @@ function _methods_by_execution!(interp::Interpreter, methodinfo, frame::Frame, i
                     pc = step_expr!(interp, frame, stmt, true)
                 end
             elseif head === :call
-                f = lookup(interp, frame, stmt.args[1])
-                if isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3
-                    T = lookup(interp, frame, stmt.args[2])
-                    lnn = lookup(interp, frame, stmt.args[3])
+                f = lookup(frame, stmt.args[1])
+                if __bpart__ && f === Core._typebody!
+                    # Handle type redefinition
+                    newtype = Base.unwrap_unionall(lookup(frame, stmt.args[3]))
+                    newtypename = newtype.name
+                    oldtype = isdefinedglobal(newtypename.module, newtypename.name) ? getglobal(newtypename.module, newtypename.name) : nothing
+                    if oldtype !== nothing
+                        nfts = lookup(frame, stmt.args[4])
+                        oldtype = Base.unwrap_unionall(oldtype)
+                        ofts = fieldtypes(oldtype)
+                        if !Core._equiv_typedef(oldtype, newtype) || !all(ab -> recursive_egal(ab..., oldtype), zip(nfts, ofts))
+                            isrequired[pc:end] .= true   # ensure we evaluate all remaining statements (probably not needed, but just in case)
+                            # Find all methods restricted to `oldtype`
+                            meths = methods_with(oldtype)
+                            # For any modules that have not yet been parsed and had their signatures extracted,
+                            # we need to do this now, before the binding changes to the new type
+                            maybe_extract_sigs_for_meths(meths)
+                            union!(reeval_methods, meths)
+                        end
+                    end
+                    pc = step_expr!(interp, frame, stmt, true)
+                elseif isdefined(Core, :_defaultctors) && f === Core._defaultctors && length(stmt.args) == 3
+                    # Create the constructors for a type (i.e., a method definition)
+                    T = lookup(frame, stmt.args[2])
+                    lnn = lookup(frame, stmt.args[3])
                     if T isa Type && lnn isa LineNumberNode
                         empty!(signatures)
                         uT = Base.unwrap_unionall(T)::DataType

src/packagedef.jl

@@ -6,6 +6,10 @@ using Base: PkgId
 using Base.Meta: isexpr
 using Core: CodeInfo, MethodTable
 
+if !isdefined(Core, :isdefinedglobal)
+    isdefinedglobal(m::Module, s::Symbol) = isdefined(m, s)
+end
+
 export revise, includet, entr, MethodSummary
 
 ## BEGIN abstract Distributed API
@@ -101,6 +105,7 @@ include("utils.jl")
 include("parsing.jl")
 include("lowered.jl")
 include("loading.jl")
+include("visit.jl")
 include("pkgs.jl")
 include("git.jl")
 include("recipes.jl")
@@ -147,6 +152,12 @@ Global variable, maps `(pkgdata, filename)` pairs that errored upon last revisio
 """
 const queue_errors = Dict{Tuple{PkgData,String},Tuple{Exception, Any}}() # locking is covered by revision_queue_lock
 
+# Can we revise types?
+const __bpart__ = Base.VERSION >= v"1.12.0-DEV.2047"
+# Julia 1.12+: when bindings switch to a new type, we need to re-evaluate method
+# definitions using the new binding resolution.
+const reeval_methods = Set{Method}()
+
 """
     Revise.NOPACKAGE
 
@@ -265,6 +276,37 @@ const silence_pkgs = Set{Symbol}()
 const depsdir = joinpath(dirname(@__DIR__), "deps")
 const silencefile = Ref(joinpath(depsdir, "silence.txt"))  # Ref so that tests don't clobber
 
+function collect_mis(sigs)
+    mis = Core.MethodInstance[]
+    world = Base.get_world_counter()
+    for tt in sigs
+        matches = Base._methods_by_ftype(tt, 10, world)::Vector
+        for mm in matches
+            m = mm.method
+            for mi in Base.specializations(m)
+                if mi.specTypes <: tt
+                    push!(mis, mi)
+                end
+            end
+        end
+    end
+    return mis
+end
+
+# from Compiler/bootstrap.jl
+const compiler_mis = if __bpart__
+    collect_mis(Any[
+        Tuple{typeof(Core.Compiler.compact!), Vararg{Any}},
+        Tuple{typeof(Core.Compiler.ssa_inlining_pass!), Core.Compiler.IRCode, Core.Compiler.InliningState{Core.Compiler.NativeInterpreter}, Bool},
+        Tuple{typeof(Core.Compiler.optimize), Core.Compiler.NativeInterpreter, Core.Compiler.OptimizationState{Core.Compiler.NativeInterpreter}, Core.Compiler.InferenceResult},
+        Tuple{typeof(Core.Compiler.typeinf_ext), Core.Compiler.NativeInterpreter, Core.MethodInstance, UInt8},
+        Tuple{typeof(Core.Compiler.typeinf), Core.Compiler.NativeInterpreter, Core.Compiler.InferenceState},
+        Tuple{typeof(Core.Compiler.typeinf_edge), Core.Compiler.NativeInterpreter, Method, Any, Core.SimpleVector, Core.Compiler.InferenceState, Bool, Bool},
+    ])
+else
+    Core.MethodInstance[]
+end
+
 ##
 ## The inputs are sets of expressions found in each file.
 ## Some of those expressions will generate methods which are identified via their signatures.
@@ -378,8 +420,8 @@ function eval_rex(rex::RelocatableExpr, exs_sigs_old::ExprsSigs, mod::Module; mo
         if rexo === nothing
             ex = rex.ex
             # ex is not present in old
-            @debug "Eval" _group="Action" time=time() deltainfo=(mod, ex)
-            mt_sigs, includes, thunk = eval_with_signatures(mod, ex; mode=mode)  # All signatures defined by `ex`
+            @debug titlecase(String(mode)) _group="Action" time=time() deltainfo=(mod, ex, mode)
+            mt_sigs, includes, thunk = eval_with_signatures(mod, ex; mode)  # All signatures defined by `ex`
             if !isexpr(thunk, :thunk)
                 thunk = ex
             end
@@ -713,7 +755,7 @@ function revise_file_now(pkgdata::PkgData, file)
         error(file, " is not currently being tracked.")
     end
     mexsnew, mexsold = handle_deletions(pkgdata, file)
-    if mexsnew != nothing
+    if mexsnew !== nothing
         _, includes = eval_new!(mexsnew, mexsold)
         fi = fileinfo(pkgdata, i)
         pkgdata.fileinfos[i] = FileInfo(mexsnew, fi)
@@ -769,8 +811,16 @@ otherwise these are only logged.
 function revise(; throw::Bool=false)
     active[] || return nothing
     sleep(0.01)  # in case the file system isn't quite done writing out the new files
+    # To ensure we don't just call `Core.Compiler.bootstrap!()` for reasons of invalidation rather than redefinition,
+    # record the pre-revision max world ages of the compiler methods.
+    # This means that if those methods get invalidated by loading packages, automatic revision of the compiler won't
+    # work anymore---Revise's changes won't lower the world age to anything less than it already is.
+    # But people testing the compiler often do so in a fresh & slim session, so there's still some value in
+    # automatic revision. One can always call `Compiler.bootstrap!()` manually to reinitialize the compiler.
+    cmaxworlds = Dict(mi => mi.cache.max_world for mi in compiler_mis)
     lock(revision_queue_lock) do
         have_queue_errors = !isempty(queue_errors)
+        empty!(reeval_methods)
 
         # Do all the deletion first. This ensures that a method that moved from one file to another
         # won't get redefined first and deleted second.
@@ -840,6 +890,54 @@ function revise(; throw::Bool=false)
                 queue_errors[(pkgdata, file)] = (err, catch_backtrace())
             end
         end
+        # Handle binding invalidations
+        if !isempty(reeval_methods)
+            handled = Base.IdSet{Type}()
+            while !isempty(reeval_methods)
+                list = collect(reeval_methods)
+                with_logger(_debug_logger) do
+                    @debug "OldTypeMethods" _group="Bindings" time=time() deltainfo=(MethodSummary.(list),)
+                end
+                empty!(reeval_methods)
+                for m in list
+                    methinfo = get(CodeTracking.method_info, Pair{Union{Nothing, Core.MethodTable}, Type}(nothing, m.sig), missing)
+                    if methinfo === missing
+                        push!(handled, m.sig)
+                        continue
+                    end
+                    if length(methinfo) != 1 && Base.unwrap_unionall(m.sig).parameters[1] !== typeof(Core.kwcall)
+                        with_logger(_debug_logger) do
+                            @debug "FailedDeletion" _group="Action" time=time() deltainfo=(m.sig, methinfo)
+                        end
+                        continue
+                    end
+                    if isdefinedglobal(m.module, m.name)
+                        f = getglobal(m.module, m.name)
+                        if isa(f, DataType)
+                            newmeths = methods_with(f)
+                            filter!(m -> m.sig ∉ handled, newmeths)
+                            maybe_extract_sigs_for_meths(newmeths)
+                            union!(reeval_methods, newmeths)
+                        end
+                    end
+                    !iszero(m.dispatch_status) && with_logger(_debug_logger) do
+                        @debug "DeleteMethod" _group="Action" time=time() deltainfo=(m.sig, MethodSummary(m))
+                        Base.delete_method(m)  # ensure that "old data" doesn't get run with "old methods"
+                        delete!(CodeTracking.method_info, m.sig)
+                        _, ex = methinfo[1]
+                        @debug "Eval" _group="Action" time=time() deltainfo=(mod, ex)
+                        invokelatest(eval_with_signatures, m.module, ex; mode=:eval)
+                    end
+                    push!(handled, m.sig)
+                end
+            end
+        end
+        # If needed, reinitialize the compiler
+        init_compiler = false
+        for mi in compiler_mis
+            init_compiler |= mi.cache.max_world < cmaxworlds[mi]
+        end
+        init_compiler && Core.Compiler.bootstrap!()
         if interrupt
             for pkgfile in finished
                 haskey(queue_errors, pkgfile) || delete!(revision_queue, pkgfile)
@@ -948,7 +1046,7 @@ function track(mod::Module, file::AbstractString; mode=:sigs, kwargs...)
         if mode === :includet
             mode = :sigs   # we already handled evaluation in `parse_source`
         end
-        instantiate_sigs!(fm; mode, kwargs...)
+        invokelatest(instantiate_sigs!, fm; mode, kwargs...)
         if !haskey(pkgdatas, id)
             # Wait a bit to see if `mod` gets initialized
             sleep(0.1)

src/pkgs.jl

@@ -129,6 +129,23 @@ function maybe_extract_sigs!(fi::FileInfo)
 end
 maybe_extract_sigs!(pkgdata::PkgData, file::AbstractString) = maybe_extract_sigs!(fileinfo(pkgdata, file))
 
+function maybe_extract_sigs_for_meths(meths)
+    for m in meths
+        methinfo = get(CodeTracking.method_info, m.sig, false)
+        if methinfo === false
+            pkgdata = get(pkgdatas, PkgId(m.module), nothing)
+            pkgdata === nothing && continue
+            for file in srcfiles(pkgdata)
+                fi = fileinfo(pkgdata, file)
+                if (isempty(fi.modexsigs) && !fi.parsed[]) && (!isempty(fi.cachefile) || !isempty(fi.cacheexprs))
+                    fi = maybe_parse_from_cache!(pkgdata, file)
+                    instantiate_sigs!(fi.modexsigs)
+                end
+            end
+        end
+    end
+end
+
 function maybe_add_includes_to_pkgdata!(pkgdata::PkgData, file::AbstractString, includes; eval_now::Bool=false)
     for (mod, inc) in includes
         inc = joinpath(splitdir(file)[1], inc)

src/precompile.jl

@@ -62,7 +62,7 @@ function _precompile_()
     mbody = bodymethod(mex)
     # use `typeof(pairs(NamedTuple()))` here since it actually differs between Julia versions
     @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, typeof(pairs(NamedTuple())), typeof(methods_by_execution!), Compiled, MI, Module, Expr})
-    @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Tuple{Symbol},NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Compiled, MI, Module, Expr})
+    @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, Bool, Iterators.Pairs{Symbol,Bool,Nothing,NamedTuple{(:skip_include,),Tuple{Bool}}}, typeof(methods_by_execution!), Compiled, MI, Module, Expr})
     mfr = which(_methods_by_execution!, (Compiled, MI, Frame, Vector{Bool}))
     mbody = bodymethod(mfr)
     @warnpcfail precompile(Tuple{mbody.sig.parameters[1], Symbol, Bool, typeof(_methods_by_execution!), Compiled, MI, Frame, Vector{Bool}})

src/recipes.jl

@@ -63,8 +63,17 @@ function _track(id::PkgId, modname::Symbol; modified_files=revision_queue)
         if pkgdata === nothing
             pkgdata = PkgData(id, srcdir)
         end
+        ret = Revise.pkg_fileinfo(id)
+        if ret !== nothing
+            cachefile, _ = ret
+            if cachefile === nothing
+                @error "unable to find cache file for $id, tracking is not possible"
+            end
+        else
+            cachefile = basesrccache
+        end
         lock(revision_queue_lock) do
-            for (submod, filename) in Iterators.drop(Base._included_files, 1)  # stepping through sysimg.jl rebuilds Base, omit it
+            for (submod, filename) in modulefiles_basestlibs(id)
                 ffilename = fixpath(filename)
                 inpath(ffilename, dirs) || continue
                 keypath = ffilename[1:last(findfirst(dirs[end], ffilename))]
@@ -74,7 +83,7 @@ function _track(id::PkgId, modname::Symbol; modified_files=revision_queue)
                     cache_file_key[fullpath] = filename
                     src_file_key[filename] = fullpath
                 end
-                push!(pkgdata, rpath=>FileInfo(submod, basesrccache))
+                push!(pkgdata, rpath=>FileInfo(submod, cachefile))
                 if mtime(ffilename) > mtcache
                     with_logger(_debug_logger) do
                         @debug "Recipe for Base/StdLib" _group="Watching" filename=filename mtime=mtime(filename) mtimeref=mtcache
@@ -205,7 +214,8 @@ const stdlib_names = Set([
 
 # This replacement is needed because the path written during compilation differs from
 # the git source path
-const stdlib_rep = joinpath("usr", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)") => "stdlib"
+const stdpath_rep = (joinpath("usr", "share", "julia", "stdlib", "v$(VERSION.major).$(VERSION.minor)") => "stdlib",
+                    joinpath("usr", "share", "julia", "Compiler") => "Compiler")
 
-const juliaf2m = Dict(normpath(replace(file, stdlib_rep))=>mod
+const juliaf2m = Dict(normpath(replace(file, stdpath_rep...))=>mod
     for (mod,file) in Base._included_files)

src/utils.jl

@@ -96,6 +96,23 @@ function unwrap_where(ex::Expr)
     return ex::Expr
 end
 
+function recursive_egal(@nospecialize(a), @nospecialize(b), @nospecialize(bskip))
+    # Like ===, except for recursive structs this unpacks all the parameters
+    (isa(a, Type) && isa(b, Type)) || return a === b
+    b === bskip && return true
+    typeof(a) === typeof(b) || return false
+    isa(a, Core.TypeofBottom) && return a === b
+    isa(a, Union) && return (recursive_egal(a.a, b.a, bskip) && recursive_egal(a.b, b.b, bskip))
+    a = Base.unwrap_unionall(a)
+    b = Base.unwrap_unionall(b)
+    length(a.parameters) === length(b.parameters) || return false
+    length(a.parameters) == 0 && return a === b
+    for (ap, bp) in zip(a.parameters, b.parameters)
+        recursive_egal(ap, bp, bskip) || return false
+    end
+    return true
+end
+
 function pushex!(exsigs::ExprsSigs, ex::Expr)
     uex = unwrap(ex)
     if is_doc_expr(uex)