Add paging by thread (and task, optionally) (#176)

Project.toml

@@ -21,9 +21,9 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 Cairo = "0.6, 0.8, 1"
 Colors = "0.9, 0.10, 0.11, 0.12"
 FileIO = "1.6"
-FlameGraphs = "0.2.8"
+FlameGraphs = "0.2.9"
 Graphics = "0.4, 1"
-Gtk = "0.18, 1"
+Gtk = "1.1.11"
 GtkObservables = "1"
 IntervalSets = "0.2, 0.3, 0.4, 0.5"
 MethodAnalysis = "0.4"

src/ProfileView.jl

@@ -50,6 +50,9 @@ Clear the Profile buffer, profile `f(args...)`, and view the result graphically.
 """
 macro profview(ex)
     return quote
+        # disable the eventloop while profiling
+        # it will be restarted upon show
+        Gtk.enable_eventloop(false)
         Profile.clear()
         @profile $(esc(ex))
         view()
@@ -70,7 +73,10 @@ function closeall()
 end
 
 const window_wrefs = WeakKeyDict{Gtk.GtkWindowLeaf,Nothing}()
+const tabname_allthreads = Symbol("All Threads")
+const tabname_alltasks = Symbol("All Tasks")
 
+NestedGraphDict = Dict{Symbol,Dict{Symbol,Node{NodeData}}}
 """
     ProfileView.view([fcolor], data=Profile.fetch(); lidict=nothing, C=false, recur=:off, fontsize=14, windowname="Profile", kwargs...)
 
@@ -79,15 +85,45 @@ You have several options to control the output, of which the major ones are:
 
 - `fcolor`: an optional coloration function. The main options are `FlameGraphs.FlameColors`
   and `FlameGraphs.StackFrameCategory`.
-- `C`: if true, the graph will include stackframes from C code called by Julia.
+- `C::Bool = false`: if true, the graph will include stackframes from C code called by Julia.
 - `recur`: on Julia 1.4+, collapse recursive calls (see `Profile.print` for more detail)
+- `expand_threads::Bool = true`: Break down profiling by thread (true by default)
+- `expand_tasks::Bool = false`: Break down profiling of each thread by task (false by default)
 
 See [FlameGraphs](https://github.com/timholy/FlameGraphs.jl) for more information.
 """
-function view(fcolor, data::Vector{UInt64}; lidict=nothing, C=false, combine=true, recur=:off, pruned=FlameGraphs.defaultpruned, kwargs...)
+function view(fcolor, data::Vector{UInt64}; lidict=nothing, C=false, combine=true, recur=:off, pruned=FlameGraphs.defaultpruned,
+                expand_threads::Bool=true, expand_tasks::Bool=false, kwargs...)
     g = flamegraph(data; lidict=lidict, C=C, combine=combine, recur=recur, pruned=pruned)
     g === nothing && return nothing
-    return view(fcolor, g; data=data, lidict=lidict, kwargs...)
+    # Dict of dicts. Outer is threads, inner is tasks
+    # Don't report the tasks at the "all threads" level because their id is thread-specific, so it's not useful
+    # to track them across thread TODO: Perhaps fix that in base, so tasks keep the same id across threads?
+    gdict = NestedGraphDict(tabname_allthreads => Dict{Symbol,Node{NodeData}}(tabname_alltasks => g))
+    if expand_threads && isdefined(Profile, :has_meta) && Profile.has_meta(data)
+        dlock_outer = Base.ReentrantLock()
+        Threads.@threads for threadid in Profile.get_thread_ids(data)
+            g = flamegraph(data; lidict=lidict, C=C, combine=combine, recur=recur, pruned=pruned, threads = threadid)
+            gdict_inner = Dict{Symbol,Node{NodeData}}(tabname_alltasks => g)
+            if expand_tasks
+                dlock_inner = Base.ReentrantLock()
+                taskids = Profile.get_task_ids(data, threadid)
+                if length(taskids) > 1
+                    # skip when there's only one task as it will be the same as "all tasks"
+                    Threads.@threads for taskid in taskids
+                        g = flamegraph(data; lidict=lidict, C=C, combine=combine, recur=recur, pruned=pruned, threads = threadid, tasks = taskid)
+                        lock(dlock_inner) do
+                            gdict_inner[Symbol(taskid)] = g
+                        end
+                    end
+                end
+            end
+            lock(dlock_outer) do
+                gdict[Symbol(threadid)] = gdict_inner
+            end
+        end
+    end
+    return view(fcolor, gdict; data=data, lidict=lidict, kwargs...)
 end
 function view(fcolor; kwargs...)
     data, lidict = Profile.retrieve()
@@ -97,6 +133,7 @@ function view(data::Vector{UInt64}; lidict=nothing, kwargs...)
     view(FlameGraphs.default_colors, data; lidict=lidict, kwargs...)
 end
 function view(; kwargs...)
+    Gtk.enable_eventloop(false)
     data, lidict = Profile.retrieve()
     view(FlameGraphs.default_colors, data; lidict=lidict, kwargs...)
 end
@@ -112,36 +149,91 @@ function view(fcolor, g::Node{NodeData}; data=nothing, lidict=nothing, kwargs...
     win, _ = viewgui(fcolor, g; data=data, lidict=lidict, kwargs...)
     Gtk.showall(win)
 end
+function view(g_or_gdict::Union{Node{NodeData},NestedGraphDict}; kwargs...)
+    view(FlameGraphs.default_colors, g_or_gdict; kwargs...)
+end
+function view(fcolor, g_or_gdict::Union{Node{NodeData},NestedGraphDict}; data=nothing, lidict=nothing, kwargs...)
+    win, _ = viewgui(fcolor, g_or_gdict; data=data, lidict=lidict, kwargs...)
+    Gtk.showall(win)
+end
+
+function viewgui(fcolor, g::Node{NodeData}; kwargs...)
+    gdict = NestedGraphDict(tabname_allthreads => Dict{Symbol,Node{NodeData}}(tabname_alltasks => g))
+    viewgui(fcolor, gdict; kwargs...)
+end
+function viewgui(fcolor, gdict::NestedGraphDict; data=nothing, lidict=nothing, windowname="Profile", kwargs...)
+    _c, _fdraw, _tb_open, _tb_save_as = nothing, nothing, nothing, nothing # needed to be returned for precompile helper
+    thread_tabs = collect(keys(gdict))
+    nb_threads = Notebook() # for holding the per-thread pages
+    Gtk.GAccessor.scrollable(nb_threads, true)
+    Gtk.GAccessor.show_tabs(nb_threads, length(thread_tabs) > 1)
+    sort!(thread_tabs, by = s -> something(tryparse(Int, string(s)), 0)) # sorts thread_tabs as [all threads, 1, 2, 3 ....]
+    for thread_tab in thread_tabs
+        gdict_thread = gdict[thread_tab]
+        task_tabs = collect(keys(gdict_thread))
+        sort!(task_tabs, by = s -> s == tabname_alltasks ? "" : string(s)) # sorts thread_tabs as [all threads, 0xds ....]
+
+        nb_tasks = if length(task_tabs) > 1
+            # only show the 2nd tab row if there is more than 1 task
+            nb_tasks = Notebook() # for holding the per-task pages
+            Gtk.GAccessor.scrollable(nb_tasks, true)
+            nb_tasks
+        else
+            nothing
+        end
+
+        task_tab_num = 1
+        @sync for task_tab in task_tabs
+            g = gdict_thread[task_tab]
+            gsig = Observable(g)  # allow substitution by the open dialog
+            c = canvas(UserUnit)
+            set_gtk_property!(widget(c), :expand, true)
+            f = Frame(c)
+            tb = Toolbar()
+            tb_open = ToolButton("gtk-open")
+            tb_save_as = ToolButton("gtk-save-as")
+            push!(tb, tb_open)
+            push!(tb, tb_save_as)
+            # FIXME: likely have to do `allkwargs` in the two below (add in C, combine, recur)
+            signal_connect(open_cb, tb_open, "clicked", Nothing, (), false, (widget(c),gsig,kwargs))
+            signal_connect(save_as_cb, tb_save_as, "clicked", Nothing, (), false, (widget(c),data,lidict,g))
+            bx = Box(:v)
+            push!(bx, tb)
+            push!(bx, f)
+            #
+            if nb_tasks !== nothing
+                # don't use the actual taskid as the tab as it's very long
+                push!(nb_tasks, bx, task_tab_num == 1 ? task_tab : Symbol(task_tab_num - 1))
+            else
+                push!(nb_threads, bx, string(thread_tab))
+            end
+            Threads.@spawn begin
+                fdraw = viewprof(fcolor, c, gsig; kwargs...)
+                GtkObservables.gc_preserve(nb_threads, c)
+                GtkObservables.gc_preserve(nb_threads, fdraw)
+                _fdraw = fdraw
+            end
+            _c, _tb_open, _tb_save_as = c, tb_open, tb_save_as
+            task_tab_num += 1
+        end
+        if nb_tasks !== nothing
+            push!(nb_threads, nb_tasks, string(thread_tab))
+        end
+    end
 
-function viewgui(fcolor, g::Node{NodeData}; data=nothing, lidict=nothing, windowname="Profile", kwargs...)
-    gsig = Observable(g)  # allow substitution by the open dialog
-    # Display in a window
-    c = canvas(UserUnit)
-    set_gtk_property!(widget(c), :expand, true)
-    f = Frame(c)
-    tb = Toolbar()
     bx = Box(:v)
-    push!(bx, tb)
-    push!(bx, f)
-    tb_open = ToolButton("gtk-open")
-    tb_save_as = ToolButton("gtk-save-as")
-    push!(tb, tb_open)
-    push!(tb, tb_save_as)
-    # FIXME: likely have to do `allkwargs` in the two below (add in C, combine, recur)
-    signal_connect(open_cb, tb_open, "clicked", Nothing, (), false, (widget(c),gsig,kwargs))
-    signal_connect(save_as_cb, tb_save_as, "clicked", Nothing, (), false, (widget(c),data,lidict,g))
+    push!(bx, nb_threads)
+
+    # Defer creating the window until here because Window includes a `show` that will unpause the Gtk eventloop
     win = Window(windowname, 800, 600)
     push!(win, bx)
-    GtkObservables.gc_preserve(win, c)
+
     # Register the window with closeall
     window_wrefs[win] = nothing
     signal_connect(win, :destroy) do w
         delete!(window_wrefs, win)
     end
 
-    fdraw = viewprof(fcolor, c, gsig; kwargs...)
-    GtkObservables.gc_preserve(win, fdraw)
-
     # Ctrl-w and Ctrl-q destroy the window
     signal_connect(win, "key-press-event") do w, evt
         if evt.state == CONTROL && (evt.keyval == UInt('q') || evt.keyval == UInt('w'))
@@ -150,7 +242,7 @@ function viewgui(fcolor, g::Node{NodeData}; data=nothing, lidict=nothing, window
         end
     end
 
-    return win, c, fdraw, (tb_open, tb_save_as)
+    return win, _c, _fdraw, (_tb_open, _tb_save_as)
 end
 
 function viewprof(fcolor, c, gsig; fontsize=14)

src/precompile.jl

@@ -17,7 +17,8 @@ function _precompile_()
                                      7=>stackframe(:f1, :file1, 2),
                                      8=>stackframe(:f6, :file3, 10))
     g = flamegraph(backtraces; lidict=lidict)
-    win, c, fdraw = viewgui(FlameGraphs.default_colors, g)
+    gdict = Dict(tabname_allthreads => Dict(tabname_alltasks => g))
+    win, c, fdraw = viewgui(FlameGraphs.default_colors, gdict)
     for obs in c.preserved
         if isa(obs, Observable) || isa(obs, Observables.ObserverFunction)
             precompile(obs)