diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml new file mode 100644 index 000000000..4ac8f6f9f --- /dev/null +++ b/.github/workflows/Benchmark.yml @@ -0,0 +1,59 @@ +name: Benchmarking + +on: + push: + branches: + - main + tags: ['*'] + pull_request: + +jobs: + profile: + name: Benchmarking Analysis Passes + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.version == 'nightly' }} + strategy: + fail-fast: false + matrix: + version: + - '1' + os: + - ubuntu-latest + arch: + - x64 + include: + - version: '1' + os: ubuntu-latest + arch: x64 + coverage: false + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + env: + RUN_MODE: profile + - uses: julia-actions/julia-processcoverage@v1 + if: matrix.coverage + - uses: codecov/codecov-action@v1 + if: matrix.coverage + with: + file: lcov.info + - uses: coverallsapp/github-action@master + if: matrix.coverage + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + path-to-lcov: lcov.info diff --git a/Project.toml b/Project.toml index 16b85facb..11fadd35d 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "0.3.0" AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" BangBang = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" diff --git a/src/JuliaBUGS.jl b/src/JuliaBUGS.jl index eba0ee836..00dad149d 100644 --- a/src/JuliaBUGS.jl +++ b/src/JuliaBUGS.jl @@ -202,6 +202,24 @@ function merge_with_coalescence( return NamedTuple{Tuple(unioned_keys)}(Tuple(coalesced_values)) end +function compute_data_transformation(scalar, array_sizes, model_def, data) + transformed_variables = Dict{Symbol,Any}() + for s in scalar + transformed_variables[s] = missing + end + for (k, v) in array_sizes + transformed_variables[k] = Array{Union{Missing,Int,Float64}}(missing, v...) + end + + has_new_val = true + while has_new_val + has_new_val, transformed_variables = analyze_program( + DataTransformation(false, transformed_variables), model_def, data + ) + end + return transformed_variables +end + """ compile(model_def[, data, initializations]) @@ -231,14 +249,9 @@ function compile(model_def::Expr, data, inits; is_transformed=true) scalars, array_sizes = analyze_program( CollectVariables(model_def, data), model_def, data ) - has_new_val, transformed_variables = analyze_program( - DataTransformation(scalars, array_sizes), model_def, data + transformed_variables = compute_data_transformation( + scalars, array_sizes, model_def, data ) - while has_new_val - has_new_val, transformed_variables = analyze_program( - DataTransformation(false, transformed_variables), model_def, data - ) - end array_bitmap, transformed_variables = analyze_program( PostChecking(data, transformed_variables), model_def, data ) diff --git a/src/compiler_pass.jl b/src/compiler_pass.jl index 1cade0e33..6cbbadfba 100644 --- a/src/compiler_pass.jl +++ b/src/compiler_pass.jl @@ -451,7 +451,7 @@ struct PostChecking <: CompilerPass logical_or_stochastic::Dict # used to identify logical or stochastic assignment end -function PostChecking(data, transformed_variables::Dict) +function PostChecking(data::NamedTuple, transformed_variables::Dict) is_data = Dict() definition_bit_map = Dict() logical_or_stochastic = Dict() diff --git a/test/profile.jl b/test/profile.jl new file mode 100644 index 000000000..395a90b47 --- /dev/null +++ b/test/profile.jl @@ -0,0 +1,172 @@ +using BenchmarkTools + +using JuliaBUGS +using JuliaBUGS: + BUGSExamples, + analyze_program, + CollectVariables, + DataTransformation, + PostChecking, + NodeFunctions, + merge_with_coalescence, + compute_data_transformation + +suite = BenchmarkGroup() + +for name in keys(BUGSExamples.VOLUME_I) + @info "Adding benchmark for $name" + model_def = BUGSExamples.VOLUME_I[name].model_def + data = BUGSExamples.VOLUME_I[name].data + + scalars, array_sizes = analyze_program( + CollectVariables(model_def, data), model_def, data + ) + + transformed_variables = compute_data_transformation( + scalars, array_sizes, model_def, data + ) + + array_bitmap, transformed_variables = analyze_program( + PostChecking(data, transformed_variables), model_def, data + ) + merged_data = merge_with_coalescence(deepcopy(data), transformed_variables) + + vars, array_sizes, array_bitmap, node_args, node_functions, dependencies = analyze_program( + NodeFunctions(array_sizes, array_bitmap), model_def, merged_data + ) + + _suite = BenchmarkGroup() + + _suite["CollectVariables"] = @benchmarkable analyze_program( + CollectVariables($model_def, $data), $model_def, $data + ) + + _suite["DataTransformation"] = @benchmarkable compute_data_transformation( + $scalars, $array_sizes, $model_def, $data + ) + + _suite["NodeFunctions"] = @benchmarkable analyze_program( + NodeFunctions($array_sizes, $array_bitmap), $model_def, $merged_data + ) + + tune!(_suite) + suite[string(name)] = _suite +end + +results = run(suite; verbose=true) + +function create_result_dict(results) + result_dict = Dict{String,Dict{String,Dict{String,String}}}() + for (name, example_suite) in results + _d = Dict{String,Dict{String,String}}() + for k in ("CollectVariables", "DataTransformation", "NodeFunctions") + __d = Dict{String,String}() + med = median(example_suite[k]) + min = minimum(example_suite[k]) + max = maximum(example_suite[k]) + for (str, val) in zip(["median", "minimum", "maximum"], [med, min, max]) + __d[str] = BenchmarkTools.prettytime(val.time) + end + __d["memory"] = BenchmarkTools.prettymemory(memory(example_suite[k])) + _d[k] = __d + end + result_dict[name] = _d + end + return result_dict +end + +function print_pure_text_table(result_dict) + # Define the table header + println( + rpad("Example Name", 25), + "|", + lpad("Category", 20), + "|", + lpad("Median Time", 15), + "|", + lpad("Minimum Time", 15), + "|", + lpad("Maximum Time", 15), + "|", + lpad("Memory Usage", 15), + ) + println("-"^105) # Adjust the number based on the total length of the header + + # Iterate through each example and its benchmarks to populate the table rows + for (name, benchmarks) in result_dict + first_category = true + for (category, results) in benchmarks + if first_category + println( + rpad(name, 25), + "|", + lpad(category, 20), + "|", + lpad(results["median"], 15), + "|", + lpad(results["minimum"], 15), + "|", + lpad(results["maximum"], 15), + "|", + lpad(results["memory"], 15), + ) + first_category = false + else + println( + rpad("", 25), + "|", + lpad(category, 20), + "|", + lpad(results["median"], 15), + "|", + lpad(results["minimum"], 15), + "|", + lpad(results["maximum"], 15), + "|", + lpad(results["memory"], 15), + ) + end + end + println("-"^105) # Adjust the number based on the total length of the header + end +end + +function print_markdown_table_to_file(result_dict, filename=nothing) + output_target = filename !== nothing ? open(filename, "w") : stdout + + try + println( + output_target, + "| Example Name | Category | Median Time | Minimum Time | Maximum Time | Memory Usage |", + ) + println( + output_target, + "|--------------|----------|-------------|--------------|--------------|--------------|", + ) + + for (name, benchmarks) in result_dict + first_category = true + for (category, results) in benchmarks + if first_category + println( + output_target, + "| $(name) | $(category) | $(results["median"]) | $(results["minimum"]) | $(results["maximum"]) | $(results["memory"]) |", + ) + first_category = false + else + println( + output_target, + "| | $(category) | $(results["median"]) | $(results["minimum"]) | $(results["maximum"]) | $(results["memory"]) |", + ) + end + end + end + finally + if filename !== nothing + close(output_target) + end + end +end + +result_dict = create_result_dict(results) +print_pure_text_table(result_dict) diff --git a/test/runtests.jl b/test/runtests.jl index 3f70d84bc..cfee4a1af 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -46,91 +46,95 @@ using UnPack AbstractMCMC.setprogress!(false) -@testset "Function Unit Tests" begin - DocMeta.setdocmeta!( - JuliaBUGS, - :DocTestSetup, - :(using JuliaBUGS: +if get(ENV, "RUN_MODE", "test") == "profile" + include("profile.jl") +else + @testset "Function Unit Tests" begin + DocMeta.setdocmeta!( JuliaBUGS, - BUGSExamples, - @bugs, - Var, - create_array_var, - replace_constants_in_expr, - evaluate_and_track_dependencies, - find_variables_on_lhs, - evaluate, - merge_with_coalescence, - scalarize, - concretize_colon_indexing, - extract_variable_names_and_numdims, - extract_variables_in_bounds_and_lhs_indices, - simple_arithmetic_eval); - recursive=true, - ) - Documenter.doctest(JuliaBUGS; manual=false) -end + :DocTestSetup, + :(using JuliaBUGS: + JuliaBUGS, + BUGSExamples, + @bugs, + Var, + create_array_var, + replace_constants_in_expr, + evaluate_and_track_dependencies, + find_variables_on_lhs, + evaluate, + merge_with_coalescence, + scalarize, + concretize_colon_indexing, + extract_variable_names_and_numdims, + extract_variables_in_bounds_and_lhs_indices, + simple_arithmetic_eval); + recursive=true, + ) + Documenter.doctest(JuliaBUGS; manual=false) + end -include("bugs_primitives.jl") + include("bugs_primitives.jl") -@testset "Parser" begin - include("parser/bugs_macro.jl") - include("parser/bugs_parser.jl") - include("parser/winbugs_examples.jl") -end + @testset "Parser" begin + include("parser/bugs_macro.jl") + include("parser/bugs_parser.jl") + include("parser/winbugs_examples.jl") + end -include("compile.jl") + include("compile.jl") -@testset "Compile WinBUGS Vol I examples: $m" for m in [ - :blockers, - :bones, - :dogs, - :dyes, - :epil, - :equiv, - :kidney, - :leuk, - :leukfr, - :lsat, - :magnesium, - :mice, - :oxford, - :pumps, - :rats, - :salm, - :seeds, - :stacks, - :surgical_simple, - :surgical_realistic, -] - model_def = JuliaBUGS.BUGSExamples.VOLUME_I[m].model_def - data = JuliaBUGS.BUGSExamples.VOLUME_I[m].data - inits = JuliaBUGS.BUGSExamples.VOLUME_I[m].inits[1] - model = compile(model_def, data, inits) -end + @testset "Compile WinBUGS Vol I examples: $m" for m in [ + :blockers, + :bones, + :dogs, + :dyes, + :epil, + :equiv, + :kidney, + :leuk, + :leukfr, + :lsat, + :magnesium, + :mice, + :oxford, + :pumps, + :rats, + :salm, + :seeds, + :stacks, + :surgical_simple, + :surgical_realistic, + ] + model_def = JuliaBUGS.BUGSExamples.VOLUME_I[m].model_def + data = JuliaBUGS.BUGSExamples.VOLUME_I[m].data + inits = JuliaBUGS.BUGSExamples.VOLUME_I[m].inits[1] + model = compile(model_def, data, inits) + end -@testset "Utils" begin - include("utils.jl") -end + @testset "Utils" begin + include("utils.jl") + end -@testset "Log Probability Test" begin - include("run_logp_tests.jl") - @testset "Single stochastic variable test" begin - @testset "test for $s" for s in [:binomial, :gamma, :lkj, :dwish, :ddirich] - include("logp_tests/$s.jl") + @testset "Log Probability Test" begin + include("run_logp_tests.jl") + @testset "Single stochastic variable test" begin + @testset "test for $s" for s in [:binomial, :gamma, :lkj, :dwish, :ddirich] + include("logp_tests/$s.jl") + end end - end - @testset "BUGS examples" begin - @testset "test for $s" for s in [:blockers, :bones, :dogs, :rats] - include("logp_tests/$s.jl") + @testset "BUGS examples" begin + @testset "test for $s" for s in [:blockers, :bones, :dogs, :rats] + include("logp_tests/$s.jl") + end end end -end -@testset "Graph data structure" begin - include("graphs.jl") -end + @testset "Graph data structure" begin + include("graphs.jl") + end -include("gibbs.jl") + include("gibbs.jl") -include("ext/mcmchains.jl") + include("ext/mcmchains.jl") +end