From bd7c92ec7e637defd0065d7a836bfa4c386e4a74 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki Date: Tue, 27 Feb 2024 17:52:43 +0900 Subject: [PATCH] 1.11: allow external abstract interpreter compilation As mentioned in #53478, the precompilation support for external abstract interpreters in v1.11 isn't perfect, and directly cherry-picking the refined test cases from #53478 into the v1.11 backport branch leads to a test failure (note that this particular problem has likely been fixed in the master branch, probably thanks to #53300). To address this, this commit does more than just cherry-pick the test case, and it also modifies the `CodeInstance(::AbstractInterpreter, ::InferenceResult)` constructor to allow precompilation for external abstract interpreters in v1.11. --- base/compiler/typeinfer.jl | 2 +- test/precompile.jl | 185 ++----------------------------------- test/precompile_absint1.jl | 73 +++++++++++++++ test/precompile_absint2.jl | 92 ++++++++++++++++++ test/precompile_utils.jl | 41 ++++++++ 5 files changed, 214 insertions(+), 179 deletions(-) create mode 100644 test/precompile_absint1.jl create mode 100644 test/precompile_absint2.jl create mode 100644 test/precompile_utils.jl diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index 747fd6e3ef705..36f121c06614d 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -321,7 +321,7 @@ function CodeInstance(interp::AbstractInterpreter, result::InferenceResult, t = @_gc_preserve_begin inferred_result relocatability = unsafe_load(unsafe_convert(Ptr{UInt8}, inferred_result), Core.sizeof(inferred_result)) @_gc_preserve_end t - elseif inferred_result === nothing + elseif !(inferred_result isa CodeInfo) relocatability = 0x1 end end diff --git a/test/precompile.jl b/test/precompile.jl index 527be616a5448..ed6e457354780 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1,11 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -original_depot_path = copy(Base.DEPOT_PATH) -original_load_path = copy(Base.LOAD_PATH) - using Test, Distributed, Random, Logging using REPL # doc lookup function +include("precompile_utils.jl") + Foo_module = :Foo4b3a94a1a081a8cb Foo2_module = :F2oo4b3a94a1a081a8cb FooBase_module = :FooBase4b3a94a1a081a8cb @@ -16,37 +15,6 @@ FooBase_module = :FooBase4b3a94a1a081a8cb end using .ConflictingBindings -function precompile_test_harness(@nospecialize(f), testset::String) - @testset "$testset" begin - precompile_test_harness(f, true) - end -end -function precompile_test_harness(@nospecialize(f), separate::Bool) - load_path = mktempdir() - load_cache_path = separate ? mktempdir() : load_path - try - pushfirst!(LOAD_PATH, load_path) - pushfirst!(DEPOT_PATH, load_cache_path) - f(load_path) - finally - try - rm(load_path, force=true, recursive=true) - catch err - @show err - end - if separate - try - rm(load_cache_path, force=true, recursive=true) - catch err - @show err - end - end - filter!((≠)(load_path), LOAD_PATH) - separate && filter!((≠)(load_cache_path), DEPOT_PATH) - end - nothing -end - # method root provenance rootid(m::Module) = Base.module_build_id(Base.parentmodule(m)) % UInt64 @@ -1712,146 +1680,10 @@ precompile_test_harness("issue #46296") do load_path (@eval (using CodeInstancePrecompile)) end -let newinterp_path = abspath("compiler/newinterp.jl") - precompile_test_harness("AbstractInterpreter caching") do load_path - write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule - basic_callee(x) = x - basic_caller(x) = basic_callee(x) - end) |> string) - - write(joinpath(load_path, "CustomAbstractInterpreterCaching.jl"), :(module CustomAbstractInterpreterCaching - import SimpleModule: basic_caller, basic_callee - - module Custom - include("$($newinterp_path)") - @newinterp PrecompileInterpreter - end - - Base.return_types((Float64,)) do x - basic_caller(x) - end - Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x - basic_caller(x) - end - Base.return_types((Vector{Float64},)) do x - sum(x) - end - Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x - sum(x) - end - end) |> string) - Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching")) - @eval let - using CustomAbstractInterpreterCaching - cache_owner = Core.Compiler.cache_owner( - CustomAbstractInterpreterCaching.Custom.PrecompileInterpreter()) - let m = only(methods(CustomAbstractInterpreterCaching.basic_callee)) - mi = only(Base.specializations(m)) - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === nothing - @test ci.max_world == typemax(UInt) - ci = ci.next - @test !isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - end - let m = only(methods(sum, (Vector{Float64},))) - found = false - for mi in Base.specializations(m) - if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - ci = ci.next - @test !isdefined(ci, :next) - @test ci.owner === nothing - @test ci.max_world == typemax(UInt) - found = true - break - end - end - @test found - end - end - - write(joinpath(load_path, "CustomAbstractInterpreterCaching2.jl"), :(module CustomAbstractInterpreterCaching2 - import SimpleModule: basic_caller, basic_callee - - module Custom - const CC = Core.Compiler - include("$($newinterp_path)") - @newinterp PrecompileInterpreter - struct CustomData - inferred - CustomData(@nospecialize inferred) = new(inferred) - end - function CC.transform_result_for_cache(interp::PrecompileInterpreter, - mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) - inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter, - mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) - return CustomData(inferred_result) - end - function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src), - @nospecialize(info::CC.CallInfo), stmt_flag::UInt32) - if src isa CustomData - src = src.inferred - end - return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any, - info::CC.CallInfo, stmt_flag::UInt32) - end - end - - Base.return_types((Float64,)) do x - basic_caller(x) - end - Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x - basic_caller(x) - end - Base.return_types((Vector{Float64},)) do x - sum(x) - end - Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x - sum(x) - end - end) |> string) - Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching2")) - @eval let - using CustomAbstractInterpreterCaching2 - cache_owner = Core.Compiler.cache_owner( - CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter()) - let m = only(methods(CustomAbstractInterpreterCaching2.basic_callee)) - mi = only(Base.specializations(m)) - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === nothing - @test ci.max_world == typemax(UInt) - ci = ci.next - @test !isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - end - let m = only(methods(sum, (Vector{Float64},))) - found = false - for mi = Base.specializations(m) - if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} - ci = mi.cache - @test isdefined(ci, :next) - @test ci.owner === cache_owner - @test ci.max_world == typemax(UInt) - ci = ci.next - @test !isdefined(ci, :next) - @test ci.owner === nothing - @test ci.max_world == typemax(UInt) - found = true - break - end - end - @test found - end - end - end +@testset "Precompile external abstract interpreter" begin + dir = @__DIR__ + @test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint1.jl`; dir); stdout, stderr)) + @test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint2.jl`; dir); stdout, stderr)) end precompile_test_harness("Recursive types") do load_path @@ -2093,7 +1925,4 @@ precompile_test_harness("Test flags") do load_path @test !Base.isprecompiled(id, ;flags=current_flags) end -empty!(Base.DEPOT_PATH) -append!(Base.DEPOT_PATH, original_depot_path) -empty!(Base.LOAD_PATH) -append!(Base.LOAD_PATH, original_load_path) +finish_precompile_test!() diff --git a/test/precompile_absint1.jl b/test/precompile_absint1.jl new file mode 100644 index 0000000000000..f47b26bce9d94 --- /dev/null +++ b/test/precompile_absint1.jl @@ -0,0 +1,73 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test + +include("precompile_utils.jl") + +precompile_test_harness() do load_path + write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule + basic_callee(x) = x + basic_caller(x) = basic_callee(x) + end) |> string) + + newinterp_path = abspath("compiler/newinterp.jl") + write(joinpath(load_path, "TestAbsIntPrecompile1.jl"), :(module TestAbsIntPrecompile1 + import SimpleModule: basic_caller, basic_callee + + module Custom + include("$($newinterp_path)") + @newinterp PrecompileInterpreter + end + + Base.return_types((Float64,)) do x + basic_caller(x) + end + Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x + basic_caller(x) + end + Base.return_types((Vector{Float64},)) do x + sum(x) + end + Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x + sum(x) + end + end) |> string) + Base.compilecache(Base.PkgId("TestAbsIntPrecompile1")) + + @eval let + using TestAbsIntPrecompile1 + cache_owner = Core.Compiler.cache_owner( + TestAbsIntPrecompile1.Custom.PrecompileInterpreter()) + let m = only(methods(TestAbsIntPrecompile1.basic_callee)) + mi = only(Base.specializations(m)) + ci = mi.cache + @test isdefined(ci, :next) + @test ci.owner === nothing + @test ci.max_world == typemax(UInt) + ci = ci.next + @test !isdefined(ci, :next) + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + end + let m = only(methods(sum, (Vector{Float64},))) + found = false + for mi in Base.specializations(m) + if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} + ci = mi.cache + @test isdefined(ci, :next) + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + ci = ci.next + @test !isdefined(ci, :next) + @test ci.owner === nothing + @test ci.max_world == typemax(UInt) + found = true + break + end + end + @test found + end + end +end + +finish_precompile_test!() diff --git a/test/precompile_absint2.jl b/test/precompile_absint2.jl new file mode 100644 index 0000000000000..2c98bb9c5135c --- /dev/null +++ b/test/precompile_absint2.jl @@ -0,0 +1,92 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test + +include("precompile_utils.jl") + +precompile_test_harness() do load_path + write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule + basic_callee(x) = x + basic_caller(x) = basic_callee(x) + end) |> string) + + newinterp_path = abspath("compiler/newinterp.jl") + write(joinpath(load_path, "TestAbsIntPrecompile2.jl"), :(module TestAbsIntPrecompile2 + import SimpleModule: basic_caller, basic_callee + + module Custom + const CC = Core.Compiler + include("$($newinterp_path)") + @newinterp PrecompileInterpreter + struct CustomData + inferred + CustomData(@nospecialize inferred) = new(inferred) + end + function CC.transform_result_for_cache(interp::PrecompileInterpreter, + mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter, + mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult) + return CustomData(inferred_result) + end + function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src), + @nospecialize(info::CC.CallInfo), stmt_flag::UInt32) + if src isa CustomData + src = src.inferred + end + return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any, + info::CC.CallInfo, stmt_flag::UInt32) + end + end + + Base.return_types((Float64,)) do x + basic_caller(x) + end + Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x + basic_caller(x) + end + Base.return_types((Vector{Float64},)) do x + sum(x) + end + Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x + sum(x) + end + end) |> string) + Base.compilecache(Base.PkgId("TestAbsIntPrecompile2")) + + @eval let + using TestAbsIntPrecompile2 + cache_owner = Core.Compiler.cache_owner( + TestAbsIntPrecompile2.Custom.PrecompileInterpreter()) + let m = only(methods(TestAbsIntPrecompile2.basic_callee)) + mi = only(Base.specializations(m)) + ci = mi.cache + @test isdefined(ci, :next) + @test ci.owner === nothing + @test ci.max_world == typemax(UInt) + ci = ci.next + @test !isdefined(ci, :next) + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + end + let m = only(methods(sum, (Vector{Float64},))) + found = false + for mi = Base.specializations(m) + if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}} + ci = mi.cache + @test isdefined(ci, :next) + @test ci.owner === cache_owner + @test ci.max_world == typemax(UInt) + ci = ci.next + @test !isdefined(ci, :next) + @test ci.owner === nothing + @test ci.max_world == typemax(UInt) + found = true + break + end + end + @test found + end + end +end + +finish_precompile_test!() diff --git a/test/precompile_utils.jl b/test/precompile_utils.jl new file mode 100644 index 0000000000000..55eba353f2ada --- /dev/null +++ b/test/precompile_utils.jl @@ -0,0 +1,41 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +function precompile_test_harness(@nospecialize(f), testset::String) + @testset "$testset" precompile_test_harness(f, true) +end +function precompile_test_harness(@nospecialize(f), separate::Bool=true) + load_path = mktempdir() + load_cache_path = separate ? mktempdir() : load_path + try + pushfirst!(LOAD_PATH, load_path) + pushfirst!(DEPOT_PATH, load_cache_path) + f(load_path) + finally + try + rm(load_path, force=true, recursive=true) + catch err + @show err + end + if separate + try + rm(load_cache_path, force=true, recursive=true) + catch err + @show err + end + end + filter!((≠)(load_path), LOAD_PATH) + separate && filter!((≠)(load_cache_path), DEPOT_PATH) + end + return nothing +end + +let original_depot_path = copy(Base.DEPOT_PATH) + original_load_path = copy(Base.LOAD_PATH) + + global function finish_precompile_test!() + empty!(Base.DEPOT_PATH) + append!(Base.DEPOT_PATH, original_depot_path) + empty!(Base.LOAD_PATH) + append!(Base.LOAD_PATH, original_load_path) + end +end