From 2eb1b5fb25b3240dda9f9710f8df2c69462f6e8f Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Mon, 4 Nov 2024 06:22:09 +0800 Subject: [PATCH] Basic MinGW-w64-based interpreter support (#15140) Implements a MinGW-based loader for `x86_64-windows-gnu` and enables the interpreter. --- .github/workflows/mingw-w64.yml | 14 +- spec/compiler/ffi/ffi_spec.cr | 10 +- spec/compiler/interpreter/lib_spec.cr | 39 ++-- spec/compiler/loader/spec_helper.cr | 3 + src/compiler/crystal/interpreter/context.cr | 8 +- src/compiler/crystal/loader.cr | 4 +- src/compiler/crystal/loader/mingw.cr | 195 ++++++++++++++++++++ src/crystal/system/win32/wmain.cr | 2 +- 8 files changed, 246 insertions(+), 29 deletions(-) create mode 100644 src/compiler/crystal/loader/mingw.cr diff --git a/.github/workflows/mingw-w64.yml b/.github/workflows/mingw-w64.yml index b32aed414f77..10841a325bf5 100644 --- a/.github/workflows/mingw-w64.yml +++ b/.github/workflows/mingw-w64.yml @@ -31,7 +31,7 @@ jobs: crystal: "1.14.0" - name: Cross-compile Crystal - run: make && make -B target=x86_64-windows-gnu release=1 + run: make && make -B target=x86_64-windows-gnu release=1 interpreter=1 - name: Upload crystal.obj uses: actions/upload-artifact@v4 @@ -63,6 +63,7 @@ jobs: mingw-w64-ucrt-x86_64-libiconv mingw-w64-ucrt-x86_64-zlib mingw-w64-ucrt-x86_64-llvm + mingw-w64-ucrt-x86_64-libffi - name: Download crystal.obj uses: actions/download-artifact@v4 @@ -80,7 +81,7 @@ jobs: run: | mkdir bin cc crystal.obj -o bin/crystal.exe \ - $(pkg-config bdw-gc libpcre2-8 iconv zlib --libs) \ + $(pkg-config bdw-gc libpcre2-8 iconv zlib libffi --libs) \ $(llvm-config --libs --system-libs --ldflags) \ -lDbgHelp -lole32 -lWS2_32 -Wl,--stack,0x800000 ldd bin/crystal.exe | grep -iv /c/windows/system32 | sed 's/.* => //; s/ (.*//' | xargs -t -i cp '{}' bin/ @@ -144,7 +145,14 @@ jobs: run: | export PATH="$(pwd)/crystal/bin:$PATH" export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" - make compiler_spec FLAGS=-Dwithout_ffi + make compiler_spec + + - name: Run interpreter specs + shell: msys2 {0} + run: | + export PATH="$(pwd)/crystal/bin:$PATH" + export CRYSTAL_SPEC_COMPILER_BIN="$(pwd)/crystal/bin/crystal.exe" + make interpreter_spec - name: Run primitives specs shell: msys2 {0} diff --git a/spec/compiler/ffi/ffi_spec.cr b/spec/compiler/ffi/ffi_spec.cr index ec644e45870d..a4718edb3501 100644 --- a/spec/compiler/ffi/ffi_spec.cr +++ b/spec/compiler/ffi/ffi_spec.cr @@ -27,7 +27,7 @@ private def dll_search_paths {% end %} end -{% if flag?(:unix) %} +{% if flag?(:unix) || (flag?(:win32) && flag?(:gnu)) %} class Crystal::Loader def self.new(search_paths : Array(String), *, dll_search_paths : Nil) new(search_paths) @@ -39,9 +39,17 @@ describe Crystal::FFI::CallInterface do before_all do FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) build_c_dynlib(compiler_datapath("ffi", "sum.c")) + + {% if flag?(:win32) && flag?(:gnu) %} + ENV["PATH"] = "#{SPEC_CRYSTAL_LOADER_LIB_PATH}#{Process::PATH_DELIMITER}#{ENV["PATH"]}" + {% end %} end after_all do + {% if flag?(:win32) && flag?(:gnu) %} + ENV["PATH"] = ENV["PATH"].delete_at(0, ENV["PATH"].index!(Process::PATH_DELIMITER) + 1) + {% end %} + FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) end diff --git a/spec/compiler/interpreter/lib_spec.cr b/spec/compiler/interpreter/lib_spec.cr index 2c1798645645..bbf6367ee6df 100644 --- a/spec/compiler/interpreter/lib_spec.cr +++ b/spec/compiler/interpreter/lib_spec.cr @@ -3,7 +3,7 @@ require "./spec_helper" require "../loader/spec_helper" private def ldflags - {% if flag?(:win32) %} + {% if flag?(:msvc) %} "/LIBPATH:#{SPEC_CRYSTAL_LOADER_LIB_PATH} sum.lib" {% else %} "-L#{SPEC_CRYSTAL_LOADER_LIB_PATH} -lsum" @@ -11,7 +11,7 @@ private def ldflags end private def ldflags_with_backtick - {% if flag?(:win32) %} + {% if flag?(:msvc) %} "/LIBPATH:#{SPEC_CRYSTAL_LOADER_LIB_PATH} `powershell.exe -C Write-Host -NoNewline sum.lib`" {% else %} "-L#{SPEC_CRYSTAL_LOADER_LIB_PATH} -l`echo sum`" @@ -19,12 +19,24 @@ private def ldflags_with_backtick end describe Crystal::Repl::Interpreter do - context "variadic calls" do - before_all do - FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) - build_c_dynlib(compiler_datapath("interpreter", "sum.c")) - end + before_all do + FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) + build_c_dynlib(compiler_datapath("interpreter", "sum.c")) + + {% if flag?(:win32) %} + ENV["PATH"] = "#{SPEC_CRYSTAL_LOADER_LIB_PATH}#{Process::PATH_DELIMITER}#{ENV["PATH"]}" + {% end %} + end + + after_all do + {% if flag?(:win32) %} + ENV["PATH"] = ENV["PATH"].delete_at(0, ENV["PATH"].index!(Process::PATH_DELIMITER) + 1) + {% end %} + + FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) + end + context "variadic calls" do it "promotes float" do interpret(<<-CRYSTAL).should eq 3.5 @[Link(ldflags: #{ldflags.inspect})] @@ -65,18 +77,9 @@ describe Crystal::Repl::Interpreter do LibSum.sum_int(2, E::ONE, F::FOUR) CRYSTAL end - - after_all do - FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) - end end context "command expansion" do - before_all do - FileUtils.mkdir_p(SPEC_CRYSTAL_LOADER_LIB_PATH) - build_c_dynlib(compiler_datapath("interpreter", "sum.c")) - end - it "expands ldflags" do interpret(<<-CRYSTAL).should eq 4 @[Link(ldflags: #{ldflags_with_backtick.inspect})] @@ -87,9 +90,5 @@ describe Crystal::Repl::Interpreter do LibSum.simple_sum_int(2, 2) CRYSTAL end - - after_all do - FileUtils.rm_rf(SPEC_CRYSTAL_LOADER_LIB_PATH) - end end end diff --git a/spec/compiler/loader/spec_helper.cr b/spec/compiler/loader/spec_helper.cr index 0db69dc19752..5b2a6454bfa1 100644 --- a/spec/compiler/loader/spec_helper.cr +++ b/spec/compiler/loader/spec_helper.cr @@ -8,6 +8,9 @@ def build_c_dynlib(c_filename, *, lib_name = nil, target_dir = SPEC_CRYSTAL_LOAD {% if flag?(:msvc) %} o_basename = o_filename.rchop(".lib") `#{ENV["CC"]? || "cl.exe"} /nologo /LD #{Process.quote(c_filename)} #{Process.quote("/Fo#{o_basename}")} #{Process.quote("/Fe#{o_basename}")}` + {% elsif flag?(:win32) && flag?(:gnu) %} + o_basename = o_filename.rchop(".a") + `#{ENV["CC"]? || "cc"} -shared -fvisibility=hidden #{Process.quote(c_filename)} -o #{Process.quote(o_basename + ".dll")} #{Process.quote("-Wl,--out-implib,#{o_basename}.a")}` {% else %} `#{ENV["CC"]? || "cc"} -shared -fvisibility=hidden #{Process.quote(c_filename)} -o #{Process.quote(o_filename)}` {% end %} diff --git a/src/compiler/crystal/interpreter/context.cr b/src/compiler/crystal/interpreter/context.cr index 50e36a3ff8b7..c2c1537e002d 100644 --- a/src/compiler/crystal/interpreter/context.cr +++ b/src/compiler/crystal/interpreter/context.cr @@ -393,14 +393,16 @@ class Crystal::Repl::Context getter(loader : Loader) { lib_flags = program.lib_flags # Execute and expand `subcommands`. - lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}` } + lib_flags = lib_flags.gsub(/`(.*?)`/) { `#{$1}`.chomp } args = Process.parse_arguments(lib_flags) # FIXME: Part 1: This is a workaround for initial integration of the interpreter: # The loader can't handle the static libgc.a usually shipped with crystal and loading as a shared library conflicts # with the compiler's own GC. - # (MSVC doesn't seem to have this issue) - args.delete("-lgc") + # (Windows doesn't seem to have this issue) + unless program.has_flag?("win32") && program.has_flag?("gnu") + args.delete("-lgc") + end # recreate the MSVC developer prompt environment, similar to how compiled # code does it in `Compiler#linker_command` diff --git a/src/compiler/crystal/loader.cr b/src/compiler/crystal/loader.cr index 5a147dad590f..84ff43d03d8e 100644 --- a/src/compiler/crystal/loader.cr +++ b/src/compiler/crystal/loader.cr @@ -1,4 +1,4 @@ -{% skip_file unless flag?(:unix) || flag?(:msvc) %} +{% skip_file unless flag?(:unix) || flag?(:win32) %} require "option_parser" # This loader component imitates the behaviour of `ld.so` for linking and loading @@ -105,4 +105,6 @@ end require "./loader/unix" {% elsif flag?(:msvc) %} require "./loader/msvc" +{% elsif flag?(:win32) && flag?(:gnu) %} + require "./loader/mingw" {% end %} diff --git a/src/compiler/crystal/loader/mingw.cr b/src/compiler/crystal/loader/mingw.cr new file mode 100644 index 000000000000..677f564cec16 --- /dev/null +++ b/src/compiler/crystal/loader/mingw.cr @@ -0,0 +1,195 @@ +{% skip_file unless flag?(:win32) && flag?(:gnu) %} + +require "crystal/system/win32/library_archive" + +# MinGW-based loader used on Windows. Assumes an MSYS2 shell. +# +# The core implementation is derived from the MSVC loader. Main deviations are: +# +# - `.parse` follows GNU `ld`'s style, rather than MSVC `link`'s; +# - `#library_filename` follows the usual naming of the MinGW linker: `.dll.a` +# for DLL import libraries, `.a` for other libraries; +# - `.default_search_paths` relies solely on `.cc_each_library_path`. +# +# TODO: The actual MinGW linker supports linking to DLLs directly, figure out +# how this is done. + +class Crystal::Loader + alias Handle = Void* + + def initialize(@search_paths : Array(String)) + end + + # Parses linker arguments in the style of `ld`. + # + # This is identical to the Unix loader. *dll_search_paths* has no effect. + def self.parse(args : Array(String), *, search_paths : Array(String) = default_search_paths, dll_search_paths : Array(String)? = nil) : self + libnames = [] of String + file_paths = [] of String + extra_search_paths = [] of String + + OptionParser.parse(args.dup) do |parser| + parser.on("-L DIRECTORY", "--library-path DIRECTORY", "Add DIRECTORY to library search path") do |directory| + extra_search_paths << directory + end + parser.on("-l LIBNAME", "--library LIBNAME", "Search for library LIBNAME") do |libname| + libnames << libname + end + parser.on("-static", "Do not link against shared libraries") do + raise LoadError.new "static libraries are not supported by Crystal's runtime loader" + end + parser.unknown_args do |args, after_dash| + file_paths.concat args + end + + parser.invalid_option do |arg| + unless arg.starts_with?("-Wl,") + raise LoadError.new "Not a recognized linker flag: #{arg}" + end + end + end + + search_paths = extra_search_paths + search_paths + + begin + loader = new(search_paths) + loader.load_all(libnames, file_paths) + loader + rescue exc : LoadError + exc.args = args + exc.search_paths = search_paths + raise exc + end + end + + def self.library_filename(libname : String) : String + "lib#{libname}.a" + end + + def find_symbol?(name : String) : Handle? + @handles.each do |handle| + address = LibC.GetProcAddress(handle, name.check_no_null_byte) + return address if address + end + end + + def load_file(path : String | ::Path) : Nil + load_file?(path) || raise LoadError.new "cannot load #{path}" + end + + def load_file?(path : String | ::Path) : Bool + if api_set?(path) + return load_dll?(path.to_s) + end + + return false unless File.file?(path) + + System::LibraryArchive.imported_dlls(path).all? do |dll| + load_dll?(dll) + end + end + + private def load_dll?(dll) + handle = open_library(dll) + return false unless handle + + @handles << handle + @loaded_libraries << (module_filename(handle) || dll) + true + end + + def load_library(libname : String) : Nil + load_library?(libname) || raise LoadError.new "cannot find #{Loader.library_filename(libname)}" + end + + def load_library?(libname : String) : Bool + if ::Path::SEPARATORS.any? { |separator| libname.includes?(separator) } + return load_file?(::Path[libname].expand) + end + + # attempt .dll.a before .a + # TODO: verify search order + @search_paths.each do |directory| + library_path = File.join(directory, Loader.library_filename(libname + ".dll")) + return true if load_file?(library_path) + + library_path = File.join(directory, Loader.library_filename(libname)) + return true if load_file?(library_path) + end + + false + end + + private def open_library(path : String) + LibC.LoadLibraryExW(System.to_wstr(path), nil, 0) + end + + def load_current_program_handle + if LibC.GetModuleHandleExW(0, nil, out hmodule) != 0 + @handles << hmodule + @loaded_libraries << (Process.executable_path || "current program handle") + end + end + + def close_all : Nil + @handles.each do |handle| + LibC.FreeLibrary(handle) + end + @handles.clear + end + + private def api_set?(dll) + dll.to_s.matches?(/^(?:api-|ext-)[a-zA-Z0-9-]*l\d+-\d+-\d+\.dll$/) + end + + private def module_filename(handle) + Crystal::System.retry_wstr_buffer do |buffer, small_buf| + len = LibC.GetModuleFileNameW(handle, buffer, buffer.size) + if 0 < len < buffer.size + break String.from_utf16(buffer[0, len]) + elsif small_buf && len == buffer.size + next 32767 # big enough. 32767 is the maximum total path length of UNC path. + else + break nil + end + end + end + + # Returns a list of directories used as the default search paths. + # + # Right now this depends on `cc` exclusively. + def self.default_search_paths : Array(String) + default_search_paths = [] of String + + cc_each_library_path do |path| + default_search_paths << path + end + + default_search_paths.uniq! + end + + # identical to the Unix loader + def self.cc_each_library_path(& : String ->) : Nil + search_dirs = begin + cc = + {% if Crystal.has_constant?("Compiler") %} + Crystal::Compiler::DEFAULT_LINKER + {% else %} + # this allows the loader to be required alone without the compiler + ENV["CC"]? || "cc" + {% end %} + + `#{cc} -print-search-dirs` + rescue IO::Error + return + end + + search_dirs.each_line do |line| + if libraries = line.lchop?("libraries: =") + libraries.split(Process::PATH_DELIMITER) do |path| + yield File.expand_path(path) + end + end + end + end +end diff --git a/src/crystal/system/win32/wmain.cr b/src/crystal/system/win32/wmain.cr index 3dd64f9c1b92..caad6748229f 100644 --- a/src/crystal/system/win32/wmain.cr +++ b/src/crystal/system/win32/wmain.cr @@ -7,7 +7,7 @@ require "c/stdlib" @[Link({{ flag?(:static) ? "libcmt" : "msvcrt" }})] {% if flag?(:msvc) %} @[Link(ldflags: "/ENTRY:wmainCRTStartup")] - {% elsif flag?(:gnu) %} + {% elsif flag?(:gnu) && !flag?(:interpreted) %} @[Link(ldflags: "-municode")] {% end %} {% end %}