Merge branch 'master' into strict_posix_util_14846

Makefile

@@ -24,18 +24,19 @@ all:
 CRYSTAL ?= crystal## which previous crystal compiler use
 LLVM_CONFIG ?=     ## llvm-config command path to use
 
-release ?=      ## Compile in release mode
-stats ?=        ## Enable statistics output
-progress ?=     ## Enable progress output
-threads ?=      ## Maximum number of threads to use
-debug ?=        ## Add symbolic debug info
-verbose ?=      ## Run specs in verbose mode
-junit_output ?= ## Path to output junit results
-static ?=       ## Enable static linking
-target ?=       ## Cross-compilation target
-interpreter ?=  ## Enable interpreter feature
-check ?=        ## Enable only check when running format
-order ?=random  ## Enable order for spec execution (values: "default" | "random" | seed number)
+release ?=        ## Compile in release mode
+stats ?=          ## Enable statistics output
+progress ?=       ## Enable progress output
+threads ?=        ## Maximum number of threads to use
+debug ?=          ## Add symbolic debug info
+verbose ?=        ## Run specs in verbose mode
+junit_output ?=   ## Path to output junit results
+static ?=         ## Enable static linking
+target ?=         ## Cross-compilation target
+interpreter ?=    ## Enable interpreter feature
+check ?=          ## Enable only check when running format
+order ?=random    ## Enable order for spec execution (values: "default" | "random" | seed number)
+deref_symlinks ?= ## Deference symbolic links for `make install`
 
 O := .build
 SOURCES := $(shell find src -name '*.cr')
@@ -167,7 +168,7 @@ install: $(O)/$(CRYSTAL_BIN) man/crystal.1.gz ## Install the compiler at DESTDIR
 	$(INSTALL) -m 0755 "$(O)/$(CRYSTAL_BIN)" "$(BINDIR)/$(CRYSTAL_BIN)"
 
 	$(INSTALL) -d -m 0755 $(DATADIR)
-	cp -R -P -p src "$(DATADIR)/src"
+	cp -R $(if $(deref_symlinks),-L --preserve=all,-P -p) src "$(DATADIR)/src"
 	rm -rf "$(DATADIR)/$(LLVM_EXT_OBJ)" # Don't install llvm_ext.o
 
 	$(INSTALL) -d -m 0755 "$(MANDIR)/man1/"

src/compiler/crystal/codegen/codegen.cr

@@ -274,7 +274,7 @@ module Crystal
                    @llvm_context : LLVM::Context = LLVM::Context.new)
       @abi = @program.target_machine.abi
       # LLVM::Context.register(@llvm_context, "main")
-      @llvm_mod = @llvm_context.new_module("main_module")
+      @llvm_mod = configure_module(@llvm_context.new_module("main_module"))
       @main_mod = @llvm_mod
       @main_llvm_context = @main_mod.context
       @llvm_typer = LLVMTyper.new(@program, @llvm_context)
@@ -345,8 +345,6 @@ module Crystal
       @unused_fun_defs = [] of FunDef
       @proc_counts = Hash(String, Int32).new(0)
 
-      @llvm_mod.data_layout = self.data_layout
-
       # We need to define __crystal_malloc and __crystal_realloc as soon as possible,
       # to avoid some memory being allocated with plain malloc.
       codegen_well_known_functions @node
@@ -367,6 +365,30 @@ module Crystal
 
     getter llvm_context
 
+    def configure_module(llvm_mod)
+      llvm_mod.data_layout = @program.target_machine.data_layout
+
+      # enable branch authentication instructions (BTI)
+      if @program.has_flag?("aarch64")
+        if @program.has_flag?("branch-protection=bti")
+          llvm_mod.add_flag(:override, "branch-target-enforcement", 1)
+        end
+      end
+
+      # enable control flow enforcement protection (CET): IBT and/or SHSTK
+      if @program.has_flag?("x86_64") || @program.has_flag?("i386")
+        if @program.has_flag?("cf-protection=branch") || @program.has_flag?("cf-protection=full")
+          llvm_mod.add_flag(:override, "cf-protection-branch", 1)
+        end
+
+        if @program.has_flag?("cf-protection=return") || @program.has_flag?("cf-protection=full")
+          llvm_mod.add_flag(:override, "cf-protection-return", 1)
+        end
+      end
+
+      llvm_mod
+    end
+
     def new_builder(llvm_context)
       wrap_builder(llvm_context.new_builder)
     end
@@ -419,10 +441,6 @@ module Crystal
       global.initializer = llvm_element_type.const_array(llvm_elements)
     end
 
-    def data_layout
-      @program.target_machine.data_layout
-    end
-
     class CodegenWellKnownFunctions < Visitor
       @codegen : CodeGenVisitor
 

src/compiler/crystal/codegen/debug.cr

@@ -42,17 +42,13 @@ module Crystal
 
       if @program.has_flag?("msvc")
         # Windows uses CodeView instead of DWARF
-        mod.add_flag(
-          LibLLVM::ModuleFlagBehavior::Warning,
-          "CodeView",
-          mod.context.int32.const_int(1)
-        )
+        mod.add_flag(LibLLVM::ModuleFlagBehavior::Warning, "CodeView", 1)
       end
 
       mod.add_flag(
         LibLLVM::ModuleFlagBehavior::Warning,
         "Debug Info Version",
-        mod.context.int32.const_int(LLVM::DEBUG_METADATA_VERSION)
+        LLVM::DEBUG_METADATA_VERSION
       )
     end
 

src/compiler/crystal/codegen/fun.cr

@@ -626,8 +626,7 @@ class Crystal::CodeGenVisitor
         # LLVM::Context.register(llvm_context, type_name)
 
         llvm_typer = LLVMTyper.new(@program, llvm_context)
-        llvm_mod = llvm_context.new_module(type_name)
-        llvm_mod.data_layout = self.data_layout
+        llvm_mod = configure_module(llvm_context.new_module(type_name))
         llvm_builder = new_builder(llvm_context)
 
         define_symbol_table llvm_mod, llvm_typer

src/compiler/crystal/compiler.cr

@@ -507,12 +507,6 @@ module Crystal
           link_flags += " -L/usr/local/lib"
         end
 
-        if program.has_flag?("openbsd")
-          # OpenBSD requires Indirect Branch Tracking by default, but we're not
-          # compatible (yet), so we disable it for now:
-          link_flags += " -Wl,-znobtcfi"
-        end
-
         {DEFAULT_LINKER, %(#{DEFAULT_LINKER} "${@}" -o #{Process.quote_posix(output_filename)} #{link_flags} #{program.lib_flags(@cross_compile)}), object_names}
       end
     end

src/compiler/crystal/semantic/flags.cr

@@ -49,7 +49,18 @@ class Crystal::Program
       flags.add "freebsd#{target.freebsd_version}"
     end
     flags.add "netbsd" if target.netbsd?
-    flags.add "openbsd" if target.openbsd?
+
+    if target.openbsd?
+      flags.add "openbsd"
+
+      case target.architecture
+      when "aarch64"
+        flags.add "branch-protection=bti" unless flags.any?(&.starts_with?("branch-protection="))
+      when "x86_64", "i386"
+        flags.add "cf-protection=branch" unless flags.any?(&.starts_with?("cf-protection="))
+      end
+    end
+
     flags.add "dragonfly" if target.dragonfly?
     flags.add "solaris" if target.solaris?
     flags.add "android" if target.android?

src/crystal/once.cr

@@ -11,9 +11,6 @@
 # :nodoc:
 class Crystal::OnceState
   @rec = [] of Bool*
-  {% if flag?(:preview_mt) %}
-    @mutex = Mutex.new(:reentrant)
-  {% end %}
 
   def once(flag : Bool*, initializer : Void*)
     unless flag.value
@@ -29,7 +26,13 @@ class Crystal::OnceState
     end
   end
 
-  {% if flag?(:preview_mt) %}
+  # on Win32, `Crystal::System::FileDescriptor#@@reader_thread` spawns a new
+  # thread even without the `preview_mt` flag, and the thread can also reference
+  # Crystal constants, leading to race conditions, so we always enable the mutex
+  # TODO: can this be improved?
+  {% if flag?(:preview_mt) || flag?(:win32) %}
+    @mutex = Mutex.new(:reentrant)
+
     def once(flag : Bool*, initializer : Void*)
       unless flag.value
         @mutex.synchronize do

src/llvm/lib_llvm/core.cr

@@ -5,7 +5,12 @@ lib LibLLVM
   # counterparts (e.g. `LLVMModuleFlagBehavior` v.s. `LLVM::Module::ModFlagBehavior`)
 
   enum ModuleFlagBehavior
-    Warning = 1
+    Error        = 0
+    Warning      = 1
+    Require      = 2
+    Override     = 3
+    Append       = 4
+    AppendUnique = 5
   end
 
   alias AttributeIndex = UInt

src/llvm/module.cr

@@ -45,6 +45,10 @@ class LLVM::Module
     GlobalCollection.new(self)
   end
 
+  def add_flag(module_flag : LibLLVM::ModuleFlagBehavior, key : String, val : Int32)
+    add_flag(module_flag, key, @context.int32.const_int(val))
+  end
+
   def add_flag(module_flag : LibLLVM::ModuleFlagBehavior, key : String, val : Value)
     LibLLVM.add_module_flag(
       self,

src/string.cr

@@ -3473,8 +3473,8 @@ class String
   # ```
   # "Hello, World".rindex('o')    # => 8
   # "Hello, World".rindex('Z')    # => nil
-  # "Hello, World".rindex("o", 5) # => 4
-  # "Hello, World".rindex("W", 2) # => nil
+  # "Hello, World".rindex('o', 5) # => 4
+  # "Hello, World".rindex('W', 2) # => nil
   # ```
   def rindex(search : Char, offset = size - 1)
     # If it's ASCII we can delegate to slice
@@ -3519,7 +3519,16 @@ class String
     end
   end
 
-  # :ditto:
+  # Returns the index of the _last_ appearance of *search* in the string,
+  # If *offset* is present, it defines the position to _end_ the search
+  # (characters beyond this point are ignored).
+  #
+  # ```
+  # "Hello, World".rindex("orld")    # => 8
+  # "Hello, World".rindex("snorlax") # => nil
+  # "Hello, World".rindex("o", 5)    # => 4
+  # "Hello, World".rindex("W", 2)    # => nil
+  # ```
   def rindex(search : String, offset = size - search.size) : Int32?
     offset += size if offset < 0
     return if offset < 0
@@ -3572,7 +3581,16 @@ class String
     end
   end
 
-  # :ditto:
+  # Returns the index of the _last_ appearance of *search* in the string,
+  # If *offset* is present, it defines the position to _end_ the search
+  # (characters beyond this point are ignored).
+  #
+  # ```
+  # "Hello, World".rindex(/world/i) # => 7
+  # "Hello, World".rindex(/world/)  # => nil
+  # "Hello, World".rindex(/o/, 5)   # => 4
+  # "Hello, World".rindex(/W/, 2)   # => nil
+  # ```
   def rindex(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32?
     offset += size if offset < 0
     return nil unless 0 <= offset <= size
@@ -3586,21 +3604,49 @@ class String
     match_result.try &.begin
   end
 
-  # :ditto:
-  #
+  # Returns the index of the _last_ appearance of *search* in the string,
+  # If *offset* is present, it defines the position to _end_ the search
+  # (characters beyond this point are ignored).
   # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`.
-  def rindex!(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32
-    rindex(search, offset, options: options) || raise Enumerable::NotFoundError.new
+  #
+  # ```
+  # "Hello, World".rindex!('o')    # => 8
+  # "Hello, World".rindex!('Z')    # raises Enumerable::NotFoundError
+  # "Hello, World".rindex!('o', 5) # => 4
+  # "Hello, World".rindex!('W', 2) # raises Enumerable::NotFoundError
+  # ```
+  def rindex!(search : Char, offset = size - 1) : Int32
+    rindex(search, offset) || raise Enumerable::NotFoundError.new
   end
 
-  # :ditto:
+  # Returns the index of the _last_ appearance of *search* in the string,
+  # If *offset* is present, it defines the position to _end_ the search
+  # (characters beyond this point are ignored).
+  # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`.
+  #
+  # ```
+  # "Hello, World".rindex!("orld")    # => 8
+  # "Hello, World".rindex!("snorlax") # raises Enumerable::NotFoundError
+  # "Hello, World".rindex!("o", 5)    # => 4
+  # "Hello, World".rindex!("W", 2)    # raises Enumerable::NotFoundError
+  # ```
   def rindex!(search : String, offset = size - search.size) : Int32
     rindex(search, offset) || raise Enumerable::NotFoundError.new
   end
 
-  # :ditto:
-  def rindex!(search : Char, offset = size - 1) : Int32
-    rindex(search, offset) || raise Enumerable::NotFoundError.new
+  # Returns the index of the _last_ appearance of *search* in the string,
+  # If *offset* is present, it defines the position to _end_ the search
+  # (characters beyond this point are ignored).
+  # Raises `Enumerable::NotFoundError` if *search* does not occur in `self`.
+  #
+  # ```
+  # "Hello, World".rindex!(/world/i) # => 7
+  # "Hello, World".rindex!(/world/)  # raises Enumerable::NotFoundError
+  # "Hello, World".rindex!(/o/, 5)   # => 4
+  # "Hello, World".rindex!(/W/, 2)   # raises Enumerable::NotFoundError
+  # ```
+  def rindex!(search : Regex, offset = size, *, options : Regex::MatchOptions = Regex::MatchOptions::None) : Int32
+    rindex(search, offset, options: options) || raise Enumerable::NotFoundError.new
   end
 
   # Searches separator or pattern (`Regex`) in the string, and returns