Do not over-commit fiber stacks on Windows (#15037)

Whenever a new fiber is spawned on Windows, currently Crystal allocates a fully committed 8 MB range of virtual memory. This commit charge stays until the stack becomes unused and reaped, even when most of the stack goes unused. Thus it is "only" possible to spawn several thousand fibers concurrently before the system runs out of virtual memory, depending on the total size of RAM and page files.

With this PR, for every fresh fiber stack, only the guard pages plus one extra initial page are committed. Spawning 100,000 idle fibers now consumes just around 7.4 GB of virtual memory, instead of 800 GB. Committed pages are also reset after a stack is returned to a pool and before it is retrieved again; this should be reasonably first, as decommitting pages doesn't alter the page contents.

Note that the guard pages reside immediately above the normal committed pages, not at the top of the whole reserved range. This is required for proper stack overflow detection.

spec/std/process_spec.cr

@@ -172,6 +172,14 @@ describe Process do
       error.to_s.should eq("hello#{newline}")
     end
 
+    it "sends long output and error to IO" do
+      output = IO::Memory.new
+      error = IO::Memory.new
+      Process.run(*shell_command("echo #{"." * 8000}"), output: output, error: error)
+      output.to_s.should eq("." * 8000 + newline)
+      error.to_s.should be_empty
+    end
+
     it "controls process in block" do
       value = Process.run(*stdin_to_stdout_command, error: :inherit) do |proc|
         proc.input.puts "hello"

src/crystal/system/fiber.cr

@@ -1,12 +1,12 @@
 module Crystal::System::Fiber
   # Allocates memory for a stack.
-  # def self.allocate_stack(stack_size : Int) : Void*
+  # def self.allocate_stack(stack_size : Int, protect : Bool) : Void*
+
+  # Prepares an existing, unused stack for use again.
+  # def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
 
   # Frees memory of a stack.
   # def self.free_stack(stack : Void*, stack_size : Int) : Nil
-
-  # Determines location of the top of the main process fiber's stack.
-  # def self.main_fiber_stack(stack_bottom : Void*) : Void*
 end
 
 {% if flag?(:wasi) %}

src/crystal/system/unix/fiber.cr

@@ -21,6 +21,9 @@ module Crystal::System::Fiber
     pointer
   end
 
+  def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
+  end
+
   def self.free_stack(stack : Void*, stack_size) : Nil
     LibC.munmap(stack, stack_size)
   end

src/crystal/system/wasi/fiber.cr

@@ -3,6 +3,9 @@ module Crystal::System::Fiber
     LibC.malloc(stack_size)
   end
 
+  def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
+  end
+
   def self.free_stack(stack : Void*, stack_size) : Nil
     LibC.free(stack)
   end

src/crystal/system/win32/fiber.cr

@@ -7,28 +7,63 @@ module Crystal::System::Fiber
   # overflow
   RESERVED_STACK_SIZE = LibC::DWORD.new(0x10000)
 
-  # the reserved stack size, plus the size of a single page
-  @@total_reserved_size : LibC::DWORD = begin
-    LibC.GetNativeSystemInfo(out system_info)
-    system_info.dwPageSize + RESERVED_STACK_SIZE
-  end
-
   def self.allocate_stack(stack_size, protect) : Void*
-    unless memory_pointer = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_COMMIT | LibC::MEM_RESERVE, LibC::PAGE_READWRITE)
-      raise RuntimeError.from_winerror("VirtualAlloc")
+    if stack_top = LibC.VirtualAlloc(nil, stack_size, LibC::MEM_RESERVE, LibC::PAGE_READWRITE)
+      if protect
+        if commit_and_guard(stack_top, stack_size)
+          return stack_top
+        end
+      else
+        # for the interpreter, the stack is just ordinary memory so the entire
+        # range is committed
+        if LibC.VirtualAlloc(stack_top, stack_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE)
+          return stack_top
+        end
+      end
+
+      # failure
+      LibC.VirtualFree(stack_top, 0, LibC::MEM_RELEASE)
     end
 
-    # Detects stack overflows by guarding the top of the stack, similar to
-    # `LibC.mprotect`. Windows will fail to allocate a new guard page for these
-    # fiber stacks and trigger a stack overflow exception
+    raise RuntimeError.from_winerror("VirtualAlloc")
+  end
+
+  def self.reset_stack(stack : Void*, stack_size : Int, protect : Bool) : Nil
     if protect
-      if LibC.VirtualProtect(memory_pointer, @@total_reserved_size, LibC::PAGE_READWRITE | LibC::PAGE_GUARD, out _) == 0
-        LibC.VirtualFree(memory_pointer, 0, LibC::MEM_RELEASE)
-        raise RuntimeError.from_winerror("VirtualProtect")
+      if LibC.VirtualFree(stack, 0, LibC::MEM_DECOMMIT) == 0
+        raise RuntimeError.from_winerror("VirtualFree")
+      end
+      unless commit_and_guard(stack, stack_size)
+        raise RuntimeError.from_winerror("VirtualAlloc")
       end
     end
+  end
+
+  # Commits the bottommost page and sets up the guard pages above it, in the
+  # same manner as each thread's main stack. When the stack hits a guard page
+  # for the first time, a page fault is generated, the page's guard status is
+  # reset, and Windows checks if a reserved page is available above. On success,
+  # a new guard page is committed, and on failure, a stack overflow exception is
+  # triggered after the `RESERVED_STACK_SIZE` portion is made available.
+  private def self.commit_and_guard(stack_top, stack_size)
+    stack_bottom = stack_top + stack_size
+
+    LibC.GetNativeSystemInfo(out system_info)
+    stack_commit_size = system_info.dwPageSize
+    stack_commit_top = stack_bottom - stack_commit_size
+    unless LibC.VirtualAlloc(stack_commit_top, stack_commit_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE)
+      return false
+    end
+
+    # the reserved stack size, plus a final guard page for when the stack
+    # overflow handler itself overflows the stack
+    stack_guard_size = system_info.dwPageSize + RESERVED_STACK_SIZE
+    stack_guard_top = stack_commit_top - stack_guard_size
+    unless LibC.VirtualAlloc(stack_guard_top, stack_guard_size, LibC::MEM_COMMIT, LibC::PAGE_READWRITE | LibC::PAGE_GUARD)
+      return false
+    end
 
-    memory_pointer
+    true
   end
 
   def self.free_stack(stack : Void*, stack_size) : Nil

src/fiber/context/x86_64-microsoft.cr

@@ -10,13 +10,17 @@ class Fiber
     @context.stack_top = (stack_ptr - (12 + 10*2)).as(Void*)
     @context.resumable = 1
 
+    # actual stack top, not including guard pages and reserved pages
+    LibC.GetNativeSystemInfo(out system_info)
+    stack_top = @stack_bottom - system_info.dwPageSize
+
     stack_ptr[0] = fiber_main.pointer # %rbx: Initial `resume` will `ret` to this address
     stack_ptr[-1] = self.as(Void*)    # %rcx: puts `self` as first argument for `fiber_main`
 
     # The following three values are stored in the Thread Information Block (NT_TIB)
     # and are used by Windows to track the current stack limits
     stack_ptr[-2] = @stack        # %gs:0x1478: Win32 DeallocationStack
-    stack_ptr[-3] = @stack        # %gs:0x10: Stack Limit
+    stack_ptr[-3] = stack_top     # %gs:0x10: Stack Limit
     stack_ptr[-4] = @stack_bottom # %gs:0x08: Stack Base
   end
 

src/fiber/stack_pool.cr

@@ -42,7 +42,11 @@ class Fiber
 
     # Removes a stack from the bottom of the pool, or allocates a new one.
     def checkout : {Void*, Void*}
-      stack = @deque.pop? || Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect)
+      if stack = @deque.pop?
+        Crystal::System::Fiber.reset_stack(stack, STACK_SIZE, @protect)
+      else
+        stack = Crystal::System::Fiber.allocate_stack(STACK_SIZE, @protect)
+      end
       {stack, stack + STACK_SIZE}
     end