Detect and warn on stack overflow (#6928)

* bindings for sys/resource.h

* Warn on potential stack overflow in SIGSEGV handler

spec/std/kernel_spec.cr

@@ -220,3 +220,50 @@ describe "at_exit" do
                            OUTPUT
   end
 end
+
+describe "seg fault" do
+  it "reports SIGSEGV" do
+    status, _, error = build_and_run <<-'CODE'
+      puts Pointer(Int64).new(0x00FEDCBA).value
+    CODE
+
+    status.success?.should be_false
+    error.should contain("Invalid memory access")
+    error.should_not contain("Stack overflow")
+  end
+
+  it "detects stack overflow on the main stack" do
+    # This spec can take some time under FreeBSD where
+    # the default stack size is 0.5G.  Setting a
+    # smaller stack size with `ulimit -s 8192`
+    # will address this.
+    status, _, error = build_and_run <<-'CODE'
+      def foo
+        y = StaticArray(Int8,512).new(0)
+        foo
+      end
+      foo
+    CODE
+
+    status.success?.should be_false
+    error.should contain("Stack overflow")
+  end
+
+  it "detects stack overflow on a fiber stack" do
+    status, _, error = build_and_run <<-'CODE'
+      def foo
+        y = StaticArray(Int8,512).new(0)
+        foo
+      end
+
+      spawn do
+        foo
+      end
+
+      sleep 60.seconds
+    CODE
+
+    status.success?.should be_false
+    error.should contain("Stack overflow")
+  end
+end

src/fiber.cr

@@ -1,4 +1,7 @@
 require "c/sys/mman"
+{% unless flag?(:win32) %}
+require "c/sys/resource"
+{% end %}
 require "thread/linked_list"
 
 # Load the arch-specific methods to create a context and to swap from one
@@ -86,11 +89,26 @@ class Fiber
   # :nodoc:
   def initialize
     @proc = Proc(Void).new { }
-    @stack = Pointer(Void).null
     @stack_top = _fiber_get_stack_top
     @stack_bottom = GC.stack_bottom
     @name = "main"
 
+    # Determine location of the top of the stack.
+    # The technique here works only for the main stack on a POSIX platform.
+    # TODO: implement for Windows with GetCurrentThreadStackLimits
+    # TODO: implement for pthreads using
+    #    linux-glibc/musl: pthread_getattr_np
+    #              macosx: pthread_get_stackaddr_np, pthread_get_stacksize_np
+    #             freebsd: pthread_attr_get_np
+    #             openbsd: pthread_stackseg_np
+    @stack = Pointer(Void).null
+    {% unless flag?(:win32) %}
+      if LibC.getrlimit(LibC::RLIMIT_STACK, out rlim) == 0
+        stack_size = rlim.rlim_cur
+        @stack = Pointer(Void).new(@stack_bottom.address - stack_size)
+      end
+    {% end %}
+
     @@fibers.push(self)
   end
 

src/lib_c/aarch64-linux-gnu/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/aarch64-linux-musl/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULongLong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/amd64-unknown-openbsd/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULongLong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/arm-linux-gnueabihf/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/i686-linux-gnu/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/i686-linux-musl/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULongLong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/x86_64-linux-gnu/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/x86_64-linux-musl/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULongLong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/x86_64-macosx-darwin/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = ULongLong
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/lib_c/x86_64-portbld-freebsd/c/sys/resource.cr

@@ -0,0 +1,12 @@
+lib LibC
+  alias RlimT = Long
+
+  struct Rlimit
+    rlim_cur : RlimT
+    rlim_max : RlimT
+  end
+
+  fun getrlimit(Int, Rlimit*) : Int
+
+  RLIMIT_STACK = 3
+end

src/signal.cr

@@ -320,7 +320,18 @@ end
 # :nodoc:
 fun __crystal_sigfault_handler(sig : LibC::Int, addr : Void*)
   # Capture fault signals (SEGV, BUS) and finish the process printing a backtrace first
-  LibC.dprintf 2, "Invalid memory access (signal %d) at address 0x%lx\n", sig, addr
+
+  # Determine if the SEGV was inside or 'near' the top of the stack
+  # to check for potential stack overflow. 'Near' is a small
+  # amount larger than a typical stack frame, 4096 bytes here.
+  stack_top = Pointer(Void).new(Fiber.current.@stack.address - 4096)
+
+  if stack_top <= addr < Fiber.current.@stack_bottom
+    LibC.dprintf 2, "Stack overflow (e.g., infinite or very deep recursion)\n"
+  else
+    LibC.dprintf 2, "Invalid memory access (signal %d) at address 0x%lx\n", sig, addr
+  end
+
   CallStack.print_backtrace
   LibC._exit(sig)
 end