Handle invalid types passed as max_nesting option

CHANGES.md

@@ -2,6 +2,8 @@
 
 ### Unreleased
 
+* Handle invalid types passed as `max_nesting` option.
+
 ### 2026-05-28 (2.19.7)
 
 * Fix some more edge cases with out of range floats.

ext/json/ext/generator/generator.c

@@ -1581,7 +1581,7 @@ static VALUE cState_max_nesting(VALUE self)
 
 static long long_config(VALUE num)
 {
-    return RTEST(num) ? FIX2LONG(num) : 0;
+    return RTEST(num) ? NUM2LONG(num) : 0;
 }
 
 // depth must never be negative; reject early with a clear error.

lib/json/truffle_ruby/generator.rb

@@ -307,6 +307,9 @@ def configure(opts)
           if !opts.key?(:max_nesting) # defaults to 100
             @max_nesting = 100
           elsif opts[:max_nesting]
+            unless opts[:max_nesting].is_a?(Integer)
+              raise TypeError, ":max_nesting must be an Integer, got: #{opts[:max_nesting].class}"
+            end
             @max_nesting = opts[:max_nesting]
           else
             @max_nesting = 0

test/json/json_generator_test.rb

@@ -601,6 +601,8 @@ def test_nesting
     assert_equal too_deep, ok
     ok = generate too_deep_ary, :max_nesting => 0
     assert_equal too_deep, ok
+
+    assert_raise(TypeError) { generate too_deep_ary, max_nesting: "garbage" }
   end
 
   def test_backslash