Backport regexp repeat and null-check ID overflow guards

Backport ruby/ruby@493d26d for CVE-2026-110264. Reject excessive range-repeat and null-check counts before their IDs wrap, retaining the existing downstream fixes. Cover the accepted 32767-count boundary and rejection at 32768 and 100000 with diagnostic assertions.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: srt32

include/ruby/onigmo.h

@@ -687,6 +687,8 @@ ONIG_EXTERN const OnigSyntaxType*   OnigDefaultSyntax;
 #define ONIGERR_NEVER_ENDING_RECURSION                       -221
 #define ONIGERR_GROUP_NUMBER_OVER_FOR_CAPTURE_HISTORY        -222
 #define ONIGERR_INVALID_CHAR_PROPERTY_NAME                   -223
+#define ONIGERR_TOO_MANY_RANGE_REPEAT                        -224
+#define ONIGERR_TOO_MANY_NULL_CHECK                          -225
 #define ONIGERR_TOO_BIG_COMPILED_PROGRAM                     -226
 #define ONIGERR_INVALID_CODE_POINT_VALUE                     -400
 #define ONIGERR_INVALID_WIDE_CHAR_VALUE                      -400

regcomp.c

@@ -398,6 +398,7 @@ compile_tree_empty_check(Node* node, regex_t* reg, int empty_info)
     r = add_mem_num(reg, reg->num_null_check); /* NULL CHECK ID */
     if (r) return r;
     reg->num_null_check++;
+    if ((MemNumType)reg->num_null_check <= 0) return ONIGERR_TOO_MANY_NULL_CHECK;
   }
 
   r = compile_tree(node, reg);
@@ -707,6 +708,7 @@ compile_range_repeat_node(QtfrNode* qn, int target_len, int empty_info,
   if (r) return r;
   r = add_mem_num(reg, num_repeat); /* OP_REPEAT ID */
   reg->num_repeat++;
+  if ((MemNumType)reg->num_repeat <= 0) return ONIGERR_TOO_MANY_RANGE_REPEAT;
   if (r) return r;
   r = add_rel_addr(reg, target_len + SIZE_OP_REPEAT_INC);
   if (r) return r;

regerror.c

@@ -165,6 +165,10 @@ onig_error_code_to_format(OnigPosition code)
     p = "group number is too big for capture history"; break;
   case ONIGERR_INVALID_CHAR_PROPERTY_NAME:
     p = "invalid character property name {%n}"; break;
+  case ONIGERR_TOO_MANY_RANGE_REPEAT:
+    p = "too many range repeat"; break;
+  case ONIGERR_TOO_MANY_NULL_CHECK:
+    p = "too many null check"; break;
   case ONIGERR_TOO_BIG_COMPILED_PROGRAM:
     p = "too big compiled program"; break;
   case ONIGERR_TOO_MANY_CAPTURE_GROUPS:

test/ruby/test_regexp.rb

@@ -2303,4 +2303,24 @@ def test_nested_repeat_expansion_overflow
       end
     end;
   end
+
+  def test_too_many_range_repeat
+    source = '(?:foobar){0,100}'
+    assert_nothing_raised { Regexp.new(source * 32767) }
+    [32768, 100000].each do |count|
+      assert_raise_with_message(RegexpError, /\Atoo many range repeat:/) do
+        Regexp.new(source * count)
+      end
+    end
+  end
+
+  def test_too_many_null_check
+    source = '(?:(?:foo)?|(?:bar)?)*'
+    assert_nothing_raised { Regexp.new(source * 32767) }
+    [32768, 100000].each do |count|
+      assert_raise_with_message(RegexpError, /\Atoo many null check:/) do
+        Regexp.new(source * count)
+      end
+    end
+  end
 end