Fix a bug that Prism.parse_stream may write too much data to internal buffer

ext/prism/extension.c

@@ -1071,9 +1071,21 @@ parse_stream_eof(void *stream) {
  */
 static char *
 parse_stream_fgets(char *string, int size, void *stream) {
-    RUBY_ASSERT(size > 0);
+    /* We should use rb_enc_mbmaxlen() of stream.external_encoding but
+     * we don't want to do it here for performance. We can detect the
+     * max character length in all available encodings by the
+     * following command line in the Ruby source directory:
+     *
+     *   grep -Eh "max (enc|byte) length" enc/*.c |
+     *     grep -Eo '[0-9]+' |
+     *     sort |
+     *     tail -n1 # => 6
+     */
+#define MAX_ENC_LEN 6
+    RUBY_ASSERT(size > MAX_ENC_LEN);
 
-    VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - 1));
+    VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - MAX_ENC_LEN - 1));
+#undef MAX_ENC_LEN
     if (NIL_P(line)) {
         return NULL;
     }

lib/prism/ffi.rb

@@ -298,7 +298,18 @@ def parse_stream(stream, **options)
         callback = -> (string, size, _) {
           raise "Expected size to be >= 0, got: #{size}" if size <= 0
 
-          if !(line = stream.gets(size - 1)).nil?
+          # gets(size - 1) may return size or more data when the last
+          # character is a multi-byte character. So we use gets(size -
+          # max_character_length - 1) here to avoid the
+          # case. max_character_length can be detected by the
+          # following command line in the Ruby source directory:
+          #
+          #   grep -Eh "max (enc|byte) length" enc/*.c |
+          #     grep -Eo '[0-9]+' |
+          #     sort |
+          #     tail -n1 # => 6
+          max_character_length = 6
+          if !(line = stream.gets(size - max_character_length - 1)).nil?
             source << line
             string.write_string("#{line}\x00", line.bytesize + 1)
           end

test/prism/api/parse_stream_test.rb

@@ -114,5 +114,19 @@ def test_nul_bytes
       assert result.success?
       assert_equal 3, result.value.statements.body.length
     end
+
+    def test_long_line
+      # The important point of this test is that the character at 4095
+      # is a multi-byte character (U+3042 HIRAGANA LETTER A) in this
+      # case. gets(4095) may return 4095 or more bytes if the last
+      # character is a multi-byte character. For example,
+      # StringIO.new("\u3042").gets(1).bytesize is 3 not 1.
+      long_string = "\"" + ("a" * 4093) + "\u3042" + "\""
+      io = StringIO.new(long_string)
+      result = Prism.parse_stream(io)
+
+      assert result.success?
+      assert_equal long_string[1..-2], result.value.statements.body[0].content
+    end
   end
 end