diff --git a/ext/prism/extension.c b/ext/prism/extension.c index 27df8dac50..c2658cefbd 100644 --- a/ext/prism/extension.c +++ b/ext/prism/extension.c @@ -1071,9 +1071,21 @@ parse_stream_eof(void *stream) { */ static char * parse_stream_fgets(char *string, int size, void *stream) { - RUBY_ASSERT(size > 0); + /* We should use rb_enc_mbmaxlen() of stream.external_encoding but + * we don't want to do it here for performance. We can detect the + * max character length in all available encodings by the + * following command line in the Ruby source directory: + * + * grep -Eh "max (enc|byte) length" enc/*.c | + * grep -Eo '[0-9]+' | + * sort | + * tail -n1 # => 6 + */ +#define MAX_ENC_LEN 6 + RUBY_ASSERT(size > MAX_ENC_LEN); - VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - 1)); + VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - MAX_ENC_LEN - 1)); +#undef MAX_ENC_LEN if (NIL_P(line)) { return NULL; } diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index 6b9bde51ea..e7ca6a2251 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -298,7 +298,18 @@ def parse_stream(stream, **options) callback = -> (string, size, _) { raise "Expected size to be >= 0, got: #{size}" if size <= 0 - if !(line = stream.gets(size - 1)).nil? + # gets(size - 1) may return size or more data when the last + # character is a multi-byte character. So we use gets(size - + # max_character_length - 1) here to avoid the + # case. max_character_length can be detected by the + # following command line in the Ruby source directory: + # + # grep -Eh "max (enc|byte) length" enc/*.c | + # grep -Eo '[0-9]+' | + # sort | + # tail -n1 # => 6 + max_character_length = 6 + if !(line = stream.gets(size - max_character_length - 1)).nil? source << line string.write_string("#{line}\x00", line.bytesize + 1) end diff --git a/test/prism/api/parse_stream_test.rb b/test/prism/api/parse_stream_test.rb index 3bc86fbd61..e9826e401c 100644 --- a/test/prism/api/parse_stream_test.rb +++ b/test/prism/api/parse_stream_test.rb @@ -114,5 +114,19 @@ def test_nul_bytes assert result.success? assert_equal 3, result.value.statements.body.length end + + def test_long_line + # The important point of this test is that the character at 4095 + # is a multi-byte character (U+3042 HIRAGANA LETTER A) in this + # case. gets(4095) may return 4095 or more bytes if the last + # character is a multi-byte character. For example, + # StringIO.new("\u3042").gets(1).bytesize is 3 not 1. + long_string = "\"" + ("a" * 4093) + "\u3042" + "\"" + io = StringIO.new(long_string) + result = Prism.parse_stream(io) + + assert result.success? + assert_equal long_string[1..-2], result.value.statements.body[0].content + end end end