From 3f377b50eed467509c41fb66b4e5e04d3d4bd7ed Mon Sep 17 00:00:00 2001 From: Yusuke Endoh Date: Sun, 25 Oct 2026 05:26:07 +0900 Subject: [PATCH] Use the external encoding for character boundaries in GzipReader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GzipReader#getc and #gets with a limit split the data before it is converted to the internal encoding, but measured characters in the internal encoding. gz = Zlib.gzip("\x82\xa0\x82\xa2".b) # "あい" in Shift_JIS Zlib::GzipReader.new(StringIO.new(gz), external_encoding: "Shift_JIS", internal_encoding: "UTF-8").each_char.to_a # expected: ["あ", "い"] # actual: ["\x82", "\xA0", "\x82", "｢"] Co-Authored-By: Claude Opus 5.5 --- ext/zlib/zlib.c | 15 +++++++++------ test/zlib/test_zlib.rb | 7 +++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ext/zlib/zlib.c b/ext/zlib/zlib.c index f13ef4d..0ba8499 100644 --- a/ext/zlib/zlib.c +++ b/ext/zlib/zlib.c @@ -2484,6 +2484,9 @@ struct gzfile { #define GZFILE_READ_SIZE 2048 +/* the encoding of the data before conversion */ +#define GZFILE_RAW_ENC(gz) ((gz)->enc2 ? (gz)->enc2 : (gz)->enc) + enum { read_raw_arg_len, read_raw_arg_buf, read_raw_arg__count}; struct read_raw_arg { VALUE io; @@ -3084,7 +3087,7 @@ gzfile_getc(struct gzfile *gz) VALUE buf, dst = 0; int len; - len = rb_enc_mbmaxlen(gz->enc); + len = rb_enc_mbmaxlen(GZFILE_RAW_ENC(gz)); while (!ZSTREAM_IS_FINISHED(&gz->z) && ZSTREAM_BUF_FILLED(&gz->z) < len) { gzfile_read_more(gz, Qnil); } @@ -3113,7 +3116,7 @@ gzfile_getc(struct gzfile *gz) } else { buf = gz->z.buf; - len = rb_enc_mbclen(RSTRING_PTR(buf), RSTRING_END(buf), gz->enc); + len = rb_enc_mbclen(RSTRING_PTR(buf), RSTRING_END(buf), GZFILE_RAW_ENC(gz)); dst = gzfile_read(gz, len, Qnil); if (NIL_P(dst)) return dst; return gzfile_newstr(gz, dst); @@ -4347,10 +4350,11 @@ gzreader_charboundary(struct gzfile *gz, rb_len_t n) { char *s = RSTRING_PTR(gz->z.buf); char *e = s + ZSTREAM_BUF_FILLED(&gz->z); - char *p = rb_enc_left_char_head(s, s + n - 1, e, gz->enc); + rb_encoding *enc = GZFILE_RAW_ENC(gz); + char *p = rb_enc_left_char_head(s, s + n - 1, e, enc); rb_len_t l = p - s; if (l < n) { - int n_bytes = rb_enc_precise_mbclen(p, e, gz->enc); + int n_bytes = rb_enc_precise_mbclen(p, e, enc); if (MBCLEN_NEEDMORE_P(n_bytes)) { if ((l = gzfile_fill(gz, n + MBCLEN_NEEDMORE_LEN(n_bytes), Qnil)) > 0) { return l; @@ -4373,8 +4377,7 @@ gzreader_gets(int argc, VALUE *argv, VALUE obj) char *p, *res; rb_len_t rslen, n, limit = -1; int rspara; - rb_encoding *enc = gz->enc; - int maxlen = rb_enc_mbmaxlen(enc); + int maxlen = rb_enc_mbmaxlen(GZFILE_RAW_ENC(gz)); if (argc == 0) { rs = rb_rs; diff --git a/test/zlib/test_zlib.rb b/test/zlib/test_zlib.rb index e1ecbf8..58c7acd 100644 --- a/test/zlib/test_zlib.rb +++ b/test/zlib/test_zlib.rb @@ -1279,6 +1279,13 @@ def test_gets } end + def test_getc_with_encoding_conversion + gz = Zlib.gzip("\x82\xa0\x82\xa2".b) + Zlib::GzipReader.wrap(StringIO.new(gz), external_encoding: "Shift_JIS", internal_encoding: "UTF-8") do |f| + assert_equal(["\u3042", "\u3044"], f.each_char.to_a) + end + end + def test_gets2 Tempfile.create("test_zlib_gzip_reader_gets2") {|t| t.close