From 51ae3b1a85a96b3729f9f893ac7523fa0329dce5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 6 Nov 2026 10:26:19 +0100 Subject: [PATCH 1/3] Skip intermediate string slicing when using MemoryReader The decoder does a lot of `read().unpack1()`, which makes sense for the file reader, but for the memory reader is a bit wasteful because we can directly call `unpack1` on the in-memory string and unpack at a given offset with zero-copy and zero allocation. --- lib/maxmind/db.rb | 13 ++++--------- lib/maxmind/db/decoder.rb | 26 ++++++++++---------------- lib/maxmind/db/file_reader.rb | 6 ++++++ lib/maxmind/db/memory_reader.rb | 22 +++++++++++++++++++--- test/test_decoder.rb | 6 ++++++ 5 files changed, 45 insertions(+), 28 deletions(-) diff --git a/lib/maxmind/db.rb b/lib/maxmind/db.rb index 8e776bd..a1ec7fa 100644 --- a/lib/maxmind/db.rb +++ b/lib/maxmind/db.rb @@ -249,27 +249,22 @@ def read_node(node_number, index) if @record_size == 24 offset = index == 0 ? base_offset : base_offset + 3 - buf = @io.read(offset, 3) - node_bytes = "\x00".b << buf - return node_bytes.unpack1('N') + return @io.unpack1(offset, 3, 'N', 4) end if @record_size == 28 if index == 0 - buf = @io.read(base_offset, 4) - n = buf.unpack1('N') + n = @io.unpack1(base_offset, 4, 'N') last24 = n >> 8 first4 = (n & 0xf0) << 20 return first4 | last24 end - buf = @io.read(base_offset + 3, 4) - return buf.unpack1('N') & 0x0fffffff + return @io.unpack1(base_offset + 3, 4, 'N') & 0x0fffffff end if @record_size == 32 offset = index == 0 ? base_offset : base_offset + 4 - node_bytes = @io.read(offset, 4) - return node_bytes.unpack1('N') + return @io.unpack1(offset, 4, 'N') end raise InvalidDatabaseError, "Unsupported record size: #{@record_size}" diff --git a/lib/maxmind/db/decoder.rb b/lib/maxmind/db/decoder.rb index 6a6c0a7..74fef45 100644 --- a/lib/maxmind/db/decoder.rb +++ b/lib/maxmind/db/decoder.rb @@ -135,14 +135,12 @@ def decode_bytes(size, offset, budget) def decode_double(size, offset) verify_size(8, size) - buf = @io.read(offset, 8) - [buf.unpack1('G'), offset + 8] + [@io.unpack1(offset, size, 'G'), offset + 8] end def decode_float(size, offset) verify_size(4, size) - buf = @io.read(offset, 4) - [buf.unpack1('g'), offset + 4] + [@io.unpack1(offset, size, 'g'), offset + 4] end def verify_size(expected, actual) @@ -177,9 +175,7 @@ def decode_int(type_code, type_size, size, offset, budget) return 0, offset if size == 0 raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 - buf = @io.read(offset, size) - buf = buf.rjust(type_size, "\x00") if size != type_size - [buf.unpack1(type_code), offset + size] + [@io.unpack1(offset, size, type_code, type_size), offset + size] end def decode_uint128(size, offset, budget) @@ -187,13 +183,13 @@ def decode_uint128(size, offset, budget) return 0, offset if size == 0 raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 - buf = @io.read(offset, size) if size <= 8 - buf = buf.rjust(8, "\x00") - return buf.unpack1('Q>'), offset + size + a = @io.unpack1(offset, size, 'Q>', 8) + return [a, offset + size] end + buf = @io.read(offset, size) a_bytes = buf[0...-8].rjust(8, "\x00") b_bytes = buf[-8...buf.length] a = a_bytes.unpack1('Q>') @@ -227,7 +223,7 @@ def decode_pointer(size, offset) pointer = ((size & 0x7) << 8) | @io.getbyte(offset) when 1 new_offset = offset + 2 - pointer = ((size & 0x7) << 16) | @io.read(offset, 2).unpack1('n') + pointer = ((size & 0x7) << 16) | @io.unpack1(offset, 2, 'n') pointer += 2048 when 2 new_offset = offset + 3 @@ -237,7 +233,7 @@ def decode_pointer(size, offset) pointer += 526_336 else new_offset = offset + 4 - pointer = @io.read(offset, 4).unpack1('N') + pointer = @io.unpack1(offset, 4, 'N') end pointer += @pointer_base [pointer, new_offset] @@ -359,13 +355,11 @@ def size_from_ctrl_byte(ctrl_byte, offset, type_num) end if size == 30 - size_bytes = @io.read(offset, 2) - size = 285 + size_bytes.unpack1('n') + size = 285 + @io.unpack1(offset, 2, 'n') return size, offset + 2 end - size_bytes = "\x00".b << @io.read(offset, 3) - size = 65_821 + size_bytes.unpack1('N') + size = 65_821 + @io.unpack1(offset, 3, 'N', 4) [size, offset + 3] end end diff --git a/lib/maxmind/db/file_reader.rb b/lib/maxmind/db/file_reader.rb index eb73b53..a1e66b2 100644 --- a/lib/maxmind/db/file_reader.rb +++ b/lib/maxmind/db/file_reader.rb @@ -58,6 +58,12 @@ def read(offset, size) buf end + + def unpack1(offset, size, type_code, type_size = size) + buf = read(offset, size) + buf = buf.rjust(type_size, "\x00") if buf.bytesize < type_size + buf.unpack1(type_code) + end end end end diff --git a/lib/maxmind/db/memory_reader.rb b/lib/maxmind/db/memory_reader.rb index eab6811..66a162f 100644 --- a/lib/maxmind/db/memory_reader.rb +++ b/lib/maxmind/db/memory_reader.rb @@ -9,12 +9,12 @@ class MemoryReader def initialize(filename, options = {}) if options[:is_buffer] @buf = filename - @size = @buf.length + @size = @buf.bytesize return end @buf = File.read(filename, mode: 'rb').freeze - @size = @buf.length + @size = @buf.bytesize end attr_reader :size @@ -34,11 +34,27 @@ def getbyte(offset) def read(offset, size) return ''.b if size == 0 - raise_bad_data if offset + size > @buf.length + raise_bad_data if offset + size > @buf.bytesize @buf.byteslice(offset, size) end + def unpack1(offset, size, type_code, type_size = size) + if type_size == size + begin + value = @buf.unpack1(type_code, offset: offset) + return value unless value.nil? + rescue ArgumentError + raise_bad_data if offset + size > @buf.bytesize + raise + end + end + + buf = read(offset, size) + buf = buf.rjust(type_size, "\x00") if buf.bytesize < type_size + buf.unpack1(type_code) + end + private def raise_bad_data diff --git a/test/test_decoder.rb b/test/test_decoder.rb index 471952e..35b8788 100644 --- a/test/test_decoder.rb +++ b/test/test_decoder.rb @@ -26,6 +26,12 @@ def read(offset, size) bytes end + + def unpack1(offset, size, type_code, type_size = size) + buf = read(offset, size) + buf = buf.rjust(type_size, "\x00") if buf.bytesize < type_size + buf.unpack1(type_code) + end end def test_arrays From fc55394466f455281c2219b0657df841ab22b8d5 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 9 Nov 2026 09:00:45 +0100 Subject: [PATCH 2/3] WIP --- lib/maxmind/db/decoder.rb | 12 +++++++++--- lib/maxmind/db/memory_reader.rb | 15 ++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/maxmind/db/decoder.rb b/lib/maxmind/db/decoder.rb index 74fef45..c8070b6 100644 --- a/lib/maxmind/db/decoder.rb +++ b/lib/maxmind/db/decoder.rb @@ -172,10 +172,16 @@ def decode_uint64(size, offset, budget) def decode_int(type_code, type_size, size, offset, budget) raise_invalid_size if size > type_size - return 0, offset if size == 0 - raise_bytes_exceeded if (budget[BUDGET_BYTES] -= size) < 0 - [@io.unpack1(offset, size, type_code, type_size), offset + size] + + case size + when 0 + return 0, offset + when 1 + return [@io.getbyte(offset), offset + 1] + else + [@io.unpack1(offset, size, type_code, type_size), offset + size] + end end def decode_uint128(size, offset, budget) diff --git a/lib/maxmind/db/memory_reader.rb b/lib/maxmind/db/memory_reader.rb index 66a162f..9430fe4 100644 --- a/lib/maxmind/db/memory_reader.rb +++ b/lib/maxmind/db/memory_reader.rb @@ -40,14 +40,15 @@ def read(offset, size) end def unpack1(offset, size, type_code, type_size = size) + raise_bad_data if offset + size > @buf.bytesize + if type_size == size - begin - value = @buf.unpack1(type_code, offset: offset) - return value unless value.nil? - rescue ArgumentError - raise_bad_data if offset + size > @buf.bytesize - raise - end + return @buf.unpack1(type_code, offset: offset) + elsif size == 3 && type_code == 'N' + # Fast path for 24bit integer. Avoid read+rjust + a, b, c = getbyte(offset), getbyte(offset + 1), getbyte(offset + 2) + + return (a << 16) + (b << 8) + c end buf = read(offset, size) From 23c8ce539ba033c2820f336d6b2fc5439abab721 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 9 Nov 2026 09:14:26 +0100 Subject: [PATCH 3/3] WIP --- lib/maxmind/db/memory_reader.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/maxmind/db/memory_reader.rb b/lib/maxmind/db/memory_reader.rb index 9430fe4..560f17f 100644 --- a/lib/maxmind/db/memory_reader.rb +++ b/lib/maxmind/db/memory_reader.rb @@ -42,11 +42,19 @@ def read(offset, size) def unpack1(offset, size, type_code, type_size = size) raise_bad_data if offset + size > @buf.bytesize + if type_code == 'N' + case size + when 4 + a, b, c, d = @buf.getbyte(offset), @buf.getbyte(offset + 1), @buf.getbyte(offset + 2), @buf.getbyte(offset + 3) + return ((a << 24) + (b << 16) + (c << 8) + d) + end + end + if type_size == size return @buf.unpack1(type_code, offset: offset) elsif size == 3 && type_code == 'N' # Fast path for 24bit integer. Avoid read+rjust - a, b, c = getbyte(offset), getbyte(offset + 1), getbyte(offset + 2) + a, b, c = @buf.getbyte(offset), @buf.getbyte(offset + 1), @buf.getbyte(offset + 2) return (a << 16) + (b << 8) + c end