diff --git a/Changes.md b/Changes.md index d766fb5d..e59808ec 100644 --- a/Changes.md +++ b/Changes.md @@ -1,3 +1,25 @@ +## next release + +- `MMDB_open()` now returns `MMDB_INVALID_METADATA_ERROR` if the top-level + metadata map or the `description` map repeats a key. Before, top-level + metadata lookups used only the first value of a repeated key, and the + `description` list kept every pair, while other readers may keep the last + value. Keys are compared up to the first NUL byte, so keys that differ only + after a NUL count as repeated. A top-level metadata map or `description` map + with more than 1,024 keys also causes `MMDB_INVALID_METADATA_ERROR`. Databases + with repeated metadata keys no longer open. +- `MMDB_open()` now reads every top-level metadata key, not only the required + keys, and skips each value without following pointers in it. A malformed key + or value, or a map that claims more entries than it has, now causes + `MMDB_INVALID_DATA_ERROR`. A value nested deeper than the depth limit, or keys + whose total size exceeds the payload limit, causes + `MMDB_INVALID_METADATA_ERROR`. Some of these databases opened before. Metadata + whose root is not a map now causes `MMDB_INVALID_METADATA_ERROR`. +- `MMDB_get_value()` and its variants, and `MMDB_open()`, now return + `MMDB_INVALID_DATA_ERROR` when they skip a nested map whose key is a map or an + array. Before, they skipped only the key's header and read the data after it + out of step. A lookup could return the wrong value. + ## 1.14.1 - 2026-09-17 - Fixed a `-Wsign-compare` warning in `MMDB_open()` that broke builds with diff --git a/doc/libmaxminddb.md b/doc/libmaxminddb.md index 092bcff7..45245255 100644 --- a/doc/libmaxminddb.md +++ b/doc/libmaxminddb.md @@ -381,8 +381,9 @@ status codes are: its major version is not 2. This library can only handle major version 2. - `MMDB_OUT_OF_MEMORY_ERROR` - a memory allocation call (`malloc`, etc.) failed. - `MMDB_INVALID_DATA_ERROR` - an entry in the data section contains invalid - data. For example, a `uint16` field is claiming to be more than 2 bytes long. - The database is probably damaged or was generated incorrectly. + data. For example, a `uint16` field is claiming to be more than 2 bytes long, + or a map has a key that is a map or an array. The database is probably damaged + or was generated incorrectly. - `MMDB_INVALID_LOOKUP_PATH_ERROR` - The lookup path passed to `MMDB_get_value`, `MMDB_vget_value`, or `MMDB_aget_value` contains an array offset that is larger than LONG_MAX or smaller than LONG_MIN. @@ -459,6 +460,15 @@ If metadata processing exceeds one of the decoder resource limits described under `MMDB_get_entry_data_list()`, this function returns `MMDB_INVALID_METADATA_ERROR`. +If the metadata root is not a map, or the top-level metadata map or the +`description` map repeats a key or has more than 1,024 keys, this function +returns `MMDB_INVALID_METADATA_ERROR`. It compares keys up to the first NUL +byte, so keys that differ only after a NUL count as repeated. + +If a top-level metadata key or value is malformed, this function returns +`MMDB_INVALID_DATA_ERROR`. It does not follow pointers in values that it does +not use, so it does not detect a bad pointer target there. + ## `MMDB_close()` ```c @@ -631,6 +641,9 @@ calls. For each of the three functions, the return value is a status code as defined above. +Map keys must be strings. If a lookup skips a nested map whose key is a map or +an array, these functions return `MMDB_INVALID_DATA_ERROR`. + ## `MMDB_get_entry_data_list()` ```c diff --git a/src/maxminddb.c b/src/maxminddb.c index 95ca7a7d..27cb9631 100644 --- a/src/maxminddb.c +++ b/src/maxminddb.c @@ -178,9 +178,17 @@ typedef struct decode_state_s { uint64_t bytes; } decode_state_s; +typedef struct metadata_key_s { + const char *name; + uint32_t size; +} metadata_key_s; + #define METADATA_MARKER "\xab\xcd\xefMaxMind.com" /* This is 128kb */ #define METADATA_BLOCK_MAX_SIZE 131072 +/* Real metadata has about 10 top-level keys and a few description languages. + * The cap bounds the key sort, because C does not bound the cost of qsort. */ +#define METADATA_MAP_MAX_KEYS 1024 // 64 leads us to allocating 4 KiB on a 64bit system. #define MMDB_POOL_INIT_SIZE 64 @@ -191,6 +199,10 @@ static const uint8_t *find_metadata(const uint8_t *file_content, uint32_t *metadata_size); static int read_metadata(MMDB_s *mmdb); static MMDB_s make_fake_metadata_db(const MMDB_s *const mmdb); +static int check_metadata_keys_unique(const MMDB_s *const metadata_db); +static int next_map_key(const MMDB_s *const mmdb, + uint32_t *offset, + MMDB_entry_data_s *key); static int value_for_key_as_uint16(MMDB_entry_s *start, char *key, uint16_t *value); static int @@ -205,6 +217,9 @@ static int populate_languages_metadata(MMDB_s *mmdb, static int populate_description_metadata(MMDB_s *mmdb, MMDB_s *metadata_db, MMDB_entry_s *metadata_start); +static metadata_key_s metadata_key(const char *name, uint32_t size); +static int check_keys_unique(metadata_key_s *keys, uint32_t count); +static int compare_metadata_keys(const void *a, const void *b); static int resolve_any_address(const char *ipstr, struct addrinfo **addresses); static int find_address_in_search_tree(const MMDB_s *const mmdb, uint8_t const *address, @@ -589,10 +604,28 @@ static int read_metadata(MMDB_s *mmdb) { want to use the same functions we use for the data section to get metadata values. */ MMDB_s metadata_db = make_fake_metadata_db(mmdb); - MMDB_entry_s metadata_start = {.mmdb = &metadata_db, .offset = 0}; - int status = value_for_key_as_uint32( + // Check the major version before the strict walk below. A newer format + // can have metadata that this version cannot decode, and the caller must + // still get MMDB_UNKNOWN_DATABASE_FORMAT_ERROR for it. If this lookup + // fails or finds 0, the full read below reports the error. + uint16_t major_version = 0; + int status = value_for_key_as_uint16( + &metadata_start, "binary_format_major_version", &major_version); + if (MMDB_SUCCESS == status && 0 != major_version && 2 != major_version) { + DEBUG_MSGF("unsupported binary_format_major_version %i", major_version); + return MMDB_UNKNOWN_DATABASE_FORMAT_ERROR; + } + + // Key lookups return the first match, but other readers can keep the + // last one. A repeated key could give each reader a different value. + status = check_metadata_keys_unique(&metadata_db); + if (MMDB_SUCCESS != status) { + return status; + } + + status = value_for_key_as_uint32( &metadata_start, "node_count", &mmdb->metadata.node_count); if (MMDB_SUCCESS != status) { return status; @@ -699,6 +732,85 @@ static MMDB_s make_fake_metadata_db(const MMDB_s *const mmdb) { return fake_metadata_db; } +static int check_metadata_keys_unique(const MMDB_s *const metadata_db) { + MMDB_entry_data_s map; + CHECKED_DECODE_ONE_FOLLOW(metadata_db, 0, &map); + if (MMDB_DATA_TYPE_MAP != map.type) { + DEBUG_MSG("metadata value is not a map"); + return MMDB_INVALID_METADATA_ERROR; + } + if (0 == map.data_size) { + return MMDB_SUCCESS; + } + // Each entry takes at least a key byte and a value byte. Reject a larger + // count before it sizes the allocation, with the error the walk would + // return when it reaches the end of the section. + if (map.offset_to_next > metadata_db->data_section_size || + map.data_size > + (metadata_db->data_section_size - map.offset_to_next) / 2) { + DEBUG_MSG("metadata map size exceeds the metadata section"); + return MMDB_INVALID_DATA_ERROR; + } + if (map.data_size > METADATA_MAP_MAX_KEYS) { + DEBUG_MSG("metadata map has too many keys"); + return MMDB_INVALID_METADATA_ERROR; + } + + metadata_key_s *keys = calloc(map.data_size, sizeof(metadata_key_s)); + if (NULL == keys) { + return MMDB_OUT_OF_MEMORY_ERROR; + } + + int status = MMDB_SUCCESS; + uint32_t offset = map.offset_to_next; + uint64_t key_bytes = 0; + for (uint32_t i = 0; i < map.data_size; i++) { + MMDB_entry_data_s key; + status = next_map_key(metadata_db, &offset, &key); + if (MMDB_SUCCESS != status) { + goto cleanup; + } + // Pointer keys can reuse one long string many times. Charge each key + // against the payload budget to bound the scan and the sort below. + key_bytes += key.data_size; + if (key_bytes > (uint64_t)(MAXIMUM_DATA_STRUCTURE_BYTES)) { + DEBUG_MSG("metadata keys exceed the payload budget"); + status = MMDB_DECODER_LIMIT_ERROR; + goto cleanup; + } + keys[i] = metadata_key(key.utf8_string, key.data_size); + } + + status = check_keys_unique(keys, map.data_size); + +cleanup: + free(keys); + return status; +} + +/* Decodes the map key at *offset, skips its value, and sets *offset to the + * next key. */ +static int next_map_key(const MMDB_s *const mmdb, + uint32_t *offset, + MMDB_entry_data_s *key) { + CHECKED_DECODE_ONE_FOLLOW(mmdb, *offset, key); + if (MMDB_DATA_TYPE_UTF8_STRING != key->type) { + return MMDB_INVALID_DATA_ERROR; + } + + // A top-level value is at depth 1, as in + // MMDB_get_metadata_as_entry_data_list, so both apply the same limit. + MMDB_entry_data_s value; + CHECKED_DECODE_ONE(mmdb, key->offset_to_next, &value); + int status = skip_map_or_array(mmdb, &value, 1); + if (MMDB_SUCCESS != status) { + return status; + } + *offset = value.offset_to_next; + + return MMDB_SUCCESS; +} + static int value_for_key_as_uint16(MMDB_entry_s *start, char *key, uint16_t *value) { MMDB_entry_data_s entry_data; @@ -864,6 +976,7 @@ static int populate_description_metadata(MMDB_s *mmdb, } MMDB_entry_data_list_s *first_member = member; + metadata_key_s *keys = NULL; uint32_t map_size = member->entry_data.data_size; mmdb->metadata.description.count = 0; @@ -874,6 +987,17 @@ static int populate_description_metadata(MMDB_s *mmdb, MAYBE_CHECK_SIZE_OVERFLOW(map_size, SIZE_MAX / sizeof(MMDB_description_s *), MMDB_INVALID_METADATA_ERROR); + if (map_size > METADATA_MAP_MAX_KEYS) { + DEBUG_MSG("description map has too many keys"); + status = MMDB_INVALID_METADATA_ERROR; + goto cleanup; + } + + keys = calloc(map_size, sizeof(metadata_key_s)); + if (NULL == keys) { + status = MMDB_OUT_OF_MEMORY_ERROR; + goto cleanup; + } mmdb->metadata.description.descriptions = calloc(map_size, sizeof(MMDB_description_s *)); @@ -901,6 +1025,8 @@ static int populate_description_metadata(MMDB_s *mmdb, goto cleanup; } + keys[i] = metadata_key(member->entry_data.utf8_string, + member->entry_data.data_size); mmdb->metadata.description.descriptions[i]->language = mmdb_strndup( member->entry_data.utf8_string, member->entry_data.data_size); @@ -925,12 +1051,51 @@ static int populate_description_metadata(MMDB_s *mmdb, } } + status = check_keys_unique(keys, map_size); + cleanup: + free(keys); MMDB_free_entry_data_list(first_member); return status; } +/* Returns the key to compare for a metadata map key. Keys are compared up to + * the first NUL, because some callers, such as the PHP extension, cut keys + * there. */ +static metadata_key_s metadata_key(const char *name, uint32_t size) { + metadata_key_s key = {.name = name, .size = size}; + const char *nul = memchr(name, '\0', size); + if (NULL != nul) { + key.size = (uint32_t)(nul - name); + } + return key; +} + +/* Sorts keys and returns MMDB_INVALID_METADATA_ERROR if two are equal. */ +static int check_keys_unique(metadata_key_s *keys, uint32_t count) { + qsort(keys, count, sizeof(metadata_key_s), compare_metadata_keys); + for (uint32_t i = 1; i < count; i++) { + if (0 == compare_metadata_keys(&keys[i - 1], &keys[i])) { + DEBUG_MSG("duplicate key in metadata"); + return MMDB_INVALID_METADATA_ERROR; + } + } + return MMDB_SUCCESS; +} + +static int compare_metadata_keys(const void *a, const void *b) { + const metadata_key_s *key_a = a; + const metadata_key_s *key_b = b; + if (key_a->size < key_b->size) { + return -1; + } + if (key_a->size > key_b->size) { + return 1; + } + return memcmp(key_a->name, key_b->name, key_a->size); +} + MMDB_lookup_result_s MMDB_lookup_string(const MMDB_s *const mmdb, const char *const ipstr, int *const gai_error, @@ -1432,6 +1597,14 @@ static int skip_map_or_array(const MMDB_s *const mmdb, while (size-- > 0) { CHECKED_DECODE_ONE( mmdb, entry_data->offset_to_next, entry_data); // key + // Keys must be strings. This loop skips only a key's header, so + // the contents of a map or array key would be read as the keys and + // values that follow it. + if (MMDB_DATA_TYPE_MAP == entry_data->type || + MMDB_DATA_TYPE_ARRAY == entry_data->type) { + DEBUG_MSG("map key is a map or an array"); + return MMDB_INVALID_DATA_ERROR; + } CHECKED_DECODE_ONE( mmdb, entry_data->offset_to_next, entry_data); // value int status = skip_map_or_array(mmdb, entry_data, depth + 1); diff --git a/t/CMakeLists.txt b/t/CMakeLists.txt index bb9c23c7..bd7f4731 100644 --- a/t/CMakeLists.txt +++ b/t/CMakeLists.txt @@ -24,6 +24,7 @@ set(TEST_TARGET_NAMES overflow_bounds_t pointer_dos_t read_node_t + repeated_metadata_key_t version_t ) @@ -33,6 +34,7 @@ if(UNIX) # or if (NOT WIN32) bad_data_size_t bad_epoch_t bad_indent_t + duplicate_metadata_key_t empty_container_metadata_t invalid_sockaddr_t max_depth_t diff --git a/t/Makefile.am b/t/Makefile.am index f2c01aca..ac09c48e 100644 --- a/t/Makefile.am +++ b/t/Makefile.am @@ -20,12 +20,13 @@ check_PROGRAMS = \ bad_pointers_t bad_databases_t bad_data_size_t bad_epoch_t bad_indent_t \ bad_search_tree_t \ basic_lookup_t data_entry_list_t \ - data-pool-t data_types_t double_close_t dump_t empty_container_metadata_t \ + data-pool-t data_types_t double_close_t dump_t duplicate_metadata_key_t \ + empty_container_metadata_t \ gai_error_t get_value_t \ get_value_pointer_bug_t invalid_sockaddr_t \ ipv4_start_cache_t ipv6_lookup_in_ipv4_t max_depth_t metadata_t \ metadata_marker_t metadata_pointers_t no_map_get_value_t \ - overflow_bounds_t pointer_dos_t read_node_t \ + overflow_bounds_t pointer_dos_t read_node_t repeated_metadata_key_t \ threads_t version_t data_pool_t_LDFLAGS = $(AM_LDFLAGS) -lm diff --git a/t/duplicate_metadata_key_t.c b/t/duplicate_metadata_key_t.c new file mode 100644 index 00000000..d850d8b3 --- /dev/null +++ b/t/duplicate_metadata_key_t.c @@ -0,0 +1,503 @@ +#define _XOPEN_SOURCE 700 + +#include "maxminddb_test_helper.h" + +#include +#include + +// One IPv4 node whose 24-bit records both mark "no data", the data section +// separator, and the metadata marker. +#define DB_PREFIX \ + "\x00\x00\x01\x00\x00\x01" \ + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \ + "\xAB\xCD\xEF" \ + "MaxMind.com" + +// The metadata keys that MMDB_open requires, except description, without the +// map header. database_type starts at metadata offset 0x29. +#define REQUIRED_KEYS_BEFORE_DESCRIPTION \ + "\x4A" \ + "node_count" \ + "\xC1\x01" \ + "\x4B" \ + "record_size" \ + "\xA1\x18" \ + "\x4A" \ + "ip_version" \ + "\xA1\x04" \ + "\x4D" \ + "database_type" \ + "\x44" \ + "Test" \ + "\x49" \ + "languages" \ + "\x00\x04" \ + "\x5B" \ + "binary_format_major_version" \ + "\xA1\x02" \ + "\x5B" \ + "binary_format_minor_version" \ + "\xA0" \ + "\x4B" \ + "build_epoch" \ + "\x01\x02\x01" + +// The nine metadata keys that MMDB_open requires, without the map header. +#define REQUIRED_METADATA_KEYS \ + REQUIRED_KEYS_BEFORE_DESCRIPTION \ + "\x4B" \ + "description" \ + "\xE0" + +#define TEST_OPEN(db, expected, message) \ + test_open(db, sizeof(db) - 1, expected, message) + +// Writes db to a new file in TMPDIR, or /tmp, and puts its path in filename. +static void write_temp_db(const char *db, + size_t size, + char *filename, + size_t filename_size) { + const char *tmpdir = getenv("TMPDIR"); + if (NULL == tmpdir || '\0' == tmpdir[0]) { + tmpdir = "/tmp"; + } + int length = snprintf( + filename, filename_size, "%s/duplicate_metadata_key_t_XXXXXX", tmpdir); + if (length < 0 || (size_t)length >= filename_size) { + BAIL_OUT("TMPDIR is too long"); + } + int fd = mkstemp(filename); + if (-1 == fd) { + BAIL_OUT("could not create a temporary file in %s", tmpdir); + } + FILE *fp = fdopen(fd, "wb"); + if (NULL == fp) { + BAIL_OUT("could not open %s for writing", filename); + } + size_t written = fwrite(db, 1, size, fp); + if (0 != fclose(fp) || written != size) { + BAIL_OUT("could not write %s", filename); + } +} + +static void +test_open(const char *db, size_t size, int expected, const char *message) { + char filename[4096]; + write_temp_db(db, size, filename, sizeof(filename)); + + MMDB_s mmdb; + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); + cmp_ok(status, "==", expected, message); + if (MMDB_SUCCESS == status) { + MMDB_close(&mmdb); + } + + if (0 != remove(filename)) { + BAIL_OUT("could not remove %s", filename); + } +} + +#ifndef MAXIMUM_DATA_STRUCTURE_DEPTH + #define MAXIMUM_DATA_STRUCTURE_DEPTH 512 +#endif + +// An extra key whose value is a chain of `arrays` nested one-element arrays +// around a uint16. The value is at depth 1, and the uint16 is at depth +// arrays + 1. +// MMDB_get_metadata_as_entry_data_list fails when a value reaches +// MAXIMUM_DATA_STRUCTURE_DEPTH, and MMDB_open must agree with it. +static void test_deep_extra_value(size_t arrays, int expected) { + const char head[] = DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x45" + "extra"; + const size_t head_size = sizeof(head) - 1; + size_t size = head_size + arrays * 2 + 1; + char *db = malloc(size); + if (NULL == db) { + BAIL_OUT("could not allocate the deep test database"); + } + memcpy(db, head, head_size); + for (size_t i = 0; i < arrays; i++) { + db[head_size + i * 2] = '\x01'; + db[head_size + i * 2 + 1] = '\x04'; + } + db[size - 1] = '\xA0'; + + char filename[4096]; + write_temp_db(db, size, filename, sizeof(filename)); + free(db); + + MMDB_s mmdb; + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); + cmp_ok(status, + "==", + expected, + "MMDB_open status for an extra value with %zu nested arrays", + arrays); + if (MMDB_SUCCESS == status) { + MMDB_entry_data_list_s *list = NULL; + status = MMDB_get_metadata_as_entry_data_list(&mmdb, &list); + cmp_ok(status, + "==", + MMDB_SUCCESS, + "MMDB_get_metadata_as_entry_data_list reads the same metadata"); + MMDB_free_entry_data_list(list); + MMDB_close(&mmdb); + } + + if (0 != remove(filename)) { + BAIL_OUT("could not remove %s", filename); + } +} + +#ifndef MAXIMUM_DATA_STRUCTURE_BYTES + #define MAXIMUM_DATA_STRUCTURE_BYTES (1U << 21) +#endif + +// Writes a map header for count pairs and returns its size. +static size_t write_map_header(char *buf, size_t count) { + if (count <= 28) { + buf[0] = (char)(0xE0 | count); + return 1; + } + if (count <= 284) { + buf[0] = '\xFD'; + buf[1] = (char)(count - 29); + return 2; + } + buf[0] = '\xFE'; + buf[1] = (char)((count - 285) >> 8); + buf[2] = (char)((count - 285) & 0xFF); + return 3; +} + +// Pointer keys into a run of 0x5E bytes. The key at run offset j is a +// 24,443-byte string that starts with key_count - j bytes of 0x5E, so the keys +// are distinct, but together they exceed MAXIMUM_DATA_STRUCTURE_BYTES. +static void test_long_pointer_keys(void) { + const size_t key_size = 285 + 0x5E5E; + const size_t key_count = MAXIMUM_DATA_STRUCTURE_BYTES / key_size + 1; + // The keys must fit the map key cap, and each pointer has a one-byte + // offset payload. + skip(key_count > 1000, + 1, + "MAXIMUM_DATA_STRUCTURE_BYTES is too large for this fixture"); + + const size_t run = key_count + 3; + const size_t blob_size = run + key_size; + const char keys[] = REQUIRED_METADATA_KEYS "\x44" + "blob" + "\x5E"; + const size_t prefix_size = sizeof(DB_PREFIX) - 1; + const size_t keys_size = sizeof(keys) - 1; + + size_t size = prefix_size + 3 + keys_size + 2 + blob_size + key_count * 3; + char *db = malloc(size); + if (NULL == db) { + BAIL_OUT("could not allocate the long-key test database"); + } + memcpy(db, DB_PREFIX, prefix_size); + size_t pos = prefix_size; + pos += write_map_header(db + pos, 10 + key_count); + memcpy(db + pos, keys, keys_size); + pos += keys_size; + db[pos++] = (char)((blob_size - 285) >> 8); + db[pos++] = (char)((blob_size - 285) & 0xFF); + // The metadata offset of the blob content. + const size_t blob_offset = pos - prefix_size; + memset(db + pos, 0x5E, run); + memset(db + pos + run, 'a', key_size); + pos += blob_size; + for (size_t j = 0; j < key_count; j++) { + size_t target = blob_offset + j; + db[pos++] = (char)(0x20 | (target >> 8)); + db[pos++] = (char)(target & 0xFF); + db[pos++] = '\xA0'; + } + + test_open(db, + pos, + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects metadata keys past the payload budget"); + free(db); + + end_skip; +} + +// Builds metadata with count distinct keys, either in the top-level map +// (counting the nine required keys) or in the description map, and checks +// the MMDB_open status. count must be at least 285. +static void test_key_count(bool in_description, + size_t count, + int expected, + const char *message) { + const char required[] = REQUIRED_METADATA_KEYS; + const char before_description[] = + REQUIRED_KEYS_BEFORE_DESCRIPTION "\x4B" + "description"; + size_t size = sizeof(DB_PREFIX) + sizeof(required) + 3 + count * 8; + char *db = malloc(size); + if (NULL == db) { + BAIL_OUT("could not allocate the key-count test database"); + } + size_t pos = sizeof(DB_PREFIX) - 1; + memcpy(db, DB_PREFIX, pos); + size_t keys = count; + if (in_description) { + db[pos++] = '\xE9'; + memcpy(db + pos, before_description, sizeof(before_description) - 1); + pos += sizeof(before_description) - 1; + } else { + keys = count - 9; + } + // A map header with the 2-byte size form, for 285 through 65,820. + db[pos++] = '\xFE'; + db[pos++] = (char)((count - 285) >> 8); + db[pos++] = (char)((count - 285) & 0xFF); + if (!in_description) { + memcpy(db + pos, required, sizeof(required) - 1); + pos += sizeof(required) - 1; + } + for (size_t i = 0; i < keys; i++) { + char key[24]; + snprintf(key, sizeof(key), "k%04zu", i); + db[pos++] = '\x45'; + memcpy(db + pos, key, 5); + pos += 5; + db[pos++] = '\x41'; + db[pos++] = 'x'; + } + + test_open(db, pos, expected, message); + free(db); +} + +// A record {"a": {["x"]: "y"}, "b": "z"}. Both search-tree records point to +// it. Looking up "b" skips the value of "a", which has an array key. +static void test_lookup_after_array_key(void) { + const char db[] = "\x00\x00\x11\x00\x00\x11" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00" + "\xE2" + "\x41" + "a" + "\xE1" + "\x01\x04" + "\x41" + "x" + "\x41" + "y" + "\x41" + "b" + "\x41" + "z" + "\xAB\xCD\xEF" + "MaxMind.com" + "\xE9" REQUIRED_METADATA_KEYS; + char filename[4096]; + write_temp_db(db, sizeof(db) - 1, filename, sizeof(filename)); + + MMDB_s mmdb; + int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); + cmp_ok(status, "==", MMDB_SUCCESS, "MMDB_open opens the array-key record"); + if (MMDB_SUCCESS == status) { + int gai_error, mmdb_error; + MMDB_lookup_result_s result = + MMDB_lookup_string(&mmdb, "1.1.1.1", &gai_error, &mmdb_error); + cmp_ok(mmdb_error, "==", MMDB_SUCCESS, "lookup succeeded"); + ok(result.found_entry, "lookup found the record"); + MMDB_entry_data_s data; + status = MMDB_get_value(&result.entry, &data, "b", NULL); + cmp_ok(status, + "==", + MMDB_INVALID_DATA_ERROR, + "MMDB_get_value rejects a nested map key that is an array"); + MMDB_close(&mmdb); + } + + if (0 != remove(filename)) { + BAIL_OUT("could not remove %s", filename); + } +} + +// Major version 3 metadata with an extra value that this version cannot +// decode. MMDB_open must report the unknown format, not the bad value. +static void test_newer_major_version(void) { + char db[] = DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x45" + "extra" + "\x44" + "ab"; + const char key[] = "binary_format_major_version"; + size_t size = sizeof(db) - 1; + size_t version = 0; + for (size_t i = 0; i + sizeof(key) + 1 < size; i++) { + if (0 == memcmp(db + i, key, sizeof(key) - 1)) { + version = i + sizeof(key) - 1 + 1; + break; + } + } + if (0 == version || '\x02' != db[version]) { + BAIL_OUT("could not find binary_format_major_version in the fixture"); + } + db[version] = '\x03'; + + test_open(db, + size, + MMDB_UNKNOWN_DATABASE_FORMAT_ERROR, + "MMDB_open reports a newer major version before bad metadata"); +} + +int main(void) { + plan(NO_PLAN); + + TEST_OPEN(DB_PREFIX "\xE9" REQUIRED_METADATA_KEYS, + MMDB_SUCCESS, + "MMDB_open accepts metadata without repeated keys"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x45" + "extra" + "\xA1\x05", + MMDB_SUCCESS, + "MMDB_open accepts an extra distinct metadata key"); + + // The key points at the database_type value, "Test", at offset 0x37. + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x20\x37" + "\xA1\x05", + MMDB_SUCCESS, + "MMDB_open accepts a distinct key reached through a pointer"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x4D" + "database_type" + "\xA1\x05", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects a repeated database_type key"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x4B" + "description" + "\xE0", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects a repeated description key"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x20\x29" + "\xA1\x05", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects a repeated key reached through a pointer"); + + TEST_OPEN(DB_PREFIX "\xE9" REQUIRED_KEYS_BEFORE_DESCRIPTION "\x4B" + "description" + "\xE2" + "\x42" + "en" + "\x41" + "A" + "\x42" + "en" + "\x41" + "B", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects a repeated key in the description map"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x4F" + "database_type\x00" + "x" + "\x44" + "Evil", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects top-level keys that differ only after a NUL"); + + TEST_OPEN(DB_PREFIX "\xE9" REQUIRED_KEYS_BEFORE_DESCRIPTION "\x4B" + "description" + "\xE2" + "\x42" + "en" + "\x41" + "A" + "\x44" + "en\x00" + "x" + "\x41" + "B", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects description keys that differ only after a " + "NUL"); + + TEST_OPEN(DB_PREFIX "\xE9" REQUIRED_KEYS_BEFORE_DESCRIPTION "\x4B" + "description" + "\xE2" + "\x42" + "en" + "\x41" + "A" + "\xA1\x01" + "\x41" + "B", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects a non-string key in the description map"); + + TEST_OPEN(DB_PREFIX "\xEB" REQUIRED_METADATA_KEYS, + MMDB_INVALID_DATA_ERROR, + "MMDB_open rejects a metadata map count larger than its entries"); + + TEST_OPEN(DB_PREFIX "\xFE\x00\x00" REQUIRED_METADATA_KEYS, + MMDB_INVALID_DATA_ERROR, + "MMDB_open rejects a metadata map count larger than the section"); + + TEST_OPEN(DB_PREFIX "\xA1\x05", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects metadata whose root is not a map"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\xA1\x05" + "\xA1\x05", + MMDB_INVALID_DATA_ERROR, + "MMDB_open rejects a non-string key after the required keys"); + + TEST_OPEN(DB_PREFIX "\xEA" REQUIRED_METADATA_KEYS "\x45" + "extra" + "\x44" + "ab", + MMDB_INVALID_DATA_ERROR, + "MMDB_open rejects a truncated value after the required keys"); + + // The extra value is {["x"]: "fake"}. Skipping only the header of the + // array key would make the walk read "fake" as a top-level key and miss + // the repeated database_type. + TEST_OPEN(DB_PREFIX "\xEB" REQUIRED_METADATA_KEYS "\x45" + "extra" + "\xE1" + "\x01\x04" + "\x41" + "x" + "\x44" + "fake" + "\x4D" + "database_type" + "\x44" + "Evil", + MMDB_INVALID_DATA_ERROR, + "MMDB_open rejects a nested map key that is an array"); + + test_deep_extra_value(MAXIMUM_DATA_STRUCTURE_DEPTH - 2, MMDB_SUCCESS); + test_deep_extra_value(MAXIMUM_DATA_STRUCTURE_DEPTH - 1, + MMDB_INVALID_METADATA_ERROR); + test_long_pointer_keys(); + test_key_count(false, + 1024, + MMDB_SUCCESS, + "MMDB_open accepts 1,024 top-level metadata keys"); + test_key_count(false, + 1025, + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects 1,025 top-level metadata keys"); + test_key_count(true, + 1024, + MMDB_SUCCESS, + "MMDB_open accepts 1,024 description languages"); + test_key_count(true, + 1025, + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects 1,025 description languages"); + + test_lookup_after_array_key(); + test_newer_major_version(); + + done_testing(); +} diff --git a/t/maxmind-db b/t/maxmind-db index 1bc9af96..4464b5f6 160000 --- a/t/maxmind-db +++ b/t/maxmind-db @@ -1 +1 @@ -Subproject commit 1bc9af960e6c57cbf5e77aa53a62d72740f21fde +Subproject commit 4464b5f65fcbdff9d3ca16e5da6a6bf5d492cbbe diff --git a/t/repeated_metadata_key_t.c b/t/repeated_metadata_key_t.c new file mode 100644 index 00000000..037038f3 --- /dev/null +++ b/t/repeated_metadata_key_t.c @@ -0,0 +1,27 @@ +#include "maxminddb_test_helper.h" + +// The repeated-key databases from MaxMind-DB. This test needs no temporary +// file, so it also runs on Windows. +static void test_bad_data_file(const char *filename) { + char *path = bad_database_path(filename); + MMDB_s mmdb; + int status = MMDB_open(path, MMDB_MODE_MMAP, &mmdb); + cmp_ok(status, + "==", + MMDB_INVALID_METADATA_ERROR, + "MMDB_open rejects %s", + filename); + if (MMDB_SUCCESS == status) { + MMDB_close(&mmdb); + } + free(path); +} + +int main(void) { + plan(NO_PLAN); + test_bad_data_file("libmaxminddb-repeated-metadata-key.mmdb"); + test_bad_data_file("libmaxminddb-repeated-description-key.mmdb"); + test_bad_data_file("libmaxminddb-repeated-node-count.mmdb"); + test_bad_data_file("libmaxminddb-repeated-pointer-key.mmdb"); + done_testing(); +}