diff --git a/tuple/CMakeLists.txt b/tuple/CMakeLists.txt index 4b0a48c7..54df11ee 100644 --- a/tuple/CMakeLists.txt +++ b/tuple/CMakeLists.txt @@ -54,4 +54,6 @@ install(FILES include/array_tuple_intersection_impl.hpp include/array_tuple_a_not_b.hpp include/array_tuple_a_not_b_impl.hpp + include/array_of_strings_sketch.hpp + include/array_of_strings_sketch_impl.hpp DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/DataSketches") diff --git a/tuple/include/array_of_strings_sketch.hpp b/tuple/include/array_of_strings_sketch.hpp new file mode 100644 index 00000000..60701834 --- /dev/null +++ b/tuple/include/array_of_strings_sketch.hpp @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ARRAY_OF_STRINGS_SKETCH_HPP_ +#define ARRAY_OF_STRINGS_SKETCH_HPP_ + +#include +#include + +#include "array_tuple_sketch.hpp" +#include "xxhash64.h" + +namespace datasketches { + +using array_of_strings = array; + +// default update policy for an array of strings +class default_array_of_strings_update_policy { +public: + default_array_of_strings_update_policy() = default; + + array_of_strings create() const; + + void update(array_of_strings& array, const array_of_strings& input) const; + + void update(array_of_strings& array, const array_of_strings* input) const; +}; + +/** + * Serializer/deserializer for an array of strings. + * + * Requirements: + * - Array size must be <= 127. + * + * This serde does not perform UTF-8 validation. Callers must ensure strings + * are valid UTF-8 before serialization to guarantee interoperability with + * Java, Go, and Rust implementations. + */ +template> +struct default_array_of_strings_serde { + using summary_allocator = typename std::allocator_traits::template rebind_alloc; + + explicit default_array_of_strings_serde(const Allocator& allocator = Allocator()); + + void serialize(std::ostream& os, const array_of_strings* items, unsigned num) const; + void deserialize(std::istream& is, array_of_strings* items, unsigned num) const; + size_t serialize(void* ptr, size_t capacity, const array_of_strings* items, unsigned num) const; + size_t deserialize(const void* ptr, size_t capacity, array_of_strings* items, unsigned num) const; + size_t size_of_item(const array_of_strings& item) const; + +private: + summary_allocator summary_allocator_; + static void check_num_nodes(uint8_t num_nodes); + static uint32_t compute_total_bytes(const array_of_strings& item); +}; + +/** + * Hashes an array of strings key the same way as Java's ArrayOfStringsTupleSketch + * (org.apache.datasketches.tuple.Util.stringArrHash), so that sketches built in C++ and Java + * from the same keys can be combined. + * + * The elements are joined with "," and the joined string is hashed with XXH64 (seed 0x7A3CCA71) + * as UTF-16 code units in little-endian byte order (UTF-16LE, no byte-order mark). + * The strings are expected to be UTF-8. Invalid UTF-8 is replaced with U+FFFD the same way + * Java's UTF-8 decoder (new String(bytes, StandardCharsets.UTF_8)) replaces it. + * + * Key arrays are hashed by joining their elements with ",". Keys that join to the same string, + * for example {"a,b"} and {"a", "b"}, or an empty array and {""}, are treated as the same key. + * If key elements may contain commas, escape or encode them before updating the sketch. + * + * @param key array of strings to hash + * @return hash of the key to pass to update() of update_array_of_strings_tuple_sketch + */ +uint64_t hash_array_of_strings_key(const array_of_strings& key); + +/** + * Extended class of compact_tuple_sketch for array of strings. + * + * Requirements: + * - Array size must be <= 127. + * + * UTF-8 compatibility: + * Serialized sketches are intended to be language and platform independent. + * Other implementations (Java, Go, Rust) enforce UTF-8 encoding for strings. + * This C++ implementation does not validate UTF-8; it is the caller's + * responsibility to ensure all strings are valid UTF-8 before calling update(). + * Non-UTF-8 strings may serialize successfully but will fail to deserialize + * in other language implementations. + */ +template> +class compact_array_of_strings_tuple_sketch: + public compact_tuple_sketch { +public: + using Base = compact_tuple_sketch; + using vector_bytes = typename Base::vector_bytes; + + /** + * Copy constructor. + * Constructs a compact sketch from another sketch (update or compact) + * @param other sketch to be constructed from + * @param ordered if true make the resulting sketch ordered + */ + template + compact_array_of_strings_tuple_sketch(const Sketch& sketch, bool ordered = true); + + /** + * This method serializes the sketch into a given stream in a binary form + * @param os output stream + * @param sd instance of a SerDe + */ + template> + void serialize(std::ostream& os, const SerDe& sd = SerDe()) const; + + /** + * This method serializes the sketch as a vector of bytes. + * An optional header can be reserved in front of the sketch. + * It is a blank space of a given size. + * @param header_size_bytes space to reserve in front of the sketch + * @param sd instance of a SerDe + * @return serialized sketch as a vector of bytes + */ + template> + vector_bytes serialize(unsigned header_size_bytes = 0, const SerDe& sd = SerDe()) const; + + /** + * This method deserializes a sketch from a given stream. + * @param is input stream + * @param seed the seed for the hash function that was used to create the sketch + * @param sd instance of a SerDe + * @param allocator instance of an Allocator + * @return an instance of the sketch + */ + template> + static compact_array_of_strings_tuple_sketch deserialize(std::istream& is, uint64_t seed = DEFAULT_SEED, + const SerDe& sd = SerDe(), const Allocator& allocator = Allocator()); + + /** + * This method deserializes a sketch from a given array of bytes. + * @param bytes pointer to the array of bytes + * @param size the size of the array + * @param seed the seed for the hash function that was used to create the sketch + * @param sd instance of a SerDe + * @param allocator instance of an Allocator + * @return an instance of the sketch + */ + template> + static compact_array_of_strings_tuple_sketch deserialize(const void* bytes, size_t size, uint64_t seed = DEFAULT_SEED, + const SerDe& sd = SerDe(), const Allocator& allocator = Allocator()); + +private: + explicit compact_array_of_strings_tuple_sketch(Base&& base); +}; + +/** + * Convenience alias for update_tuple_sketch for array of strings + */ +template, + typename Policy = default_array_of_strings_update_policy> +using update_array_of_strings_tuple_sketch = update_tuple_sketch< + array_of_strings, + array_of_strings, + Policy, + Allocator +>; + +/** + * Converts an array of strings tuple sketch to a compact sketch (ordered or unordered). + * @param sketch input sketch + * @param ordered optional flag to specify if an ordered sketch should be produced + * @return compact array of strings sketch + */ +template, typename Policy = default_array_of_strings_update_policy> +compact_array_of_strings_tuple_sketch compact_array_of_strings_sketch( + const update_array_of_strings_tuple_sketch& sketch, bool ordered = true); + +} /* namespace datasketches */ + +#include "array_of_strings_sketch_impl.hpp" + +#endif diff --git a/tuple/include/array_of_strings_sketch_impl.hpp b/tuple/include/array_of_strings_sketch_impl.hpp new file mode 100644 index 00000000..475c6523 --- /dev/null +++ b/tuple/include/array_of_strings_sketch_impl.hpp @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ARRAY_OF_STRINGS_SKETCH_IMPL_HPP_ +#define ARRAY_OF_STRINGS_SKETCH_IMPL_HPP_ + +#include + +#include "common_defs.hpp" + +namespace datasketches { + +inline array_of_strings default_array_of_strings_update_policy::create() const { + return array_of_strings(0, ""); +} + +inline void default_array_of_strings_update_policy::update( + array_of_strings& array, const array_of_strings& input +) const { + const auto length = static_cast(input.size()); + array = array_of_strings(static_cast(length), ""); + for (size_t i = 0; i < length; ++i) array[i] = input[i]; +} + +inline void default_array_of_strings_update_policy::update( + array_of_strings& array, const array_of_strings* input +) const { + if (input == nullptr) { + array = array_of_strings(0, ""); + return; + } + const auto length = static_cast(input->size()); + array = array_of_strings(static_cast(length), ""); + for (size_t i = 0; i < length; ++i) array[i] = (*input)[i]; +} + +/** + * Feeds strings to XXHash64 as UTF-16LE code units, which is how Java hashes + * a String with XxHash.hashCharArr(s.toCharArray(), ...). + * + * The input is decoded from UTF-8. Malformed input is replaced with U+FFFD exactly + * as Java's UTF-8 decoder (new String(bytes, StandardCharsets.UTF_8)) does it: + * one U+FFFD for the longest valid prefix of a malformed sequence (at least one byte), + * and one U+FFFD for a complete 3-byte sequence that encodes a surrogate. + */ +class array_of_strings_key_hasher { +public: + explicit array_of_strings_key_hasher(uint64_t seed): hasher_(seed), size_(0) {} + + void add_utf8(const std::string& str) { + const uint8_t* bytes = reinterpret_cast(str.data()); + const size_t length = str.size(); + size_t i = 0; + while (i < length) { + const uint8_t lead = bytes[i++]; + if (lead < 0x80) { // ASCII + add_code_unit(lead); + continue; + } + uint8_t second_min; + uint8_t second_max; + const unsigned num_continuation = get_sequence_rule(lead, second_min, second_max); + if (num_continuation == 0) { // Invalid UTF8 + add_code_unit(REPLACEMENT_CHAR); + continue; + } + uint32_t code_point = lead & (0x3F >> num_continuation); + unsigned num_read = 0; + while (num_read < num_continuation && i < length) { + const uint8_t byte = bytes[i]; + const bool valid = num_read == 0 ? byte >= second_min && byte <= second_max : (byte & 0xC0) == 0x80; + if (!valid) break; + code_point = (code_point << 6) | (byte & 0x3F); + ++i; + ++num_read; + } + if (num_read < num_continuation) { + // the bytes consumed so far are the longest valid prefix, resume at the next byte + add_code_unit(REPLACEMENT_CHAR); + } else { + add_code_point(code_point); + } + } + } + + void add_code_unit(uint16_t unit) { + if (size_ == sizeof(buffer_)) flush(); + buffer_[size_++] = static_cast(unit); + buffer_[size_++] = static_cast(unit >> 8); + } + + uint64_t hash() { + flush(); + return hasher_.hash(); + } + +private: + enum : uint16_t { REPLACEMENT_CHAR = 0xFFFD }; + + XXHash64 hasher_; + uint8_t buffer_[256]; + size_t size_; + + void flush() { + hasher_.add(buffer_, size_); + size_ = 0; + } + + void add_code_point(uint32_t code_point) { + if (code_point >= 0xD800 && code_point <= 0xDFFF) { + add_code_unit(REPLACEMENT_CHAR); + } else if (code_point < 0x10000) { + add_code_unit(static_cast(code_point)); + } else { + code_point -= 0x10000; + add_code_unit(static_cast(0xD800 | (code_point >> 10))); + add_code_unit(static_cast(0xDC00 | (code_point & 0x3FF))); + } + } + + /** + * Returns the number of continuation bytes that follow a UTF-8 lead byte, + * or 0 if the byte cannot start a sequence (80-C1, F5-FF). + * Also returns the range the second byte must be in, as Java's decoder checks it. + */ + static unsigned get_sequence_rule(uint8_t lead, uint8_t& second_min, uint8_t& second_max) { + second_min = 0x80; + second_max = 0xBF; + if (lead >= 0xC2 && lead <= 0xDF) return 1; + if (lead == 0xE0) second_min = 0xA0; // no overlong forms + // ED A0-BF encodes a surrogate, which Java's decoder accepts here and replaces once decoded + if (lead >= 0xE0 && lead <= 0xEF) return 2; + if (lead == 0xF0) second_min = 0x90; // no overlong forms + if (lead == 0xF4) second_max = 0x8F; // nothing above U+10FFFF + if (lead >= 0xF0 && lead <= 0xF4) return 3; + return 0; + } +}; + +inline uint64_t hash_array_of_strings_key(const array_of_strings& key) { + // Matches Java Util.PRIME for ArrayOfStrings key hashing. + static constexpr uint64_t STRING_ARR_HASH_SEED = 0x7A3CCA71ULL; + array_of_strings_key_hasher hasher(STRING_ARR_HASH_SEED); + const auto size = static_cast(key.size()); + for (size_t i = 0; i < size; ++i) { + if (i > 0) hasher.add_code_unit(','); + hasher.add_utf8(key[i]); + } + return hasher.hash(); +} + +template +compact_array_of_strings_tuple_sketch compact_array_of_strings_sketch( + const update_array_of_strings_tuple_sketch& sketch, bool ordered +) { + return compact_array_of_strings_tuple_sketch(sketch, ordered); +} + +template +template +compact_array_of_strings_tuple_sketch::compact_array_of_strings_tuple_sketch( + const Sketch& sketch, bool ordered +): Base(sketch, ordered) {} + +template +compact_array_of_strings_tuple_sketch::compact_array_of_strings_tuple_sketch( + Base&& base +): Base(std::move(base)) {} + +template +template +void compact_array_of_strings_tuple_sketch::serialize(std::ostream& os, const SerDe& sd) const { + Base::serialize(os, sd); +} + +template +template +auto compact_array_of_strings_tuple_sketch::serialize( + unsigned header_size_bytes, const SerDe& sd +) const -> vector_bytes { + return Base::serialize(header_size_bytes, sd); +} + +template +template +auto compact_array_of_strings_tuple_sketch::deserialize( + std::istream& is, uint64_t seed, const SerDe& sd, const Allocator& allocator +) -> compact_array_of_strings_tuple_sketch { + auto base = Base::deserialize(is, seed, sd, allocator); + return compact_array_of_strings_tuple_sketch(std::move(base)); +} + +template +template +auto compact_array_of_strings_tuple_sketch::deserialize( + const void* bytes, size_t size, uint64_t seed, const SerDe& sd, const Allocator& allocator +) -> compact_array_of_strings_tuple_sketch { + auto base = Base::deserialize(bytes, size, seed, sd, allocator); + return compact_array_of_strings_tuple_sketch(std::move(base)); +} + +template +default_array_of_strings_serde::default_array_of_strings_serde(const Allocator& allocator): + summary_allocator_(allocator) {} + +template +void default_array_of_strings_serde::serialize( + std::ostream& os, const array_of_strings* items, unsigned num +) const { + unsigned i = 0; + try { + for (; i < num; ++i) { + const uint32_t total_bytes = compute_total_bytes(items[i]); + const uint8_t num_nodes = items[i].size(); + write(os, total_bytes); + write(os, num_nodes); + const std::string* data = items[i].data(); + for (uint8_t j = 0; j < num_nodes; ++j) { + const uint32_t length = static_cast(data[j].size()); + write(os, length); + os.write(data[j].data(), length); + } + } + } catch (std::runtime_error& e) { + if (std::string(e.what()).find("size exceeds 127") != std::string::npos) throw; + throw std::runtime_error("array_of_strings stream write failed at item " + std::to_string(i)); + } +} + +template +void default_array_of_strings_serde::deserialize( + std::istream& is, array_of_strings* items, unsigned num +) const { + unsigned i = 0; + bool failure = false; + try { + for (; i < num; ++i) { + read(is); // total_bytes + if (!is) { failure = true; break; } + const uint8_t num_nodes = read(is); + if (!is) { failure = true; break; } + check_num_nodes(num_nodes); + array_of_strings array(num_nodes, ""); + for (uint8_t j = 0; j < num_nodes; ++j) { + const uint32_t length = read(is); + if (!is) { failure = true; break; } + std::string value(length, '\0'); + if (length != 0) { + is.read(&value[0], length); + if (!is) { failure = true; break; } + } + array[j] = std::move(value); + } + if (failure) break; + summary_allocator alloc(summary_allocator_); + std::allocator_traits::construct(alloc, &items[i], std::move(array)); + } + } catch (std::exception& e) { + summary_allocator alloc(summary_allocator_); + for (unsigned j = 0; j < i; ++j) { + std::allocator_traits::destroy(alloc, &items[j]); + } + if (std::string(e.what()).find("size exceeds 127") != std::string::npos) throw; + throw std::runtime_error("array_of_strings stream read failed at item " + std::to_string(i)); + } + if (failure) { + summary_allocator alloc(summary_allocator_); + for (unsigned j = 0; j < i; ++j) { + std::allocator_traits::destroy(alloc, &items[j]); + } + throw std::runtime_error("array_of_strings stream read failed at item " + std::to_string(i)); + } +} + +template +size_t default_array_of_strings_serde::serialize( + void* ptr, size_t capacity, const array_of_strings* items, unsigned num +) const { + uint8_t* ptr8 = static_cast(ptr); + size_t bytes_written = 0; + + for (unsigned i = 0; i < num; ++i) { + const uint32_t total_bytes = compute_total_bytes(items[i]); + const uint8_t num_nodes = static_cast(items[i].size()); + check_memory_size(bytes_written + total_bytes, capacity); + bytes_written += copy_to_mem(total_bytes, ptr8 + bytes_written); + bytes_written += copy_to_mem(num_nodes, ptr8 + bytes_written); + const std::string* data = items[i].data(); + for (uint8_t j = 0; j < num_nodes; ++j) { + const uint32_t length = static_cast(data[j].size()); + + bytes_written += copy_to_mem(length, ptr8 + bytes_written); + bytes_written += copy_to_mem(data[j].data(), ptr8 + bytes_written, length); + } + } + return bytes_written; +} + +template +size_t default_array_of_strings_serde::deserialize( + const void* ptr, size_t capacity, array_of_strings* items, unsigned num +) const { + const uint8_t* ptr8 = static_cast(ptr); + size_t bytes_read = 0; + + unsigned i = 0; + + try { + for (; i < num; ++i) { + check_memory_size(bytes_read + sizeof(uint32_t), capacity); + const size_t item_start = bytes_read; + uint32_t total_bytes; + bytes_read += copy_from_mem(ptr8 + bytes_read, total_bytes); + check_memory_size(item_start + total_bytes, capacity); + + check_memory_size(bytes_read + sizeof(uint8_t), capacity); + uint8_t num_nodes; + bytes_read += copy_from_mem(ptr8 + bytes_read, num_nodes); + check_num_nodes(num_nodes); + + array_of_strings array(num_nodes, ""); + for (uint8_t j = 0; j < num_nodes; ++j) { + check_memory_size(bytes_read + sizeof(uint32_t), capacity); + uint32_t length; + bytes_read += copy_from_mem(ptr8 + bytes_read, length); + + check_memory_size(bytes_read + length, capacity); + std::string value(length, '\0'); + if (length != 0) { + bytes_read += copy_from_mem(ptr8 + bytes_read, &value[0], length); + } + array[j] = std::move(value); + } + summary_allocator alloc(summary_allocator_); + std::allocator_traits::construct(alloc, &items[i], std::move(array)); + } + } catch (std::exception& e) { + summary_allocator alloc(summary_allocator_); + for (unsigned j = 0; j < i; ++j) { + std::allocator_traits::destroy(alloc, &items[j]); + } + if (std::string(e.what()).find("size exceeds 127") != std::string::npos) throw; + throw std::runtime_error("array_of_strings bytes read failed at item " + std::to_string(i)); + } + return bytes_read; +} + +template +size_t default_array_of_strings_serde::size_of_item(const array_of_strings& item) const { + return compute_total_bytes(item); +} + +template +void default_array_of_strings_serde::check_num_nodes(uint8_t num_nodes) { + if (num_nodes > 127) { + throw std::runtime_error("array_of_strings size exceeds 127"); + } +} + +template +uint32_t default_array_of_strings_serde::compute_total_bytes(const array_of_strings& item) { + const auto count = item.size(); + check_num_nodes(static_cast(count)); + size_t total = sizeof(uint32_t) + sizeof(uint8_t) + count * sizeof(uint32_t); + const std::string* data = item.data(); + for (uint32_t j = 0; j < count; ++j) { + total += data[j].size(); + } + return static_cast(total); +} + +} /* namespace datasketches */ + +#endif diff --git a/tuple/test/CMakeLists.txt b/tuple/test/CMakeLists.txt index 4ca6a503..3d7ccca3 100644 --- a/tuple/test/CMakeLists.txt +++ b/tuple/test/CMakeLists.txt @@ -44,6 +44,7 @@ target_sources(tuple_test tuple_a_not_b_test.cpp tuple_jaccard_similarity_test.cpp array_of_doubles_sketch_test.cpp + array_of_strings_sketch_test.cpp engagement_test.cpp ) @@ -52,6 +53,7 @@ target_sources(tuple_test PRIVATE aod_sketch_deserialize_from_java_test.cpp tuple_sketch_deserialize_from_java_test.cpp + aos_sketch_deserialize_from_java_test.cpp ) endif() @@ -60,5 +62,6 @@ target_sources(tuple_test PRIVATE aod_sketch_serialize_for_java.cpp tuple_sketch_serialize_for_java.cpp + aos_sketch_serialize_for_java.cpp ) endif() diff --git a/tuple/test/aos_sketch_deserialize_from_java_test.cpp b/tuple/test/aos_sketch_deserialize_from_java_test.cpp new file mode 100644 index 00000000..1e8fc29a --- /dev/null +++ b/tuple/test/aos_sketch_deserialize_from_java_test.cpp @@ -0,0 +1,376 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "array_of_strings_sketch.hpp" + +namespace datasketches { + // assume the binary sketches for this test have been generated by datasketches-java code + // in the subdirectory "serialization_test_data/java_generated_files" in the root directory of this project + static std::string testBinaryInputPath = std::string(TEST_BINARY_INPUT_PATH) + "../../serialization_test_data/java_generated_files/"; + + static std::vector read_binary_file(const std::string& path) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + is.seekg(0, std::ios::end); + const auto size = static_cast(is.tellg()); + is.seekg(0, std::ios::beg); + std::vector bytes(size); + if (size != 0) { + is.read(reinterpret_cast(bytes.data()), size); + } + return bytes; + } + + TEST_CASE("aos sketch one value", "[serde_compat]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + const auto path = testBinaryInputPath + "aos_1_n" + std::to_string(n) + "_java.sk"; + SECTION("stream") { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + is, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.is_estimation_mode() == (n > 1000)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 1); + } + } + SECTION("bytes") { + const auto bytes = read_binary_file(path); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.is_estimation_mode() == (n > 1000)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 1); + } + } + } + } + + TEST_CASE("aos sketch three values", "[serde_compat]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + const auto path = testBinaryInputPath + "aos_3_n" + std::to_string(n) + "_java.sk"; + SECTION("stream") { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + is, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.is_estimation_mode() == (n > 1000)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 3); + } + } + SECTION("bytes") { + const auto bytes = read_binary_file(path); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.is_estimation_mode() == (n > 1000)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 3); + } + } + } + } + + TEST_CASE("aos sketch non-empty no entries", "[serde_compat]") { + const auto path = testBinaryInputPath + "aos_1_non_empty_no_entries_java.sk"; + SECTION("stream") { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + is, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE(sketch.get_num_retained() == 0); + } + SECTION("bytes") { + const auto bytes = read_binary_file(path); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE(sketch.get_num_retained() == 0); + } + } + + TEST_CASE("aos sketch multi keys strings", "[serde_compat]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + const auto path = testBinaryInputPath + "aos_multikey_n" + std::to_string(n) + "_java.sk"; + SECTION("stream") { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + is, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.is_estimation_mode() == (n > 1000)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 1); + } + } + SECTION("bytes") { + const auto bytes = read_binary_file(path); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.is_estimation_mode() == (n > 1000)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 1); + } + } + } + } + + TEST_CASE("aos sketch unicode strings", "[serde_compat]") { + const auto path = testBinaryInputPath + "aos_unicode_java.sk"; + auto check = [](const compact_array_of_strings_tuple_sketch<>& sketch) { + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE_FALSE(sketch.is_estimation_mode()); + REQUIRE(sketch.get_num_retained() == 3); + + const std::vector> expected_values = { + {"밸류", "값"}, + {"📦", "🎁"}, + {"ценить1", "ценить2"} + }; + std::vector matched(expected_values.size(), false); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + REQUIRE(entry.second.size() == 2); + + bool found = false; + for (size_t i = 0; i < expected_values.size(); ++i) { + if (matched[i]) continue; + const auto& expected = expected_values[i]; + if (entry.second.size() != expected.size()) continue; + bool equal = true; + for (size_t j = 0; j < expected.size(); ++j) { + if (entry.second[j] != expected[j]) { + equal = false; + break; + } + } + if (equal) { + matched[i] = true; + found = true; + break; + } + } + REQUIRE(found); + } + for (bool found: matched) REQUIRE(found); + }; + SECTION("stream") { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + is, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + check(sketch); + } + SECTION("bytes") { + const auto bytes = read_binary_file(path); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + check(sketch); + } + } + + TEST_CASE("aos sketch empty strings", "[serde_compat]") { + const auto path = testBinaryInputPath + "aos_empty_strings_java.sk"; + auto check = [](const compact_array_of_strings_tuple_sketch<>& sketch) { + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE_FALSE(sketch.is_estimation_mode()); + REQUIRE(sketch.get_num_retained() == 3); + const std::vector> expected_values = { + {"empty_key_value"}, + {""}, + {"", ""} + }; + std::vector matched(expected_values.size(), false); + for (const auto& entry: sketch) { + REQUIRE(entry.first < sketch.get_theta64()); + + bool found = false; + for (size_t i = 0; i < expected_values.size(); ++i) { + if (matched[i]) continue; + const auto& expected = expected_values[i]; + if (entry.second.size() != expected.size()) continue; + bool equal = true; + for (size_t j = 0; j < expected.size(); ++j) { + if (entry.second[j] != expected[j]) { + equal = false; + break; + } + } + if (equal) { + matched[i] = true; + found = true; + break; + } + } + REQUIRE(found); + } + for (bool found: matched) REQUIRE(found); + }; + SECTION("stream") { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(path, std::ios::binary); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + is, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + check(sketch); + } + SECTION("bytes") { + const auto bytes = read_binary_file(path); + const auto sketch = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + check(sketch); + } + } + + using aos_update = std::function; + + static array_of_strings make_array(std::initializer_list items) { + array_of_strings array(static_cast(items.size()), ""); + uint8_t i = 0; + for (const auto& item: items) array[i++] = item; + return array; + } + + // Updates the sketch with the same keys and values as Java's AosSketchCrossLanguageTest and + // requires it to retain exactly the hashes and summaries of the Java snapshot. + static void check_java_hash_compat( + const std::string& name, + update_array_of_strings_tuple_sketch<> sketch, + const std::function& updates + ) { + INFO(name); + updates([&sketch](const array_of_strings& key, const array_of_strings& values) { + sketch.update(hash_array_of_strings_key(key), values); + }); + const auto cpp_sketch = compact_array_of_strings_sketch(sketch); + + const auto bytes = read_binary_file(testBinaryInputPath + name + "_java.sk"); + const auto java_sketch = compact_array_of_strings_tuple_sketch<>::deserialize(bytes.data(), bytes.size()); + + REQUIRE(cpp_sketch.get_theta64() == java_sketch.get_theta64()); + REQUIRE(cpp_sketch.get_num_retained() == java_sketch.get_num_retained()); + std::map cpp_entries; + for (const auto& entry: cpp_sketch) cpp_entries.emplace(entry.first, entry.second); + for (const auto& entry: java_sketch) { + const auto it = cpp_entries.find(entry.first); + REQUIRE(it != cpp_entries.end()); + REQUIRE(it->second == entry.second); + } + } + + TEST_CASE("aos sketch java hash compat", "[serde_compat]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + const std::string sfx = "_n" + std::to_string(n); + check_java_hash_compat("aos_1" + sfx, update_array_of_strings_tuple_sketch<>::builder().build(), + [n](const aos_update& update) { + for (unsigned i = 0; i < n; ++i) { + update(make_array({std::to_string(i)}), make_array({"value" + std::to_string(i)})); + } + }); + check_java_hash_compat("aos_3" + sfx, update_array_of_strings_tuple_sketch<>::builder().build(), + [n](const aos_update& update) { + for (unsigned i = 0; i < n; ++i) { + const std::string s = std::to_string(i); + update(make_array({s}), make_array({"a" + s, "b" + s, "c" + s})); + } + }); + check_java_hash_compat("aos_multikey" + sfx, update_array_of_strings_tuple_sketch<>::builder().build(), + [n](const aos_update& update) { + for (unsigned i = 0; i < n; ++i) { + update( + make_array({"key" + std::to_string(i), "subkey" + std::to_string(i % 10)}), + make_array({"value" + std::to_string(i)}) + ); + } + }); + } + + check_java_hash_compat("aos_1_non_empty_no_entries", + update_array_of_strings_tuple_sketch<>::builder() + .set_lg_k(12) + .set_resize_factor(resize_factor::X8) + .set_p(0.01f) + .build(), + [](const aos_update& update) { + update(make_array({"key1"}), make_array({"value1"})); + }); + + check_java_hash_compat("aos_unicode", update_array_of_strings_tuple_sketch<>::builder().build(), + [](const aos_update& update) { + update(make_array({"키", "열쇠"}), make_array({"밸류", "값"})); + // "🗝️" is U+1F5DD followed by the variation selector U+FE0F, as in Java + update(make_array({"🔑", "🗝️"}), make_array({"📦", "🎁"})); + update(make_array({"ключ1", "ключ2"}), make_array({"ценить1", "ценить2"})); + }); + + check_java_hash_compat("aos_empty_strings", update_array_of_strings_tuple_sketch<>::builder().build(), + [](const aos_update& update) { + update(make_array({""}), make_array({"empty_key_value"})); + update(make_array({"empty_value_key"}), make_array({""})); + update(make_array({"", ""}), make_array({"", ""})); + }); + } +} diff --git a/tuple/test/aos_sketch_serialize_for_java.cpp b/tuple/test/aos_sketch_serialize_for_java.cpp new file mode 100644 index 00000000..c6eb0dfc --- /dev/null +++ b/tuple/test/aos_sketch_serialize_for_java.cpp @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include + +#include "array_of_strings_sketch.hpp" + +namespace datasketches { + +using aos_sketch = update_array_of_strings_tuple_sketch<>; +using array_of_strings = array; + +static array_of_strings make_array(std::initializer_list items) { + array_of_strings array(static_cast(items.size()), ""); + size_t i = 0; + for (const auto& item: items) { + array[static_cast(i)] = item; + ++i; + } + return array; +} + +TEST_CASE("aos sketch generate one value", "[serialize_for_java]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + auto sketch = aos_sketch::builder().build(); + for (unsigned i = 0; i < n; ++i) { + array_of_strings key(1, ""); + key[0] = std::to_string(i); + array_of_strings value(1, ""); + value[0] = "value" + std::to_string(i); + sketch.update(hash_array_of_strings_key(key), value); + } + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + std::ofstream os("aos_1_n" + std::to_string(n) + "_cpp.sk", std::ios::binary); + compact_array_of_strings_sketch(sketch).serialize(os, default_array_of_strings_serde<>()); + } +} + +TEST_CASE("aos sketch generate three values", "[serialize_for_java]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + auto sketch = aos_sketch::builder().build(); + for (unsigned i = 0; i < n; ++i) { + array_of_strings key(1, ""); + key[0] = std::to_string(i); + array_of_strings value(3, ""); + value[0] = "a" + std::to_string(i); + value[1] = "b" + std::to_string(i); + value[2] = "c" + std::to_string(i); + sketch.update(hash_array_of_strings_key(key), value); + } + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + std::ofstream os("aos_3_n" + std::to_string(n) + "_cpp.sk", std::ios::binary); + compact_array_of_strings_sketch(sketch).serialize(os, default_array_of_strings_serde<>()); + } +} + +TEST_CASE("aos sketch generate non-empty no entries", "[serialize_for_java]") { + auto sketch = aos_sketch::builder() + .set_lg_k(12) + .set_resize_factor(resize_factor::X8) + .set_p(0.01f) + .build(); + array_of_strings key(1, ""); + key[0] = "key1"; + array_of_strings value(1, ""); + value[0] = "value1"; + sketch.update(hash_array_of_strings_key(key), value); + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE(sketch.get_num_retained() == 0); + std::ofstream os("aos_1_non_empty_no_entries_cpp.sk", std::ios::binary); + compact_array_of_strings_sketch(sketch).serialize(os, default_array_of_strings_serde<>()); +} + +TEST_CASE("aos sketch generate multi key strings", "[serialize_for_java]") { + const unsigned n_arr[] = {0, 1, 10, 100, 1000, 10000, 100000, 1000000}; + for (const unsigned n: n_arr) { + auto sketch = aos_sketch::builder().build(); + for (unsigned i = 0; i < n; ++i) { + array_of_strings key(2, ""); + key[0] = "key" + std::to_string(i); + key[1] = "subkey" + std::to_string(i % 10); + array_of_strings value(1, ""); + value[0] = "value" + std::to_string(i); + sketch.update(hash_array_of_strings_key(key), value); + } + REQUIRE(sketch.is_empty() == (n == 0)); + REQUIRE(sketch.get_estimate() == Approx(n).margin(n * 0.03)); + std::ofstream os("aos_multikey_n" + std::to_string(n) + "_cpp.sk", std::ios::binary); + compact_array_of_strings_sketch(sketch).serialize(os, default_array_of_strings_serde<>()); + } +} + +TEST_CASE("aos sketch generate unicode strings", "[serialize_for_java]") { + auto sketch = aos_sketch::builder().build(); + sketch.update( + hash_array_of_strings_key(make_array({u8"키", u8"열쇠"})), + make_array({u8"밸류", u8"값"}) + ); + sketch.update( + hash_array_of_strings_key(make_array({u8"🔑", u8"🗝️"})), + make_array({u8"📦", u8"🎁"}) + ); + sketch.update( + hash_array_of_strings_key(make_array({u8"ключ1", u8"ключ2"})), + make_array({u8"ценить1", u8"ценить2"}) + ); + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE(sketch.get_num_retained() == 3); + std::ofstream os("aos_unicode_cpp.sk", std::ios::binary); + compact_array_of_strings_sketch(sketch).serialize(os, default_array_of_strings_serde<>()); +} + +TEST_CASE("aos sketch generate empty strings", "[serialize_for_java]") { + auto sketch = aos_sketch::builder().build(); + sketch.update(hash_array_of_strings_key(make_array({""})), make_array({"empty_key_value"})); + sketch.update(hash_array_of_strings_key(make_array({"empty_value_key"})), make_array({""})); + sketch.update(hash_array_of_strings_key(make_array({"", ""})), make_array({"", ""})); + REQUIRE_FALSE(sketch.is_empty()); + REQUIRE(sketch.get_num_retained() == 3); + std::ofstream os("aos_empty_strings_cpp.sk", std::ios::binary); + compact_array_of_strings_sketch(sketch).serialize(os, default_array_of_strings_serde<>()); +} + +} /* namespace datasketches */ diff --git a/tuple/test/array_of_strings_sketch_test.cpp b/tuple/test/array_of_strings_sketch_test.cpp new file mode 100644 index 00000000..0347a857 --- /dev/null +++ b/tuple/test/array_of_strings_sketch_test.cpp @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include + +#include + +#include "array_of_strings_sketch.hpp" + +namespace datasketches { + +TEST_CASE("aos update policy", "[tuple_sketch]") { + default_array_of_strings_update_policy policy; + + SECTION("create empty") { + auto values = policy.create(); + REQUIRE(values.size() == 0); + } + + SECTION("replace array") { + auto values = policy.create(); + + array_of_strings input(2, "", std::allocator()); + input[0] = "alpha"; + input[1] = "beta"; + policy.update(values, input); + REQUIRE(values.size() == 2); + REQUIRE(values[0] == "alpha"); + REQUIRE(values[1] == "beta"); + input[0] = "changed"; + REQUIRE(values[0] == "alpha"); + + array_of_strings input2(1, "", std::allocator()); + input2[0] = "gamma"; + policy.update(values, input2); + REQUIRE(values.size() == 1); + REQUIRE(values[0] == "gamma"); + } + + SECTION("nullptr clears") { + array_of_strings values(2, "", std::allocator()); + values[0] = "one"; + values[1] = "two"; + + policy.update(values, nullptr); + REQUIRE(values.size() == 0); + } + + SECTION("pointer input copies") { + auto values = policy.create(); + + array_of_strings input(2, "", std::allocator()); + input[0] = "first"; + input[1] = "second"; + policy.update(values, &input); + REQUIRE(values.size() == 2); + REQUIRE(values[1] == "second"); + input[1] = "changed"; + REQUIRE(values[1] == "second"); + } +} + +TEST_CASE("aos sketch update", "[tuple_sketch]") { + auto make_array = [](std::initializer_list entries) { + array_of_strings array(static_cast(entries.size()), "", std::allocator()); + uint8_t i = 0; + for (const auto* entry: entries) array[i++] = entry; + return array; + }; + + SECTION("same key replaces summary") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + + sketch.update( + hash_array_of_strings_key(make_array({"alpha", "beta"})), + make_array({"first"}) + ); + sketch.update( + hash_array_of_strings_key(make_array({"alpha", "beta"})), + make_array({"second", "third"}) + ); + + REQUIRE(sketch.get_num_retained() == 1); + + auto it = sketch.begin(); + REQUIRE(it != sketch.end()); + REQUIRE(it->second.size() == 2); + REQUIRE(it->second[0] == "second"); + REQUIRE(it->second[1] == "third"); + } + + SECTION("distinct keys retain multiple entries") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + + sketch.update( + hash_array_of_strings_key(make_array({"a", "bc"})), + make_array({"one"}) + ); + sketch.update( + hash_array_of_strings_key(make_array({"ab", "c"})), + make_array({"two"}) + ); + + REQUIRE(sketch.get_num_retained() == 2); + + bool saw_one = false; + bool saw_two = false; + for (const auto& entry: sketch) { + REQUIRE(entry.second.size() == 1); + if (entry.second[0] == "one") saw_one = true; + if (entry.second[0] == "two") saw_two = true; + } + REQUIRE(saw_one); + REQUIRE(saw_two); + } + + SECTION("empty key") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + + sketch.update(hash_array_of_strings_key(make_array({})), make_array({"value"})); + REQUIRE(sketch.get_num_retained() == 1); + + auto it = sketch.begin(); + REQUIRE(it != sketch.end()); + REQUIRE(it->second.size() == 1); + REQUIRE(it->second[0] == "value"); + } +} + +static array_of_strings to_array(const std::vector& items) { + array_of_strings array(static_cast(items.size()), ""); + for (size_t i = 0; i < items.size(); ++i) array[static_cast(i)] = items[i]; + return array; +} + +// Java's Util.stringArrHash: XXH64 with seed 0x7A3CCA71 over the UTF-16LE code units of the joined key +static uint64_t java_key_hash(const std::vector& code_units) { + std::vector bytes; + for (const uint16_t unit: code_units) { + bytes.push_back(static_cast(unit)); + bytes.push_back(static_cast(unit >> 8)); + } + return XXHash64::hash(bytes.data(), bytes.size(), 0x7A3CCA71); +} + +TEST_CASE("aos key hash: UTF-16LE as in Java", "[tuple_sketch]") { + struct test_case { + const char* name; + std::vector key; + std::vector code_units; + }; + const std::vector test_cases = { + {"empty array", {}, {}}, + {"empty string", {""}, {}}, + {"ascii with separator", {"a", "b"}, {0x61, 0x2C, 0x62}}, + {"bmp", {"\xED\x82\xA4"}, {0xD0A4}}, // U+D0A4 + {"surrogate pair", {"\xF0\x9F\x94\x91"}, {0xD83D, 0xDD11}}, // U+1F511 + {"mixed with trailing empty element", {"a", "\xF0\x9F\x94\x91", ""}, {0x61, 0x2C, 0xD83D, 0xDD11, 0x2C}}, + {"last code point before surrogates", {"\xED\x9F\xBF"}, {0xD7FF}}, + {"first code point after surrogates", {"\xEE\x80\x80"}, {0xE000}}, + {"replacement character", {"\xEF\xBF\xBD"}, {0xFFFD}}, + {"first supplementary code point", {"\xF0\x90\x80\x80"}, {0xD800, 0xDC00}}, + {"last code point", {"\xF4\x8F\xBF\xBF"}, {0xDBFF, 0xDFFF}}, + }; + for (const auto& test_case: test_cases) { + INFO(test_case.name); + REQUIRE(hash_array_of_strings_key(to_array(test_case.key)) == java_key_hash(test_case.code_units)); + } + + SECTION("longer than the internal buffer") { + // the surrogate pair of U+1F511 straddles the 256-byte buffer boundary + const std::string str = std::string(127, 'a') + "\xF0\x9F\x94\x91" + std::string(200, 'b'); + std::vector code_units(127, 0x61); + code_units.push_back(0xD83D); + code_units.push_back(0xDD11); + code_units.insert(code_units.end(), 200, 0x62); + REQUIRE(hash_array_of_strings_key(to_array({str})) == java_key_hash(code_units)); + } +} + +TEST_CASE("aos key hash: invalid UTF-8 replaced as in Java", "[tuple_sketch]") { + // expected code units are what new String(bytes, StandardCharsets.UTF_8) gives in Java 11-26 + struct test_case { + const char* name; + std::string str; + std::vector code_units; + }; + const std::vector test_cases = { + {"invalid lead byte", "\xFF", {0xFFFD}}, + {"lone continuation byte", "\x80", {0xFFFD}}, + {"overlong 2-byte", "\xC0\x80", {0xFFFD, 0xFFFD}}, + {"truncated 3-byte at end", "\xE2\x82", {0xFFFD}}, + {"truncated 3-byte", "\xE2\x82" "A", {0xFFFD, 0x41}}, + {"encoded surrogate", "\xED\xA0\x80", {0xFFFD}}, + {"truncated encoded surrogate", "\xED\xA0" "A", {0xFFFD, 0x41}}, + {"overlong 3-byte", "\xE0\x80\x80", {0xFFFD, 0xFFFD, 0xFFFD}}, + {"truncated 4-byte", "\xF0\x9F\x94" "A", {0xFFFD, 0x41}}, + {"above U+10FFFF", "\xF4\x90\x80\x80", {0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD}}, + {"5-byte form", "\xF8\x88\x80\x80\x80", {0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD}}, + {"between valid characters", "a\xE2\x82" "b", {0x61, 0xFFFD, 0x62}}, + }; + for (const auto& test_case: test_cases) { + INFO(test_case.name); + REQUIRE(hash_array_of_strings_key(to_array({test_case.str})) == java_key_hash(test_case.code_units)); + } +} + +TEST_CASE("aos key hash: elements joined with commas", "[tuple_sketch]") { + // elements are joined without escaping, as in Java + REQUIRE(hash_array_of_strings_key(to_array({"a,b"})) == hash_array_of_strings_key(to_array({"a", "b"}))); + REQUIRE(hash_array_of_strings_key(to_array({})) == hash_array_of_strings_key(to_array({""}))); + REQUIRE(hash_array_of_strings_key(to_array({"a", "b"})) != hash_array_of_strings_key(to_array({"b", "a"}))); +} + +TEST_CASE("aos sketch: serialize deserialize", "[tuple_sketch]") { + auto make_array = [](std::initializer_list entries) { + array_of_strings array(static_cast(entries.size()), "", std::allocator()); + uint8_t i = 0; + for (const auto& entry: entries) array[i++] = entry; + return array; + }; + + auto collect_entries = [](const compact_array_of_strings_tuple_sketch<>& sketch) { + typedef std::pair entry_type; + std::vector entries; + for (const auto& entry: sketch) entries.push_back(entry); + struct entry_less { + bool operator()(const entry_type& lhs, const entry_type& rhs) const { + return lhs.first < rhs.first; + } + }; + std::sort(entries.begin(), entries.end(), entry_less()); + return entries; + }; + + auto check_round_trip = [&](const compact_array_of_strings_tuple_sketch<>& compact_sketch) { + std::stringstream ss; + ss.exceptions(std::ios::failbit | std::ios::badbit); + compact_sketch.serialize(ss, default_array_of_strings_serde<>()); + auto deserialized_stream = compact_array_of_strings_tuple_sketch<>::deserialize( + ss, DEFAULT_SEED, default_array_of_strings_serde<>() + ); + + auto bytes = compact_sketch.serialize(0, default_array_of_strings_serde<>()); + auto deserialized_bytes = compact_array_of_strings_tuple_sketch<>::deserialize( + bytes.data(), bytes.size(), DEFAULT_SEED, default_array_of_strings_serde<>() + ); + + const compact_array_of_strings_tuple_sketch<>* deserialized_list[2] = { + &deserialized_stream, + &deserialized_bytes + }; + for (int list_index = 0; list_index < 2; ++list_index) { + const compact_array_of_strings_tuple_sketch<>* deserialized = deserialized_list[list_index]; + REQUIRE(compact_sketch.is_empty() == deserialized->is_empty()); + REQUIRE(compact_sketch.is_estimation_mode() == deserialized->is_estimation_mode()); + REQUIRE(compact_sketch.is_ordered() == deserialized->is_ordered()); + REQUIRE(compact_sketch.get_num_retained() == deserialized->get_num_retained()); + REQUIRE(compact_sketch.get_theta() == Approx(deserialized->get_theta()).margin(1e-10)); + REQUIRE(compact_sketch.get_estimate() == Approx(deserialized->get_estimate()).margin(1e-10)); + REQUIRE(compact_sketch.get_lower_bound(1) == Approx(deserialized->get_lower_bound(1)).margin(1e-10)); + REQUIRE(compact_sketch.get_upper_bound(1) == Approx(deserialized->get_upper_bound(1)).margin(1e-10)); + + auto original_entries = collect_entries(compact_sketch); + auto round_trip_entries = collect_entries(*deserialized); + REQUIRE(original_entries.size() == round_trip_entries.size()); + for (size_t i = 0; i < original_entries.size(); ++i) { + REQUIRE(original_entries[i].first == round_trip_entries[i].first); + REQUIRE(original_entries[i].second.size() == round_trip_entries[i].second.size()); + for (size_t j = 0; j < original_entries[i].second.size(); ++j) { + REQUIRE(original_entries[i].second[static_cast(j)] == + round_trip_entries[i].second[static_cast(j)]); + } + } + } + }; + + auto run_tests = [&](const update_array_of_strings_tuple_sketch<>& sketch) { + auto ordered = compact_array_of_strings_sketch(sketch, true); + auto unordered = compact_array_of_strings_sketch(sketch, false); + check_round_trip(ordered); + check_round_trip(unordered); + }; + + SECTION("empty sketch") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + run_tests(sketch); + } + + SECTION("single entry sketch") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + sketch.update(hash_array_of_strings_key(make_array({"key"})), make_array({"value"})); + run_tests(sketch); + } + + SECTION("multiple entries exact mode") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().set_lg_k(8).build(); + for (int i = 0; i < 50; ++i) { + sketch.update( + hash_array_of_strings_key(make_array({std::string("key-") + std::to_string(i)})), + make_array({std::string("value-") + std::to_string(i), "extra"}) + ); + } + REQUIRE_FALSE(sketch.is_estimation_mode()); + run_tests(sketch); + } + + SECTION("multiple entries estimation mode") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + for (int i = 0; i < 10000; ++i) { + sketch.update( + hash_array_of_strings_key(make_array({std::string("key-") + std::to_string(i)})), + make_array({std::string("value-") + std::to_string(i)}) + ); + } + REQUIRE(sketch.is_estimation_mode()); + run_tests(sketch); + } +} + +TEST_CASE("aos sketch: serialize with default serde", "[tuple_sketch]") { + auto sketch = update_array_of_strings_tuple_sketch<>::builder().build(); + sketch.update(hash_array_of_strings_key(to_array({"key"})), to_array({"value1", "value2"})); + const auto compact_sketch = compact_array_of_strings_sketch(sketch); + + std::stringstream ss; + ss.exceptions(std::ios::failbit | std::ios::badbit); + compact_sketch.serialize(ss); + const auto bytes = compact_sketch.serialize(); + REQUIRE(bytes.size() == ss.str().size()); + + const auto from_stream = compact_array_of_strings_tuple_sketch<>::deserialize(ss); + const auto from_bytes = compact_array_of_strings_tuple_sketch<>::deserialize(bytes.data(), bytes.size()); + for (const auto* deserialized: {&from_stream, &from_bytes}) { + REQUIRE(deserialized->get_num_retained() == 1); + const auto& entry = *deserialized->begin(); + REQUIRE(entry.first == compact_sketch.begin()->first); + REQUIRE(entry.second == to_array({"value1", "value2"})); + } +} + +TEST_CASE("aos serde validation", "[tuple_sketch]") { + default_array_of_strings_serde<> serde; + + SECTION("too many nodes rejected") { + array_of_strings array(128, "", std::allocator()); + std::stringstream ss; + ss.exceptions(std::ios::failbit | std::ios::badbit); + REQUIRE_THROWS_WITH( + serde.serialize(ss, &array, 1), + Catch::Matchers::Contains("size exceeds 127") + ); + } +} + +} /* namespace datasketches */