diff --git a/common/test/CMakeLists.txt b/common/test/CMakeLists.txt index d7b89838..6db1fc3b 100644 --- a/common/test/CMakeLists.txt +++ b/common/test/CMakeLists.txt @@ -46,6 +46,7 @@ target_sources(common_test_lib INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/test_allocator.hpp ${CMAKE_CURRENT_SOURCE_DIR}/test_type.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/undersized_serde.hpp PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/catch_runner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_allocator.cpp diff --git a/common/test/undersized_serde.hpp b/common/test/undersized_serde.hpp new file mode 100644 index 00000000..e0806e09 --- /dev/null +++ b/common/test/undersized_serde.hpp @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef UNDERSIZED_SERDE_HPP_ +#define UNDERSIZED_SERDE_HPP_ + +#include + +#include "serde.hpp" + +namespace datasketches { + +// Reports each string as one byte shorter than it serializes it, so a sketch that serializes to bytes +// allocates too little. The capacity check of the serde must then throw std::out_of_range, +// which it can only do if the sketch passes it the true end of the buffer. +struct undersized_string_serde: serde { + size_t size_of_item(const std::string& item) const { + return serde::size_of_item(item) - 1; + } +}; + +} /* namespace datasketches */ + +#endif diff --git a/density/include/density_sketch_impl.hpp b/density/include/density_sketch_impl.hpp index 019b0bf6..1e9113d1 100755 --- a/density/include/density_sketch_impl.hpp +++ b/density/include/density_sketch_impl.hpp @@ -226,7 +226,7 @@ auto density_sketch::serialize(unsigned header_size_bytes) const -> vec vector_bytes bytes(size, 0, levels_.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; ptr += copy_to_mem(preamble_ints, ptr); const uint8_t ser_ver = SERIAL_VERSION; diff --git a/density/test/density_sketch_test.cpp b/density/test/density_sketch_test.cpp index 6fd88c9e..9837b65e 100755 --- a/density/test/density_sketch_test.cpp +++ b/density/test/density_sketch_test.cpp @@ -480,4 +480,22 @@ TEST_CASE("density sketch: reject invalid family and ebpps headers", "[density_s } } +TEST_CASE("density sketch: serialize with a header", "[density_sketch]") { + // serialize() used to check its output against an end that included the header twice, so it threw + for (int n: {0, 1, 1000}) { + CAPTURE(n); + density_sketch sketch(10, 3); + for (int i = 0; i < n; ++i) { + const float x = static_cast(i); + sketch.update(std::vector({x, x, x})); + } + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + REQUIRE(bytes.size() == header_size + sketch.serialize().size()); + const auto copy = density_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + } +} + } /* namespace datasketches */ diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp index 78df6033..ae77ae77 100644 --- a/fi/include/frequent_items_sketch_impl.hpp +++ b/fi/include/frequent_items_sketch_impl.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include "memory_operations.hpp" @@ -197,22 +198,22 @@ void frequent_items_sketch::serialize(std::ostream& os, const Ser write(os, total_weight); write(os, offset); - // copy active items and their weights to use batch serialization - using AllocW = typename std::allocator_traits::template rebind_alloc; - AllocW aw(map.get_allocator()); - W* weights = aw.allocate(num_items); - A alloc(map.get_allocator()); - T* items = alloc.allocate(num_items); - uint32_t i = 0; - for (auto it: map) { - new (&items[i]) T(it.first); - weights[i++] = it.second; + // copy active items and their weights to use batch serialization; + // the vectors release the copies if a copy or the serde throws. + // a non-empty sketch can have no active items (e.g. after a purge) + if (num_items > 0) { + using AllocW = typename std::allocator_traits::template rebind_alloc; + std::vector weights(map.get_allocator()); + weights.reserve(num_items); + std::vector items(map.get_allocator()); + items.reserve(num_items); + for (auto it: map) { + items.push_back(it.first); + weights.push_back(it.second); + } + write(os, weights.data(), sizeof(W) * num_items); + sd.serialize(os, items.data(), num_items); } - write(os, weights, sizeof(W) * num_items); - aw.deallocate(weights, num_items); - sd.serialize(os, items, num_items); - for (i = 0; i < num_items; i++) items[i].~T(); - alloc.deallocate(items, num_items); } } @@ -231,7 +232,7 @@ auto frequent_items_sketch::serialize(unsigned header_size_bytes, const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, map.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; - uint8_t* end_ptr = ptr + size; + uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY; ptr += copy_to_mem(preamble_longs, ptr); @@ -256,23 +257,23 @@ auto frequent_items_sketch::serialize(unsigned header_size_bytes, ptr += copy_to_mem(total_weight, ptr); ptr += copy_to_mem(offset, ptr); - // copy active items and their weights to use batch serialization - using AllocW = typename std::allocator_traits::template rebind_alloc; - AllocW aw(map.get_allocator()); - W* weights = aw.allocate(num_items); - A alloc(map.get_allocator()); - T* items = alloc.allocate(num_items); - uint32_t i = 0; - for (auto it: map) { - new (&items[i]) T(it.first); - weights[i++] = it.second; + // copy active items and their weights to use batch serialization; + // the vectors release the copies if a copy or the serde throws. + // a non-empty sketch can have no active items (e.g. after a purge) + if (num_items > 0) { + using AllocW = typename std::allocator_traits::template rebind_alloc; + std::vector weights(map.get_allocator()); + weights.reserve(num_items); + std::vector items(map.get_allocator()); + items.reserve(num_items); + for (auto it: map) { + items.push_back(it.first); + weights.push_back(it.second); + } + ptr += copy_to_mem(weights.data(), ptr, sizeof(W) * num_items); + const size_t bytes_remaining = end_ptr - ptr; + ptr += sd.serialize(ptr, bytes_remaining, items.data(), num_items); } - ptr += copy_to_mem(weights, ptr, sizeof(W) * num_items); - aw.deallocate(weights, num_items); - const size_t bytes_remaining = end_ptr - ptr; - ptr += sd.serialize(ptr, bytes_remaining, items, num_items); - for (i = 0; i < num_items; i++) items[i].~T(); - alloc.deallocate(items, num_items); } return bytes; } diff --git a/fi/test/frequent_items_sketch_custom_type_test.cpp b/fi/test/frequent_items_sketch_custom_type_test.cpp index 85e0d6ba..eb5ec777 100644 --- a/fi/test/frequent_items_sketch_custom_type_test.cpp +++ b/fi/test/frequent_items_sketch_custom_type_test.cpp @@ -129,4 +129,33 @@ TEST_CASE("frequent items: a purge does not allocate", "[frequent_items_sketch]" REQUIRE(s.get_total_weight() == 100000); } +namespace { + +// a serde whose serialize() always throws +struct throwing_test_type_serde: test_type_serde { + void serialize(std::ostream&, const test_type*, unsigned) const { + throw std::runtime_error("serialize failed"); + } + size_t serialize(void*, size_t, const test_type*, unsigned) const { + throw std::runtime_error("serialize failed"); + } +}; + +} // namespace + +TEST_CASE("frequent items: a serde that throws leaks nothing", "[frequent_items_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + frequent_test_type_sketch sketch(3, frequent_test_type_sketch::LG_MIN_MAP_SIZE, test_type_equal(), 0); + for (int i = 0; i < 10; ++i) sketch.update(i); + // serialize() copies the items for the serde, and must release the copies when the serde throws + REQUIRE_THROWS_AS(sketch.serialize(0, throwing_test_type_serde()), std::runtime_error); + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + REQUIRE_THROWS_AS(sketch.serialize(s, throwing_test_type_serde()), std::runtime_error); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + } /* namespace datasketches */ diff --git a/fi/test/frequent_items_sketch_test.cpp b/fi/test/frequent_items_sketch_test.cpp index 9e5bef17..b2035439 100644 --- a/fi/test/frequent_items_sketch_test.cpp +++ b/fi/test/frequent_items_sketch_test.cpp @@ -18,11 +18,13 @@ */ #include +#include #include #include #include #include "frequent_items_sketch.hpp" +#include #ifdef TEST_BINARY_INPUT_PATH static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH; @@ -435,4 +437,19 @@ TEST_CASE("frequent items: string deserialize single item buffer overrun", "[fre REQUIRE_THROWS_AS(frequent_items_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range); } +TEST_CASE("frequent items sketch: serialize with a header", "[frequent_items_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + frequent_items_sketch sketch(3); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = frequent_items_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_total_weight() == sketch.get_total_weight()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } /* namespace datasketches */ diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index e1e8e57b..397bfbff 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -550,7 +550,7 @@ auto kll_sketch::serialize(unsigned header_size_bytes, const SerDe& sd) const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL); ptr += copy_to_mem(preamble_ints, ptr); const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1); diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index eaec784e..85b24727 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include +#include namespace datasketches { @@ -1584,4 +1586,19 @@ TEST_CASE("kll sketch: compaction and merge use the sketch's comparator", "[kll_ REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(RANK_EPS_FOR_K_200 * 10)); } +TEST_CASE("kll sketch: serialize with a header", "[kll_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + kll_sketch sketch(200); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = kll_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } /* namespace datasketches */ diff --git a/quantiles/include/quantiles_sketch_impl.hpp b/quantiles/include/quantiles_sketch_impl.hpp index 6b28f56b..65e0e032 100644 --- a/quantiles/include/quantiles_sketch_impl.hpp +++ b/quantiles/include/quantiles_sketch_impl.hpp @@ -353,7 +353,7 @@ auto quantiles_sketch::serialize(unsigned header_size_bytes, const SerD const size_t size = get_serialized_size_bytes(serde) + header_size_bytes; vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL; ptr += copy_to_mem(preamble_longs, ptr); diff --git a/quantiles/test/quantiles_sketch_test.cpp b/quantiles/test/quantiles_sketch_test.cpp index 0a23a7c7..627332ee 100644 --- a/quantiles/test/quantiles_sketch_test.cpp +++ b/quantiles/test/quantiles_sketch_test.cpp @@ -18,6 +18,8 @@ */ #include +#include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#include namespace datasketches { @@ -957,6 +960,21 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") { } +TEST_CASE("quantiles sketch: serialize with a header", "[quantiles_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + quantiles_sketch sketch(128); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = quantiles_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + TEST_CASE("quantiles sketch: deserialize serial version 1 truncated after min and max", "[quantiles_sketch]") { quantiles_sketch sketch(16); for (int i = 0; i < 1000; ++i) sketch.update(i); diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index d6f4c8e4..8d121872 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -354,8 +354,8 @@ size_t req_sketch::get_serialized_size_bytes(const SerDe& sd) const { if (is_estimation_mode()) { size += sizeof(n_) + sizeof(TT) * 2; // min and max } - if (n_ == 1) { - size += sizeof(TT); + if (n_ <= req_constants::MIN_K) { // raw items, without a compactor header + size += sizeof(TT) * static_cast(n_); } else { for (const auto& compactor: compactors_) size += compactor.get_serialized_size_bytes(sd); } @@ -373,8 +373,8 @@ size_t req_sketch::get_serialized_size_bytes(const SerDe& sd) const { size += sd.size_of_item(*min_item_); size += sd.size_of_item(*max_item_); } - if (n_ == 1) { - size += sd.size_of_item(*compactors_[0].begin()); + if (n_ <= req_constants::MIN_K) { // raw items, without a compactor header + for (auto it = compactors_[0].begin(); it != compactors_[0].end(); ++it) size += sd.size_of_item(*it); } else { for (const auto& compactor: compactors_) size += compactor.get_serialized_size_bytes(sd); } @@ -422,7 +422,7 @@ auto req_sketch::serialize(unsigned header_size_bytes, const SerDe& sd) const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_ints = is_estimation_mode() ? 4 : 2; ptr += copy_to_mem(preamble_ints, ptr); diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 3db56247..c706801c 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -444,6 +445,42 @@ TEST_CASE("req sketch: serialize deserialize stream and bytes equivalence", "[re REQUIRE(sketch2.get_max_item() == sketch.get_max_item()); } +TEST_CASE("req sketch: serialized size of raw items", "[req_sketch]") { + // up to 4 items are written without a compactor header, to bytes as to a stream + for (int n = 1; n <= 6; ++n) { + CAPTURE(n); + req_sketch floats(12); + req_sketch strings(12); + for (int i = 0; i < n; ++i) { + floats.update(static_cast(i)); + strings.update(std::to_string(i)); + } + std::stringstream floats_stream(std::ios::in | std::ios::out | std::ios::binary); + floats.serialize(floats_stream); + REQUIRE(floats.serialize().size() == floats_stream.str().size()); + REQUIRE(floats.get_serialized_size_bytes() == floats_stream.str().size()); + std::stringstream strings_stream(std::ios::in | std::ios::out | std::ios::binary); + strings.serialize(strings_stream); + REQUIRE(strings.serialize().size() == strings_stream.str().size()); + REQUIRE(strings.get_serialized_size_bytes() == strings_stream.str().size()); + } +} + +TEST_CASE("req sketch: serialize with a header", "[req_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + req_sketch sketch(12); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = req_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + TEST_CASE("req sketch: merge into empty", "[req_sketch]") { req_sketch sketch1(40); diff --git a/sampling/include/var_opt_sketch_impl.hpp b/sampling/include/var_opt_sketch_impl.hpp index 24ddd411..b660d75b 100644 --- a/sampling/include/var_opt_sketch_impl.hpp +++ b/sampling/include/var_opt_sketch_impl.hpp @@ -327,7 +327,7 @@ std::vector> var_opt_sketch::serialize(unsigned header const size_t size = header_size_bytes + get_serialized_size_bytes(sd); std::vector> bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - uint8_t* end_ptr = ptr + size; + uint8_t* end_ptr = bytes.data() + size; bool empty = is_empty(); uint8_t preLongs = (empty ? PREAMBLE_LONGS_EMPTY diff --git a/sampling/test/var_opt_sketch_test.cpp b/sampling/test/var_opt_sketch_test.cpp index c8dd5c18..472700e9 100644 --- a/sampling/test/var_opt_sketch_test.cpp +++ b/sampling/test/var_opt_sketch_test.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #ifdef TEST_BINARY_INPUT_PATH static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH; @@ -560,4 +561,19 @@ TEST_CASE("varopt sketch: update after deserialize in exact mode", "[var_opt_ske } } +TEST_CASE("varopt sketch: serialize with a header", "[var_opt_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + var_opt_sketch sketch(10); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = var_opt_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } diff --git a/tuple/include/tuple_sketch_impl.hpp b/tuple/include/tuple_sketch_impl.hpp index 0c64a3c8..025079fb 100644 --- a/tuple/include/tuple_sketch_impl.hpp +++ b/tuple/include/tuple_sketch_impl.hpp @@ -441,7 +441,7 @@ auto compact_tuple_sketch::serialize(unsigned header_size_bytes, const Ser + sizeof(uint64_t) * entries_.size() + get_serialized_size_summaries_bytes(sd); vector_bytes bytes(size, 0, entries_.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; ptr += copy_to_mem(preamble_longs, ptr); const uint8_t serial_version = SERIAL_VERSION; diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp index dd0c3f1d..15a85afe 100644 --- a/tuple/test/tuple_sketch_test.cpp +++ b/tuple/test/tuple_sketch_test.cpp @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -35,6 +36,7 @@ std::ostream& operator<<(std::ostream& os, const three_doubles& tuple) { #include #include +#include namespace datasketches { @@ -430,4 +432,20 @@ TEST_CASE("tuple sketch: deserialize bounds-checks each entry key", "[tuple_sket std::out_of_range); } +TEST_CASE("tuple sketch: serialize with a header", "[tuple_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + auto update_sketch = update_tuple_sketch::builder().build(); + for (int i = 0; i < n; ++i) update_sketch.update(i, std::to_string(i)); + const auto sketch = update_sketch.compact(); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = compact_tuple_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } /* namespace datasketches */