From 5ab7b88daa7c38e77f7cb13090b2cc7ad2de8e0e Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 10:26:00 -0800 Subject: [PATCH 1/3] Use the true end of the buffer in serialize() with a header serialize(header_size_bytes) set end_ptr to ptr + size, where ptr is already past the header and size includes it, so the end was header_size_bytes past the buffer. The same as #591 fixed for EBPPS, in var_opt, tuple, KLL, density, REQ, frequent items and classic quantiles: - The capacity checks of the serde used that end, so a serde that wrote more than size_of_item() reported could write past the buffer. - Density compared its output with that end and threw whenever a header size was given. REQ: get_serialized_size_bytes() counted a compactor header for 2 to 4 items, which are written as raw items without one, so serialize() to bytes returned 20 trailing zero bytes that the stream form doesn't have. Tests: serialize with a header and deserialize the bytes after it, and check that a serde that under-reports the size of its items is stopped by its capacity check. A test helper, undersized_string_serde, is added to common/test. Co-Authored-By: Claude Opus 5.5 --- common/test/CMakeLists.txt | 1 + common/test/undersized_serde.hpp | 40 +++++++++++++++++++++ density/include/density_sketch_impl.hpp | 2 +- density/test/density_sketch_test.cpp | 18 ++++++++++ fi/include/frequent_items_sketch_impl.hpp | 2 +- fi/test/frequent_items_sketch_test.cpp | 17 +++++++++ kll/include/kll_sketch_impl.hpp | 2 +- kll/test/kll_sketch_test.cpp | 17 +++++++++ quantiles/include/quantiles_sketch_impl.hpp | 2 +- quantiles/test/quantiles_sketch_test.cpp | 18 ++++++++++ req/include/req_sketch_impl.hpp | 10 +++--- req/test/req_sketch_test.cpp | 37 +++++++++++++++++++ sampling/include/var_opt_sketch_impl.hpp | 2 +- sampling/test/var_opt_sketch_test.cpp | 16 +++++++++ tuple/include/tuple_sketch_impl.hpp | 2 +- tuple/test/tuple_sketch_test.cpp | 18 ++++++++++ 16 files changed, 193 insertions(+), 11 deletions(-) create mode 100644 common/test/undersized_serde.hpp diff --git a/common/test/CMakeLists.txt b/common/test/CMakeLists.txt index d7b898385..6db1fc3b8 100644 --- a/common/test/CMakeLists.txt +++ b/common/test/CMakeLists.txt @@ -46,6 +46,7 @@ target_sources(common_test_lib INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/test_allocator.hpp ${CMAKE_CURRENT_SOURCE_DIR}/test_type.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/undersized_serde.hpp PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/catch_runner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_allocator.cpp diff --git a/common/test/undersized_serde.hpp b/common/test/undersized_serde.hpp new file mode 100644 index 000000000..e0806e099 --- /dev/null +++ b/common/test/undersized_serde.hpp @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef UNDERSIZED_SERDE_HPP_ +#define UNDERSIZED_SERDE_HPP_ + +#include + +#include "serde.hpp" + +namespace datasketches { + +// Reports each string as one byte shorter than it serializes it, so a sketch that serializes to bytes +// allocates too little. The capacity check of the serde must then throw std::out_of_range, +// which it can only do if the sketch passes it the true end of the buffer. +struct undersized_string_serde: serde { + size_t size_of_item(const std::string& item) const { + return serde::size_of_item(item) - 1; + } +}; + +} /* namespace datasketches */ + +#endif diff --git a/density/include/density_sketch_impl.hpp b/density/include/density_sketch_impl.hpp index 019b0bf64..1e9113d1e 100755 --- a/density/include/density_sketch_impl.hpp +++ b/density/include/density_sketch_impl.hpp @@ -226,7 +226,7 @@ auto density_sketch::serialize(unsigned header_size_bytes) const -> vec vector_bytes bytes(size, 0, levels_.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; ptr += copy_to_mem(preamble_ints, ptr); const uint8_t ser_ver = SERIAL_VERSION; diff --git a/density/test/density_sketch_test.cpp b/density/test/density_sketch_test.cpp index 6fd88c9e1..9837b65e4 100755 --- a/density/test/density_sketch_test.cpp +++ b/density/test/density_sketch_test.cpp @@ -480,4 +480,22 @@ TEST_CASE("density sketch: reject invalid family and ebpps headers", "[density_s } } +TEST_CASE("density sketch: serialize with a header", "[density_sketch]") { + // serialize() used to check its output against an end that included the header twice, so it threw + for (int n: {0, 1, 1000}) { + CAPTURE(n); + density_sketch sketch(10, 3); + for (int i = 0; i < n; ++i) { + const float x = static_cast(i); + sketch.update(std::vector({x, x, x})); + } + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + REQUIRE(bytes.size() == header_size + sketch.serialize().size()); + const auto copy = density_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + } +} + } /* namespace datasketches */ diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp index 78df6033f..770b05d27 100644 --- a/fi/include/frequent_items_sketch_impl.hpp +++ b/fi/include/frequent_items_sketch_impl.hpp @@ -231,7 +231,7 @@ auto frequent_items_sketch::serialize(unsigned header_size_bytes, const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, map.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; - uint8_t* end_ptr = ptr + size; + uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY; ptr += copy_to_mem(preamble_longs, ptr); diff --git a/fi/test/frequent_items_sketch_test.cpp b/fi/test/frequent_items_sketch_test.cpp index 9e5bef170..b2035439a 100644 --- a/fi/test/frequent_items_sketch_test.cpp +++ b/fi/test/frequent_items_sketch_test.cpp @@ -18,11 +18,13 @@ */ #include +#include #include #include #include #include "frequent_items_sketch.hpp" +#include #ifdef TEST_BINARY_INPUT_PATH static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH; @@ -435,4 +437,19 @@ TEST_CASE("frequent items: string deserialize single item buffer overrun", "[fre REQUIRE_THROWS_AS(frequent_items_sketch::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range); } +TEST_CASE("frequent items sketch: serialize with a header", "[frequent_items_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + frequent_items_sketch sketch(3); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = frequent_items_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_total_weight() == sketch.get_total_weight()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } /* namespace datasketches */ diff --git a/kll/include/kll_sketch_impl.hpp b/kll/include/kll_sketch_impl.hpp index e1e8e57b7..397bfbff7 100644 --- a/kll/include/kll_sketch_impl.hpp +++ b/kll/include/kll_sketch_impl.hpp @@ -550,7 +550,7 @@ auto kll_sketch::serialize(unsigned header_size_bytes, const SerDe& sd) const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL); ptr += copy_to_mem(preamble_ints, ptr); const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1); diff --git a/kll/test/kll_sketch_test.cpp b/kll/test/kll_sketch_test.cpp index eaec784e9..85b247270 100644 --- a/kll/test/kll_sketch_test.cpp +++ b/kll/test/kll_sketch_test.cpp @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include +#include namespace datasketches { @@ -1584,4 +1586,19 @@ TEST_CASE("kll sketch: compaction and merge use the sketch's comparator", "[kll_ REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(RANK_EPS_FOR_K_200 * 10)); } +TEST_CASE("kll sketch: serialize with a header", "[kll_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + kll_sketch sketch(200); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = kll_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } /* namespace datasketches */ diff --git a/quantiles/include/quantiles_sketch_impl.hpp b/quantiles/include/quantiles_sketch_impl.hpp index 6b28f56bf..65e0e0325 100644 --- a/quantiles/include/quantiles_sketch_impl.hpp +++ b/quantiles/include/quantiles_sketch_impl.hpp @@ -353,7 +353,7 @@ auto quantiles_sketch::serialize(unsigned header_size_bytes, const SerD const size_t size = get_serialized_size_bytes(serde) + header_size_bytes; vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL; ptr += copy_to_mem(preamble_longs, ptr); diff --git a/quantiles/test/quantiles_sketch_test.cpp b/quantiles/test/quantiles_sketch_test.cpp index 0a23a7c73..627332ee2 100644 --- a/quantiles/test/quantiles_sketch_test.cpp +++ b/quantiles/test/quantiles_sketch_test.cpp @@ -18,6 +18,8 @@ */ #include +#include +#include #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#include namespace datasketches { @@ -957,6 +960,21 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") { } +TEST_CASE("quantiles sketch: serialize with a header", "[quantiles_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + quantiles_sketch sketch(128); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = quantiles_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + TEST_CASE("quantiles sketch: deserialize serial version 1 truncated after min and max", "[quantiles_sketch]") { quantiles_sketch sketch(16); for (int i = 0; i < 1000; ++i) sketch.update(i); diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index d6f4c8e4f..8d1218723 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -354,8 +354,8 @@ size_t req_sketch::get_serialized_size_bytes(const SerDe& sd) const { if (is_estimation_mode()) { size += sizeof(n_) + sizeof(TT) * 2; // min and max } - if (n_ == 1) { - size += sizeof(TT); + if (n_ <= req_constants::MIN_K) { // raw items, without a compactor header + size += sizeof(TT) * static_cast(n_); } else { for (const auto& compactor: compactors_) size += compactor.get_serialized_size_bytes(sd); } @@ -373,8 +373,8 @@ size_t req_sketch::get_serialized_size_bytes(const SerDe& sd) const { size += sd.size_of_item(*min_item_); size += sd.size_of_item(*max_item_); } - if (n_ == 1) { - size += sd.size_of_item(*compactors_[0].begin()); + if (n_ <= req_constants::MIN_K) { // raw items, without a compactor header + for (auto it = compactors_[0].begin(); it != compactors_[0].end(); ++it) size += sd.size_of_item(*it); } else { for (const auto& compactor: compactors_) size += compactor.get_serialized_size_bytes(sd); } @@ -422,7 +422,7 @@ auto req_sketch::serialize(unsigned header_size_bytes, const SerDe& sd) const size_t size = header_size_bytes + get_serialized_size_bytes(sd); vector_bytes bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; const uint8_t preamble_ints = is_estimation_mode() ? 4 : 2; ptr += copy_to_mem(preamble_ints, ptr); diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 3db562473..c706801ca 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -444,6 +445,42 @@ TEST_CASE("req sketch: serialize deserialize stream and bytes equivalence", "[re REQUIRE(sketch2.get_max_item() == sketch.get_max_item()); } +TEST_CASE("req sketch: serialized size of raw items", "[req_sketch]") { + // up to 4 items are written without a compactor header, to bytes as to a stream + for (int n = 1; n <= 6; ++n) { + CAPTURE(n); + req_sketch floats(12); + req_sketch strings(12); + for (int i = 0; i < n; ++i) { + floats.update(static_cast(i)); + strings.update(std::to_string(i)); + } + std::stringstream floats_stream(std::ios::in | std::ios::out | std::ios::binary); + floats.serialize(floats_stream); + REQUIRE(floats.serialize().size() == floats_stream.str().size()); + REQUIRE(floats.get_serialized_size_bytes() == floats_stream.str().size()); + std::stringstream strings_stream(std::ios::in | std::ios::out | std::ios::binary); + strings.serialize(strings_stream); + REQUIRE(strings.serialize().size() == strings_stream.str().size()); + REQUIRE(strings.get_serialized_size_bytes() == strings_stream.str().size()); + } +} + +TEST_CASE("req sketch: serialize with a header", "[req_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + req_sketch sketch(12); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = req_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + TEST_CASE("req sketch: merge into empty", "[req_sketch]") { req_sketch sketch1(40); diff --git a/sampling/include/var_opt_sketch_impl.hpp b/sampling/include/var_opt_sketch_impl.hpp index 24ddd4116..b660d75b3 100644 --- a/sampling/include/var_opt_sketch_impl.hpp +++ b/sampling/include/var_opt_sketch_impl.hpp @@ -327,7 +327,7 @@ std::vector> var_opt_sketch::serialize(unsigned header const size_t size = header_size_bytes + get_serialized_size_bytes(sd); std::vector> bytes(size, 0, allocator_); uint8_t* ptr = bytes.data() + header_size_bytes; - uint8_t* end_ptr = ptr + size; + uint8_t* end_ptr = bytes.data() + size; bool empty = is_empty(); uint8_t preLongs = (empty ? PREAMBLE_LONGS_EMPTY diff --git a/sampling/test/var_opt_sketch_test.cpp b/sampling/test/var_opt_sketch_test.cpp index c8dd5c18a..472700e97 100644 --- a/sampling/test/var_opt_sketch_test.cpp +++ b/sampling/test/var_opt_sketch_test.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #ifdef TEST_BINARY_INPUT_PATH static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH; @@ -560,4 +561,19 @@ TEST_CASE("varopt sketch: update after deserialize in exact mode", "[var_opt_ske } } +TEST_CASE("varopt sketch: serialize with a header", "[var_opt_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + var_opt_sketch sketch(10); + for (int i = 0; i < n; ++i) sketch.update(std::to_string(i)); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = var_opt_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_n() == sketch.get_n()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } diff --git a/tuple/include/tuple_sketch_impl.hpp b/tuple/include/tuple_sketch_impl.hpp index 0c64a3c84..025079fb3 100644 --- a/tuple/include/tuple_sketch_impl.hpp +++ b/tuple/include/tuple_sketch_impl.hpp @@ -441,7 +441,7 @@ auto compact_tuple_sketch::serialize(unsigned header_size_bytes, const Ser + sizeof(uint64_t) * entries_.size() + get_serialized_size_summaries_bytes(sd); vector_bytes bytes(size, 0, entries_.get_allocator()); uint8_t* ptr = bytes.data() + header_size_bytes; - const uint8_t* end_ptr = ptr + size; + const uint8_t* end_ptr = bytes.data() + size; ptr += copy_to_mem(preamble_longs, ptr); const uint8_t serial_version = SERIAL_VERSION; diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp index dd0c3f1d3..15a85afe2 100644 --- a/tuple/test/tuple_sketch_test.cpp +++ b/tuple/test/tuple_sketch_test.cpp @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -35,6 +36,7 @@ std::ostream& operator<<(std::ostream& os, const three_doubles& tuple) { #include #include +#include namespace datasketches { @@ -430,4 +432,20 @@ TEST_CASE("tuple sketch: deserialize bounds-checks each entry key", "[tuple_sket std::out_of_range); } +TEST_CASE("tuple sketch: serialize with a header", "[tuple_sketch]") { + // with a header, the bytes after it round-trip, and the serde is given the true end of the buffer: + // one that writes more than it reported must be stopped by its capacity check + for (int n: {1, 3, 1000}) { + CAPTURE(n); + auto update_sketch = update_tuple_sketch::builder().build(); + for (int i = 0; i < n; ++i) update_sketch.update(i, std::to_string(i)); + const auto sketch = update_sketch.compact(); + const unsigned header_size = 16; + const auto bytes = sketch.serialize(header_size); + const auto copy = compact_tuple_sketch::deserialize(bytes.data() + header_size, bytes.size() - header_size); + REQUIRE(copy.get_num_retained() == sketch.get_num_retained()); + REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range); + } +} + } /* namespace datasketches */ From 0a8b021b418f66d2724faea2619c5c7bbb81822b Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 11:27:28 -0800 Subject: [PATCH 2/3] Frequent items: release the item copies when serialize() throws Both serialize() overloads copied the items and weights into raw allocations for the serde, and freed them only after it returned, so a serde or copy that threw leaked them. The new header test in this PR made the serde throw and LeakSanitizer on Linux caught it. The copies are now held in vectors. The serialized bytes are unchanged. Co-Authored-By: Claude Opus 5.5 --- fi/include/frequent_items_sketch_impl.hpp | 47 +++++++++---------- ...frequent_items_sketch_custom_type_test.cpp | 29 ++++++++++++ 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp index 770b05d27..36b15671e 100644 --- a/fi/include/frequent_items_sketch_impl.hpp +++ b/fi/include/frequent_items_sketch_impl.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include "memory_operations.hpp" @@ -197,22 +198,19 @@ void frequent_items_sketch::serialize(std::ostream& os, const Ser write(os, total_weight); write(os, offset); - // copy active items and their weights to use batch serialization + // copy active items and their weights to use batch serialization; + // the vectors release the copies if a copy or the serde throws using AllocW = typename std::allocator_traits::template rebind_alloc; - AllocW aw(map.get_allocator()); - W* weights = aw.allocate(num_items); - A alloc(map.get_allocator()); - T* items = alloc.allocate(num_items); - uint32_t i = 0; + std::vector weights(map.get_allocator()); + weights.reserve(num_items); + std::vector items(map.get_allocator()); + items.reserve(num_items); for (auto it: map) { - new (&items[i]) T(it.first); - weights[i++] = it.second; + items.push_back(it.first); + weights.push_back(it.second); } - write(os, weights, sizeof(W) * num_items); - aw.deallocate(weights, num_items); - sd.serialize(os, items, num_items); - for (i = 0; i < num_items; i++) items[i].~T(); - alloc.deallocate(items, num_items); + write(os, weights.data(), sizeof(W) * num_items); + sd.serialize(os, items.data(), num_items); } } @@ -256,23 +254,20 @@ auto frequent_items_sketch::serialize(unsigned header_size_bytes, ptr += copy_to_mem(total_weight, ptr); ptr += copy_to_mem(offset, ptr); - // copy active items and their weights to use batch serialization + // copy active items and their weights to use batch serialization; + // the vectors release the copies if a copy or the serde throws using AllocW = typename std::allocator_traits::template rebind_alloc; - AllocW aw(map.get_allocator()); - W* weights = aw.allocate(num_items); - A alloc(map.get_allocator()); - T* items = alloc.allocate(num_items); - uint32_t i = 0; + std::vector weights(map.get_allocator()); + weights.reserve(num_items); + std::vector items(map.get_allocator()); + items.reserve(num_items); for (auto it: map) { - new (&items[i]) T(it.first); - weights[i++] = it.second; + items.push_back(it.first); + weights.push_back(it.second); } - ptr += copy_to_mem(weights, ptr, sizeof(W) * num_items); - aw.deallocate(weights, num_items); + ptr += copy_to_mem(weights.data(), ptr, sizeof(W) * num_items); const size_t bytes_remaining = end_ptr - ptr; - ptr += sd.serialize(ptr, bytes_remaining, items, num_items); - for (i = 0; i < num_items; i++) items[i].~T(); - alloc.deallocate(items, num_items); + ptr += sd.serialize(ptr, bytes_remaining, items.data(), num_items); } return bytes; } diff --git a/fi/test/frequent_items_sketch_custom_type_test.cpp b/fi/test/frequent_items_sketch_custom_type_test.cpp index 85e0d6ba8..eb5ec777a 100644 --- a/fi/test/frequent_items_sketch_custom_type_test.cpp +++ b/fi/test/frequent_items_sketch_custom_type_test.cpp @@ -129,4 +129,33 @@ TEST_CASE("frequent items: a purge does not allocate", "[frequent_items_sketch]" REQUIRE(s.get_total_weight() == 100000); } +namespace { + +// a serde whose serialize() always throws +struct throwing_test_type_serde: test_type_serde { + void serialize(std::ostream&, const test_type*, unsigned) const { + throw std::runtime_error("serialize failed"); + } + size_t serialize(void*, size_t, const test_type*, unsigned) const { + throw std::runtime_error("serialize failed"); + } +}; + +} // namespace + +TEST_CASE("frequent items: a serde that throws leaks nothing", "[frequent_items_sketch]") { + test_allocator_total_bytes = 0; + test_allocator_net_allocations = 0; + { + frequent_test_type_sketch sketch(3, frequent_test_type_sketch::LG_MIN_MAP_SIZE, test_type_equal(), 0); + for (int i = 0; i < 10; ++i) sketch.update(i); + // serialize() copies the items for the serde, and must release the copies when the serde throws + REQUIRE_THROWS_AS(sketch.serialize(0, throwing_test_type_serde()), std::runtime_error); + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + REQUIRE_THROWS_AS(sketch.serialize(s, throwing_test_type_serde()), std::runtime_error); + } + REQUIRE(test_allocator_total_bytes == 0); + REQUIRE(test_allocator_net_allocations == 0); +} + } /* namespace datasketches */ From 3c78c6263dccf189d9f54fcc7227c74ab9e07d05 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 13:14:53 -0800 Subject: [PATCH 3/3] Frequent items: skip the item copies when no counters are active A purge can clear every counter while the sketch stays non-empty. serialize() then reserved zero-length vectors and passed their null data() to memcpy, which UBSan reports on Linux in "purge clearing all counters serialized form". Co-Authored-By: Claude Opus 5.5 --- fi/include/frequent_items_sketch_impl.hpp | 52 +++++++++++++---------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/fi/include/frequent_items_sketch_impl.hpp b/fi/include/frequent_items_sketch_impl.hpp index 36b15671e..ae77ae77c 100644 --- a/fi/include/frequent_items_sketch_impl.hpp +++ b/fi/include/frequent_items_sketch_impl.hpp @@ -199,18 +199,21 @@ void frequent_items_sketch::serialize(std::ostream& os, const Ser write(os, offset); // copy active items and their weights to use batch serialization; - // the vectors release the copies if a copy or the serde throws - using AllocW = typename std::allocator_traits::template rebind_alloc; - std::vector weights(map.get_allocator()); - weights.reserve(num_items); - std::vector items(map.get_allocator()); - items.reserve(num_items); - for (auto it: map) { - items.push_back(it.first); - weights.push_back(it.second); + // the vectors release the copies if a copy or the serde throws. + // a non-empty sketch can have no active items (e.g. after a purge) + if (num_items > 0) { + using AllocW = typename std::allocator_traits::template rebind_alloc; + std::vector weights(map.get_allocator()); + weights.reserve(num_items); + std::vector items(map.get_allocator()); + items.reserve(num_items); + for (auto it: map) { + items.push_back(it.first); + weights.push_back(it.second); + } + write(os, weights.data(), sizeof(W) * num_items); + sd.serialize(os, items.data(), num_items); } - write(os, weights.data(), sizeof(W) * num_items); - sd.serialize(os, items.data(), num_items); } } @@ -255,19 +258,22 @@ auto frequent_items_sketch::serialize(unsigned header_size_bytes, ptr += copy_to_mem(offset, ptr); // copy active items and their weights to use batch serialization; - // the vectors release the copies if a copy or the serde throws - using AllocW = typename std::allocator_traits::template rebind_alloc; - std::vector weights(map.get_allocator()); - weights.reserve(num_items); - std::vector items(map.get_allocator()); - items.reserve(num_items); - for (auto it: map) { - items.push_back(it.first); - weights.push_back(it.second); + // the vectors release the copies if a copy or the serde throws. + // a non-empty sketch can have no active items (e.g. after a purge) + if (num_items > 0) { + using AllocW = typename std::allocator_traits::template rebind_alloc; + std::vector weights(map.get_allocator()); + weights.reserve(num_items); + std::vector items(map.get_allocator()); + items.reserve(num_items); + for (auto it: map) { + items.push_back(it.first); + weights.push_back(it.second); + } + ptr += copy_to_mem(weights.data(), ptr, sizeof(W) * num_items); + const size_t bytes_remaining = end_ptr - ptr; + ptr += sd.serialize(ptr, bytes_remaining, items.data(), num_items); } - ptr += copy_to_mem(weights.data(), ptr, sizeof(W) * num_items); - const size_t bytes_remaining = end_ptr - ptr; - ptr += sd.serialize(ptr, bytes_remaining, items.data(), num_items); } return bytes; }