From b1dcf1d2ea8757e6e13b0ab6a7429bfccb53a7d5 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 11:43:06 -0800 Subject: [PATCH] REQ: reject n that isn't the total weight on deserialize An estimation-mode image stores n separately from the compactors, and deserialize() took it as given. Compaction keeps the total weight of the retained items, so n must equal it; a corrupt image could otherwise give a sketch whose n disagrees with its items. Both deserializers now check it, and also reject a total weight that overflows. Matches the REQ check in apache/datasketches-java#810. Co-Authored-By: Claude Opus 5.5 --- req/include/req_sketch.hpp | 2 ++ req/include/req_sketch_impl.hpp | 21 +++++++++++++++++++++ req/test/req_sketch_test.cpp | 10 ++++++++++ 3 files changed, 33 insertions(+) diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp index 8e3468f0..f2742aac 100755 --- a/req/include/req_sketch.hpp +++ b/req/include/req_sketch.hpp @@ -496,6 +496,8 @@ class req_sketch { static void check_num_levels(uint8_t num_levels, bool raw_items); static void check_serial_version(uint8_t serial_version); static void check_family_id(uint8_t family_id); + // in estimation mode, n is stored and must be the total weight of the retained items + static void check_n(uint64_t n, const std::vector& compactors); template::value, int>::type = 0> static inline bool check_update_item(const TT& item) { diff --git a/req/include/req_sketch_impl.hpp b/req/include/req_sketch_impl.hpp index d6f4c8e4..62a54520 100644 --- a/req/include/req_sketch_impl.hpp +++ b/req/include/req_sketch_impl.hpp @@ -20,6 +20,7 @@ #ifndef REQ_SKETCH_IMPL_HPP_ #define REQ_SKETCH_IMPL_HPP_ +#include #include #include #include @@ -513,6 +514,7 @@ req_sketch req_sketch::deserialize(std::istream& is, const Ser k, static_cast(i))); } } + if (num_levels > 1) check_n(n, compactors); if (num_levels == 1) { if (compactors[0].get_num_items() == 0) throw std::invalid_argument("Possible corruption: non-empty sketch with no items"); const auto begin = compactors[0].begin(); @@ -602,6 +604,7 @@ req_sketch req_sketch::deserialize(const void* bytes, size_t s ptr += pair.second; } } + if (num_levels > 1) check_n(n, compactors); if (num_levels == 1) { if (compactors[0].get_num_items() == 0) throw std::invalid_argument("Possible corruption: non-empty sketch with no items"); const auto begin = compactors[0].begin(); @@ -620,6 +623,24 @@ req_sketch req_sketch::deserialize(const void* bytes, size_t s return req_sketch(k, hra, n, std::move(min_item), std::move(max_item), std::move(compactors), comparator); } +template +void req_sketch::check_n(uint64_t n, const std::vector& compactors) { + // compaction keeps the total weight, so it is always n + uint64_t total_weight = 0; + for (const auto& compactor: compactors) { + const uint64_t num_items = compactor.get_num_items(); + const uint8_t lg_weight = compactor.get_lg_weight(); + if (num_items > 0 && (lg_weight >= 64 || num_items > ((std::numeric_limits::max() - total_weight) >> lg_weight))) { + throw std::invalid_argument("Possible corruption: the total weight of the retained items overflows"); + } + total_weight += num_items << lg_weight; + } + if (n != total_weight) { + throw std::invalid_argument("Possible corruption: n " + std::to_string(n) + + " is not the total weight of the retained items " + std::to_string(total_weight)); + } +} + template void req_sketch::grow() { const uint8_t lg_weight = get_num_levels(); diff --git a/req/test/req_sketch_test.cpp b/req/test/req_sketch_test.cpp index 3db56247..301e9d89 100755 --- a/req/test/req_sketch_test.cpp +++ b/req/test/req_sketch_test.cpp @@ -726,6 +726,16 @@ TEST_CASE("req sketch: deserialize rejects inconsistent compactors", "[req_sketc check_rejected(bytes); } + SECTION("n is not the total weight") { + const size_t n_offset = 8; // after the preamble, in estimation mode + for (const int64_t delta: {-1, 1}) { + CAPTURE(delta); + auto bytes = estimation.serialize(); + put_value(bytes, n_offset, static_cast(static_cast(estimation.get_n()) + delta)); + check_rejected(bytes); + } + } + SECTION("compactor header beyond the input") { auto bytes = estimation.serialize(); bytes.resize(estimation_compactor + 10);