Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ target_sources(common_test_lib
INTERFACE
${CMAKE_CURRENT_SOURCE_DIR}/test_allocator.hpp
${CMAKE_CURRENT_SOURCE_DIR}/test_type.hpp
${CMAKE_CURRENT_SOURCE_DIR}/undersized_serde.hpp
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/catch_runner.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_allocator.cpp
Expand Down
40 changes: 40 additions & 0 deletions common/test/undersized_serde.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#ifndef UNDERSIZED_SERDE_HPP_
#define UNDERSIZED_SERDE_HPP_

#include <string>

#include "serde.hpp"

namespace datasketches {

// Reports each string as one byte shorter than it serializes it, so a sketch that serializes to bytes
// allocates too little. The capacity check of the serde must then throw std::out_of_range,
// which it can only do if the sketch passes it the true end of the buffer.
struct undersized_string_serde: serde<std::string> {
size_t size_of_item(const std::string& item) const {
return serde<std::string>::size_of_item(item) - 1;
}
};

} /* namespace datasketches */

#endif
2 changes: 1 addition & 1 deletion density/include/density_sketch_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ auto density_sketch<T, K, A>::serialize(unsigned header_size_bytes) const -> vec

vector_bytes bytes(size, 0, levels_.get_allocator());
uint8_t* ptr = bytes.data() + header_size_bytes;
const uint8_t* end_ptr = ptr + size;
const uint8_t* end_ptr = bytes.data() + size;

ptr += copy_to_mem(preamble_ints, ptr);
const uint8_t ser_ver = SERIAL_VERSION;
Expand Down
18 changes: 18 additions & 0 deletions density/test/density_sketch_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -480,4 +480,22 @@ TEST_CASE("density sketch: reject invalid family and ebpps headers", "[density_s
}
}

TEST_CASE("density sketch: serialize with a header", "[density_sketch]") {
// serialize() used to check its output against an end that included the header twice, so it threw
for (int n: {0, 1, 1000}) {
CAPTURE(n);
density_sketch<float> sketch(10, 3);
for (int i = 0; i < n; ++i) {
const float x = static_cast<float>(i);
sketch.update(std::vector<float>({x, x, x}));
}
const unsigned header_size = 16;
const auto bytes = sketch.serialize(header_size);
REQUIRE(bytes.size() == header_size + sketch.serialize().size());
const auto copy = density_sketch<float>::deserialize(bytes.data() + header_size, bytes.size() - header_size);
REQUIRE(copy.get_n() == sketch.get_n());
REQUIRE(copy.get_num_retained() == sketch.get_num_retained());
}
}

} /* namespace datasketches */
65 changes: 33 additions & 32 deletions fi/include/frequent_items_sketch_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <limits>
#include <sstream>
#include <stdexcept>
#include <vector>

#include "memory_operations.hpp"

Expand Down Expand Up @@ -197,22 +198,22 @@ void frequent_items_sketch<T, W, H, E, A>::serialize(std::ostream& os, const Ser
write(os, total_weight);
write(os, offset);

// copy active items and their weights to use batch serialization
using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
AllocW aw(map.get_allocator());
W* weights = aw.allocate(num_items);
A alloc(map.get_allocator());
T* items = alloc.allocate(num_items);
uint32_t i = 0;
for (auto it: map) {
new (&items[i]) T(it.first);
weights[i++] = it.second;
// copy active items and their weights to use batch serialization;
// the vectors release the copies if a copy or the serde throws.
// a non-empty sketch can have no active items (e.g. after a purge)
if (num_items > 0) {
using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
std::vector<W, AllocW> weights(map.get_allocator());
weights.reserve(num_items);
std::vector<T, A> items(map.get_allocator());
items.reserve(num_items);
for (auto it: map) {
items.push_back(it.first);
weights.push_back(it.second);
}
write(os, weights.data(), sizeof(W) * num_items);
sd.serialize(os, items.data(), num_items);
}
write(os, weights, sizeof(W) * num_items);
aw.deallocate(weights, num_items);
sd.serialize(os, items, num_items);
for (i = 0; i < num_items; i++) items[i].~T();
alloc.deallocate(items, num_items);
}
}

Expand All @@ -231,7 +232,7 @@ auto frequent_items_sketch<T, W, H, E, A>::serialize(unsigned header_size_bytes,
const size_t size = header_size_bytes + get_serialized_size_bytes(sd);
vector_bytes bytes(size, 0, map.get_allocator());
uint8_t* ptr = bytes.data() + header_size_bytes;
uint8_t* end_ptr = ptr + size;
uint8_t* end_ptr = bytes.data() + size;

const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_EMPTY : PREAMBLE_LONGS_NONEMPTY;
ptr += copy_to_mem(preamble_longs, ptr);
Expand All @@ -256,23 +257,23 @@ auto frequent_items_sketch<T, W, H, E, A>::serialize(unsigned header_size_bytes,
ptr += copy_to_mem(total_weight, ptr);
ptr += copy_to_mem(offset, ptr);

// copy active items and their weights to use batch serialization
using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
AllocW aw(map.get_allocator());
W* weights = aw.allocate(num_items);
A alloc(map.get_allocator());
T* items = alloc.allocate(num_items);
uint32_t i = 0;
for (auto it: map) {
new (&items[i]) T(it.first);
weights[i++] = it.second;
// copy active items and their weights to use batch serialization;
// the vectors release the copies if a copy or the serde throws.
// a non-empty sketch can have no active items (e.g. after a purge)
if (num_items > 0) {
using AllocW = typename std::allocator_traits<A>::template rebind_alloc<W>;
std::vector<W, AllocW> weights(map.get_allocator());
weights.reserve(num_items);
std::vector<T, A> items(map.get_allocator());
items.reserve(num_items);
for (auto it: map) {
items.push_back(it.first);
weights.push_back(it.second);
}
ptr += copy_to_mem(weights.data(), ptr, sizeof(W) * num_items);
const size_t bytes_remaining = end_ptr - ptr;
ptr += sd.serialize(ptr, bytes_remaining, items.data(), num_items);
}
ptr += copy_to_mem(weights, ptr, sizeof(W) * num_items);
aw.deallocate(weights, num_items);
const size_t bytes_remaining = end_ptr - ptr;
ptr += sd.serialize(ptr, bytes_remaining, items, num_items);
for (i = 0; i < num_items; i++) items[i].~T();
alloc.deallocate(items, num_items);
}
return bytes;
}
Expand Down
29 changes: 29 additions & 0 deletions fi/test/frequent_items_sketch_custom_type_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -129,4 +129,33 @@ TEST_CASE("frequent items: a purge does not allocate", "[frequent_items_sketch]"
REQUIRE(s.get_total_weight() == 100000);
}

namespace {

// a serde whose serialize() always throws
struct throwing_test_type_serde: test_type_serde {
void serialize(std::ostream&, const test_type*, unsigned) const {
throw std::runtime_error("serialize failed");
}
size_t serialize(void*, size_t, const test_type*, unsigned) const {
throw std::runtime_error("serialize failed");
}
};

} // namespace

TEST_CASE("frequent items: a serde that throws leaks nothing", "[frequent_items_sketch]") {
test_allocator_total_bytes = 0;
test_allocator_net_allocations = 0;
{
frequent_test_type_sketch sketch(3, frequent_test_type_sketch::LG_MIN_MAP_SIZE, test_type_equal(), 0);
for (int i = 0; i < 10; ++i) sketch.update(i);
// serialize() copies the items for the serde, and must release the copies when the serde throws
REQUIRE_THROWS_AS(sketch.serialize(0, throwing_test_type_serde()), std::runtime_error);
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
REQUIRE_THROWS_AS(sketch.serialize(s, throwing_test_type_serde()), std::runtime_error);
}
REQUIRE(test_allocator_total_bytes == 0);
REQUIRE(test_allocator_net_allocations == 0);
}

} /* namespace datasketches */
17 changes: 17 additions & 0 deletions fi/test/frequent_items_sketch_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@
*/

#include <catch2/catch.hpp>
#include <string>
#include <sstream>
#include <fstream>
#include <stdexcept>

#include "frequent_items_sketch.hpp"
#include <undersized_serde.hpp>

#ifdef TEST_BINARY_INPUT_PATH
static std::string testBinaryInputPath = TEST_BINARY_INPUT_PATH;
Expand Down Expand Up @@ -435,4 +437,19 @@ TEST_CASE("frequent items: string deserialize single item buffer overrun", "[fre
REQUIRE_THROWS_AS(frequent_items_sketch<std::string>::deserialize(bytes.data(), bytes.size() - 1), std::out_of_range);
}

TEST_CASE("frequent items sketch: serialize with a header", "[frequent_items_sketch]") {
// with a header, the bytes after it round-trip, and the serde is given the true end of the buffer:
// one that writes more than it reported must be stopped by its capacity check
for (int n: {1, 3, 1000}) {
CAPTURE(n);
frequent_items_sketch<std::string> sketch(3);
for (int i = 0; i < n; ++i) sketch.update(std::to_string(i));
const unsigned header_size = 16;
const auto bytes = sketch.serialize(header_size);
const auto copy = frequent_items_sketch<std::string>::deserialize(bytes.data() + header_size, bytes.size() - header_size);
REQUIRE(copy.get_total_weight() == sketch.get_total_weight());
REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range);
}
}

} /* namespace datasketches */
2 changes: 1 addition & 1 deletion kll/include/kll_sketch_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -550,7 +550,7 @@ auto kll_sketch<T, C, A>::serialize(unsigned header_size_bytes, const SerDe& sd)
const size_t size = header_size_bytes + get_serialized_size_bytes(sd);
vector_bytes bytes(size, 0, allocator_);
uint8_t* ptr = bytes.data() + header_size_bytes;
const uint8_t* end_ptr = ptr + size;
const uint8_t* end_ptr = bytes.data() + size;
const uint8_t preamble_ints(is_empty() || is_single_item ? PREAMBLE_INTS_SHORT : PREAMBLE_INTS_FULL);
ptr += copy_to_mem(preamble_ints, ptr);
const uint8_t serial_version(is_single_item ? SERIAL_VERSION_2 : SERIAL_VERSION_1);
Expand Down
17 changes: 17 additions & 0 deletions kll/test/kll_sketch_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/

#include <catch2/catch.hpp>
#include <string>
#include <algorithm>
#include <cmath>
#include <cstring>
Expand All @@ -28,6 +29,7 @@

#include <kll_sketch.hpp>
#include <test_allocator.hpp>
#include <undersized_serde.hpp>

namespace datasketches {

Expand Down Expand Up @@ -1584,4 +1586,19 @@ TEST_CASE("kll sketch: compaction and merge use the sketch's comparator", "[kll_
REQUIRE(sketch.get_rank(1000) == Approx(0.5).margin(RANK_EPS_FOR_K_200 * 10));
}

TEST_CASE("kll sketch: serialize with a header", "[kll_sketch]") {
// with a header, the bytes after it round-trip, and the serde is given the true end of the buffer:
// one that writes more than it reported must be stopped by its capacity check
for (int n: {1, 3, 1000}) {
CAPTURE(n);
kll_sketch<std::string> sketch(200);
for (int i = 0; i < n; ++i) sketch.update(std::to_string(i));
const unsigned header_size = 16;
const auto bytes = sketch.serialize(header_size);
const auto copy = kll_sketch<std::string>::deserialize(bytes.data() + header_size, bytes.size() - header_size);
REQUIRE(copy.get_n() == sketch.get_n());
REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range);
}
}

} /* namespace datasketches */
2 changes: 1 addition & 1 deletion quantiles/include/quantiles_sketch_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ auto quantiles_sketch<T, C, A>::serialize(unsigned header_size_bytes, const SerD
const size_t size = get_serialized_size_bytes(serde) + header_size_bytes;
vector_bytes bytes(size, 0, allocator_);
uint8_t* ptr = bytes.data() + header_size_bytes;
const uint8_t* end_ptr = ptr + size;
const uint8_t* end_ptr = bytes.data() + size;

const uint8_t preamble_longs = is_empty() ? PREAMBLE_LONGS_SHORT : PREAMBLE_LONGS_FULL;
ptr += copy_to_mem(preamble_longs, ptr);
Expand Down
18 changes: 18 additions & 0 deletions quantiles/test/quantiles_sketch_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@
*/

#include <catch2/catch.hpp>
#include <string>
#include <stdexcept>
#include <cmath>
#include <sstream>
#include <fstream>

#include <quantiles_sketch.hpp>
#include <test_allocator.hpp>
#include <common_defs.hpp>
#include <undersized_serde.hpp>

namespace datasketches {

Expand Down Expand Up @@ -957,6 +960,21 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
}


TEST_CASE("quantiles sketch: serialize with a header", "[quantiles_sketch]") {
// with a header, the bytes after it round-trip, and the serde is given the true end of the buffer:
// one that writes more than it reported must be stopped by its capacity check
for (int n: {1, 3, 1000}) {
CAPTURE(n);
quantiles_sketch<std::string> sketch(128);
for (int i = 0; i < n; ++i) sketch.update(std::to_string(i));
const unsigned header_size = 16;
const auto bytes = sketch.serialize(header_size);
const auto copy = quantiles_sketch<std::string>::deserialize(bytes.data() + header_size, bytes.size() - header_size);
REQUIRE(copy.get_n() == sketch.get_n());
REQUIRE_THROWS_AS(sketch.serialize(header_size, undersized_string_serde()), std::out_of_range);
}
}

TEST_CASE("quantiles sketch: deserialize serial version 1 truncated after min and max", "[quantiles_sketch]") {
quantiles_sketch<int> sketch(16);
for (int i = 0; i < 1000; ++i) sketch.update(i);
Expand Down
10 changes: 5 additions & 5 deletions req/include/req_sketch_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,8 @@ size_t req_sketch<T, C, A>::get_serialized_size_bytes(const SerDe& sd) const {
if (is_estimation_mode()) {
size += sizeof(n_) + sizeof(TT) * 2; // min and max
}
if (n_ == 1) {
size += sizeof(TT);
if (n_ <= req_constants::MIN_K) { // raw items, without a compactor header
size += sizeof(TT) * static_cast<size_t>(n_);
} else {
for (const auto& compactor: compactors_) size += compactor.get_serialized_size_bytes(sd);
}
Expand All @@ -373,8 +373,8 @@ size_t req_sketch<T, C, A>::get_serialized_size_bytes(const SerDe& sd) const {
size += sd.size_of_item(*min_item_);
size += sd.size_of_item(*max_item_);
}
if (n_ == 1) {
size += sd.size_of_item(*compactors_[0].begin());
if (n_ <= req_constants::MIN_K) { // raw items, without a compactor header
for (auto it = compactors_[0].begin(); it != compactors_[0].end(); ++it) size += sd.size_of_item(*it);
} else {
for (const auto& compactor: compactors_) size += compactor.get_serialized_size_bytes(sd);
}
Expand Down Expand Up @@ -422,7 +422,7 @@ auto req_sketch<T, C, A>::serialize(unsigned header_size_bytes, const SerDe& sd)
const size_t size = header_size_bytes + get_serialized_size_bytes(sd);
vector_bytes bytes(size, 0, allocator_);
uint8_t* ptr = bytes.data() + header_size_bytes;
const uint8_t* end_ptr = ptr + size;
const uint8_t* end_ptr = bytes.data() + size;

const uint8_t preamble_ints = is_estimation_mode() ? 4 : 2;
ptr += copy_to_mem(preamble_ints, ptr);
Expand Down
Loading
Loading