From 67cf69f80f76e1f498105d581a471d70d8ba2f9e Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Wed, 28 Feb 2024 15:31:27 -0800 Subject: [PATCH 1/4] Adding some asserts for self-merging and long overruns. --- .../datasketches/kll/KllDoublesHelper.java | 7 +++++-- .../datasketches/kll/KllDoublesSketch.java | 3 ++- .../org/apache/datasketches/kll/KllHelper.java | 1 + .../quantilescommon/QuantilesAPI.java | 1 + .../apache/datasketches/common/Shuffle.java | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java index 88da49e38..8a5c98b92 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java @@ -123,8 +123,7 @@ private static void compressWhileUpdatingSketch(final KllDoublesSketch dblSk) { } //assumes readOnly = false and UPDATABLE, called from KllDoublesSketch::merge - static void mergeDoubleImpl(final KllDoublesSketch mySketch, - final KllDoublesSketch otherDblSk) { + static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSketch otherDblSk) { if (otherDblSk.isEmpty()) { return; } //capture my key mutable fields before doing any merging @@ -133,6 +132,8 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final double myMax = myEmpty ? Double.NaN : mySketch.getMaxItem(); final int myMinK = mySketch.getMinK(); final long finalN = mySketch.getN() + otherDblSk.getN(); + assert finalN <= Long.MAX_VALUE && finalN >= 0 : + "The input count has exceeded the capacity of a long and the capability of this sketch."; //buffers that are referenced multiple times final int otherNumLevels = otherDblSk.getNumLevels(); @@ -400,6 +401,8 @@ private static int[] generalDoublesCompress( // If we are at the current top level, add an empty level above it for convenience, // but do not increment numLevels until later if (curLevel == (numLevels - 1)) { + assert curLevel + 2 < 60 : + "The number of levels has exceeded the capability of this sketch."; inLevels[curLevel + 2] = inLevels[curLevel + 1]; } diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java index 3cef84710..8883734c9 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesSketch.java @@ -276,7 +276,8 @@ public QuantilesDoublesSketchIterator iterator() { @Override public final void merge(final KllSketch other) { if (readOnly || sketchStructure != UPDATABLE) { throw new SketchesArgumentException(TGT_IS_READ_ONLY_MSG); } - final KllDoublesSketch othDblSk = (KllDoublesSketch)other; //check cast first + if (this == other) { throw new SketchesArgumentException(SELF_MERGE_MSG); } + final KllDoublesSketch othDblSk = (KllDoublesSketch)other; if (othDblSk.isEmpty()) { return; } //then check empty KllDoublesHelper.mergeDoubleImpl(this, othDblSk); kllDoublesSV = null; diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index f1d1df9b7..ae829506e 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -632,6 +632,7 @@ else if (sketchType == FLOATS_SKETCH) { //grow levels arr by one and copy the old data to the new array, extra space at the top. myNewLevelsArr = Arrays.copyOf(myCurLevelsArr, myCurNumLevels + 2); assert myNewLevelsArr.length == myCurLevelsArr.length + 1; + assert myNewLevelsArr.length <= 60 : "The number of levels has exceeded the capability of this sketch."; myNewNumLevels = myCurNumLevels + 1; sketch.incNumLevels(); //increment for off-heap } else { diff --git a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java index 38502ecaa..f433519e2 100644 --- a/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java +++ b/src/main/java/org/apache/datasketches/quantilescommon/QuantilesAPI.java @@ -209,6 +209,7 @@ public interface QuantilesAPI { static String NOT_SINGLE_ITEM_MSG = "Sketch does not have just one item. "; static String MEM_REQ_SVR_NULL_MSG = "MemoryRequestServer must not be null. "; static String TGT_IS_READ_ONLY_MSG = "Target sketch is Read Only, cannot write. "; + static String SELF_MERGE_MSG = "A sketch cannot merge with itself. "; /** * Gets the user configured parameter k, which controls the accuracy of the sketch diff --git a/src/test/java/org/apache/datasketches/common/Shuffle.java b/src/test/java/org/apache/datasketches/common/Shuffle.java index 992d15ab7..223e3486e 100644 --- a/src/test/java/org/apache/datasketches/common/Shuffle.java +++ b/src/test/java/org/apache/datasketches/common/Shuffle.java @@ -99,4 +99,22 @@ private static void swap(final int[] array, final int i1, final int i2) { array[i2] = value; } + /** + * Shuffle the given input array of type T + * @param array input array + */ + public static void shuffle(final T[] array) { + final int arrLen = array.length; + for (int i = 0; i < arrLen; i++) { + final int r = rand.nextInt(i + 1); + swap(array, i, r); + } + } + + private static void swap(final T[] array, final int i1, final int i2) { + final T value = array[i1]; + array[i1] = array[i2]; + array[i2] = value; + } + } From 4ac7757366e9ea1ab5a27530324d0dca2fd3ca41 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 29 Feb 2024 11:42:49 -0800 Subject: [PATCH 2/4] Update the built in shuffle class. Includes generic type arrays and also gives the option of providing a user source of randomness derived from java.util.Random. --- .../apache/datasketches/common/Shuffle.java | 67 ++++++++++++++++--- 1 file changed, 57 insertions(+), 10 deletions(-) diff --git a/src/test/java/org/apache/datasketches/common/Shuffle.java b/src/test/java/org/apache/datasketches/common/Shuffle.java index 223e3486e..af6c4f8b7 100644 --- a/src/test/java/org/apache/datasketches/common/Shuffle.java +++ b/src/test/java/org/apache/datasketches/common/Shuffle.java @@ -28,10 +28,19 @@ public final class Shuffle { private static final Random rand = new Random(); /** - * Shuffle the given input float array - * @param array input array + * Shuffle the given input array using a default source of randomness. + * @param array the array to be shuffled. */ public static void shuffle(final float[] array) { + shuffle(array, rand); + } + + /** + * Shuffle the given input array using the given source of randomness. + * @param array the array to be shuffled. + * @param rand the source of randomness used to shuffle the list. + */ + public static void shuffle(final float[] array, final Random rand) { final int arrLen = array.length; for (int i = 0; i < arrLen; i++) { final int r = rand.nextInt(i + 1); @@ -46,10 +55,19 @@ private static void swap(final float[] array, final int i1, final int i2) { } /** - * Shuffle the given input double array - * @param array input array + * Shuffle the given input array using a default source of randomness. + * @param array the array to be shuffled. */ public static void shuffle(final double[] array) { + shuffle(array, rand); + } + + /** + * Shuffle the given input array using the given source of randomness. + * @param array the array to be shuffled. + * @param rand the source of randomness used to shuffle the list. + */ + public static void shuffle(final double[] array, final Random rand) { final int arrLen = array.length; for (int i = 0; i < arrLen; i++) { final int r = rand.nextInt(i + 1); @@ -64,10 +82,19 @@ private static void swap(final double[] array, final int i1, final int i2) { } /** - * Shuffle the given input long array - * @param array input array + * Shuffle the given input array using a default source of randomness. + * @param array the array to be shuffled. */ public static void shuffle(final long[] array) { + shuffle(array, rand); + } + + /** + * Shuffle the given input array using the given source of randomness. + * @param array the array to be shuffled. + * @param rand the source of randomness used to shuffle the list. + */ + public static void shuffle(final long[] array, final Random rand) { final int arrLen = array.length; for (int i = 0; i < arrLen; i++) { final int r = rand.nextInt(i + 1); @@ -82,10 +109,19 @@ private static void swap(final long[] array, final int i1, final int i2) { } /** - * Shuffle the given input int array - * @param array input array + * Shuffle the given input array using a default source of randomness. + * @param array the array to be shuffled. */ public static void shuffle(final int[] array) { + shuffle(array, rand); + } + + /** + * Shuffle the given input array using the given source of randomness. + * @param array the array to be shuffled. + * @param rand the source of randomness used to shuffle the list. + */ + public static void shuffle(final int[] array, final Random rand) { final int arrLen = array.length; for (int i = 0; i < arrLen; i++) { final int r = rand.nextInt(i + 1); @@ -100,10 +136,21 @@ private static void swap(final int[] array, final int i1, final int i2) { } /** - * Shuffle the given input array of type T - * @param array input array + * Shuffle the given input array using a default source of randomness. + * @param array the array to be shuffled. + * @param the component type of the given array. */ public static void shuffle(final T[] array) { + shuffle(array, rand); + } + + /** + * Shuffle the given input array using the given source of randomness. + * @param array the array to be shuffled. + * @param rand the source of randomness used to shuffle the list. + * @param the component type of the given array. + */ + public static void shuffle(final T[] array, final Random rand) { final int arrLen = array.length; for (int i = 0; i < arrLen; i++) { final int r = rand.nextInt(i + 1); From ab41c7de4a63aa277dcb6a9a180449c18783636f Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Thu, 29 Feb 2024 14:38:00 -0800 Subject: [PATCH 3/4] Added comments to assert --- src/main/java/org/apache/datasketches/kll/KllHelper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index ae829506e..10e41a406 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -313,9 +313,9 @@ static int getNumRetainedAboveLevelZero(final int numLevels, final int[] levels) * @return the capacity of a specific level */ static int levelCapacity(final int k, final int numLevels, final int level, final int m) { - assert (k <= (1 << 29)); - assert (numLevels >= 1) && (numLevels <= 61); - assert (level >= 0) && (level < numLevels); + assert (k <= (1 << 29)) : "The given k is > 2^29."; + assert (numLevels >= 1) && (numLevels <= 61) : "The given numLevels is < 1 or > 61"; + assert (level >= 0) && (level < numLevels) : "The given level is < 0 or >= numLevels."; final int depth = numLevels - level - 1; //depth is # levels from the top level (= 0) return (int) Math.max(m, intCapAux(k, depth)); } From 88dd9428f53011b29b43d7ed60e041ace3c40c54 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Tue, 5 Mar 2024 19:22:36 -0800 Subject: [PATCH 4/4] Removed two asserts, converted one to ArithmeticException when long N overflows. Turns out there is a convenient Math.addExact(a, b) that is the perfect solution. --- .../org/apache/datasketches/kll/KllDoublesHelper.java | 8 ++------ src/main/java/org/apache/datasketches/kll/KllHelper.java | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java index 8a5c98b92..ecb8c99dd 100644 --- a/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllDoublesHelper.java @@ -131,9 +131,7 @@ static void mergeDoubleImpl(final KllDoublesSketch mySketch, final KllDoublesSke final double myMin = myEmpty ? Double.NaN : mySketch.getMinItem(); final double myMax = myEmpty ? Double.NaN : mySketch.getMaxItem(); final int myMinK = mySketch.getMinK(); - final long finalN = mySketch.getN() + otherDblSk.getN(); - assert finalN <= Long.MAX_VALUE && finalN >= 0 : - "The input count has exceeded the capacity of a long and the capability of this sketch."; + final long finalN = Math.addExact(mySketch.getN(), otherDblSk.getN()); //buffers that are referenced multiple times final int otherNumLevels = otherDblSk.getNumLevels(); @@ -399,10 +397,8 @@ private static int[] generalDoublesCompress( curLevel++; // start out at level 0 // If we are at the current top level, add an empty level above it for convenience, - // but do not increment numLevels until later + // but do not actually increment numLevels until later if (curLevel == (numLevels - 1)) { - assert curLevel + 2 < 60 : - "The number of levels has exceeded the capability of this sketch."; inLevels[curLevel + 2] = inLevels[curLevel + 1]; } diff --git a/src/main/java/org/apache/datasketches/kll/KllHelper.java b/src/main/java/org/apache/datasketches/kll/KllHelper.java index 10e41a406..b3894810b 100644 --- a/src/main/java/org/apache/datasketches/kll/KllHelper.java +++ b/src/main/java/org/apache/datasketches/kll/KllHelper.java @@ -632,7 +632,6 @@ else if (sketchType == FLOATS_SKETCH) { //grow levels arr by one and copy the old data to the new array, extra space at the top. myNewLevelsArr = Arrays.copyOf(myCurLevelsArr, myCurNumLevels + 2); assert myNewLevelsArr.length == myCurLevelsArr.length + 1; - assert myNewLevelsArr.length <= 60 : "The number of levels has exceeded the capability of this sketch."; myNewNumLevels = myCurNumLevels + 1; sketch.incNumLevels(); //increment for off-heap } else {