diff --git a/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java b/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java index f02da0a21..bbff6fbd2 100644 --- a/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java @@ -52,21 +52,25 @@ * the compact() method of an UpdatableTupleSketch or as a result of the getResult() * method of a set operation like TupleUnion, TupleIntersection or TupleAnotB. * CompactTupleSketch consists of a compact list (i.e. no intervening spaces) of hash values, - * corresponding list of Summaries, and a value for theta. The lists may or may - * not be ordered. CompactTupleSketch is read-only. + * corresponding list of Summaries, and a value for theta. A CompactTupleSketch created by this + * library is ordered: its entries are sorted by hash value, as in C++. One deserialized from an + * unordered image, such as those written by earlier Java releases, keeps the order of that image. + * CompactTupleSketch is read-only. * * @param type of Summary */ public final class CompactTupleSketch extends TupleSketch { private final long[] hashArr_; private S[] summaryArr_; + private final boolean ordered_; private enum FlagsLegacy { IS_BIG_ENDIAN, IS_EMPTY, HAS_ENTRIES, IS_THETA_INCLUDED } private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED } /** - * Create a CompactTupleSketch from correct components + * Create a CompactTupleSketch from correct components. The entries are sorted by hash value. If they are + * not already sorted, the given arrays are not changed: sorted copies are used instead. * @param hashArr compacted hash array * @param summaryArr compacted summary array * @param thetaLong long value of theta @@ -78,8 +82,60 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED super(thetaLong, empty, null, seedHash); super.thetaLong_ = thetaLong; super.empty_ = empty; - hashArr_ = hashArr; - summaryArr_ = summaryArr; + if ((hashArr == null) || (summaryArr == null) || isSorted(hashArr)) { //callers pass both or neither + hashArr_ = hashArr; + summaryArr_ = summaryArr; + } else { + hashArr_ = hashArr.clone(); + summaryArr_ = summaryArr.clone(); + sortByHash(hashArr_, summaryArr_, hashArr_.clone(), summaryArr_.clone(), 0, hashArr_.length); + } + ordered_ = true; + } + + private static boolean isSorted(final long[] hashArr) { + for (int i = 1; i < hashArr.length; i++) { + if (hashArr[i - 1] > hashArr[i]) { return false; } + } + return true; + } + + // Sorts hashArr[lo, hi) and moves the summaries with their hashes: a merge sort, with tmpHash and + // tmpSum as scratch space of the same length. + private static void sortByHash(final long[] hashArr, final T[] sumArr, final long[] tmpHash, + final T[] tmpSum, final int lo, final int hi) { + if ((hi - lo) <= 16) { //insertion sort + for (int i = lo + 1; i < hi; i++) { + final long h = hashArr[i]; + final T sm = sumArr[i]; + int j = i - 1; + while ((j >= lo) && (hashArr[j] > h)) { + hashArr[j + 1] = hashArr[j]; + sumArr[j + 1] = sumArr[j]; + j--; + } + hashArr[j + 1] = h; + sumArr[j + 1] = sm; + } + return; + } + final int mid = (lo + hi) >>> 1; + sortByHash(hashArr, sumArr, tmpHash, tmpSum, lo, mid); + sortByHash(hashArr, sumArr, tmpHash, tmpSum, mid, hi); + if (hashArr[mid - 1] <= hashArr[mid]) { return; } //already in order + System.arraycopy(hashArr, lo, tmpHash, lo, hi - lo); + System.arraycopy(sumArr, lo, tmpSum, lo, hi - lo); + int i = lo; + int j = mid; + for (int k = lo; k < hi; k++) { + if ((j >= hi) || ((i < mid) && (tmpHash[i] <= tmpHash[j]))) { + hashArr[k] = tmpHash[i]; + sumArr[k] = tmpSum[i++]; + } else { + hashArr[k] = tmpHash[j]; + sumArr[k] = tmpSum[j++]; + } + } } /** @@ -139,6 +195,7 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED hashArr_ = new long[0]; summaryArr_ = null; } + ordered_ = hashArr_.length <= 1; // the legacy format has no ordered flag } else { // current serial format checkBounds(0, (long) preambleLongs * Long.BYTES, seg.byteSize()); final byte flags = seg.get(JAVA_BYTE, COMPACT_FLAGS_BYTE); @@ -166,6 +223,7 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED offset += Long.BYTES; offset += readSummary(seg, offset, i, count, deserializer); } + ordered_ = (count <= 1) || ((flags & (1 << Flags.IS_ORDERED.ordinal())) > 0); } } @@ -201,6 +259,14 @@ public CompactTupleSketch compact() { return this; } + /** + * Returns true if the entries are sorted by hash value. A sketch with no more than one entry is ordered. + * @return true if the entries are sorted by hash value + */ + boolean isOrdered() { + return ordered_; + } + long[] getHashArr() { return hashArr_; } @@ -249,6 +315,7 @@ public byte[] toByteArray() { (1 << Flags.IS_COMPACT.ordinal()) | (1 << Flags.IS_READ_ONLY.ordinal()) | (isEmpty() ? 1 << Flags.IS_EMPTY.ordinal() : 0) + | (ordered_ ? 1 << Flags.IS_ORDERED.ordinal() : 0) ); // an empty image carries zero, as in theta ByteArrayUtil.putShortLE(bytes, SEED_HASH_SHORT, isEmpty() ? 0 : seedHash_); diff --git a/src/main/java/org/apache/datasketches/tuple/JaccardSimilarity.java b/src/main/java/org/apache/datasketches/tuple/JaccardSimilarity.java index 0f4a3d0a8..75a4274f6 100644 --- a/src/main/java/org/apache/datasketches/tuple/JaccardSimilarity.java +++ b/src/main/java/org/apache/datasketches/tuple/JaccardSimilarity.java @@ -518,7 +518,7 @@ public static boolean dissimilarityTest( * @param summarySetOps instance of SummarySetOperations used to unify summaries. * @param seed the update seed * @param Summary - * @return the unordered union of sketch A and sketch B + * @return the ordered union of sketch A and sketch B */ private static TupleSketch computeUnion( final TupleSketch sketchA, @@ -540,7 +540,7 @@ private static TupleSketch computeUnion( * @param summarySetOps instance of SummarySetOperations used to unify summaries. * @param seed the update seed * @param Summary - * @return the unordered union of sketch A and sketch B + * @return the ordered union of sketch A and sketch B */ private static TupleSketch computeUnion( final TupleSketch sketchA, diff --git a/src/main/java/org/apache/datasketches/tuple/PreambleUtil.java b/src/main/java/org/apache/datasketches/tuple/PreambleUtil.java index 412afb936..4d7324128 100644 --- a/src/main/java/org/apache/datasketches/tuple/PreambleUtil.java +++ b/src/main/java/org/apache/datasketches/tuple/PreambleUtil.java @@ -42,7 +42,8 @@ *

This form is binary compatible with the C++ and Go compact tuple sketches, as long as * every language uses the same Summary encoding.

* - *

Flags: ReadOnly, Compact, Empty if empty. The Ordered flag is defined but Java does not set it. + *

Flags: ReadOnly, Compact, Empty if empty, Ordered if the entries are sorted by hash value. A sketch with + * at most one entry is always ordered, as in C++. * The seed hash is zero when the sketch is empty, as in theta.

* *

An empty sketch, or a single-item sketch that is not in estimation mode, uses 1 preamble long. diff --git a/src/main/java/org/apache/datasketches/tuple/TupleAnotB.java b/src/main/java/org/apache/datasketches/tuple/TupleAnotB.java index 2f5776c82..8c7249c60 100644 --- a/src/main/java/org/apache/datasketches/tuple/TupleAnotB.java +++ b/src/main/java/org/apache/datasketches/tuple/TupleAnotB.java @@ -295,7 +295,7 @@ public void notB(final ThetaSketch skB) { * * @param reset If true, clears this operator to the empty state after this result is * returned. Set this to false if you wish to obtain an intermediate result. - * @return the result of this operation as an unordered {@link CompactTupleSketch}. + * @return the result of this operation as an ordered {@link CompactTupleSketch}. */ public CompactTupleSketch getResult(final boolean reset) { final CompactTupleSketch result; @@ -331,7 +331,7 @@ public CompactTupleSketch getResult(final boolean reset) { * @param skA The incoming TupleSketch for the first argument * @param skB The incoming TupleSketch for the second argument * @param Type of Summary - * @return the result as an unordered {@link CompactTupleSketch} + * @return the result as an ordered {@link CompactTupleSketch} */ @SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", justification = "hashArr and summaryArr are guaranteed to be valid due to the switch on CornerCase") @@ -349,7 +349,7 @@ public static CompactTupleSketch aNotB( * @param skB The incoming TupleSketch for the second argument * @param seed See Update Hash Seed * @param Type of Summary - * @return the result as an unordered {@link CompactTupleSketch} + * @return the result as an ordered {@link CompactTupleSketch} * @throws SketchesArgumentException if the seed hash of a non-empty argument does not match the given seed */ @SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", @@ -447,7 +447,7 @@ public static CompactTupleSketch aNotB( * @param skA The incoming TupleSketch for the first argument * @param skB The incoming ThetaSketch for the second argument * @param Type of Summary - * @return the result as an unordered {@link CompactTupleSketch} + * @return the result as an ordered {@link CompactTupleSketch} */ @SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", justification = "hashArr and summaryArr are guaranteed to be valid due to the switch on CornerCase") @@ -465,7 +465,7 @@ public static CompactTupleSketch aNotB( * @param skB The incoming ThetaSketch for the second argument * @param seed See Update Hash Seed * @param Type of Summary - * @return the result as an unordered {@link CompactTupleSketch} + * @return the result as an ordered {@link CompactTupleSketch} * @throws SketchesArgumentException if the seed hash of a non-empty argument does not match the given seed */ @SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR", diff --git a/src/main/java/org/apache/datasketches/tuple/TupleIntersection.java b/src/main/java/org/apache/datasketches/tuple/TupleIntersection.java index 5aa61b81d..e58833f1c 100644 --- a/src/main/java/org/apache/datasketches/tuple/TupleIntersection.java +++ b/src/main/java/org/apache/datasketches/tuple/TupleIntersection.java @@ -75,10 +75,10 @@ public TupleIntersection(final SummarySetOperations summarySetOps, final long /** * Perform a stateless intersect set operation on the two given TupleSketches and returns the - * result as an unordered CompactTupleSketch on the heap. + * result as an ordered CompactTupleSketch on the heap. * @param tupleSketchA The first sketch argument. It must not be null. * @param tupleSketchB The second sketch argument. It must not be null. - * @return an unordered CompactTupleSketch on the heap + * @return an ordered CompactTupleSketch on the heap */ public CompactTupleSketch intersect( final TupleSketch tupleSketchA, @@ -93,12 +93,12 @@ public CompactTupleSketch intersect( /** * Perform a stateless intersect set operation on a TupleSketch and a ThetaSketch and returns the - * result as an unordered CompactTupleSketch on the heap. + * result as an ordered CompactTupleSketch on the heap. * @param tupleSketch The first sketch argument. It must not be null. * @param thetaSketch The second sketch argument. It must not be null. * @param summary the given proxy summary for the ThetaSketch, which doesn't have one. * This must not be null. - * @return an unordered CompactTupleSketch on the heap + * @return an ordered CompactTupleSketch on the heap */ public CompactTupleSketch intersect( final TupleSketch tupleSketch, @@ -201,7 +201,7 @@ public void intersect(final ThetaSketch thetaSketch, final S summary) { } /** - * Gets the internal set as an unordered CompactTupleSketch + * Gets the internal set as an ordered CompactTupleSketch * @return result of the intersections so far */ public CompactTupleSketch getResult() { diff --git a/src/main/java/org/apache/datasketches/tuple/TupleUnion.java b/src/main/java/org/apache/datasketches/tuple/TupleUnion.java index 549ec8c13..2f5c0a058 100644 --- a/src/main/java/org/apache/datasketches/tuple/TupleUnion.java +++ b/src/main/java/org/apache/datasketches/tuple/TupleUnion.java @@ -156,7 +156,7 @@ public void union(final ThetaSketch thetaSketch, final S summary) { } /** - * Gets the result of a sequence of stateful union operations as an unordered CompactTupleSketch + * Gets the result of a sequence of stateful union operations as an ordered CompactTupleSketch * @return result of the stateful unions so far. The state of this operation is not reset after the * result is returned. */ @@ -165,7 +165,7 @@ public CompactTupleSketch getResult() { } /** - * Gets the result of a sequence of stateful union operations as an unordered CompactTupleSketch. + * Gets the result of a sequence of stateful union operations as an ordered CompactTupleSketch. * @param reset If true, clears this operator to the empty state after this result is * returned. Set this to false if you wish to obtain an intermediate result. * @return result of the stateful union diff --git a/src/test/java/org/apache/datasketches/tuple/CompactTupleOrderedTest.java b/src/test/java/org/apache/datasketches/tuple/CompactTupleOrderedTest.java new file mode 100644 index 000000000..70643c62d --- /dev/null +++ b/src/test/java/org/apache/datasketches/tuple/CompactTupleOrderedTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.datasketches.tuple; + +import static org.apache.datasketches.tuple.PreambleUtil.COMPACT_FLAGS_BYTE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +import java.lang.foreign.MemorySegment; +import java.util.HashMap; +import java.util.Map; + +import org.apache.datasketches.tuple.adouble.DoubleSummary; +import org.apache.datasketches.tuple.adouble.DoubleSummaryDeserializer; +import org.apache.datasketches.tuple.adouble.DoubleSummaryFactory; +import org.apache.datasketches.tuple.adouble.DoubleSummarySetOperations; +import org.testng.annotations.Test; + +/** + * Checks that compact tuple sketches are ordered by hash, as in C++, and that the image says so. + */ +public class CompactTupleOrderedTest { + private static final int READ_ONLY = 1 << 1; + private static final int EMPTY = 1 << 2; + private static final int COMPACT = 1 << 3; + private static final int ORDERED = 1 << 4; + + private static UpdatableTupleSketch newSketch(final int n, final int offset) { + final UpdatableTupleSketch sk = + new UpdatableTupleSketchBuilder<>(new DoubleSummaryFactory(DoubleSummary.Mode.Sum)).build(); + for (int i = 0; i < n; i++) { sk.update(i + offset, (double) (i + offset)); } + return sk; + } + + //The hashes are strictly increasing. If src is given, every hash keeps the summary it has in src. + private static void checkOrdered(final CompactTupleSketch csk, final TupleSketch src) { + assertTrue(csk.isOrdered()); + final Map expected = new HashMap<>(); + if (src != null) { + final TupleSketchIterator sit = src.iterator(); + while (sit.next()) { expected.put(sit.getHash(), sit.getSummary().getValue()); } + } + final TupleSketchIterator it = csk.iterator(); + long prev = -1; + int count = 0; + while (it.next()) { + assertTrue(it.getHash() > prev); + prev = it.getHash(); + if (src != null) { assertEquals(it.getSummary().getValue(), expected.get(it.getHash())); } + count++; + } + assertEquals(count, csk.getRetainedEntries()); + } + + @Test + public void compactIsOrdered() { + for (final int n : new int[] {0, 1, 2, 17, 1000, 100_000}) { + final UpdatableTupleSketch sk = newSketch(n, 0); + final CompactTupleSketch csk = sk.compact(); + checkOrdered(csk, sk); + final int flags = csk.toByteArray()[COMPACT_FLAGS_BYTE] & 0xFF; + assertEquals(flags, READ_ONLY | COMPACT | ORDERED | ((n == 0) ? EMPTY : 0), "n = " + n); + } + } + + @Test + public void setOperationResultsAreOrdered() { + final UpdatableTupleSketch a = newSketch(10_000, 0); + final UpdatableTupleSketch b = newSketch(10_000, 5_000); + final DoubleSummarySetOperations ops = new DoubleSummarySetOperations(); + final TupleUnion union = new TupleUnion<>(ops); + union.union(a); + union.union(b); + //union and intersection combine summaries, so only the order is checked + checkOrdered(union.getResult(), null); + checkOrdered(new TupleUnion<>(ops).union(a, b), null); + checkOrdered(new TupleIntersection<>(ops).intersect(a, b), null); + checkOrdered(TupleAnotB.aNotB(a, b), a); + final TupleAnotB anotb = new TupleAnotB<>(); + anotb.setA(a); + anotb.notB(b); + final CompactTupleSketch r1 = anotb.getResult(false); + checkOrdered(r1, a); + assertEquals(anotb.getResult(false).toByteArray(), r1.toByteArray()); //the operator's own state is intact + } + + @Test + public void inputArraysAreNotChanged() { + final long[] hashes = {30, 10, 20}; + final DoubleSummary[] sums = {new DoubleSummary(DoubleSummary.Mode.Sum).update(3.0), + new DoubleSummary(DoubleSummary.Mode.Sum).update(1.0), new DoubleSummary(DoubleSummary.Mode.Sum).update(2.0)}; + final CompactTupleSketch csk = + new CompactTupleSketch<>(hashes, sums, Long.MAX_VALUE, false, Util.DEFAULT_SEED_HASH); + assertEquals(hashes, new long[] {30, 10, 20}); + assertEquals(sums[0].getValue(), 3.0); + final TupleSketchIterator it = csk.iterator(); + double v = 0; + while (it.next()) { + v++; + assertEquals(it.getHash(), (long) (v * 10)); + assertEquals(it.getSummary().getValue(), v); + } + } + + //An image without the ordered flag, as earlier Java releases wrote, reads back unordered and is written back + //unchanged. With at most one entry it is ordered regardless of the flag. + @Test + public void unorderedImageStaysUnordered() { + final DoubleSummaryDeserializer deser = new DoubleSummaryDeserializer(); + final byte[] bytes = newSketch(1000, 0).compact().toByteArray(); + bytes[COMPACT_FLAGS_BYTE] &= (byte) ~ORDERED; + final CompactTupleSketch csk = + (CompactTupleSketch) TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), deser); + assertFalse(csk.isOrdered()); + assertEquals(csk.toByteArray(), bytes); + for (final int n : new int[] {0, 1}) { + final byte[] small = newSketch(n, 0).compact().toByteArray(); + small[COMPACT_FLAGS_BYTE] &= (byte) ~ORDERED; + final CompactTupleSketch s = + (CompactTupleSketch) TupleSketch.heapifySketch(MemorySegment.ofArray(small), deser); + assertTrue(s.isOrdered()); + } + } +} diff --git a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java index 895213499..2c7bbbfe0 100644 --- a/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java +++ b/src/test/java/org/apache/datasketches/tuple/TupleCrossLanguageTest.java @@ -64,6 +64,22 @@ public void checkCpp() { deserializeTupleIntegerSummary(GroupLanguage.CPP); } + //Java sorts a compact tuple sketch by hash and sets the ordered flag, as C++ does, so the images are identical. + @Test(groups = {CHECK_CPP_FILES}) + public void checkCppBytesIdentical() { + final int[] nArr = {0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000}; + for (int n: nArr) { + final byte[] cppBytes = getFileBytes(GroupLanguage.CPP.pth, "tuple_int_n" + n + "_cpp.sk"); + if (cppBytes.length == 0) { continue; } + final UpdatableTupleSketch sk = + new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build(); + for (int i = 0; i < n; i++) { + sk.update(i, i); + } + Assert.assertEquals(sk.compact().toByteArray(), cppBytes, "n = " + n); + } + } + @Test(groups = {CHECK_GO_FILES}) public void checkGo() { deserializeTupleIntegerSummary(GroupLanguage.GO);