Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -52,21 +52,25 @@
* the compact() method of an UpdatableTupleSketch or as a result of the getResult()
* method of a set operation like TupleUnion, TupleIntersection or TupleAnotB.
* CompactTupleSketch consists of a compact list (i.e. no intervening spaces) of hash values,
* corresponding list of Summaries, and a value for theta. The lists may or may
* not be ordered. CompactTupleSketch is read-only.
* corresponding list of Summaries, and a value for theta. A CompactTupleSketch created by this
* library is ordered: its entries are sorted by hash value, as in C++. One deserialized from an
* unordered image, such as those written by earlier Java releases, keeps the order of that image.
* CompactTupleSketch is read-only.
*
* @param <S> type of Summary
*/
public final class CompactTupleSketch<S extends Summary> extends TupleSketch<S> {
private final long[] hashArr_;
private S[] summaryArr_;
private final boolean ordered_;

private enum FlagsLegacy { IS_BIG_ENDIAN, IS_EMPTY, HAS_ENTRIES, IS_THETA_INCLUDED }

private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED }

/**
* Create a CompactTupleSketch from correct components
* Create a CompactTupleSketch from correct components. The entries are sorted by hash value. If they are
* not already sorted, the given arrays are not changed: sorted copies are used instead.
* @param hashArr compacted hash array
* @param summaryArr compacted summary array
* @param thetaLong long value of theta
Expand All @@ -78,8 +82,60 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED
super(thetaLong, empty, null, seedHash);
super.thetaLong_ = thetaLong;
super.empty_ = empty;
hashArr_ = hashArr;
summaryArr_ = summaryArr;
if ((hashArr == null) || (summaryArr == null) || isSorted(hashArr)) { //callers pass both or neither
hashArr_ = hashArr;
summaryArr_ = summaryArr;
} else {
hashArr_ = hashArr.clone();
summaryArr_ = summaryArr.clone();
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
sortByHash(hashArr_, summaryArr_, hashArr_.clone(), summaryArr_.clone(), 0, hashArr_.length);
}
ordered_ = true;
}

private static boolean isSorted(final long[] hashArr) {
for (int i = 1; i < hashArr.length; i++) {
if (hashArr[i - 1] > hashArr[i]) { return false; }
}
return true;
}

// Sorts hashArr[lo, hi) and moves the summaries with their hashes: a merge sort, with tmpHash and
// tmpSum as scratch space of the same length.
private static <T> void sortByHash(final long[] hashArr, final T[] sumArr, final long[] tmpHash,
final T[] tmpSum, final int lo, final int hi) {
if ((hi - lo) <= 16) { //insertion sort
for (int i = lo + 1; i < hi; i++) {
final long h = hashArr[i];
final T sm = sumArr[i];
int j = i - 1;
while ((j >= lo) && (hashArr[j] > h)) {
hashArr[j + 1] = hashArr[j];
sumArr[j + 1] = sumArr[j];
j--;
}
hashArr[j + 1] = h;
sumArr[j + 1] = sm;
}
return;
}
final int mid = (lo + hi) >>> 1;
sortByHash(hashArr, sumArr, tmpHash, tmpSum, lo, mid);
sortByHash(hashArr, sumArr, tmpHash, tmpSum, mid, hi);
if (hashArr[mid - 1] <= hashArr[mid]) { return; } //already in order
System.arraycopy(hashArr, lo, tmpHash, lo, hi - lo);
System.arraycopy(sumArr, lo, tmpSum, lo, hi - lo);
int i = lo;
int j = mid;
for (int k = lo; k < hi; k++) {
if ((j >= hi) || ((i < mid) && (tmpHash[i] <= tmpHash[j]))) {
hashArr[k] = tmpHash[i];
sumArr[k] = tmpSum[i++];
} else {
hashArr[k] = tmpHash[j];
sumArr[k] = tmpSum[j++];
}
}
}

/**
Expand Down Expand Up @@ -139,6 +195,7 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED
hashArr_ = new long[0];
summaryArr_ = null;
}
ordered_ = hashArr_.length <= 1; // the legacy format has no ordered flag
} else { // current serial format
checkBounds(0, (long) preambleLongs * Long.BYTES, seg.byteSize());
final byte flags = seg.get(JAVA_BYTE, COMPACT_FLAGS_BYTE);
Expand Down Expand Up @@ -166,6 +223,7 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED
offset += Long.BYTES;
offset += readSummary(seg, offset, i, count, deserializer);
}
ordered_ = (count <= 1) || ((flags & (1 << Flags.IS_ORDERED.ordinal())) > 0);
}
}

Expand Down Expand Up @@ -201,6 +259,14 @@ public CompactTupleSketch<S> compact() {
return this;
}

/**
* Returns true if the entries are sorted by hash value. A sketch with no more than one entry is ordered.
* @return true if the entries are sorted by hash value
*/
boolean isOrdered() {
return ordered_;
}

long[] getHashArr() {
return hashArr_;
}
Expand Down Expand Up @@ -249,6 +315,7 @@ public byte[] toByteArray() {
(1 << Flags.IS_COMPACT.ordinal())
| (1 << Flags.IS_READ_ONLY.ordinal())
| (isEmpty() ? 1 << Flags.IS_EMPTY.ordinal() : 0)
| (ordered_ ? 1 << Flags.IS_ORDERED.ordinal() : 0)
);
// an empty image carries zero, as in theta
ByteArrayUtil.putShortLE(bytes, SEED_HASH_SHORT, isEmpty() ? 0 : seedHash_);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ public static <S extends Summary> boolean dissimilarityTest(
* @param summarySetOps instance of SummarySetOperations used to unify summaries.
* @param seed the update seed
* @param <S> Summary
* @return the unordered union of sketch A and sketch B
* @return the ordered union of sketch A and sketch B
*/
private static <S extends Summary> TupleSketch<S> computeUnion(
final TupleSketch<S> sketchA,
Expand All @@ -540,7 +540,7 @@ private static <S extends Summary> TupleSketch<S> computeUnion(
* @param summarySetOps instance of SummarySetOperations used to unify summaries.
* @param seed the update seed
* @param <S> Summary
* @return the unordered union of sketch A and sketch B
* @return the ordered union of sketch A and sketch B
*/
private static <S extends Summary> TupleSketch<S> computeUnion(
final TupleSketch<S> sketchA,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@
* <p>This form is binary compatible with the C++ and Go compact tuple sketches, as long as
* every language uses the same Summary encoding.</p>
*
* <p>Flags: ReadOnly, Compact, Empty if empty. The Ordered flag is defined but Java does not set it.
* <p>Flags: ReadOnly, Compact, Empty if empty, Ordered if the entries are sorted by hash value. A sketch with
* at most one entry is always ordered, as in C++.
* The seed hash is zero when the sketch is empty, as in theta.</p>
*
* <p>An empty sketch, or a single-item sketch that is not in estimation mode, uses 1 preamble long.
Expand Down
10 changes: 5 additions & 5 deletions src/main/java/org/apache/datasketches/tuple/TupleAnotB.java
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ public void notB(final ThetaSketch skB) {
*
* @param reset If <i>true</i>, clears this operator to the empty state after this result is
* returned. Set this to <i>false</i> if you wish to obtain an intermediate result.
* @return the result of this operation as an unordered {@link CompactTupleSketch}.
* @return the result of this operation as an ordered {@link CompactTupleSketch}.
*/
public CompactTupleSketch<S> getResult(final boolean reset) {
final CompactTupleSketch<S> result;
Expand Down Expand Up @@ -331,7 +331,7 @@ public CompactTupleSketch<S> getResult(final boolean reset) {
* @param skA The incoming TupleSketch for the first argument
* @param skB The incoming TupleSketch for the second argument
* @param <S> Type of Summary
* @return the result as an unordered {@link CompactTupleSketch}
* @return the result as an ordered {@link CompactTupleSketch}
*/
@SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR",
justification = "hashArr and summaryArr are guaranteed to be valid due to the switch on CornerCase")
Expand All @@ -349,7 +349,7 @@ public static <S extends Summary> CompactTupleSketch<S> aNotB(
* @param skB The incoming TupleSketch for the second argument
* @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See Update Hash Seed</a>
* @param <S> Type of Summary
* @return the result as an unordered {@link CompactTupleSketch}
* @return the result as an ordered {@link CompactTupleSketch}
* @throws SketchesArgumentException if the seed hash of a non-empty argument does not match the given seed
*/
@SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR",
Expand Down Expand Up @@ -447,7 +447,7 @@ public static <S extends Summary> CompactTupleSketch<S> aNotB(
* @param skA The incoming TupleSketch for the first argument
* @param skB The incoming ThetaSketch for the second argument
* @param <S> Type of Summary
* @return the result as an unordered {@link CompactTupleSketch}
* @return the result as an ordered {@link CompactTupleSketch}
*/
@SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR",
justification = "hashArr and summaryArr are guaranteed to be valid due to the switch on CornerCase")
Expand All @@ -465,7 +465,7 @@ public static <S extends Summary> CompactTupleSketch<S> aNotB(
* @param skB The incoming ThetaSketch for the second argument
* @param seed <a href="{@docRoot}/resources/dictionary.html#seed">See Update Hash Seed</a>
* @param <S> Type of Summary
* @return the result as an unordered {@link CompactTupleSketch}
* @return the result as an ordered {@link CompactTupleSketch}
* @throws SketchesArgumentException if the seed hash of a non-empty argument does not match the given seed
*/
@SuppressFBWarnings(value = "UWF_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ public TupleIntersection(final SummarySetOperations<S> summarySetOps, final long

/**
* Perform a stateless intersect set operation on the two given TupleSketches and returns the
* result as an unordered CompactTupleSketch on the heap.
* result as an ordered CompactTupleSketch on the heap.
* @param tupleSketchA The first sketch argument. It must not be null.
* @param tupleSketchB The second sketch argument. It must not be null.
* @return an unordered CompactTupleSketch on the heap
* @return an ordered CompactTupleSketch on the heap
*/
public CompactTupleSketch<S> intersect(
final TupleSketch<S> tupleSketchA,
Expand All @@ -93,12 +93,12 @@ public CompactTupleSketch<S> intersect(

/**
* Perform a stateless intersect set operation on a TupleSketch and a ThetaSketch and returns the
* result as an unordered CompactTupleSketch on the heap.
* result as an ordered CompactTupleSketch on the heap.
* @param tupleSketch The first sketch argument. It must not be null.
* @param thetaSketch The second sketch argument. It must not be null.
* @param summary the given proxy summary for the ThetaSketch, which doesn't have one.
* This must not be null.
* @return an unordered CompactTupleSketch on the heap
* @return an ordered CompactTupleSketch on the heap
*/
public CompactTupleSketch<S> intersect(
final TupleSketch<S> tupleSketch,
Expand Down Expand Up @@ -201,7 +201,7 @@ public void intersect(final ThetaSketch thetaSketch, final S summary) {
}

/**
* Gets the internal set as an unordered CompactTupleSketch
* Gets the internal set as an ordered CompactTupleSketch
* @return result of the intersections so far
*/
public CompactTupleSketch<S> getResult() {
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/apache/datasketches/tuple/TupleUnion.java
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ public void union(final ThetaSketch thetaSketch, final S summary) {
}

/**
* Gets the result of a sequence of stateful <i>union</i> operations as an unordered CompactTupleSketch
* Gets the result of a sequence of stateful <i>union</i> operations as an ordered CompactTupleSketch
* @return result of the stateful unions so far. The state of this operation is not reset after the
* result is returned.
*/
Expand All @@ -165,7 +165,7 @@ public CompactTupleSketch<S> getResult() {
}

/**
* Gets the result of a sequence of stateful <i>union</i> operations as an unordered CompactTupleSketch.
* Gets the result of a sequence of stateful <i>union</i> operations as an ordered CompactTupleSketch.
* @param reset If <i>true</i>, clears this operator to the empty state after this result is
* returned. Set this to <i>false</i> if you wish to obtain an intermediate result.
* @return result of the stateful union
Expand Down
Loading
Loading