Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,10 @@ public T getMaxItem() {
return serDe.deserializeFromMemory(mem, DATA_START_ADR_SINGLE_ITEM, 1)[0];
}
//sketchStructure == COMPACT_FULL
final int offset = DATA_START_ADR + getNumLevels() * Integer.BYTES;
return serDe.deserializeFromMemory(mem, offset, 2)[1];
final int baseOffset = DATA_START_ADR + getNumLevels() * Integer.BYTES;
final int offset = baseOffset + serDe.sizeOf(mem, baseOffset, 1); //size of minItem

return serDe.deserializeFromMemory(mem, offset, 1)[0];
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,7 @@ DoublesSketchSortedView getSV() {
cumWeights = new long[numQuantiles];
populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles);
return new DoublesSketchSortedView(
quantiles, cumWeights, getN(), getMaxItemInternal(), getMinItemInternal());
quantiles, cumWeights, KllDoublesSketch.this);
}

private void populateFromSketch(final double[] srcQuantiles, final int[] srcLevels,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -560,8 +560,7 @@ FloatsSketchSortedView getSV() {
quantiles = new float[numQuantiles];
cumWeights = new long[numQuantiles];
populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles);
return new FloatsSketchSortedView(
quantiles, cumWeights, getN(), getMaxItemInternal(), getMinItemInternal());
return new FloatsSketchSortedView(quantiles, cumWeights, KllFloatsSketch.this);
}

private void populateFromSketch(final float[] srcQuantiles, final int[] srcLevels,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ void incNumLevels() {
void setDoubleItemsArrayAt(final int index, final double item) { this.doubleItems[index] = item; }

@Override
void setDoubleItemsArrayAt(final int dstIndex, final double[] srcItems, final int srcOffset, final int length) { //TODO
void setDoubleItemsArrayAt(final int dstIndex, final double[] srcItems, final int srcOffset, final int length) {
System.arraycopy(srcItems, srcOffset, doubleItems, dstIndex, length);
}

Expand Down
33 changes: 17 additions & 16 deletions src/main/java/org/apache/datasketches/kll/KllItemsSketch.java
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ public byte[] toByteArray() {
@Override
public String toString(final boolean withLevels, final boolean withLevelsAndItems) {
KllSketch sketch = this;
if (withLevelsAndItems && sketchStructure != UPDATABLE) {
if (hasMemory()) {
final Memory mem = getWritableMemory();
assert mem != null;
sketch = KllItemsSketch.heapify((Memory)getWritableMemory(), comparator, serDe);
Expand Down Expand Up @@ -422,49 +422,50 @@ private final ItemsSketchSortedView<T> refreshSortedView() {
@SuppressWarnings({"rawtypes"})
private final class CreateSortedView {
T[] quantiles;
long[] cumWeights;
long[] cumWeights; //The new cumWeights array

ItemsSketchSortedView<T> getSV() {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
if (getN() == 0) { throw new SketchesArgumentException(EMPTY_MSG); }
if (isEmpty() || getN() == 0) { throw new SketchesArgumentException(EMPTY_MSG); }
final T[] srcQuantiles = getTotalItemsArray();
final int[] srcLevels = levelsArr;
final int[] srcLevelsArr = levelsArr;
final int srcNumLevels = getNumLevels();

if (!isLevelZeroSorted()) {
Arrays.sort(srcQuantiles, srcLevels[0], srcLevels[1], comparator);
Arrays.sort(srcQuantiles, srcLevelsArr[0], srcLevelsArr[1], comparator);
if (!hasMemory()) { setLevelZeroSorted(true); }
}
final int numQuantiles = getNumRetained();
quantiles = (T[]) Array.newInstance(serDe.getClassOfT(), numQuantiles);
cumWeights = new long[numQuantiles];
populateFromSketch(srcQuantiles, srcLevels, srcNumLevels, numQuantiles);
populateFromSketch(srcQuantiles, srcLevelsArr, srcNumLevels, numQuantiles);
final QuantilesGenericAPI<T> sk = KllItemsSketch.this;
return new ItemsSketchSortedView(quantiles, cumWeights, sk);
}

private void populateFromSketch(final Object[] srcQuantiles, final int[] srcLevels,
private void populateFromSketch(final Object[] srcQuantiles, final int[] srcLevelsArr,
final int srcNumLevels, final int numItems) {
final int[] myLevels = new int[srcNumLevels + 1];
final int offset = srcLevels[0];
System.arraycopy(srcQuantiles, offset, quantiles, 0, numItems);
//Remove free space from both itemsArray and levels array
final int[] myLevelsArr = new int[srcLevelsArr.length];

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unclear why using this instead of srcNumLevels+1 as before, but gonna trust it's fine either way

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without studying closely, the srcNumLevels + 1 makes it seem like we are increasing the size of the levelsArr, when we are not. It is the same size as the original, but with different offsets.

final int offset = srcLevelsArr[0];
System.arraycopy(srcQuantiles, offset, quantiles, 0, numItems); //remove free space from quantiles arr
//fill the new cumWeights array with the correct weights and adjust the levels array to match.
int srcLevel = 0;
int dstLevel = 0;
long weight = 1;
while (srcLevel < srcNumLevels) {
final int fromIndex = srcLevels[srcLevel] - offset;
final int toIndex = srcLevels[srcLevel + 1] - offset; // exclusive
final int fromIndex = srcLevelsArr[srcLevel] - offset;
final int toIndex = srcLevelsArr[srcLevel + 1] - offset; // exclusive
if (fromIndex < toIndex) { // if equal, skip empty level
Arrays.fill(cumWeights, fromIndex, toIndex, weight);
myLevels[dstLevel] = fromIndex;
myLevels[dstLevel + 1] = toIndex;
myLevelsArr[dstLevel] = fromIndex;
myLevelsArr[dstLevel + 1] = toIndex;
dstLevel++;
}
srcLevel++;
weight *= 2;
}
final int numLevels = dstLevel;
blockyTandemMergeSort(quantiles, cumWeights, myLevels, numLevels, comparator); //create unit weights
blockyTandemMergeSort(quantiles, cumWeights, myLevelsArr, numLevels, comparator); //create unit weights
KllHelper.convertToCumulative(cumWeights);
}
} //End of class CreateSortedView
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ static int computeSketchBytes( //for COMPACT_FULL or UPDATABLE only
final int levelsLen = updatable ? levelsArr.length : levelsArr.length - 1;
final int numItems = updatable ? capacityItems : retainedItems;

int offsetBytes = DATA_START_ADR + levelsLen * Integer.BYTES;
int offsetBytes = DATA_START_ADR + levelsLen * Integer.BYTES; //levels array
if (sketchType == ITEMS_SKETCH) {
if (serDe instanceof ArrayOfBooleansSerDe) {
offsetBytes += serDe.sizeOf(srcMem, offsetBytes, numItems) + 2; //2 for min & max
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@ private DoublesSketchSortedView getSV() {
if (convertToCumulative(svCumWeights) != totalN) {
throw new SketchesStateException("Sorted View is misconfigured. TotalN does not match cumWeights.");
}
return new DoublesSketchSortedView(svQuantiles, svCumWeights, totalN, getMaxItem(), getMinItem());
return new DoublesSketchSortedView(svQuantiles, svCumWeights, this);
}

private final static void populateFromDoublesSketch(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ static <T> void processFullBaseBuffer(final ItemsSketch<T> sketch) {
assert (n / (2L * sketch.getK())) == sketch.getBitPattern(); // internal consistency check
}

//TODO change booleans to "withLevels", "withLevelsAndItems"
static <T> String toString(final boolean withLevels, final boolean withLevelsAndItems,
final ItemsSketch<T> sk) {
final StringBuilder sb = new StringBuilder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.datasketches.quantilescommon;

import static org.apache.datasketches.quantilescommon.IncludeMinMax.DoublesPair;
import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE;
import static org.apache.datasketches.quantilescommon.QuantilesAPI.EMPTY_MSG;
import static org.apache.datasketches.quantilescommon.QuantilesUtil.getNaturalRank;
Expand All @@ -34,24 +35,36 @@ public final class DoublesSketchSortedView implements DoublesSortedView {
private final double[] quantiles;
private final long[] cumWeights; //cumulative natural weights
private final long totalN;
private final double maxItem;
private final double minItem;

/**
* Construct from elements, also used in testing.
* @param quantiles sorted array of quantiles
* @param cumWeights sorted, monotonically increasing cumulative weights.
* @param totalN the total number of items presented to the sketch.
* @param maxItem of type double
* @param minItem of type double
* @param sk the underlying quantile sketch.
*/
public DoublesSketchSortedView(final double[] quantiles, final long[] cumWeights, final long totalN,
final double maxItem, final double minItem) {
this.quantiles = quantiles;
this.cumWeights = cumWeights;
public DoublesSketchSortedView(
final double[] quantiles,
final long[] cumWeights,
final QuantilesDoublesAPI sk) {
final DoublesPair dPair =
IncludeMinMax.includeDoublesMinMax(quantiles, cumWeights, sk.getMaxItem(), sk.getMinItem());
this.quantiles = dPair.quantiles;
this.cumWeights = dPair.cumWeights;
this.totalN = sk.getN();
}

//Used for testing

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Constructors like this are ... not great. We should really be trying to find ways to test w/o special constructors like this. Not gonna "require changes" because of it, but just noting it.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some of the tests would be very difficult to reproduce without this.

DoublesSketchSortedView(
final double[] quantiles,
final long[] cumWeights,
final long totalN,
final double maxItem,
final double minItem) {
final DoublesPair dPair =
IncludeMinMax.includeDoublesMinMax(quantiles, cumWeights, maxItem, minItem);
this.quantiles = dPair.quantiles;
this.cumWeights = dPair.cumWeights;
this.totalN = totalN;
this.maxItem = maxItem;
this.minItem = minItem;
}

@Override
Expand All @@ -61,12 +74,13 @@ public long[] getCumulativeWeights() {

@Override
public double getMaxItem() {
return maxItem;
final int top = quantiles.length - 1;
return quantiles[top];
}

@Override
public double getMinItem() {
return minItem;
return quantiles[0];
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.datasketches.quantilescommon;

import static org.apache.datasketches.quantilescommon.IncludeMinMax.FloatsPair;
import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE;
import static org.apache.datasketches.quantilescommon.QuantilesAPI.EMPTY_MSG;
import static org.apache.datasketches.quantilescommon.QuantilesUtil.getNaturalRank;
Expand All @@ -34,24 +35,36 @@ public class FloatsSketchSortedView implements FloatsSortedView {
private final float[] quantiles;
private final long[] cumWeights; //cumulative natural weights
private final long totalN;
private final float maxItem;
private final float minItem;

/**
* Construct from elements, also used in testing.
* Construct Sorted View.
* @param quantiles sorted array of quantiles
* @param cumWeights sorted, monotonically increasing cumulative weights.
* @param totalN the total number of items presented to the sketch.
* @param maxItem of type double
* @param minItem of type double
* @param sk the underlying quantile sketch.
*/
public FloatsSketchSortedView(final float[] quantiles, final long[] cumWeights, final long totalN,
final float maxItem, final float minItem) {
this.quantiles = quantiles;
this.cumWeights = cumWeights;
public FloatsSketchSortedView(
final float[] quantiles,
final long[] cumWeights,
final QuantilesFloatsAPI sk) {
final FloatsPair fPair =
IncludeMinMax.includeFloatsMinMax(quantiles, cumWeights, sk.getMaxItem(), sk.getMinItem());
this.quantiles = fPair.quantiles;
this.cumWeights = fPair.cumWeights;
this.totalN = sk.getN();
}

//Used for testing
FloatsSketchSortedView(
final float[] quantiles,
final long[] cumWeights,
final long totalN,
final float maxItem,
final float minItem) {
final FloatsPair fPair =
IncludeMinMax.includeFloatsMinMax(quantiles, cumWeights, maxItem, minItem);
this.quantiles = fPair.quantiles;
this.cumWeights = fPair.cumWeights;
this.totalN = totalN;
this.maxItem = maxItem;
this.minItem = minItem;
}

@Override
Expand All @@ -61,12 +74,13 @@ public long[] getCumulativeWeights() {

@Override
public float getMaxItem() {
return maxItem;
final int top = quantiles.length - 1;
return quantiles[top];
}

@Override
public float getMinItem() {
return minItem;
return quantiles[0];
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.apache.datasketches.quantilescommon;

import static org.apache.datasketches.quantilescommon.QuantilesAPI.EMPTY_MSG;

import java.util.Comparator;

import org.apache.datasketches.common.SketchesArgumentException;
Expand Down Expand Up @@ -67,7 +69,23 @@ public interface GenericSortedView<T> extends PartitioningFeature<T>, SketchPar
* @return a discrete CDF array of m+1 double ranks (or cumulative probabilities) on the interval [0.0, 1.0].
* @throws IllegalArgumentException if sketch is empty.
*/
double[] getCDF(T[] splitPoints, QuantileSearchCriteria searchCrit);
default double[] getCDF(final T[] splitPoints, final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
GenericSortedView.validateItems(splitPoints, getComparator());
final int len = splitPoints.length + 1;
final double[] buckets = new double[len];
for (int i = 0; i < len - 1; i++) {
buckets[i] = getRank(splitPoints[i], searchCrit);
}
buckets[len - 1] = 1.0;
return buckets;
}

/**
* Gets the Comparator for this generic type.
* @return the Comparator for this generic type.
*/
Comparator<? super T> getComparator();

/**
* Returns the maximum item of the stream. This may be distinct from the largest item retained by the
Expand Down Expand Up @@ -127,7 +145,15 @@ public interface GenericSortedView<T> extends PartitioningFeature<T>, SketchPar
* @return a PMF array of m+1 probability masses as doubles on the interval [0.0, 1.0].
* @throws IllegalArgumentException if sketch is empty.
*/
double[] getPMF(T[] splitPoints, QuantileSearchCriteria searchCrit);
default double[] getPMF(final T[] splitPoints, final QuantileSearchCriteria searchCrit) {
if (isEmpty()) { throw new SketchesArgumentException(EMPTY_MSG); }
GenericSortedView.validateItems(splitPoints, getComparator());
final double[] buckets = getCDF(splitPoints, searchCrit);
for (int i = buckets.length; i-- > 1; ) {
buckets[i] -= buckets[i - 1];
}
return buckets;
}

/**
* Gets the approximate quantile of the given normalized rank and the given search criterion.
Expand Down
Loading