Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,13 @@ final class DirectHll8Array extends DirectHllArray {
//Called by HllSketch.writableWrap(), DirectCouponList.promoteListOrSetToHll
DirectHll8Array(final int lgConfigK, final MemorySegment wseg) {
super(lgConfigK, TgtHllType.HLL_8, wseg);
Hll8Array.checkSlots(wseg, lgConfigK);
}

//Called by HllSketch.wrap(MemorySegment)
DirectHll8Array(final int lgConfigK, final MemorySegment seg, final boolean readOnly) {
super(lgConfigK, TgtHllType.HLL_8, seg, readOnly);
Hll8Array.checkSlots(seg, lgConfigK);
}

@Override
Expand Down
30 changes: 30 additions & 0 deletions src/main/java/org/apache/datasketches/hll/Hll8Array.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,16 @@

package org.apache.datasketches.hll;

import static java.lang.foreign.ValueLayout.JAVA_BYTE;
import static org.apache.datasketches.hll.HllUtil.EMPTY;
import static org.apache.datasketches.hll.HllUtil.KEY_BITS_26;
import static org.apache.datasketches.hll.HllUtil.VAL_MASK_6;
import static org.apache.datasketches.hll.PreambleUtil.HLL_BYTE_ARR_START;
import static org.apache.datasketches.hll.PreambleUtil.extractLgK;

import java.lang.foreign.MemorySegment;

import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.common.SketchesStateException;

/**
Expand Down Expand Up @@ -55,9 +58,36 @@ static Hll8Array heapify(final MemorySegment seg) {
final int lgConfigK = extractLgK(seg);
final Hll8Array hll8Array = new Hll8Array(lgConfigK);
HllArray.extractCommonHll(seg, hll8Array);
checkSlots(hll8Array.hllByteArr);
return hll8Array;
}

/**
* A valid HLL8 rank is {@code 0..63}. Readers mask with {@link HllUtil#VAL_MASK_6}, so a
* byte of 64 would be treated as an empty slot and the estimate would drop. Reject it here.
* @param slots the HLL8 register bytes
*/
static void checkSlots(final byte[] slots) {
for (int i = 0; i < slots.length; i++) {
final int value = slots[i] & 0xFF;
if (value > VAL_MASK_6) {
throw new SketchesArgumentException(
"Possible Corruption: HLL8 slot " + i + " value " + value + " > " + VAL_MASK_6);
}
}
}

static void checkSlots(final MemorySegment seg, final int lgConfigK) {
final int bytes = hll8ArrBytes(lgConfigK);
for (int i = 0; i < bytes; i++) {
final int value = seg.get(JAVA_BYTE, HLL_BYTE_ARR_START + i) & 0xFF;
if (value > VAL_MASK_6) {
throw new SketchesArgumentException(
"Possible Corruption: HLL8 slot " + i + " value " + value + " > " + VAL_MASK_6);
}
}
}

@Override
Hll8Array copy() {
return new Hll8Array(this);
Expand Down
11 changes: 11 additions & 0 deletions src/test/java/org/apache/datasketches/hll/HllCorruptImageTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,17 @@ public void checkValidImagesStillWork() {
}
}

@Test
public void checkHll8SlotAbove63() {
final byte[] bytes = sketch(TgtHllType.HLL_8, 10_000).toUpdatableByteArray();
assertEquals(HllSketch.heapify(bytes).getCurMode(), CurMode.HLL);
bytes[bytes.length - 1] = 64; // bit 6 set; the 6-bit mask would read this as 0
final SketchesArgumentException e = expectThrows(
SketchesArgumentException.class, () -> HllSketch.heapify(bytes));
assertTrue(e.getMessage().contains("HLL8 slot"));
checkAllPaths(bytes);
}

@Test
public void checkLgKTooLarge() {
for (final TgtHllType type : TgtHllType.values()) {
Expand Down