Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,11 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED
final byte flags = seg.get(JAVA_BYTE, COMPACT_FLAGS_BYTE);
final short seedHash = seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT);
empty_ = (flags & (1 << Flags.IS_EMPTY.ordinal())) > 0;
if (!empty_) { checkSeedHashes(seedHash_, seedHash); } // an empty image carries zero, as in theta
if (empty_) {
checkEmptyPre0(seg.get(JAVA_LONG_UNALIGNED, 0)); // the seed hash of an empty image is ignored
} else {
checkSeedHashes(seedHash_, seedHash);
}
thetaLong_ = Long.MAX_VALUE;
int count = 0;
if (!empty_) {
Expand All @@ -169,6 +173,26 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED
}
}

// The MASK selects which bits of the first 8 bytes of an empty image are examined.
// The TEST gives the required values of the examined bits; it lies within the MASK.
//
// An empty image must have
// bytes: PreLongs=1, SerVer=3, Family=TUPLE(9), SketchType=CompactTupleSketch(1), byte 4 = 0 (unused);
// flags: read-only, empty, compact set; reserved bits 0, 5, 6, 7 clear.
// The ordered flag is ignored: older Java writers omit it, C++ and Go set it.
// The seed hash bytes are ignored: they may be 0 or the seed hash, depending on the writer.
// Every Java (since 3.1.0), C++ and Go writer of this serial version writes empty images in this form.
static final long EMPTY_SKETCH_MASK = 0X00_00_EF_FF_FF_FF_FF_FFL;
static final long EMPTY_SKETCH_TEST = 0X00_00_0E_00_01_09_03_01L;

private static void checkEmptyPre0(final long pre0) {
if ((pre0 & EMPTY_SKETCH_MASK) != EMPTY_SKETCH_TEST) {
throw new SketchesArgumentException("Possible corruption: an empty CompactTupleSketch image must have "
+ "masked Pre0 " + Long.toHexString(EMPTY_SKETCH_TEST) + ", found: "
+ Long.toHexString(pre0 & EMPTY_SKETCH_MASK));
}
}

/**
* Checks the retained entries count read from an image. Each entry is at least an 8-byte hash,
* so the remaining bytes of the image bound the count.
Expand Down
124 changes: 124 additions & 0 deletions src/test/java/org/apache/datasketches/tuple/TupleEmptyImageTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.datasketches.tuple;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;

import java.lang.foreign.MemorySegment;

import org.apache.datasketches.common.SketchesArgumentException;
import org.testng.annotations.Test;

/**
* Checks that the CompactTupleSketch reader accepts every known form of an empty serial version 3 image
* and rejects an empty image whose first 8 bytes do not match that form.
*/
public class TupleEmptyImageTest {

// PreLongs=1, SerVer=3, Family=9, SketchType=1, unused=0, flags, seed hash (2 bytes, little-endian)
private static byte[] emptyImage(final int flags, final int seedHash) {
return new byte[] {1, 3, 9, 1, 0, (byte) flags, (byte) seedHash, (byte) (seedHash >>> 8)};
}

private static TupleSketch<IntegerSummary> heapify(final byte[] bytes) {
return TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer());
}

private static TupleSketch<IntegerSummary> heapify(final byte[] bytes, final long seed) {
return TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer(), seed);
}

private static void assertEmpty(final TupleSketch<IntegerSummary> sk) {
assertTrue(sk.isEmpty());
assertEquals(sk.getRetainedEntries(), 0);
assertEquals(sk.getThetaLong(), Long.MAX_VALUE);
}

private static void assertRejected(final byte[] bytes) {
try {
heapify(bytes);
fail("expected the empty image to be rejected");
} catch (final SketchesArgumentException e) {
//expected
}
}

@Test
public void acceptsKnownEmptyForms() {
final int defaultSeedHash = Short.toUnsignedInt(Util.DEFAULT_SEED_HASH);
assertEmpty(heapify(emptyImage(0x0e, 0))); //Java since 9.1.0
assertEmpty(heapify(emptyImage(0x0e, defaultSeedHash))); //Java 3.1.0 to 9.0.0
assertEmpty(heapify(emptyImage(0x1e, 0))); //C++, and Java with the ordered flag
assertEmpty(heapify(emptyImage(0x1e, defaultSeedHash))); //Go
}

@Test
public void seedHashOfEmptyImageIsIgnored() {
assertEmpty(heapify(emptyImage(0x0e, 0x1234), 12345L));
assertEmpty(heapify(emptyImage(0x1e, 0xffff), 12345L));
}

@Test
public void acceptsWhatJavaWrites() {
final byte[] bytes = new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build()
.compact().toByteArray();
assertEmpty(heapify(bytes));
assertEmpty(heapify(TupleSketch.<IntegerSummary>createEmptySketch().toByteArray()));
}

@Test
public void acceptsTrailingBytes() {
final byte[] bytes = new byte[24];
System.arraycopy(emptyImage(0x0e, 0), 0, bytes, 0, 8);
assertEmpty(heapify(bytes));
}

@Test
public void rejectsWrongPreambleLongs() {
for (final int preLongs : new int[] {2, 3}) {
final byte[] bytes = new byte[preLongs * Long.BYTES];
System.arraycopy(emptyImage(0x0e, 0), 0, bytes, 0, 8);
bytes[0] = (byte) preLongs;
assertRejected(bytes);
}
}

@Test
public void rejectsNonZeroUnusedByte() {
final byte[] bytes = emptyImage(0x0e, 0);
bytes[4] = 1;
assertRejected(bytes);
}

@Test
public void rejectsMissingFlags() {
assertRejected(emptyImage(0x0c, 0)); //not read-only
assertRejected(emptyImage(0x06, 0)); //not compact
}

@Test
public void rejectsReservedFlags() {
for (final int bit : new int[] {0, 5, 6, 7}) {
assertRejected(emptyImage(0x0e | (1 << bit), 0));
}
}
}
Loading