From aa45c7099e405bd5806a2c7e9c10b5e6b4ca5253 Mon Sep 17 00:00:00 2001 From: Lee Rhodes Date: Fri, 20 Nov 2026 13:18:56 -0800 Subject: [PATCH] Tuple: check empty compact images strictly, as theta does An empty serial version 3 CompactTupleSketch image must now have PreLongs=1, SerVer=3, Family=TUPLE, SketchType=CompactTupleSketch, a zero unused byte, and the read-only, empty and compact flags set with the reserved bits clear. The ordered flag and the seed hash are ignored, as in theta (#776), so every image written by Java since 3.1.0, C++ and Go still reads. Co-Authored-By: Claude Opus 5.5 --- .../tuple/CompactTupleSketch.java | 26 +++- .../tuple/TupleEmptyImageTest.java | 124 ++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/apache/datasketches/tuple/TupleEmptyImageTest.java diff --git a/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java b/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java index f02da0a21..750b8f962 100644 --- a/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java +++ b/src/main/java/org/apache/datasketches/tuple/CompactTupleSketch.java @@ -144,7 +144,11 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED final byte flags = seg.get(JAVA_BYTE, COMPACT_FLAGS_BYTE); final short seedHash = seg.get(JAVA_SHORT_UNALIGNED, SEED_HASH_SHORT); empty_ = (flags & (1 << Flags.IS_EMPTY.ordinal())) > 0; - if (!empty_) { checkSeedHashes(seedHash_, seedHash); } // an empty image carries zero, as in theta + if (empty_) { + checkEmptyPre0(seg.get(JAVA_LONG_UNALIGNED, 0)); // the seed hash of an empty image is ignored + } else { + checkSeedHashes(seedHash_, seedHash); + } thetaLong_ = Long.MAX_VALUE; int count = 0; if (!empty_) { @@ -169,6 +173,26 @@ private enum Flags { IS_RESERVED, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED } } + // The MASK selects which bits of the first 8 bytes of an empty image are examined. + // The TEST gives the required values of the examined bits; it lies within the MASK. + // + // An empty image must have + // bytes: PreLongs=1, SerVer=3, Family=TUPLE(9), SketchType=CompactTupleSketch(1), byte 4 = 0 (unused); + // flags: read-only, empty, compact set; reserved bits 0, 5, 6, 7 clear. + // The ordered flag is ignored: older Java writers omit it, C++ and Go set it. + // The seed hash bytes are ignored: they may be 0 or the seed hash, depending on the writer. + // Every Java (since 3.1.0), C++ and Go writer of this serial version writes empty images in this form. + static final long EMPTY_SKETCH_MASK = 0X00_00_EF_FF_FF_FF_FF_FFL; + static final long EMPTY_SKETCH_TEST = 0X00_00_0E_00_01_09_03_01L; + + private static void checkEmptyPre0(final long pre0) { + if ((pre0 & EMPTY_SKETCH_MASK) != EMPTY_SKETCH_TEST) { + throw new SketchesArgumentException("Possible corruption: an empty CompactTupleSketch image must have " + + "masked Pre0 " + Long.toHexString(EMPTY_SKETCH_TEST) + ", found: " + + Long.toHexString(pre0 & EMPTY_SKETCH_MASK)); + } + } + /** * Checks the retained entries count read from an image. Each entry is at least an 8-byte hash, * so the remaining bytes of the image bound the count. diff --git a/src/test/java/org/apache/datasketches/tuple/TupleEmptyImageTest.java b/src/test/java/org/apache/datasketches/tuple/TupleEmptyImageTest.java new file mode 100644 index 000000000..a3a81d19c --- /dev/null +++ b/src/test/java/org/apache/datasketches/tuple/TupleEmptyImageTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datasketches.tuple; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import static org.testng.Assert.fail; + +import java.lang.foreign.MemorySegment; + +import org.apache.datasketches.common.SketchesArgumentException; +import org.testng.annotations.Test; + +/** + * Checks that the CompactTupleSketch reader accepts every known form of an empty serial version 3 image + * and rejects an empty image whose first 8 bytes do not match that form. + */ +public class TupleEmptyImageTest { + + // PreLongs=1, SerVer=3, Family=9, SketchType=1, unused=0, flags, seed hash (2 bytes, little-endian) + private static byte[] emptyImage(final int flags, final int seedHash) { + return new byte[] {1, 3, 9, 1, 0, (byte) flags, (byte) seedHash, (byte) (seedHash >>> 8)}; + } + + private static TupleSketch heapify(final byte[] bytes) { + return TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer()); + } + + private static TupleSketch heapify(final byte[] bytes, final long seed) { + return TupleSketch.heapifySketch(MemorySegment.ofArray(bytes), new IntegerSummaryDeserializer(), seed); + } + + private static void assertEmpty(final TupleSketch sk) { + assertTrue(sk.isEmpty()); + assertEquals(sk.getRetainedEntries(), 0); + assertEquals(sk.getThetaLong(), Long.MAX_VALUE); + } + + private static void assertRejected(final byte[] bytes) { + try { + heapify(bytes); + fail("expected the empty image to be rejected"); + } catch (final SketchesArgumentException e) { + //expected + } + } + + @Test + public void acceptsKnownEmptyForms() { + final int defaultSeedHash = Short.toUnsignedInt(Util.DEFAULT_SEED_HASH); + assertEmpty(heapify(emptyImage(0x0e, 0))); //Java since 9.1.0 + assertEmpty(heapify(emptyImage(0x0e, defaultSeedHash))); //Java 3.1.0 to 9.0.0 + assertEmpty(heapify(emptyImage(0x1e, 0))); //C++, and Java with the ordered flag + assertEmpty(heapify(emptyImage(0x1e, defaultSeedHash))); //Go + } + + @Test + public void seedHashOfEmptyImageIsIgnored() { + assertEmpty(heapify(emptyImage(0x0e, 0x1234), 12345L)); + assertEmpty(heapify(emptyImage(0x1e, 0xffff), 12345L)); + } + + @Test + public void acceptsWhatJavaWrites() { + final byte[] bytes = new UpdatableTupleSketchBuilder<>(new IntegerSummaryFactory()).build() + .compact().toByteArray(); + assertEmpty(heapify(bytes)); + assertEmpty(heapify(TupleSketch.createEmptySketch().toByteArray())); + } + + @Test + public void acceptsTrailingBytes() { + final byte[] bytes = new byte[24]; + System.arraycopy(emptyImage(0x0e, 0), 0, bytes, 0, 8); + assertEmpty(heapify(bytes)); + } + + @Test + public void rejectsWrongPreambleLongs() { + for (final int preLongs : new int[] {2, 3}) { + final byte[] bytes = new byte[preLongs * Long.BYTES]; + System.arraycopy(emptyImage(0x0e, 0), 0, bytes, 0, 8); + bytes[0] = (byte) preLongs; + assertRejected(bytes); + } + } + + @Test + public void rejectsNonZeroUnusedByte() { + final byte[] bytes = emptyImage(0x0e, 0); + bytes[4] = 1; + assertRejected(bytes); + } + + @Test + public void rejectsMissingFlags() { + assertRejected(emptyImage(0x0c, 0)); //not read-only + assertRejected(emptyImage(0x06, 0)); //not compact + } + + @Test + public void rejectsReservedFlags() { + for (final int bit : new int[] {0, 5, 6, 7}) { + assertRejected(emptyImage(0x0e | (1 << bit), 0)); + } + } +}