From 8acb012f56b357af7bb52b48ae106aec46ee6e55 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Thu, 12 Nov 2026 22:16:19 +0100 Subject: [PATCH] Decode the decimal min/max of a flat node's stats Rust writes a decimal bound as the little-endian two's-complement bytes of its unscaled value (1, 2, 4, 8, 16 or 32 bytes, ScalarValue::to_proto). The parser read every bytes_value as text, so a decimal bound never ordered and a filtered scan could not prune on it. ArrayStatsParser.parse takes the column dtype and builds a BigDecimal from the dtype's scale; an invalid width is rejected as Rust's bytes_from_proto does, which readFlatStats already reads as no stats. No vortex-jni cross-check: 0.93 writes no min/max on a decimal root node (byte parts or plain), so the path is only reachable from a file Rust writes with those stats; the unit tests pin Rust's encoding at every width. Fixes #710 Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../dfa1/vortex/reader/ArrayStatsParser.java | 55 ++++++- .../dfa1/vortex/reader/ScanIterator.java | 16 +- .../vortex/reader/ArrayStatsParserTest.java | 138 ++++++++++++++++++ .../ScanIteratorMalformedStatsTest.java | 79 ++++++++++ 5 files changed, 279 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daa9d8092..bfa1137d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Fixed +- A decimal min or max in a flat node's stats is decoded as a `BigDecimal` instead of text, so a filtered scan can prune on it and `columnStats()` reports it. ([#710](https://github.com/dfa1/vortex-java/issues/710)) - A `vortex.uuid` column's file statistics carry its min and max, as Rust writes them from 0.92, and `WriteSummary` reports them as `UUID`s. ([#730](https://github.com/dfa1/vortex-java/pull/730)) ### Changed diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java index 154902eb8..42e1b30a6 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java @@ -4,10 +4,13 @@ import io.github.dfa1.vortex.core.fbs.FbsArrayStats; import io.github.dfa1.vortex.core.fbs.FbsPrecision; import io.github.dfa1.vortex.core.model.ArrayStats; +import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.io.IOException; import java.lang.foreign.MemorySegment; +import java.math.BigDecimal; +import java.math.BigInteger; import java.nio.charset.StandardCharsets; /// Parses the wire `ArrayStats` FlatBuffer table into an [ArrayStats]: Rust's `from_flatbuffer`. @@ -23,11 +26,23 @@ private ArrayStatsParser() { /// @return parsed stats, or an empty instance if no usable data is present /// @throws VortexException if a precision field or a scalar value is malformed public static ArrayStats parse(FbsArrayStats fbs) { + return parse(fbs, null); + } + + /// Parses stats like [#parse(FbsArrayStats)], decoding the min and max against the column's + /// dtype: Rust writes a decimal bound as the little-endian two's-complement bytes of its unscaled + /// value, which only the dtype's scale turns into a number. + /// + /// @param fbs the FlatBuffers stats table, or `null` + /// @param dtype the dtype of the array the stats describe, or `null` when unknown + /// @return parsed stats, or an empty instance if no usable data is present + /// @throws VortexException if a precision field or a scalar value is malformed + public static ArrayStats parse(FbsArrayStats fbs, DType dtype) { if (fbs == null) { return ArrayStats.empty(); } - Object min = decodeScalar(fbs.minAsSegment()); - Object max = decodeScalar(fbs.maxAsSegment()); + Object min = decodeBound(fbs.minAsSegment(), dtype); + Object max = decodeBound(fbs.maxAsSegment(), dtype); ArrayStats stats = new ArrayStats( min, min != null && isExact(fbs.minPrecision(), "min_precision"), @@ -53,6 +68,42 @@ private static boolean isExact(int precision, String field) { }; } + private static Object decodeBound(MemorySegment seg, DType dtype) { + Object value = decodeScalar(seg); + // Rust's from_proto reads an extension's scalar as its storage dtype's. + while (dtype instanceof DType.Extension extension) { + dtype = extension.storageDType(); + } + if (!(dtype instanceof DType.Decimal decimal) || value == null) { + return value; + } + // Rust reads a decimal's BytesValue only (bytes_from_proto): any other kind is a corrupted + // bound, and must not reach a fold that orders BigDecimal against it. + byte[] bytes = decodeBytes(seg); + if (bytes == null) { + throw new VortexException("decimal bound in array stats is not a byte string: " + value); + } + // bytes_from_proto's match: the storage width, 1, 2, 4, 8, 16 or 32 bytes. + int width = bytes.length; + switch (width) { + case 1, 2, 4, 8, 16, 32 -> { } + default -> throw new VortexException("invalid decimal byte length in array stats: " + width); + } + byte[] bigEndian = new byte[width]; + for (int i = 0; i < width; i++) { + bigEndian[i] = bytes[width - 1 - i]; + } + return new BigDecimal(new BigInteger(bigEndian), decimal.scale()); + } + + private static byte[] decodeBytes(MemorySegment seg) { + try { + return ProtoScalarValue.decode(seg, 0, seg.byteSize()).bytes_value(); + } catch (IOException e) { + throw new VortexException("invalid scalar value in array stats", e); + } + } + private static Object decodeScalar(MemorySegment seg) { if (seg == null || seg.byteSize() == 0) { return null; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 504bde90b..306c39fac 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -626,12 +626,12 @@ public List chunkStats(ColumnName name) { if (layout != null && layout.isDict() && !layout.children().isEmpty()) { // child[0] = values, child[1] = codes — the order DictLayoutDecoder decodes. Layout values = layout.children().getFirst(); - ArrayStats pool = values.isFlat() ? readFlatStats(values) : ArrayStats.empty(); + ArrayStats pool = values.isFlat() ? readFlatStats(values, columnDType(name)) : ArrayStats.empty(); out.add(new ArrayStats(pool.min(), pool.minExact(), pool.max(), pool.maxExact(), null, null, null, null, null, null, null)); continue; } - out.add(layout == null || !layout.isFlat() ? ArrayStats.empty() : readFlatStats(layout)); + out.add(layout == null || !layout.isFlat() ? ArrayStats.empty() : readFlatStats(layout, columnDType(name))); } return out; } @@ -1165,7 +1165,7 @@ private boolean canPruneChunk(ChunkSpec chunk, ScanFilter filter) { // zones in a larger chunk), and the IS NOT NULL test compares nullCount against it. List zones = zonesOverlapping(chunk, col); if (zones == null) { - yield canPrune(predicate, readFlatStats(flat), flat.rowCount(), columnDType(col)) + yield canPrune(predicate, readFlatStats(flat, columnDType(col)), flat.rowCount(), columnDType(col)) || dictMatchesNothing(flat, col, predicate); } // The window has no matching row iff none of the zones covering it has one. @@ -1266,7 +1266,7 @@ private static boolean rejectsNull(Predicate predicate) { /// Returns the zones of `col` that together cover `chunk`'s window, in row order, or `null` when /// the column has no usable zone map or its zones do not reach the window's end — in which case /// the caller falls back to the chunk's own embedded stats. Reading here decodes (and caches) one - /// small zone-map segment per column for the whole scan, instead of [#readFlatStats(Layout)]'s + /// small zone-map segment per column for the whole scan, instead of [#readFlatStats(Layout, DType)]'s /// per-chunk read of the chunk's full data segment — the difference that makes pruning over /// HTTP actually cheaper than not pruning. private List zonesOverlapping(ChunkSpec chunk, ColumnName col) { @@ -1432,8 +1432,8 @@ static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DT Object min = stats.min(); Object max = stats.max(); if (ct instanceof DType.Decimal) { - // A flat node's stats scalar is decoded without the column's dtype, so a decimal bound - // arrives as text: not a bound this column can order, hence no bound at all. + // Only a BigDecimal orders against a decimal column: a bound of any other type (a stats + // scalar decoded without its dtype) is no bound at all. if (!(min instanceof java.math.BigDecimal)) { min = null; } @@ -1482,7 +1482,7 @@ static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DT }; } - private ArrayStats readFlatStats(Layout flat) { + private ArrayStats readFlatStats(Layout flat, DType dtype) { if (flat.segments().isEmpty()) { return ArrayStats.empty(); } @@ -1511,7 +1511,7 @@ private ArrayStats readFlatStats(Layout flat) { if (root == null) { return ArrayStats.empty(); } - return ArrayStatsParser.parse(root.stats()); + return ArrayStatsParser.parse(root.stats(), dtype); } catch (VortexException _) { // The FlatBuffer verifier rejected the stats segment: malformed stats read as none. return ArrayStats.empty(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java index 0380db247..671d77f14 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java @@ -3,9 +3,22 @@ import io.github.dfa1.vortex.core.fbs.FbsBuilder; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.ArrayStats; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ExtensionId; +import io.github.dfa1.vortex.core.model.OpaqueMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -258,6 +271,131 @@ void malformedScalarBytes_throwsVortexException() { } } + @Nested + class ParseDecimal { + + // Rust writes a decimal bound as the little-endian two's-complement bytes of its unscaled + // value, at the storage width: negative values and every width catch a sign or byte-order slip. + @ParameterizedTest + @ValueSource(ints = {1, 2, 4, 8, 16, 32}) + void bound_decodesAtEveryStorageWidth(int width) { + // Given + BigInteger min = BigInteger.valueOf(-100); + BigInteger max = BigInteger.valueOf(99); + var fbs = buildFbs( + ProtoScalarValue.ofBytesValue(littleEndian(min, width)).encode(), + ProtoScalarValue.ofBytesValue(littleEndian(max, width)).encode()); + + // When + ArrayStats result = ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) 2, false)); + + // Then + assertThat(result.min()).isEqualTo(new BigDecimal("-1.00")); + assertThat(result.max()).isEqualTo(new BigDecimal("0.99")); + } + + @Test + void negativeScale_keepsTheUnscaledValue() { + // Given + var fbs = buildFbs( + ProtoScalarValue.ofBytesValue(littleEndian(BigInteger.valueOf(5), 4)).encode(), + ProtoScalarValue.ofBytesValue(littleEndian(BigInteger.valueOf(7), 4)).encode()); + + // When + ArrayStats result = ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) -3, false)); + + // Then + assertThat(result.min()).isEqualTo(new BigDecimal("5E+3")); + assertThat(result.max()).isEqualTo(new BigDecimal("7E+3")); + } + + @ParameterizedTest + @ValueSource(ints = {0, 3, 5, 33}) + void bound_ofAnInvalidWidth_isRejected(int width) { + // Given + var fbs = buildFbs(ProtoScalarValue.ofBytesValue(new byte[width]).encode(), null); + + // When / Then + assertThatThrownBy(() -> ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) 2, false))) + .isInstanceOf(VortexException.class) + .hasMessageContaining("invalid decimal byte length"); + } + + // Rust reads only a BytesValue for a decimal (bytes_from_proto): a crafted file with another kind + // must raise VortexException, not an NPE, and never reach a fold ordering it against a BigDecimal. + @ParameterizedTest + @MethodSource("nonBytesScalars") + void bound_thatIsNotBytes_isRejected(ProtoScalarValue scalar) { + // Given + var fbs = buildFbs(scalar.encode(), null); + + // When / Then + assertThatThrownBy(() -> ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) 2, false))) + .isInstanceOf(VortexException.class) + .hasMessageContaining("not a byte string"); + } + + static Stream nonBytesScalars() { + return Stream.of( + ProtoScalarValue.ofStringValue("1.5"), + ProtoScalarValue.ofInt64Value(7L), + ProtoScalarValue.ofBoolValue(true)); + } + + // The small values above fit one byte; past a long only the 16- and 32-byte widths carry them. + @ParameterizedTest + @ValueSource(ints = {16, 32}) + void bound_beyondALong_decodes(int width) { + // Given + BigInteger max = BigInteger.TWO.pow(100); + var fbs = buildFbs( + ProtoScalarValue.ofBytesValue(littleEndian(max.negate(), width)).encode(), + ProtoScalarValue.ofBytesValue(littleEndian(max, width)).encode()); + + // When + ArrayStats result = ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 38, (byte) 0, false)); + + // Then + assertThat(result.min()).isEqualTo(new BigDecimal(max.negate())); + assertThat(result.max()).isEqualTo(new BigDecimal(max)); + } + + @Test + void bound_ofAnExtensionOverADecimal_decodesAsTheStorage() { + // Given + DType storage = new DType.Decimal((byte) 9, (byte) 2, false); + DType extension = new DType.Extension(ExtensionId.parse("acme.money"), storage, OpaqueMetadata.EMPTY, false); + var fbs = buildFbs(ProtoScalarValue.ofBytesValue(littleEndian(BigInteger.valueOf(150), 4)).encode(), null); + + // When + ArrayStats result = ArrayStatsParser.parse(fbs, extension); + + // Then + assertThat(result.min()).isEqualTo(new BigDecimal("1.50")); + } + + @Test + void bound_withoutADtype_staysTheRawText() { + // Given + var fbs = buildFbs(ProtoScalarValue.ofBytesValue(new byte[]{'1'}).encode(), null); + + // When + ArrayStats result = ArrayStatsParser.parse(fbs); + + // Then + assertThat(result.min()).isEqualTo("1"); + } + + private static byte[] littleEndian(BigInteger value, int width) { + // The shifts are arithmetic, so the limbs sign-extend and the truncation is two's complement. + ByteBuffer buffer = ByteBuffer.allocate(32).order(ByteOrder.LITTLE_ENDIAN); + for (int limb = 0; limb < 4; limb++) { + buffer.putLong(value.shiftRight(64 * limb).longValue()); + } + return Arrays.copyOf(buffer.array(), width); + } + } + private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes) { return buildFbs(minBytes, maxBytes, null); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java index 45dc36316..e6093e9a7 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java @@ -1,11 +1,16 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.fbs.FbsArray; +import io.github.dfa1.vortex.core.fbs.FbsArrayNode; +import io.github.dfa1.vortex.core.fbs.FbsArrayStats; +import io.github.dfa1.vortex.core.fbs.FbsBuilder; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.LayoutId; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.layout.Layout; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -13,6 +18,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.lang.foreign.MemorySegment; +import java.math.BigDecimal; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -54,4 +60,77 @@ void statsSegmentWithCorruptFlatBuffer_degradesToEmptyStats() { // Then assertThat(result).containsExactly(ArrayStats.empty()); } + + // The flat node's stats scalar is dtype-blind on the wire: only the column's dtype turns Rust's + // little-endian decimal bytes into a number, so the dtype must reach the parser (#710). + @Test + void decimalColumn_readsTheBoundsOfAFlatNodeAsBigDecimals() { + // Given + DType.Decimal decimal = new DType.Decimal((byte) 9, (byte) 2, false); + given(file.dtype()).willReturn(new DType.Struct(List.of(COLUMN), List.of(decimal), false)); + given(file.layout()).willReturn(new Layout(LayoutId.STRUCT, 5, null, + List.of(new Layout(LayoutId.FLAT, 5, null, List.of(), List.of(0))), List.of())); + MemorySegment seg = flatSegmentWithBounds( + ProtoScalarValue.ofBytesValue(new byte[]{(byte) 0x9C, (byte) 0xFF}).encode(), // -100 as an i16 + ProtoScalarValue.ofBytesValue(new byte[]{99, 0}).encode()); + given(file.footer()).willReturn(new Footer(List.of(), List.of(), + List.of(new SegmentSpec(0, seg.byteSize(), (byte) 0, CompressionScheme.NONE)), List.of())); + given(file.rawSegment(any())).willReturn(seg); + + // When + List result; + try (ScanIterator sut = new ScanIterator(file, ScanOptions.all())) { + result = sut.chunkStats(COLUMN); + } + + // Then + assertThat(result).hasSize(1); + assertThat(result.getFirst().min()).isEqualTo(new BigDecimal("-1.00")); + assertThat(result.getFirst().max()).isEqualTo(new BigDecimal("0.99")); + } + + @Test + void decimalColumn_withABoundThatIsNotBytes_degradesToEmptyStats() { + // Given — an NPE here would escape a scan, chunkStats and columnStats alike + DType.Decimal decimal = new DType.Decimal((byte) 9, (byte) 2, false); + given(file.dtype()).willReturn(new DType.Struct(List.of(COLUMN), List.of(decimal), false)); + given(file.layout()).willReturn(new Layout(LayoutId.STRUCT, 5, null, + List.of(new Layout(LayoutId.FLAT, 5, null, List.of(), List.of(0))), List.of())); + MemorySegment seg = flatSegmentWithBounds(ProtoScalarValue.ofStringValue("1.5").encode(), null); + given(file.footer()).willReturn(new Footer(List.of(), List.of(), + List.of(new SegmentSpec(0, seg.byteSize(), (byte) 0, CompressionScheme.NONE)), List.of())); + given(file.rawSegment(any())).willReturn(seg); + + // When + List result; + try (ScanIterator sut = new ScanIterator(file, ScanOptions.all())) { + result = sut.chunkStats(COLUMN); + } + + // Then + assertThat(result).containsExactly(ArrayStats.empty()); + } + + /// A data-less flat segment: the array FlatBuffer (one node, no buffers) then its 4-byte length, + /// the root node carrying `min`/`max` as its stats. + private static MemorySegment flatSegmentWithBounds(byte[] min, byte[] max) { + FbsBuilder fbb = new FbsBuilder(128); + int minOff = min == null ? 0 : FbsArrayStats.createMinVector(fbb, min); + int maxOff = max == null ? 0 : FbsArrayStats.createMaxVector(fbb, max); + FbsArrayStats.startFbsArrayStats(fbb); + if (min != null) { + FbsArrayStats.addMin(fbb, minOff); + } + if (max != null) { + FbsArrayStats.addMax(fbb, maxOff); + } + int statsOff = FbsArrayStats.endFbsArrayStats(fbb); + int nodeOff = FbsArrayNode.createFbsArrayNode(fbb, 0, 0, 0, 0, statsOff); + FbsArray.finishFbsArrayBuffer(fbb, FbsArray.createFbsArray(fbb, nodeOff, 0)); + byte[] flatBuffer = fbb.sizedByteArray(); + MemorySegment seg = MemorySegment.ofArray(new byte[flatBuffer.length + 4]); + MemorySegment.copy(MemorySegment.ofArray(flatBuffer), 0, seg, 0, flatBuffer.length); + seg.set(VortexFormat.LE_INT, flatBuffer.length, flatBuffer.length); + return seg; + } }