From 5e07ffd93df3fec69063a967b1dfb7d901b4f6e7 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 7 Nov 2026 22:38:58 +0100 Subject: [PATCH] fix(reader): columnStats reports decimal min/max, and decimal columns compare by value (#583) The zone-table decode boxed primitives, bools and strings but not decimals, so a decimal column (Java- or Rust-written) had no min/max in VortexReader.columnStats(), and zone-map pruning of decimal filters never fired. Compare also treated a BigDecimal as a Number and fell to the long compare, so 1.5 and 1.2 were equal. Decimal and unresolved-column compares now go by value, against another decimal or a long, big integer or finite float operand (anything else throws VortexException); primitive columns keep their own order, so an unsigned or floating column is unchanged whatever the operand. ZoneMapPruningTest checks each zone's decoded decimal bounds and the rows of nine filtered scans; DecimalBytePartsInteropIntegrationTest checks column stats on a vortex-jni file and on Java's own, and Java's filtered read of both. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011fRbYsrGtpKHp9APhU8qDV --- CHANGELOG.md | 1 + docs/compatibility.md | 4 +- ...ecimalBytePartsInteropIntegrationTest.java | 90 ++++++++++++++++++ .../dfa1/vortex/reader/ScanIterator.java | 2 + .../dfa1/vortex/reader/compute/Compare.java | 30 +++++- .../vortex/reader/compute/CompareTest.java | 72 ++++++++++++++ .../vortex/writer/ZoneMapPruningTest.java | 94 +++++++++++++++++++ 7 files changed, 288 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2653cdb37..8e6b8de79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - FSST symbol tables are trained as Rust's `fsst-rs` does (its sample, generations, candidate gains and cut), so a text column without a null gets Rust's FSST encoding up to ties of equal gain, its offsets a `vortex.sequence` where Rust's cascade finds one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650)) - A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) diff --git a/docs/compatibility.md b/docs/compatibility.md index 839b8ff3a..c67b128d1 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -168,8 +168,8 @@ it cascades into alike: So a Java-written string column has no bound in its flat segments, as a Rust-written one has none: `VortexReader.columnStats()` reports no string `min`/`max` for either, and a string bound lives only in the -zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the -zone table's are not decoded yet. +zoned layout's stats table. A decimal column's min and max are decoded from the zone table too: `columnStats()` reports them and +zone-map pruning uses them, compared by value at the column's scale. The nodes below the root follow the same table, plus three shapes of their own: a `vortex.sequence` carries `is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP, ALP-RD and bit-packed diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java index 6be80a00c..6cb469671 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java @@ -12,6 +12,10 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.reader.ScanFilter; +import io.github.dfa1.vortex.reader.array.DecimalArray; +import io.github.dfa1.vortex.reader.VortexReader; import io.github.dfa1.vortex.writer.WriteOptions; import io.github.dfa1.vortex.writer.encode.NullableData; import org.apache.arrow.c.ArrowArray; @@ -125,6 +129,92 @@ void jniReadsJavaFile(String name, int precision, int scale, IntToLongFunction u assertThat(range).isEqualTo(expected.subList(1000, 20_000)); } + /// The file's decimal min, max and null count, which Rust keeps in the zone table: read by Java + /// from a vortex-jni file and from its own, they are the extremes of the valid rows. + @ParameterizedTest(name = "{0}") + @MethodSource("nullableShapes") + void columnStats_areTheExtremesOfTheValidRows(String name, int precision, int scale, IntToLongFunction unscaled, + IntPredicate isNull, @TempDir Path tmp) throws Exception { + // Given + Path jniFile = tmp.resolve("jni.vortex"); + Path javaFile = tmp.resolve("java.vortex"); + writeWithJni(jniFile, precision, scale, unscaled, isNull); + writeWithJava(javaFile, precision, scale, unscaled, isNull); + BigDecimal min = null; + BigDecimal max = null; + long nulls = 0; + for (int i = 0; i < ROWS; i++) { + if (isNull.test(i)) { + nulls++; + continue; + } + BigDecimal value = BigDecimal.valueOf(unscaled.applyAsLong(i), scale); + min = min == null || value.compareTo(min) < 0 ? value : min; + max = max == null || value.compareTo(max) > 0 ? value : max; + } + + // When + ArrayStats fromJni = columnStats(jniFile); + ArrayStats fromJava = columnStats(javaFile); + + // Then + for (ArrayStats stats : List.of(fromJni, fromJava)) { + assertThat((BigDecimal) stats.min()).isEqualByComparingTo(min); + assertThat((BigDecimal) stats.max()).isEqualByComparingTo(max); + assertThat(stats.nullCount()).isEqualTo(nulls); + } + } + + /// Pruning trusts each zone's decimal bounds, so a filtered read by Java of a vortex-jni file (whose + /// zone table Rust wrote) and of Java's own must return exactly the matching rows. + @ParameterizedTest(name = "{0}") + @MethodSource("shapes") + void filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision, int scale, + IntToLongFunction unscaled, @TempDir Path tmp) + throws Exception { + // Given + Path jniFile = tmp.resolve("jni.vortex"); + Path javaFile = tmp.resolve("java.vortex"); + writeWithJni(jniFile, precision, scale, unscaled); + writeWithJava(javaFile, precision, scale, unscaled); + List all = new ArrayList<>(ROWS); + for (int i = 0; i < ROWS; i++) { + all.add(BigDecimal.valueOf(unscaled.applyAsLong(i), scale)); + } + BigDecimal threshold = all.get(ROWS / 2); + + // When + List fromJni = readWithJava(jniFile, ScanFilter.gte(D, threshold)); + List fromJava = readWithJava(javaFile, ScanFilter.gte(D, threshold)); + + // Then + List expected = all.stream().filter(v -> v.compareTo(threshold) >= 0).toList(); + assertThat(fromJni).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); + assertThat(fromJava).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); + } + + private static List readWithJava(Path file, ScanFilter filter) throws IOException { + List out = new ArrayList<>(); + try (var reader = VortexReader.open(file); + var scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all().withFilter(filter))) { + while (scan.hasNext()) { + try (var chunk = scan.next()) { + DecimalArray array = chunk.column(D); + for (int i = 0; i < chunk.rowCount(); i++) { + out.add(array.getDecimal(i)); + } + } + } + } + return out; + } + + private static ArrayStats columnStats(Path file) throws IOException { + try (var reader = VortexReader.open(file)) { + return reader.columnStats().get(D); + } + } + static Stream nullableShapes() { return Stream.of( // nullable but every row valid: written without a validity, as Rust's AllValid diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index dd019126c..773dd826d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -16,6 +16,7 @@ import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.CanonicalArrays; +import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FixedSizeListArray; @@ -821,6 +822,7 @@ private static Object boxedScalar(Array array, long i) { case ByteArray a -> unsigned ? a.getByte(i) & 0xFFL : (Object) a.getByte(i); case BoolArray a -> a.getBoolean(i); case VarBinArray a -> a.getString(i); + case DecimalArray a -> a.getDecimal(i); default -> null; }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java index 8734c181d..7948e2560 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java @@ -4,13 +4,17 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; +import java.math.BigDecimal; +import java.math.BigInteger; + /// Dtype-aware three-way comparison of two boxed scalar values, the ordering primitive the compute /// kernels lower every value predicate and `MIN` / `MAX` reduction onto. /// /// The comparison mode is keyed off the *column* dtype, never the boxed operand type: integer /// columns compare as longs (unsigned columns via [Long#compareUnsigned(long, long)]), float columns -/// as doubles, strings in UTF-8 byte order ([Utf8Order]), and everything else through the natural -/// [Comparable] order. Keying off the column +/// as doubles, strings in UTF-8 byte order ([Utf8Order]), decimals by value ([BigDecimal#compareTo]), +/// and everything else through the natural [Comparable] order. With no column, a [BigDecimal] +/// operand makes it a decimal compare. Keying off the column /// keeps the compare width-agnostic — a caller may box a filter value at any integer width — and /// never routes an integer column through double-compare, which would lose precision past 2^53. /// @@ -26,13 +30,18 @@ private Compare() { /// /// @param a the left value, typically the array element under test /// @param b the right value, typically the predicate operand - /// @param column the dtype that decides the comparison mode (unsigned, floating, or natural), or + /// @param column the dtype that decides the comparison mode (unsigned, floating, decimal, or natural), or /// `null` to compare width-agnostically off the operands (numbers as longs unless /// either side is floating, otherwise the natural [Comparable] order) /// @return a negative, zero, or positive integer as `a` is less than, equal to, or greater than `b` /// @throws VortexException if the two values are not mutually comparable @SuppressWarnings("unchecked") public static int values(Object a, Object b, DType column) { + // a decimal is a Number, but its order is its value, never the long it truncates to; a + // primitive column keeps its own order, whatever the operand's type + if ((a instanceof BigDecimal || b instanceof BigDecimal) && !(column instanceof DType.Primitive)) { + return decimal(a).compareTo(decimal(b)); + } if (a instanceof Number na && b instanceof Number nb) { if (column instanceof DType.Primitive prim) { if (prim.ptype().isFloating()) { @@ -64,6 +73,21 @@ public static int values(Object a, Object b, DType column) { } } + /// A decimal-column operand as a [BigDecimal]: a decimal itself, an integer of any width, or a + /// finite float or double. Anything else, a NaN or an infinity among them, has no decimal value. + private static BigDecimal decimal(Object value) { + return switch (value) { + case null -> throw new VortexException("compute value of type null is not comparable to a decimal"); + case BigDecimal d -> d; + case BigInteger b -> new BigDecimal(b); + case Byte _, Short _, Integer _, Long _ -> BigDecimal.valueOf(((Number) value).longValue()); + case Double d when Double.isFinite(d) -> BigDecimal.valueOf(d); + case Float f when Float.isFinite(f) -> new BigDecimal(f.toString()); + default -> throw new VortexException("compute value of type " + className(value) + + " (" + value + ") is not comparable to a decimal"); + }; + } + /// Returns the simple class name of `value`, or `"null"` when it is absent, for error messages. /// /// @param value the value to describe diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java index 560e19f33..f551ace8e 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java @@ -1,8 +1,16 @@ package io.github.dfa1.vortex.reader.compute; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -101,4 +109,68 @@ void incomparableValuesThrow() { .isInstanceOf(VortexException.class) .hasMessageContaining("not comparable"); } + + static Stream decimalPairs() { + DType decimal = new DType.Decimal((byte) 10, (byte) 2, false); + DType wide = new DType.Decimal((byte) 38, (byte) 0, false); + BigDecimal i128 = new BigDecimal("170141183460469231731687303715884105727"); + return Stream.of( + // 1.5 and 1.2 are both 1 as a long + Arguments.of("by value, not the truncated long", decimal, new BigDecimal("1.5"), new BigDecimal("1.2"), 1), + Arguments.of("same value, two scales", decimal, new BigDecimal("1.50"), new BigDecimal("1.5"), 0), + Arguments.of("negative", decimal, new BigDecimal("-0.01"), new BigDecimal("0.00"), -1), + Arguments.of("negative scale", decimal, new BigDecimal("1E+2"), 100L, 0), + Arguments.of("long operand", decimal, new BigDecimal("2.50"), 2L, 1), + Arguments.of("long on the left", decimal, 1L, new BigDecimal("1.00"), 0), + Arguments.of("double operand", decimal, new BigDecimal("2.50"), 2.75d, -1), + Arguments.of("float operand", decimal, new BigDecimal("2.50"), 2.25f, 1), + Arguments.of("past a long", wide, i128, Long.MAX_VALUE, 1), + Arguments.of("big integer operand", decimal, new BigDecimal("1"), BigInteger.TWO.pow(64), -1), + Arguments.of("no column", null, new BigDecimal("1.5"), new BigDecimal("1.2"), 1)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("decimalPairs") + void decimalsCompareByValue(String name, DType column, Object a, Object b, int sign) { + // Given a decimal column (or none) and operands of mixed types + + // When + int result = Compare.values(a, b, column); + + // Then + assertThat(Integer.signum(result)).isEqualTo(sign); + } + + static Stream notDecimalValues() { + DType decimal = new DType.Decimal((byte) 10, (byte) 2, false); + return Stream.of( + Arguments.of("NaN", Double.NaN), + Arguments.of("infinity", Double.POSITIVE_INFINITY), + Arguments.of("float NaN", Float.NaN), + Arguments.of("text", "x"), + Arguments.of("null", null)).map(a -> Arguments.of(a.get()[0], decimal, a.get()[1])); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("notDecimalValues") + void decimalAgainstAValueWithNoDecimalMeaningThrows(String name, DType column, Object operand) { + // Given / When / Then + assertThatThrownBy(() -> Compare.values(new BigDecimal("1"), operand, column)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("not comparable to a decimal"); + } + + @Test + void primitiveColumnsKeepTheirOwnOrderWhateverTheOperand() { + // Given a U64 value past 2^63 (a negative long), and a float column holding NaN + long big = -1L; + + // When + int unsigned = Compare.values(big, new BigDecimal("2"), DTypes.U64); + int nan = Compare.values(Double.NaN, new BigDecimal("1.5"), DTypes.F64); + + // Then the unsigned order and the double order hold, no decimal conversion + assertThat(unsigned).isPositive(); + assertThat(nan).isPositive(); + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java index 794a08c0f..61b5cd89d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java @@ -4,11 +4,13 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.ScanFilter; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.compute.Predicate; @@ -20,6 +22,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.math.BigDecimal; import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -749,4 +752,95 @@ void filterOnAnAbsentValue_prunesEveryChunk(@TempDir Path tmp) throws IOExceptio assertThat(scanRowCounts(file, ScanFilter.eq(ColumnName.of("s"), "nope"))).isEmpty(); } } + + @Nested + class Decimals { + + // Three 8192-row zones: 1.00..1.49, 5.00..5.49, 9.00..9.49 at scale 2, the longs 1, 5 and 9 apart + private Path writeDecimalChunks(Path tmp) throws IOException { + DType.Struct schema = new DType.Struct(List.of(ColumnName.of("d")), + List.of(new DType.Decimal((byte) 10, (byte) 2, false)), false); + Path file = tmp.resolve("decimals.vtx"); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var sut = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + BigDecimal[] values = new BigDecimal[3 * 8192]; + for (int i = 0; i < values.length; i++) { + values[i] = BigDecimal.valueOf(100 + (i / 8192) * 400L + i % 50, 2); + } + sut.writeChunk(Map.of(ColumnName.of("d"), values)); + } + return file; + } + + @Test + void zoneStats_areEachZonesDecimalBounds(@TempDir Path tmp) throws IOException { + // Given + Path file = writeDecimalChunks(tmp); + + // When + List zones; + try (var vf = VortexReader.open(file, registry()); + var iter = vf.scan(ScanOptions.all())) { + zones = iter.columnZoneStats(ColumnName.of("d")); + } + + // Then + assertThat(zones).hasSize(3); + assertThat((BigDecimal) zones.get(0).min()).isEqualByComparingTo("1.00"); + assertThat((BigDecimal) zones.get(0).max()).isEqualByComparingTo("1.49"); + assertThat((BigDecimal) zones.get(1).min()).isEqualByComparingTo("5.00"); + assertThat((BigDecimal) zones.get(1).max()).isEqualByComparingTo("5.49"); + assertThat((BigDecimal) zones.get(2).min()).isEqualByComparingTo("9.00"); + assertThat((BigDecimal) zones.get(2).max()).isEqualByComparingTo("9.49"); + } + + static Stream filters() { + ColumnName d = ColumnName.of("d"); + return Stream.of( + Arguments.of("gt 5.20", ScanFilter.gt(d, new BigDecimal("5.20")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.20")) > 0), + Arguments.of("gt 5.49", ScanFilter.gt(d, new BigDecimal("5.49")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.49")) > 0), + Arguments.of("gte 5.49", ScanFilter.gte(d, new BigDecimal("5.49")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.49")) >= 0), + Arguments.of("lt 1.00", ScanFilter.lt(d, new BigDecimal("1.00")), (java.util.function.Predicate) v -> false), + Arguments.of("lte 1.00", ScanFilter.lte(d, new BigDecimal("1.00")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("1.00")) <= 0), + Arguments.of("eq 5.25", ScanFilter.eq(d, new BigDecimal("5.25")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.25")) == 0), + // a literal at another scale is the same value + Arguments.of("eq 5.250", ScanFilter.eq(d, new BigDecimal("5.250")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.25")) == 0), + // a long, 1.49 < 5 <= 5.00 and 5.49 < 9 + Arguments.of("gte long 5", ScanFilter.gte(d, 5L), (java.util.function.Predicate) v -> v.compareTo(BigDecimal.valueOf(5)) >= 0), + Arguments.of("gt long 9", ScanFilter.gt(d, 9L), (java.util.function.Predicate) v -> v.compareTo(BigDecimal.valueOf(9)) > 0)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("filters") + void filteredScan_returnsExactlyTheMatchingDecimals(String name, ScanFilter filter, + java.util.function.Predicate matches, @TempDir Path tmp) + throws IOException { + // Given + Path file = writeDecimalChunks(tmp); + List expected = new ArrayList<>(); + for (int i = 0; i < 3 * 8192; i++) { + BigDecimal value = BigDecimal.valueOf(100 + (i / 8192) * 400L + i % 50, 2); + if (matches.test(value)) { + expected.add(value); + } + } + + // When + List rows = new ArrayList<>(); + try (var vf = VortexReader.open(file, registry()); + var iter = vf.scan(ScanOptions.all().withFilter(filter))) { + while (iter.hasNext()) { + try (Chunk chunk = iter.next()) { + DecimalArray array = chunk.column(ColumnName.of("d")); + for (int i = 0; i < chunk.rowCount(); i++) { + rows.add(array.getDecimal(i)); + } + } + } + } + + // Then + assertThat(rows).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); + } + } }