diff --git a/CHANGELOG.md b/CHANGELOG.md index 2653cdb37..8e6b8de79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - FSST symbol tables are trained as Rust's `fsst-rs` does (its sample, generations, candidate gains and cut), so a text column without a null gets Rust's FSST encoding up to ties of equal gain, its offsets a `vortex.sequence` where Rust's cascade finds one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650)) - A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) diff --git a/docs/compatibility.md b/docs/compatibility.md index 839b8ff3a..c67b128d1 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -168,8 +168,8 @@ it cascades into alike: So a Java-written string column has no bound in its flat segments, as a Rust-written one has none: `VortexReader.columnStats()` reports no string `min`/`max` for either, and a string bound lives only in the -zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the -zone table's are not decoded yet. +zoned layout's stats table. A decimal column's min and max are decoded from the zone table too: `columnStats()` reports them and +zone-map pruning uses them, compared by value at the column's scale. The nodes below the root follow the same table, plus three shapes of their own: a `vortex.sequence` carries `is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP, ALP-RD and bit-packed diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java index 6be80a00c..6cb469671 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java @@ -12,6 +12,10 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.reader.ScanFilter; +import io.github.dfa1.vortex.reader.array.DecimalArray; +import io.github.dfa1.vortex.reader.VortexReader; import io.github.dfa1.vortex.writer.WriteOptions; import io.github.dfa1.vortex.writer.encode.NullableData; import org.apache.arrow.c.ArrowArray; @@ -125,6 +129,92 @@ void jniReadsJavaFile(String name, int precision, int scale, IntToLongFunction u assertThat(range).isEqualTo(expected.subList(1000, 20_000)); } + /// The file's decimal min, max and null count, which Rust keeps in the zone table: read by Java + /// from a vortex-jni file and from its own, they are the extremes of the valid rows. + @ParameterizedTest(name = "{0}") + @MethodSource("nullableShapes") + void columnStats_areTheExtremesOfTheValidRows(String name, int precision, int scale, IntToLongFunction unscaled, + IntPredicate isNull, @TempDir Path tmp) throws Exception { + // Given + Path jniFile = tmp.resolve("jni.vortex"); + Path javaFile = tmp.resolve("java.vortex"); + writeWithJni(jniFile, precision, scale, unscaled, isNull); + writeWithJava(javaFile, precision, scale, unscaled, isNull); + BigDecimal min = null; + BigDecimal max = null; + long nulls = 0; + for (int i = 0; i < ROWS; i++) { + if (isNull.test(i)) { + nulls++; + continue; + } + BigDecimal value = BigDecimal.valueOf(unscaled.applyAsLong(i), scale); + min = min == null || value.compareTo(min) < 0 ? value : min; + max = max == null || value.compareTo(max) > 0 ? value : max; + } + + // When + ArrayStats fromJni = columnStats(jniFile); + ArrayStats fromJava = columnStats(javaFile); + + // Then + for (ArrayStats stats : List.of(fromJni, fromJava)) { + assertThat((BigDecimal) stats.min()).isEqualByComparingTo(min); + assertThat((BigDecimal) stats.max()).isEqualByComparingTo(max); + assertThat(stats.nullCount()).isEqualTo(nulls); + } + } + + /// Pruning trusts each zone's decimal bounds, so a filtered read by Java of a vortex-jni file (whose + /// zone table Rust wrote) and of Java's own must return exactly the matching rows. + @ParameterizedTest(name = "{0}") + @MethodSource("shapes") + void filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision, int scale, + IntToLongFunction unscaled, @TempDir Path tmp) + throws Exception { + // Given + Path jniFile = tmp.resolve("jni.vortex"); + Path javaFile = tmp.resolve("java.vortex"); + writeWithJni(jniFile, precision, scale, unscaled); + writeWithJava(javaFile, precision, scale, unscaled); + List all = new ArrayList<>(ROWS); + for (int i = 0; i < ROWS; i++) { + all.add(BigDecimal.valueOf(unscaled.applyAsLong(i), scale)); + } + BigDecimal threshold = all.get(ROWS / 2); + + // When + List fromJni = readWithJava(jniFile, ScanFilter.gte(D, threshold)); + List fromJava = readWithJava(javaFile, ScanFilter.gte(D, threshold)); + + // Then + List expected = all.stream().filter(v -> v.compareTo(threshold) >= 0).toList(); + assertThat(fromJni).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); + assertThat(fromJava).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); + } + + private static List readWithJava(Path file, ScanFilter filter) throws IOException { + List out = new ArrayList<>(); + try (var reader = VortexReader.open(file); + var scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all().withFilter(filter))) { + while (scan.hasNext()) { + try (var chunk = scan.next()) { + DecimalArray array = chunk.column(D); + for (int i = 0; i < chunk.rowCount(); i++) { + out.add(array.getDecimal(i)); + } + } + } + } + return out; + } + + private static ArrayStats columnStats(Path file) throws IOException { + try (var reader = VortexReader.open(file)) { + return reader.columnStats().get(D); + } + } + static Stream nullableShapes() { return Stream.of( // nullable but every row valid: written without a validity, as Rust's AllValid diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index dd019126c..773dd826d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -16,6 +16,7 @@ import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.CanonicalArrays; +import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.DictIntArray; import io.github.dfa1.vortex.reader.array.DoubleArray; import io.github.dfa1.vortex.reader.array.FixedSizeListArray; @@ -821,6 +822,7 @@ private static Object boxedScalar(Array array, long i) { case ByteArray a -> unsigned ? a.getByte(i) & 0xFFL : (Object) a.getByte(i); case BoolArray a -> a.getBoolean(i); case VarBinArray a -> a.getString(i); + case DecimalArray a -> a.getDecimal(i); default -> null; }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java index 8734c181d..7948e2560 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java @@ -4,13 +4,17 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; +import java.math.BigDecimal; +import java.math.BigInteger; + /// Dtype-aware three-way comparison of two boxed scalar values, the ordering primitive the compute /// kernels lower every value predicate and `MIN` / `MAX` reduction onto. /// /// The comparison mode is keyed off the *column* dtype, never the boxed operand type: integer /// columns compare as longs (unsigned columns via [Long#compareUnsigned(long, long)]), float columns -/// as doubles, strings in UTF-8 byte order ([Utf8Order]), and everything else through the natural -/// [Comparable] order. Keying off the column +/// as doubles, strings in UTF-8 byte order ([Utf8Order]), decimals by value ([BigDecimal#compareTo]), +/// and everything else through the natural [Comparable] order. With no column, a [BigDecimal] +/// operand makes it a decimal compare. Keying off the column /// keeps the compare width-agnostic — a caller may box a filter value at any integer width — and /// never routes an integer column through double-compare, which would lose precision past 2^53. /// @@ -26,13 +30,18 @@ private Compare() { /// /// @param a the left value, typically the array element under test /// @param b the right value, typically the predicate operand - /// @param column the dtype that decides the comparison mode (unsigned, floating, or natural), or + /// @param column the dtype that decides the comparison mode (unsigned, floating, decimal, or natural), or /// `null` to compare width-agnostically off the operands (numbers as longs unless /// either side is floating, otherwise the natural [Comparable] order) /// @return a negative, zero, or positive integer as `a` is less than, equal to, or greater than `b` /// @throws VortexException if the two values are not mutually comparable @SuppressWarnings("unchecked") public static int values(Object a, Object b, DType column) { + // a decimal is a Number, but its order is its value, never the long it truncates to; a + // primitive column keeps its own order, whatever the operand's type + if ((a instanceof BigDecimal || b instanceof BigDecimal) && !(column instanceof DType.Primitive)) { + return decimal(a).compareTo(decimal(b)); + } if (a instanceof Number na && b instanceof Number nb) { if (column instanceof DType.Primitive prim) { if (prim.ptype().isFloating()) { @@ -64,6 +73,21 @@ public static int values(Object a, Object b, DType column) { } } + /// A decimal-column operand as a [BigDecimal]: a decimal itself, an integer of any width, or a + /// finite float or double. Anything else, a NaN or an infinity among them, has no decimal value. + private static BigDecimal decimal(Object value) { + return switch (value) { + case null -> throw new VortexException("compute value of type null is not comparable to a decimal"); + case BigDecimal d -> d; + case BigInteger b -> new BigDecimal(b); + case Byte _, Short _, Integer _, Long _ -> BigDecimal.valueOf(((Number) value).longValue()); + case Double d when Double.isFinite(d) -> BigDecimal.valueOf(d); + case Float f when Float.isFinite(f) -> new BigDecimal(f.toString()); + default -> throw new VortexException("compute value of type " + className(value) + + " (" + value + ") is not comparable to a decimal"); + }; + } + /// Returns the simple class name of `value`, or `"null"` when it is absent, for error messages. /// /// @param value the value to describe diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java index 560e19f33..f551ace8e 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java @@ -1,8 +1,16 @@ package io.github.dfa1.vortex.reader.compute; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -101,4 +109,68 @@ void incomparableValuesThrow() { .isInstanceOf(VortexException.class) .hasMessageContaining("not comparable"); } + + static Stream decimalPairs() { + DType decimal = new DType.Decimal((byte) 10, (byte) 2, false); + DType wide = new DType.Decimal((byte) 38, (byte) 0, false); + BigDecimal i128 = new BigDecimal("170141183460469231731687303715884105727"); + return Stream.of( + // 1.5 and 1.2 are both 1 as a long + Arguments.of("by value, not the truncated long", decimal, new BigDecimal("1.5"), new BigDecimal("1.2"), 1), + Arguments.of("same value, two scales", decimal, new BigDecimal("1.50"), new BigDecimal("1.5"), 0), + Arguments.of("negative", decimal, new BigDecimal("-0.01"), new BigDecimal("0.00"), -1), + Arguments.of("negative scale", decimal, new BigDecimal("1E+2"), 100L, 0), + Arguments.of("long operand", decimal, new BigDecimal("2.50"), 2L, 1), + Arguments.of("long on the left", decimal, 1L, new BigDecimal("1.00"), 0), + Arguments.of("double operand", decimal, new BigDecimal("2.50"), 2.75d, -1), + Arguments.of("float operand", decimal, new BigDecimal("2.50"), 2.25f, 1), + Arguments.of("past a long", wide, i128, Long.MAX_VALUE, 1), + Arguments.of("big integer operand", decimal, new BigDecimal("1"), BigInteger.TWO.pow(64), -1), + Arguments.of("no column", null, new BigDecimal("1.5"), new BigDecimal("1.2"), 1)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("decimalPairs") + void decimalsCompareByValue(String name, DType column, Object a, Object b, int sign) { + // Given a decimal column (or none) and operands of mixed types + + // When + int result = Compare.values(a, b, column); + + // Then + assertThat(Integer.signum(result)).isEqualTo(sign); + } + + static Stream notDecimalValues() { + DType decimal = new DType.Decimal((byte) 10, (byte) 2, false); + return Stream.of( + Arguments.of("NaN", Double.NaN), + Arguments.of("infinity", Double.POSITIVE_INFINITY), + Arguments.of("float NaN", Float.NaN), + Arguments.of("text", "x"), + Arguments.of("null", null)).map(a -> Arguments.of(a.get()[0], decimal, a.get()[1])); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("notDecimalValues") + void decimalAgainstAValueWithNoDecimalMeaningThrows(String name, DType column, Object operand) { + // Given / When / Then + assertThatThrownBy(() -> Compare.values(new BigDecimal("1"), operand, column)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("not comparable to a decimal"); + } + + @Test + void primitiveColumnsKeepTheirOwnOrderWhateverTheOperand() { + // Given a U64 value past 2^63 (a negative long), and a float column holding NaN + long big = -1L; + + // When + int unsigned = Compare.values(big, new BigDecimal("2"), DTypes.U64); + int nan = Compare.values(Double.NaN, new BigDecimal("1.5"), DTypes.F64); + + // Then the unsigned order and the double order hold, no decimal conversion + assertThat(unsigned).isPositive(); + assertThat(nan).isPositive(); + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java index 794a08c0f..61b5cd89d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java @@ -4,11 +4,13 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.ScanFilter; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.reader.compute.Predicate; @@ -20,6 +22,7 @@ import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.math.BigDecimal; import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -749,4 +752,95 @@ void filterOnAnAbsentValue_prunesEveryChunk(@TempDir Path tmp) throws IOExceptio assertThat(scanRowCounts(file, ScanFilter.eq(ColumnName.of("s"), "nope"))).isEmpty(); } } + + @Nested + class Decimals { + + // Three 8192-row zones: 1.00..1.49, 5.00..5.49, 9.00..9.49 at scale 2, the longs 1, 5 and 9 apart + private Path writeDecimalChunks(Path tmp) throws IOException { + DType.Struct schema = new DType.Struct(List.of(ColumnName.of("d")), + List.of(new DType.Decimal((byte) 10, (byte) 2, false)), false); + Path file = tmp.resolve("decimals.vtx"); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var sut = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + BigDecimal[] values = new BigDecimal[3 * 8192]; + for (int i = 0; i < values.length; i++) { + values[i] = BigDecimal.valueOf(100 + (i / 8192) * 400L + i % 50, 2); + } + sut.writeChunk(Map.of(ColumnName.of("d"), values)); + } + return file; + } + + @Test + void zoneStats_areEachZonesDecimalBounds(@TempDir Path tmp) throws IOException { + // Given + Path file = writeDecimalChunks(tmp); + + // When + List zones; + try (var vf = VortexReader.open(file, registry()); + var iter = vf.scan(ScanOptions.all())) { + zones = iter.columnZoneStats(ColumnName.of("d")); + } + + // Then + assertThat(zones).hasSize(3); + assertThat((BigDecimal) zones.get(0).min()).isEqualByComparingTo("1.00"); + assertThat((BigDecimal) zones.get(0).max()).isEqualByComparingTo("1.49"); + assertThat((BigDecimal) zones.get(1).min()).isEqualByComparingTo("5.00"); + assertThat((BigDecimal) zones.get(1).max()).isEqualByComparingTo("5.49"); + assertThat((BigDecimal) zones.get(2).min()).isEqualByComparingTo("9.00"); + assertThat((BigDecimal) zones.get(2).max()).isEqualByComparingTo("9.49"); + } + + static Stream filters() { + ColumnName d = ColumnName.of("d"); + return Stream.of( + Arguments.of("gt 5.20", ScanFilter.gt(d, new BigDecimal("5.20")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.20")) > 0), + Arguments.of("gt 5.49", ScanFilter.gt(d, new BigDecimal("5.49")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.49")) > 0), + Arguments.of("gte 5.49", ScanFilter.gte(d, new BigDecimal("5.49")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.49")) >= 0), + Arguments.of("lt 1.00", ScanFilter.lt(d, new BigDecimal("1.00")), (java.util.function.Predicate) v -> false), + Arguments.of("lte 1.00", ScanFilter.lte(d, new BigDecimal("1.00")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("1.00")) <= 0), + Arguments.of("eq 5.25", ScanFilter.eq(d, new BigDecimal("5.25")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.25")) == 0), + // a literal at another scale is the same value + Arguments.of("eq 5.250", ScanFilter.eq(d, new BigDecimal("5.250")), (java.util.function.Predicate) v -> v.compareTo(new BigDecimal("5.25")) == 0), + // a long, 1.49 < 5 <= 5.00 and 5.49 < 9 + Arguments.of("gte long 5", ScanFilter.gte(d, 5L), (java.util.function.Predicate) v -> v.compareTo(BigDecimal.valueOf(5)) >= 0), + Arguments.of("gt long 9", ScanFilter.gt(d, 9L), (java.util.function.Predicate) v -> v.compareTo(BigDecimal.valueOf(9)) > 0)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("filters") + void filteredScan_returnsExactlyTheMatchingDecimals(String name, ScanFilter filter, + java.util.function.Predicate matches, @TempDir Path tmp) + throws IOException { + // Given + Path file = writeDecimalChunks(tmp); + List expected = new ArrayList<>(); + for (int i = 0; i < 3 * 8192; i++) { + BigDecimal value = BigDecimal.valueOf(100 + (i / 8192) * 400L + i % 50, 2); + if (matches.test(value)) { + expected.add(value); + } + } + + // When + List rows = new ArrayList<>(); + try (var vf = VortexReader.open(file, registry()); + var iter = vf.scan(ScanOptions.all().withFilter(filter))) { + while (iter.hasNext()) { + try (Chunk chunk = iter.next()) { + DecimalArray array = chunk.column(ColumnName.of("d")); + for (int i = 0; i < chunk.rowCount(); i++) { + rows.add(array.getDecimal(i)); + } + } + } + } + + // Then + assertThat(rows).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); + } + } }