diff --git a/CHANGELOG.md b/CHANGELOG.md index 8db2dc510..8bc001dad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650)) - 8-bit, 16-bit and `F32` columns get Rust's dict layout when its probe picks one, so a nullable low-cardinality `SMALLINT` is written up to 30% smaller. ([#651](https://github.com/dfa1/vortex-java/issues/651)) +- A filtered read of a decimal column whose flat node carries min/max stats no longer fails with `VortexException`; those bounds are not used for pruning. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696)) - **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java index b5f56a3c8..f5337d4ec 100644 --- a/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/DecimalSumTest.java @@ -63,6 +63,11 @@ static void writeFile() throws Exception { "select sum(amount), avg(amount), count(amount) from vtx.t where id < 5 " + "| select sum(case when id > -1 then amount end), avg(case when id > -1 then amount end), " + "count(case when id > -1 then amount end) from vtx.t where id + 0 < 5", + "select min(amount), max(amount) from vtx.t " + + "| select min(case when id > -1 then amount end), max(case when id > -1 then amount end) from vtx.t", + "select min(amount), max(amount) from vtx.t where id < 5 " + + "| select min(case when id > -1 then amount end), max(case when id > -1 then amount end) " + + "from vtx.t where id + 0 < 5", "select s, sum(amount), min(amount), max(amount) from vtx.t group by s order by s " + "| select s, sum(case when id > -1 then amount end), min(case when id > -1 then amount end), " + "max(case when id > -1 then amount end) from vtx.t group by s order by s"}) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java index 6cb469671..fdafe31b1 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java @@ -193,6 +193,34 @@ void filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision assertThat(fromJava).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected); } + /// A column too small for a zone map reads its bounds from the flat node's stats, which Rust writes + /// for a decimal as its own scalar; Java must prune with them or not at all, never misread them. + @ParameterizedTest(name = "{0}") + @MethodSource("shapes") + void smallFile_filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision, int scale, + IntToLongFunction unscaled, @TempDir Path tmp) + throws Exception { + // Given + int rows = 100; + Path jniFile = tmp.resolve("jni.vortex"); + writeWithJni(jniFile, precision, scale, unscaled, null, rows); + List all = new ArrayList<>(rows); + for (int i = 0; i < rows; i++) { + all.add(BigDecimal.valueOf(unscaled.applyAsLong(i), scale)); + } + BigDecimal median = all.stream().sorted().toList().get(rows / 2); + BigDecimal above = all.stream().max(BigDecimal::compareTo).orElseThrow().add(BigDecimal.ONE); + + // When + List atLeastMedian = readWithJava(jniFile, ScanFilter.gte(D, median)); + List aboveMax = readWithJava(jniFile, ScanFilter.gte(D, above)); + + // Then + assertThat(atLeastMedian).usingElementComparator(BigDecimal::compareTo) + .isEqualTo(all.stream().filter(v -> v.compareTo(median) >= 0).toList()); + assertThat(aboveMax).isEmpty(); + } + private static List readWithJava(Path file, ScanFilter filter) throws IOException { List out = new ArrayList<>(); try (var reader = VortexReader.open(file); @@ -296,23 +324,28 @@ private static void writeWithJni(Path path, int precision, int scale, IntToLongF /// `isNull` null writes a non-nullable column, any predicate a nullable one. private static void writeWithJni(Path path, int precision, int scale, IntToLongFunction unscaled, IntPredicate isNull) throws Exception { + writeWithJni(path, precision, scale, unscaled, isNull, ROWS); + } + + private static void writeWithJni(Path path, int precision, int scale, IntToLongFunction unscaled, + IntPredicate isNull, int rows) throws Exception { ArrowType decimal = new ArrowType.Decimal(precision, scale, 128); Schema schema = new Schema(List.of(isNull == null ? Field.notNullable("d", decimal) : Field.nullable("d", decimal))); try (VortexWriter writer = VortexWriter.builder(SESSION, path.toAbsolutePath().toUri().toString(), schema, ALLOCATOR).build(); VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) { DecimalVector vector = (DecimalVector) root.getVector("d"); - vector.setInitialCapacity(ROWS); + vector.setInitialCapacity(rows); vector.allocateNew(); - for (int i = 0; i < ROWS; i++) { + for (int i = 0; i < rows; i++) { if (isNull != null && isNull.test(i)) { vector.setNull(i); } else { vector.setSafe(i, BigDecimal.valueOf(unscaled.applyAsLong(i), scale)); } } - vector.setValueCount(ROWS); - root.setRowCount(ROWS); + vector.setValueCount(rows); + root.setRowCount(rows); try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema arrowSchema = ArrowSchema.allocateNew(ALLOCATOR)) { Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, arrowSchema); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 773dd826d..9d47af885 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -1427,9 +1427,19 @@ private List zonesFor(ColumnName col, boolean includeBounds) { /// @param rowCount the chunk's row count /// @param ct the column's dtype, deciding the comparison mode /// @return `true` if no row can match and the chunk may be pruned - private static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DType ct) { + static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DType ct) { Object min = stats.min(); Object max = stats.max(); + if (ct instanceof DType.Decimal) { + // A flat node's stats scalar is decoded without the column's dtype, so a decimal bound + // arrives as text: not a bound this column can order, hence no bound at all. + if (!(min instanceof java.math.BigDecimal)) { + min = null; + } + if (!(max instanceof java.math.BigDecimal)) { + max = null; + } + } Long nullCount = stats.nullCount(); if (ct instanceof DType.Primitive p && p.ptype().isFloating() && (stats.nanCount() == null || stats.nanCount() != 0)) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java new file mode 100644 index 000000000..97010405d --- /dev/null +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java @@ -0,0 +1,38 @@ +package io.github.dfa1.vortex.reader; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.compute.Predicate; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.assertj.core.api.Assertions.assertThat; + +class ScanIteratorDecimalStatsTest { + + private static final DType DECIMAL = new DType.Decimal((byte) 10, (byte) 2, false); + + @Test + void canPrune_decimalBoundsAsText_doesNotPruneOrThrow() { + // Given a flat node's stats, whose decimal scalar was read without the column's dtype + ArrayStats stats = stats("\u0001\u0002", "\u0003\u0004"); + + // When / Then + assertThat(ScanIterator.canPrune(new Predicate.Gte(new BigDecimal("1.00")), stats, 10, DECIMAL)).isFalse(); + assertThat(ScanIterator.canPrune(new Predicate.Eq(new BigDecimal("1.00")), stats, 10, DECIMAL)).isFalse(); + } + + @Test + void canPrune_decimalBounds_prunesOutsideTheRange() { + // Given + ArrayStats stats = stats(new BigDecimal("1.00"), new BigDecimal("5.00")); + + // When / Then + assertThat(ScanIterator.canPrune(new Predicate.Gte(new BigDecimal("5.01")), stats, 10, DECIMAL)).isTrue(); + assertThat(ScanIterator.canPrune(new Predicate.Gte(new BigDecimal("5.00")), stats, 10, DECIMAL)).isFalse(); + } + + private static ArrayStats stats(Object min, Object max) { + return new ArrayStats(min, true, max, true, null, null, null, null, 0L, null, null); + } +}