Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650))
- 8-bit, 16-bit and `F32` columns get Rust's dict layout when its probe picks one, so a nullable low-cardinality `SMALLINT` is written up to 30% smaller. ([#651](https://github.com/dfa1/vortex-java/issues/651))
- A filtered read of a decimal column whose flat node carries min/max stats no longer fails with `VortexException`; those bounds are not used for pruning. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696))
- **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701))
- A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ static void writeFile() throws Exception {
"select sum(amount), avg(amount), count(amount) from vtx.t where id < 5 "
+ "| select sum(case when id > -1 then amount end), avg(case when id > -1 then amount end), "
+ "count(case when id > -1 then amount end) from vtx.t where id + 0 < 5",
"select min(amount), max(amount) from vtx.t "
+ "| select min(case when id > -1 then amount end), max(case when id > -1 then amount end) from vtx.t",
"select min(amount), max(amount) from vtx.t where id < 5 "
+ "| select min(case when id > -1 then amount end), max(case when id > -1 then amount end) "
+ "from vtx.t where id + 0 < 5",
"select s, sum(amount), min(amount), max(amount) from vtx.t group by s order by s "
+ "| select s, sum(case when id > -1 then amount end), min(case when id > -1 then amount end), "
+ "max(case when id > -1 then amount end) from vtx.t group by s order by s"})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,34 @@ void filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision
assertThat(fromJava).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected);
}

/// A column too small for a zone map reads its bounds from the flat node's stats, which Rust writes
/// for a decimal as its own scalar; Java must prune with them or not at all, never misread them.
@ParameterizedTest(name = "{0}")
@MethodSource("shapes")
void smallFile_filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision, int scale,
IntToLongFunction unscaled, @TempDir Path tmp)
throws Exception {
// Given
int rows = 100;
Path jniFile = tmp.resolve("jni.vortex");
writeWithJni(jniFile, precision, scale, unscaled, null, rows);
List<BigDecimal> all = new ArrayList<>(rows);
for (int i = 0; i < rows; i++) {
all.add(BigDecimal.valueOf(unscaled.applyAsLong(i), scale));
}
BigDecimal median = all.stream().sorted().toList().get(rows / 2);
BigDecimal above = all.stream().max(BigDecimal::compareTo).orElseThrow().add(BigDecimal.ONE);

// When
List<BigDecimal> atLeastMedian = readWithJava(jniFile, ScanFilter.gte(D, median));
List<BigDecimal> aboveMax = readWithJava(jniFile, ScanFilter.gte(D, above));

// Then
assertThat(atLeastMedian).usingElementComparator(BigDecimal::compareTo)
.isEqualTo(all.stream().filter(v -> v.compareTo(median) >= 0).toList());
assertThat(aboveMax).isEmpty();
}

private static List<BigDecimal> readWithJava(Path file, ScanFilter filter) throws IOException {
List<BigDecimal> out = new ArrayList<>();
try (var reader = VortexReader.open(file);
Expand Down Expand Up @@ -296,23 +324,28 @@ private static void writeWithJni(Path path, int precision, int scale, IntToLongF
/// `isNull` null writes a non-nullable column, any predicate a nullable one.
private static void writeWithJni(Path path, int precision, int scale, IntToLongFunction unscaled,
IntPredicate isNull) throws Exception {
writeWithJni(path, precision, scale, unscaled, isNull, ROWS);
}

private static void writeWithJni(Path path, int precision, int scale, IntToLongFunction unscaled,
IntPredicate isNull, int rows) throws Exception {
ArrowType decimal = new ArrowType.Decimal(precision, scale, 128);
Schema schema = new Schema(List.of(isNull == null ? Field.notNullable("d", decimal) : Field.nullable("d", decimal)));
try (VortexWriter writer = VortexWriter.builder(SESSION, path.toAbsolutePath().toUri().toString(), schema,
ALLOCATOR).build();
VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) {
DecimalVector vector = (DecimalVector) root.getVector("d");
vector.setInitialCapacity(ROWS);
vector.setInitialCapacity(rows);
vector.allocateNew();
for (int i = 0; i < ROWS; i++) {
for (int i = 0; i < rows; i++) {
if (isNull != null && isNull.test(i)) {
vector.setNull(i);
} else {
vector.setSafe(i, BigDecimal.valueOf(unscaled.applyAsLong(i), scale));
}
}
vector.setValueCount(ROWS);
root.setRowCount(ROWS);
vector.setValueCount(rows);
root.setRowCount(rows);
try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR);
ArrowSchema arrowSchema = ArrowSchema.allocateNew(ALLOCATOR)) {
Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, arrowSchema);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1427,9 +1427,19 @@ private List<Zone> zonesFor(ColumnName col, boolean includeBounds) {
/// @param rowCount the chunk's row count
/// @param ct the column's dtype, deciding the comparison mode
/// @return `true` if no row can match and the chunk may be pruned
private static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DType ct) {
static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DType ct) {
Object min = stats.min();
Object max = stats.max();
if (ct instanceof DType.Decimal) {
// A flat node's stats scalar is decoded without the column's dtype, so a decimal bound
// arrives as text: not a bound this column can order, hence no bound at all.
if (!(min instanceof java.math.BigDecimal)) {
min = null;
}
if (!(max instanceof java.math.BigDecimal)) {
max = null;
}
}
Long nullCount = stats.nullCount();
if (ct instanceof DType.Primitive p && p.ptype().isFloating()
&& (stats.nanCount() == null || stats.nanCount() != 0)) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package io.github.dfa1.vortex.reader;

import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.compute.Predicate;
import org.junit.jupiter.api.Test;

import java.math.BigDecimal;

import static org.assertj.core.api.Assertions.assertThat;

class ScanIteratorDecimalStatsTest {

private static final DType DECIMAL = new DType.Decimal((byte) 10, (byte) 2, false);

@Test
void canPrune_decimalBoundsAsText_doesNotPruneOrThrow() {
// Given a flat node's stats, whose decimal scalar was read without the column's dtype
ArrayStats stats = stats("\u0001\u0002", "\u0003\u0004");

// When / Then
assertThat(ScanIterator.canPrune(new Predicate.Gte(new BigDecimal("1.00")), stats, 10, DECIMAL)).isFalse();
assertThat(ScanIterator.canPrune(new Predicate.Eq(new BigDecimal("1.00")), stats, 10, DECIMAL)).isFalse();
}

@Test
void canPrune_decimalBounds_prunesOutsideTheRange() {
// Given
ArrayStats stats = stats(new BigDecimal("1.00"), new BigDecimal("5.00"));

// When / Then
assertThat(ScanIterator.canPrune(new Predicate.Gte(new BigDecimal("5.01")), stats, 10, DECIMAL)).isTrue();
assertThat(ScanIterator.canPrune(new Predicate.Gte(new BigDecimal("5.00")), stats, 10, DECIMAL)).isFalse();
}

private static ArrayStats stats(Object min, Object max) {
return new ArrayStats(min, true, max, true, null, null, null, null, 0L, null, null);
}
}
Loading