Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed
- FSST symbol tables are trained as Rust's `fsst-rs` does (its sample, generations, candidate gains and cut), so a text column without a null gets Rust's FSST encoding up to ties of equal gain, its offsets a `vortex.sequence` where Rust's cascade finds one. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650))
- A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696))
- A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694))
Expand Down
4 changes: 2 additions & 2 deletions docs/compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,8 @@ it cascades into alike:

So a Java-written string column has no bound in its flat segments, as a Rust-written one has none:
`VortexReader.columnStats()` reports no string `min`/`max` for either, and a string bound lives only in the
zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the
zone table's are not decoded yet.
zoned layout's stats table. A decimal column's min and max are decoded from the zone table too: `columnStats()` reports them and
zone-map pruning uses them, compared by value at the column's scale.

The nodes below the root follow the same table, plus three shapes of their own: a `vortex.sequence` carries
`is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP, ALP-RD and bit-packed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
import dev.vortex.jni.NativeLoader;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.ScanFilter;
import io.github.dfa1.vortex.reader.array.DecimalArray;
import io.github.dfa1.vortex.reader.VortexReader;
import io.github.dfa1.vortex.writer.WriteOptions;
import io.github.dfa1.vortex.writer.encode.NullableData;
import org.apache.arrow.c.ArrowArray;
Expand Down Expand Up @@ -125,6 +129,92 @@ void jniReadsJavaFile(String name, int precision, int scale, IntToLongFunction u
assertThat(range).isEqualTo(expected.subList(1000, 20_000));
}

/// The file's decimal min, max and null count, which Rust keeps in the zone table: read by Java
/// from a vortex-jni file and from its own, they are the extremes of the valid rows.
@ParameterizedTest(name = "{0}")
@MethodSource("nullableShapes")
void columnStats_areTheExtremesOfTheValidRows(String name, int precision, int scale, IntToLongFunction unscaled,
IntPredicate isNull, @TempDir Path tmp) throws Exception {
// Given
Path jniFile = tmp.resolve("jni.vortex");
Path javaFile = tmp.resolve("java.vortex");
writeWithJni(jniFile, precision, scale, unscaled, isNull);
writeWithJava(javaFile, precision, scale, unscaled, isNull);
BigDecimal min = null;
BigDecimal max = null;
long nulls = 0;
for (int i = 0; i < ROWS; i++) {
if (isNull.test(i)) {
nulls++;
continue;
}
BigDecimal value = BigDecimal.valueOf(unscaled.applyAsLong(i), scale);
min = min == null || value.compareTo(min) < 0 ? value : min;
max = max == null || value.compareTo(max) > 0 ? value : max;
}

// When
ArrayStats fromJni = columnStats(jniFile);
ArrayStats fromJava = columnStats(javaFile);

// Then
for (ArrayStats stats : List.of(fromJni, fromJava)) {
assertThat((BigDecimal) stats.min()).isEqualByComparingTo(min);
assertThat((BigDecimal) stats.max()).isEqualByComparingTo(max);
assertThat(stats.nullCount()).isEqualTo(nulls);
}
}

/// Pruning trusts each zone's decimal bounds, so a filtered read by Java of a vortex-jni file (whose
/// zone table Rust wrote) and of Java's own must return exactly the matching rows.
@ParameterizedTest(name = "{0}")
@MethodSource("shapes")
void filteredReadByJava_returnsExactlyTheMatchingRows(String name, int precision, int scale,
IntToLongFunction unscaled, @TempDir Path tmp)
throws Exception {
// Given
Path jniFile = tmp.resolve("jni.vortex");
Path javaFile = tmp.resolve("java.vortex");
writeWithJni(jniFile, precision, scale, unscaled);
writeWithJava(javaFile, precision, scale, unscaled);
List<BigDecimal> all = new ArrayList<>(ROWS);
for (int i = 0; i < ROWS; i++) {
all.add(BigDecimal.valueOf(unscaled.applyAsLong(i), scale));
}
BigDecimal threshold = all.get(ROWS / 2);

// When
List<BigDecimal> fromJni = readWithJava(jniFile, ScanFilter.gte(D, threshold));
List<BigDecimal> fromJava = readWithJava(javaFile, ScanFilter.gte(D, threshold));

// Then
List<BigDecimal> expected = all.stream().filter(v -> v.compareTo(threshold) >= 0).toList();
assertThat(fromJni).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected);
assertThat(fromJava).usingElementComparator(BigDecimal::compareTo).isEqualTo(expected);
}

private static List<BigDecimal> readWithJava(Path file, ScanFilter filter) throws IOException {
List<BigDecimal> out = new ArrayList<>();
try (var reader = VortexReader.open(file);
var scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all().withFilter(filter))) {
while (scan.hasNext()) {
try (var chunk = scan.next()) {
DecimalArray array = chunk.column(D);
for (int i = 0; i < chunk.rowCount(); i++) {
out.add(array.getDecimal(i));
}
}
}
}
return out;
}

private static ArrayStats columnStats(Path file) throws IOException {
try (var reader = VortexReader.open(file)) {
return reader.columnStats().get(D);
}
}

static Stream<Arguments> nullableShapes() {
return Stream.of(
// nullable but every row valid: written without a validity, as Rust's AllValid
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import io.github.dfa1.vortex.reader.array.BoolArray;
import io.github.dfa1.vortex.reader.array.ByteArray;
import io.github.dfa1.vortex.reader.array.CanonicalArrays;
import io.github.dfa1.vortex.reader.array.DecimalArray;
import io.github.dfa1.vortex.reader.array.DictIntArray;
import io.github.dfa1.vortex.reader.array.DoubleArray;
import io.github.dfa1.vortex.reader.array.FixedSizeListArray;
Expand Down Expand Up @@ -821,6 +822,7 @@ private static Object boxedScalar(Array array, long i) {
case ByteArray a -> unsigned ? a.getByte(i) & 0xFFL : (Object) a.getByte(i);
case BoolArray a -> a.getBoolean(i);
case VarBinArray a -> a.getString(i);
case DecimalArray a -> a.getDecimal(i);
default -> null;
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,17 @@
import io.github.dfa1.vortex.core.error.VortexException;
import io.github.dfa1.vortex.core.model.DType;

import java.math.BigDecimal;
import java.math.BigInteger;

/// Dtype-aware three-way comparison of two boxed scalar values, the ordering primitive the compute
/// kernels lower every value predicate and `MIN` / `MAX` reduction onto.
///
/// The comparison mode is keyed off the *column* dtype, never the boxed operand type: integer
/// columns compare as longs (unsigned columns via [Long#compareUnsigned(long, long)]), float columns
/// as doubles, strings in UTF-8 byte order ([Utf8Order]), and everything else through the natural
/// [Comparable] order. Keying off the column
/// as doubles, strings in UTF-8 byte order ([Utf8Order]), decimals by value ([BigDecimal#compareTo]),
/// and everything else through the natural [Comparable] order. With no column, a [BigDecimal]
/// operand makes it a decimal compare. Keying off the column
/// keeps the compare width-agnostic — a caller may box a filter value at any integer width — and
/// never routes an integer column through double-compare, which would lose precision past 2^53.
///
Expand All @@ -26,13 +30,18 @@ private Compare() {
///
/// @param a the left value, typically the array element under test
/// @param b the right value, typically the predicate operand
/// @param column the dtype that decides the comparison mode (unsigned, floating, or natural), or
/// @param column the dtype that decides the comparison mode (unsigned, floating, decimal, or natural), or
/// `null` to compare width-agnostically off the operands (numbers as longs unless
/// either side is floating, otherwise the natural [Comparable] order)
/// @return a negative, zero, or positive integer as `a` is less than, equal to, or greater than `b`
/// @throws VortexException if the two values are not mutually comparable
@SuppressWarnings("unchecked")
public static int values(Object a, Object b, DType column) {
// a decimal is a Number, but its order is its value, never the long it truncates to; a
// primitive column keeps its own order, whatever the operand's type
if ((a instanceof BigDecimal || b instanceof BigDecimal) && !(column instanceof DType.Primitive)) {
return decimal(a).compareTo(decimal(b));
}
if (a instanceof Number na && b instanceof Number nb) {
if (column instanceof DType.Primitive prim) {
if (prim.ptype().isFloating()) {
Expand Down Expand Up @@ -64,6 +73,21 @@ public static int values(Object a, Object b, DType column) {
}
}

/// A decimal-column operand as a [BigDecimal]: a decimal itself, an integer of any width, or a
/// finite float or double. Anything else, a NaN or an infinity among them, has no decimal value.
private static BigDecimal decimal(Object value) {
return switch (value) {
case null -> throw new VortexException("compute value of type null is not comparable to a decimal");
case BigDecimal d -> d;
case BigInteger b -> new BigDecimal(b);
case Byte _, Short _, Integer _, Long _ -> BigDecimal.valueOf(((Number) value).longValue());
case Double d when Double.isFinite(d) -> BigDecimal.valueOf(d);
case Float f when Float.isFinite(f) -> new BigDecimal(f.toString());
default -> throw new VortexException("compute value of type " + className(value)
+ " (" + value + ") is not comparable to a decimal");
};
}

/// Returns the simple class name of `value`, or `"null"` when it is absent, for error messages.
///
/// @param value the value to describe
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
package io.github.dfa1.vortex.reader.compute;

import io.github.dfa1.vortex.core.error.VortexException;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
Expand Down Expand Up @@ -101,4 +109,68 @@ void incomparableValuesThrow() {
.isInstanceOf(VortexException.class)
.hasMessageContaining("not comparable");
}

static Stream<Arguments> decimalPairs() {
DType decimal = new DType.Decimal((byte) 10, (byte) 2, false);
DType wide = new DType.Decimal((byte) 38, (byte) 0, false);
BigDecimal i128 = new BigDecimal("170141183460469231731687303715884105727");
return Stream.of(
// 1.5 and 1.2 are both 1 as a long
Arguments.of("by value, not the truncated long", decimal, new BigDecimal("1.5"), new BigDecimal("1.2"), 1),
Arguments.of("same value, two scales", decimal, new BigDecimal("1.50"), new BigDecimal("1.5"), 0),
Arguments.of("negative", decimal, new BigDecimal("-0.01"), new BigDecimal("0.00"), -1),
Arguments.of("negative scale", decimal, new BigDecimal("1E+2"), 100L, 0),
Arguments.of("long operand", decimal, new BigDecimal("2.50"), 2L, 1),
Arguments.of("long on the left", decimal, 1L, new BigDecimal("1.00"), 0),
Arguments.of("double operand", decimal, new BigDecimal("2.50"), 2.75d, -1),
Arguments.of("float operand", decimal, new BigDecimal("2.50"), 2.25f, 1),
Arguments.of("past a long", wide, i128, Long.MAX_VALUE, 1),
Arguments.of("big integer operand", decimal, new BigDecimal("1"), BigInteger.TWO.pow(64), -1),
Arguments.of("no column", null, new BigDecimal("1.5"), new BigDecimal("1.2"), 1));
}

@ParameterizedTest(name = "{0}")
@MethodSource("decimalPairs")
void decimalsCompareByValue(String name, DType column, Object a, Object b, int sign) {
// Given a decimal column (or none) and operands of mixed types

// When
int result = Compare.values(a, b, column);

// Then
assertThat(Integer.signum(result)).isEqualTo(sign);
}

static Stream<Arguments> notDecimalValues() {
DType decimal = new DType.Decimal((byte) 10, (byte) 2, false);
return Stream.of(
Arguments.of("NaN", Double.NaN),
Arguments.of("infinity", Double.POSITIVE_INFINITY),
Arguments.of("float NaN", Float.NaN),
Arguments.of("text", "x"),
Arguments.of("null", null)).map(a -> Arguments.of(a.get()[0], decimal, a.get()[1]));
}

@ParameterizedTest(name = "{0}")
@MethodSource("notDecimalValues")
void decimalAgainstAValueWithNoDecimalMeaningThrows(String name, DType column, Object operand) {
// Given / When / Then
assertThatThrownBy(() -> Compare.values(new BigDecimal("1"), operand, column))
.isInstanceOf(VortexException.class)
.hasMessageContaining("not comparable to a decimal");
}

@Test
void primitiveColumnsKeepTheirOwnOrderWhateverTheOperand() {
// Given a U64 value past 2^63 (a negative long), and a float column holding NaN
long big = -1L;

// When
int unsigned = Compare.values(big, new BigDecimal("2"), DTypes.U64);
int nan = Compare.values(Double.NaN, new BigDecimal("1.5"), DTypes.F64);

// Then the unsigned order and the double order hold, no decimal conversion
assertThat(unsigned).isPositive();
assertThat(nan).isPositive();
}
}
Loading
Loading