diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e6b8de79..319f2b715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650)) - A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696)) +- **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) ### Added diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/FractionalLiteralTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/FractionalLiteralTest.java new file mode 100644 index 000000000..68bd7d49e --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/FractionalLiteralTest.java @@ -0,0 +1,76 @@ +package io.github.dfa1.vortex.calcite; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A fractional literal against an integer column (#701): the reader's integer compare truncated the +/// operand to a long, so `v >= 1.5` matched `v = 1`. Whether SQL reaches that path depends on how +/// Calcite types the comparison, so every operator runs with a `DECIMAL` and a `DOUBLE` literal. +class FractionalLiteralTest { + + private static final ColumnName COLUMN = ColumnName.of("v"); + + @TempDir + Path tmp; + + @ParameterizedTest(name = "v {0} -> {1}") + @CsvSource(delimiter = '|', value = { + // v holds 0, 1, 2, 3; 1.5 lies strictly between two of them, so truncation shows up on + // both sides of the boundary + ">= 1.5 | 2", + "> 1.5 | 2", + "<= 1.5 | 2", + "< 1.5 | 2", + "= 1.5 | 0", + "<> 1.5 | 4", + ">= 1.5e0 | 2", + "> 1.5e0 | 2", + "<= 1.5e0 | 2", + "< 1.5e0 | 2", + "= 1.5e0 | 0", + "<> 1.5e0 | 4", + ">= -0.5 | 4", + "< -0.5 | 0"}) + void count_fractionalLiteral_comparesByValue(String predicate, long expected) throws Exception { + // Given + Path file = write(); + + // When + long result; + try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file)); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery("select count(*) from vtx.t where v " + predicate)) { + rs.next(); + result = rs.getLong(1); + } + + // Then + assertThat(result).isEqualTo(expected); + } + + private Path write() throws Exception { + DType.Struct schema = DType.structBuilder().field(COLUMN, DTypes.I64).build(); + Path file = tmp.resolve("ints.vortex"); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var w = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(Map.of(COLUMN, new long[]{0, 1, 2, 3})); + } + return file; + } +} diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/FilterCommandTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/FilterCommandTest.java index e7f1d11f8..0dccb52e1 100644 --- a/cli/src/test/java/io/github/dfa1/vortex/cli/FilterCommandTest.java +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/FilterCommandTest.java @@ -117,16 +117,16 @@ void unknownColumn_returnsErrorNotCrash() { } @Test - void doubleValueAgainstLongColumn_returnsOk() { - // Given — a fractional literal parses as Double, compared against an I64 column - // (exercises parseValue's Double branch and compareNumeric's Double path) + void doubleValueAgainstLongColumn_returnsError() { + // Given — a fractional literal parses as Double, compared against an I64 column. The scan + // refuses it, as vortex-jni does, instead of truncating 1.5 to 1 (#701) // When CliTestSupport.Captured result = capture(() -> FilterCommand.run(new String[]{"filter", file.toString(), "id", ">", "1.5"})); - // Then — id in {2, 3} match - assertThat(result.status()).isEqualTo(ExitStatus.OK); - assertThat(result.stdout()).startsWith("id"); + // Then + assertThat(result.status()).isEqualTo(ExitStatus.ERROR); + assertThat(result.stderr()).contains("an integer column compares only with an integer value"); } @Test diff --git a/docs/compatibility.md b/docs/compatibility.md index c67b128d1..4942f5149 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -65,6 +65,10 @@ Separate from the Vector API, found while comparing against the Rust source: without a `nan_count` (the legacy `vortex.stats` table) may hold a `NaN`, so it is never pruned on its maximum. Pinned by `JavaWritesRustReadsIntegrationTest#javaWriter_jniReader_filteredFloats_matchJni` ([#518](https://github.com/dfa1/vortex-java/issues/518)). +- A float or decimal filter operand on an integer column is refused (`VortexException`), as Rust's + compare refuses operands of different dtypes. Integer operands of any width are still accepted, a + deliberate leniency: Rust would refuse an `i32` literal against an `i64` column too. Pinned by + `FractionalOperandInteropIntegrationTest` ([#701](https://github.com/dfa1/vortex-java/issues/701)). - `U64` min/max and the dense-counter span in `ArrayStats` are computed in unsigned order, as Rust's `typed_int_stats` does. - The zoned float `Sum` skips `NaN`, as Rust's `skip_nans()` binding does; an all-`NaN` zone sums to `0.0` diff --git a/docs/reference.md b/docs/reference.md index 5935f3ae4..864601600 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -233,6 +233,10 @@ cannot match are skipped entirely; in the rest every row is tested, so a scan re matching rows, and a limit counts them, as Rust's scan does. A chunk with no matching row is not returned. The filter's columns are decoded even when they are not projected. +An integer column takes an integer operand of any width (`Long`, `Integer`, `Short`, `Byte`). A +`Double`, `Float` or `BigDecimal` operand on an integer column fails the scan with +`VortexException`, as vortex-jni refuses it ("Cannot compare different DTypes i64 and f64"). + | Static factory | Builds | |------------------------------|------------------------------------| | `ScanFilter.eq(col, val)` | `Column` bound to `Predicate.Eq` | diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/FractionalOperandInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/FractionalOperandInteropIntegrationTest.java new file mode 100644 index 000000000..06945dced --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/FractionalOperandInteropIntegrationTest.java @@ -0,0 +1,94 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.DataSource; +import dev.vortex.api.Expression; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.ScanFilter; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/// A fractional operand against an integer column (#701). vortex-java truncated it to a long, so +/// `v >= 1.5` matched `v = 1`. Rust's compare requires both sides to share a dtype and refuses an +/// `i64` column against an `f64` literal; both readers must now refuse it. +class FractionalOperandInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final ColumnName V = ColumnName.of("v"); + + static { + NativeLoader.loadJni(); + } + + @Test + void fractionalOperandOnIntegerColumn_bothReadersReject(@TempDir Path tmp) throws Exception { + // Given — 0, 1, 2, 3: 1.5 falls between two values, so a truncating compare would select + // rows instead of failing + Path file = tmp.resolve("ints.vortex"); + DType.Struct schema = DType.structBuilder().field(V, DTypes.I64).build(); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(Map.of(V, new long[]{0, 1, 2, 3})); + } + + // When / Then + assertThatThrownBy(() -> scanJni(file)) + .as("vortex-jni") + .hasMessageContaining("Cannot compare different DTypes i64 and f64"); + assertThatThrownBy(() -> scanJava(file)) + .as("vortex-java") + .isInstanceOf(VortexException.class) + .hasMessageContaining("an integer column compares only with an integer value"); + } + + private static void scanJava(Path file) throws Exception { + try (VortexReader reader = VortexReader.open(file); + ScanIterator scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all() + .withFilter(ScanFilter.gte(V, 1.5)))) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + chunk.columns(); + } + } + } + } + + private static void scanJni(Path file) throws Exception { + ScanOptions opts = ScanOptions.builder() + .projection(Expression.select(new String[]{"v"}, Expression.root())) + .filter(Expression.binary(Expression.BinaryOp.GTE, Expression.column("v"), Expression.literal(1.5))) + .build(); + DataSource ds = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString()); + Scan scan = ds.scan(opts); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) { + while (reader.loadNextBatch()) { + reader.getVectorSchemaRoot(); + } + } + } + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java index 7948e2560..148767fe0 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/Compare.java @@ -50,9 +50,9 @@ public static int values(Object a, Object b, DType column) { // U64 values store the raw 64 bits, so a value >= 2^63 is a negative Long; an // unsigned column must compare unsigned. U8/U16/U32 are zero-extended to a positive // Long where signed == unsigned, so this stays correct for them too. - return column.isUnsigned() - ? Long.compareUnsigned(na.longValue(), nb.longValue()) - : Long.compare(na.longValue(), nb.longValue()); + long la = integer(a); + long lb = integer(b); + return column.isUnsigned() ? Long.compareUnsigned(la, lb) : Long.compare(la, lb); } // Column type unresolved — fall back to a width-agnostic compare keyed off the operands // so two valid numbers never drop into the throwing path. @@ -88,6 +88,24 @@ private static BigDecimal decimal(Object value) { }; } + /// An integer-column operand as a long. Integers of any width are accepted, as everywhere in the + /// compute API; a float, double or decimal is rejected, as Rust's compare requires both sides to + /// share a dtype: truncating it would make `x >= 1.5` match `x = 1` (#701). + /// + /// @param value the operand + /// @return its long value + /// @throws VortexException if `value` is not a `Long`, `Integer`, `Short` or `Byte` + static long integer(Object value) { + return switch (value) { + case Long l -> l; + case Integer i -> i; + case Short s -> s; + case Byte b -> b; + default -> throw new VortexException("an integer column compares only with an integer value, got " + + className(value) + " (" + value + ")"); + }; + } + /// Returns the simple class name of `value`, or `"null"` when it is absent, for error messages. /// /// @param value the value to describe diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilter.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilter.java index 5e951329c..fe601c385 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilter.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilter.java @@ -50,44 +50,44 @@ static boolean predicateOk(Predicate predicate) { /// @param predicate the comparison leaf (Eq / Neq / Lt / Lte / Gt / Gte / Between) /// @param flip the order-preserving XOR flip (`Long.MIN_VALUE` unsigned, `0` signed) /// @return the inclusive range on the flipped longs and whether to select its complement + /// @throws io.github.dfa1.vortex.core.error.VortexException if an operand is not an integer static LongRange lowerLong(Predicate predicate, long flip) { // Every leaf lowers to a single inclusive range on the flipped longs; Neq is that same // range (the point [c, c]) tested negated, so the match is the complement of equality. return switch (predicate) { case Predicate.Eq(var value) -> { - long c = ((Number) value).longValue() ^ flip; + long c = Compare.integer(value) ^ flip; yield new LongRange(c, c, false); } case Predicate.Neq(var value) -> { - long c = ((Number) value).longValue() ^ flip; + long c = Compare.integer(value) ^ flip; yield new LongRange(c, c, true); } case Predicate.Lt(var value) -> { - long c = ((Number) value).longValue() ^ flip; + long c = Compare.integer(value) ^ flip; yield c == Long.MIN_VALUE ? new LongRange(0L, -1L, false) : new LongRange(Long.MIN_VALUE, c - 1, false); } case Predicate.Lte(var value) -> { // v <= c → inclusive [MIN, c]; no underflow guard needed (c is the upper bound). - long c = ((Number) value).longValue() ^ flip; + long c = Compare.integer(value) ^ flip; yield new LongRange(Long.MIN_VALUE, c, false); } case Predicate.Gt(var value) -> { - long c = ((Number) value).longValue() ^ flip; + long c = Compare.integer(value) ^ flip; yield c == Long.MAX_VALUE ? new LongRange(0L, -1L, false) : new LongRange(c + 1, Long.MAX_VALUE, false); } case Predicate.Gte(var value) -> { // v >= c → inclusive [c, MAX]; no overflow guard needed (c is the lower bound). - long c = ((Number) value).longValue() ^ flip; + long c = Compare.integer(value) ^ flip; yield new LongRange(c, Long.MAX_VALUE, false); } default -> { Predicate.Between between = (Predicate.Between) predicate; - yield new LongRange(((Number) between.lo()).longValue() ^ flip, - ((Number) between.hi()).longValue() ^ flip, false); + yield new LongRange(Compare.integer(between.lo()) ^ flip, Compare.integer(between.hi()) ^ flip, false); } }; } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java index f551ace8e..0b7ba0382 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/CompareTest.java @@ -17,6 +17,36 @@ class CompareTest { + static Stream fractionalOperands() { + return Stream.of( + Arguments.of(1.5, DTypes.I64), + Arguments.of(1.5f, DTypes.I32), + Arguments.of(new BigDecimal("1.5"), DTypes.U8), + // a whole double is rejected too: Rust refuses the f64 dtype, not the fraction + Arguments.of(2.0, DTypes.I64)); + } + + @ParameterizedTest(name = "{0} vs {1}") + @MethodSource("fractionalOperands") + void integerColumn_nonIntegerOperand_throwsVortexException(Object operand, DType column) { + // Given an integer column value and a float/double/decimal operand. The compare used to + // truncate it to a long, so `x >= 1.5` matched `x = 1` (#701) + // When / Then + assertThatThrownBy(() -> Compare.values(1L, operand, column)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("an integer column compares only with an integer value"); + } + + @Test + void integerColumn_narrowIntegerOperands_compareByValue() { + // Given operands boxed at every integer width, which callers may pass for any integer column + // When + int result = Compare.values((byte) 1, (short) 2, DTypes.I64) + Compare.values(3, 3L, DTypes.I64); + + // Then — 1 < 2 contributes -1, 3 == 3 contributes 0 + assertThat(result).isEqualTo(-1); + } + @Test void stringsOrderByUtf8BytesNotUtf16() { // Given a supplementary character (a surrogate pair in UTF-16, 4 bytes in UTF-8) and a BMP @@ -162,11 +192,12 @@ void decimalAgainstAValueWithNoDecimalMeaningThrows(String name, DType column, O @Test void primitiveColumnsKeepTheirOwnOrderWhateverTheOperand() { - // Given a U64 value past 2^63 (a negative long), and a float column holding NaN + // Given a U64 value past 2^63 (a negative long), and a float column holding NaN. A decimal + // operand on the integer column is rejected (#701), so that side uses an integer one long big = -1L; // When - int unsigned = Compare.values(big, new BigDecimal("2"), DTypes.U64); + int unsigned = Compare.values(big, 2L, DTypes.U64); int nan = Compare.values(Double.NaN, new BigDecimal("1.5"), DTypes.F64); // Then the unsigned order and the double order hold, no decimal conversion diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilterTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilterTest.java index b211c1f1e..8f4a0945f 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilterTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/PrimitiveFilterTest.java @@ -1,9 +1,11 @@ package io.github.dfa1.vortex.reader.compute; +import io.github.dfa1.vortex.core.error.VortexException; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /// Covers [PrimitiveFilter#matchDouble(double, PrimitiveFilter.DoubleOp, double, double)]'s float /// ordering (#406 gap 3, #518): Rust's total order, the same as `Double.compare`, where `-0.0 < 0.0` @@ -97,4 +99,30 @@ void lowersEachComparisonKindToItsMatchingOperator() { .isEqualTo(new PrimitiveFilter.DoubleBound(PrimitiveFilter.DoubleOp.BETWEEN, 0.0, 10.0)); } } + + @Nested + class LowerLong { + + @Test + void fractionalBound_throwsVortexException() { + // Given a Between whose upper bound is a double. The fused kernels lower integer + // columns here, and truncating 2.5 to 2 silently changed the range (#701) + Predicate predicate = new Predicate.Between(1L, 2.5); + + // When / Then + assertThatThrownBy(() -> PrimitiveFilter.lowerLong(predicate, 0L)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("an integer column compares only with an integer value"); + } + + @Test + void integerBound_lowersToItsRange() { + // Given an Integer operand, a narrower box than the column's long domain + // When + PrimitiveFilter.LongRange result = PrimitiveFilter.lowerLong(new Predicate.Gte(7), 0L); + + // Then + assertThat(result).isEqualTo(new PrimitiveFilter.LongRange(7L, Long.MAX_VALUE, false)); + } + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java index 61b5cd89d..4488cff64 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java @@ -503,22 +503,24 @@ void f16ColumnPrunesLikeF32(String name, Object value, @TempDir Path tmp) throws } /// Comparison mode keys off the *column* type, not the filter value's boxing — so an integer - /// column never routes through double-compare, which would lose precision past 2^53. + /// column never routes through double-compare, which would lose precision past 2^53. A + /// floating operand on an integer column is refused outright, as Rust's compare refuses an + /// operand of another dtype (#701). @Nested class IntegerColumnFloatFilter { @Test - void doubleFilterOnI64ComparesInIntegerDomain(@TempDir Path tmp) throws IOException { - // Given — a chunk whose max (2^53 + 1) is NOT exactly representable as a double + void doubleFilterOnI64_throwsVortexException(@TempDir Path tmp) throws IOException { + // Given — a chunk whose max (2^53 + 1) is NOT exactly representable as a double, so a + // double-domain compare would round it down and wrongly prune the matching row long max = (1L << 53) + 1; Path file = writeI64Chunk(tmp, max - 2, max - 1, max); + ScanFilter filter = ScanFilter.gt(ColumnName.of("id"), (double) (1L << 53)); - // When — gt with a Double threshold of 2^53; only the row 2^53+1 satisfies it - List rowCounts = scanRowCounts(file, ScanFilter.gt(ColumnName.of("id"), (double) (1L << 53))); - - // Then — chunk kept. A double-domain compare would round 2^53+1 down to 2^53 and wrongly - // prune the chunk, dropping the one matching row. - assertThat(rowCounts).containsExactly(3L); + // When / Then — refused even though 2^53 is whole: the dtype differs, not the value + assertThatThrownBy(() -> scanRowCounts(file, filter)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("an integer column compares only with an integer value"); } }