Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `VortexReader.columnStats()` reports the min and max of a decimal column, and a decimal filter compares by value instead of by the long it truncates to. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- A nullable text column without a null is written without a validity, as Rust writes it, instead of under `vortex.masked` (with cascading on). ([#650](https://github.com/dfa1/vortex-java/issues/650))
- A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696))
- **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701))
- A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694))

### Added
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.writer.VortexWriter;
import io.github.dfa1.vortex.writer.WriteOptions;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;

/// A fractional literal against an integer column (#701): the reader's integer compare truncated the
/// operand to a long, so `v >= 1.5` matched `v = 1`. Whether SQL reaches that path depends on how
/// Calcite types the comparison, so every operator runs with a `DECIMAL` and a `DOUBLE` literal.
class FractionalLiteralTest {

private static final ColumnName COLUMN = ColumnName.of("v");

@TempDir
Path tmp;

@ParameterizedTest(name = "v {0} -> {1}")
@CsvSource(delimiter = '|', value = {
// v holds 0, 1, 2, 3; 1.5 lies strictly between two of them, so truncation shows up on
// both sides of the boundary
">= 1.5 | 2",
"> 1.5 | 2",
"<= 1.5 | 2",
"< 1.5 | 2",
"= 1.5 | 0",
"<> 1.5 | 4",
">= 1.5e0 | 2",
"> 1.5e0 | 2",
"<= 1.5e0 | 2",
"< 1.5e0 | 2",
"= 1.5e0 | 0",
"<> 1.5e0 | 4",
">= -0.5 | 4",
"< -0.5 | 0"})
void count_fractionalLiteral_comparesByValue(String predicate, long expected) throws Exception {
// Given
Path file = write();

// When
long result;
try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file));
Statement st = conn.createStatement();
ResultSet rs = st.executeQuery("select count(*) from vtx.t where v " + predicate)) {
rs.next();
result = rs.getLong(1);
}

// Then
assertThat(result).isEqualTo(expected);
}

private Path write() throws Exception {
DType.Struct schema = DType.structBuilder().field(COLUMN, DTypes.I64).build();
Path file = tmp.resolve("ints.vortex");
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
var w = VortexWriter.create(ch, schema, WriteOptions.defaults())) {
w.writeChunk(Map.of(COLUMN, new long[]{0, 1, 2, 3}));
}
return file;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -117,16 +117,16 @@ void unknownColumn_returnsErrorNotCrash() {
}

@Test
void doubleValueAgainstLongColumn_returnsOk() {
// Given — a fractional literal parses as Double, compared against an I64 column
// (exercises parseValue's Double branch and compareNumeric's Double path)
void doubleValueAgainstLongColumn_returnsError() {
// Given — a fractional literal parses as Double, compared against an I64 column. The scan
// refuses it, as vortex-jni does, instead of truncating 1.5 to 1 (#701)
// When
CliTestSupport.Captured result = capture(() ->
FilterCommand.run(new String[]{"filter", file.toString(), "id", ">", "1.5"}));

// Then — id in {2, 3} match
assertThat(result.status()).isEqualTo(ExitStatus.OK);
assertThat(result.stdout()).startsWith("id");
// Then
assertThat(result.status()).isEqualTo(ExitStatus.ERROR);
assertThat(result.stderr()).contains("an integer column compares only with an integer value");
}

@Test
Expand Down
4 changes: 4 additions & 0 deletions docs/compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ Separate from the Vector API, found while comparing against the Rust source:
without a `nan_count` (the legacy `vortex.stats` table) may hold a `NaN`, so it is never pruned on its
maximum. Pinned by `JavaWritesRustReadsIntegrationTest#javaWriter_jniReader_filteredFloats_matchJni`
([#518](https://github.com/dfa1/vortex-java/issues/518)).
- A float or decimal filter operand on an integer column is refused (`VortexException`), as Rust's
compare refuses operands of different dtypes. Integer operands of any width are still accepted, a
deliberate leniency: Rust would refuse an `i32` literal against an `i64` column too. Pinned by
`FractionalOperandInteropIntegrationTest` ([#701](https://github.com/dfa1/vortex-java/issues/701)).
- `U64` min/max and the dense-counter span in `ArrayStats` are computed in unsigned order, as Rust's
`typed_int_stats` does.
- The zoned float `Sum` skips `NaN`, as Rust's `skip_nans()` binding does; an all-`NaN` zone sums to `0.0`
Expand Down
4 changes: 4 additions & 0 deletions docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,10 @@ cannot match are skipped entirely; in the rest every row is tested, so a scan re
matching rows, and a limit counts them, as Rust's scan does. A chunk with no matching row is not
returned. The filter's columns are decoded even when they are not projected.

An integer column takes an integer operand of any width (`Long`, `Integer`, `Short`, `Byte`). A
`Double`, `Float` or `BigDecimal` operand on an integer column fails the scan with
`VortexException`, as vortex-jni refuses it ("Cannot compare different DTypes i64 and f64").

| Static factory | Builds |
|------------------------------|------------------------------------|
| `ScanFilter.eq(col, val)` | `Column` bound to `Predicate.Eq` |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package io.github.dfa1.vortex.integration;

import dev.vortex.api.DataSource;
import dev.vortex.api.Expression;
import dev.vortex.api.Partition;
import dev.vortex.api.Scan;
import dev.vortex.api.ScanOptions;
import dev.vortex.api.Session;
import dev.vortex.arrow.ArrowAllocation;
import dev.vortex.jni.NativeLoader;
import io.github.dfa1.vortex.core.error.VortexException;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.reader.Chunk;
import io.github.dfa1.vortex.reader.ScanFilter;
import io.github.dfa1.vortex.reader.ScanIterator;
import io.github.dfa1.vortex.reader.VortexReader;
import io.github.dfa1.vortex.writer.WriteOptions;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Map;

import static org.assertj.core.api.Assertions.assertThatThrownBy;

/// A fractional operand against an integer column (#701). vortex-java truncated it to a long, so
/// `v >= 1.5` matched `v = 1`. Rust's compare requires both sides to share a dtype and refuses an
/// `i64` column against an `f64` literal; both readers must now refuse it.
class FractionalOperandInteropIntegrationTest {

private static final Session SESSION = Session.create();
private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
private static final ColumnName V = ColumnName.of("v");

static {
NativeLoader.loadJni();
}

@Test
void fractionalOperandOnIntegerColumn_bothReadersReject(@TempDir Path tmp) throws Exception {
// Given — 0, 1, 2, 3: 1.5 falls between two values, so a truncating compare would select
// rows instead of failing
Path file = tmp.resolve("ints.vortex");
DType.Struct schema = DType.structBuilder().field(V, DTypes.I64).build();
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) {
w.writeChunk(Map.of(V, new long[]{0, 1, 2, 3}));
}

// When / Then
assertThatThrownBy(() -> scanJni(file))
.as("vortex-jni")
.hasMessageContaining("Cannot compare different DTypes i64 and f64");
assertThatThrownBy(() -> scanJava(file))
.as("vortex-java")
.isInstanceOf(VortexException.class)
.hasMessageContaining("an integer column compares only with an integer value");
}

private static void scanJava(Path file) throws Exception {
try (VortexReader reader = VortexReader.open(file);
ScanIterator scan = reader.scan(io.github.dfa1.vortex.reader.ScanOptions.all()
.withFilter(ScanFilter.gte(V, 1.5)))) {
while (scan.hasNext()) {
try (Chunk chunk = scan.next()) {
chunk.columns();
}
}
}
}

private static void scanJni(Path file) throws Exception {
ScanOptions opts = ScanOptions.builder()
.projection(Expression.select(new String[]{"v"}, Expression.root()))
.filter(Expression.binary(Expression.BinaryOp.GTE, Expression.column("v"), Expression.literal(1.5)))
.build();
DataSource ds = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString());
Scan scan = ds.scan(opts);
while (scan.hasNext()) {
Partition partition = scan.next();
try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
while (reader.loadNextBatch()) {
reader.getVectorSchemaRoot();
}
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ public static int values(Object a, Object b, DType column) {
// U64 values store the raw 64 bits, so a value >= 2^63 is a negative Long; an
// unsigned column must compare unsigned. U8/U16/U32 are zero-extended to a positive
// Long where signed == unsigned, so this stays correct for them too.
return column.isUnsigned()
? Long.compareUnsigned(na.longValue(), nb.longValue())
: Long.compare(na.longValue(), nb.longValue());
long la = integer(a);
long lb = integer(b);
return column.isUnsigned() ? Long.compareUnsigned(la, lb) : Long.compare(la, lb);
}
// Column type unresolved — fall back to a width-agnostic compare keyed off the operands
// so two valid numbers never drop into the throwing path.
Expand Down Expand Up @@ -88,6 +88,24 @@ private static BigDecimal decimal(Object value) {
};
}

/// An integer-column operand as a long. Integers of any width are accepted, as everywhere in the
/// compute API; a float, double or decimal is rejected, as Rust's compare requires both sides to
/// share a dtype: truncating it would make `x >= 1.5` match `x = 1` (#701).
///
/// @param value the operand
/// @return its long value
/// @throws VortexException if `value` is not a `Long`, `Integer`, `Short` or `Byte`
static long integer(Object value) {
return switch (value) {
case Long l -> l;
case Integer i -> i;
case Short s -> s;
case Byte b -> b;
default -> throw new VortexException("an integer column compares only with an integer value, got "
+ className(value) + " (" + value + ")");
};
}

/// Returns the simple class name of `value`, or `"null"` when it is absent, for error messages.
///
/// @param value the value to describe
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,44 +50,44 @@ static boolean predicateOk(Predicate predicate) {
/// @param predicate the comparison leaf (Eq / Neq / Lt / Lte / Gt / Gte / Between)
/// @param flip the order-preserving XOR flip (`Long.MIN_VALUE` unsigned, `0` signed)
/// @return the inclusive range on the flipped longs and whether to select its complement
/// @throws io.github.dfa1.vortex.core.error.VortexException if an operand is not an integer
static LongRange lowerLong(Predicate predicate, long flip) {
// Every leaf lowers to a single inclusive range on the flipped longs; Neq is that same
// range (the point [c, c]) tested negated, so the match is the complement of equality.
return switch (predicate) {
case Predicate.Eq(var value) -> {
long c = ((Number) value).longValue() ^ flip;
long c = Compare.integer(value) ^ flip;
yield new LongRange(c, c, false);
}
case Predicate.Neq(var value) -> {
long c = ((Number) value).longValue() ^ flip;
long c = Compare.integer(value) ^ flip;
yield new LongRange(c, c, true);
}
case Predicate.Lt(var value) -> {
long c = ((Number) value).longValue() ^ flip;
long c = Compare.integer(value) ^ flip;
yield c == Long.MIN_VALUE
? new LongRange(0L, -1L, false)
: new LongRange(Long.MIN_VALUE, c - 1, false);
}
case Predicate.Lte(var value) -> {
// v <= c → inclusive [MIN, c]; no underflow guard needed (c is the upper bound).
long c = ((Number) value).longValue() ^ flip;
long c = Compare.integer(value) ^ flip;
yield new LongRange(Long.MIN_VALUE, c, false);
}
case Predicate.Gt(var value) -> {
long c = ((Number) value).longValue() ^ flip;
long c = Compare.integer(value) ^ flip;
yield c == Long.MAX_VALUE
? new LongRange(0L, -1L, false)
: new LongRange(c + 1, Long.MAX_VALUE, false);
}
case Predicate.Gte(var value) -> {
// v >= c → inclusive [c, MAX]; no overflow guard needed (c is the lower bound).
long c = ((Number) value).longValue() ^ flip;
long c = Compare.integer(value) ^ flip;
yield new LongRange(c, Long.MAX_VALUE, false);
}
default -> {
Predicate.Between between = (Predicate.Between) predicate;
yield new LongRange(((Number) between.lo()).longValue() ^ flip,
((Number) between.hi()).longValue() ^ flip, false);
yield new LongRange(Compare.integer(between.lo()) ^ flip, Compare.integer(between.hi()) ^ flip, false);
}
};
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,36 @@

class CompareTest {

static Stream<Arguments> fractionalOperands() {
return Stream.of(
Arguments.of(1.5, DTypes.I64),
Arguments.of(1.5f, DTypes.I32),
Arguments.of(new BigDecimal("1.5"), DTypes.U8),
// a whole double is rejected too: Rust refuses the f64 dtype, not the fraction
Arguments.of(2.0, DTypes.I64));
}

@ParameterizedTest(name = "{0} vs {1}")
@MethodSource("fractionalOperands")
void integerColumn_nonIntegerOperand_throwsVortexException(Object operand, DType column) {
// Given an integer column value and a float/double/decimal operand. The compare used to
// truncate it to a long, so `x >= 1.5` matched `x = 1` (#701)
// When / Then
assertThatThrownBy(() -> Compare.values(1L, operand, column))
.isInstanceOf(VortexException.class)
.hasMessageContaining("an integer column compares only with an integer value");
}

@Test
void integerColumn_narrowIntegerOperands_compareByValue() {
// Given operands boxed at every integer width, which callers may pass for any integer column
// When
int result = Compare.values((byte) 1, (short) 2, DTypes.I64) + Compare.values(3, 3L, DTypes.I64);

// Then — 1 < 2 contributes -1, 3 == 3 contributes 0
assertThat(result).isEqualTo(-1);
}

@Test
void stringsOrderByUtf8BytesNotUtf16() {
// Given a supplementary character (a surrogate pair in UTF-16, 4 bytes in UTF-8) and a BMP
Expand Down Expand Up @@ -162,11 +192,12 @@ void decimalAgainstAValueWithNoDecimalMeaningThrows(String name, DType column, O

@Test
void primitiveColumnsKeepTheirOwnOrderWhateverTheOperand() {
// Given a U64 value past 2^63 (a negative long), and a float column holding NaN
// Given a U64 value past 2^63 (a negative long), and a float column holding NaN. A decimal
// operand on the integer column is rejected (#701), so that side uses an integer one
long big = -1L;

// When
int unsigned = Compare.values(big, new BigDecimal("2"), DTypes.U64);
int unsigned = Compare.values(big, 2L, DTypes.U64);
int nan = Compare.values(Double.NaN, new BigDecimal("1.5"), DTypes.F64);

// Then the unsigned order and the double order hold, no decimal conversion
Expand Down
Loading
Loading