Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]

### Fixed
- A decimal min or max in a flat node's stats is decoded as a `BigDecimal` instead of text, so a filtered scan can prune on it and `columnStats()` reports it. ([#710](https://github.com/dfa1/vortex-java/issues/710))
- A `vortex.uuid` column's file statistics carry its min and max, as Rust writes them from 0.92, and `WriteSummary` reports them as `UUID`s. ([#730](https://github.com/dfa1/vortex-java/pull/730))

### Changed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,13 @@
import io.github.dfa1.vortex.core.fbs.FbsArrayStats;
import io.github.dfa1.vortex.core.fbs.FbsPrecision;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.proto.ProtoScalarValue;

import java.io.IOException;
import java.lang.foreign.MemorySegment;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;

/// Parses the wire `ArrayStats` FlatBuffer table into an [ArrayStats]: Rust's `from_flatbuffer`.
Expand All @@ -23,11 +26,23 @@ private ArrayStatsParser() {
/// @return parsed stats, or an empty instance if no usable data is present
/// @throws VortexException if a precision field or a scalar value is malformed
public static ArrayStats parse(FbsArrayStats fbs) {
return parse(fbs, null);
}

/// Parses stats like [#parse(FbsArrayStats)], decoding the min and max against the column's
/// dtype: Rust writes a decimal bound as the little-endian two's-complement bytes of its unscaled
/// value, which only the dtype's scale turns into a number.
///
/// @param fbs the FlatBuffers stats table, or `null`
/// @param dtype the dtype of the array the stats describe, or `null` when unknown
/// @return parsed stats, or an empty instance if no usable data is present
/// @throws VortexException if a precision field or a scalar value is malformed
public static ArrayStats parse(FbsArrayStats fbs, DType dtype) {
if (fbs == null) {
return ArrayStats.empty();
}
Object min = decodeScalar(fbs.minAsSegment());
Object max = decodeScalar(fbs.maxAsSegment());
Object min = decodeBound(fbs.minAsSegment(), dtype);
Object max = decodeBound(fbs.maxAsSegment(), dtype);
ArrayStats stats = new ArrayStats(
min,
min != null && isExact(fbs.minPrecision(), "min_precision"),
Expand All @@ -53,6 +68,42 @@ private static boolean isExact(int precision, String field) {
};
}

private static Object decodeBound(MemorySegment seg, DType dtype) {
Object value = decodeScalar(seg);
// Rust's from_proto reads an extension's scalar as its storage dtype's.
while (dtype instanceof DType.Extension extension) {
dtype = extension.storageDType();
}
if (!(dtype instanceof DType.Decimal decimal) || value == null) {
return value;
}
// Rust reads a decimal's BytesValue only (bytes_from_proto): any other kind is a corrupted
// bound, and must not reach a fold that orders BigDecimal against it.
byte[] bytes = decodeBytes(seg);
if (bytes == null) {
throw new VortexException("decimal bound in array stats is not a byte string: " + value);
}
// bytes_from_proto's match: the storage width, 1, 2, 4, 8, 16 or 32 bytes.
int width = bytes.length;
switch (width) {
case 1, 2, 4, 8, 16, 32 -> { }
default -> throw new VortexException("invalid decimal byte length in array stats: " + width);
}
byte[] bigEndian = new byte[width];
for (int i = 0; i < width; i++) {
bigEndian[i] = bytes[width - 1 - i];
}
return new BigDecimal(new BigInteger(bigEndian), decimal.scale());
}

private static byte[] decodeBytes(MemorySegment seg) {
try {
return ProtoScalarValue.decode(seg, 0, seg.byteSize()).bytes_value();
} catch (IOException e) {
throw new VortexException("invalid scalar value in array stats", e);
}
}

private static Object decodeScalar(MemorySegment seg) {
if (seg == null || seg.byteSize() == 0) {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -626,12 +626,12 @@ public List<ArrayStats> chunkStats(ColumnName name) {
if (layout != null && layout.isDict() && !layout.children().isEmpty()) {
// child[0] = values, child[1] = codes — the order DictLayoutDecoder decodes.
Layout values = layout.children().getFirst();
ArrayStats pool = values.isFlat() ? readFlatStats(values) : ArrayStats.empty();
ArrayStats pool = values.isFlat() ? readFlatStats(values, columnDType(name)) : ArrayStats.empty();
out.add(new ArrayStats(pool.min(), pool.minExact(), pool.max(), pool.maxExact(),
null, null, null, null, null, null, null));
continue;
}
out.add(layout == null || !layout.isFlat() ? ArrayStats.empty() : readFlatStats(layout));
out.add(layout == null || !layout.isFlat() ? ArrayStats.empty() : readFlatStats(layout, columnDType(name)));
}
return out;
}
Expand Down Expand Up @@ -1165,7 +1165,7 @@ private boolean canPruneChunk(ChunkSpec chunk, ScanFilter filter) {
// zones in a larger chunk), and the IS NOT NULL test compares nullCount against it.
List<Zone> zones = zonesOverlapping(chunk, col);
if (zones == null) {
yield canPrune(predicate, readFlatStats(flat), flat.rowCount(), columnDType(col))
yield canPrune(predicate, readFlatStats(flat, columnDType(col)), flat.rowCount(), columnDType(col))
|| dictMatchesNothing(flat, col, predicate);
}
// The window has no matching row iff none of the zones covering it has one.
Expand Down Expand Up @@ -1266,7 +1266,7 @@ private static boolean rejectsNull(Predicate predicate) {
/// Returns the zones of `col` that together cover `chunk`'s window, in row order, or `null` when
/// the column has no usable zone map or its zones do not reach the window's end — in which case
/// the caller falls back to the chunk's own embedded stats. Reading here decodes (and caches) one
/// small zone-map segment per column for the whole scan, instead of [#readFlatStats(Layout)]'s
/// small zone-map segment per column for the whole scan, instead of [#readFlatStats(Layout, DType)]'s
/// per-chunk read of the chunk's full data segment — the difference that makes pruning over
/// HTTP actually cheaper than not pruning.
private List<Zone> zonesOverlapping(ChunkSpec chunk, ColumnName col) {
Expand Down Expand Up @@ -1432,8 +1432,8 @@ static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DT
Object min = stats.min();
Object max = stats.max();
if (ct instanceof DType.Decimal) {
// A flat node's stats scalar is decoded without the column's dtype, so a decimal bound
// arrives as text: not a bound this column can order, hence no bound at all.
// Only a BigDecimal orders against a decimal column: a bound of any other type (a stats
// scalar decoded without its dtype) is no bound at all.
if (!(min instanceof java.math.BigDecimal)) {
min = null;
}
Expand Down Expand Up @@ -1482,7 +1482,7 @@ static boolean canPrune(Predicate predicate, ArrayStats stats, long rowCount, DT
};
}

private ArrayStats readFlatStats(Layout flat) {
private ArrayStats readFlatStats(Layout flat, DType dtype) {
if (flat.segments().isEmpty()) {
return ArrayStats.empty();
}
Expand Down Expand Up @@ -1511,7 +1511,7 @@ private ArrayStats readFlatStats(Layout flat) {
if (root == null) {
return ArrayStats.empty();
}
return ArrayStatsParser.parse(root.stats());
return ArrayStatsParser.parse(root.stats(), dtype);
} catch (VortexException _) {
// The FlatBuffer verifier rejected the stats segment: malformed stats read as none.
return ArrayStats.empty();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,22 @@
import io.github.dfa1.vortex.core.fbs.FbsBuilder;
import io.github.dfa1.vortex.core.error.VortexException;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.ExtensionId;
import io.github.dfa1.vortex.core.model.OpaqueMetadata;
import io.github.dfa1.vortex.core.proto.ProtoScalarValue;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.stream.Stream;


import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -258,6 +271,131 @@ void malformedScalarBytes_throwsVortexException() {
}
}

@Nested
class ParseDecimal {

// Rust writes a decimal bound as the little-endian two's-complement bytes of its unscaled
// value, at the storage width: negative values and every width catch a sign or byte-order slip.
@ParameterizedTest
@ValueSource(ints = {1, 2, 4, 8, 16, 32})
void bound_decodesAtEveryStorageWidth(int width) {
// Given
BigInteger min = BigInteger.valueOf(-100);
BigInteger max = BigInteger.valueOf(99);
var fbs = buildFbs(
ProtoScalarValue.ofBytesValue(littleEndian(min, width)).encode(),
ProtoScalarValue.ofBytesValue(littleEndian(max, width)).encode());

// When
ArrayStats result = ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) 2, false));

// Then
assertThat(result.min()).isEqualTo(new BigDecimal("-1.00"));
assertThat(result.max()).isEqualTo(new BigDecimal("0.99"));
}

@Test
void negativeScale_keepsTheUnscaledValue() {
// Given
var fbs = buildFbs(
ProtoScalarValue.ofBytesValue(littleEndian(BigInteger.valueOf(5), 4)).encode(),
ProtoScalarValue.ofBytesValue(littleEndian(BigInteger.valueOf(7), 4)).encode());

// When
ArrayStats result = ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) -3, false));

// Then
assertThat(result.min()).isEqualTo(new BigDecimal("5E+3"));
assertThat(result.max()).isEqualTo(new BigDecimal("7E+3"));
}

@ParameterizedTest
@ValueSource(ints = {0, 3, 5, 33})
void bound_ofAnInvalidWidth_isRejected(int width) {
// Given
var fbs = buildFbs(ProtoScalarValue.ofBytesValue(new byte[width]).encode(), null);

// When / Then
assertThatThrownBy(() -> ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) 2, false)))
.isInstanceOf(VortexException.class)
.hasMessageContaining("invalid decimal byte length");
}

// Rust reads only a BytesValue for a decimal (bytes_from_proto): a crafted file with another kind
// must raise VortexException, not an NPE, and never reach a fold ordering it against a BigDecimal.
@ParameterizedTest
@MethodSource("nonBytesScalars")
void bound_thatIsNotBytes_isRejected(ProtoScalarValue scalar) {
// Given
var fbs = buildFbs(scalar.encode(), null);

// When / Then
assertThatThrownBy(() -> ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 9, (byte) 2, false)))
.isInstanceOf(VortexException.class)
.hasMessageContaining("not a byte string");
}

static Stream<ProtoScalarValue> nonBytesScalars() {
return Stream.of(
ProtoScalarValue.ofStringValue("1.5"),
ProtoScalarValue.ofInt64Value(7L),
ProtoScalarValue.ofBoolValue(true));
}

// The small values above fit one byte; past a long only the 16- and 32-byte widths carry them.
@ParameterizedTest
@ValueSource(ints = {16, 32})
void bound_beyondALong_decodes(int width) {
// Given
BigInteger max = BigInteger.TWO.pow(100);
var fbs = buildFbs(
ProtoScalarValue.ofBytesValue(littleEndian(max.negate(), width)).encode(),
ProtoScalarValue.ofBytesValue(littleEndian(max, width)).encode());

// When
ArrayStats result = ArrayStatsParser.parse(fbs, new DType.Decimal((byte) 38, (byte) 0, false));

// Then
assertThat(result.min()).isEqualTo(new BigDecimal(max.negate()));
assertThat(result.max()).isEqualTo(new BigDecimal(max));
}

@Test
void bound_ofAnExtensionOverADecimal_decodesAsTheStorage() {
// Given
DType storage = new DType.Decimal((byte) 9, (byte) 2, false);
DType extension = new DType.Extension(ExtensionId.parse("acme.money"), storage, OpaqueMetadata.EMPTY, false);
var fbs = buildFbs(ProtoScalarValue.ofBytesValue(littleEndian(BigInteger.valueOf(150), 4)).encode(), null);

// When
ArrayStats result = ArrayStatsParser.parse(fbs, extension);

// Then
assertThat(result.min()).isEqualTo(new BigDecimal("1.50"));
}

@Test
void bound_withoutADtype_staysTheRawText() {
// Given
var fbs = buildFbs(ProtoScalarValue.ofBytesValue(new byte[]{'1'}).encode(), null);

// When
ArrayStats result = ArrayStatsParser.parse(fbs);

// Then
assertThat(result.min()).isEqualTo("1");
}

private static byte[] littleEndian(BigInteger value, int width) {
// The shifts are arithmetic, so the limbs sign-extend and the truncation is two's complement.
ByteBuffer buffer = ByteBuffer.allocate(32).order(ByteOrder.LITTLE_ENDIAN);
for (int limb = 0; limb < 4; limb++) {
buffer.putLong(value.shiftRight(64 * limb).longValue());
}
return Arrays.copyOf(buffer.array(), width);
}
}

private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes) {
return buildFbs(minBytes, maxBytes, null);
}
Expand Down
Loading
Loading