Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ fsst — io.github.dfa1.vortex.fsst: standalone FSST (Fast Static Symbol Tabl
core — everything lives under `io.github.dfa1.vortex.core.*`:
core.model DType, DTypes, PType, TimeUnit, EncodingId, LayoutId, ColumnName, ExtensionId, TimeDtype, TimestampDtype,
ExtensionMetadata, OpaqueMetadata, TimeUnitMetadata, TimestampMetadata,
EditionId, Edition, EditionFamily, Editions
EditionId, Edition, EditionFamily, Editions, ArrayStats
core.io IoBounds, PTypeIO, VortexFormat, MemorySize
core.error VortexException
core.jfr JFR events (FileOpened, HttpRange, SegmentDecoded, SegmentEncoded, ChunkFilter, Compress, Candidate, Winner, …)
Expand All @@ -46,7 +46,7 @@ core — everything lives under `io.github.dfa1.vortex.core.*`:
VectorApiSimdOperations (opt-in via --add-modules jdk.incubator.vector; Rust-parity results,
see docs/compatibility.md), SimdOperationsSupport#preferred() (picks one)
core.fbs / core.proto — generated wire codecs + their runtimes
reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStats, Zone,
reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStatsParser, Zone,
ScanOptions, ScanFilter; file internals (Footer, Trailer, PostscriptParser, …)
reader.array — Array + all subtypes (decode outputs)
reader.decode — EncodingDecoder, DecodeContext, ArrayNode + *EncodingDecoder impls
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Changed
- `EncodeContext` has a new last component, `trace`, so code calling its canonical constructor passes `EncodeTrace.NONE`. ([#601](https://github.com/dfa1/vortex-java/issues/601))
- **Breaking:** `ArrayStats` moves from `io.github.dfa1.vortex.reader` to `io.github.dfa1.vortex.core.model`, and `ArrayStats.fromFbs` becomes `ArrayStatsParser.parse`. ([#664](https://github.com/dfa1/vortex-java/issues/664))

### Removed
- `CompressorBuilder#seed(long)` of the `vortex-fsst` module: training takes Rust's fixed sample, not a seeded one. ([#583](https://github.com/dfa1/vortex-java/issues/583))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package io.github.dfa1.vortex.calcite;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.Chunk;
import io.github.dfa1.vortex.reader.ScanIterator;
import io.github.dfa1.vortex.reader.ScanOptions;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

import io.github.dfa1.vortex.core.compute.LongIntMap;
import io.github.dfa1.vortex.core.error.VortexException;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.ExtensionId;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.Chunk;
import io.github.dfa1.vortex.reader.ScanFilter;
import io.github.dfa1.vortex.reader.ScanIterator;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.github.dfa1.vortex.cli;

import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.VortexReader;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package io.github.dfa1.vortex.core.model;

/// Per-array statistics embedded in the encoding tree: one component per field of the wire
/// `ArrayStats` table (Rust's `Stat` enum). A bool column's true count is its `sum`, as in Rust.
///
/// `min`/`max` may be inexact (Rust's `Precision::Inexact`): still a valid bound, so pruning on a
/// range may use it, but not the true extreme (Rust's flat writer truncates a string bound longer
/// than 64 bytes). Anything that reports the value, or needs `min == max`, requires
/// `minExact`/`maxExact`.
///
/// The counts are `u64` on the wire, read as a signed `Long`.
///
/// @param min minimum value in the array, or `null` if unknown
/// @param minExact `true` if `min` is the exact minimum, not only a lower bound
/// @param max maximum value in the array, or `null` if unknown
/// @param maxExact `true` if `max` is the exact maximum, not only an upper bound
/// @param sum sum of values (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown
/// @param isSorted `true` if the values are sorted ascending with nulls first (Rust orders null lowest), or `null` if unknown
/// @param isStrictSorted `true` if also no two neighbors are equal, two nulls included, or `null` if unknown
/// @param isConstant `true` if every value is null, or none is and all are equal, or `null` if unknown
/// @param nullCount number of null values, or `null` if unknown
/// @param uncompressedSizeInBytes the array's uncompressed size in bytes, or `null` if unknown
/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown
public record ArrayStats(
Object min,
boolean minExact,
Object max,
boolean maxExact,
Object sum,
Boolean isSorted,
Boolean isStrictSorted,
Boolean isConstant,
Long nullCount,
Long uncompressedSizeInBytes,
Long nanCount) {
private static final ArrayStats EMPTY =
new ArrayStats(null, false, null, false, null, null, null, null, null, null, null);

/// Returns an empty stats instance with all fields set to `null`.
///
/// @return an empty [ArrayStats] with no known statistics
public static ArrayStats empty() {
return EMPTY;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package io.github.dfa1.vortex.core.model;

import org.junit.jupiter.api.Test;

import static org.assertj.core.api.Assertions.assertThat;

class ArrayStatsTest {

@Test
void empty_returnsCanonicalInstance() {
// Given / When / Then — empty() is a singleton; callers may identity-compare.
assertThat(ArrayStats.empty()).isSameAs(ArrayStats.empty());
}

@Test
void empty_hasAllNullComponents() {
// Given / When
ArrayStats sut = ArrayStats.empty();

// Then
assertThat(sut.min()).isNull();
assertThat(sut.max()).isNull();
assertThat(sut.sum()).isNull();
assertThat(sut.isSorted()).isNull();
assertThat(sut.isStrictSorted()).isNull();
assertThat(sut.isConstant()).isNull();
assertThat(sut.nullCount()).isNull();
assertThat(sut.uncompressedSizeInBytes()).isNull();
assertThat(sut.nanCount()).isNull();
}
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.ScanIterator;
import io.github.dfa1.vortex.reader.ScanOptions;
import io.github.dfa1.vortex.reader.VortexHandle;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.ArrayStats;

import java.util.ArrayList;
import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.ExtensionId;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.SegmentSpec;
import io.github.dfa1.vortex.reader.extension.DateExtensionDecoder;
import io.github.dfa1.vortex.reader.extension.ExtensionStorage;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@

import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT;

import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.error.VortexException;
import io.github.dfa1.vortex.core.model.EncodingId;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.ArrayStatsParser;
import io.github.dfa1.vortex.reader.Footer;
import io.github.dfa1.vortex.reader.layout.Layout;
import io.github.dfa1.vortex.reader.ScanIterator;
Expand Down Expand Up @@ -408,7 +409,7 @@ private static Peek peekFlatRoot(MemorySegment seg, List<EncodingId> arraySpecs)
}
Set<String> nested = new LinkedHashSet<>();
collectEncodings(root, arraySpecs, nested, new int[]{0});
return new Peek(resolveEncodingId(arraySpecs, root.encoding()), ArrayStats.fromFbs(root.stats()),
return new Peek(resolveEncodingId(arraySpecs, root.encoding()), ArrayStatsParser.parse(root.stats()),
Set.copyOf(nested));
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.compute.Utf8Order;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.layout.Layout;
import io.github.dfa1.vortex.reader.SegmentSpec;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.reader.ArrayStats;
import org.junit.jupiter.api.Test;

import java.util.List;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.Zone;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.core.model.LayoutId;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.core.model.TimestampDtype;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.CompressionScheme;
import io.github.dfa1.vortex.reader.SegmentSpec;
import io.github.dfa1.vortex.reader.layout.Layout;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ void buildShallow_skipsAllSlicesAndStillNamesColumns() {
assertThat(sut.root().children().get(1).fieldName()).contains("value");
assertThat(sut.usedEncodings()).isEmpty();
assertThat(sut.root().children().get(0).usedEncodings()).isEmpty();
assertThat(sut.root().children().getFirst().stats()).isEqualTo(io.github.dfa1.vortex.reader.ArrayStats.empty());
assertThat(sut.root().children().getFirst().stats()).isEqualTo(io.github.dfa1.vortex.core.model.ArrayStats.empty());
// rawSegment is reserved for lazy peek; shallow build must never call it
org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).rawSegment(
org.mockito.Mockito.any());
Expand All @@ -223,7 +223,7 @@ void peek_nonFlatNode_returnsEmptyWithoutSlicing() {
// out without slicing, so navigating to them doesn't hit the network.
Layout structLayout = struct(0, List.of());
InspectorTree.Node node = new InspectorTree.Node(structLayout, java.util.Optional.empty(),
Set.of(), io.github.dfa1.vortex.reader.ArrayStats.empty(), List.of());
Set.of(), io.github.dfa1.vortex.core.model.ArrayStats.empty(), List.of());

// When
InspectorTree.Peek result = InspectorTree.peek(node, handle);
Expand All @@ -240,7 +240,7 @@ void peek_compressedFlatSegment_returnsEmptyWithoutSlicing() {
// their FlatBuffer can be parsed; peek skips them rather than slicing garbage.
Layout flat = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0));
InspectorTree.Node node = new InspectorTree.Node(flat, java.util.Optional.empty(),
Set.of(), io.github.dfa1.vortex.reader.ArrayStats.empty(), List.of());
Set.of(), io.github.dfa1.vortex.core.model.ArrayStats.empty(), List.of());
given(handle.footer()).willReturn(new io.github.dfa1.vortex.reader.Footer(
List.of(EncodingId.parse("vortex.flat")), List.of(),
List.of(new SegmentSpec(0, 100, (byte) 0, CompressionScheme.ZSTD)),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package io.github.dfa1.vortex.inspect;

import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.reader.CompressionScheme;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import dev.vortex.jni.NativeLoader;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.reader.ScanFilter;
import io.github.dfa1.vortex.reader.array.DecimalArray;
import io.github.dfa1.vortex.reader.VortexReader;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -968,7 +968,7 @@ void javaWriter_jniReader_filteredUtf8_prunesOnTruncatedBounds(@TempDir Path tmp
// When
List<Object> result = readColumnFiltered(file, "k", Expression.binary(Expression.BinaryOp.EQ,
Expression.column("k"), Expression.literal(target)));
io.github.dfa1.vortex.reader.ArrayStats stats;
io.github.dfa1.vortex.core.model.ArrayStats stats;
try (var reader = io.github.dfa1.vortex.reader.VortexReader.open(file)) {
stats = reader.columnStats().get(ColumnName.of("k"));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import io.github.dfa1.vortex.reader.array.LongArray;
import io.github.dfa1.vortex.reader.array.MaskedArray;
import io.github.dfa1.vortex.reader.array.VarBinArray;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.reader.ReadRegistry;
import io.github.dfa1.vortex.reader.VortexReader;
import org.apache.arrow.c.ArrowArray;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.reader.ArrayStats;
import io.github.dfa1.vortex.core.model.ArrayStats;
import io.github.dfa1.vortex.reader.ReadRegistry;
import io.github.dfa1.vortex.writer.WriteOptions;
import io.github.dfa1.vortex.inspect.ColumnDistribution;
Expand Down
Loading
Loading