diff --git a/AGENTS.md b/AGENTS.md index 86295eacd..6a4625c98 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ fsst — io.github.dfa1.vortex.fsst: standalone FSST (Fast Static Symbol Tabl core — everything lives under `io.github.dfa1.vortex.core.*`: core.model DType, DTypes, PType, TimeUnit, EncodingId, LayoutId, ColumnName, ExtensionId, TimeDtype, TimestampDtype, ExtensionMetadata, OpaqueMetadata, TimeUnitMetadata, TimestampMetadata, - EditionId, Edition, EditionFamily, Editions + EditionId, Edition, EditionFamily, Editions, ArrayStats core.io IoBounds, PTypeIO, VortexFormat, MemorySize core.error VortexException core.jfr JFR events (FileOpened, HttpRange, SegmentDecoded, SegmentEncoded, ChunkFilter, Compress, Candidate, Winner, …) @@ -46,7 +46,7 @@ core — everything lives under `io.github.dfa1.vortex.core.*`: VectorApiSimdOperations (opt-in via --add-modules jdk.incubator.vector; Rust-parity results, see docs/compatibility.md), SimdOperationsSupport#preferred() (picks one) core.fbs / core.proto — generated wire codecs + their runtimes -reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStats, Zone, +reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStatsParser, Zone, ScanOptions, ScanFilter; file internals (Footer, Trailer, PostscriptParser, …) reader.array — Array + all subtypes (decode outputs) reader.decode — EncodingDecoder, DecodeContext, ArrayNode + *EncodingDecoder impls diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bc001dad..acd8d3abe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `EncodeContext` has a new last component, `trace`, so code calling its canonical constructor passes `EncodeTrace.NONE`. ([#601](https://github.com/dfa1/vortex-java/issues/601)) +- **Breaking:** `ArrayStats` moves from `io.github.dfa1.vortex.reader` to `io.github.dfa1.vortex.core.model`, and `ArrayStats.fromFbs` becomes `ArrayStatsParser.parse`. ([#664](https://github.com/dfa1/vortex-java/issues/664)) ### Removed - `CompressorBuilder#seed(long)` of the `vortex-fsst` module: training takes Rust's fixed sample, not a seeded one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregates.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregates.java index 01165a364..78ccd2b12 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregates.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexAggregates.java @@ -1,9 +1,9 @@ package io.github.dfa1.vortex.calcite; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.ScanIterator; import io.github.dfa1.vortex.reader.ScanOptions; diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java index 36e692700..7dc73c6b6 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java @@ -2,11 +2,11 @@ import io.github.dfa1.vortex.core.compute.LongIntMap; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.ScanFilter; import io.github.dfa1.vortex.reader.ScanIterator; diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/StatsCommand.java b/cli/src/main/java/io/github/dfa1/vortex/cli/StatsCommand.java index a0adbecee..6d905bd22 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/StatsCommand.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/StatsCommand.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.cli; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.VortexReader; diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/ArrayStats.java b/core/src/main/java/io/github/dfa1/vortex/core/model/ArrayStats.java new file mode 100644 index 000000000..32632478c --- /dev/null +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/ArrayStats.java @@ -0,0 +1,45 @@ +package io.github.dfa1.vortex.core.model; + +/// Per-array statistics embedded in the encoding tree: one component per field of the wire +/// `ArrayStats` table (Rust's `Stat` enum). A bool column's true count is its `sum`, as in Rust. +/// +/// `min`/`max` may be inexact (Rust's `Precision::Inexact`): still a valid bound, so pruning on a +/// range may use it, but not the true extreme (Rust's flat writer truncates a string bound longer +/// than 64 bytes). Anything that reports the value, or needs `min == max`, requires +/// `minExact`/`maxExact`. +/// +/// The counts are `u64` on the wire, read as a signed `Long`. +/// +/// @param min minimum value in the array, or `null` if unknown +/// @param minExact `true` if `min` is the exact minimum, not only a lower bound +/// @param max maximum value in the array, or `null` if unknown +/// @param maxExact `true` if `max` is the exact maximum, not only an upper bound +/// @param sum sum of values (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown +/// @param isSorted `true` if the values are sorted ascending with nulls first (Rust orders null lowest), or `null` if unknown +/// @param isStrictSorted `true` if also no two neighbors are equal, two nulls included, or `null` if unknown +/// @param isConstant `true` if every value is null, or none is and all are equal, or `null` if unknown +/// @param nullCount number of null values, or `null` if unknown +/// @param uncompressedSizeInBytes the array's uncompressed size in bytes, or `null` if unknown +/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown +public record ArrayStats( + Object min, + boolean minExact, + Object max, + boolean maxExact, + Object sum, + Boolean isSorted, + Boolean isStrictSorted, + Boolean isConstant, + Long nullCount, + Long uncompressedSizeInBytes, + Long nanCount) { + private static final ArrayStats EMPTY = + new ArrayStats(null, false, null, false, null, null, null, null, null, null, null); + + /// Returns an empty stats instance with all fields set to `null`. + /// + /// @return an empty [ArrayStats] with no known statistics + public static ArrayStats empty() { + return EMPTY; + } +} diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/ArrayStatsTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/ArrayStatsTest.java new file mode 100644 index 000000000..818542a30 --- /dev/null +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/ArrayStatsTest.java @@ -0,0 +1,31 @@ +package io.github.dfa1.vortex.core.model; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ArrayStatsTest { + + @Test + void empty_returnsCanonicalInstance() { + // Given / When / Then — empty() is a singleton; callers may identity-compare. + assertThat(ArrayStats.empty()).isSameAs(ArrayStats.empty()); + } + + @Test + void empty_hasAllNullComponents() { + // Given / When + ArrayStats sut = ArrayStats.empty(); + + // Then + assertThat(sut.min()).isNull(); + assertThat(sut.max()).isNull(); + assertThat(sut.sum()).isNull(); + assertThat(sut.isSorted()).isNull(); + assertThat(sut.isStrictSorted()).isNull(); + assertThat(sut.isConstant()).isNull(); + assertThat(sut.nullCount()).isNull(); + assertThat(sut.uncompressedSizeInBytes()).isNull(); + assertThat(sut.nanCount()).isNull(); + } +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java index ee1465829..3cf4e31f2 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java @@ -1,8 +1,8 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.ScanIterator; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexHandle; diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java index 4e868dc5d..c548fed44 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.reader.ArrayStats; import java.util.ArrayList; import java.util.List; diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java index 831dd30ad..049cc576d 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java @@ -1,10 +1,10 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.reader.extension.DateExtensionDecoder; import io.github.dfa1.vortex.reader.extension.ExtensionStorage; diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index bb62f82fd..f22604ba5 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -2,11 +2,12 @@ import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStatsParser; import io.github.dfa1.vortex.reader.Footer; import io.github.dfa1.vortex.reader.layout.Layout; import io.github.dfa1.vortex.reader.ScanIterator; @@ -408,7 +409,7 @@ private static Peek peekFlatRoot(MemorySegment seg, List arraySpecs) } Set nested = new LinkedHashSet<>(); collectEncodings(root, arraySpecs, nested, new int[]{0}); - return new Peek(resolveEncodingId(arraySpecs, root.encoding()), ArrayStats.fromFbs(root.stats()), + return new Peek(resolveEncodingId(arraySpecs, root.encoding()), ArrayStatsParser.parse(root.stats()), Set.copyOf(nested)); } diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java index fc9e9bf42..50a887441 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.inspect; import io.github.dfa1.vortex.core.compute.Utf8Order; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.layout.Layout; import io.github.dfa1.vortex.reader.SegmentSpec; diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java index ef8f87f2e..d9fd554df 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.DTypes; -import io.github.dfa1.vortex.reader.ArrayStats; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java index f6c8d3b17..fcf195c36 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java @@ -1,8 +1,8 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Zone; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java index 8f1e8f374..22e099df2 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java @@ -1,12 +1,12 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.TimestampDtype; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.CompressionScheme; import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.reader.layout.Layout; diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java index e2679989d..dab360499 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/InspectorTreeTest.java @@ -210,7 +210,7 @@ void buildShallow_skipsAllSlicesAndStillNamesColumns() { assertThat(sut.root().children().get(1).fieldName()).contains("value"); assertThat(sut.usedEncodings()).isEmpty(); assertThat(sut.root().children().get(0).usedEncodings()).isEmpty(); - assertThat(sut.root().children().getFirst().stats()).isEqualTo(io.github.dfa1.vortex.reader.ArrayStats.empty()); + assertThat(sut.root().children().getFirst().stats()).isEqualTo(io.github.dfa1.vortex.core.model.ArrayStats.empty()); // rawSegment is reserved for lazy peek; shallow build must never call it org.mockito.Mockito.verify(handle, org.mockito.Mockito.never()).rawSegment( org.mockito.Mockito.any()); @@ -223,7 +223,7 @@ void peek_nonFlatNode_returnsEmptyWithoutSlicing() { // out without slicing, so navigating to them doesn't hit the network. Layout structLayout = struct(0, List.of()); InspectorTree.Node node = new InspectorTree.Node(structLayout, java.util.Optional.empty(), - Set.of(), io.github.dfa1.vortex.reader.ArrayStats.empty(), List.of()); + Set.of(), io.github.dfa1.vortex.core.model.ArrayStats.empty(), List.of()); // When InspectorTree.Peek result = InspectorTree.peek(node, handle); @@ -240,7 +240,7 @@ void peek_compressedFlatSegment_returnsEmptyWithoutSlicing() { // their FlatBuffer can be parsed; peek skips them rather than slicing garbage. Layout flat = new Layout(LayoutId.parse("vortex.flat"), 10, null, List.of(), List.of(0)); InspectorTree.Node node = new InspectorTree.Node(flat, java.util.Optional.empty(), - Set.of(), io.github.dfa1.vortex.reader.ArrayStats.empty(), List.of()); + Set.of(), io.github.dfa1.vortex.core.model.ArrayStats.empty(), List.of()); given(handle.footer()).willReturn(new io.github.dfa1.vortex.reader.Footer( List.of(EncodingId.parse("vortex.flat")), List.of(), List.of(new SegmentSpec(0, 100, (byte) 0, CompressionScheme.ZSTD)), diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java index 4f2e19e28..a842d7157 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.inspect; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.CompressionScheme; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java index fdafe31b1..9e2b733c8 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/DecimalBytePartsInteropIntegrationTest.java @@ -12,7 +12,7 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.reader.ScanFilter; import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.VortexReader; diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index a98ee7564..6b350cdcb 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -968,7 +968,7 @@ void javaWriter_jniReader_filteredUtf8_prunesOnTruncatedBounds(@TempDir Path tmp // When List result = readColumnFiltered(file, "k", Expression.binary(Expression.BinaryOp.EQ, Expression.column("k"), Expression.literal(target))); - io.github.dfa1.vortex.reader.ArrayStats stats; + io.github.dfa1.vortex.core.model.ArrayStats stats; try (var reader = io.github.dfa1.vortex.reader.VortexReader.open(file)) { stats = reader.columnStats().get(ColumnName.of("k")); } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java index 5072db57d..b823d0843 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java @@ -18,7 +18,7 @@ import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.VarBinArray; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.VortexReader; import org.apache.arrow.c.ArrowArray; diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java index f5f270e1d..c511c8e2c 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.writer.WriteOptions; import io.github.dfa1.vortex.inspect.ColumnDistribution; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java similarity index 50% rename from reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java rename to reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java index fb458a0ab..154902eb8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStatsParser.java @@ -1,63 +1,30 @@ package io.github.dfa1.vortex.reader; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.fbs.FbsArrayStats; +import io.github.dfa1.vortex.core.fbs.FbsPrecision; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.nio.charset.StandardCharsets; -/// Per-array statistics embedded in the encoding tree: one component per field of the wire -/// `ArrayStats` table (Rust's `Stat` enum). A bool column's true count is its `sum`, as in Rust. -/// -/// `min`/`max` may be inexact (Rust's `Precision::Inexact`): still a valid bound, so pruning on a -/// range may use it, but not the true extreme (Rust's flat writer truncates a string bound longer -/// than 64 bytes). Anything that reports the value, or needs `min == max`, requires -/// `minExact`/`maxExact`. -/// -/// The counts are `u64` on the wire, read as a signed `Long`. -/// -/// @param min minimum value in the array, or `null` if unknown -/// @param minExact `true` if `min` is the exact minimum, not only a lower bound -/// @param max maximum value in the array, or `null` if unknown -/// @param maxExact `true` if `max` is the exact maximum, not only an upper bound -/// @param sum sum of values (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown -/// @param isSorted `true` if the values are sorted ascending with nulls first (Rust orders null lowest), or `null` if unknown -/// @param isStrictSorted `true` if also no two neighbors are equal, two nulls included, or `null` if unknown -/// @param isConstant `true` if every value is null, or none is and all are equal, or `null` if unknown -/// @param nullCount number of null values, or `null` if unknown -/// @param uncompressedSizeInBytes the array's uncompressed size in bytes, or `null` if unknown -/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown -public record ArrayStats( - Object min, - boolean minExact, - Object max, - boolean maxExact, - Object sum, - Boolean isSorted, - Boolean isStrictSorted, - Boolean isConstant, - Long nullCount, - Long uncompressedSizeInBytes, - Long nanCount) { - private static final ArrayStats EMPTY = - new ArrayStats(null, false, null, false, null, null, null, null, null, null, null); +/// Parses the wire `ArrayStats` FlatBuffer table into an [ArrayStats]: Rust's `from_flatbuffer`. +public final class ArrayStatsParser { - /// Returns an empty stats instance with all fields set to `null`. - /// - /// @return an empty [ArrayStats] with no known statistics - public static ArrayStats empty() { - return EMPTY; + private ArrayStatsParser() { } - /// Parses stats from a FlatBuffers [io.github.dfa1.vortex.core.fbs.FbsArrayStats] table. - /// Returns an empty instance when `fbs` is `null` or carries no statistic. + /// Parses stats from a FlatBuffers [FbsArrayStats] table. + /// Returns [ArrayStats#empty()] when `fbs` is `null` or carries no statistic. /// /// @param fbs the FlatBuffers stats table, or `null` /// @return parsed stats, or an empty instance if no usable data is present - public static ArrayStats fromFbs(io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs) { + /// @throws VortexException if a precision field or a scalar value is malformed + public static ArrayStats parse(FbsArrayStats fbs) { if (fbs == null) { - return EMPTY; + return ArrayStats.empty(); } Object min = decodeScalar(fbs.minAsSegment()); Object max = decodeScalar(fbs.maxAsSegment()); @@ -73,15 +40,15 @@ public static ArrayStats fromFbs(io.github.dfa1.vortex.core.fbs.FbsArrayStats fb fbs.hasNullCount() ? fbs.nullCount() : null, fbs.hasUncompressedSizeInBytes() ? fbs.uncompressedSizeInBytes() : null, fbs.hasNanCount() ? fbs.nanCount() : null); - return stats.equals(EMPTY) ? EMPTY : stats; + return stats.equals(ArrayStats.empty()) ? ArrayStats.empty() : stats; } /// Rust's `from_flatbuffer`: an absent precision is the FlatBuffer default `Inexact`, and any /// value other than `Inexact`/`Exact` is a corrupted field. Read only when its bound is present. private static boolean isExact(int precision, String field) { return switch (precision) { - case io.github.dfa1.vortex.core.fbs.FbsPrecision.Inexact -> false; - case io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact -> true; + case FbsPrecision.Inexact -> false; + case FbsPrecision.Exact -> true; default -> throw new VortexException("corrupted " + field + " in array stats: " + precision); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 9d47af885..504bde90b 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -2,6 +2,7 @@ import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import io.github.dfa1.vortex.core.jfr.ChunkFilterEvent; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; @@ -1510,7 +1511,7 @@ private ArrayStats readFlatStats(Layout flat) { if (root == null) { return ArrayStats.empty(); } - return ArrayStats.fromFbs(root.stats()); + return ArrayStatsParser.parse(root.stats()); } catch (VortexException _) { // The FlatBuffer verifier rejected the stats segment: malformed stats read as none. return ArrayStats.empty(); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java index 6d01b0a95..81673759b 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.compute.Compare; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/Zone.java b/reader/src/main/java/io/github/dfa1/vortex/reader/Zone.java index b7c520ab9..4fc068baa 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/Zone.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/Zone.java @@ -1,5 +1,7 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.model.ArrayStats; + /// One row of a column's zone-map table, together with the rows of the column it describes. /// /// @param firstRow first row of the column this zone covers diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/ZoneReducer.java b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/ZoneReducer.java index 7230c06ad..72457d66c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/compute/ZoneReducer.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/compute/ZoneReducer.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.compute; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.ScanIterator; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java similarity index 85% rename from reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java rename to reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java index 486a77640..0380db247 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsParserTest.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.fbs.FbsBuilder; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -10,42 +11,15 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -class ArrayStatsTest { +class ArrayStatsParserTest { @Nested - class Empty { - - @Test - void returnsCanonicalInstance() { - // Given / When / Then — empty() is a singleton; callers may identity-compare. - assertThat(ArrayStats.empty()).isSameAs(ArrayStats.empty()); - } - - @Test - void hasAllNullComponents() { - // Given / When - ArrayStats sut = ArrayStats.empty(); - - // Then - assertThat(sut.min()).isNull(); - assertThat(sut.max()).isNull(); - assertThat(sut.sum()).isNull(); - assertThat(sut.isSorted()).isNull(); - assertThat(sut.isStrictSorted()).isNull(); - assertThat(sut.isConstant()).isNull(); - assertThat(sut.nullCount()).isNull(); - assertThat(sut.uncompressedSizeInBytes()).isNull(); - assertThat(sut.nanCount()).isNull(); - } - } - - @Nested - class FromFbs { + class Parse { @Test void nullFbs_returnsEmpty() { // Given / When - ArrayStats sut = ArrayStats.fromFbs(null); + ArrayStats sut = ArrayStatsParser.parse(null); // Then — same instance as empty(), not just an equal one assertThat(sut).isSameAs(ArrayStats.empty()); @@ -57,7 +31,7 @@ void fbsWithNoMinOrMax_returnsEmpty() { io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs(null, null); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut).isSameAs(ArrayStats.empty()); @@ -71,7 +45,7 @@ void int64Scalar_decodes() { ProtoScalarValue.ofInt64Value(42L).encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo(-7L); @@ -86,7 +60,7 @@ void uint64Scalar_decodes() { ProtoScalarValue.ofUint64Value(Long.MAX_VALUE).encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo(0L); @@ -101,7 +75,7 @@ void f32Scalar_decodes() { ProtoScalarValue.ofF32Value(3.25f).encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo(1.5f); @@ -116,7 +90,7 @@ void f64Scalar_decodes() { ProtoScalarValue.ofF64Value(99.875).encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo(-0.5); @@ -131,7 +105,7 @@ void boolScalar_decodes() { ProtoScalarValue.ofBoolValue(true).encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo(false); @@ -146,7 +120,7 @@ void stringScalar_decodes() { ProtoScalarValue.ofStringValue("omega").encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo("alpha"); @@ -162,7 +136,7 @@ void bytesScalar_decodesAsUtf8String() { ProtoScalarValue.ofBytesValue("zz".getBytes()).encode()); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo("aa"); @@ -176,7 +150,7 @@ void minOnly_setsMaxToNull() { ProtoScalarValue.ofInt64Value(1L).encode(), null); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut.min()).isEqualTo(1L); @@ -189,7 +163,7 @@ void emptyByteVector_treatedAsAbsent() { io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs(new byte[0], new byte[0]); // When - ArrayStats sut = ArrayStats.fromFbs(fbs); + ArrayStats sut = ArrayStatsParser.parse(fbs); // Then assertThat(sut).isSameAs(ArrayStats.empty()); @@ -217,7 +191,7 @@ void everyNonScalarStat_decodes() { var fbs = io.github.dfa1.vortex.core.fbs.FbsArrayStats.getRootAsFbsArrayStats(builder.dataSegment()); // When - ArrayStats result = ArrayStats.fromFbs(fbs); + ArrayStats result = ArrayStatsParser.parse(fbs); // Then assertThat(result).isEqualTo( @@ -233,7 +207,7 @@ void exactPrecision_marksBoundsExact() { (int) io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); // When - ArrayStats result = ArrayStats.fromFbs(fbs); + ArrayStats result = ArrayStatsParser.parse(fbs); // Then assertThat(result.minExact()).isTrue(); @@ -249,7 +223,7 @@ void absentPrecision_readsInexact() { ProtoScalarValue.ofStringValue("omega").encode()); // When - ArrayStats result = ArrayStats.fromFbs(fbs); + ArrayStats result = ArrayStatsParser.parse(fbs); // Then assertThat(result.min()).isEqualTo("alpha"); @@ -267,7 +241,7 @@ void corruptedPrecision_throwsVortexException() { 2); // When / Then - assertThatThrownBy(() -> ArrayStats.fromFbs(fbs)) + assertThatThrownBy(() -> ArrayStatsParser.parse(fbs)) .isInstanceOf(VortexException.class) .hasMessageContaining("corrupted min_precision"); } @@ -278,7 +252,7 @@ void malformedScalarBytes_throwsVortexException() { io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs(new byte[]{(byte) 0x80}, null); // When / Then - assertThatThrownBy(() -> ArrayStats.fromFbs(fbs)) + assertThatThrownBy(() -> ArrayStatsParser.parse(fbs)) .isInstanceOf(VortexException.class) .hasMessageContaining("invalid scalar value"); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java index 97010405d..53a85931d 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorDecimalStatsTest.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.compute.Predicate; import org.junit.jupiter.api.Test; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java index 1b6bbcc60..45dc36316 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorMalformedStatsTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader; import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorZoneCountAdversarialTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorZoneCountAdversarialTest.java index 24d02ac68..dc462d610 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorZoneCountAdversarialTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ScanIteratorZoneCountAdversarialTest.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ColumnZoneStatsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ColumnZoneStatsTest.java index 9f5881c75..398848907 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ColumnZoneStatsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ColumnZoneStatsTest.java @@ -1,9 +1,9 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.VortexReader; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java index 12f3d2eb9..6d4be28a9 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/NullableDecimalWriteTest.java @@ -1,9 +1,9 @@ package io.github.dfa1.vortex.writer; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.ScanIterator; import io.github.dfa1.vortex.reader.ScanOptions; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java index 28ce63b68..d86e18eba 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java @@ -13,7 +13,7 @@ import io.github.dfa1.vortex.core.model.Editions; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.layout.Layout; -import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.reader.ScanIterator; import io.github.dfa1.vortex.reader.ScanOptions; import io.github.dfa1.vortex.reader.SegmentSpec; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java index 4488cff64..9873f08d6 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneMapPruningTest.java @@ -1,10 +1,10 @@ package io.github.dfa1.vortex.writer; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ArrayStats; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.DTypes; -import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.ScanFilter;