diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4209986..ddef5fb37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index 83a296306..2a21b99c8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -156,7 +156,8 @@ decimals ([#430](https://github.com/dfa1/vortex-java/pull/430)) and nulls in nul A flat segment's `ArrayNode` tree carries array stats, and Rust's flat layout does not compute them: it keeps the ones the compressor had cached on the array, and each encoding passes on only some -(#583). The writer mirrors that for the **root node**: +(#583). The writer mirrors that for every node of the tree the compressor encodes, the root and the children +it cascades into alike: | Column | Root encoding | Stats written | |---|---|---| @@ -168,9 +169,16 @@ the ones the compressor had cached on the array, and each encoding passes on onl So a Java-written string column has no bound in its flat segments, as a Rust-written one has none: `VortexReader.columnStats()` reports no string `min`/`max` for either, and a string bound lives only in the zoned layout's stats table. The decimal bounds the reader can read come from the same two places, and the -zone table's are not decoded yet. Child nodes carry no stats yet, and `vortex.sequence` does not write its sorted -flags yet ([#583](https://github.com/dfa1/vortex-java/issues/583)); `NodeStatsInteropIntegrationTest` compares -every flat root against vortex-jni. +zone table's are not decoded yet. + +The nodes below the root follow the same table, plus four shapes of their own: a `vortex.sequence` carries +`is_sorted` and `is_strict_sorted` (false for a falling step), the primitive values of ALP and ALP-RD patches +carry `is_constant = false`, and the `vortex.bool` validity bitmap of a nullable primitive carries `is_constant` +when every row is valid and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. +`NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against +vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol +table) and the zone-table segments of string and bool columns are not covered yet +([#583](https://github.com/dfa1/vortex-java/issues/583)). ### Decode shape diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java index 4a26d8c2d..41e5a194d 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -40,8 +40,8 @@ /// data (#583). Rust's flat layout keeps only the stats the compressor had cached on the array, and each /// encoding passes on some of them: integers keep min, max, `is_constant`, `null_count` and the /// uncompressed size under `vortex.primitive` and `fastlanes.bitpacked`; floats and strings keep none -/// under ALP, FSST or onpair. `vortex.sequence` is compared by encoding only: its sorted flags are not -/// written yet. +/// under ALP, FSST or onpair. A sequence carries its sorted flags, and the validity bitmap of a +/// nullable integer its true count. class NodeStatsInteropIntegrationTest { private static final Session SESSION = Session.create(); @@ -67,12 +67,16 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { boolean[] bool = new boolean[N]; int[] nullable = new int[N]; boolean[] valid = new boolean[N]; + double[] patched = new double[N]; + long[] descending = new long[N]; for (int i = 0; i < N; i++) { sorted[i] = i; random[i] = r.nextLong(); small[i] = r.nextInt(100); rf[i] = r.nextDouble() * 1000; sf[i] = i * 0.5; + patched[i] = i % 100 == 0 ? r.nextDouble() * 1e9 : i * 0.01; + descending[i] = 100_000L - i * 3L; low[i] = "k" + r.nextInt(10); high[i] = "value-" + Long.toString(r.nextLong() & 0xffffffffL, 36); bool[i] = r.nextBoolean(); @@ -86,7 +90,9 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { Field.notNullable("sf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), Field.notNullable("low", ArrowType.Utf8.INSTANCE), Field.notNullable("high", ArrowType.Utf8.INSTANCE), Field.notNullable("bool", ArrowType.Bool.INSTANCE), - Field.nullable("nullable", new ArrowType.Int(32, true)))); + Field.nullable("nullable", new ArrowType.Int(32, true)), + Field.notNullable("patched", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("descending", new ArrowType.Int(64, true)))); Path jni = tmp.resolve("jni.vortex"); try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { @@ -99,6 +105,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { VarCharVector g = (VarCharVector) root.getVector("high"); BitVector h = (BitVector) root.getVector("bool"); IntVector nv = (IntVector) root.getVector("nullable"); + Float8Vector pv = (Float8Vector) root.getVector("patched"); + BigIntVector dv = (BigIntVector) root.getVector("descending"); root.allocateNew(); for (int i = 0; i < N; i++) { a.setSafe(i, sorted[i]); @@ -109,6 +117,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { f.setSafe(i, low[i].getBytes(StandardCharsets.UTF_8)); g.setSafe(i, high[i].getBytes(StandardCharsets.UTF_8)); h.setSafe(i, bool[i] ? 1 : 0); + pv.setSafe(i, patched[i]); + dv.setSafe(i, descending[i]); if (valid[i]) { nv.setSafe(i, nullable[i]); } else { @@ -123,31 +133,58 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { } Path java = tmp.resolve("java.vortex"); DType.Struct schema = new DType.Struct( - List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable").stream().map(ColumnName::of).toList(), - List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable()), false); + List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending").stream().map(ColumnName::of).toList(), + List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64), false); try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { - w.writeChunk(Map.of(ColumnName.of("sorted"), sorted, ColumnName.of("random"), random, - ColumnName.of("small"), small, ColumnName.of("rf"), rf, ColumnName.of("sf"), sf, - ColumnName.of("low"), low, ColumnName.of("high"), high, ColumnName.of("bool"), bool, - ColumnName.of("nullable"), new NullableData(nullable, valid))); + Map chunk = new java.util.LinkedHashMap<>(); + chunk.put(ColumnName.of("sorted"), sorted); + chunk.put(ColumnName.of("random"), random); + chunk.put(ColumnName.of("small"), small); + chunk.put(ColumnName.of("rf"), rf); + chunk.put(ColumnName.of("sf"), sf); + chunk.put(ColumnName.of("low"), low); + chunk.put(ColumnName.of("high"), high); + chunk.put(ColumnName.of("bool"), bool); + chunk.put(ColumnName.of("nullable"), new NullableData(nullable, valid)); + chunk.put(ColumnName.of("patched"), patched); + chunk.put(ColumnName.of("descending"), descending); + w.writeChunk(chunk); } - List expected = flatRoots(EncodingTrees.withStats(jni)); + List expected = EncodingTrees.withStats(jni); // When - List result = flatRoots(EncodingTrees.withStats(java)); + List result = EncodingTrees.withStats(java); - // Then - assertThat(result).isEqualTo(expected); + // Then — every flat segment's root, and the whole tree (children and zone table) of the columns + // whose encodings do not depend on how each writer trains its string symbol table + assertThat(flatRoots(result)).isEqualTo(flatRoots(expected)); + List> expectedColumns = columns(expected); + List> resultColumns = columns(result); + assertThat(resultColumns).hasSameSizeAs(expectedColumns); + for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10}) { + assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c)); + } + } + + /// The tree lines of each top-level column, in file order. + private static List> columns(List tree) { + List> columns = new java.util.ArrayList<>(); + for (String line : tree.subList(1, tree.size())) { + if (!line.startsWith(" ")) { + columns.add(new java.util.ArrayList<>()); + } + columns.getLast().add(line); + } + return columns; } - /// The root node line of every flat segment, in file order; a sequence's stats are blanked. + /// The root node line of every flat segment, in file order. private static List flatRoots(List tree) { List roots = new java.util.ArrayList<>(); for (int i = 0; i + 1 < tree.size(); i++) { if (tree.get(i).strip().equals("vortex.flat")) { - String root = tree.get(i + 1).strip(); - roots.add(root.startsWith("vortex.sequence") ? "vortex.sequence" : root); + roots.add(tree.get(i + 1).strip()); } } return roots; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index d9f861581..0cdfbc462 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -24,6 +24,7 @@ import io.github.dfa1.vortex.writer.encode.EncodedBuffer; import io.github.dfa1.vortex.writer.encode.MinMax; import io.github.dfa1.vortex.writer.encode.NodeStats; +import io.github.dfa1.vortex.writer.encode.RootNodeStats; import io.github.dfa1.vortex.writer.encode.NullableData; import io.github.dfa1.vortex.writer.encode.StructData; import io.github.dfa1.vortex.writer.encode.StructEncodingEncoder; @@ -959,9 +960,12 @@ private EncodedSegment encodeSegment(DType dtype, Object data, EncodingEncoder e } SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount); // the stats the flat segment's root node carries are Rust's, not the zone map's - EncodeNode root = result.rootNode().withStats(RootNodeStats.of(dtype, result.rootNode().encodingId(), - min, max, arrayLength(data), nullCount)); - result = new EncodeResult(root, result.encodedBuffers(), result.statsMin(), result.statsMax()); + NodeStats own = result.rootNode().stats(); + if (own == null || own.isEmpty()) { + EncodeNode root = result.rootNode().withStats(RootNodeStats.of(dtype, result.rootNode().encodingId(), + min, max, arrayLength(data), nullCount)); + result = new EncodeResult(root, result.encodedBuffers(), result.statsMin(), result.statsMax()); + } return new EncodedSegment(result, arena, stats); } catch (RuntimeException | Error e) { arena.close(); @@ -1141,6 +1145,8 @@ private static int buildStats(FbsBuilder fbb, NodeStats stats) { : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(fbb, stats.min().encode()); int maxVec = stats.max() == null ? 0 : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(fbb, stats.max().encode()); + int sumVec = stats.sum() == null ? 0 + : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createSumVector(fbb, stats.sum().encode()); // forceDefaults only while building ArrayStats, so a scalar equal to its default is still // serialized; reset immediately so the Array/ArrayNode tables keep their normal layout fbb.forceDefaults(true); @@ -1153,6 +1159,9 @@ private static int buildStats(FbsBuilder fbb, NodeStats stats) { io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(fbb, maxVec); io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); } + if (stats.sum() != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addSum(fbb, sumVec); + } if (stats.isSorted() != null) { io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsSorted(fbb, stats.isSorted()); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index 20bf31559..fe097294e 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -49,6 +49,7 @@ private static int dataLength(Object data) { return switch (data) { case StructData(var fieldArrays) -> fieldArrays.isEmpty() ? 0 : dataLength(fieldArrays.getFirst()); case NullableData nullable -> nullable.length(); + case boolean[] a -> a.length; case byte[] a -> a.length; case short[] a -> a.length; case int[] a -> a.length; @@ -257,6 +258,28 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { } private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx) { + EncodeResult result = encodeArray(dtype, data, ctx); + NodeStats own = result.rootNode().stats(); + if (ctx.sample() || own != null && !own.isEmpty()) { + return result; + } + return new EncodeResult(result.rootNode().withStats(nodeStats(dtype, data, result.rootNode().encodingId())), + result.encodedBuffers(), result.statsMin(), result.statsMax()); + } + + /// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root + /// as much as the root itself ([RootNodeStats]). + private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { + if (!(dtype instanceof DType.Primitive || dtype instanceof DType.Bool)) { + return NodeStats.NONE; + } + MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null; + long nullCount = data instanceof NullableData nullable ? nullable.nullCount() : 0L; + return RootNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(), + dataLength(data), nullCount); + } + + private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { if (keepsValidity(dtype) && data instanceof NullableData nullable) { return encodeNullablePrimitive(dtype, nullable, ctx); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java index 4ca3ddb1b..3db1af72d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/EncodeResult.java @@ -72,7 +72,8 @@ public EncodeResult withStats(MinMax stats) { EncodeResult withValidity(boolean[] validity, EncodeContext ctx) { EncodeResult bool = new BoolEncodingEncoder().encode(DTypes.BOOL, validity, ctx); EncodeNode[] children = Arrays.copyOf(rootNode.children(), rootNode.children().length + 1); - children[children.length - 1] = EncodeNode.remapBufferIndices(bool.rootNode(), encodedBuffers.size()); + children[children.length - 1] = EncodeNode.remapBufferIndices(bool.rootNode(), encodedBuffers.size()) + .withStats(RootNodeStats.ofValidity(validity)); List buffers = new ArrayList<>(encodedBuffers); buffers.addAll(bool.encodedBuffers()); EncodeNode root = new EncodeNode(rootNode.encodingId(), rootNode.metadata(), children, rootNode.bufferIndices()); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java index 8f1dcd6e9..7cb8b076d 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/NodeStats.java @@ -16,6 +16,7 @@ /// @param nullCount number of null rows, or `null` /// @param uncompressedSize size of the canonical array in bytes, or `null` /// @param nanCount number of NaN values, or `null` +/// @param sum exact sum of the values, or `null` public record NodeStats( ProtoScalarValue min, ProtoScalarValue max, @@ -24,17 +25,33 @@ public record NodeStats( Boolean isStrictSorted, Long nullCount, Long uncompressedSize, - Long nanCount + Long nanCount, + ProtoScalarValue sum ) { + /// Stats without a sum. + /// + /// @param min exact minimum of the array's values, or `null` + /// @param max exact maximum of the array's values, or `null` + /// @param isConstant whether every value is equal, or `null` + /// @param isSorted whether the values are sorted, or `null` + /// @param isStrictSorted whether the values are strictly sorted, or `null` + /// @param nullCount number of null rows, or `null` + /// @param uncompressedSize size of the canonical array in bytes, or `null` + /// @param nanCount number of NaN values, or `null` + public NodeStats(ProtoScalarValue min, ProtoScalarValue max, Boolean isConstant, Boolean isSorted, + Boolean isStrictSorted, Long nullCount, Long uncompressedSize, Long nanCount) { + this(min, max, isConstant, isSorted, isStrictSorted, nullCount, uncompressedSize, nanCount, null); + } + /// A node that carries no stats. - public static final NodeStats NONE = new NodeStats(null, null, null, null, null, null, null, null); + public static final NodeStats NONE = new NodeStats(null, null, null, null, null, null, null, null, null); /// Whether this node carries no stat at all. /// /// @return `true` if every component is `null` public boolean isEmpty() { return min == null && max == null && isConstant == null && isSorted == null && isStrictSorted == null - && nullCount == null && uncompressedSize == null && nanCount == null; + && nullCount == null && uncompressedSize == null && nanCount == null && sum == null; } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchBuffers.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchBuffers.java index 47727fcff..14e9e88e2 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchBuffers.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchBuffers.java @@ -73,7 +73,7 @@ EncodeNode[] append(Object values, PType ptype, List buffers, Enc } /// Appends the patch values to `buffers` as Rust's `compress_patches` leaves them: a - /// `vortex.constant` when every value is equal, a `vortex.primitive` otherwise. + /// `vortex.constant` when every value is equal, a `vortex.primitive` otherwise, marked not constant. /// /// @param values the patch values: a primitive array of `ptype`, or for an integer `ptype` its /// values widened to a `long[]` as `PrimitiveArrays.toLongs` widens them @@ -98,7 +98,9 @@ static EncodeNode valuesNode(Object values, PType ptype, List buf segment = PrimitiveArrays.toSegment(values, ptype, ctx.arena()); } buffers.add(EncodedBuffer.of(segment, ptype)); - return EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, first); + // the constant check that found them unequal leaves its answer on the array + return EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, first) + .withStats(new NodeStats(null, null, false, null, null, null, null, null)); } // Writes widened integers back at their own width, in one copy. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RootNodeStats.java similarity index 76% rename from writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java rename to writer/src/main/java/io/github/dfa1/vortex/writer/encode/RootNodeStats.java index 7301130bd..f1d75e9e1 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/RootNodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RootNodeStats.java @@ -1,9 +1,8 @@ -package io.github.dfa1.vortex.writer; +package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; -import io.github.dfa1.vortex.writer.encode.NodeStats; import java.util.Arrays; @@ -17,7 +16,7 @@ /// on: a canonical primitive or bool does, as does `fastlanes.bitpacked` /// (`bitpacking/array/bitpack_compress`); `vortex.alp`, `vortex.alprd`, `vortex.fsst`, /// `vortex.onpair`, `vortex.constant` and the rest do not, so their roots carry nothing. -final class RootNodeStats { +public final class RootNodeStats { private RootNodeStats() { } @@ -31,7 +30,7 @@ private RootNodeStats() { /// @param rows the number of rows /// @param nullCount the number of null rows /// @return the root's stats, [NodeStats#NONE] when the encoding passes none on - static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows, + public static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoScalarValue max, long rows, long nullCount) { return switch (dtype) { case DType.Primitive p when p.ptype().isFloating() && root.equals(EncodingId.VORTEX_PRIMITIVE) -> @@ -45,6 +44,26 @@ static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoSca }; } + /// The stats of the `vortex.bool` array that holds a nullable array's validity. Rust builds it + /// from the mask: an array of one value is constant, a mixed one carries both bounds, its + /// valid-row count as the sum and no nulls. + /// + /// @param validity per-row validity + /// @return the validity child's stats + public static NodeStats ofValidity(boolean[] validity) { + long valid = 0; + for (boolean v : validity) { + if (v) { + valid++; + } + } + if (valid == 0 || valid == validity.length) { + return new NodeStats(null, null, true, null, null, null, null, null); + } + return new NodeStats(ProtoScalarValue.ofBoolValue(false), ProtoScalarValue.ofBoolValue(true), null, null, null, + 0L, null, null, ProtoScalarValue.ofUint64Value(valid)); + } + private static boolean retainsIntegerStats(EncodingId root) { return root.equals(EncodingId.VORTEX_PRIMITIVE) || root.equals(EncodingId.FASTLANES_BITPACKED); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java index 572f3c504..ff2710448 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoder.java @@ -116,8 +116,10 @@ private static EncodeResult encodeInteger(PType pt, Object data) { } long last = previous; boolean baseIsMin = pt.isUnsigned() ? Long.compareUnsigned(base, last) <= 0 : base <= last; + // Rust's `Sequence::is_sorted`: a step is never zero here, so only a negative one breaks the order + boolean sorted = n <= 1 || pt.isUnsigned() || multiplier > 0; return buildResult(buildIntScalar(pt, base), buildIntScalar(pt, multiplier), - buildIntScalar(pt, baseIsMin ? base : last), buildIntScalar(pt, baseIsMin ? last : base)); + buildIntScalar(pt, baseIsMin ? base : last), buildIntScalar(pt, baseIsMin ? last : base), sorted); } /// `a - b` in `pt`, or not applicable when the result does not fit `pt`. @@ -176,10 +178,11 @@ private static VortexException notInType(PType pt) { return new VortexException(EncodingId.VORTEX_SEQUENCE, "sequence step overflows " + pt); } - private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, ProtoScalarValue statsMin, ProtoScalarValue statsMax) { + private static EncodeResult buildResult(ProtoScalarValue base, ProtoScalarValue mul, ProtoScalarValue statsMin, ProtoScalarValue statsMax, boolean sorted) { ProtoSequenceMetadata meta = new ProtoSequenceMetadata(base, mul); MemorySegment metaBuf = MemorySegment.ofArray(meta.encode()); - EncodeNode node = new EncodeNode(EncodingId.VORTEX_SEQUENCE, metaBuf, new EncodeNode[0], new int[]{}); + EncodeNode node = new EncodeNode(EncodingId.VORTEX_SEQUENCE, metaBuf, new EncodeNode[0], new int[]{}) + .withStats(new NodeStats(null, null, null, sorted, sorted, null, null, null)); return new EncodeResult(node, List.of(), statsMin, statsMax); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RootNodeStatsTest.java similarity index 79% rename from writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java rename to writer/src/test/java/io/github/dfa1/vortex/writer/encode/RootNodeStatsTest.java index 326fd5a12..cb32c36db 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/RootNodeStatsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RootNodeStatsTest.java @@ -1,9 +1,8 @@ -package io.github.dfa1.vortex.writer; +package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; -import io.github.dfa1.vortex.writer.encode.NodeStats; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -85,4 +84,25 @@ void stringRoots_carryNone() { assertThat(varbin).isEqualTo(NodeStats.NONE); assertThat(fsst).isEqualTo(NodeStats.NONE); } + + @Test + void validityBitmap_mixedCarriesBoundsAndTheValidCount() { + // Given / When + NodeStats result = RootNodeStats.ofValidity(new boolean[]{true, false, true, true}); + + // Then + assertThat(result).isEqualTo(new NodeStats(ProtoScalarValue.ofBoolValue(false), + ProtoScalarValue.ofBoolValue(true), null, null, null, 0L, null, null, ProtoScalarValue.ofUint64Value(3L))); + } + + @Test + void validityBitmap_ofOneValueIsConstant() { + // Given / When + NodeStats allValid = RootNodeStats.ofValidity(new boolean[]{true, true}); + NodeStats noneValid = RootNodeStats.ofValidity(new boolean[]{false, false}); + + // Then + assertThat(allValid).isEqualTo(new NodeStats(null, null, true, null, null, null, null, null)); + assertThat(noneValid).isEqualTo(allValid); + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index c2207e12b..8430521e5 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -73,6 +73,36 @@ void accepts_float_false(PType ptype) { assertThat(result).isFalse(); } + @Test + void encode_risingStep_isSortedAndStrictlySorted() { + // Given / When + NodeStats stats = ENCODER.encode(DTypes.I64, new long[]{5, 8, 11}, EncodeTestHelper.testCtx()).rootNode().stats(); + + // Then — Rust's Sequence::stats + assertThat(stats.isSorted()).isTrue(); + assertThat(stats.isStrictSorted()).isTrue(); + } + + @Test + void encode_fallingStep_isNeitherSorted() { + // Given / When + NodeStats stats = ENCODER.encode(DTypes.I64, new long[]{11, 8, 5}, EncodeTestHelper.testCtx()).rootNode().stats(); + + // Then + assertThat(stats.isSorted()).isFalse(); + assertThat(stats.isStrictSorted()).isFalse(); + } + + @Test + void encode_singleValue_isSortedWhateverTheStep() { + // Given / When + NodeStats stats = ENCODER.encode(DTypes.I64, new long[]{7}, EncodeTestHelper.testCtx()).rootNode().stats(); + + // Then — no adjacent pair to compare + assertThat(stats.isSorted()).isTrue(); + assertThat(stats.isStrictSorted()).isTrue(); + } + @Test void encode_float_throwsVortexException() { // Given