diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayAndStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayAndStats.java index 3cd46bffb..759db4d7f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayAndStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ArrayAndStats.java @@ -60,6 +60,15 @@ public ArrayStats stats() { return stats; } + /// The stats if something has already computed them, without computing them now: a caller that only + /// wants the min and max is better off scanning for them than forcing the run, distinct and + /// frequency counts of the whole set. + /// + /// @return the stats, or `null` if [#stats()] was never asked for + ArrayStats statsIfComputed() { + return stats; + } + /// Returns the number of rows, null or not. /// /// @return the array length diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index 3182ad304..5dc0ef2ab 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -275,7 +275,13 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx) private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx, boolean traced) { var event = new CompressEvent(); event.begin(); - EncodeResult result = encodeArray(dtype, data, ctx); + // a primitive's stats are shared by the competition and its node stats: Rust caches them on the + // array, and reads min and max from there instead of scanning again + ArrayAndStats input = null; + if (dtype instanceof DType.Primitive) { + input = withStats(dtype, data, ctx); + } + EncodeResult result = encodeArray(dtype, data, ctx, input); if (traced && !ctx.sample()) { CompressorEvents.compress(event, ctx, dtype, data, result); } @@ -283,13 +289,20 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx, if (ctx.sample() || own != null && !own.isEmpty()) { return result; } - return new EncodeResult(result.rootNode().withStats(nodeStats(dtype, data, result.rootNode().encodingId())), + return new EncodeResult(result.rootNode().withStats(nodeStats(dtype, data, result.rootNode().encodingId(), input)), result.encodedBuffers(), result.statsMin(), result.statsMax()); } /// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root /// as much as the root itself ([CachedNodeStats]). - private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { + /// + /// @param dtype the array's dtype + /// @param data the array + /// @param root the encoding the array was compressed to + /// @param input the array's stats if it has any, whose min and max are reused when the competition + /// already computed them + /// @return the stats the node carries + private static NodeStats nodeStats(DType dtype, Object data, EncodingId root, ArrayAndStats input) { if (dtype instanceof DType.Bool) { if (!root.equals(EncodingId.VORTEX_BOOL) || !(data instanceof boolean[] bools)) { return NodeStats.NONE; @@ -306,15 +319,32 @@ private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { if (!(dtype instanceof DType.Primitive)) { return NodeStats.NONE; } - MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null; + MinMax bounds = null; + if (((DType.Primitive) dtype).ptype().isInteger()) { + bounds = integerBounds((DType.Primitive) dtype, data, input); + } long nullCount = data instanceof NullableData nullable ? nullable.nullCount() : 0L; return CachedNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(), RowCount.of(data), nullCount); } - private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { + /// The bounds of an integer array's valid values: from the stats the competition computed when it + /// did, else by scanning for them. + static MinMax integerBounds(DType.Primitive dtype, Object data, ArrayAndStats input) { + ArrayStats computed = null; + if (input != null) { + computed = input.statsIfComputed(); + } + // no values (empty, or every row null) is the EMPTY stats, and has no bounds to reuse + if (computed != null && computed.valueCount() > 0) { + return PrimitiveEncodingEncoder.integerMinMax(dtype.ptype(), computed.min(), computed.max()); + } + return ZoneMapStats.of(dtype, data); + } + + private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx, ArrayAndStats input) { if (keepsValidity(dtype) && data instanceof NullableData nullable) { - return encodeNullablePrimitive(dtype, nullable, ctx); + return encodeNullablePrimitive(dtype, nullable, ctx, input); } // An all-null decimal is one null constant, before any scheme, as for a primitive. if (dtype instanceof DType.Decimal && data instanceof NullableData nullable @@ -376,8 +406,7 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { return spliceResult(findPrimitiveEncoding(dtype, data, ctx.excluded()), dtype, data, ctx); } - return competeAndEncode(dtype, withStats(dtype, data, ctx), ctx, - sampleSize -> primitiveBytes(dtype, sampleSize)); + return competeAndEncode(dtype, input, ctx, sampleSize -> primitiveBytes(dtype, sampleSize)); } /// A bool array stays `vortex.bool` unless it is one value and that is smaller as a @@ -591,7 +620,8 @@ static long totalBytes(EncodeResult result) { /// both decided by the compressor before any scheme (`compressor/cascade.rs`). Otherwise the /// schemes compete with the validity in their arrays: a sampled scheme is measured with its /// validity child, against the values plus a packed bitmap. - private EncodeResult encodeNullablePrimitive(DType dtype, NullableData data, EncodeContext ctx) { + private EncodeResult encodeNullablePrimitive(DType dtype, NullableData data, EncodeContext ctx, + ArrayAndStats input) { int n = data.length(); if (MaskedEncodingEncoder.isNullConstant(data.validity(), ctx)) { return MaskedEncodingEncoder.nullConstant(); @@ -604,7 +634,6 @@ private EncodeResult encodeNullablePrimitive(DType dtype, NullableData data, Enc return MaskedEncodingEncoder.maskedConstant(dtype, data, valid, ctx); } } - ArrayAndStats input = withStats(dtype, data, ctx); int sampleSize = sampleSize(n, ctx); Object sample = sampleSize < n ? stratifiedSample(data, sampleSize, ctx.sampleSeed()) : data; Choice choice = chooseBest(dtype, input, ctx, uncompressedBytes(dtype, sample), () -> sample); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index 5dfb91455..56fb1fcfc 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -87,6 +87,21 @@ public static MinMax minMaxStats(PType ptype, Object data) { }; } + /// The bounds of an integer array whose min and max are already known, as scalars: what + /// [#minMaxStats(PType, Object)] makes of the same kernel's answer. + /// + /// @param ptype an integer type + /// @param min the minimum, widened (raw 64 bits for `U64`) + /// @param max the maximum, widened + /// @return the serialized bounds + static MinMax integerMinMax(PType ptype, long min, long max) { + return switch (ptype) { + case I8, I16, I32, I64 -> new MinMax(scalarI64(min), scalarI64(max)); + case U8, U16, U32, U64 -> new MinMax(scalarU64(min), scalarU64(max)); + default -> throw new IllegalArgumentException(ptype + " is not an integer type"); + }; + } + /// The min half of a [#minMaxStats] result, or `null` when `stats` itself is `null` (empty /// data). Saves callers the repeated `stats == null ? null : stats.min()` idiom. /// diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java index 30905397d..d11060d96 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import io.github.dfa1.vortex.writer.WriteRegistry; @@ -190,6 +191,44 @@ void dictChildren_carryTheirBoundsAndNullCount() { assertThat(distinct.nullCount()).isZero(); } + static Stream boundedColumns() { + int rows = 4096; + int[] small = new int[rows]; + long[] smallLongs = new long[rows]; + short[] smallShorts = new short[rows]; + boolean[] everySeventhNull = new boolean[rows]; + for (int i = 0; i < rows; i++) { + small[i] = (i * 7) % 16; + smallLongs[i] = (i * 13L) % 50; + smallShorts[i] = (short) ((i * 11) % 200); + everySeventhNull[i] = i % 7 != 0; + } + return Stream.of( + Arguments.of("i32", new DType.Primitive(PType.I32, false), small), + Arguments.of("i32 with nulls", new DType.Primitive(PType.I32, true), + new NullableData(small, everySeventhNull)), + Arguments.of("u16", new DType.Primitive(PType.U16, false), smallShorts), + Arguments.of("u64 with nulls", new DType.Primitive(PType.U64, true), + new NullableData(smallLongs, everySeventhNull)), + Arguments.of("i64", new DType.Primitive(PType.I64, false), smallLongs)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("boundedColumns") + void integerColumn_rootCarriesTheBoundsOfItsValidValues(String name, DType dtype, Object data) { + // Given + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + MinMax expected = ZoneMapStats.of(dtype, data); + + // When + EncodeResult result = sut.encode(dtype, data, ctx(3)); + + // Then the root node, a bit-packed array here, has the bounds a scan of the valid rows finds + NodeStats root = result.rootNode().stats(); + assertThat(root.min()).isNotNull().isEqualTo(expected.min()); + assertThat(root.max()).isNotNull().isEqualTo(expected.max()); + } + @Test void constantBoolColumn_staysABoolUntilTheConstantIsSmaller() { // Given — 16 rows pack into 2 bytes, as small as a constant; 17 rows pack into 3 diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NodeStatsBoundsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NodeStatsBoundsTest.java new file mode 100644 index 000000000..1446ed871 --- /dev/null +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NodeStatsBoundsTest.java @@ -0,0 +1,71 @@ +package io.github.dfa1.vortex.writer.encode; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A node's min and max come from the stats the scheme competition computed when it did, as Rust reads +/// them off the array's cache, and from a scan when it did not; the two must be the same bounds. +class NodeStatsBoundsTest { + + static Stream columns() { + boolean[] someNull = {true, false, true, true, false, true}; + boolean[] allNull = {false, false, false}; + return Stream.of( + Arguments.of("i8", PType.I8, new byte[]{5, -3, 9, 0, 120, -128}), + Arguments.of("u16", PType.U16, new short[]{5, 3, (short) 65_535, 0}), + Arguments.of("i32", PType.I32, new int[]{7, -2_000_000_000, 2_000_000_000, 0}), + Arguments.of("u32", PType.U32, new int[]{7, -1, 3}), + Arguments.of("i64", PType.I64, new long[]{Long.MIN_VALUE, 0, Long.MAX_VALUE}), + // a value past 2^63 is a negative long, and the maximum + Arguments.of("u64 past 2^63", PType.U64, new long[]{5, -1L, 1L << 63, 2}), + Arguments.of("i32, with nulls", PType.I32, + new NullableData(new int[]{100, 0, -50, 70, 0, 9}, someNull)), + Arguments.of("u64, with nulls", PType.U64, + new NullableData(new long[]{3, 0, -1L, 8, 0, 4}, someNull)), + Arguments.of("i16, all null", PType.I16, new NullableData(new short[]{0, 0, 0}, allNull)), + Arguments.of("i64, one value", PType.I64, new long[]{42}), + Arguments.of("i32, empty", PType.I32, new int[0])); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("columns") + void integerBounds_fromComputedStats_areTheScannedBounds(String name, PType ptype, Object data) { + // Given one array whose stats were computed, as the competition does, and one whose were not + DType dtype = new DType.Primitive(ptype, data instanceof NullableData); + ArrayAndStats computed = new ArrayAndStats(dtype, data, StatsOptions.defaults()); + computed.stats(); + ArrayAndStats untouched = new ArrayAndStats(dtype, data, StatsOptions.defaults()); + DType.Primitive primitive = (DType.Primitive) dtype; + + // When + MinMax reused = CascadingCompressor.integerBounds(primitive, data, computed); + MinMax scanned = CascadingCompressor.integerBounds(primitive, data, untouched); + MinMax expected = ZoneMapStats.of(dtype, data); + + // Then + assertThat(reused).isEqualTo(expected); + assertThat(scanned).isEqualTo(expected); + assertThat(untouched.statsIfComputed()).as("the scan does not force the stats").isNull(); + } + + @Test + void integerBounds_withoutAnInput_scans() { + // Given + DType dtype = new DType.Primitive(PType.I32, false); + int[] data = {4, 1, 9}; + + // When + MinMax result = CascadingCompressor.integerBounds((DType.Primitive) dtype, data, null); + + // Then + assertThat(result).isEqualTo(ZoneMapStats.of(dtype, data)); + } +}