Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ public ArrayStats stats() {
return stats;
}

/// The stats if something has already computed them, without computing them now: a caller that only
/// wants the min and max is better off scanning for them than forcing the run, distinct and
/// frequency counts of the whole set.
///
/// @return the stats, or `null` if [#stats()] was never asked for
ArrayStats statsIfComputed() {
return stats;
}

/// Returns the number of rows, null or not.
///
/// @return the array length
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -275,21 +275,34 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx)
private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx, boolean traced) {
var event = new CompressEvent();
event.begin();
EncodeResult result = encodeArray(dtype, data, ctx);
// a primitive's stats are shared by the competition and its node stats: Rust caches them on the
// array, and reads min and max from there instead of scanning again
ArrayAndStats input = null;
if (dtype instanceof DType.Primitive) {
input = withStats(dtype, data, ctx);
}
EncodeResult result = encodeArray(dtype, data, ctx, input);
if (traced && !ctx.sample()) {
CompressorEvents.compress(event, ctx, dtype, data, result);
}
NodeStats own = result.rootNode().stats();
if (ctx.sample() || own != null && !own.isEmpty()) {
return result;
}
return new EncodeResult(result.rootNode().withStats(nodeStats(dtype, data, result.rootNode().encodingId())),
return new EncodeResult(result.rootNode().withStats(nodeStats(dtype, data, result.rootNode().encodingId(), input)),
result.encodedBuffers(), result.statsMin(), result.statsMax());
}

/// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root
/// as much as the root itself ([CachedNodeStats]).
private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) {
///
/// @param dtype the array's dtype
/// @param data the array
/// @param root the encoding the array was compressed to
/// @param input the array's stats if it has any, whose min and max are reused when the competition
/// already computed them
/// @return the stats the node carries
private static NodeStats nodeStats(DType dtype, Object data, EncodingId root, ArrayAndStats input) {
if (dtype instanceof DType.Bool) {
if (!root.equals(EncodingId.VORTEX_BOOL) || !(data instanceof boolean[] bools)) {
return NodeStats.NONE;
Expand All @@ -306,15 +319,32 @@ private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) {
if (!(dtype instanceof DType.Primitive)) {
return NodeStats.NONE;
}
MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null;
MinMax bounds = null;
if (((DType.Primitive) dtype).ptype().isInteger()) {
bounds = integerBounds((DType.Primitive) dtype, data, input);
}
long nullCount = data instanceof NullableData nullable ? nullable.nullCount() : 0L;
return CachedNodeStats.of(dtype, root, bounds == null ? null : bounds.min(), bounds == null ? null : bounds.max(),
RowCount.of(data), nullCount);
}

private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) {
/// The bounds of an integer array's valid values: from the stats the competition computed when it
/// did, else by scanning for them.
static MinMax integerBounds(DType.Primitive dtype, Object data, ArrayAndStats input) {
ArrayStats computed = null;
if (input != null) {
computed = input.statsIfComputed();
}
// no values (empty, or every row null) is the EMPTY stats, and has no bounds to reuse
if (computed != null && computed.valueCount() > 0) {
return PrimitiveEncodingEncoder.integerMinMax(dtype.ptype(), computed.min(), computed.max());
}
return ZoneMapStats.of(dtype, data);
}

private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx, ArrayAndStats input) {
if (keepsValidity(dtype) && data instanceof NullableData nullable) {
return encodeNullablePrimitive(dtype, nullable, ctx);
return encodeNullablePrimitive(dtype, nullable, ctx, input);
}
// An all-null decimal is one null constant, before any scheme, as for a primitive.
if (dtype instanceof DType.Decimal && data instanceof NullableData nullable
Expand Down Expand Up @@ -376,8 +406,7 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) {
return spliceResult(findPrimitiveEncoding(dtype, data, ctx.excluded()), dtype, data, ctx);
}

return competeAndEncode(dtype, withStats(dtype, data, ctx), ctx,
sampleSize -> primitiveBytes(dtype, sampleSize));
return competeAndEncode(dtype, input, ctx, sampleSize -> primitiveBytes(dtype, sampleSize));
}

/// A bool array stays `vortex.bool` unless it is one value and that is smaller as a
Expand Down Expand Up @@ -591,7 +620,8 @@ static long totalBytes(EncodeResult result) {
/// both decided by the compressor before any scheme (`compressor/cascade.rs`). Otherwise the
/// schemes compete with the validity in their arrays: a sampled scheme is measured with its
/// validity child, against the values plus a packed bitmap.
private EncodeResult encodeNullablePrimitive(DType dtype, NullableData data, EncodeContext ctx) {
private EncodeResult encodeNullablePrimitive(DType dtype, NullableData data, EncodeContext ctx,
ArrayAndStats input) {
int n = data.length();
if (MaskedEncodingEncoder.isNullConstant(data.validity(), ctx)) {
return MaskedEncodingEncoder.nullConstant();
Expand All @@ -604,7 +634,6 @@ private EncodeResult encodeNullablePrimitive(DType dtype, NullableData data, Enc
return MaskedEncodingEncoder.maskedConstant(dtype, data, valid, ctx);
}
}
ArrayAndStats input = withStats(dtype, data, ctx);
int sampleSize = sampleSize(n, ctx);
Object sample = sampleSize < n ? stratifiedSample(data, sampleSize, ctx.sampleSeed()) : data;
Choice choice = chooseBest(dtype, input, ctx, uncompressedBytes(dtype, sample), () -> sample);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,21 @@ public static MinMax minMaxStats(PType ptype, Object data) {
};
}

/// The bounds of an integer array whose min and max are already known, as scalars: what
/// [#minMaxStats(PType, Object)] makes of the same kernel's answer.
///
/// @param ptype an integer type
/// @param min the minimum, widened (raw 64 bits for `U64`)
/// @param max the maximum, widened
/// @return the serialized bounds
static MinMax integerMinMax(PType ptype, long min, long max) {
return switch (ptype) {
case I8, I16, I32, I64 -> new MinMax(scalarI64(min), scalarI64(max));
case U8, U16, U32, U64 -> new MinMax(scalarU64(min), scalarU64(max));
default -> throw new IllegalArgumentException(ptype + " is not an integer type");
};
}

/// The min half of a [#minMaxStats] result, or `null` when `stats` itself is `null` (empty
/// data). Saves callers the repeated `stats == null ? null : stats.min()` idiom.
///
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;

import io.github.dfa1.vortex.writer.WriteRegistry;
Expand Down Expand Up @@ -190,6 +191,44 @@ void dictChildren_carryTheirBoundsAndNullCount() {
assertThat(distinct.nullCount()).isZero();
}

static Stream<Arguments> boundedColumns() {
int rows = 4096;
int[] small = new int[rows];
long[] smallLongs = new long[rows];
short[] smallShorts = new short[rows];
boolean[] everySeventhNull = new boolean[rows];
for (int i = 0; i < rows; i++) {
small[i] = (i * 7) % 16;
smallLongs[i] = (i * 13L) % 50;
smallShorts[i] = (short) ((i * 11) % 200);
everySeventhNull[i] = i % 7 != 0;
}
return Stream.of(
Arguments.of("i32", new DType.Primitive(PType.I32, false), small),
Arguments.of("i32 with nulls", new DType.Primitive(PType.I32, true),
new NullableData(small, everySeventhNull)),
Arguments.of("u16", new DType.Primitive(PType.U16, false), smallShorts),
Arguments.of("u64 with nulls", new DType.Primitive(PType.U64, true),
new NullableData(smallLongs, everySeventhNull)),
Arguments.of("i64", new DType.Primitive(PType.I64, false), smallLongs));
}

@ParameterizedTest(name = "{0}")
@MethodSource("boundedColumns")
void integerColumn_rootCarriesTheBoundsOfItsValidValues(String name, DType dtype, Object data) {
// Given
CascadingCompressor sut = new CascadingCompressor(ALL_CODECS);
MinMax expected = ZoneMapStats.of(dtype, data);

// When
EncodeResult result = sut.encode(dtype, data, ctx(3));

// Then the root node, a bit-packed array here, has the bounds a scan of the valid rows finds
NodeStats root = result.rootNode().stats();
assertThat(root.min()).isNotNull().isEqualTo(expected.min());
assertThat(root.max()).isNotNull().isEqualTo(expected.max());
}

@Test
void constantBoolColumn_staysABoolUntilTheConstantIsSmaller() {
// Given — 16 rows pack into 2 bytes, as small as a constant; 17 rows pack into 3
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package io.github.dfa1.vortex.writer.encode;

import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.PType;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;

import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;

/// A node's min and max come from the stats the scheme competition computed when it did, as Rust reads
/// them off the array's cache, and from a scan when it did not; the two must be the same bounds.
class NodeStatsBoundsTest {

static Stream<Arguments> columns() {
boolean[] someNull = {true, false, true, true, false, true};
boolean[] allNull = {false, false, false};
return Stream.of(
Arguments.of("i8", PType.I8, new byte[]{5, -3, 9, 0, 120, -128}),
Arguments.of("u16", PType.U16, new short[]{5, 3, (short) 65_535, 0}),
Arguments.of("i32", PType.I32, new int[]{7, -2_000_000_000, 2_000_000_000, 0}),
Arguments.of("u32", PType.U32, new int[]{7, -1, 3}),
Arguments.of("i64", PType.I64, new long[]{Long.MIN_VALUE, 0, Long.MAX_VALUE}),
// a value past 2^63 is a negative long, and the maximum
Arguments.of("u64 past 2^63", PType.U64, new long[]{5, -1L, 1L << 63, 2}),
Arguments.of("i32, with nulls", PType.I32,
new NullableData(new int[]{100, 0, -50, 70, 0, 9}, someNull)),
Arguments.of("u64, with nulls", PType.U64,
new NullableData(new long[]{3, 0, -1L, 8, 0, 4}, someNull)),
Arguments.of("i16, all null", PType.I16, new NullableData(new short[]{0, 0, 0}, allNull)),
Arguments.of("i64, one value", PType.I64, new long[]{42}),
Arguments.of("i32, empty", PType.I32, new int[0]));
}

@ParameterizedTest(name = "{0}")
@MethodSource("columns")
void integerBounds_fromComputedStats_areTheScannedBounds(String name, PType ptype, Object data) {
// Given one array whose stats were computed, as the competition does, and one whose were not
DType dtype = new DType.Primitive(ptype, data instanceof NullableData);
ArrayAndStats computed = new ArrayAndStats(dtype, data, StatsOptions.defaults());
computed.stats();
ArrayAndStats untouched = new ArrayAndStats(dtype, data, StatsOptions.defaults());
DType.Primitive primitive = (DType.Primitive) dtype;

// When
MinMax reused = CascadingCompressor.integerBounds(primitive, data, computed);
MinMax scanned = CascadingCompressor.integerBounds(primitive, data, untouched);
MinMax expected = ZoneMapStats.of(dtype, data);

// Then
assertThat(reused).isEqualTo(expected);
assertThat(scanned).isEqualTo(expected);
assertThat(untouched.statsIfComputed()).as("the scan does not force the stats").isNull();
}

@Test
void integerBounds_withoutAnInput_scans() {
// Given
DType dtype = new DType.Primitive(PType.I32, false);
int[] data = {4, 1, 9};

// When
MinMax result = CascadingCompressor.integerBounds((DType.Primitive) dtype, data, null);

// Then
assertThat(result).isEqualTo(ZoneMapStats.of(dtype, data));
}
}
Loading