diff --git a/CHANGELOG.md b/CHANGELOG.md index ddef5fb37..c175630b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- The zone table of a column holds Rust's shape: no validity or mask on a column without nulls, and a constant bool stays a `vortex.bool` until the constant is smaller. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index 83c7eb245..c482c6bd8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -177,7 +177,7 @@ patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. `NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol -table) and the zone-table segments of string and bool columns are not covered yet +table) and the zone-table segments of string columns, which wait for Rust's `vortex.varbinview` canonical form, are not covered yet ([#583](https://github.com/dfa1/vortex-java/issues/583)). ### Decode shape diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java new file mode 100644 index 000000000..9a4619df7 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java @@ -0,0 +1,80 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A bool array of one value becomes `vortex.constant` only when that is smaller than its packed +/// bits: Rust's compressor keeps a short one as `vortex.bool`, so the three flags of a zone table, +/// and a handful of rows, stay bool while a long constant column does not (#583). +class ConstantBoolInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + + static { + NativeLoader.loadJni(); + } + + @ParameterizedTest + @ValueSource(ints = {1, 3, 8, 9, 16, 17, 24, 64, 65, 512, 20_000}) + void javaWritesAConstantBoolAsJniDoes(int rows, @TempDir Path tmp) throws Exception { + // Given — one column of identical bools + Schema arrow = new Schema(List.of(Field.notNullable("flag", ArrowType.Bool.INSTANCE))); + Path jni = tmp.resolve("jni.vortex"); + try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + BitVector flag = (BitVector) root.getVector("flag"); + root.allocateNew(); + for (int i = 0; i < rows; i++) { + flag.setSafe(i, 1); + } + root.setRowCount(rows); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema as = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, as); + w.writeBatch(array.memoryAddress(), as.memoryAddress()); + } + } + Path written = tmp.resolve("java.vortex"); + DType.Struct schema = new DType.Struct(List.of(ColumnName.of("flag")), List.of(DTypes.BOOL), false); + boolean[] flags = new boolean[rows]; + Arrays.fill(flags, true); + try (FileChannel ch = FileChannel.open(written, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(Map.of(ColumnName.of("flag"), flags)); + } + + // When + List result = EncodingTrees.withStats(written); + + // Then + List expected = EncodingTrees.withStats(jni); + assertThat(result).isEqualTo(expected); + } +} diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java index 8c0e775c3..63d7bc33b 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -71,6 +71,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { long[] descending = new long[N]; double[] nullableFloat = new double[N]; boolean[] validFloat = new boolean[N]; + boolean[] flag = new boolean[N]; + String[] noisy = new String[N]; for (int i = 0; i < N; i++) { sorted[i] = i; random[i] = r.nextLong(); @@ -79,6 +81,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { sf[i] = i * 0.5; patched[i] = i % 100 == 0 ? r.nextDouble() * 1e9 : i * 0.01; descending[i] = 100_000L - i * 3L; + flag[i] = true; + noisy[i] = Long.toHexString(r.nextLong()) + Long.toHexString(r.nextLong()); low[i] = "k" + r.nextInt(10); high[i] = "value-" + Long.toString(r.nextLong() & 0xffffffffL, 36); bool[i] = r.nextBoolean(); @@ -97,7 +101,9 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { Field.nullable("nullable", new ArrowType.Int(32, true)), Field.notNullable("patched", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), Field.notNullable("descending", new ArrowType.Int(64, true)), - Field.nullable("nullf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)))); + Field.nullable("nullf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("flag", ArrowType.Bool.INSTANCE), + Field.notNullable("noisy", ArrowType.Utf8.INSTANCE))); Path jni = tmp.resolve("jni.vortex"); try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { @@ -113,6 +119,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { Float8Vector pv = (Float8Vector) root.getVector("patched"); BigIntVector dv = (BigIntVector) root.getVector("descending"); Float8Vector nf = (Float8Vector) root.getVector("nullf"); + BitVector fv = (BitVector) root.getVector("flag"); + VarCharVector nz = (VarCharVector) root.getVector("noisy"); root.allocateNew(); for (int i = 0; i < N; i++) { a.setSafe(i, sorted[i]); @@ -130,6 +138,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { } else { nf.setNull(i); } + fv.setSafe(i, 1); + nz.setSafe(i, noisy[i].getBytes(StandardCharsets.UTF_8)); if (valid[i]) { nv.setSafe(i, nullable[i]); } else { @@ -144,8 +154,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { } Path java = tmp.resolve("java.vortex"); DType.Struct schema = new DType.Struct( - List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending", "nullf").stream().map(ColumnName::of).toList(), - List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64, DTypes.F64.asNullable()), false); + List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending", "nullf", "flag", "noisy").stream().map(ColumnName::of).toList(), + List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64, DTypes.F64.asNullable(), DTypes.BOOL, DTypes.UTF8), false); try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { Map chunk = new java.util.LinkedHashMap<>(); @@ -161,6 +171,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { chunk.put(ColumnName.of("patched"), patched); chunk.put(ColumnName.of("descending"), descending); chunk.put(ColumnName.of("nullf"), new NullableData(nullableFloat, validFloat)); + chunk.put(ColumnName.of("flag"), flag); + chunk.put(ColumnName.of("noisy"), noisy); w.writeChunk(chunk); } List expected = EncodingTrees.withStats(jni); @@ -174,9 +186,22 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { List> expectedColumns = columns(expected); List> resultColumns = columns(result); assertThat(resultColumns).hasSameSizeAs(expectedColumns); - for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11}) { + for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11, 12}) { assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c)); } + // the string columns' zone tables wait for Rust's varbinview canonical form (#583) + for (int c = 0; c < expectedColumns.size(); c++) { + if (c == 5 || c == 6 || c == 13) { + continue; + } + assertThat(zoneTable(resultColumns.get(c))).as("zone table of column %d", c) + .isEqualTo(zoneTable(expectedColumns.get(c))); + } + } + + /// The lines of a column's stats-table segment: the last flat under the zoned layout. + private static List zoneTable(List column) { + return column.subList(column.lastIndexOf(" vortex.flat"), column.size()); } /// The tree lines of each top-level column, in file order. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java index 1ded62f95..99553dea9 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java @@ -194,14 +194,11 @@ StructData tableData() { return new StructData(fields); } - private static NullableData counts(List counts) { - long[] values = counts.stream().mapToLong(Long::longValue).toArray(); - boolean[] valid = new boolean[values.length]; - Arrays.fill(valid, true); - return new NullableData(values, valid); + private static long[] counts(List counts) { + return counts.stream().mapToLong(Long::longValue).toArray(); } - private NullableData boundedMaxColumn(int zones) { + private Object boundedMaxColumn(int zones) { // The state struct is null for a zone with no non-null value; otherwise `unknown` says // whether any bound fit, and `bound` holds it when one did. boolean[] stateValid = new boolean[zones]; @@ -213,11 +210,23 @@ private NullableData boundedMaxColumn(int zones) { unknown[i] = maxUnknown.get(i); bounds.add(maxs.get(i)); } - return new NullableData(new StructData(List.of(valueColumn(bounds), unknown)), stateValid); + return withValidity(new StructData(List.of(valueColumn(bounds), unknown)), stateValid); + } + + /// `values` with its validity, or on its own when every row is valid: Rust builds each stats-table + /// column with a builder whose validity stays `AllValid` unless a null is appended, so a column + /// without nulls carries no validity child and is not masked. + private static Object withValidity(Object values, boolean[] valid) { + for (boolean v : valid) { + if (!v) { + return new NullableData(values, valid); + } + } + return values; } /// A per-zone value column in the column's own carrier, `null` entries invalid. - private NullableData valueColumn(List perZone) { + private Object valueColumn(List perZone) { int zones = perZone.size(); boolean[] valid = new boolean[zones]; for (int i = 0; i < zones; i++) { @@ -263,7 +272,7 @@ private NullableData valueColumn(List perZone) { case BINARY -> perZone.toArray(new byte[0][]); case NULL_COUNT_ONLY -> throw new IllegalStateException("no value column for " + dtype); }; - return new NullableData(values, valid); + return withValidity(values, valid); } // ── per-zone stats ───────────────────────────────────────────────────── diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java index b456de854..5f4bd0a37 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java @@ -38,12 +38,24 @@ static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoSca case DType.Primitive p when !p.ptype().isFloating() && retainsIntegerStats(root) -> new NodeStats(min, max, isConstant(min, max, nullCount), null, null, nullCount, canonicalSize(p, rows, nullCount), null); - case DType.Bool _ when root.equals(EncodingId.VORTEX_BOOL) -> - new NodeStats(null, null, false, null, null, null, (rows + 7) / 8, null); + case DType.Bool _ when root.equals(EncodingId.VORTEX_BOOL) -> ofBool(rows, false); default -> NodeStats.NONE; }; } + /// The stats of a `vortex.bool` array: constant when every bit is equal, and nothing else then; the + /// packed size otherwise. + /// + /// @param rows the number of rows + /// @param constant whether every row is equal + /// @return the bool array's stats + static NodeStats ofBool(long rows, boolean constant) { + if (constant) { + return new NodeStats(null, null, true, null, null, null, null, null); + } + return new NodeStats(null, null, false, null, null, null, (rows + 7) / 8, null); + } + /// The stats of the `vortex.bool` array that holds a nullable array's validity. Rust builds it /// from the mask: an array of one value is constant, a mixed one carries both bounds, its /// valid-row count as the sum and no nulls. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index 43c03f687..92fe801a9 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -32,6 +32,8 @@ public final class CascadingCompressor { private static final EncodingEncoder CANONICAL_PRIMITIVE = new PrimitiveEncodingEncoder(); private static final EncodingEncoder CANONICAL_BOOL = new BoolEncodingEncoder(); + /// Bytes of a `vortex.constant` bool: Rust keeps a bool array whose packed bits are no larger. + private static final int CONSTANT_BOOL_BYTES = 2; private static final EncodingEncoder CANONICAL_VARBIN = new VarBinEncodingEncoder(); private static final EncodingEncoder CANONICAL_DECIMAL = new DecimalEncodingEncoder(); private static final EncodingEncoder CANONICAL_EXTENSION = new ExtEncodingEncoder(); @@ -270,7 +272,13 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx) /// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root /// as much as the root itself ([CachedNodeStats]). private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { - if (!(dtype instanceof DType.Primitive || dtype instanceof DType.Bool)) { + if (dtype instanceof DType.Bool) { + if (!root.equals(EncodingId.VORTEX_BOOL) || !(data instanceof boolean[] bools)) { + return NodeStats.NONE; + } + return CachedNodeStats.ofBool(bools.length, SimdOperationsSupport.preferred().allEqual(bools)); + } + if (!(dtype instanceof DType.Primitive)) { return NodeStats.NONE; } MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null; @@ -336,6 +344,9 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { if (dtype instanceof DType.Decimal decimal) { return encodeDecimal(decimal, data, ctx); } + if (dtype instanceof DType.Bool && data instanceof boolean[] bools) { + return encodeBool(bools, ctx); + } if (!(dtype instanceof DType.Primitive)) { return spliceResult(findPrimitiveEncoding(dtype, data, ctx.excluded()), dtype, data, ctx); } @@ -344,6 +355,18 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { sampleSize -> primitiveBytes(dtype, sampleSize)); } + /// A bool array stays `vortex.bool` unless it is one value and that is smaller as a + /// `vortex.constant`: Rust's `ConstantScheme` only wins when the constant is smaller than the packed + /// bits, which it is from 17 rows (3 bytes) on. A zone table's three flags, or a few rows, stay bool. + private EncodeResult encodeBool(boolean[] bools, EncodeContext ctx) { + boolean constantPays = (bools.length + 7) / 8 > CONSTANT_BOOL_BYTES; + if (constantPays && !ctx.sample() && !ctx.excluded().contains(EncodingId.VORTEX_CONSTANT) + && SimdOperationsSupport.preferred().allEqual(bools)) { + return new ConstantEncodingEncoder().encode(DTypes.BOOL, bools, ctx); + } + return CANONICAL_BOOL.encode(DTypes.BOOL, bools, ctx); + } + /// Rust's constant check for a decimal array, run in the compressor before any scheme and never /// on a sample: when every valid value is equal the array is one `vortex.constant`, wrapped in /// `vortex.masked` with the validity when some row is null, and kept only if that is smaller than diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java index 2522d884f..374b8440f 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java @@ -58,6 +58,20 @@ void allNullZoneHasNullMinMaxAndCountsItsNulls() { assertThat(values(result, 2)).isEqualTo(new long[]{3}); } + @Test + void columnsWithoutNullsCarryNoValidity() { + // Given two zones, each with a value: Rust's builders stay AllValid, so nothing is masked + ZoneAccumulator sut = new ZoneAccumulator(DTypes.I64.withNullable(true)); + sut.add(new NullableData(new long[]{1, 2, 3}, new boolean[]{true, false, true}), 3); + + // When + sut.finish(); + StructData result = sut.tableData(); + + // Then + assertThat(result.fieldArrays()).noneMatch(NullableData.class::isInstance); + } + @Test void floatsSkipNanInMinMaxAndCountIt() { // Given a leading NaN — Rust's max/min skip NaN and record them in nan_count @@ -69,8 +83,8 @@ void floatsSkipNanInMinMaxAndCountIt() { StructData result = sut.tableData(); // Then — fields: max, min, nan_count, null_count - assertThat((double[]) ((NullableData) result.fieldArrays().get(0)).values()).containsExactly(2.0); - assertThat((double[]) ((NullableData) result.fieldArrays().get(1)).values()).containsExactly(-1.0); + assertThat((double[]) result.fieldArrays().get(0)).containsExactly(2.0); + assertThat((double[]) result.fieldArrays().get(1)).containsExactly(-1.0); assertThat(values(result, 2)).isEqualTo(new long[]{2}); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java index 269307e2f..a1016993d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java @@ -133,4 +133,22 @@ void withRootStats_fillsAnEmptyRootWithTheCachedStats() { assertThat(result.rootNode().stats()) .isEqualTo(CachedNodeStats.of(DTypes.I64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0)); } + + @Test + void ofBool_constantCarriesOnlyTheFlag() { + // Given / When + NodeStats result = CachedNodeStats.ofBool(20_000, true); + + // Then + assertThat(result).isEqualTo(new NodeStats(null, null, true, null, null, null, null, null)); + } + + @Test + void ofBool_otherwiseCarriesTheFlagAndThePackedSize() { + // Given / When + NodeStats result = CachedNodeStats.ofBool(17, false); + + // Then + assertThat(result).isEqualTo(new NodeStats(null, null, false, null, null, null, 3L, null)); + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java index 2a0edd1c6..6fb71fd10 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java @@ -190,6 +190,20 @@ void dictChildren_carryTheirBoundsAndNullCount() { assertThat(distinct.nullCount()).isZero(); } + @Test + void constantBoolColumn_staysABoolUntilTheConstantIsSmaller() { + // Given — 16 rows pack into 2 bytes, as small as a constant; 17 rows pack into 3 + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult small = sut.encode(DTypes.BOOL, new boolean[16], ctx(3)); + EncodeResult large = sut.encode(DTypes.BOOL, new boolean[17], ctx(3)); + + // Then + assertThat(small.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_BOOL); + assertThat(large.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_CONSTANT); + } + @Test void nullableIntegerValidity_isABoolChildCarryingTheValidCount() { // Given