From 1b8ae554ad7375d4f435223fa76566afb08af6ce Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Fri, 6 Nov 2026 19:41:46 +0100 Subject: [PATCH] fix(writer): the zone table of a column has Rust's shape: no validity without nulls, constant bool by size (#583) Rust's stats-table builders stay AllValid unless a null is appended, so a zone-table column without nulls carries no validity child and no mask (ZoneAccumulator). A bool column is a vortex.constant only when its packed bytes exceed the constant's two (17 rows and up), and its node carries the stats Rust caches. Interop tests compare the numeric, bool, flag and nullable float zone tables, and constant bools of 1 to 20000 rows, with vortex-jni. The string columns' zone tables wait for Rust's vortex.varbinview canonical form. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011fRbYsrGtpKHp9APhU8qDV --- CHANGELOG.md | 1 + docs/compatibility.md | 2 +- .../ConstantBoolInteropIntegrationTest.java | 80 +++++++++++++++++++ .../NodeStatsInteropIntegrationTest.java | 33 +++++++- .../dfa1/vortex/writer/ZoneAccumulator.java | 27 ++++--- .../vortex/writer/encode/CachedNodeStats.java | 16 +++- .../writer/encode/CascadingCompressor.java | 25 +++++- .../vortex/writer/ZoneAccumulatorTest.java | 18 ++++- .../writer/encode/CachedNodeStatsTest.java | 18 +++++ .../encode/CascadingCompressorTest.java | 14 ++++ 10 files changed, 215 insertions(+), 19 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ddef5fb37..c175630b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- The zone table of a column holds Rust's shape: no validity or mask on a column without nulls, and a constant bool stays a `vortex.bool` until the constant is smaller. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index 83c7eb245..c482c6bd8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -177,7 +177,7 @@ patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. `NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol -table) and the zone-table segments of string and bool columns are not covered yet +table) and the zone-table segments of string columns, which wait for Rust's `vortex.varbinview` canonical form, are not covered yet ([#583](https://github.com/dfa1/vortex-java/issues/583)). ### Decode shape diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java new file mode 100644 index 000000000..9a4619df7 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/ConstantBoolInteropIntegrationTest.java @@ -0,0 +1,80 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A bool array of one value becomes `vortex.constant` only when that is smaller than its packed +/// bits: Rust's compressor keeps a short one as `vortex.bool`, so the three flags of a zone table, +/// and a handful of rows, stay bool while a long constant column does not (#583). +class ConstantBoolInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + + static { + NativeLoader.loadJni(); + } + + @ParameterizedTest + @ValueSource(ints = {1, 3, 8, 9, 16, 17, 24, 64, 65, 512, 20_000}) + void javaWritesAConstantBoolAsJniDoes(int rows, @TempDir Path tmp) throws Exception { + // Given — one column of identical bools + Schema arrow = new Schema(List.of(Field.notNullable("flag", ArrowType.Bool.INSTANCE))); + Path jni = tmp.resolve("jni.vortex"); + try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + BitVector flag = (BitVector) root.getVector("flag"); + root.allocateNew(); + for (int i = 0; i < rows; i++) { + flag.setSafe(i, 1); + } + root.setRowCount(rows); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema as = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, as); + w.writeBatch(array.memoryAddress(), as.memoryAddress()); + } + } + Path written = tmp.resolve("java.vortex"); + DType.Struct schema = new DType.Struct(List.of(ColumnName.of("flag")), List.of(DTypes.BOOL), false); + boolean[] flags = new boolean[rows]; + Arrays.fill(flags, true); + try (FileChannel ch = FileChannel.open(written, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(Map.of(ColumnName.of("flag"), flags)); + } + + // When + List result = EncodingTrees.withStats(written); + + // Then + List expected = EncodingTrees.withStats(jni); + assertThat(result).isEqualTo(expected); + } +} diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java index 8c0e775c3..63d7bc33b 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -71,6 +71,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { long[] descending = new long[N]; double[] nullableFloat = new double[N]; boolean[] validFloat = new boolean[N]; + boolean[] flag = new boolean[N]; + String[] noisy = new String[N]; for (int i = 0; i < N; i++) { sorted[i] = i; random[i] = r.nextLong(); @@ -79,6 +81,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { sf[i] = i * 0.5; patched[i] = i % 100 == 0 ? r.nextDouble() * 1e9 : i * 0.01; descending[i] = 100_000L - i * 3L; + flag[i] = true; + noisy[i] = Long.toHexString(r.nextLong()) + Long.toHexString(r.nextLong()); low[i] = "k" + r.nextInt(10); high[i] = "value-" + Long.toString(r.nextLong() & 0xffffffffL, 36); bool[i] = r.nextBoolean(); @@ -97,7 +101,9 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { Field.nullable("nullable", new ArrowType.Int(32, true)), Field.notNullable("patched", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), Field.notNullable("descending", new ArrowType.Int(64, true)), - Field.nullable("nullf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)))); + Field.nullable("nullf", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("flag", ArrowType.Bool.INSTANCE), + Field.notNullable("noisy", ArrowType.Utf8.INSTANCE))); Path jni = tmp.resolve("jni.vortex"); try (VortexWriter w = VortexWriter.builder(SESSION, jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { @@ -113,6 +119,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { Float8Vector pv = (Float8Vector) root.getVector("patched"); BigIntVector dv = (BigIntVector) root.getVector("descending"); Float8Vector nf = (Float8Vector) root.getVector("nullf"); + BitVector fv = (BitVector) root.getVector("flag"); + VarCharVector nz = (VarCharVector) root.getVector("noisy"); root.allocateNew(); for (int i = 0; i < N; i++) { a.setSafe(i, sorted[i]); @@ -130,6 +138,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { } else { nf.setNull(i); } + fv.setSafe(i, 1); + nz.setSafe(i, noisy[i].getBytes(StandardCharsets.UTF_8)); if (valid[i]) { nv.setSafe(i, nullable[i]); } else { @@ -144,8 +154,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { } Path java = tmp.resolve("java.vortex"); DType.Struct schema = new DType.Struct( - List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending", "nullf").stream().map(ColumnName::of).toList(), - List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64, DTypes.F64.asNullable()), false); + List.of("sorted", "random", "small", "rf", "sf", "low", "high", "bool", "nullable", "patched", "descending", "nullf", "flag", "noisy").stream().map(ColumnName::of).toList(), + List.of(DTypes.I64, DTypes.I64, DTypes.I32, DTypes.F64, DTypes.F64, DTypes.UTF8, DTypes.UTF8, DTypes.BOOL, DTypes.I32.asNullable(), DTypes.F64, DTypes.I64, DTypes.F64.asNullable(), DTypes.BOOL, DTypes.UTF8), false); try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE); var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { Map chunk = new java.util.LinkedHashMap<>(); @@ -161,6 +171,8 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { chunk.put(ColumnName.of("patched"), patched); chunk.put(ColumnName.of("descending"), descending); chunk.put(ColumnName.of("nullf"), new NullableData(nullableFloat, validFloat)); + chunk.put(ColumnName.of("flag"), flag); + chunk.put(ColumnName.of("noisy"), noisy); w.writeChunk(chunk); } List expected = EncodingTrees.withStats(jni); @@ -174,9 +186,22 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { List> expectedColumns = columns(expected); List> resultColumns = columns(result); assertThat(resultColumns).hasSameSizeAs(expectedColumns); - for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11}) { + for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11, 12}) { assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c)); } + // the string columns' zone tables wait for Rust's varbinview canonical form (#583) + for (int c = 0; c < expectedColumns.size(); c++) { + if (c == 5 || c == 6 || c == 13) { + continue; + } + assertThat(zoneTable(resultColumns.get(c))).as("zone table of column %d", c) + .isEqualTo(zoneTable(expectedColumns.get(c))); + } + } + + /// The lines of a column's stats-table segment: the last flat under the zoned layout. + private static List zoneTable(List column) { + return column.subList(column.lastIndexOf(" vortex.flat"), column.size()); } /// The tree lines of each top-level column, in file order. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java index 1ded62f95..99553dea9 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java @@ -194,14 +194,11 @@ StructData tableData() { return new StructData(fields); } - private static NullableData counts(List counts) { - long[] values = counts.stream().mapToLong(Long::longValue).toArray(); - boolean[] valid = new boolean[values.length]; - Arrays.fill(valid, true); - return new NullableData(values, valid); + private static long[] counts(List counts) { + return counts.stream().mapToLong(Long::longValue).toArray(); } - private NullableData boundedMaxColumn(int zones) { + private Object boundedMaxColumn(int zones) { // The state struct is null for a zone with no non-null value; otherwise `unknown` says // whether any bound fit, and `bound` holds it when one did. boolean[] stateValid = new boolean[zones]; @@ -213,11 +210,23 @@ private NullableData boundedMaxColumn(int zones) { unknown[i] = maxUnknown.get(i); bounds.add(maxs.get(i)); } - return new NullableData(new StructData(List.of(valueColumn(bounds), unknown)), stateValid); + return withValidity(new StructData(List.of(valueColumn(bounds), unknown)), stateValid); + } + + /// `values` with its validity, or on its own when every row is valid: Rust builds each stats-table + /// column with a builder whose validity stays `AllValid` unless a null is appended, so a column + /// without nulls carries no validity child and is not masked. + private static Object withValidity(Object values, boolean[] valid) { + for (boolean v : valid) { + if (!v) { + return new NullableData(values, valid); + } + } + return values; } /// A per-zone value column in the column's own carrier, `null` entries invalid. - private NullableData valueColumn(List perZone) { + private Object valueColumn(List perZone) { int zones = perZone.size(); boolean[] valid = new boolean[zones]; for (int i = 0; i < zones; i++) { @@ -263,7 +272,7 @@ private NullableData valueColumn(List perZone) { case BINARY -> perZone.toArray(new byte[0][]); case NULL_COUNT_ONLY -> throw new IllegalStateException("no value column for " + dtype); }; - return new NullableData(values, valid); + return withValidity(values, valid); } // ── per-zone stats ───────────────────────────────────────────────────── diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java index b456de854..5f4bd0a37 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java @@ -38,12 +38,24 @@ static NodeStats of(DType dtype, EncodingId root, ProtoScalarValue min, ProtoSca case DType.Primitive p when !p.ptype().isFloating() && retainsIntegerStats(root) -> new NodeStats(min, max, isConstant(min, max, nullCount), null, null, nullCount, canonicalSize(p, rows, nullCount), null); - case DType.Bool _ when root.equals(EncodingId.VORTEX_BOOL) -> - new NodeStats(null, null, false, null, null, null, (rows + 7) / 8, null); + case DType.Bool _ when root.equals(EncodingId.VORTEX_BOOL) -> ofBool(rows, false); default -> NodeStats.NONE; }; } + /// The stats of a `vortex.bool` array: constant when every bit is equal, and nothing else then; the + /// packed size otherwise. + /// + /// @param rows the number of rows + /// @param constant whether every row is equal + /// @return the bool array's stats + static NodeStats ofBool(long rows, boolean constant) { + if (constant) { + return new NodeStats(null, null, true, null, null, null, null, null); + } + return new NodeStats(null, null, false, null, null, null, (rows + 7) / 8, null); + } + /// The stats of the `vortex.bool` array that holds a nullable array's validity. Rust builds it /// from the mask: an array of one value is constant, a mixed one carries both bounds, its /// valid-row count as the sum and no nulls. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index 43c03f687..92fe801a9 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -32,6 +32,8 @@ public final class CascadingCompressor { private static final EncodingEncoder CANONICAL_PRIMITIVE = new PrimitiveEncodingEncoder(); private static final EncodingEncoder CANONICAL_BOOL = new BoolEncodingEncoder(); + /// Bytes of a `vortex.constant` bool: Rust keeps a bool array whose packed bits are no larger. + private static final int CONSTANT_BOOL_BYTES = 2; private static final EncodingEncoder CANONICAL_VARBIN = new VarBinEncodingEncoder(); private static final EncodingEncoder CANONICAL_DECIMAL = new DecimalEncodingEncoder(); private static final EncodingEncoder CANONICAL_EXTENSION = new ExtEncodingEncoder(); @@ -270,7 +272,13 @@ private EncodeResult encodeWithCtx(DType dtype, Object data, EncodeContext ctx) /// The stats Rust keeps on the node of an array it compressed, here a child of the segment's root /// as much as the root itself ([CachedNodeStats]). private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { - if (!(dtype instanceof DType.Primitive || dtype instanceof DType.Bool)) { + if (dtype instanceof DType.Bool) { + if (!root.equals(EncodingId.VORTEX_BOOL) || !(data instanceof boolean[] bools)) { + return NodeStats.NONE; + } + return CachedNodeStats.ofBool(bools.length, SimdOperationsSupport.preferred().allEqual(bools)); + } + if (!(dtype instanceof DType.Primitive)) { return NodeStats.NONE; } MinMax bounds = dtype instanceof DType.Primitive p && !p.ptype().isFloating() ? ZoneMapStats.of(dtype, data) : null; @@ -336,6 +344,9 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { if (dtype instanceof DType.Decimal decimal) { return encodeDecimal(decimal, data, ctx); } + if (dtype instanceof DType.Bool && data instanceof boolean[] bools) { + return encodeBool(bools, ctx); + } if (!(dtype instanceof DType.Primitive)) { return spliceResult(findPrimitiveEncoding(dtype, data, ctx.excluded()), dtype, data, ctx); } @@ -344,6 +355,18 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { sampleSize -> primitiveBytes(dtype, sampleSize)); } + /// A bool array stays `vortex.bool` unless it is one value and that is smaller as a + /// `vortex.constant`: Rust's `ConstantScheme` only wins when the constant is smaller than the packed + /// bits, which it is from 17 rows (3 bytes) on. A zone table's three flags, or a few rows, stay bool. + private EncodeResult encodeBool(boolean[] bools, EncodeContext ctx) { + boolean constantPays = (bools.length + 7) / 8 > CONSTANT_BOOL_BYTES; + if (constantPays && !ctx.sample() && !ctx.excluded().contains(EncodingId.VORTEX_CONSTANT) + && SimdOperationsSupport.preferred().allEqual(bools)) { + return new ConstantEncodingEncoder().encode(DTypes.BOOL, bools, ctx); + } + return CANONICAL_BOOL.encode(DTypes.BOOL, bools, ctx); + } + /// Rust's constant check for a decimal array, run in the compressor before any scheme and never /// on a sample: when every valid value is equal the array is one `vortex.constant`, wrapped in /// `vortex.masked` with the validity when some row is null, and kept only if that is smaller than diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java index 2522d884f..374b8440f 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/ZoneAccumulatorTest.java @@ -58,6 +58,20 @@ void allNullZoneHasNullMinMaxAndCountsItsNulls() { assertThat(values(result, 2)).isEqualTo(new long[]{3}); } + @Test + void columnsWithoutNullsCarryNoValidity() { + // Given two zones, each with a value: Rust's builders stay AllValid, so nothing is masked + ZoneAccumulator sut = new ZoneAccumulator(DTypes.I64.withNullable(true)); + sut.add(new NullableData(new long[]{1, 2, 3}, new boolean[]{true, false, true}), 3); + + // When + sut.finish(); + StructData result = sut.tableData(); + + // Then + assertThat(result.fieldArrays()).noneMatch(NullableData.class::isInstance); + } + @Test void floatsSkipNanInMinMaxAndCountIt() { // Given a leading NaN — Rust's max/min skip NaN and record them in nan_count @@ -69,8 +83,8 @@ void floatsSkipNanInMinMaxAndCountIt() { StructData result = sut.tableData(); // Then — fields: max, min, nan_count, null_count - assertThat((double[]) ((NullableData) result.fieldArrays().get(0)).values()).containsExactly(2.0); - assertThat((double[]) ((NullableData) result.fieldArrays().get(1)).values()).containsExactly(-1.0); + assertThat((double[]) result.fieldArrays().get(0)).containsExactly(2.0); + assertThat((double[]) result.fieldArrays().get(1)).containsExactly(-1.0); assertThat(values(result, 2)).isEqualTo(new long[]{2}); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java index 269307e2f..a1016993d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CachedNodeStatsTest.java @@ -133,4 +133,22 @@ void withRootStats_fillsAnEmptyRootWithTheCachedStats() { assertThat(result.rootNode().stats()) .isEqualTo(CachedNodeStats.of(DTypes.I64, EncodingId.VORTEX_PRIMITIVE, MIN, MAX, 100, 0)); } + + @Test + void ofBool_constantCarriesOnlyTheFlag() { + // Given / When + NodeStats result = CachedNodeStats.ofBool(20_000, true); + + // Then + assertThat(result).isEqualTo(new NodeStats(null, null, true, null, null, null, null, null)); + } + + @Test + void ofBool_otherwiseCarriesTheFlagAndThePackedSize() { + // Given / When + NodeStats result = CachedNodeStats.ofBool(17, false); + + // Then + assertThat(result).isEqualTo(new NodeStats(null, null, false, null, null, null, 3L, null)); + } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java index 2a0edd1c6..6fb71fd10 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java @@ -190,6 +190,20 @@ void dictChildren_carryTheirBoundsAndNullCount() { assertThat(distinct.nullCount()).isZero(); } + @Test + void constantBoolColumn_staysABoolUntilTheConstantIsSmaller() { + // Given — 16 rows pack into 2 bytes, as small as a constant; 17 rows pack into 3 + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult small = sut.encode(DTypes.BOOL, new boolean[16], ctx(3)); + EncodeResult large = sut.encode(DTypes.BOOL, new boolean[17], ctx(3)); + + // Then + assertThat(small.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_BOOL); + assertThat(large.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_CONSTANT); + } + @Test void nullableIntegerValidity_isABoolChildCarryingTheValidCount() { // Given