From 358dda557b5c27a17c472d0a2a1c8b58f41f8805 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Fri, 6 Nov 2026 20:12:26 +0100 Subject: [PATCH] fix(writer): text is a vortex.varbinview unless a scheme is smaller, as Rust's canonical form (#583) Rust's canonical text array is vortex.varbinview and the baseline every string scheme must beat; VarBinScheme re-encodes binary only. The Utf8 competition now scores against the views' size, leaves vortex.varbin and vortex.varbinview out as candidates for Utf8 (varbinview for binary too), and falls back to varbinview. The node carries is_constant, null_count and the size of its views, and an empty array's views buffer is empty, which Rust checks against the row count. The string zone tables now match vortex-jni, bar one whose three bounds Rust's trained FSST table compresses. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011fRbYsrGtpKHp9APhU8qDV --- CHANGELOG.md | 1 + docs/compatibility.md | 3 +- .../NodeStatsInteropIntegrationTest.java | 5 ++- .../vortex/writer/encode/CachedNodeStats.java | 10 +++++ .../writer/encode/CascadingCompressor.java | 39 ++++++++++++++++++- .../encode/VarBinViewEncodingEncoder.java | 2 +- .../encode/CascadingCompressorTest.java | 39 +++++++++++++++++++ .../encode/VarBinViewEncodingEncoderTest.java | 13 +++++++ 8 files changed, 106 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c175630b8..3f7c3b177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The zone table of a column holds Rust's shape: no validity or mask on a column without nulls, and a constant bool stays a `vortex.bool` until the constant is smaller. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- Text columns stay `vortex.varbinview`, Rust's canonical form, unless a scheme is smaller, and a `vortex.varbin` is only chosen for binary. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index c482c6bd8..227942f9b 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -177,7 +177,8 @@ patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. `NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol -table) and the zone-table segments of string columns, which wait for Rust's `vortex.varbinview` canonical form, are not covered yet +table) are not covered, nor is the zone table of a column whose few bounds Rust's trained table compresses and ours does not. +Text is `vortex.varbinview` unless a scheme beats its size (`vortex.varbin` is a binary-only scheme, as in Rust), and that node carries `is_constant`, `null_count` and the size of its views ([#583](https://github.com/dfa1/vortex-java/issues/583)). ### Decode shape diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java index 63d7bc33b..22d0ce3f0 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -189,9 +189,10 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11, 12}) { assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c)); } - // the string columns' zone tables wait for Rust's varbinview canonical form (#583) + // the zone table of `high` holds three bounds, which Rust's trained symbol table compresses and ours + // does not: the string children depend on the trained table, which is not ported byte for byte for (int c = 0; c < expectedColumns.size(); c++) { - if (c == 5 || c == 6 || c == 13) { + if (c == 6) { continue; } assertThat(zoneTable(resultColumns.get(c))).as("zone table of column %d", c) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java index 5f4bd0a37..017e99a5b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java @@ -89,6 +89,16 @@ private static Boolean isConstant(ProtoScalarValue min, ProtoScalarValue max, lo return min != null && max != null && Arrays.equals(min.encode(), max.encode()); } + /// The stats of a canonical text array, a `vortex.varbinview`: the flag, the null count and the size + /// of its views and out-of-line bytes. + /// + /// @param nullCount the number of null rows + /// @param viewBytes the exact size of the canonical array in bytes + /// @return the node's stats + static NodeStats ofView(long nullCount, long viewBytes) { + return new NodeStats(null, null, notConstant(nullCount), null, null, nullCount, viewBytes, null); + } + private static Boolean notConstant(long nullCount) { if (nullCount > 0) { return null; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index 92fe801a9..b4eb079e3 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -31,6 +31,7 @@ public final class CascadingCompressor { private static final DType OFFSETS = DTypes.I64; private static final EncodingEncoder CANONICAL_PRIMITIVE = new PrimitiveEncodingEncoder(); + private static final EncodingEncoder CANONICAL_VARBINVIEW = new VarBinViewEncodingEncoder(); private static final EncodingEncoder CANONICAL_BOOL = new BoolEncodingEncoder(); /// Bytes of a `vortex.constant` bool: Rust keeps a bool array whose packed bits are no larger. private static final int CONSTANT_BOOL_BYTES = 2; @@ -214,6 +215,14 @@ private static boolean eligible(EncodingEncoder enc, DType dtype, Object data, E if (!enc.accepts(dtype) || ctx.excluded().contains(enc.encodingId())) { return false; } + // Rust's canonical text is `vortex.varbinview`, the baseline every scheme must beat, and + // its `VarBinScheme` re-encodes binary only + if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { + EncodingId id = enc.encodingId(); + if (id.equals(EncodingId.VORTEX_VARBINVIEW) || id.equals(EncodingId.VORTEX_VARBIN) && dtype instanceof DType.Utf8) { + return false; + } + } return !(data instanceof NullableData) || enc.acceptsNullable(dtype); } @@ -278,6 +287,13 @@ private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { } return CachedNodeStats.ofBool(bools.length, SimdOperationsSupport.preferred().allEqual(bools)); } + if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { + if (!root.equals(EncodingId.VORTEX_VARBINVIEW)) { + return NodeStats.NONE; + } + long nulls = data instanceof NullableData nullable ? nullable.nullCount() : 0L; + return CachedNodeStats.ofView(nulls, viewBytes(data, Integer.MAX_VALUE, 0L)); + } if (!(dtype instanceof DType.Primitive)) { return NodeStats.NONE; } @@ -330,7 +346,7 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { // practice). if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { return competeAndEncode(dtype, new ArrayAndStats(dtype, data, StatsOptions.defaults()), ctx, - sampleSize -> Long.MAX_VALUE); + sampleSize -> viewBytes(data, sampleSize, ctx.sampleSeed())); } // Remaining non-primitives (extension types, List, ...): find the accepting encoding and @@ -509,6 +525,21 @@ && totalBytes(result) >= uncompressedBytes(dtype, data)) { return result; } + /// The size of the canonical text array of the `sampleSize` rows a trial encodes: Rust's + /// `vortex.varbinview` is a 16-byte view per row plus the bytes of each value too long to be + /// inlined in it. + private static long viewBytes(Object data, int sampleSize, long seed) { + Object sample = sampleSize < dataLength(data) ? stratifiedSample(data, sampleSize, seed) : data; + if (sample instanceof NullableData nullable) { + sample = nullable.values(); + } + long total = 0; + for (byte[] row : VarBinBytes.toByteArrays(sample)) { + total += 16 + (row.length > 12 ? row.length : 0); + } + return total; + } + private static long totalBytes(EncodeResult result) { long total = 0; for (EncodedBuffer buffer : result.encodedBuffers()) { @@ -767,6 +798,10 @@ private EncodingEncoder findPrimitiveEncoding(DType dtype, Object data, Set CANONICAL_PRIMITIVE; case DType.Bool _ -> CANONICAL_BOOL; - case DType.Utf8 _, DType.Binary _ -> CANONICAL_VARBIN; + case DType.Utf8 _, DType.Binary _ -> CANONICAL_VARBINVIEW; case DType.Decimal _ -> CANONICAL_DECIMAL; case DType.Extension _ -> CANONICAL_EXTENSION; default -> throw new UnsupportedOperationException("no encoder for dtype: " + dtype); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java index 23f75f6fe..5aab1242f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java @@ -70,7 +70,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { for (int b = 0; b < dataBufs.length; b++) { dataBufs[b] = arena.allocate(bufferSizes.get(b)); } - MemorySegment viewsBuf = arena.allocate(n > 0 ? (long) n * VIEW_SIZE : 1); + MemorySegment viewsBuf = arena.allocate((long) n * VIEW_SIZE, VIEW_SIZE); int bufferIndex = 0; long dataOffset = 0; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java index 6fb71fd10..4c60f40b0 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java @@ -248,6 +248,45 @@ void nullableFloatValidity_isABoolChildCarryingTheValidCount() { } + @Nested + class TextCanonical { + + @Test + void incompressibleUtf8_staysTheCanonicalVarBinView() { + // Given — distinct random 8-byte strings: no scheme beats the 16-byte views + Random random = new Random(3); + String[] values = new String[2048]; + for (int i = 0; i < values.length; i++) { + values[i] = Long.toString(random.nextLong() & 0xffffffffL, 36); + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(DTypes.UTF8, values, ctx(0)); + + // Then — Rust's `VarBinScheme` re-encodes binary only + assertThat(result.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_VARBINVIEW); + } + + @Test + void varBinViewRoot_carriesTheFlagNullCountAndViewSize() { + // Given + Random random = new Random(3); + String[] values = new String[2048]; + for (int i = 0; i < values.length; i++) { + values[i] = Long.toString(random.nextLong() & 0xffffffffL, 36); + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(DTypes.UTF8, values, ctx(0)); + + // Then — a 16-byte view per row, values of up to 12 bytes inline + assertThat(result.rootNode().stats()) + .isEqualTo(new NodeStats(null, null, false, null, null, 0L, 2048L * 16, null)); + } + } + @Nested class RoundTrip { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java index 1aa6dc9c2..a6b206026 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java @@ -54,6 +54,19 @@ void accepts_binary_true() { assertThat(result).isTrue(); } + @Test + void encode_noRows_hasAnEmptyViewsBuffer() { + // Given + VarBinViewEncodingEncoder sut = new VarBinViewEncodingEncoder(); + + // When + EncodeResult result = sut.encode(DTypes.UTF8, new String[0], EncodeTestHelper.testCtx()); + + // Then — Rust checks the views buffer against the row count, so an empty array has none + assertThat(result.encodedBuffers()).hasSize(1); + assertThat(result.encodedBuffers().getFirst().data().byteSize()).isZero(); + } + @Test void accepts_primitive_false() { // Given