diff --git a/CHANGELOG.md b/CHANGELOG.md index c175630b8..3f7c3b177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The zone table of a column holds Rust's shape: no validity or mask on a column without nulls, and a constant bool stays a `vortex.bool` until the constant is smaller. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- Text columns stay `vortex.varbinview`, Rust's canonical form, unless a scheme is smaller, and a `vortex.varbin` is only chosen for binary. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index c482c6bd8..227942f9b 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -177,7 +177,8 @@ patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`. `NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol -table) and the zone-table segments of string columns, which wait for Rust's `vortex.varbinview` canonical form, are not covered yet +table) are not covered, nor is the zone table of a column whose few bounds Rust's trained table compresses and ours does not. +Text is `vortex.varbinview` unless a scheme beats its size (`vortex.varbin` is a binary-only scheme, as in Rust), and that node carries `is_constant`, `null_count` and the size of its views ([#583](https://github.com/dfa1/vortex-java/issues/583)). ### Decode shape diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java index 63d7bc33b..22d0ce3f0 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NodeStatsInteropIntegrationTest.java @@ -189,9 +189,10 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception { for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11, 12}) { assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c)); } - // the string columns' zone tables wait for Rust's varbinview canonical form (#583) + // the zone table of `high` holds three bounds, which Rust's trained symbol table compresses and ours + // does not: the string children depend on the trained table, which is not ported byte for byte for (int c = 0; c < expectedColumns.size(); c++) { - if (c == 5 || c == 6 || c == 13) { + if (c == 6) { continue; } assertThat(zoneTable(resultColumns.get(c))).as("zone table of column %d", c) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java index 5f4bd0a37..017e99a5b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CachedNodeStats.java @@ -89,6 +89,16 @@ private static Boolean isConstant(ProtoScalarValue min, ProtoScalarValue max, lo return min != null && max != null && Arrays.equals(min.encode(), max.encode()); } + /// The stats of a canonical text array, a `vortex.varbinview`: the flag, the null count and the size + /// of its views and out-of-line bytes. + /// + /// @param nullCount the number of null rows + /// @param viewBytes the exact size of the canonical array in bytes + /// @return the node's stats + static NodeStats ofView(long nullCount, long viewBytes) { + return new NodeStats(null, null, notConstant(nullCount), null, null, nullCount, viewBytes, null); + } + private static Boolean notConstant(long nullCount) { if (nullCount > 0) { return null; diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java index 92fe801a9..b4eb079e3 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/CascadingCompressor.java @@ -31,6 +31,7 @@ public final class CascadingCompressor { private static final DType OFFSETS = DTypes.I64; private static final EncodingEncoder CANONICAL_PRIMITIVE = new PrimitiveEncodingEncoder(); + private static final EncodingEncoder CANONICAL_VARBINVIEW = new VarBinViewEncodingEncoder(); private static final EncodingEncoder CANONICAL_BOOL = new BoolEncodingEncoder(); /// Bytes of a `vortex.constant` bool: Rust keeps a bool array whose packed bits are no larger. private static final int CONSTANT_BOOL_BYTES = 2; @@ -214,6 +215,14 @@ private static boolean eligible(EncodingEncoder enc, DType dtype, Object data, E if (!enc.accepts(dtype) || ctx.excluded().contains(enc.encodingId())) { return false; } + // Rust's canonical text is `vortex.varbinview`, the baseline every scheme must beat, and + // its `VarBinScheme` re-encodes binary only + if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { + EncodingId id = enc.encodingId(); + if (id.equals(EncodingId.VORTEX_VARBINVIEW) || id.equals(EncodingId.VORTEX_VARBIN) && dtype instanceof DType.Utf8) { + return false; + } + } return !(data instanceof NullableData) || enc.acceptsNullable(dtype); } @@ -278,6 +287,13 @@ private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) { } return CachedNodeStats.ofBool(bools.length, SimdOperationsSupport.preferred().allEqual(bools)); } + if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { + if (!root.equals(EncodingId.VORTEX_VARBINVIEW)) { + return NodeStats.NONE; + } + long nulls = data instanceof NullableData nullable ? nullable.nullCount() : 0L; + return CachedNodeStats.ofView(nulls, viewBytes(data, Integer.MAX_VALUE, 0L)); + } if (!(dtype instanceof DType.Primitive)) { return NodeStats.NONE; } @@ -330,7 +346,7 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) { // practice). if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) { return competeAndEncode(dtype, new ArrayAndStats(dtype, data, StatsOptions.defaults()), ctx, - sampleSize -> Long.MAX_VALUE); + sampleSize -> viewBytes(data, sampleSize, ctx.sampleSeed())); } // Remaining non-primitives (extension types, List, ...): find the accepting encoding and @@ -509,6 +525,21 @@ && totalBytes(result) >= uncompressedBytes(dtype, data)) { return result; } + /// The size of the canonical text array of the `sampleSize` rows a trial encodes: Rust's + /// `vortex.varbinview` is a 16-byte view per row plus the bytes of each value too long to be + /// inlined in it. + private static long viewBytes(Object data, int sampleSize, long seed) { + Object sample = sampleSize < dataLength(data) ? stratifiedSample(data, sampleSize, seed) : data; + if (sample instanceof NullableData nullable) { + sample = nullable.values(); + } + long total = 0; + for (byte[] row : VarBinBytes.toByteArrays(sample)) { + total += 16 + (row.length > 12 ? row.length : 0); + } + return total; + } + private static long totalBytes(EncodeResult result) { long total = 0; for (EncodedBuffer buffer : result.encodedBuffers()) { @@ -767,6 +798,10 @@ private EncodingEncoder findPrimitiveEncoding(DType dtype, Object data, Set CANONICAL_PRIMITIVE; case DType.Bool _ -> CANONICAL_BOOL; - case DType.Utf8 _, DType.Binary _ -> CANONICAL_VARBIN; + case DType.Utf8 _, DType.Binary _ -> CANONICAL_VARBINVIEW; case DType.Decimal _ -> CANONICAL_DECIMAL; case DType.Extension _ -> CANONICAL_EXTENSION; default -> throw new UnsupportedOperationException("no encoder for dtype: " + dtype); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java index 23f75f6fe..5aab1242f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java @@ -70,7 +70,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { for (int b = 0; b < dataBufs.length; b++) { dataBufs[b] = arena.allocate(bufferSizes.get(b)); } - MemorySegment viewsBuf = arena.allocate(n > 0 ? (long) n * VIEW_SIZE : 1); + MemorySegment viewsBuf = arena.allocate((long) n * VIEW_SIZE, VIEW_SIZE); int bufferIndex = 0; long dataOffset = 0; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java index 6fb71fd10..4c60f40b0 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/CascadingCompressorTest.java @@ -248,6 +248,45 @@ void nullableFloatValidity_isABoolChildCarryingTheValidCount() { } + @Nested + class TextCanonical { + + @Test + void incompressibleUtf8_staysTheCanonicalVarBinView() { + // Given — distinct random 8-byte strings: no scheme beats the 16-byte views + Random random = new Random(3); + String[] values = new String[2048]; + for (int i = 0; i < values.length; i++) { + values[i] = Long.toString(random.nextLong() & 0xffffffffL, 36); + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(DTypes.UTF8, values, ctx(0)); + + // Then — Rust's `VarBinScheme` re-encodes binary only + assertThat(result.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_VARBINVIEW); + } + + @Test + void varBinViewRoot_carriesTheFlagNullCountAndViewSize() { + // Given + Random random = new Random(3); + String[] values = new String[2048]; + for (int i = 0; i < values.length; i++) { + values[i] = Long.toString(random.nextLong() & 0xffffffffL, 36); + } + CascadingCompressor sut = new CascadingCompressor(ALL_CODECS); + + // When + EncodeResult result = sut.encode(DTypes.UTF8, values, ctx(0)); + + // Then — a 16-byte view per row, values of up to 12 bytes inline + assertThat(result.rootNode().stats()) + .isEqualTo(new NodeStats(null, null, false, null, null, 0L, 2048L * 16, null)); + } + } + @Nested class RoundTrip { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java index 1aa6dc9c2..a6b206026 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java @@ -54,6 +54,19 @@ void accepts_binary_true() { assertThat(result).isTrue(); } + @Test + void encode_noRows_hasAnEmptyViewsBuffer() { + // Given + VarBinViewEncodingEncoder sut = new VarBinViewEncodingEncoder(); + + // When + EncodeResult result = sut.encode(DTypes.UTF8, new String[0], EncodeTestHelper.testCtx()); + + // Then — Rust checks the views buffer against the row count, so an empty array has none + assertThat(result.encodedBuffers()).hasSize(1); + assertThat(result.encodedBuffers().getFirst().data().byteSize()).isZero(); + } + @Test void accepts_primitive_false() { // Given