diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7c3b177..194147ee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - The zone table of a column holds Rust's shape: no validity or mask on a column without nulls, and a constant bool stays a `vortex.bool` until the constant is smaller. ([#583](https://github.com/dfa1/vortex-java/issues/583)) - Text columns stay `vortex.varbinview`, Rust's canonical form, unless a scheme is smaller, and a `vortex.varbin` is only chosen for binary. ([#583](https://github.com/dfa1/vortex-java/issues/583)) +- A text column keeps its dict layout past 32,768 distinct values, starting a new dictionary at 65,535 entries or 1 MiB as Rust does, and is admitted by its first 8,192 rows. ([#650](https://github.com/dfa1/vortex-java/issues/650)) - The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636)) - A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646)) - `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665)) diff --git a/docs/compatibility.md b/docs/compatibility.md index 227942f9b..f8b6feee0 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -125,7 +125,7 @@ decimals ([#430](https://github.com/dfa1/vortex-java/pull/430)) and nulls in nul | `vortex.varbinview` | `VarBinViewEncodingDecoder` | `VarBinViewEncodingEncoder` | ✅ | ✅ | Utf8, Binary | | `vortex.alp` | `AlpEncodingDecoder` | `AlpEncodingEncoder` | ✅ | ✅ | F64, F32 | | `vortex.alprd` | `AlpRdEncodingDecoder` | `AlpRdEncodingEncoder` | ✅ | ✅ | F64, F32 | -| `vortex.dict` | `DictEncodingDecoder` | `DictEncodingEncoder` | ✅ | ✅ | Primitive, Utf8/Binary. Writes `DictMetadata` + children `[codes, values]`, both cascaded (a float pool goes through ALP). With `globalDict` on (the default) the writer's data cascade never dictionary-encodes integers: Rust's file writer excludes `IntDictScheme` there and dictionary-encodes suitable columns once, as a layout. A column first needs under half distinct values in its first batch, at most 2,048 of them (32,768 for strings), and a type the dict layout reads (no `I8`/`U8`/`I16`/`U16`/`F16`/`F32`); Rust has no such cap and skips only above half. A non-null primitive column that passes then gets the layout only when the cascade, with the integer dictionary back in, picks `vortex.dict` for its first rows: Rust's `DictStrategy` probe. Rust probes exactly the first 8192 rows across batches; Java probes the first batch's first 8192, so a first batch under 8192 rows can decide differently. Nullable and string columns skip the probe and decide by the cardinality check alone, which admits some nullable integer columns Rust bit-packs. With it off (the CSV and Parquet importers stream), integer chunks keep their per-chunk dictionary. Decode also accepts the pre-0.14.5 Java shape (1-byte metadata, `[values, codes]`) so older Java-written files stay readable; that shape was never readable by Rust | +| `vortex.dict` | `DictEncodingDecoder` | `DictEncodingEncoder` | ✅ | ✅ | Primitive, Utf8/Binary. Writes `DictMetadata` + children `[codes, values]`, both cascaded (a float pool goes through ALP). With `globalDict` on (the default) the writer's data cascade never dictionary-encodes integers: Rust's file writer excludes `IntDictScheme` there and dictionary-encodes suitable columns once, as a layout. A column first needs a type the dict layout reads (no `I8`/`U8`/`I16`/`U16`/`F16`/`F32`) and under half distinct values in its first batch, at most 2,048 of them for numbers (Rust has no such cap and skips only above half); a text column has no cap either: it needs under half of its first 8,192 rows distinct, and starts a new dictionary whenever the open one reaches 65,535 entries or 1 MiB, as Rust's dict layout does. A non-null primitive column that passes then gets the layout only when the cascade, with the integer dictionary back in, picks `vortex.dict` for its first rows: Rust's `DictStrategy` probe. Rust probes exactly the first 8192 rows across batches; Java probes the first batch's first 8192, so a first batch under 8192 rows can decide differently. Nullable and string columns skip the probe and decide by the distinct-count check alone, which admits some nullable integer columns Rust bit-packs. With it off (the CSV and Parquet importers stream), integer chunks keep their per-chunk dictionary. Decode also accepts the pre-0.14.5 Java shape (1-byte metadata, `[values, codes]`) so older Java-written files stay readable; that shape was never readable by Rust | | `vortex.sparse` | `SparseEncodingDecoder` | `SparseEncodingEncoder` | ✅ | ✅ | Primitive, Bool, Utf8/Binary (decode); Primitive (encode) | | `vortex.sequence` | `SequenceEncodingDecoder` | `SequenceEncodingEncoder` | ✅ | ✅ | Primitive (encode: integer only, as Rust; decode also float) | | `vortex.struct` | `StructEncodingDecoder` | `StructEncodingEncoder` | ✅ | ✅ | Struct | diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/DictLayoutStringInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/DictLayoutStringInteropIntegrationTest.java new file mode 100644 index 000000000..b5f17f885 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/DictLayoutStringInteropIntegrationTest.java @@ -0,0 +1,165 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.reader.ReadRegistry; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Which text columns get Rust's dict layout, and how many dictionaries it makes (#650). Rust's +/// `DictStrategy` admits a column by its first 8192 rows (not more than half distinct) and starts a +/// new dictionary whenever the open one reaches 65,535 entries or 1 MiB, so a column of any total +/// cardinality can keep its dict layout. +class DictLayoutStringInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final int CHUNK = 65_536; + + static { + NativeLoader.loadJni(); + } + + @ParameterizedTest(name = "pool of {0} values") + @ValueSource(ints = {100, 3_000, 20_000, 150_000}) + void admitsTheColumnsRustAdmits(int poolSize, @TempDir Path tmp) throws Exception { + // Given — 3 chunks drawing at random from a pool: the pool decides how distinct the first 8192 rows are + Random r = new Random(11); + String[] pool = new String[poolSize]; + for (int i = 0; i < pool.length; i++) { + pool[i] = "https://example.org/path/" + Long.toString(r.nextLong() & 0xffffffffffL, 36); + } + String[] values = new String[CHUNK * 3]; + for (int i = 0; i < values.length; i++) { + values[i] = pool[r.nextInt(pool.length)]; + } + + // When + Path java = writeJava(tmp, values); + Path jni = writeJni(tmp, values); + + // Then + assertThat(dictLayouts(EncodingTrees.of(java))).isEqualTo(dictLayouts(EncodingTrees.of(jni))); + } + + @Test + void startsANewDictionaryWhenTheOpenOneIsFull(@TempDir Path tmp) throws Exception { + // Given — 8 chunks, each drawing 3000 values from a window that slides 2500 values on: well under + // half distinct in the first 8192 rows, but 23000 values of 96 bytes overall, past 1 MiB + Random r = new Random(5); + int chunks = 8; + String[] pool = new String[2500 * chunks + 3000]; + for (int i = 0; i < pool.length; i++) { + pool[i] = "https://example.org/some/long/path/segment/" + Long.toString(r.nextLong() & 0xffffffffffL, 36) + + "/index.html?utm=" + i; + } + String[] values = new String[CHUNK * chunks]; + for (int i = 0; i < values.length; i++) { + values[i] = pool[(i / CHUNK) * 2500 + r.nextInt(3000)]; + } + + // When + Path java = writeJava(tmp, values); + Path jni = writeJni(tmp, values); + + // Then — Rust fills one dictionary and starts the next, and so do we + assertThat(dictLayouts(EncodingTrees.of(jni))).isEqualTo(2); + assertThat(dictLayouts(EncodingTrees.of(java))).isEqualTo(2); + assertThat(readAll(java)).containsExactly(values); + } + + private static Path writeJava(Path tmp, String[] values) throws IOException { + Path file = tmp.resolve("java.vortex"); + DType.Struct schema = new DType.Struct(List.of(ColumnName.of("url")), List.of(DTypes.UTF8), false); + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var w = io.github.dfa1.vortex.writer.VortexWriter.create(ch, schema, WriteOptions.defaults())) { + for (int from = 0; from < values.length; from += CHUNK) { + String[] chunk = new String[CHUNK]; + System.arraycopy(values, from, chunk, 0, CHUNK); + w.writeChunk(Map.of(ColumnName.of("url"), chunk)); + } + } + return file; + } + + private static Path writeJni(Path tmp, String[] values) throws Exception { + Path file = tmp.resolve("jni.vortex"); + Schema arrow = new Schema(List.of(Field.notNullable("url", ArrowType.Utf8.INSTANCE))); + try (VortexWriter w = VortexWriter.builder(SESSION, file.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build()) { + for (int from = 0; from < values.length; from += CHUNK) { + try (VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + VarCharVector v = (VarCharVector) root.getVector("url"); + root.allocateNew(); + for (int i = 0; i < CHUNK; i++) { + v.setSafe(i, values[from + i].getBytes(StandardCharsets.UTF_8)); + } + root.setRowCount(CHUNK); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema as = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, as); + w.writeBatch(array.memoryAddress(), as.memoryAddress()); + } + } + } + } + return file; + } + + /// How many dict layouts a tree holds: a `vortex.dict` whose child is a layout, not an array encoding. + private static long dictLayouts(List tree) { + long count = 0; + for (int i = 0; i + 1 < tree.size(); i++) { + if (tree.get(i).strip().equals("vortex.dict") && tree.get(i + 1).strip().equals("vortex.flat")) { + count++; + } + } + return count; + } + + private static String[] readAll(Path file) throws IOException { + List out = new ArrayList<>(); + try (var reader = VortexReader.open(file, ReadRegistry.loadAll()); + var iter = reader.scan(ScanOptions.columns(ColumnName.of("url")))) { + while (iter.hasNext()) { + try (var chunk = iter.next()) { + VarBinArray strings = (VarBinArray) chunk.column(ColumnName.of("url")); + for (long i = 0; i < strings.length(); i++) { + out.add(strings.getString(i)); + } + } + } + } + return out.toArray(String[]::new); + } +} diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/DictValuesPoolCompressionIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/DictValuesPoolCompressionIntegrationTest.java index 67550a4fc..8c25610b3 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/DictValuesPoolCompressionIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/DictValuesPoolCompressionIntegrationTest.java @@ -94,12 +94,12 @@ void globalDictUtf8_valuesPool_isFsstCompressed_andRoundTrips(@TempDir Path tmp) @Test void utf8_cardinalityAbovePriorCap_admittedToGlobalDict(@TempDir Path tmp) throws IOException { - // Given — 8 000 distinct values: above the old 2048 numeric-era cap (so previously demoted - // to per-chunk), below the 32768 Utf8 cap, with >50% dedup so the ratio gate admits it. + // Given — 3 000 distinct values: above the old 2048 numeric-era cap (so previously demoted + // to per-chunk), with under half of the first 8192 rows distinct, which is Rust's gate. Path file = tmp.resolve("high_card.vortex"); int rowsPerChunk = 20_000; int chunkCount = 4; - int distinct = 8_000; + int distinct = 3_000; List expected = new ArrayList<>(); try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); @@ -113,10 +113,10 @@ void utf8_cardinalityAbovePriorCap_admittedToGlobalDict(@TempDir Path tmp) throw } // Then — the column is stored as one global dictionary (a DICT layout node), not a per-chunk - // fallback. Before the type-aware cap a 8 000-distinct column produced no DICT layout. + // fallback. try (var vf = VortexReader.open(file, ReadRegistry.loadAll())) { assertThat(hasDictLayout(InspectorTree.build(vf).root())) - .as("8 000-distinct Utf8 column admitted to the global dictionary") + .as("3 000-distinct Utf8 column admitted to the global dictionary") .isTrue(); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java index 2b82a7930..ffd29c2ca 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java @@ -46,7 +46,14 @@ final class DictColumnState { /// the codes' raw width, and on U8 codes Sparse's estimate beat bit-packing where on Rust's /// U16 codes it does not (taxi Airport_fee). static final PType CODES_PTYPE = PType.U16; - static final int GLOBAL_DICT_MAX_CARDINALITY_UTF8 = 32_768; + + /// Rust's `DictLayoutConstraints` defaults for a Utf8/Binary dictionary: at most `u16::MAX` + /// entries and 1 MiB of views plus out-of-line bytes. A column that reaches either starts a new + /// dictionary, so a text column's cardinality is not a reason to give up the dict layout. + static final int MAX_VARBIN_ENTRIES = 65_535; + static final long MAX_VARBIN_DICT_BYTES = 1L << 20; + private static final int VIEW_BYTES = 16; + private static final int MAX_INLINED_BYTES = 12; private static final int INDEX_MIN_CAPACITY = 64; // The null entry's key in valueToCode: identity-compared, so no real value can collide with it. @@ -82,6 +89,8 @@ final class DictColumnState { private final List chunkStatsMax = new ArrayList<>(); private final List chunkStatsSum = new ArrayList<>(); private long codeArrayBytes; + // Rust's `dict_bytes()`: a 16-byte view per entry plus the bytes of each value too long to inline. + private long dictBytes; DictColumnState(DType dtype) { this.dtype = dtype; @@ -200,28 +209,27 @@ Object uniques() { }; } - /// Ingests one chunk into this candidate column's cardinality-bounded dict state (ADR 0021): dedups - /// each valid value into the shared value-to-code map, appends a per-chunk `short[]` code array, - /// and captures the chunk's row/null counts and min/max/sum stats before the raw array is - /// discarded. Null slots buffer the null entry's code, assigned at the first one. + /// Ingests one primitive chunk into this candidate column's cardinality-bounded dict state (ADR + /// 0021): dedups each valid value into the shared value-to-code map, appends a per-chunk `short[]` + /// code array, and captures the chunk's row/null counts and min/max/sum stats before the raw array + /// is discarded. Null slots buffer the null entry's code, assigned at the first one. /// /// Returns `false` — without mutating this state — the moment a new distinct value would push the /// map past [#GLOBAL_DICT_MAX_CARDINALITY]; the caller then demotes the column to per-chunk /// encoding. This moves the cap check from `close()` to a continuous, mid-file guard so a column /// whose distinct set grows past the cap never accumulates unbounded memory first. /// - /// @param data the chunk data (primitive array, `String[]`, `byte[][]`, or a [NullableData] wrapper) + /// @param data the chunk data (a primitive array or a [NullableData] wrapper) /// @return `true` if the chunk was ingested within the cardinality cap; `false` if the column /// must be demoted boolean ingestDictChunk(Object data) { boolean nullableData = data instanceof NullableData; Object values = nullableData ? ((NullableData) data).values() : data; boolean[] validity = nullableData ? ((NullableData) data).validity() : null; - int len = varBin ? ((Object[]) values).length : primitiveArrayLen(values, ptype); - int cap = dictMaxCardinality(varBin); + int len = primitiveArrayLen(values, ptype); + int cap = GLOBAL_DICT_MAX_CARDINALITY; int startSize = valueToCode.size(); int startNullCode = nullCode; - Object[] strings = varBin ? (Object[]) values : null; // One pass: insert new values and build the per-chunk code array. Ingest stays // all-or-nothing — a chunk that would breach the cap rolls back the entries it added (the @@ -229,46 +237,95 @@ boolean ingestDictChunk(Object data) { // earlier shape ran a non-mutating counting pass first to get that atomicity, which cost a // second boxed map probe for every row in the file. short[] codes = new short[len]; - boolean statsStrings = strings != null && !binary; + // Probe the unboxed bits index and box only when inserting a value the dictionary has not + // seen before (at most `cap` times for the whole file). The boxed key itself still comes + // from the source array, so I32 vs I64 and NaN/-0.0 keying stay exactly what a + // Double/Long-keyed map gave. + long[] bits = rawBits(values, ptype, len); + for (int i = 0; i < len; i++) { + if (validity != null && !validity[i]) { + if (nullCode < 0 && !addNullEntry(cap)) { + rollbackTo(startSize, startNullCode); + return false; + } + codes[i] = (short) nullCode; + continue; + } + int code = bitsIndex.get(bits[i]); + if (code < 0) { + if (valueToCode.size() == cap) { + rollbackTo(startSize, startNullCode); + return false; + } + code = valueToCode.size(); + valueToCode.put(readPrimitiveElement(values, ptype, i), code); + bitsIndex.put(bits[i], code); + } + codes[i] = (short) code; + } + + MinMax mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); + record(codes, validity, len, mm != null ? mm.min() : null, mm != null ? mm.max() : null, + PrimitiveEncodingEncoder.sumStat(ptype, values)); + return true; + } + + /// Ingests the leading rows of a text chunk that fit this dictionary under Rust's constraints + /// (`DictLayoutConstraints`: [#MAX_VARBIN_ENTRIES] entries, [#MAX_VARBIN_DICT_BYTES] bytes), as + /// Rust's bytes dict builder does: the first entry is always admitted, and the row that needs an + /// entry that no longer fits ends the prefix. The caller starts a new dictionary with the rest. + /// + /// @param data the chunk data (`String[]`, `byte[][]`, or a [NullableData] wrapper) + /// @return how many leading rows were ingested, `0` when the first row already needs a new + /// dictionary + int ingestVarBinChunk(Object data) { + boolean nullableData = data instanceof NullableData; + Object[] strings = (Object[]) (nullableData ? ((NullableData) data).values() : data); + boolean[] validity = nullableData ? ((NullableData) data).validity() : null; + int len = strings.length; + short[] codes = new short[len]; + boolean statsStrings = !binary; String minString = null; String maxString = null; int chunkStamp = ++stamp; if (statsStrings && seenInChunk == null) { seenInChunk = new int[16]; } - if (strings != null) { - for (int i = 0; i < len; i++) { - // Nullable Utf8/Binary keeps a real null at invalid positions (ChunkImpl.adaptUtf8, - // adaptBinary); either way the slot is null and points at the null entry. - if ((validity != null && !validity[i]) || strings[i] == null) { - if (nullCode < 0 && !addNullEntry(cap)) { - rollbackTo(startSize, startNullCode); - return false; - } - codes[i] = (short) nullCode; - continue; + int consumed = len; + for (int i = 0; i < len; i++) { + // Nullable Utf8/Binary keeps a real null at invalid positions (ChunkImpl.adaptUtf8, + // adaptBinary); either way the slot is null and points at the null entry. + if ((validity != null && !validity[i]) || strings[i] == null) { + if (nullCode < 0 && !addVarBinNullEntry()) { + consumed = i; + break; } - Object v = dictKey(strings[i]); - Integer code = valueToCode.get(v); - if (code == null) { - if (valueToCode.size() == cap) { - rollbackTo(startSize, startNullCode); - return false; - } - code = valueToCode.size(); - valueToCode.put(v, code); + codes[i] = (short) nullCode; + continue; + } + Object v = dictKey(strings[i]); + Integer code = valueToCode.get(v); + if (code == null) { + long added = entryBytes(v); + boolean first = valueToCode.isEmpty(); + if (!first && (valueToCode.size() >= MAX_VARBIN_ENTRIES || dictBytes + added > MAX_VARBIN_DICT_BYTES)) { + consumed = i; + break; } - codes[i] = code.shortValue(); - // Min/max depend on the distinct values only, so compare each code once per - // chunk instead of twice per row (UTF-8 order compare was 4% of a cascading write). - if (statsStrings) { - if (code >= seenInChunk.length) { - // Grown with the dictionary, not sized to the cap: a low-cardinality column - // must not hold 128 KiB the retained-bytes budget cannot see. - seenInChunk = Arrays.copyOf(seenInChunk, Math.max(code + 1, seenInChunk.length * 2)); - } + code = valueToCode.size(); + valueToCode.put(v, code); + dictBytes += added; + } + codes[i] = code.shortValue(); + // Min/max depend on the distinct values only, so compare each code once per chunk + // instead of twice per row (UTF-8 order compare was 4% of a cascading write). + if (statsStrings) { + if (code >= seenInChunk.length) { + // Grown with the dictionary, not sized to the cap: a low-cardinality column + // must not hold 128 KiB the retained-bytes budget cannot see. + seenInChunk = Arrays.copyOf(seenInChunk, Math.max(code + 1, seenInChunk.length * 2)); } - if (statsStrings && seenInChunk[code] != chunkStamp) { + if (seenInChunk[code] != chunkStamp) { seenInChunk[code] = chunkStamp; String s = (String) v; if (minString == null || Utf8Order.compare(s, minString) < 0) { @@ -279,57 +336,49 @@ boolean ingestDictChunk(Object data) { } } } - } else { - // Primitive path: probe the unboxed bits index and box only when inserting a value the - // dictionary has not seen before (at most `cap` times for the whole file). The boxed - // key itself still comes from the source array, so I32 vs I64 and NaN/-0.0 keying stay - // exactly what a Double/Long-keyed map gave. - long[] bits = rawBits(values, ptype, len); - for (int i = 0; i < len; i++) { - if (validity != null && !validity[i]) { - if (nullCode < 0 && !addNullEntry(cap)) { - rollbackTo(startSize, startNullCode); - return false; - } - codes[i] = (short) nullCode; - continue; - } - int code = bitsIndex.get(bits[i]); - if (code < 0) { - if (valueToCode.size() == cap) { - rollbackTo(startSize, startNullCode); - return false; - } - code = valueToCode.size(); - valueToCode.put(readPrimitiveElement(values, ptype, i), code); - bitsIndex.put(bits[i], code); - } - codes[i] = (short) code; - } } + if (consumed == 0) { + return 0; + } + if (consumed < len) { + codes = Arrays.copyOf(codes, consumed); + validity = validity != null ? Arrays.copyOf(validity, consumed) : null; + } + // No min/max for Binary, matching the per-chunk VarBin path (VarBinEncodingEncoder). + MinMax mm = binary ? null : VarBinEncodingEncoder.encodeMinMax(minString, maxString); + record(codes, validity, consumed, mm != null ? mm.min() : null, mm != null ? mm.max() : null, null); + return consumed; + } + + // The bytes a new entry adds to the dictionary: its view, and the value itself when too long to inline. + private long entryBytes(Object key) { + int length = key instanceof String s ? s.getBytes(StandardCharsets.UTF_8).length : ((ByteBuffer) key).remaining(); + return length > MAX_INLINED_BYTES ? VIEW_BYTES + length : VIEW_BYTES; + } + + // Gives the null entry the next code under the text constraints (Rust's `encode_null`); false + // when it no longer fits. + private boolean addVarBinNullEntry() { + if (!valueToCode.isEmpty() + && (valueToCode.size() >= MAX_VARBIN_ENTRIES || dictBytes + VIEW_BYTES > MAX_VARBIN_DICT_BYTES)) { + return false; + } + nullCode = valueToCode.size(); + valueToCode.put(NULL_KEY, nullCode); + dictBytes += VIEW_BYTES; + return true; + } + private void record(short[] codes, boolean[] validity, int len, ProtoScalarValue min, ProtoScalarValue max, + ProtoScalarValue sum) { chunkCodes.add(codes); chunkValidity.add(validity); chunkRowCounts.add((long) len); chunkNullCounts.add(validity != null ? VortexWriter.countNulls(validity) : 0L); - if (binary) { - // No min/max for Binary, matching the per-chunk VarBin path (VarBinEncodingEncoder). - chunkStatsMin.add(null); - chunkStatsMax.add(null); - chunkStatsSum.add(null); - } else if (varBin) { - MinMax mm = VarBinEncodingEncoder.encodeMinMax(minString, maxString); - chunkStatsMin.add(mm != null ? mm.min() : null); - chunkStatsMax.add(mm != null ? mm.max() : null); - chunkStatsSum.add(null); - } else { - MinMax mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); - chunkStatsMin.add(mm != null ? mm.min() : null); - chunkStatsMax.add(mm != null ? mm.max() : null); - chunkStatsSum.add(PrimitiveEncodingEncoder.sumStat(ptype, values)); - } + chunkStatsMin.add(min); + chunkStatsMax.add(max); + chunkStatsSum.add(sum); codeArrayBytes += 2L * len; - return true; } // Gives the null entry the next code, as Rust's dict builder does at the first null; false @@ -478,23 +527,23 @@ static boolean isVarBinDictCandidate(Object[] data) { /// The ratio denominator stays the total row count (not the valid-row count), matching the /// per-chunk encoders' convention that null placeholders occupy a row like any other value. /// + /// Rust's `StringDictScheme` skips a sample with more than half its values distinct, and has no + /// cap on the count: a column that outgrows one dictionary starts another. + /// /// @param data the Utf8 (`String[]`) or Binary (`byte[][]`) values; null elements at invalid /// positions are skipped /// @param validity per-row validity bitmap, or `null` meaning every row is valid - /// @return `true` if the column's distinct valid-value count is low enough to dictionary-encode + /// @return `true` if the sample's distinct valid-value count is low enough to dictionary-encode static boolean isVarBinDictCandidate(Object[] data, boolean[] validity) { if (data.length == 0) { return false; } - var seen = HashSet.newHashSet(Math.min(GLOBAL_DICT_MAX_CARDINALITY_UTF8, data.length)); + var seen = HashSet.newHashSet(data.length); for (int i = 0; i < data.length; i++) { if ((validity != null && !validity[i]) || data[i] == null) { continue; } seen.add(varBinKey(data[i])); - if (seen.size() > GLOBAL_DICT_MAX_CARDINALITY_UTF8) { - return false; - } } if (seen.isEmpty()) { return false; @@ -600,11 +649,6 @@ private static int countDistinctCapped(long[] bits, int n, boolean[] validity) { return distinct; } - // The global-dict cardinality cap for a column, by whether it is Utf8/Binary (see the constants above). - private static int dictMaxCardinality(boolean varBin) { - return varBin ? GLOBAL_DICT_MAX_CARDINALITY_UTF8 : GLOBAL_DICT_MAX_CARDINALITY; - } - // The dictionary key of one Utf8/Binary row. Utf8 keys are always Strings, a UTF-8 row decoded, // so a column written partly as String[] and partly as byte[][] still codes each value once. private Object dictKey(Object value) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/Repartitioner.java b/writer/src/main/java/io/github/dfa1/vortex/writer/Repartitioner.java index 3fd30d371..aef3ae99a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/Repartitioner.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/Repartitioner.java @@ -91,6 +91,15 @@ static Object head(Object data, int rows) { return concat(List.of(new Piece(data, 0, rows, 0))); } + /// A batch without its first `from` rows, in the batch's carrier shape. + /// + /// @param data a batch of a column [#supports(DType)] accepts + /// @param from how many leading rows to drop, less than the batch's row count + /// @return the remaining rows + static Object tail(Object data, int from) { + return concat(List.of(new Piece(data, from, (int) VortexWriter.arrayLength(data) - from, 0))); + } + /// Whether sizing a batch of `data` reads its values, as for strings and binaries -- the /// only batches worth [#sizePieces(Object,long,long)] ahead of time. /// diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index 1d345f188..fa9d00fa2 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -171,7 +171,10 @@ public final class VortexWriter implements Closeable { // layout. When a column's distinct set would exceed the cap mid-file, it demotes immediately. private final Set dictCandidates = new LinkedHashSet<>(); private final Map dictStates = new LinkedHashMap<>(); - private final Map dictColRefs = new LinkedHashMap<>(); + // Dictionaries a text column has already filled to Rust's constraints (a new one starts when it + // reaches them); the open one is in dictStates. + private final Map> dictRuns = new LinkedHashMap<>(); + private final Map> dictColRefs = new LinkedHashMap<>(); // Code-array bytes retained per global-dict-candidate column, and their running sum; together // they guard the aggregate memory budget (dictRetainedBudget) as a secondary safety net (ADR // 0021) so no set of mis-detected columns can pin the heap. Now that codes cost ~2 B/row instead @@ -489,7 +492,7 @@ private static long rowCountForValidation(ColumnName colName, Object data) { } } - private static long arrayLength(Object data) { + static long arrayLength(Object data) { return switch (data) { case byte[] a -> a.length; case short[] a -> a.length; @@ -667,12 +670,14 @@ public void writeChunk(Map columns) throws IOException { if (colDtype instanceof DType.Primitive p) { candidate = DictColumnState.isDictCandidate(p.ptype(), values, validity); } else if (colDtype instanceof DType.Utf8 || colDtype instanceof DType.Binary) { - // Rust's dict layout admits Primitive | Utf8 | Binary (dict_layout_supported). - candidate = DictColumnState.isVarBinDictCandidate((Object[]) values, validity); + // Rust's dict layout admits Primitive | Utf8 | Binary (dict_layout_supported), by what + // its first 8192 rows do: not more than half distinct. The cascade probe stays with + // primitives: ours picks OnPair for strings where Rust's StringDictScheme wins. + int probeRows = Math.min(Repartitioner.BLOCK_ROWS, ((Object[]) values).length); + candidate = DictColumnState.isVarBinDictCandidate( + Arrays.copyOf((Object[]) values, probeRows), + validity != null ? Arrays.copyOf(validity, probeRows) : null); } - // ponytail: only primitives take Rust's probe. Strings pick OnPair where Rust's - // StringDictScheme wins, so for them the cardinality check above still matches - // Rust more often. if (candidate && colDtype instanceof DType.Primitive) { candidate = probeCompressesToDict(colDtype, data); } @@ -688,8 +693,13 @@ public void writeChunk(Map columns) throws IOException { // the per-chunk stats now (before the raw array is discarded). If a new distinct // value would push the map past GLOBAL_DICT_MAX_CARDINALITY, ingestDictChunk returns // false and we demote the column immediately (mid-file cap breach, ADR 0021). - DictColumnState state = dictStates.get(colName); - boolean admitted = state.ingestDictChunk(data); + boolean admitted; + if (colDtype instanceof DType.Primitive) { + admitted = dictStates.get(colName).ingestDictChunk(data); + } else { + ingestVarBinChunk(colName, colDtype, data); + admitted = true; + } if (!admitted) { // Cap breached by this chunk: demote (replaying the already-buffered chunks per // -chunk) then write this chunk — which ingest rejected without buffering — too. @@ -697,7 +707,7 @@ public void writeChunk(Map columns) throws IOException { appendChunk(colName, colDtype, data); } else { long before = dictRetainedBytes.getOrDefault(colName, 0L); - long after = state.retainedBytes(); + long after = retainedBytes(colName); dictRetainedTotal += after - before; dictRetainedBytes.put(colName, after); if (dictRetainedTotal > dictRetainedBudget) { @@ -1249,15 +1259,24 @@ private void flushZoneMaps() throws IOException { // Dict-encoded columns (one zone per code chunk). MIN/MAX/SUM come from each chunk's logical // values (computed at dict-build time); NULL_COUNT always. Matches Rust, whose zone-map // stats are computed on the logical column dtype, independent of the dict encoding. - for (Map.Entry e : dictColRefs.entrySet()) { - DictColRef ref = e.getValue(); + for (Map.Entry> e : dictColRefs.entrySet()) { DType colDtype = columnDtype(e.getKey()); DType minMaxDtype = ZoneMapStatCodec.zoneMinMaxDtype(colDtype); - long[] nullCounts = ref.chunkNullCounts().stream().mapToLong(Long::longValue).toArray(); - emitZoneMap(e.getKey(), minMaxDtype, - ref.chunkStatsMin(), ref.chunkStatsMax(), - ZoneMapStatCodec.zoneSumDtype(colDtype), ref.chunkStatsSum(), nullCounts, - ref.chunkRowCounts().stream().mapToLong(Long::longValue).toArray()); + List mins = new ArrayList<>(); + List maxs = new ArrayList<>(); + List sums = new ArrayList<>(); + List nulls = new ArrayList<>(); + List rows = new ArrayList<>(); + for (DictColRef ref : e.getValue()) { + mins.addAll(ref.chunkStatsMin()); + maxs.addAll(ref.chunkStatsMax()); + sums.addAll(ref.chunkStatsSum()); + nulls.addAll(ref.chunkNullCounts()); + rows.addAll(ref.chunkRowCounts()); + } + emitZoneMap(e.getKey(), minMaxDtype, mins, maxs, ZoneMapStatCodec.zoneSumDtype(colDtype), sums, + nulls.stream().mapToLong(Long::longValue).toArray(), + rows.stream().mapToLong(Long::longValue).toArray()); } } @@ -1472,12 +1491,23 @@ private ByteBuffer buildLayout() { for (int c = 0; c < colCount; c++) { ColumnName colName = schema.fieldNames().get(c); - DictColRef ref = dictColRefs.get(colName); - if (ref != null) { - int dictLayout = buildDictColLayout(fbb, ref); - colLayouts[c] = wrapZoneMap(fbb, colName, dictLayout, ref.totalRows()); + List refs = dictColRefs.get(colName); + if (refs != null) { + // one dictionary is the column's layout; several are a chunked layout of them + int[] dictLayouts = new int[refs.size()]; + long colRows = 0; + for (int i = 0; i < dictLayouts.length; i++) { + dictLayouts[i] = buildDictColLayout(fbb, refs.get(i)); + colRows += refs.get(i).totalRows(); + } + int dictLayout = dictLayouts[0]; + if (dictLayouts.length > 1) { + dictLayout = FbsLayout.createFbsLayout(fbb, LAYOUT_CHUNKED, colRows, 0, + FbsLayout.createChildrenVector(fbb, dictLayouts), 0); + } + colLayouts[c] = wrapZoneMap(fbb, colName, dictLayout, colRows); if (totalRows == 0) { - totalRows = ref.totalRows(); + totalRows = colRows; } } else { List chunks = colChunks.get(colName); @@ -1553,6 +1583,33 @@ private static byte[] buildDictLayoutMetaBytes(PType codePType, boolean nullable // orchestration that needs this writer's segment-writing machinery (writeSegment, colChunks, // schema, options). + /// Feeds one text chunk to the column's open dictionary; whatever does not fit under Rust's + /// constraints closes it and starts the next one (`DictStrategy`: "when the dict constraints are + /// hit, a new dictionary is created"), so a text column never gives up its dict layout for its + /// cardinality. + private void ingestVarBinChunk(ColumnName colName, DType colDtype, Object data) { + Object rest = data; + while (true) { + DictColumnState state = dictStates.get(colName); + int consumed = state.ingestVarBinChunk(rest); + long length = arrayLength(rest); + if (consumed == length) { + return; + } + dictRuns.computeIfAbsent(colName, _ -> new ArrayList<>()).add(state); + dictStates.put(colName, new DictColumnState(colDtype)); + rest = consumed == 0 ? rest : Repartitioner.tail(rest, consumed); + } + } + + private long retainedBytes(ColumnName colName) { + long total = dictStates.get(colName).retainedBytes(); + for (DictColumnState run : dictRuns.getOrDefault(colName, List.of())) { + total += run.retainedBytes(); + } + return total; + } + /// Demotes the largest-retained global-dict candidate columns to per-chunk encoding, one at a /// time (largest first), until the aggregate retained bytes fall back under the budget. Demoting /// the largest column frees the most memory per eviction, so the fewest columns lose their shared @@ -1584,32 +1641,42 @@ private void evictLargestDictColumnsUntilUnderBudget() throws IOException { /// @param colName the column being demoted from global-dict to per-chunk encoding /// @throws IOException if writing a flushed segment fails private void demoteDictColumn(ColumnName colName) throws IOException { - DictColumnState state = dictStates.remove(colName); + DictColumnState open = dictStates.remove(colName); + List states = new ArrayList<>(dictRuns.getOrDefault(colName, List.of())); + dictRuns.remove(colName); dictCandidates.remove(colName); Long freed = dictRetainedBytes.remove(colName); if (freed != null) { dictRetainedTotal -= freed; } - if (state == null) { + if (open == null) { return; } + states.add(open); DType colDtype = schema.fieldTypes().get(schema.fieldNames().indexOf(colName)); - Object[] inverse = state.buildInverseMap(); - for (int c = 0; c < state.chunkCount(); c++) { - appendChunk(colName, colDtype, state.reconstructChunk(inverse, c)); + for (DictColumnState state : states) { + Object[] inverse = state.buildInverseMap(); + for (int c = 0; c < state.chunkCount(); c++) { + appendChunk(colName, colDtype, state.reconstructChunk(inverse, c)); + } } } private void flushDictColumns() throws IOException { for (ColumnName colName : dictCandidates) { - DictColumnState state = dictStates.get(colName); - if (state == null || state.chunkCount() == 0 || state.cardinality() == 0) { - continue; + List states = new ArrayList<>(dictRuns.getOrDefault(colName, List.of())); + if (dictStates.get(colName) != null) { + states.add(dictStates.get(colName)); } - if (state.varBin()) { - writeGlobalDictVarBinColumn(colName, state); - } else { - writeGlobalDictColumn(colName, state); + for (DictColumnState state : states) { + if (state.chunkCount() == 0 || state.cardinality() == 0) { + continue; + } + if (state.varBin()) { + writeGlobalDictVarBinColumn(colName, state); + } else { + writeGlobalDictColumn(colName, state); + } } } } @@ -1676,8 +1743,8 @@ private void writeGlobalDictCodes(ColumnName colName, DictColumnState state, int codesSegIdxes.add(writeSegment(codesDtype, last)); codesRowCounts.add((long) ((short[]) last).length); } - dictColRefs.put(colName, new DictColRef(valuesSegIdx, state.cardinality(), codesSegIdxes, codesRowCounts, - state.chunkRowCounts(), state.chunkNullCounts(), + dictColRefs.computeIfAbsent(colName, _ -> new ArrayList<>()).add(new DictColRef(valuesSegIdx, state.cardinality(), + codesSegIdxes, codesRowCounts, state.chunkRowCounts(), state.chunkNullCounts(), state.chunkStatsMin(), state.chunkStatsMax(), state.chunkStatsSum(), false)); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java index 558bb2653..d78903347 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java @@ -13,7 +13,6 @@ import java.util.stream.Stream; import static io.github.dfa1.vortex.writer.DictColumnState.GLOBAL_DICT_MAX_CARDINALITY; -import static io.github.dfa1.vortex.writer.DictColumnState.GLOBAL_DICT_MAX_CARDINALITY_UTF8; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -102,12 +101,10 @@ static Stream utf8DictCandidateCases() { arguments("empty", new String[0], false), arguments("ratio exactly 50%", new String[]{"a", "b", "a", "b"}, false), arguments("ratio under 50%", new String[]{"a", "b", "a", "b", "a"}, true), - // Utf8 uses the raised type-aware cap (#299), not the numeric 2048. - arguments("Utf8 admitted above numeric cap", - distinctStrings(GLOBAL_DICT_MAX_CARDINALITY + 1), true), - arguments("cardinality at Utf8 MAX", distinctStrings(GLOBAL_DICT_MAX_CARDINALITY_UTF8), true), - arguments("cardinality over Utf8 MAX", - distinctStrings(GLOBAL_DICT_MAX_CARDINALITY_UTF8 + 1), false)); + // no cardinality cap as for numbers: Rust skips a sample only above half distinct, and a + // column that outgrows one dictionary starts another + arguments("Utf8 admitted above numeric cap", repeated(GLOBAL_DICT_MAX_CARDINALITY + 1), true), + arguments("cardinality of a full dictionary", repeated(70_000), true)); } @ParameterizedTest(name = "{0}") @@ -178,11 +175,11 @@ void ingestDictChunk_binary_dedupsByContentAndReconstructs() { boolean[] validity = {true, true, false, true, true}; // When - boolean admitted = sut.ingestDictChunk(new NullableData(values, validity)); + int admitted = sut.ingestVarBinChunk(new NullableData(values, validity)); Object result = sut.reconstructChunk(sut.buildInverseMap(), 0); // Then — two values plus the null entry, and demotion rebuilds the original rows (null kept null) - assertThat(admitted).isTrue(); + assertThat(admitted).isEqualTo(5); assertThat(sut.cardinality()).isEqualTo(3); assertThat((byte[][]) sut.uniques()).isDeepEqualTo(new byte[][]{{1, 2}, {3}, null}); assertThat(result).isInstanceOfSatisfying(NullableData.class, nd -> { @@ -291,9 +288,9 @@ void ingestDictChunkReportsPerChunkStringMinMax() { String emoji = new String(Character.toChars(0x1F600)); // When - sut.ingestDictChunk(new String[]{"b", "a", "b", "A", emoji, "a"}); - sut.ingestDictChunk(new String[]{"c", null, "c", "d"}); - sut.ingestDictChunk(new String[]{"b", "a", "b"}); + sut.ingestVarBinChunk(new String[]{"b", "a", "b", "A", emoji, "a"}); + sut.ingestVarBinChunk(new String[]{"c", null, "c", "d"}); + sut.ingestVarBinChunk(new String[]{"b", "a", "b"}); // Then assertThat(decodeStrings(sut.chunkStatsMin())).containsExactly("a", "c", "a"); @@ -322,4 +319,77 @@ private static String[] distinctStrings(int distinct) { } return a; } + + // A column of `distinct` values each repeated three times: under half distinct, whatever its size. + private static String[] repeated(int distinct) { + String[] out = new String[distinct * 3]; + for (int i = 0; i < out.length; i++) { + out[i] = "value-" + (i / 3); + } + return out; + } + + // ── ingestVarBinChunk (Rust's dict constraints) ────────────────────────────── + + @Test + void ingestVarBinChunk_stopsAtTheEntryThatNoLongerFits() { + // Given — a dictionary filled to u16::MAX entries + DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + String[] full = new String[DictColumnState.MAX_VARBIN_ENTRIES]; + for (int i = 0; i < full.length; i++) { + full[i] = "v" + i; + } + sut.ingestVarBinChunk(full); + + // When — known values still fit, the first new one ends the prefix + int consumed = sut.ingestVarBinChunk(new String[]{"v0", "v1", "new", "v2"}); + + // Then + assertThat(consumed).isEqualTo(2); + assertThat(sut.cardinality()).isEqualTo(DictColumnState.MAX_VARBIN_ENTRIES); + assertThat(sut.chunkRowCounts()).containsExactly((long) full.length, 2L); + } + + @Test + void ingestVarBinChunk_countsViewsAndOutOfLineBytesAgainstTheByteLimit() { + // Given — 20-byte values cost a 16-byte view plus themselves: 36 bytes, so 29127 fit in 1 MiB + DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + String[] values = new String[40_000]; + for (int i = 0; i < values.length; i++) { + values[i] = String.format("%020d", i); + } + + // When + int consumed = sut.ingestVarBinChunk(values); + + // Then + assertThat(consumed).isEqualTo((int) (DictColumnState.MAX_VARBIN_DICT_BYTES / 36)); + } + + @Test + void ingestVarBinChunk_admitsTheFirstEntryWhateverItsSize() { + // Given — one value past the byte limit on its own + DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + String huge = "x".repeat((int) DictColumnState.MAX_VARBIN_DICT_BYTES + 1); + + // When + int consumed = sut.ingestVarBinChunk(new String[]{huge, "other"}); + + // Then — the caller makes progress, and the next value starts a new dictionary + assertThat(consumed).isEqualTo(1); + } + + @Test + void ingestVarBinChunk_aFullDictionaryTakesNothingNew() { + // Given + DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + sut.ingestVarBinChunk(new String[]{"x".repeat((int) DictColumnState.MAX_VARBIN_DICT_BYTES + 1)}); + + // When + int consumed = sut.ingestVarBinChunk(new String[]{"new"}); + + // Then + assertThat(consumed).isZero(); + assertThat(sut.chunkCount()).isEqualTo(1); + } }