From 06f83dff226b1b187bc283ee692c1d938efbc934 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Thu, 12 Nov 2026 22:06:30 +0100 Subject: [PATCH] Write struct columns as Rust's vortex.struct layout Each field of a struct column is its own chunked, repartitioned, dict-able column under a validity child when the struct is nullable; fields are zoned only while no struct above them is nullable, as in Rust. Fixes #724 Co-Authored-By: Claude Sonnet 5 --- AGENTS.md | 3 + CHANGELOG.md | 3 + .../StructLayoutInteropIntegrationTest.java | 205 ++++++++++++++++++ .../github/dfa1/vortex/writer/LayoutTree.java | 81 ++++--- .../dfa1/vortex/writer/StructColumns.java | 80 +++++++ .../dfa1/vortex/writer/VortexWriter.java | 101 ++++++++- .../dfa1/vortex/writer/ZoneMapWriter.java | 10 +- 7 files changed, 438 insertions(+), 45 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/StructLayoutInteropIntegrationTest.java create mode 100644 writer/src/main/java/io/github/dfa1/vortex/writer/StructColumns.java diff --git a/AGENTS.md b/AGENTS.md index 34077a325..539808bcf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -218,6 +218,9 @@ top-level field, Rust 0.90's per-batch fold, `FileStatsAccumulator`) blobs elsew Layout tree: `Struct → Zoned(Stats) → Chunked → [Flat, Flat, ...]` - **Flat** single encoded segment · **Chunked** sequence of Flats · **Struct** one child/column +- **Struct** also stands for a struct *column*: a validity child when nullable, then one child per field, each its own + Chunked/Zoned/Dict layout (`StructColumns` flattens fields to `parent/field` leaf columns). Rust zones the fields only + while no struct above them is nullable - **Zoned** wraps a child with a per-zone stats table for zone-map pruning. By default the writer emits Rust's `vortex.zoned`: fixed 8192-row zones independent of `writeChunk` batches (`ZoneAccumulator`), Rust's per-dtype aggregate set, no zone sum (#447). Targets before diff --git a/CHANGELOG.md b/CHANGELOG.md index f9b48e0ff..549538c74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- A struct column is written as Rust's `vortex.struct` layout, each field chunked and compressed on its own, so files are smaller and a reader can prune one field. ([#724](https://github.com/dfa1/vortex-java/issues/724)) + ## [0.20.0] — 2026-11-10 ### Changed diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/StructLayoutInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/StructLayoutInteropIntegrationTest.java new file mode 100644 index 000000000..111965232 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/StructLayoutInteropIntegrationTest.java @@ -0,0 +1,205 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.layout.Layout; +import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullableData; +import io.github.dfa1.vortex.writer.encode.StructData; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A struct column is Rust's `vortex.struct` layout: each field is its own chunked, compressed +/// column, under a validity child when the struct is nullable, and zoned only while no struct above +/// it is nullable. Java wrote one flat array (under `vortex.masked` when nullable): 2.2% bigger on +/// Raincloud `mnist`, and no field could be pruned. The same column through vortex-jni and +/// vortex-java must lay out the same tree, and vortex-jni must read Java's file. +class StructLayoutInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final ColumnName STRUCT = ColumnName.of("s"); + // past one chunk of every field, so the layout is chunked, and past 8192 rows for zone maps + private static final int ROWS = 300_000; + + static { + NativeLoader.loadJni(); + } + + @ParameterizedTest(name = "nullable struct: {0}") + @ValueSource(booleans = {true, false}) + void javaLaysOutTheStructAsJniDoes(boolean nullable, @TempDir Path tmp) throws Exception { + // Given + Path jniFile = tmp.resolve("jni.vortex"); + Path javaFile = tmp.resolve("java.vortex"); + writeWithJni(jniFile, nullable); + + // When + writeWithJava(javaFile, nullable); + + // Then + assertThat(shape(javaFile)).isEqualTo(shape(jniFile)); + } + + @ParameterizedTest(name = "nullable struct: {0}") + @ValueSource(booleans = {true, false}) + void jniReadsTheStructJavaWrote(boolean nullable, @TempDir Path tmp) throws Exception { + // Given + Path javaFile = tmp.resolve("java.vortex"); + writeWithJava(javaFile, nullable); + + // When + List result = jniReadsField(javaFile); + + // Then + List expected = new ArrayList<>(); + for (int i = 0; i < ROWS; i++) { + expected.add(nullable && isNull(i) ? null : (long) i); + } + assertThat(result).isEqualTo(expected); + } + + private static boolean isNull(int row) { + return row % 7 == 0; + } + + /// The layout tree without segment indices or metadata: Rust's order of segments is no layout. + private static String shape(Path file) throws IOException { + try (VortexReader reader = VortexReader.open(file)) { + StringBuilder out = new StringBuilder(); + shape(reader.layout(), 0, out); + return out.toString(); + } + } + + private static void shape(Layout layout, int depth, StringBuilder out) { + out.append(" ".repeat(depth)).append(layout.layoutId()).append(' ').append(layout.rowCount()).append('\n'); + for (Layout child : layout.children()) { + shape(child, depth + 1, out); + } + } + + private static void writeWithJava(Path path, boolean nullable) throws IOException { + DType.Struct struct = new DType.Struct( + List.of(ColumnName.of("bytes"), ColumnName.of("path"), ColumnName.of("n")), + List.of(new DType.Binary(true), new DType.Utf8(true), DTypes.I64), nullable); + DType.Struct schema = new DType.Struct(List.of(STRUCT), List.of(struct), false); + byte[][] bytes = new byte[ROWS][]; + String[] paths = new String[ROWS]; + long[] n = new long[ROWS]; + boolean[] valid = new boolean[ROWS]; + for (int i = 0; i < ROWS; i++) { + bytes[i] = ("bytes" + i * 31L).getBytes(StandardCharsets.UTF_8); + paths[i] = "p" + i % 10; + n[i] = i; + valid[i] = !(nullable && isNull(i)); + } + boolean[] allValid = new boolean[ROWS]; + Arrays.fill(allValid, true); + StructData fields = new StructData(List.of( + new NullableData(bytes, allValid), new NullableData(paths, allValid.clone()), n)); + Object column = fields; + if (nullable) { + column = new NullableData(fields, valid); + } + try (FileChannel channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + io.github.dfa1.vortex.writer.VortexWriter writer = + io.github.dfa1.vortex.writer.VortexWriter.create(channel, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(STRUCT, column)); + } + } + + private static void writeWithJni(Path path, boolean nullable) throws Exception { + Field struct = new Field("s", nullable ? FieldType.nullable(ArrowType.Struct.INSTANCE) + : FieldType.notNullable(ArrowType.Struct.INSTANCE), + List.of(Field.nullable("bytes", ArrowType.Binary.INSTANCE), Field.nullable("path", ArrowType.Utf8.INSTANCE), + Field.notNullable("n", new ArrowType.Int(64, true)))); + Schema schema = new Schema(List.of(struct)); + try (VortexWriter writer = VortexWriter.builder(SESSION, path.toAbsolutePath().toUri().toString(), schema, + ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) { + StructVector s = (StructVector) root.getVector("s"); + s.allocateNew(); + VarBinaryVector bytes = (VarBinaryVector) s.getChild("bytes"); + VarCharVector paths = (VarCharVector) s.getChild("path"); + BigIntVector n = (BigIntVector) s.getChild("n"); + for (int i = 0; i < ROWS; i++) { + bytes.setSafe(i, ("bytes" + i * 31L).getBytes(StandardCharsets.UTF_8)); + paths.setSafe(i, ("p" + i % 10).getBytes(StandardCharsets.UTF_8)); + n.setSafe(i, i); + if (nullable && isNull(i)) { + s.setNull(i); + } else { + s.setIndexDefined(i); + } + } + bytes.setValueCount(ROWS); + paths.setValueCount(ROWS); + n.setValueCount(ROWS); + s.setValueCount(ROWS); + root.setRowCount(ROWS); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, arrowSchema); + writer.writeBatch(array.memoryAddress(), arrowSchema.memoryAddress()); + } + } + } + + /// Field `n` of every row through vortex-jni, `null` where the struct is null. + private static List jniReadsField(Path file) throws IOException { + List out = new ArrayList<>(); + DataSource ds = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString()); + Scan scan = ds.scan(ScanOptions.of()); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) { + while (reader.loadNextBatch()) { + StructVector s = (StructVector) reader.getVectorSchemaRoot().getVector("s"); + BigIntVector n = (BigIntVector) s.getChild("n"); + for (int i = 0; i < s.getValueCount(); i++) { + out.add(s.isNull(i) ? null : n.get(i)); + } + } + } + } + return out; + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java b/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java index a174f5ff8..8e8677d85 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java @@ -14,6 +14,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -22,7 +23,7 @@ /// /// The root is a struct with one child per column: a chunked layout of flat segments (the flat /// itself for one chunk), or the column's dict layout (several in a chunked layout), each wrapped in -/// its zone map when it has one. +/// its zone map when it has one. A struct column is a struct layout of its fields in turn. final class LayoutTree { // Indices into the footer's layout_specs list, in the order specs() writes them @@ -49,49 +50,65 @@ static int specs(FbsBuilder fbb, LayoutId zoned) { /// The serialized layout tree of a file with `schema`'s columns. /// - /// @param chunks each column's written chunks, for the columns not in `dicts` - /// @param dicts the columns written as global dictionaries - /// @param zoneMaps the columns that carry a zone map - static ByteBuffer build(DType.Struct schema, Map> chunks, - Map> dicts, Map zoneMaps) { + /// @param rows the file's row count + /// @param chunks each leaf column's written chunks, for the leaves not in `dicts` + /// @param validities the written validity of each nullable struct, by its key + /// @param dicts the leaf columns written as global dictionaries + /// @param zoneMaps the leaf columns that carry a zone map + static ByteBuffer build(DType.Struct schema, long rows, Map> chunks, + Map validities, Map> dicts, + Map zoneMaps) { var fbb = new FbsBuilder(256); int colCount = schema.fieldNames().size(); int[] colLayouts = new int[colCount]; - long totalRows = 0; - for (int c = 0; c < colCount; c++) { - ColumnName colName = schema.fieldNames().get(c); - List refs = dicts.get(colName); - if (refs != null) { - long colRows = rows(refs); - colLayouts[c] = zoned(fbb, zoneMaps.get(colName), dictLayout(fbb, refs), colRows); - if (totalRows == 0) { - totalRows = colRows; - } - } else { - List colChunks = chunks.get(colName); - long colRows = 0; - int[] flats = new int[colChunks.size()]; - for (int i = 0; i < colChunks.size(); i++) { - ChunkRef cr = colChunks.get(i); - int segV = FbsLayout.createSegmentsVector(fbb, new long[]{cr.segIdx()}); - flats[i] = FbsLayout.createFbsLayout(fbb, FLAT, cr.rowCount(), 0, 0, segV); - colRows += cr.rowCount(); - } - colLayouts[c] = zoned(fbb, zoneMaps.get(colName), chunked(fbb, flats, colRows), colRows); - if (totalRows == 0) { - totalRows = colRows; - } - } + colLayouts[c] = column(fbb, schema.fieldNames().get(c), schema.fieldTypes().get(c), rows, chunks, + validities, dicts, zoneMaps); } int rootChildV = FbsLayout.createChildrenVector(fbb, colLayouts); - int rootLayout = FbsLayout.createFbsLayout(fbb, STRUCT, totalRows, 0, rootChildV, 0); + int rootLayout = FbsLayout.createFbsLayout(fbb, STRUCT, rows, 0, rootChildV, 0); FbsLayout.finishFbsLayoutBuffer(fbb, rootLayout); return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } + /// A column's layout: a struct is Rust's struct layout, a validity child (when nullable) then one + /// child per field; any other column is its zoned, chunked or dict layout. + private static int column(FbsBuilder fbb, ColumnName key, DType dtype, long rows, + Map> chunks, Map validities, + Map> dicts, Map zoneMaps) { + if (StructColumns.isLayoutStruct(dtype)) { + DType.Struct struct = (DType.Struct) dtype; + List children = new ArrayList<>(); + if (struct.nullable()) { + ChunkRef validity = validities.get(key); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{validity.segIdx()}); + children.add(FbsLayout.createFbsLayout(fbb, FLAT, validity.rowCount(), 0, 0, segV)); + } + for (int i = 0; i < struct.fieldNames().size(); i++) { + children.add(column(fbb, StructColumns.fieldKey(key, struct.fieldNames().get(i)), + struct.fieldTypes().get(i), rows, chunks, validities, dicts, zoneMaps)); + } + int childV = FbsLayout.createChildrenVector(fbb, children.stream().mapToInt(Integer::intValue).toArray()); + return FbsLayout.createFbsLayout(fbb, STRUCT, rows, 0, childV, 0); + } + List refs = dicts.get(key); + if (refs != null) { + return zoned(fbb, zoneMaps.get(key), dictLayout(fbb, refs), rows(refs)); + } + List colChunks = chunks.get(key); + long colRows = 0; + int[] flats = new int[colChunks.size()]; + for (int i = 0; i < colChunks.size(); i++) { + ChunkRef cr = colChunks.get(i); + int segV = FbsLayout.createSegmentsVector(fbb, new long[]{cr.segIdx()}); + flats[i] = FbsLayout.createFbsLayout(fbb, FLAT, cr.rowCount(), 0, 0, segV); + colRows += cr.rowCount(); + } + return zoned(fbb, zoneMaps.get(key), chunked(fbb, flats, colRows), colRows); + } + /// Wraps a column's data layout in a zoned layout when a zone-map was emitted for it (`zm` /// not `null`); otherwise returns the data layout unchanged. private static int zoned(FbsBuilder fbb, ZoneMapRef zm, int dataLayout, long colRows) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/StructColumns.java b/writer/src/main/java/io/github/dfa1/vortex/writer/StructColumns.java new file mode 100644 index 000000000..8a41b4402 --- /dev/null +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/StructColumns.java @@ -0,0 +1,80 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/// A struct column is written as Rust's `vortex.struct` layout: each field is its own column, +/// chunked and compressed on its own, under a validity child when the struct is nullable. The +/// writer's per-column machinery (repartition, global dictionaries, zone maps, segment sizes) is +/// keyed by [ColumnName], so each field of a struct, at any depth, is a *leaf column* named +/// `parent/field`; a struct with no fields stays a leaf, written as one array. +/// +/// Rust zones the fields of a struct only while no struct above them is nullable. +final class StructColumns { + + private final DType.Struct leafSchema; + private final Set unzoned; + private final Map owners; + + /// @param schema the file's schema + /// @throws IllegalArgumentException if a leaf's `parent/field` name equals another column's + StructColumns(DType.Struct schema) { + List names = new ArrayList<>(); + List types = new ArrayList<>(); + this.unzoned = new LinkedHashSet<>(); + this.owners = new LinkedHashMap<>(); + for (int i = 0; i < schema.fieldNames().size(); i++) { + flatten(schema.fieldNames().get(i), schema.fieldNames().get(i), schema.fieldTypes().get(i), true, names, types); + } + this.leafSchema = new DType.Struct(names, types, false); + } + + /// Whether `dtype` is written as a struct layout rather than as one array. + static boolean isLayoutStruct(DType dtype) { + return dtype instanceof DType.Struct struct && !struct.fieldNames().isEmpty(); + } + + /// The leaf column of `field` in the struct column `parent`. + static ColumnName fieldKey(ColumnName parent, ColumnName field) { + return ColumnName.of(parent.value() + "/" + field.value()); + } + + /// The columns actually written, one per leaf in schema order, with their dtypes. + DType.Struct leafSchema() { + return leafSchema; + } + + /// The leaves that carry no zone map: those under a nullable struct. + Set unzoned() { + return unzoned; + } + + /// The top-level column a leaf belongs to, for attributing its bytes to that column. + ColumnName owner(ColumnName leaf) { + return owners.getOrDefault(leaf, leaf); + } + + private void flatten(ColumnName column, ColumnName key, DType dtype, boolean zoned, List names, List types) { + if (!isLayoutStruct(dtype)) { + names.add(key); + owners.put(key, column); + types.add(dtype); + if (!zoned) { + unzoned.add(key); + } + return; + } + DType.Struct struct = (DType.Struct) dtype; + for (int i = 0; i < struct.fieldNames().size(); i++) { + flatten(column, fieldKey(key, struct.fieldNames().get(i)), struct.fieldTypes().get(i), + zoned && !struct.nullable(), names, types); + } + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index dd38f046a..64d31026a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -3,18 +3,22 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; import io.github.dfa1.vortex.core.model.Edition; import io.github.dfa1.vortex.core.model.EditionFamily; import io.github.dfa1.vortex.core.model.Editions; import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.LayoutId; import io.github.dfa1.vortex.writer.encode.EncodingEncoder; +import io.github.dfa1.vortex.writer.encode.NullableData; import io.github.dfa1.vortex.writer.encode.RowCount; +import io.github.dfa1.vortex.writer.encode.StructData; import java.io.Closeable; import java.io.IOException; import java.nio.channels.WritableByteChannel; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; @@ -50,6 +54,7 @@ public final class VortexWriter implements Closeable { private static final WriteRegistry DEFAULT_EXTENSIONS = WriteRegistry.builder().registerDefaultExtensions().build(); private final DType.Struct schema; + private final StructColumns structColumns; private final WriteOptions options; // The only writer state the encoding tasks on WriteOptions#executor() touch; immutable. private final SegmentEncoder segmentEncoder; @@ -66,7 +71,11 @@ public final class VortexWriter implements Closeable { private final ZoneMapWriter zoneMaps; // Rust's footer file statistics, one per top-level field in schema order, fed the raw batches. private final List fileStats; + // Every leaf column's chunks: a struct column's fields (see StructColumns) are columns of their own. private final Map> colChunks; + // Each nullable struct's validity, buffered as the batches arrive and written whole at finish + // (Rust's struct layout holds it in one flat child), by the struct's leaf-style key. + private final Map structValidities; // Rust's repartition: each column's batches coalesce into ~1 MB chunks of 8192-row multiples. private final Map repartitioners; // The batch being written: its columns' piece sizes, measured on the executor ahead of appendChunk. @@ -101,12 +110,15 @@ private VortexWriter( options.enableZoneMaps() && !zonedLayout); this.sink = new SegmentSink(channel, segmentEncoder, options.executor()); this.globalDict = new GlobalDictWriter(options, segmentEncoder, sink, zonedLayout, this::appendChunk); - this.zoneMaps = new ZoneMapWriter(schema, options, zonedLayout, segmentEncoder, sink); + this.structColumns = new StructColumns(schema); + this.zoneMaps = new ZoneMapWriter(structColumns.leafSchema(), options, zonedLayout, segmentEncoder, sink, + structColumns.unzoned()); this.fileStats = schema.fieldTypes().stream().map(FileStatsAccumulator::new).toList(); this.colChunks = new LinkedHashMap<>(); - for (ColumnName name : schema.fieldNames()) { + for (ColumnName name : structColumns.leafSchema().fieldNames()) { colChunks.put(name, new ArrayList<>()); } + this.structValidities = new LinkedHashMap<>(); this.repartitioners = new LinkedHashMap<>(); this.pieceSizes = new HashMap<>(); this.firstChunkSeen = false; @@ -326,18 +338,61 @@ public void writeChunk(Map columns) throws IOException { DType colDtype = schema.fieldTypes().get(i); Object data = adapted.get(colName); - zoneMaps.add(colName, colDtype, data); fileStats.get(i).add(data, expectedLen); + writeColumn(colName, colName, colDtype, data, true); + } + firstChunkSeen = true; + } + + /// Writes one batch of `key`: a struct is split into its fields, each written as a column of its + /// own, and its validity buffered; any other column is a leaf, written whole. + /// + /// @param column the top-level column `key` belongs to + /// @param zoned whether Rust zones this column: no struct above it is nullable + private void writeColumn(ColumnName column, ColumnName key, DType dtype, Object data, boolean zoned) + throws IOException { + if (!StructColumns.isLayoutStruct(dtype)) { + if (zoned) { + zoneMaps.add(key, dtype, data); + } if (!firstChunkSeen) { - globalDict.detect(colName, colDtype, data); + globalDict.detect(key, dtype, data); } - if (globalDict.isCandidate(colName)) { - globalDict.ingest(colName, colDtype, data); + if (globalDict.isCandidate(key)) { + globalDict.ingest(key, dtype, data); } else { - appendChunk(colName, colDtype, data); + appendChunk(key, dtype, data); } + return; } - firstChunkSeen = true; + DType.Struct struct = (DType.Struct) dtype; + StructData fields; + if (data instanceof NullableData nullable) { + fields = (StructData) nullable.values(); + structValidities.computeIfAbsent(key, _ -> new StructValidity(column)).batches().add(nullable.validity()); + } else { + fields = (StructData) data; + if (struct.nullable()) { + boolean[] allValid = new boolean[RowCount.of(data)]; + Arrays.fill(allValid, true); + structValidities.computeIfAbsent(key, _ -> new StructValidity(column)).batches().add(allValid); + } + } + for (int i = 0; i < struct.fieldNames().size(); i++) { + writeColumn(column, StructColumns.fieldKey(key, struct.fieldNames().get(i)), struct.fieldTypes().get(i), + fields.fieldArrays().get(i), zoned && !struct.nullable()); + } + } + + /// Writes each nullable struct's validity as one segment, a flat bool child of its layout. + private Map flushStructValidities() throws IOException { + Map written = new LinkedHashMap<>(); + for (Map.Entry e : structValidities.entrySet()) { + boolean[] all = e.getValue().concat(); + written.put(e.getKey(), new ChunkRef( + sink.write(e.getValue().column(), DTypes.BOOL, all, null, Set.of()), all.length)); + } + return written; } /// Hands one batch of a column to its [Repartitioner], writing every chunk it completes; a @@ -395,6 +450,7 @@ public WriteSummary finish() throws IOException { } finishing = true; flushRepartitioners(); + Map validities = flushStructValidities(); globalDict.flush(); zoneMaps.flush(colChunks, globalDict.written()); // the zoned layout: Rust's "vortex.zoned" when the edition has it, else the legacy @@ -404,8 +460,9 @@ public WriteSummary finish() throws IOException { zoned = LayoutId.ZONED; } long fileSize = sink.finish(zoned, dtypeSerializer.buildDType(schema), - LayoutTree.build(schema, colChunks, globalDict.written(), zoneMaps.written()), fileStats); - Map sizes = sink.columnSizes(); + LayoutTree.build(schema, rowCount, colChunks, validities, globalDict.written(), zoneMaps.written()), fileStats); + Map sizes = new LinkedHashMap<>(); + sink.columnSizes().forEach((column, bytes) -> sizes.merge(structColumns.owner(column), bytes, Long::sum)); Map columns = new LinkedHashMap<>(); for (int i = 0; i < schema.fieldNames().size(); i++) { ColumnName name = schema.fieldNames().get(i); @@ -429,7 +486,29 @@ private int writeSegment(ColumnName column, DType dtype, Object data, EncodingEn } private DType columnDtype(ColumnName colName) { - return schema.fieldTypes().get(schema.fieldNames().indexOf(colName)); + DType.Struct leaves = structColumns.leafSchema(); + return leaves.fieldTypes().get(leaves.fieldNames().indexOf(colName)); + } + + /// A nullable struct's validity batches so far, and the top-level column its bytes count towards. + private record StructValidity(ColumnName column, List batches) { + StructValidity(ColumnName column) { + this(column, new ArrayList<>()); + } + + boolean[] concat() { + int rows = 0; + for (boolean[] batch : batches) { + rows += batch.length; + } + boolean[] all = new boolean[rows]; + int at = 0; + for (boolean[] batch : batches) { + System.arraycopy(batch, 0, all, at, batch.length); + at += batch.length; + } + return all; + } } /// One written chunk of a column: its segment and row count. diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapWriter.java index 502fac5c8..9778b3eb2 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneMapWriter.java @@ -37,6 +37,8 @@ final class ZoneMapWriter { private final boolean zonedLayout; private final SegmentEncoder encoder; private final SegmentSink sink; + // leaves that carry no zone map (under a nullable struct) + private final Set unzoned; // Per-column zone-maps, populated by flush() in close(). private final Map zoneMaps; private final Map zoneAccumulators; @@ -45,12 +47,13 @@ final class ZoneMapWriter { private final Map> zoneTails; ZoneMapWriter(DType.Struct schema, WriteOptions options, boolean zonedLayout, SegmentEncoder encoder, - SegmentSink sink) { + SegmentSink sink, Set unzoned) { this.schema = schema; this.options = options; this.zonedLayout = zonedLayout; this.encoder = encoder; this.sink = sink; + this.unzoned = unzoned; this.zoneMaps = new LinkedHashMap<>(); this.zoneAccumulators = new LinkedHashMap<>(); this.zoneTails = new LinkedHashMap<>(); @@ -93,7 +96,7 @@ void flush(Map> colChunks, Map> e : colChunks.entrySet()) { ColumnName colName = e.getKey(); List chunks = e.getValue(); - if (chunks.isEmpty()) { + if (chunks.isEmpty() || unzoned.contains(colName)) { continue; } DType colDtype = columnDtype(colName); @@ -114,6 +117,9 @@ void flush(Map> colChunks, Map> e : dicts.entrySet()) { + if (unzoned.contains(e.getKey())) { + continue; + } DType colDtype = columnDtype(e.getKey()); DType minMaxDtype = ZoneMapStatCodec.zoneMinMaxDtype(colDtype); List mins = new ArrayList<>();