From ff0410e20a5c6d78d65fecc583b2be40fe4a4990 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 8 Nov 2026 14:05:02 +0100 Subject: [PATCH 1/2] fix(writer): no chunked layout around a single chunk Rust's ChunkedLayoutStrategy returns the only child itself when a column has one chunk (vortex-layout chunked/writer.rs at 0.90.0: `if child_layouts.len() == 1`). LayoutTree always wrapped the column's flats, and a dict's codes, in vortex.chunked: one extra layout node per single-chunk column, which every Rust and Java reader accepted, so no interop test noticed. Found by FooterFieldParityIntegrationTest (next commit). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../github/dfa1/vortex/writer/LayoutTree.java | 30 ++++++++++--------- .../writer/GlobalDictPrimitiveTest.java | 5 ++-- .../vortex/writer/GlobalDictUtf8Test.java | 10 +++---- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecaddf00b..a69dc73b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) - Written files carry the per-column file statistics (min, max, sum, null and NaN counts) in the footer, as vortex-jni writes them. ([#664](https://github.com/dfa1/vortex-java/issues/664)) +- A column written as one chunk has no `vortex.chunked` layout around it, as Rust writes it. ([#711](https://github.com/dfa1/vortex-java/pull/711)) ### Added - JDK Flight Recorder events for file open, HTTP range requests, segment decode and encode, chunk filtering, writer back-pressure and the compressor's scheme choices, which mirror Rust's compressor trace. ([#601](https://github.com/dfa1/vortex-java/issues/601)) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java b/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java index 059d03b7e..39e2da030 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java @@ -20,8 +20,9 @@ /// The file's layout tree, and the footer's `layout_specs` list its nodes index into: every /// layout node the writer emits is built here. /// -/// The root is a struct with one child per column: a chunked layout of flat segments, or the -/// column's dict layout (several in a chunked layout), each wrapped in its zone map when it has one. +/// The root is a struct with one child per column: a chunked layout of flat segments (the flat +/// itself for one chunk), or the column's dict layout (several in a chunked layout), each wrapped in +/// its zone map when it has one. final class LayoutTree { // Indices into the footer's layout_specs list, in the order specs() writes them @@ -78,9 +79,7 @@ static ByteBuffer build(DType.Struct schema, Map> chu flats[i] = FbsLayout.createFbsLayout(fbb, FLAT, cr.rowCount(), 0, 0, segV); colRows += cr.rowCount(); } - int childV = FbsLayout.createChildrenVector(fbb, flats); - int dataChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, colRows, 0, childV, 0); - colLayouts[c] = zoned(fbb, zoneMaps.get(colName), dataChunked, colRows); + colLayouts[c] = zoned(fbb, zoneMaps.get(colName), chunked(fbb, flats, colRows), colRows); if (totalRows == 0) { totalRows = colRows; } @@ -106,6 +105,15 @@ private static int zoned(FbsBuilder fbb, ZoneMapRef zm, int dataLayout, long col return FbsLayout.createFbsLayout(fbb, ZONED, colRows, metaV, childV, 0); } + /// A chunked layout of `children`, or the only child itself: Rust's `ChunkedLayoutStrategy` + /// writes no chunked node around a single chunk. + private static int chunked(FbsBuilder fbb, int[] children, long rows) { + if (children.length == 1) { + return children[0]; + } + return FbsLayout.createFbsLayout(fbb, CHUNKED, rows, 0, FbsLayout.createChildrenVector(fbb, children), 0); + } + /// A column's dict layout: one dictionary is the column's layout; several are a chunked layout /// of them. private static int dictLayout(FbsBuilder fbb, List refs) { @@ -113,12 +121,7 @@ private static int dictLayout(FbsBuilder fbb, List refs) { for (int i = 0; i < dictLayouts.length; i++) { dictLayouts[i] = dictColLayout(fbb, refs.get(i)); } - int dictLayout = dictLayouts[0]; - if (dictLayouts.length > 1) { - dictLayout = FbsLayout.createFbsLayout(fbb, CHUNKED, rows(refs), 0, - FbsLayout.createChildrenVector(fbb, dictLayouts), 0); - } - return dictLayout; + return chunked(fbb, dictLayouts, rows(refs)); } private static long rows(List refs) { @@ -130,7 +133,7 @@ private static long rows(List refs) { } private static int dictColLayout(FbsBuilder fbb, DictColRef ref) { - // Build codes Chunked layout: children are one Flat per original chunk + // Build the codes layout: one Flat per original chunk, chunked when there are several int numChunks = ref.codesSegIdxes().size(); int[] codesFlats = new int[numChunks]; long totalCodesRows = 0; @@ -140,8 +143,7 @@ private static int dictColLayout(FbsBuilder fbb, DictColRef ref) { codesFlats[j] = FbsLayout.createFbsLayout(fbb, FLAT, rowCount, 0, 0, segV); totalCodesRows += rowCount; } - int codesChildV = FbsLayout.createChildrenVector(fbb, codesFlats); - int codesChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, totalCodesRows, 0, codesChildV, 0); + int codesChunked = chunked(fbb, codesFlats, totalCodesRows); // Build values Flat layout int valSegV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.valuesSegIdx()}); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java index ba6bbfb85..7900dd9cb 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java @@ -264,9 +264,8 @@ void cardinalityGrowsPastCapMidFile_nullableI64_demotesAndRestoresNulls(@TempDir var columnLayout = unwrapZoned(vf.layout().children().getFirst()); assertThat(columnLayout.isDict()).as("cardinality-demoted column must not be a global dict").isFalse(); // The replayed batches coalesce like any column's (Rust's repartition, #470): well - // under 1 MB, so one Flat. - assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk") - .singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue()); + // under 1 MB, so one Flat, written without a chunked node around it as Rust does. + assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue(); // And every value AND every null round-trips exactly, confirming the nullable // reconstruction path (validity restore + NullableData re-wrap) end-to-end. diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java index 006adc6b3..c5b901f90 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java @@ -288,9 +288,8 @@ void retainedBytesBudgetExceeded_utf8_demotesToPerChunkChunkedLayout(@TempDir Pa var columnLayout = unwrapZoned(vf.layout().children().getFirst()); assertThat(columnLayout.isDict()).as("demoted column must not be a global dict").isFalse(); // The replayed batches coalesce like any column's (Rust's repartition, #470): well - // under 1 MB, so one Flat. - assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk") - .singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue()); + // under 1 MB, so one Flat, written without a chunked node around it as Rust does. + assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue(); // And every value round-trips exactly across all chunks despite the mid-file demotion. List got = readAllStrings(vf, "status"); @@ -338,9 +337,8 @@ void retainedBytesBudgetExceeded_nullableUtf8_demotesAndRestoresNulls(@TempDir P var columnLayout = unwrapZoned(vf.layout().children().getFirst()); assertThat(columnLayout.isDict()).as("demoted column must not be a global dict").isFalse(); // The replayed batches coalesce like any column's (Rust's repartition, #470): well - // under 1 MB, so one Flat. - assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk") - .singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue()); + // under 1 MB, so one Flat, written without a chunked node around it as Rust does. + assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue(); // And every value AND every null round-trips exactly, confirming the nullable // reconstruction path (validity restore + NullableData re-wrap) end-to-end. From 9becd2d4db065d7b2a1410d857cec7d28cc0f9de Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 8 Nov 2026 14:05:02 +0100 Subject: [PATCH 2/2] test(integration): fitness function for metadata field parity with vortex-jni Every interop test compares what a file holds, never which optional FlatBuffer fields its metadata carries, so a field the Java writer leaves out fails nothing: the footer's file statistics went unwritten for releases (#706). This writes the same rows through both writers and asserts the set of fields present in the postscript, footer, dtype, layout and file statistics is equal, walking the generated readers by reflection (hasX() / non-null table accessors), so a field added to a schema is covered without editing the test. Co-Authored-By: Claude Sonnet 5 --- .../FooterFieldParityIntegrationTest.java | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java new file mode 100644 index 000000000..e3875288b --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java @@ -0,0 +1,213 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.fbs.FbsDType; +import io.github.dfa1.vortex.core.fbs.FbsFileStatistics; +import io.github.dfa1.vortex.core.fbs.FbsFooter; +import io.github.dfa1.vortex.core.fbs.FbsLayout; +import io.github.dfa1.vortex.core.fbs.FbsPostscript; +import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.foreign.MemorySegment; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.function.Function; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A fitness function for the file's metadata (#664): every optional field of the postscript and the +/// blobs it points to (footer, dtype, layout, file statistics) that vortex-jni sets for a file, the +/// Java writer sets for the same rows, and the other way round. +/// +/// The interop tests compare what a file holds, never which of these fields it carries, and every +/// field is optional on the wire: a reader treats a missing one as "not written". So the footer's +/// file statistics went unwritten for releases without any test failing. This walks the generated +/// FlatBuffer readers by reflection rather than naming the fields, so a field added to a schema is +/// checked too. Nested layout fields give the tree's depth away as well: it first caught a +/// `vortex.chunked` node Java wrote around a single chunk, which Rust writes as the chunk itself. +class FooterFieldParityIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + // past one 8192-row zone, so the zone maps hold more than one row + private static final int ROWS = 20_000; + + @BeforeAll + static void loadJni() { + NativeLoader.loadJni(); + } + + @Test + void javaSetsTheMetadataFieldsJniSets(@TempDir Path tmp) throws Exception { + // Given — the same rows through both writers + Path jni = tmp.resolve("jni.vortex"); + writeJni(jni); + Path java = tmp.resolve("java.vortex"); + writeJava(java); + + // When + TreeSet result = presentFields(java); + + // Then + assertThat(result).isEqualTo(presentFields(jni)); + } + + private static long i64(int row) { + return row * 3L - 100; + } + + private static boolean valid(int row) { + return row % 7 != 0; + } + + private static String text(int row) { + return "k" + row % 50; + } + + private static void writeJni(Path file) throws Exception { + Schema arrow = new Schema(List.of( + Field.notNullable("i64", new ArrowType.Int(64, true)), + Field.nullable("i32n", new ArrowType.Int(32, true)), + Field.notNullable("f64", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("text", ArrowType.Utf8.INSTANCE), + Field.notNullable("flag", ArrowType.Bool.INSTANCE))); + try (dev.vortex.api.VortexWriter w = dev.vortex.api.VortexWriter.builder(SESSION, + file.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + root.allocateNew(); + for (int i = 0; i < ROWS; i++) { + ((BigIntVector) root.getVector("i64")).setSafe(i, i64(i)); + IntVector i32n = (IntVector) root.getVector("i32n"); + if (valid(i)) { + i32n.setSafe(i, i % 1000); + } else { + i32n.setNull(i); + } + ((Float8Vector) root.getVector("f64")).setSafe(i, i * 0.5); + ((VarCharVector) root.getVector("text")).setSafe(i, text(i).getBytes(StandardCharsets.UTF_8)); + ((BitVector) root.getVector("flag")).setSafe(i, i % 2); + } + root.setRowCount(ROWS); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, schema); + w.writeBatch(array.memoryAddress(), schema.memoryAddress()); + } + } + } + + private static void writeJava(Path file) throws Exception { + List names = List.of("i64", "i32n", "f64", "text", "flag").stream().map(ColumnName::of).toList(); + DType.Struct schema = new DType.Struct(names, + List.of(DTypes.I64, DTypes.I32.asNullable(), DTypes.F64, DTypes.UTF8, DTypes.BOOL), false); + long[] i64 = new long[ROWS]; + Integer[] i32n = new Integer[ROWS]; + double[] f64 = new double[ROWS]; + String[] text = new String[ROWS]; + boolean[] flag = new boolean[ROWS]; + for (int i = 0; i < ROWS; i++) { + i64[i] = i64(i); + i32n[i] = valid(i) ? i % 1000 : null; + f64[i] = i * 0.5; + text[i] = text(i); + flag[i] = i % 2 == 1; + } + Map chunk = new LinkedHashMap<>(); + List values = List.of(i64, i32n, f64, text, flag); + for (int c = 0; c < names.size(); c++) { + chunk.put(names.get(c), values.get(c)); + } + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter w = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(chunk); + } + } + + /// The path of every field set in the postscript and the blobs it points to; a vector of tables + /// contributes the union of its elements' fields. + private static TreeSet presentFields(Path file) throws Exception { + MemorySegment seg = MemorySegment.ofArray(Files.readAllBytes(file)); + long size = seg.byteSize(); + int psLen = Short.toUnsignedInt(seg.get(VortexFormat.LE_SHORT, size - VortexFormat.TRAILER_SIZE + 2)); + FbsPostscript ps = FbsPostscript.getRootAsFbsPostscript(seg.asSlice(size - VortexFormat.TRAILER_SIZE - psLen, psLen)); + TreeSet fields = new TreeSet<>(); + collect(ps, "postscript", fields); + blob(seg, ps.footer(), "footer", FbsFooter::getRootAsFbsFooter, fields); + blob(seg, ps.dtype(), "dtype", FbsDType::getRootAsFbsDType, fields); + blob(seg, ps.layout(), "layout", FbsLayout::getRootAsFbsLayout, fields); + blob(seg, ps.statistics(), "statistics", FbsFileStatistics::getRootAsFbsFileStatistics, fields); + return fields; + } + + private static void blob(MemorySegment file, FbsPostscriptSegment at, String name, + Function root, TreeSet fields) throws Exception { + if (at != null) { + collect(root.apply(file.asSlice(at.offset(), at.length())), name, fields); + } + } + + /// Adds the fields `table` has set: a `hasX()` that is true (vectors, optional scalars), or a + /// table-typed `x()` that is not `null`, recursing into tables and vectors of tables. + private static void collect(Object table, String path, TreeSet fields) throws Exception { + for (Method m : table.getClass().getMethods()) { + if (Modifier.isStatic(m.getModifiers()) || m.getDeclaringClass() != table.getClass()) { + continue; + } + String name = m.getName(); + if (m.getParameterCount() == 0 && name.startsWith("has") && m.getReturnType() == boolean.class) { + if ((boolean) m.invoke(table)) { + fields.add(path + "." + Character.toLowerCase(name.charAt(3)) + name.substring(4)); + } + } else if (m.getParameterCount() == 0 && isGenerated(m.getReturnType())) { + Object child = m.invoke(table); + if (child != null) { + fields.add(path + "." + name); + collect(child, path + "." + name, fields); + } + } else if (m.getParameterCount() == 1 && m.getParameterTypes()[0] == int.class && isGenerated(m.getReturnType())) { + Method length = table.getClass().getMethod(name + "Length"); + int n = (int) length.invoke(table); + for (int j = 0; j < n; j++) { + collect(m.invoke(table, j), path + "." + name + "[]", fields); + } + } + } + } + + private static boolean isGenerated(Class type) { + return type.getPackage() == FbsFooter.class.getPackage() && type.getSimpleName().startsWith("Fbs"); + } +}