diff --git a/CHANGELOG.md b/CHANGELOG.md index ecaddf00b..a69dc73b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) - Written files carry the per-column file statistics (min, max, sum, null and NaN counts) in the footer, as vortex-jni writes them. ([#664](https://github.com/dfa1/vortex-java/issues/664)) +- A column written as one chunk has no `vortex.chunked` layout around it, as Rust writes it. ([#711](https://github.com/dfa1/vortex-java/pull/711)) ### Added - JDK Flight Recorder events for file open, HTTP range requests, segment decode and encode, chunk filtering, writer back-pressure and the compressor's scheme choices, which mirror Rust's compressor trace. ([#601](https://github.com/dfa1/vortex-java/issues/601)) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java new file mode 100644 index 000000000..e3875288b --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/FooterFieldParityIntegrationTest.java @@ -0,0 +1,213 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.fbs.FbsDType; +import io.github.dfa1.vortex.core.fbs.FbsFileStatistics; +import io.github.dfa1.vortex.core.fbs.FbsFooter; +import io.github.dfa1.vortex.core.fbs.FbsLayout; +import io.github.dfa1.vortex.core.fbs.FbsPostscript; +import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.foreign.MemorySegment; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.function.Function; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A fitness function for the file's metadata (#664): every optional field of the postscript and the +/// blobs it points to (footer, dtype, layout, file statistics) that vortex-jni sets for a file, the +/// Java writer sets for the same rows, and the other way round. +/// +/// The interop tests compare what a file holds, never which of these fields it carries, and every +/// field is optional on the wire: a reader treats a missing one as "not written". So the footer's +/// file statistics went unwritten for releases without any test failing. This walks the generated +/// FlatBuffer readers by reflection rather than naming the fields, so a field added to a schema is +/// checked too. Nested layout fields give the tree's depth away as well: it first caught a +/// `vortex.chunked` node Java wrote around a single chunk, which Rust writes as the chunk itself. +class FooterFieldParityIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + // past one 8192-row zone, so the zone maps hold more than one row + private static final int ROWS = 20_000; + + @BeforeAll + static void loadJni() { + NativeLoader.loadJni(); + } + + @Test + void javaSetsTheMetadataFieldsJniSets(@TempDir Path tmp) throws Exception { + // Given — the same rows through both writers + Path jni = tmp.resolve("jni.vortex"); + writeJni(jni); + Path java = tmp.resolve("java.vortex"); + writeJava(java); + + // When + TreeSet result = presentFields(java); + + // Then + assertThat(result).isEqualTo(presentFields(jni)); + } + + private static long i64(int row) { + return row * 3L - 100; + } + + private static boolean valid(int row) { + return row % 7 != 0; + } + + private static String text(int row) { + return "k" + row % 50; + } + + private static void writeJni(Path file) throws Exception { + Schema arrow = new Schema(List.of( + Field.notNullable("i64", new ArrowType.Int(64, true)), + Field.nullable("i32n", new ArrowType.Int(32, true)), + Field.notNullable("f64", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("text", ArrowType.Utf8.INSTANCE), + Field.notNullable("flag", ArrowType.Bool.INSTANCE))); + try (dev.vortex.api.VortexWriter w = dev.vortex.api.VortexWriter.builder(SESSION, + file.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + root.allocateNew(); + for (int i = 0; i < ROWS; i++) { + ((BigIntVector) root.getVector("i64")).setSafe(i, i64(i)); + IntVector i32n = (IntVector) root.getVector("i32n"); + if (valid(i)) { + i32n.setSafe(i, i % 1000); + } else { + i32n.setNull(i); + } + ((Float8Vector) root.getVector("f64")).setSafe(i, i * 0.5); + ((VarCharVector) root.getVector("text")).setSafe(i, text(i).getBytes(StandardCharsets.UTF_8)); + ((BitVector) root.getVector("flag")).setSafe(i, i % 2); + } + root.setRowCount(ROWS); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, schema); + w.writeBatch(array.memoryAddress(), schema.memoryAddress()); + } + } + } + + private static void writeJava(Path file) throws Exception { + List names = List.of("i64", "i32n", "f64", "text", "flag").stream().map(ColumnName::of).toList(); + DType.Struct schema = new DType.Struct(names, + List.of(DTypes.I64, DTypes.I32.asNullable(), DTypes.F64, DTypes.UTF8, DTypes.BOOL), false); + long[] i64 = new long[ROWS]; + Integer[] i32n = new Integer[ROWS]; + double[] f64 = new double[ROWS]; + String[] text = new String[ROWS]; + boolean[] flag = new boolean[ROWS]; + for (int i = 0; i < ROWS; i++) { + i64[i] = i64(i); + i32n[i] = valid(i) ? i % 1000 : null; + f64[i] = i * 0.5; + text[i] = text(i); + flag[i] = i % 2 == 1; + } + Map chunk = new LinkedHashMap<>(); + List values = List.of(i64, i32n, f64, text, flag); + for (int c = 0; c < names.size(); c++) { + chunk.put(names.get(c), values.get(c)); + } + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter w = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + w.writeChunk(chunk); + } + } + + /// The path of every field set in the postscript and the blobs it points to; a vector of tables + /// contributes the union of its elements' fields. + private static TreeSet presentFields(Path file) throws Exception { + MemorySegment seg = MemorySegment.ofArray(Files.readAllBytes(file)); + long size = seg.byteSize(); + int psLen = Short.toUnsignedInt(seg.get(VortexFormat.LE_SHORT, size - VortexFormat.TRAILER_SIZE + 2)); + FbsPostscript ps = FbsPostscript.getRootAsFbsPostscript(seg.asSlice(size - VortexFormat.TRAILER_SIZE - psLen, psLen)); + TreeSet fields = new TreeSet<>(); + collect(ps, "postscript", fields); + blob(seg, ps.footer(), "footer", FbsFooter::getRootAsFbsFooter, fields); + blob(seg, ps.dtype(), "dtype", FbsDType::getRootAsFbsDType, fields); + blob(seg, ps.layout(), "layout", FbsLayout::getRootAsFbsLayout, fields); + blob(seg, ps.statistics(), "statistics", FbsFileStatistics::getRootAsFbsFileStatistics, fields); + return fields; + } + + private static void blob(MemorySegment file, FbsPostscriptSegment at, String name, + Function root, TreeSet fields) throws Exception { + if (at != null) { + collect(root.apply(file.asSlice(at.offset(), at.length())), name, fields); + } + } + + /// Adds the fields `table` has set: a `hasX()` that is true (vectors, optional scalars), or a + /// table-typed `x()` that is not `null`, recursing into tables and vectors of tables. + private static void collect(Object table, String path, TreeSet fields) throws Exception { + for (Method m : table.getClass().getMethods()) { + if (Modifier.isStatic(m.getModifiers()) || m.getDeclaringClass() != table.getClass()) { + continue; + } + String name = m.getName(); + if (m.getParameterCount() == 0 && name.startsWith("has") && m.getReturnType() == boolean.class) { + if ((boolean) m.invoke(table)) { + fields.add(path + "." + Character.toLowerCase(name.charAt(3)) + name.substring(4)); + } + } else if (m.getParameterCount() == 0 && isGenerated(m.getReturnType())) { + Object child = m.invoke(table); + if (child != null) { + fields.add(path + "." + name); + collect(child, path + "." + name, fields); + } + } else if (m.getParameterCount() == 1 && m.getParameterTypes()[0] == int.class && isGenerated(m.getReturnType())) { + Method length = table.getClass().getMethod(name + "Length"); + int n = (int) length.invoke(table); + for (int j = 0; j < n; j++) { + collect(m.invoke(table, j), path + "." + name + "[]", fields); + } + } + } + } + + private static boolean isGenerated(Class type) { + return type.getPackage() == FbsFooter.class.getPackage() && type.getSimpleName().startsWith("Fbs"); + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java b/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java index 059d03b7e..39e2da030 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java @@ -20,8 +20,9 @@ /// The file's layout tree, and the footer's `layout_specs` list its nodes index into: every /// layout node the writer emits is built here. /// -/// The root is a struct with one child per column: a chunked layout of flat segments, or the -/// column's dict layout (several in a chunked layout), each wrapped in its zone map when it has one. +/// The root is a struct with one child per column: a chunked layout of flat segments (the flat +/// itself for one chunk), or the column's dict layout (several in a chunked layout), each wrapped in +/// its zone map when it has one. final class LayoutTree { // Indices into the footer's layout_specs list, in the order specs() writes them @@ -78,9 +79,7 @@ static ByteBuffer build(DType.Struct schema, Map> chu flats[i] = FbsLayout.createFbsLayout(fbb, FLAT, cr.rowCount(), 0, 0, segV); colRows += cr.rowCount(); } - int childV = FbsLayout.createChildrenVector(fbb, flats); - int dataChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, colRows, 0, childV, 0); - colLayouts[c] = zoned(fbb, zoneMaps.get(colName), dataChunked, colRows); + colLayouts[c] = zoned(fbb, zoneMaps.get(colName), chunked(fbb, flats, colRows), colRows); if (totalRows == 0) { totalRows = colRows; } @@ -106,6 +105,15 @@ private static int zoned(FbsBuilder fbb, ZoneMapRef zm, int dataLayout, long col return FbsLayout.createFbsLayout(fbb, ZONED, colRows, metaV, childV, 0); } + /// A chunked layout of `children`, or the only child itself: Rust's `ChunkedLayoutStrategy` + /// writes no chunked node around a single chunk. + private static int chunked(FbsBuilder fbb, int[] children, long rows) { + if (children.length == 1) { + return children[0]; + } + return FbsLayout.createFbsLayout(fbb, CHUNKED, rows, 0, FbsLayout.createChildrenVector(fbb, children), 0); + } + /// A column's dict layout: one dictionary is the column's layout; several are a chunked layout /// of them. private static int dictLayout(FbsBuilder fbb, List refs) { @@ -113,12 +121,7 @@ private static int dictLayout(FbsBuilder fbb, List refs) { for (int i = 0; i < dictLayouts.length; i++) { dictLayouts[i] = dictColLayout(fbb, refs.get(i)); } - int dictLayout = dictLayouts[0]; - if (dictLayouts.length > 1) { - dictLayout = FbsLayout.createFbsLayout(fbb, CHUNKED, rows(refs), 0, - FbsLayout.createChildrenVector(fbb, dictLayouts), 0); - } - return dictLayout; + return chunked(fbb, dictLayouts, rows(refs)); } private static long rows(List refs) { @@ -130,7 +133,7 @@ private static long rows(List refs) { } private static int dictColLayout(FbsBuilder fbb, DictColRef ref) { - // Build codes Chunked layout: children are one Flat per original chunk + // Build the codes layout: one Flat per original chunk, chunked when there are several int numChunks = ref.codesSegIdxes().size(); int[] codesFlats = new int[numChunks]; long totalCodesRows = 0; @@ -140,8 +143,7 @@ private static int dictColLayout(FbsBuilder fbb, DictColRef ref) { codesFlats[j] = FbsLayout.createFbsLayout(fbb, FLAT, rowCount, 0, 0, segV); totalCodesRows += rowCount; } - int codesChildV = FbsLayout.createChildrenVector(fbb, codesFlats); - int codesChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, totalCodesRows, 0, codesChildV, 0); + int codesChunked = chunked(fbb, codesFlats, totalCodesRows); // Build values Flat layout int valSegV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.valuesSegIdx()}); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java index ba6bbfb85..7900dd9cb 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java @@ -264,9 +264,8 @@ void cardinalityGrowsPastCapMidFile_nullableI64_demotesAndRestoresNulls(@TempDir var columnLayout = unwrapZoned(vf.layout().children().getFirst()); assertThat(columnLayout.isDict()).as("cardinality-demoted column must not be a global dict").isFalse(); // The replayed batches coalesce like any column's (Rust's repartition, #470): well - // under 1 MB, so one Flat. - assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk") - .singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue()); + // under 1 MB, so one Flat, written without a chunked node around it as Rust does. + assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue(); // And every value AND every null round-trips exactly, confirming the nullable // reconstruction path (validity restore + NullableData re-wrap) end-to-end. diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java index 006adc6b3..c5b901f90 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictUtf8Test.java @@ -288,9 +288,8 @@ void retainedBytesBudgetExceeded_utf8_demotesToPerChunkChunkedLayout(@TempDir Pa var columnLayout = unwrapZoned(vf.layout().children().getFirst()); assertThat(columnLayout.isDict()).as("demoted column must not be a global dict").isFalse(); // The replayed batches coalesce like any column's (Rust's repartition, #470): well - // under 1 MB, so one Flat. - assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk") - .singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue()); + // under 1 MB, so one Flat, written without a chunked node around it as Rust does. + assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue(); // And every value round-trips exactly across all chunks despite the mid-file demotion. List got = readAllStrings(vf, "status"); @@ -338,9 +337,8 @@ void retainedBytesBudgetExceeded_nullableUtf8_demotesAndRestoresNulls(@TempDir P var columnLayout = unwrapZoned(vf.layout().children().getFirst()); assertThat(columnLayout.isDict()).as("demoted column must not be a global dict").isFalse(); // The replayed batches coalesce like any column's (Rust's repartition, #470): well - // under 1 MB, so one Flat. - assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk") - .singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue()); + // under 1 MB, so one Flat, written without a chunked node around it as Rust does. + assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue(); // And every value AND every null round-trips exactly, confirming the nullable // reconstruction path (validity restore + NullableData re-wrap) end-to-end.