Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701))
- A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694))
- Written files carry the per-column file statistics (min, max, sum, null and NaN counts) in the footer, as vortex-jni writes them. ([#664](https://github.com/dfa1/vortex-java/issues/664))
- A column written as one chunk has no `vortex.chunked` layout around it, as Rust writes it. ([#711](https://github.com/dfa1/vortex-java/pull/711))

### Added
- JDK Flight Recorder events for file open, HTTP range requests, segment decode and encode, chunk filtering, writer back-pressure and the compressor's scheme choices, which mirror Rust's compressor trace. ([#601](https://github.com/dfa1/vortex-java/issues/601))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
package io.github.dfa1.vortex.integration;

import dev.vortex.api.Session;
import dev.vortex.arrow.ArrowAllocation;
import dev.vortex.jni.NativeLoader;
import io.github.dfa1.vortex.core.fbs.FbsDType;
import io.github.dfa1.vortex.core.fbs.FbsFileStatistics;
import io.github.dfa1.vortex.core.fbs.FbsFooter;
import io.github.dfa1.vortex.core.fbs.FbsLayout;
import io.github.dfa1.vortex.core.fbs.FbsPostscript;
import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment;
import io.github.dfa1.vortex.core.io.VortexFormat;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.DTypes;
import io.github.dfa1.vortex.writer.VortexWriter;
import io.github.dfa1.vortex.writer.WriteOptions;
import org.apache.arrow.c.ArrowArray;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.lang.foreign.MemorySegment;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import java.util.function.Function;

import static org.assertj.core.api.Assertions.assertThat;

/// A fitness function for the file's metadata (#664): every optional field of the postscript and the
/// blobs it points to (footer, dtype, layout, file statistics) that vortex-jni sets for a file, the
/// Java writer sets for the same rows, and the other way round.
///
/// The interop tests compare what a file holds, never which of these fields it carries, and every
/// field is optional on the wire: a reader treats a missing one as "not written". So the footer's
/// file statistics went unwritten for releases without any test failing. This walks the generated
/// FlatBuffer readers by reflection rather than naming the fields, so a field added to a schema is
/// checked too. Nested layout fields give the tree's depth away as well: it first caught a
/// `vortex.chunked` node Java wrote around a single chunk, which Rust writes as the chunk itself.
class FooterFieldParityIntegrationTest {

private static final Session SESSION = Session.create();
private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
// past one 8192-row zone, so the zone maps hold more than one row
private static final int ROWS = 20_000;

@BeforeAll
static void loadJni() {
NativeLoader.loadJni();
}

@Test
void javaSetsTheMetadataFieldsJniSets(@TempDir Path tmp) throws Exception {
// Given — the same rows through both writers
Path jni = tmp.resolve("jni.vortex");
writeJni(jni);
Path java = tmp.resolve("java.vortex");
writeJava(java);

// When
TreeSet<String> result = presentFields(java);

// Then
assertThat(result).isEqualTo(presentFields(jni));
}

private static long i64(int row) {
return row * 3L - 100;
}

private static boolean valid(int row) {
return row % 7 != 0;
}

private static String text(int row) {
return "k" + row % 50;
}

private static void writeJni(Path file) throws Exception {
Schema arrow = new Schema(List.of(
Field.notNullable("i64", new ArrowType.Int(64, true)),
Field.nullable("i32n", new ArrowType.Int(32, true)),
Field.notNullable("f64", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
Field.notNullable("text", ArrowType.Utf8.INSTANCE),
Field.notNullable("flag", ArrowType.Bool.INSTANCE)));
try (dev.vortex.api.VortexWriter w = dev.vortex.api.VortexWriter.builder(SESSION,
file.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build();
VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) {
root.allocateNew();
for (int i = 0; i < ROWS; i++) {
((BigIntVector) root.getVector("i64")).setSafe(i, i64(i));
IntVector i32n = (IntVector) root.getVector("i32n");
if (valid(i)) {
i32n.setSafe(i, i % 1000);
} else {
i32n.setNull(i);
}
((Float8Vector) root.getVector("f64")).setSafe(i, i * 0.5);
((VarCharVector) root.getVector("text")).setSafe(i, text(i).getBytes(StandardCharsets.UTF_8));
((BitVector) root.getVector("flag")).setSafe(i, i % 2);
}
root.setRowCount(ROWS);
try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) {
Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, schema);
w.writeBatch(array.memoryAddress(), schema.memoryAddress());
}
}
}

private static void writeJava(Path file) throws Exception {
List<ColumnName> names = List.of("i64", "i32n", "f64", "text", "flag").stream().map(ColumnName::of).toList();
DType.Struct schema = new DType.Struct(names,
List.of(DTypes.I64, DTypes.I32.asNullable(), DTypes.F64, DTypes.UTF8, DTypes.BOOL), false);
long[] i64 = new long[ROWS];
Integer[] i32n = new Integer[ROWS];
double[] f64 = new double[ROWS];
String[] text = new String[ROWS];
boolean[] flag = new boolean[ROWS];
for (int i = 0; i < ROWS; i++) {
i64[i] = i64(i);
i32n[i] = valid(i) ? i % 1000 : null;
f64[i] = i * 0.5;
text[i] = text(i);
flag[i] = i % 2 == 1;
}
Map<ColumnName, Object> chunk = new LinkedHashMap<>();
List<Object> values = List.of(i64, i32n, f64, text, flag);
for (int c = 0; c < names.size(); c++) {
chunk.put(names.get(c), values.get(c));
}
try (FileChannel ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
VortexWriter w = VortexWriter.create(ch, schema, WriteOptions.defaults())) {
w.writeChunk(chunk);
}
}

/// The path of every field set in the postscript and the blobs it points to; a vector of tables
/// contributes the union of its elements' fields.
private static TreeSet<String> presentFields(Path file) throws Exception {
MemorySegment seg = MemorySegment.ofArray(Files.readAllBytes(file));
long size = seg.byteSize();
int psLen = Short.toUnsignedInt(seg.get(VortexFormat.LE_SHORT, size - VortexFormat.TRAILER_SIZE + 2));
FbsPostscript ps = FbsPostscript.getRootAsFbsPostscript(seg.asSlice(size - VortexFormat.TRAILER_SIZE - psLen, psLen));
TreeSet<String> fields = new TreeSet<>();
collect(ps, "postscript", fields);
blob(seg, ps.footer(), "footer", FbsFooter::getRootAsFbsFooter, fields);
blob(seg, ps.dtype(), "dtype", FbsDType::getRootAsFbsDType, fields);
blob(seg, ps.layout(), "layout", FbsLayout::getRootAsFbsLayout, fields);
blob(seg, ps.statistics(), "statistics", FbsFileStatistics::getRootAsFbsFileStatistics, fields);
return fields;
}

private static void blob(MemorySegment file, FbsPostscriptSegment at, String name,
Function<MemorySegment, Object> root, TreeSet<String> fields) throws Exception {
if (at != null) {
collect(root.apply(file.asSlice(at.offset(), at.length())), name, fields);
}
}

/// Adds the fields `table` has set: a `hasX()` that is true (vectors, optional scalars), or a
/// table-typed `x()` that is not `null`, recursing into tables and vectors of tables.
private static void collect(Object table, String path, TreeSet<String> fields) throws Exception {
for (Method m : table.getClass().getMethods()) {
if (Modifier.isStatic(m.getModifiers()) || m.getDeclaringClass() != table.getClass()) {
continue;
}
String name = m.getName();
if (m.getParameterCount() == 0 && name.startsWith("has") && m.getReturnType() == boolean.class) {
if ((boolean) m.invoke(table)) {
fields.add(path + "." + Character.toLowerCase(name.charAt(3)) + name.substring(4));
}
} else if (m.getParameterCount() == 0 && isGenerated(m.getReturnType())) {
Object child = m.invoke(table);
if (child != null) {
fields.add(path + "." + name);
collect(child, path + "." + name, fields);
}
} else if (m.getParameterCount() == 1 && m.getParameterTypes()[0] == int.class && isGenerated(m.getReturnType())) {
Method length = table.getClass().getMethod(name + "Length");
int n = (int) length.invoke(table);
for (int j = 0; j < n; j++) {
collect(m.invoke(table, j), path + "." + name + "[]", fields);
}
}
}
}

private static boolean isGenerated(Class<?> type) {
return type.getPackage() == FbsFooter.class.getPackage() && type.getSimpleName().startsWith("Fbs");
}
}
30 changes: 16 additions & 14 deletions writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@
/// The file's layout tree, and the footer's `layout_specs` list its nodes index into: every
/// layout node the writer emits is built here.
///
/// The root is a struct with one child per column: a chunked layout of flat segments, or the
/// column's dict layout (several in a chunked layout), each wrapped in its zone map when it has one.
/// The root is a struct with one child per column: a chunked layout of flat segments (the flat
/// itself for one chunk), or the column's dict layout (several in a chunked layout), each wrapped in
/// its zone map when it has one.
final class LayoutTree {

// Indices into the footer's layout_specs list, in the order specs() writes them
Expand Down Expand Up @@ -78,9 +79,7 @@ static ByteBuffer build(DType.Struct schema, Map<ColumnName, List<ChunkRef>> chu
flats[i] = FbsLayout.createFbsLayout(fbb, FLAT, cr.rowCount(), 0, 0, segV);
colRows += cr.rowCount();
}
int childV = FbsLayout.createChildrenVector(fbb, flats);
int dataChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, colRows, 0, childV, 0);
colLayouts[c] = zoned(fbb, zoneMaps.get(colName), dataChunked, colRows);
colLayouts[c] = zoned(fbb, zoneMaps.get(colName), chunked(fbb, flats, colRows), colRows);
if (totalRows == 0) {
totalRows = colRows;
}
Expand All @@ -106,19 +105,23 @@ private static int zoned(FbsBuilder fbb, ZoneMapRef zm, int dataLayout, long col
return FbsLayout.createFbsLayout(fbb, ZONED, colRows, metaV, childV, 0);
}

/// A chunked layout of `children`, or the only child itself: Rust's `ChunkedLayoutStrategy`
/// writes no chunked node around a single chunk.
private static int chunked(FbsBuilder fbb, int[] children, long rows) {
if (children.length == 1) {
return children[0];
}
return FbsLayout.createFbsLayout(fbb, CHUNKED, rows, 0, FbsLayout.createChildrenVector(fbb, children), 0);
}

/// A column's dict layout: one dictionary is the column's layout; several are a chunked layout
/// of them.
private static int dictLayout(FbsBuilder fbb, List<DictColRef> refs) {
int[] dictLayouts = new int[refs.size()];
for (int i = 0; i < dictLayouts.length; i++) {
dictLayouts[i] = dictColLayout(fbb, refs.get(i));
}
int dictLayout = dictLayouts[0];
if (dictLayouts.length > 1) {
dictLayout = FbsLayout.createFbsLayout(fbb, CHUNKED, rows(refs), 0,
FbsLayout.createChildrenVector(fbb, dictLayouts), 0);
}
return dictLayout;
return chunked(fbb, dictLayouts, rows(refs));
}

private static long rows(List<DictColRef> refs) {
Expand All @@ -130,7 +133,7 @@ private static long rows(List<DictColRef> refs) {
}

private static int dictColLayout(FbsBuilder fbb, DictColRef ref) {
// Build codes Chunked layout: children are one Flat per original chunk
// Build the codes layout: one Flat per original chunk, chunked when there are several
int numChunks = ref.codesSegIdxes().size();
int[] codesFlats = new int[numChunks];
long totalCodesRows = 0;
Expand All @@ -140,8 +143,7 @@ private static int dictColLayout(FbsBuilder fbb, DictColRef ref) {
codesFlats[j] = FbsLayout.createFbsLayout(fbb, FLAT, rowCount, 0, 0, segV);
totalCodesRows += rowCount;
}
int codesChildV = FbsLayout.createChildrenVector(fbb, codesFlats);
int codesChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, totalCodesRows, 0, codesChildV, 0);
int codesChunked = chunked(fbb, codesFlats, totalCodesRows);

// Build values Flat layout
int valSegV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.valuesSegIdx()});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -264,9 +264,8 @@ void cardinalityGrowsPastCapMidFile_nullableI64_demotesAndRestoresNulls(@TempDir
var columnLayout = unwrapZoned(vf.layout().children().getFirst());
assertThat(columnLayout.isDict()).as("cardinality-demoted column must not be a global dict").isFalse();
// The replayed batches coalesce like any column's (Rust's repartition, #470): well
// under 1 MB, so one Flat.
assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk")
.singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue());
// under 1 MB, so one Flat, written without a chunked node around it as Rust does.
assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue();

// And every value AND every null round-trips exactly, confirming the nullable
// reconstruction path (validity restore + NullableData re-wrap) end-to-end.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -288,9 +288,8 @@ void retainedBytesBudgetExceeded_utf8_demotesToPerChunkChunkedLayout(@TempDir Pa
var columnLayout = unwrapZoned(vf.layout().children().getFirst());
assertThat(columnLayout.isDict()).as("demoted column must not be a global dict").isFalse();
// The replayed batches coalesce like any column's (Rust's repartition, #470): well
// under 1 MB, so one Flat.
assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk")
.singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue());
// under 1 MB, so one Flat, written without a chunked node around it as Rust does.
assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue();

// And every value round-trips exactly across all chunks despite the mid-file demotion.
List<String> got = readAllStrings(vf, "status");
Expand Down Expand Up @@ -338,9 +337,8 @@ void retainedBytesBudgetExceeded_nullableUtf8_demotesAndRestoresNulls(@TempDir P
var columnLayout = unwrapZoned(vf.layout().children().getFirst());
assertThat(columnLayout.isDict()).as("demoted column must not be a global dict").isFalse();
// The replayed batches coalesce like any column's (Rust's repartition, #470): well
// under 1 MB, so one Flat.
assertThat(columnLayout.children()).as("demoted rows coalesce into one chunk")
.singleElement().satisfies(child -> assertThat(child.isFlat()).isTrue());
// under 1 MB, so one Flat, written without a chunked node around it as Rust does.
assertThat(columnLayout.isFlat()).as("demoted rows coalesce into one chunk").isTrue();

// And every value AND every null round-trips exactly, confirming the nullable
// reconstruction path (validity restore + NullableData re-wrap) end-to-end.
Expand Down
Loading