From 897898e585fd2e025809b99a32e6b9fcf8e8c20b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 8 Nov 2026 10:45:02 +0100 Subject: [PATCH] fix(writer): write Rust's file statistics into the footer (#664) vortex-jni writes a FileStatistics segment, pointed to by the postscript: one ArrayStats per top-level field with Rust 0.90.0's PRUNING_STATS (min, max, sum, null count, NaN count). vortex-java wrote none, although the schema declares it. FileStatsAccumulator ports 0.90.0's FileStatsAccumulator: each stat is taken per writeChunk batch and folded at close, so - a UTF-8/binary bound is cut to 64 bytes per batch and is inexact when any batch cut it; - a UTF-8/binary max is dropped when any batch has none (all null, or no 64-byte upper bound); - a sum is dropped on overflow, a decimal(p) one past p+10 digits per batch and p+20 in the fold, serialized at the p+20 width; - extensions get storage min/max only, nested types the null count, variants nothing, and a column without rows an empty entry. Scalar kinds follow Rust (f16_value for F16, decimal LE bytes, u64 for unsigned and bool sums): Rust checks them against the dtype on open, so a wrong one makes the file unreadable. ZoneAccumulator's per-zone scan becomes the shared `extremes`, reused per batch; a non-nullable primitive batch uses the vectorized minMax. Cross-checked byte for byte against vortex-jni's footer for the same batches (FileStatisticsInteropIntegrationTest), and vortex-jni reads the Java file. Cost: one more pass over each batch on the caller thread, as Rust does. JavaVsJniWriteBenchmark.javaWrite (OHLC 10M rows, depth 3, interleaved, noisy machine): 1.08/1.18 -> 0.94/1.05 ops/s (~11%), mostly the sequential float sum (kept sequential for Rust's bits) and the string scan. Co-Authored-By: Claude Sonnet 5 --- AGENTS.md | 3 +- CHANGELOG.md | 1 + docs/explanation.md | 6 +- docs/reference.md | 4 +- .../FileStatisticsInteropIntegrationTest.java | 397 ++++++++++++++++++ .../vortex/writer/FileStatsAccumulator.java | 381 +++++++++++++++++ .../dfa1/vortex/writer/SegmentSink.java | 59 ++- .../dfa1/vortex/writer/VortexWriter.java | 6 +- .../dfa1/vortex/writer/ZoneAccumulator.java | 78 +++- .../writer/FileStatsAccumulatorTest.java | 87 ++++ 10 files changed, 989 insertions(+), 33 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/FileStatisticsInteropIntegrationTest.java create mode 100644 writer/src/main/java/io/github/dfa1/vortex/writer/FileStatsAccumulator.java create mode 100644 writer/src/test/java/io/github/dfa1/vortex/writer/FileStatsAccumulatorTest.java diff --git a/AGENTS.md b/AGENTS.md index 6a4625c98..d8890c758 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -212,7 +212,8 @@ The tag push also creates the GitHub release, from the version's CHANGELOG secti 8-byte trailer at EOF: `version(u16 LE) | postscriptLen(u16 LE) | magic(VTXF)`. The postscript (FlatBuffer, immediately before the trailer) points (offset+length) to the Footer (FlatBuffer), -DType (FlatBuffer), and Layout (FlatBuffer) blobs elsewhere in the file. +DType (FlatBuffer), Layout (FlatBuffer) and file statistics (FlatBuffer: one `ArrayStats` per +top-level field, Rust 0.90's per-batch fold, `FileStatsAccumulator`) blobs elsewhere in the file. Layout tree: `Struct → Zoned(Stats) → Chunked → [Flat, Flat, ...]` - **Flat** single encoded segment · **Chunked** sequence of Flats · **Struct** one child/column diff --git a/CHANGELOG.md b/CHANGELOG.md index acd8d3abe..54d341c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A struct with duplicate field names now fails with `VortexException` when it is built, at any nesting level; the writer used to check only top-level names. ([#696](https://github.com/dfa1/vortex-java/pull/696)) - **Breaking:** a `Double`, `Float` or `BigDecimal` filter operand on an integer column fails with `VortexException`, as vortex-jni does, instead of being truncated so that `x >= 1.5` matched `x = 1`. ([#701](https://github.com/dfa1/vortex-java/issues/701)) - A `vortex.variant` column whose shredded dtype has a control character in a field or variant name fails with `VortexException` instead of `IllegalArgumentException`. ([#694](https://github.com/dfa1/vortex-java/issues/694)) +- Written files carry the per-column file statistics (min, max, sum, null and NaN counts) in the footer, as vortex-jni writes them. ([#664](https://github.com/dfa1/vortex-java/issues/664)) ### Added - JDK Flight Recorder events for file open, HTTP range requests, segment decode and encode, chunk filtering, writer back-pressure and the compressor's scheme choices, which mirror Rust's compressor trace. ([#601](https://github.com/dfa1/vortex-java/issues/601)) diff --git a/docs/explanation.md b/docs/explanation.md index 13bfe5641..d4000ddf2 100644 --- a/docs/explanation.md +++ b/docs/explanation.md @@ -254,8 +254,11 @@ self-describing tail. A reader bootstraps from the last 8 bytes — no scanning │ │ leaves point into Footer's │ │ SegmentSpec[] by index. ├──────────────────────────────────────────────┤ + │ Statistics (FlatBuffer) │ ← min/max/sum/null count per + │ │ top-level column. + ├──────────────────────────────────────────────┤ │ Postscript (FlatBuffer) │ ← (offset,length) of Footer, - │ │ DType, Layout above. + │ │ DType, Layout, Statistics. ├──────────────────────────────────────────────┤ │ Trailer (8 bytes, little-endian) │ │ u16 version │ u16 postscriptLen │ "VTXF" │ ← magic confirms file type; @@ -501,6 +504,7 @@ writer.close() └─ write DType blob (FlatBuffer) └─ write Footer blob (FlatBuffer) → SegmentSpec[] + ArraySpec[] └─ write Layout blob (FlatBuffer) → Struct → Zoned(Stats) → Chunked → [Flat …] + └─ write Statistics (FlatBuffer) → per-column min/max/sum/null count, folded per batch └─ write Postscript (FlatBuffer) → blob offsets + lengths └─ write 8-byte trailer → version · postscriptLen · magic (VTXF) ``` diff --git a/docs/reference.md b/docs/reference.md index 864601600..dcac242a0 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -849,6 +849,8 @@ version (u16 LE) | postscriptLen (u16 LE) | magic ("VTXF") ``` The postscript is a FlatBuffer blob immediately before the trailer. It points (offset + length) to: -the Footer (FlatBuffer), the DType (FlatBuffer), and the Layout (FlatBuffer) — each stored elsewhere in the file. +the Footer (FlatBuffer), the DType (FlatBuffer), the Layout (FlatBuffer) and the file statistics +(FlatBuffer, one `ArrayStats` per top-level column: min, max, sum, null and NaN counts, as vortex-jni +writes them) — each stored elsewhere in the file. See [explanation.md#memory-model](explanation.md#memory-model) for the mmap lifecycle. diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/FileStatisticsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/FileStatisticsInteropIntegrationTest.java new file mode 100644 index 000000000..5ea7eabf7 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/FileStatisticsInteropIntegrationTest.java @@ -0,0 +1,397 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.DataSource; +import dev.vortex.api.Partition; +import dev.vortex.api.Scan; +import dev.vortex.api.ScanOptions; +import dev.vortex.api.Session; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.fbs.FbsArrayStats; +import io.github.dfa1.vortex.core.fbs.FbsFileStatistics; +import io.github.dfa1.vortex.core.fbs.FbsPostscript; +import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.core.model.ExtensionId; +import io.github.dfa1.vortex.core.model.TimeUnit; +import io.github.dfa1.vortex.core.model.TimeUnitMetadata; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.NullEncodingEncoder; +import io.github.dfa1.vortex.writer.encode.NullableData; +import io.github.dfa1.vortex.writer.encode.StructData; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.BitVector; +import org.apache.arrow.vector.DateDayVector; +import org.apache.arrow.vector.DecimalVector; +import org.apache.arrow.vector.Float2Vector; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.Float8Vector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.UInt1Vector; +import org.apache.arrow.vector.UInt8Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.apache.arrow.vector.types.DateUnit; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.math.BigDecimal; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// The file statistics in the postscript's `statistics` segment, against what vortex-jni writes for the +/// same batches (#664): one `ArrayStats` per top-level field holding Rust's `PRUNING_STATS` (min, max, +/// sum, null count, NaN count), each taken per batch and folded. The data hits every rule of that +/// fold: a string min cut to 64 bytes in one batch only (inexact min, exact max), a string column +/// with an all-null batch and a binary max that cannot be rounded up (both lose their max), an `i64` +/// sum that overflows (no sum), decimals of two widths (the sum is wider than the values), an +/// all-null float column, an extension (storage min/max, no sum) and a nullable struct (null count +/// only). +class FileStatisticsInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final int ROWS = 1000; + private static final int BATCHES = 2; + private static final DType.Extension DATE = new DType.Extension(ExtensionId.VORTEX_DATE, DTypes.I32, + new TimeUnitMetadata(TimeUnit.Days), false); + private static final List NAMES = List.of("i64", "i32n", "f64nan", "f16", "f32", "u8", "u64", "ovf", + "utf8", "utf8nb", "bin", "booln", "dec10", "dec5", "date", "f64null", "st").stream().map(ColumnName::of).toList(); + + @BeforeAll + static void loadJni() { + NativeLoader.loadJni(); + } + + @Test + void javaWritesTheFileStatisticsJniWrites(@TempDir Path tmp) throws Exception { + // Given — the same two batches through both writers + Path jni = tmp.resolve("jni.vortex"); + writeJni(jni); + Path java = tmp.resolve("java.vortex"); + writeJava(java); + + // When + List result = fileStatistics(java); + + // Then — field by field, byte for byte: each scalar's kind and width, each precision + assertThat(result).isEqualTo(fileStatistics(jni)); + } + + @Test + void jniReadsAJavaFileWithFileStatistics(@TempDir Path tmp) throws Exception { + // Given — Rust parses the statistics segment when it opens the file, checking every scalar + // against its field's dtype: a wrong scalar kind makes the whole file unreadable + Path java = tmp.resolve("java.vortex"); + writeJava(java); + + // When + long result = jniRowCount(java); + + // Then + assertThat(result).isEqualTo((long) ROWS * BATCHES); + } + + @Test + void nullColumn_countsEveryRowNull(@TempDir Path tmp) throws Exception { + // Given — a Null column on its own: the default encoders have none for it. Every row is + // null, though the carrier has no validity to say so + Schema arrow = new Schema(List.of(Field.nullable("n", ArrowType.Null.INSTANCE))); + Path jni = tmp.resolve("jni.vortex"); + try (dev.vortex.api.VortexWriter w = dev.vortex.api.VortexWriter.builder(SESSION, + jni.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR); + ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) { + root.setRowCount(ROWS); + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, schema); + w.writeBatch(array.memoryAddress(), schema.memoryAddress()); + } + Path java = tmp.resolve("java.vortex"); + DType.Struct schema = new DType.Struct(List.of(ColumnName.of("n")), List.of(DTypes.NULL), false); + try (FileChannel ch = FileChannel.open(java, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter w = VortexWriter.create(ch, schema, WriteOptions.defaults(), List.of(new NullEncodingEncoder()))) { + w.writeChunk(Map.of(ColumnName.of("n"), new long[ROWS])); + } + + // When + List result = fileStatistics(java); + + // Then + assertThat(result).isEqualTo(fileStatistics(jni)); + } + + // ── data ──────────────────────────────────────────────────────────────── + + private static boolean i32Valid(int row) { + return row % 7 != 0; + } + + private static double f64(int row) { + return row % 11 == 0 ? Double.NaN : row * 0.5; + } + + private static String utf8(int batch, int i) { + // only batch 0 has a value past 64 bytes, and it is the column minimum + return batch == 0 && i == 0 ? "a".repeat(80) : "m" + (batch * ROWS + i); + } + + private static String utf8nb(int batch, int i) { + return batch == 1 || i % 3 == 0 ? null : "v" + i; + } + + private static byte[] bin(int batch, int i) { + // batch 1's max is 70 0xFF bytes: no 64-byte upper bound exists + if (batch == 1 && i == 5) { + byte[] ff = new byte[70]; + Arrays.fill(ff, (byte) 0xFF); + return ff; + } + return new byte[]{1, (byte) i}; + } + + private static Boolean bool(int row) { + return row % 3 == 0 ? null : row % 2 == 0; + } + + private static BigDecimal dec10(int row) { + return row % 4 == 0 ? null : BigDecimal.valueOf(row * 101L, 2); + } + + private static BigDecimal dec5(int row) { + return BigDecimal.valueOf(row % 999 * 7L, 2); + } + + private static boolean structValid(int row) { + return row % 5 != 0; + } + + // ── writers ───────────────────────────────────────────────────────────── + + private static void writeJni(Path file) throws Exception { + Field struct = new Field("st", FieldType.nullable(ArrowType.Struct.INSTANCE), + List.of(Field.nullable("a", new ArrowType.Int(32, true)))); + Schema arrow = new Schema(List.of( + Field.notNullable("i64", new ArrowType.Int(64, true)), + Field.nullable("i32n", new ArrowType.Int(32, true)), + Field.notNullable("f64nan", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + Field.notNullable("f16", new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)), + Field.notNullable("f32", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)), + Field.notNullable("u8", new ArrowType.Int(8, false)), + Field.notNullable("u64", new ArrowType.Int(64, false)), + Field.notNullable("ovf", new ArrowType.Int(64, true)), + Field.notNullable("utf8", ArrowType.Utf8.INSTANCE), + Field.nullable("utf8nb", ArrowType.Utf8.INSTANCE), + Field.notNullable("bin", ArrowType.Binary.INSTANCE), + Field.nullable("booln", ArrowType.Bool.INSTANCE), + Field.nullable("dec10", new ArrowType.Decimal(10, 2, 128)), + Field.notNullable("dec5", new ArrowType.Decimal(5, 2, 128)), + Field.notNullable("date", new ArrowType.Date(DateUnit.DAY)), + Field.nullable("f64null", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)), + struct)); + try (dev.vortex.api.VortexWriter w = dev.vortex.api.VortexWriter.builder(SESSION, + file.toAbsolutePath().toUri().toString(), arrow, ALLOCATOR).build()) { + for (int batch = 0; batch < BATCHES; batch++) { + try (VectorSchemaRoot root = VectorSchemaRoot.create(arrow, ALLOCATOR)) { + root.allocateNew(); + fillArrow(root, batch); + root.setRowCount(ROWS); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, schema); + w.writeBatch(array.memoryAddress(), schema.memoryAddress()); + } + } + } + } + } + + private static void fillArrow(VectorSchemaRoot root, int batch) { + StructVector st = (StructVector) root.getVector("st"); + IntVector stA = (IntVector) st.getChild("a"); + for (int i = 0; i < ROWS; i++) { + int row = batch * ROWS + i; + ((BigIntVector) root.getVector("i64")).setSafe(i, row * 3L - 100); + IntVector i32n = (IntVector) root.getVector("i32n"); + if (i32Valid(row)) { + i32n.setSafe(i, row % 1000); + } else { + i32n.setNull(i); + } + ((Float8Vector) root.getVector("f64nan")).setSafe(i, f64(row)); + ((Float2Vector) root.getVector("f16")).setSafe(i, Float.floatToFloat16(row * 0.25f)); + ((Float4Vector) root.getVector("f32")).setSafe(i, row * 1.5f); + ((UInt1Vector) root.getVector("u8")).setSafe(i, row % 200); + ((UInt8Vector) root.getVector("u64")).setSafe(i, row * 7L); + ((BigIntVector) root.getVector("ovf")).setSafe(i, Long.MAX_VALUE / 3); + ((VarCharVector) root.getVector("utf8")).setSafe(i, utf8(batch, i).getBytes(StandardCharsets.UTF_8)); + VarCharVector utf8nb = (VarCharVector) root.getVector("utf8nb"); + if (utf8nb(batch, i) == null) { + utf8nb.setNull(i); + } else { + utf8nb.setSafe(i, utf8nb(batch, i).getBytes(StandardCharsets.UTF_8)); + } + ((VarBinaryVector) root.getVector("bin")).setSafe(i, bin(batch, i)); + BitVector booln = (BitVector) root.getVector("booln"); + if (bool(row) == null) { + booln.setNull(i); + } else { + booln.setSafe(i, bool(row) ? 1 : 0); + } + DecimalVector dec10 = (DecimalVector) root.getVector("dec10"); + if (dec10(row) == null) { + dec10.setNull(i); + } else { + dec10.setSafe(i, dec10(row)); + } + ((DecimalVector) root.getVector("dec5")).setSafe(i, dec5(row)); + ((DateDayVector) root.getVector("date")).setSafe(i, 19000 + row); + ((Float8Vector) root.getVector("f64null")).setNull(i); + if (structValid(row)) { + st.setIndexDefined(i); + stA.setSafe(i, row); + } else { + st.setNull(i); + } + } + } + + private static void writeJava(Path file) throws IOException { + DType.Struct schema = new DType.Struct(NAMES, List.of( + DTypes.I64, DTypes.I32.asNullable(), DTypes.F64, DTypes.F16, DTypes.F32, DTypes.U8, DTypes.U64, DTypes.I64, + DTypes.UTF8, DTypes.UTF8.asNullable(), DTypes.BINARY, DTypes.BOOL.asNullable(), + DType.Decimal.of(10, 2, true), DType.Decimal.of(5, 2, false), DATE, DTypes.F64.asNullable(), + new DType.Struct(List.of(ColumnName.of("a")), List.of(DTypes.I32.asNullable()), true)), false); + try (FileChannel ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + VortexWriter w = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + for (int batch = 0; batch < BATCHES; batch++) { + w.writeChunk(javaBatch(batch)); + } + } + } + + private static Map javaBatch(int batch) { + long[] i64 = new long[ROWS]; + Integer[] i32n = new Integer[ROWS]; + double[] f64nan = new double[ROWS]; + short[] f16 = new short[ROWS]; + float[] f32 = new float[ROWS]; + byte[] u8 = new byte[ROWS]; + long[] u64 = new long[ROWS]; + long[] ovf = new long[ROWS]; + String[] utf8 = new String[ROWS]; + String[] utf8nb = new String[ROWS]; + byte[][] bin = new byte[ROWS][]; + Boolean[] booln = new Boolean[ROWS]; + BigDecimal[] dec10 = new BigDecimal[ROWS]; + BigDecimal[] dec5 = new BigDecimal[ROWS]; + int[] date = new int[ROWS]; + Double[] f64null = new Double[ROWS]; + int[] stA = new int[ROWS]; + boolean[] stValid = new boolean[ROWS]; + for (int i = 0; i < ROWS; i++) { + int row = batch * ROWS + i; + i64[i] = row * 3L - 100; + i32n[i] = i32Valid(row) ? row % 1000 : null; + f64nan[i] = f64(row); + f16[i] = Float.floatToFloat16(row * 0.25f); + f32[i] = row * 1.5f; + u8[i] = (byte) (row % 200); + u64[i] = row * 7L; + ovf[i] = Long.MAX_VALUE / 3; + utf8[i] = utf8(batch, i); + utf8nb[i] = utf8nb(batch, i); + bin[i] = bin(batch, i); + booln[i] = bool(row); + dec10[i] = dec10(row); + dec5[i] = dec5(row); + date[i] = 19000 + row; + stValid[i] = structValid(row); + stA[i] = stValid[i] ? row : 0; + } + List values = List.of(i64, i32n, f64nan, f16, f32, u8, u64, ovf, utf8, utf8nb, bin, booln, dec10, dec5, + date, f64null, new NullableData(new StructData(List.of(new NullableData(stA, stValid))), stValid)); + Map chunk = new LinkedHashMap<>(); + for (int c = 0; c < NAMES.size(); c++) { + chunk.put(NAMES.get(c), values.get(c)); + } + return chunk; + } + + // ── readers ───────────────────────────────────────────────────────────── + + /// Each field's `ArrayStats` table as written: every stat's presence, the raw scalar bytes and + /// each bound's precision. + private static List fileStatistics(Path file) throws IOException { + MemorySegment seg = MemorySegment.ofArray(Files.readAllBytes(file)); + long size = seg.byteSize(); + int psLen = Short.toUnsignedInt(seg.get(VortexFormat.LE_SHORT, size - VortexFormat.TRAILER_SIZE + 2)); + FbsPostscript ps = FbsPostscript.getRootAsFbsPostscript(seg.asSlice(size - VortexFormat.TRAILER_SIZE - psLen, psLen)); + FbsPostscriptSegment stats = ps.statistics(); + assertThat(stats).as("statistics segment").isNotNull(); + FbsFileStatistics fs = FbsFileStatistics.getRootAsFbsFileStatistics(seg.asSlice(stats.offset(), stats.length())); + List fields = new ArrayList<>(); + for (int j = 0; j < fs.fieldStatsLength(); j++) { + FbsArrayStats a = fs.fieldStats(j); + fields.add(NAMES.get(j) + ": min=" + hex(a.minAsSegment()) + "/" + a.minPrecision() + + " max=" + hex(a.maxAsSegment()) + "/" + a.maxPrecision() + + " sum=" + hex(a.sumAsSegment()) + + " nulls=" + (a.hasNullCount() ? a.nullCount() : "-") + + " nans=" + (a.hasNanCount() ? a.nanCount() : "-") + + " size=" + (a.hasUncompressedSizeInBytes() ? a.uncompressedSizeInBytes() : "-") + + " sorted=" + (a.hasIsSorted() ? a.isSorted() : "-") + + " strict=" + (a.hasIsStrictSorted() ? a.isStrictSorted() : "-") + + " constant=" + (a.hasIsConstant() ? a.isConstant() : "-")); + } + return fields; + } + + private static String hex(MemorySegment bytes) { + if (bytes == null) { + return "-"; + } + return HexFormat.of().formatHex(bytes.toArray(java.lang.foreign.ValueLayout.JAVA_BYTE)); + } + + private static long jniRowCount(Path file) throws IOException { + long rows = 0; + Scan scan = DataSource.open(SESSION, file.toAbsolutePath().toUri().toString()).scan(ScanOptions.builder().build()); + while (scan.hasNext()) { + Partition partition = scan.next(); + try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) { + while (reader.loadNextBatch()) { + rows += reader.getVectorSchemaRoot().getRowCount(); + } + } + } + return rows; + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/FileStatsAccumulator.java b/writer/src/main/java/io/github/dfa1/vortex/writer/FileStatsAccumulator.java new file mode 100644 index 000000000..27a78c150 --- /dev/null +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/FileStatsAccumulator.java @@ -0,0 +1,381 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.PType; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.core.simd.SimdOperationsSupport; +import io.github.dfa1.vortex.writer.ZoneAccumulator.Extremes; +import io.github.dfa1.vortex.writer.ZoneAccumulator.Kind; +import io.github.dfa1.vortex.writer.encode.NodeStats; +import io.github.dfa1.vortex.writer.encode.NullableData; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.OptionalLong; + +/// One top-level column's file statistics, written into the postscript's `statistics` segment: +/// Rust 0.90.0's `FileStatsAccumulator` (`vortex-layout/src/layouts/file_stats.rs`) over the default +/// `PRUNING_STATS` (min, max, sum, null count, NaN count), as vortex-jni writes them. +/// +/// Rust takes every stat per `writeChunk` batch and folds the batches at the end, so this does too: +/// - a UTF-8 or binary bound is cut to 64 bytes per batch, and it is inexact when any batch's bound +/// was cut, even a batch that does not hold the extreme; +/// - a UTF-8 or binary max is dropped when any batch has none (all null, or a bound that cannot be +/// rounded up): skipping that batch would be unsound; +/// - a sum is dropped when any batch's sum, or the fold, overflows: integers in `i64`/`u64`, a +/// decimal of precision `p` past `p + 10` digits in a batch and `p + 20` in the fold (Rust sums +/// the batch sums with a second `Sum`). +/// +/// An extension column has its storage's min and max but no sum or NaN count, a struct, list or +/// fixed-size list only its null count, and a variant column no statistic at all. +final class FileStatsAccumulator { + + /// Rust's `MAX_PRECISION` for decimals. + private static final int MAX_DECIMAL_PRECISION = 76; + + private final Kind kind; + private final PType ptype; + private final DType.Decimal decimal; + private final boolean extension; + private final boolean nullDtype; + private final boolean variant; + private boolean seenBatch; + private Object min; + private boolean minTruncated; + private Object max; + private boolean maxTruncated; + private boolean maxDropped; + private Object sum; + private boolean sumOverflowed; + private long nullCount; + private long nanCount; + + FileStatsAccumulator(DType dtype) { + DType storage = dtype instanceof DType.Extension ext ? ext.storageDType() : dtype; + this.kind = ZoneAccumulator.kindOf(storage); + this.ptype = storage instanceof DType.Primitive p ? p.ptype() : null; + this.decimal = storage instanceof DType.Decimal d ? d : null; + this.extension = dtype instanceof DType.Extension; + this.nullDtype = storage instanceof DType.Null; + this.variant = storage instanceof DType.Variant; + this.seenBatch = false; + this.min = null; + this.minTruncated = false; + this.max = null; + this.maxTruncated = false; + this.maxDropped = false; + this.sum = null; + this.sumOverflowed = false; + this.nullCount = 0; + this.nanCount = 0; + } + + /// Adds one `writeChunk` batch of the column; an empty batch is skipped, as Rust filters it out. + /// + /// @param batch the batch's values, [NullableData]-wrapped when the column is nullable + /// @param rows the batch's row count + void add(Object batch, long rows) { + if (rows == 0 || variant) { + return; + } + seenBatch = true; + if (nullDtype) { + nullCount += rows; + return; + } + int n = (int) rows; + Object data = kind == Kind.NULL_COUNT_ONLY ? ZoneAccumulator.validityOnly(batch, n) : ZoneAccumulator.comparable(batch); + if (ptype != null && !(data instanceof NullableData)) { + // the common case, on the writer's critical path: the vectorized kernel, no widened copy + addPrimitive(data); + } else { + Extremes extremes = ZoneAccumulator.extremes(kind, ptype, data, 0, n); + nullCount += extremes.nulls(); + nanCount += extremes.nans(); + if (kind == Kind.UTF8 || kind == Kind.BINARY) { + addBounds((byte[]) extremes.min(), (byte[]) extremes.max()); + } else if (extremes.min() != null) { + addExtremes(extremes.min(), extremes.max()); + } + } + if (summable() && !sumOverflowed) { + Object values = data; + boolean[] validity = null; + if (data instanceof NullableData nd) { + values = nd.values(); + validity = nd.validity(); + } + addSum(values, validity); + } + } + + /// The folded stats, every one exact unless [#minExact()] or [#maxExact()] says otherwise. + /// + /// @return the column's stats; [NodeStats#NONE] for a variant column or one without rows, whose + /// per-batch stats Rust drops as all null + NodeStats stats() { + if (!seenBatch) { + return NodeStats.NONE; + } + ProtoScalarValue sumScalar = null; + if (summable() && !sumOverflowed) { + sumScalar = sumScalar(sum); + } + Long nans = null; + if (!extension && (kind == Kind.F16 || kind == Kind.FLOAT)) { + nans = nanCount; + } + Object maxValue = maxDropped ? null : max; + return new NodeStats(scalar(min), scalar(maxValue), null, null, null, nullCount, null, nans, sumScalar); + } + + /// Whether the min is the column's exact minimum: no batch cut its bound. + /// + /// @return `false` when a UTF-8 or binary batch minimum was longer than 64 bytes + boolean minExact() { + return !minTruncated; + } + + /// Whether the max is the column's exact maximum: no batch cut its bound. + /// + /// @return `false` when a UTF-8 or binary batch maximum was longer than 64 bytes + boolean maxExact() { + return !maxTruncated; + } + + private void addExtremes(Object lo, Object hi) { + if (min == null || compare(lo, min) < 0) { + min = lo; + } + if (max == null || compare(hi, max) > 0) { + max = hi; + } + } + + /// A batch of a primitive column without nulls: its extremes as Rust's `skip_nans` min/max, + /// `-0.0` below `0.0`. + private void addPrimitive(Object values) { + long[] extremes = SimdOperationsSupport.preferred().minMax(values, ptype); + if (kind == Kind.F16 || kind == Kind.FLOAT) { + nanCount += nanCount(values); + } + if (extremes.length == 2) { + addExtremes(fromBits(extremes[0]), fromBits(extremes[1])); + } + } + + /// A [io.github.dfa1.vortex.core.simd.SimdOperations#minMax(Object, PType)] element as the boxed + /// value the fold compares: a float's raw bits + /// back to its value. + private Object fromBits(long bits) { + return switch (ptype) { + case F16 -> (double) Float.float16ToFloat((short) bits); + case F32 -> (double) Float.intBitsToFloat((int) bits); + case F64 -> Double.longBitsToDouble(bits); + default -> bits; + }; + } + + private static long nanCount(Object values) { + long nans = 0; + switch (values) { + case short[] a -> { + for (short v : a) { + // an F16 NaN: all exponent bits set and a non-zero mantissa + if ((v & 0x7FFF) > 0x7C00) { + nans++; + } + } + } + case float[] a -> { + for (float v : a) { + if (Float.isNaN(v)) { + nans++; + } + } + } + case double[] a -> { + for (double v : a) { + if (Double.isNaN(v)) { + nans++; + } + } + } + default -> throw new IllegalStateException("not a float array: " + values.getClass().getSimpleName()); + } + return nans; + } + + private void addBounds(byte[] lo, byte[] hi) { + if (lo == null) { + // an all-null batch: its null max makes the column max unsound, its null min is skipped + maxDropped = true; + return; + } + int limit = ZoneAccumulator.BOUNDED_MAX_BYTES; + byte[] lower = kind == Kind.UTF8 ? ZoneAccumulator.utf8LowerBound(lo) : Arrays.copyOf(lo, Math.min(lo.length, limit)); + byte[] upper = kind == Kind.UTF8 ? ZoneAccumulator.utf8UpperBound(hi) : ZoneAccumulator.binaryUpperBound(hi); + minTruncated |= lo.length > limit; + if (min == null || Arrays.compareUnsigned(lower, (byte[]) min) < 0) { + min = lower; + } + if (upper == null) { + maxDropped = true; + return; + } + maxTruncated |= hi.length > limit; + if (max == null || Arrays.compareUnsigned(upper, (byte[]) max) > 0) { + max = upper; + } + } + + private boolean summable() { + return !extension && (kind == Kind.INT || kind == Kind.F16 || kind == Kind.FLOAT + || kind == Kind.BOOL || kind == Kind.DECIMAL); + } + + private void addSum(Object values, boolean[] validity) { + Object batchSum = switch (kind) { + case INT -> { + OptionalLong s = SimdOperationsSupport.preferred().sum(values, ptype); + yield s.isPresent() ? s.getAsLong() : null; + } + case F16, FLOAT -> SimdOperationsSupport.preferred().sumFloating(values, ptype); + case BOOL -> trueCount((boolean[]) values, validity); + case DECIMAL -> decimalSum((BigDecimal[]) values, validity, decimal.precision() + 10); + default -> throw new IllegalStateException("no sum for " + kind); + }; + Object folded = batchSum == null ? null : fold(sum == null ? zeroSum() : sum, batchSum); + if (folded == null) { + sumOverflowed = true; + } + sum = folded; + } + + /// `acc + batch` in the sum's type, or `null` on overflow. + private Object fold(Object acc, Object batch) { + return switch (kind) { + case INT -> ptype.isUnsigned() ? addUnsigned((Long) acc, (Long) batch) : addSigned((Long) acc, (Long) batch); + case BOOL -> addUnsigned((Long) acc, (Long) batch); + case F16, FLOAT -> (Double) acc + (Double) batch; + case DECIMAL -> fitsPrecision(((BigDecimal) acc).add((BigDecimal) batch), decimal.precision() + 20); + default -> throw new IllegalStateException("no sum for " + kind); + }; + } + + private Object zeroSum() { + return switch (kind) { + case F16, FLOAT -> 0.0; + case DECIMAL -> BigDecimal.ZERO.setScale(decimal.scale()); + default -> 0L; + }; + } + + private static Long addSigned(long a, long b) { + long r = a + b; + // overflow iff both operands have the sign the result lacks + if (((a ^ r) & (b ^ r)) < 0) { + return null; + } + return r; + } + + private static Long addUnsigned(long a, long b) { + long r = a + b; + if (Long.compareUnsigned(r, a) < 0) { + return null; + } + return r; + } + + private static long trueCount(boolean[] values, boolean[] validity) { + long count = 0; + for (int i = 0; i < values.length; i++) { + if (values[i] && (validity == null || validity[i])) { + count++; + } + } + return count; + } + + private static BigDecimal decimalSum(BigDecimal[] values, boolean[] validity, int precision) { + BigDecimal total = BigDecimal.ZERO; + for (int i = 0; i < values.length; i++) { + if (values[i] != null && (validity == null || validity[i])) { + total = total.add(values[i]); + } + } + return fitsPrecision(total, precision); + } + + /// `value`, or `null` when its unscaled value has more digits than `precision` (capped at + /// Rust's maximum): Rust's `fits_in_precision` check on a decimal sum. + private static BigDecimal fitsPrecision(BigDecimal value, int precision) { + int digits = Math.min(precision, MAX_DECIMAL_PRECISION); + if (value.unscaledValue().abs().compareTo(BigInteger.TEN.pow(digits)) >= 0) { + return null; + } + return value; + } + + private int compare(Object a, Object b) { + return switch (kind) { + case INT -> ptype.isUnsigned() ? Long.compareUnsigned((Long) a, (Long) b) : Long.compare((Long) a, (Long) b); + case F16, FLOAT -> Double.compare((Double) a, (Double) b); + case BOOL -> Boolean.compare((Boolean) a, (Boolean) b); + case DECIMAL -> ((BigDecimal) a).compareTo((BigDecimal) b); + default -> throw new IllegalStateException("no order for " + kind); + }; + } + + /// A min or max as the scalar Rust writes for the column's storage type. + private ProtoScalarValue scalar(Object value) { + if (value == null) { + return null; + } + return switch (kind) { + case INT -> ptype.isUnsigned() ? ProtoScalarValue.ofUint64Value((Long) value) : ProtoScalarValue.ofInt64Value((Long) value); + case F16 -> ProtoScalarValue.ofF16Value(Short.toUnsignedLong(Float.floatToFloat16(((Double) value).floatValue()))); + case FLOAT -> { + if (ptype == PType.F32) { + yield ProtoScalarValue.ofF32Value(((Double) value).floatValue()); + } + yield ProtoScalarValue.ofF64Value((Double) value); + } + case BOOL -> ProtoScalarValue.ofBoolValue((Boolean) value); + case DECIMAL -> ProtoScalarValue.ofBytesValue(littleEndian(((BigDecimal) value).unscaledValue(), decimal.byteWidth())); + case UTF8 -> ProtoScalarValue.ofStringValue(new String((byte[]) value, StandardCharsets.UTF_8)); + case BINARY -> ProtoScalarValue.ofBytesValue((byte[]) value); + case NULL_COUNT_ONLY -> throw new IllegalStateException("no min or max for " + kind); + }; + } + + /// A sum as the scalar of Rust's sum dtype: `i64`, `u64` (unsigned and bool), `f64`, or a + /// decimal of precision `p + 20`. + private ProtoScalarValue sumScalar(Object value) { + return switch (kind) { + case INT -> ptype.isUnsigned() ? ProtoScalarValue.ofUint64Value((Long) value) : ProtoScalarValue.ofInt64Value((Long) value); + case BOOL -> ProtoScalarValue.ofUint64Value((Long) value); + case F16, FLOAT -> ProtoScalarValue.ofF64Value((Double) value); + case DECIMAL -> { + int precision = Math.min(decimal.precision() + 20, MAX_DECIMAL_PRECISION); + int width = DType.Decimal.of(precision, decimal.scale(), true).byteWidth(); + yield ProtoScalarValue.ofBytesValue(littleEndian(((BigDecimal) value).unscaledValue(), width)); + } + default -> throw new IllegalStateException("no sum for " + kind); + }; + } + + private static byte[] littleEndian(BigInteger unscaled, int width) { + // toByteArray is big-endian and minimal: reverse it, then sign-extend to the width + byte[] be = unscaled.toByteArray(); + byte fill = unscaled.signum() < 0 ? (byte) -1 : 0; + byte[] le = new byte[width]; + for (int b = 0; b < width; b++) { + le[b] = b < be.length ? be[be.length - 1 - b] : fill; + } + return le; + } +} diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentSink.java b/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentSink.java index e21433469..771f15e26 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentSink.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentSink.java @@ -7,6 +7,7 @@ import io.github.dfa1.vortex.core.fbs.FbsArraySpec; import io.github.dfa1.vortex.core.fbs.FbsArrayStats; import io.github.dfa1.vortex.core.fbs.FbsBuilder; +import io.github.dfa1.vortex.core.fbs.FbsFileStatistics; import io.github.dfa1.vortex.core.fbs.FbsFooter; import io.github.dfa1.vortex.core.fbs.FbsPostscript; import io.github.dfa1.vortex.core.fbs.FbsPostscriptSegment; @@ -102,10 +103,14 @@ SegmentStats stats(int segIdx) { return segs.get(segIdx).stats(); } - /// Writes the remaining segments, then the footer, `dtype`, `layout`, postscript and trailer. + /// Writes the remaining segments, then the footer, `dtype`, `layout`, file statistics, + /// postscript and trailer. /// /// @param zonedLayout the layout id [LayoutTree]'s zoned nodes stand for - void finish(LayoutId zonedLayout, ByteBuffer dtype, ByteBuffer layout) throws IOException { + /// @param fileStats one accumulator per top-level field, in schema order; a schema with no + /// field gets no statistics segment, as Rust writes none + void finish(LayoutId zonedLayout, ByteBuffer dtype, ByteBuffer layout, List fileStats) + throws IOException { drain(0); ByteBuffer footerBuf = buildFooter(zonedLayout); long footerOff = bytesWritten; @@ -117,10 +122,19 @@ void finish(LayoutId zonedLayout, ByteBuffer dtype, ByteBuffer layout) throws IO long layoutOff = bytesWritten; write(layout); + long statsOff = bytesWritten; + int statsLen = 0; + if (!fileStats.isEmpty()) { + ByteBuffer statsBuf = buildFileStatistics(fileStats); + statsLen = statsBuf.capacity(); + write(statsBuf); + } + ByteBuffer psBuf = buildPostscript( footerOff, footerBuf.capacity(), dtypeOff, dtype.capacity(), - layoutOff, layout.capacity()); + layoutOff, layout.capacity(), + statsOff, statsLen); write(psBuf); // Trailer: version(u16 LE) | postscriptLen(u16 LE) | magic(4) @@ -310,6 +324,12 @@ private static int buildStats(FbsBuilder fbb, NodeStats stats) { if (stats == null || stats.isEmpty()) { return 0; } + return buildStats(fbb, stats, true, true); + } + + /// The `ArrayStats` table of `stats`, an empty table when it carries none, with each bound's + /// precision: the file statistics hold one table per field, so even a field without stats has one. + private static int buildStats(FbsBuilder fbb, NodeStats stats, boolean minExact, boolean maxExact) { // vectors first: FlatBuffers builds bottom-up int minVec = 0; if (stats.min() != null) { @@ -329,11 +349,11 @@ private static int buildStats(FbsBuilder fbb, NodeStats stats) { FbsArrayStats.startFbsArrayStats(fbb); if (stats.min() != null) { FbsArrayStats.addMin(fbb, minVec); - FbsArrayStats.addMinPrecision(fbb, FbsPrecision.Exact); + FbsArrayStats.addMinPrecision(fbb, precision(minExact)); } if (stats.max() != null) { FbsArrayStats.addMax(fbb, maxVec); - FbsArrayStats.addMaxPrecision(fbb, FbsPrecision.Exact); + FbsArrayStats.addMaxPrecision(fbb, precision(maxExact)); } if (stats.sum() != null) { FbsArrayStats.addSum(fbb, sumVec); @@ -361,6 +381,26 @@ private static int buildStats(FbsBuilder fbb, NodeStats stats) { return off; } + private static int precision(boolean exact) { + if (exact) { + return FbsPrecision.Exact; + } + return FbsPrecision.Inexact; + } + + /// The `FileStatistics` FlatBuffer: one `ArrayStats` per top-level field, in schema order. + private static ByteBuffer buildFileStatistics(List fields) { + var fbb = new FbsBuilder(256 + 64 * fields.size()); + int[] offsets = new int[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + FileStatsAccumulator field = fields.get(i); + offsets[i] = buildStats(fbb, field.stats(), field.minExact(), field.maxExact()); + } + int vec = FbsFileStatistics.createFieldStatsVector(fbb, offsets); + fbb.finish(FbsFileStatistics.createFbsFileStatistics(fbb, vec)); + return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); + } + private int buildArrayNodeFlatBuffer(FbsBuilder fbb, EncodeNode node) { // Build children first (FlatBuffers bottom-up: nested objects before parent table) int[] childOffsets = new int[node.children().length]; @@ -418,7 +458,8 @@ private ByteBuffer buildFooter(LayoutId zonedLayout) { private static ByteBuffer buildPostscript( long footerOff, int footerLen, long dtypeOff, int dtypeLen, - long layoutOff, int layoutLen) { + long layoutOff, int layoutLen, + long statsOff, int statsLen) { var fbb = new FbsBuilder(256); int footerSegOff = FbsPostscriptSegment.createFbsPostscriptSegment( @@ -427,8 +468,12 @@ private static ByteBuffer buildPostscript( fbb, dtypeOff, dtypeLen, 0, 0, 0); int layoutSegOff = FbsPostscriptSegment.createFbsPostscriptSegment( fbb, layoutOff, layoutLen, 0, 0, 0); + int statsSegOff = 0; + if (statsLen > 0) { + statsSegOff = FbsPostscriptSegment.createFbsPostscriptSegment(fbb, statsOff, statsLen, 0, 0, 0); + } - int psOff = FbsPostscript.createFbsPostscript(fbb, dtypeSegOff, layoutSegOff, 0, footerSegOff); + int psOff = FbsPostscript.createFbsPostscript(fbb, dtypeSegOff, layoutSegOff, statsSegOff, footerSegOff); FbsPostscript.finishFbsPostscriptBuffer(fbb, psOff); return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index ee07065b8..c65d8fd91 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -64,6 +64,8 @@ public final class VortexWriter implements Closeable { private final boolean zonedLayout; private final GlobalDictWriter globalDict; private final ZoneMapWriter zoneMaps; + // Rust's footer file statistics, one per top-level field in schema order, fed the raw batches. + private final List fileStats; private final Map> colChunks; // Rust's repartition: each column's batches coalesce into ~1 MB chunks of 8192-row multiples. private final Map repartitioners; @@ -94,6 +96,7 @@ private VortexWriter( this.sink = new SegmentSink(channel, segmentEncoder, options.executor()); this.globalDict = new GlobalDictWriter(options, segmentEncoder, sink, zonedLayout, this::appendChunk); this.zoneMaps = new ZoneMapWriter(schema, options, zonedLayout, segmentEncoder, sink); + this.fileStats = schema.fieldTypes().stream().map(FileStatsAccumulator::new).toList(); this.colChunks = new LinkedHashMap<>(); for (ColumnName name : schema.fieldNames()) { colChunks.put(name, new ArrayList<>()); @@ -309,6 +312,7 @@ public void writeChunk(Map columns) throws IOException { Object data = adapted.get(colName); zoneMaps.add(colName, colDtype, data); + fileStats.get(i).add(data, expectedLen); if (!firstChunkSeen) { globalDict.detect(colName, colDtype, data); } @@ -363,7 +367,7 @@ public void close() throws IOException { zoned = LayoutId.ZONED; } sink.finish(zoned, dtypeSerializer.buildDType(schema), - LayoutTree.build(schema, colChunks, globalDict.written(), zoneMaps.written())); + LayoutTree.build(schema, colChunks, globalDict.written(), zoneMaps.written()), fileStats); } private int writeSegment(ColumnName column, DType dtype, Object data, EncodingEncoder encodingOverride) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java index f5e5dbbfe..529ad49ca 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/ZoneAccumulator.java @@ -32,9 +32,21 @@ final class ZoneAccumulator { static final int ZONE_LEN = 8192; /// Rust's `default_bounded_stat_max_bytes()`. - private static final int BOUNDED_MAX_BYTES = 64; + static final int BOUNDED_MAX_BYTES = 64; - private enum Kind { INT, F16, FLOAT, BOOL, DECIMAL, UTF8, BINARY, NULL_COUNT_ONLY } + /// How a column's values are compared, by its storage dtype. + enum Kind { INT, F16, FLOAT, BOOL, DECIMAL, UTF8, BINARY, NULL_COUNT_ONLY } + + /// A slice's extremes over its valid values (floats skipping NaN), as [#extremes] finds them. + /// + /// @param min the smallest value: a `Long` (integer bits), `Double` (F16 and floats), + /// `Boolean`, `BigDecimal` or `byte[]` (UTF-8 or binary, untruncated); `null` + /// when no value is valid + /// @param max the largest value, likewise + /// @param nulls the number of null rows + /// @param nans the number of NaN values + record Extremes(Object min, Object max, long nulls, long nans) { + } private final DType dtype; private final Kind kind; @@ -51,7 +63,12 @@ private enum Kind { INT, F16, FLOAT, BOOL, DECIMAL, UTF8, BINARY, NULL_COUNT_ONL this.dtype = dtype; DType storage = dtype instanceof DType.Extension ext ? ext.storageDType() : dtype; this.ptype = storage instanceof DType.Primitive p ? p.ptype() : null; - this.kind = switch (storage) { + this.kind = kindOf(storage); + } + + /// The [Kind] of a column whose storage dtype is `storage`. + static Kind kindOf(DType storage) { + return switch (storage) { case DType.Primitive p when p.ptype() == PType.F16 -> Kind.F16; case DType.Primitive p when p.ptype().isFloating() -> Kind.FLOAT; case DType.Primitive _ -> Kind.INT; @@ -280,18 +297,46 @@ private Object valueColumn(List perZone) { /// Records the zone of `len` rows starting at `off` in `data`. private void zone(Object data, int off, int len) { + Extremes extremes = extremes(kind, ptype, data, off, len); + Object min = extremes.min(); + Object max = extremes.max(); + boolean unknown = false; + if (min != null && kind == Kind.F16) { + // The F16 table column is stored as raw half-float bits like the data. + min = (long) Float.floatToFloat16(((Double) min).floatValue()); + max = (long) Float.floatToFloat16(((Double) max).floatValue()); + } else if (min != null && (kind == Kind.UTF8 || kind == Kind.BINARY)) { + byte[] lo = (byte[]) min; + byte[] hi = (byte[]) max; + min = kind == Kind.UTF8 ? utf8LowerBound(lo) : Arrays.copyOf(lo, Math.min(lo.length, BOUNDED_MAX_BYTES)); + max = kind == Kind.UTF8 ? utf8UpperBound(hi) : binaryUpperBound(hi); + unknown = max == null; + } + mins.add(min); + maxs.add(max); + maxUnknown.add(unknown); + nullCounts.add(extremes.nulls()); + nanCounts.add(extremes.nans()); + } + + /// The extremes and counts of the `len` rows starting at `off` in `data`, a carrier of `kind`. + static Extremes extremes(Kind kind, PType ptype, Object data, int off, int len) { Object values = data instanceof NullableData nd ? nd.values() : data; boolean[] validity = data instanceof NullableData nd ? nd.validity() : null; long nulls = 0; long nans = 0; Object min = null; Object max = null; - boolean unknown = false; // INT, FLOAT and BOOL track their bounds as primitives and box once per zone: boxing on every // new bound allocated a Long per row of an ascending column such as a row id. switch (kind) { case INT -> { - long[] longs = PrimitiveArrays.toLongs(slice(values, null, off, len), ptype, EncodingId.VORTEX_PRIMITIVE); + // a whole batch (the file statistics) needs no copy, and an I64/U64 one no widening either + Object range = values; + if (off != 0 || len != java.lang.reflect.Array.getLength(values)) { + range = slice(values, null, off, len); + } + long[] longs = PrimitiveArrays.toLongs(range, ptype, EncodingId.VORTEX_PRIMITIVE); boolean unsigned = ptype.isUnsigned(); boolean seen = false; long lo = 0; @@ -341,11 +386,7 @@ private void zone(Object data, int off, int len) { } seen = true; } - if (seen && kind == Kind.F16) { - // The F16 table column is stored as raw half-float bits like the data. - min = (long) Float.floatToFloat16((float) lo); - max = (long) Float.floatToFloat16((float) hi); - } else if (seen) { + if (seen) { min = lo; max = hi; } @@ -426,11 +467,8 @@ private void zone(Object data, int off, int len) { } } } - if (lo != null) { - min = kind == Kind.UTF8 ? utf8LowerBound(lo) : Arrays.copyOf(lo, Math.min(lo.length, BOUNDED_MAX_BYTES)); - max = kind == Kind.UTF8 ? utf8UpperBound(hi) : binaryUpperBound(hi); - unknown = max == null; - } + min = lo; + max = hi; } case NULL_COUNT_ONLY -> { if (validity != null) { @@ -442,11 +480,7 @@ private void zone(Object data, int off, int len) { } } } - mins.add(min); - maxs.add(max); - maxUnknown.add(unknown); - nullCounts.add(nulls); - nanCounts.add(nans); + return new Extremes(min, max, nulls, nans); } private static int compare(long a, long b, boolean unsigned) { @@ -526,14 +560,14 @@ private static int utf8Width(int codePoint) { /// Unwraps a pre-transformed carrier (e.g. the Parquet importer's [ComparableValues] /// `DateTimePartsData`) to the original values its stats are taken over. - private static Object comparable(Object batch) { + static Object comparable(Object batch) { if (batch instanceof NullableData(ComparableValues cv, boolean[] validity)) { return new NullableData(cv.values(), validity); } return batch instanceof ComparableValues cv ? cv.values() : batch; } - private static Object validityOnly(Object batch, int rows) { + static Object validityOnly(Object batch, int rows) { boolean[] placeholder = new boolean[rows]; return batch instanceof NullableData nd ? new NullableData(placeholder, nd.validity()) : placeholder; } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/FileStatsAccumulatorTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/FileStatsAccumulatorTest.java new file mode 100644 index 000000000..4471a5bf9 --- /dev/null +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/FileStatsAccumulatorTest.java @@ -0,0 +1,87 @@ +package io.github.dfa1.vortex.writer; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.DTypes; +import io.github.dfa1.vortex.core.proto.ProtoScalarValue; +import io.github.dfa1.vortex.writer.encode.NodeStats; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +/// The fold rules of [FileStatsAccumulator] that `FileStatisticsInteropIntegrationTest` cannot reach +/// with a small file: sums that overflow only past millions of rows, and columns without any batch. +class FileStatsAccumulatorTest { + + @Test + void noBatch_carriesNoStat() { + // Given — Rust drops every per-batch stat of a column it never saw a batch of: even the null + // count and the sum, which a column with rows always has + FileStatsAccumulator sut = new FileStatsAccumulator(DTypes.I64); + sut.add(new long[0], 0); + + // When + NodeStats result = sut.stats(); + + // Then + assertThat(result).isEqualTo(NodeStats.NONE); + } + + @Test + void variant_carriesNoStat() { + // Given — Rust's supports_file_stats excludes variant columns entirely + FileStatsAccumulator sut = new FileStatsAccumulator(DTypes.VARIANT); + sut.add(new Object(), 3); + + // When + NodeStats result = sut.stats(); + + // Then + assertThat(result).isEqualTo(NodeStats.NONE); + } + + @Test + void unsignedSumOverflowAcrossBatches_dropsSum() { + // Given — each batch fits u64, their total does not: the fold is checked too + FileStatsAccumulator sut = new FileStatsAccumulator(DTypes.U64); + sut.add(new long[]{-2L}, 1); + sut.add(new long[]{2L}, 1); + + // When + NodeStats result = sut.stats(); + + // Then — min and max are unaffected + assertThat(result.sum()).isNull(); + assertThat(result.max()).isEqualTo(ProtoScalarValue.ofUint64Value(-2L)); + } + + @Test + void decimalSumPastMaxPrecision_dropsSum() { + // Given — at Rust's maximum precision a sum cannot widen: two values near 10^76 overflow it + BigDecimal big = new BigDecimal(BigInteger.TEN.pow(76).subtract(BigInteger.ONE)); + FileStatsAccumulator sut = new FileStatsAccumulator(DType.Decimal.of(76, 0, false)); + sut.add(new BigDecimal[]{big, big}, 2); + + // When + NodeStats result = sut.stats(); + + // Then + assertThat(result.sum()).isNull(); + assertThat(result.min()).isNotNull(); + } + + @Test + void nullDtype_countsEveryRowNull() { + // Given + FileStatsAccumulator sut = new FileStatsAccumulator(DTypes.NULL); + sut.add(new Object(), 4); + + // When + NodeStats result = sut.stats(); + + // Then + assertThat(result).isEqualTo(new NodeStats(null, null, null, null, null, 4L, null, null, null)); + } +}