Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,14 @@ reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, A
reader.layout — Layout, LayoutDecoder, LayoutDecodeContext, LayoutRegistry
+ built-in *LayoutDecoder impls, ZonedStatsSchema
writer — VortexWriter, WriteRegistry, WriteOptions, ColumnEncoding, ExtensionEncoder
writer.encode — EncodingEncoder, EncodeContext, EncodeResult, EncodedBuffer, NullableData
VortexWriter orchestrates package-private parts (one concern each):
SegmentEncoder — picks and runs encoders; immutable, the only state executor tasks touch
SegmentSink — ordered segment writes to the channel, footer, postscript, trailer
LayoutTree — the layout tree + footer layout_specs; the only place FbsLayout is built
GlobalDictWriter — global dictionary columns: detect, buffer, demote, flush
ZoneMapWriter — zone maps (vortex.zoned, legacy vortex.stats)
writer.encode — EncodingEncoder, EncodeContext, EncodeResult, EncodedBuffer, NullableData,
RowCount (`RowCount.of(data)`: rows of any column carrier — never re-implement it)
+ *EncodingEncoder impls,
extension encoders
csv, parquet, jdbc — import/export between Vortex files and CSV, Parquet, and JDBC sources
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Changed
- `VortexWriter` is split into smaller internal parts and writes byte-identical files. ([#663](https://github.com/dfa1/vortex-java/issues/663))

### Fixed
- The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -373,13 +373,23 @@ private void record(short[] codes, boolean[] validity, int len, ProtoScalarValue
chunkCodes.add(codes);
chunkValidity.add(validity);
chunkRowCounts.add((long) len);
chunkNullCounts.add(validity != null ? VortexWriter.countNulls(validity) : 0L);
chunkNullCounts.add(validity != null ? countNulls(validity) : 0L);
chunkStatsMin.add(min);
chunkStatsMax.add(max);
chunkStatsSum.add(sum);
codeArrayBytes += 2L * len;
}

private static long countNulls(boolean[] validity) {
long nulls = 0;
for (boolean valid : validity) {
if (!valid) {
nulls++;
}
}
return nulls;
}

// Gives the null entry the next code, as Rust's `encode_null`; false when it no longer fits.
private boolean addNullEntry(int cap) {
if (!valueToCode.isEmpty() && valueToCode.size() >= cap) {
Expand Down

Large diffs are not rendered by default.

175 changes: 175 additions & 0 deletions writer/src/main/java/io/github/dfa1/vortex/writer/LayoutTree.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
package io.github.dfa1.vortex.writer;

import io.github.dfa1.vortex.core.fbs.FbsBuilder;
import io.github.dfa1.vortex.core.fbs.FbsFooter;
import io.github.dfa1.vortex.core.fbs.FbsLayout;
import io.github.dfa1.vortex.core.fbs.FbsLayoutSpec;
import io.github.dfa1.vortex.core.model.ColumnName;
import io.github.dfa1.vortex.core.model.DType;
import io.github.dfa1.vortex.core.model.LayoutId;
import io.github.dfa1.vortex.core.model.PType;
import io.github.dfa1.vortex.writer.GlobalDictWriter.DictColRef;
import io.github.dfa1.vortex.writer.VortexWriter.ChunkRef;
import io.github.dfa1.vortex.writer.ZoneMapWriter.ZoneMapRef;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;
import java.util.Map;

/// The file's layout tree, and the footer's `layout_specs` list its nodes index into: every
/// layout node the writer emits is built here.
///
/// The root is a struct with one child per column: a chunked layout of flat segments, or the
/// column's dict layout (several in a chunked layout), each wrapped in its zone map when it has one.
final class LayoutTree {

// Indices into the footer's layout_specs list, in the order specs() writes them
private static final int FLAT = 0;
private static final int CHUNKED = 1;
private static final int STRUCT = 2;
private static final int DICT = 3;
private static final int ZONED = 4;

private LayoutTree() {
}

/// The footer's `layout_specs` vector, in index order: FLAT, CHUNKED, STRUCT, DICT, then the
/// zoned layout: Rust's `vortex.zoned` when the edition has it, else the legacy `vortex.stats`
/// alias every reader accepts.
static int specs(FbsBuilder fbb, LayoutId zoned) {
int ls0 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(LayoutId.FLAT.id()));
int ls1 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(LayoutId.CHUNKED.id()));
int ls2 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(LayoutId.STRUCT.id()));
int ls3 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(LayoutId.DICT.id()));
int ls4 = FbsLayoutSpec.createFbsLayoutSpec(fbb, fbb.createString(zoned.id()));
return FbsFooter.createLayoutSpecsVector(fbb, new int[]{ls0, ls1, ls2, ls3, ls4});
}

/// The serialized layout tree of a file with `schema`'s columns.
///
/// @param chunks each column's written chunks, for the columns not in `dicts`
/// @param dicts the columns written as global dictionaries
/// @param zoneMaps the columns that carry a zone map
static ByteBuffer build(DType.Struct schema, Map<ColumnName, List<ChunkRef>> chunks,
Map<ColumnName, List<DictColRef>> dicts, Map<ColumnName, ZoneMapRef> zoneMaps) {
var fbb = new FbsBuilder(256);
int colCount = schema.fieldNames().size();

int[] colLayouts = new int[colCount];
long totalRows = 0;

for (int c = 0; c < colCount; c++) {
ColumnName colName = schema.fieldNames().get(c);
List<DictColRef> refs = dicts.get(colName);
if (refs != null) {
long colRows = rows(refs);
colLayouts[c] = zoned(fbb, zoneMaps.get(colName), dictLayout(fbb, refs), colRows);
if (totalRows == 0) {
totalRows = colRows;
}
} else {
List<ChunkRef> colChunks = chunks.get(colName);
long colRows = 0;
int[] flats = new int[colChunks.size()];
for (int i = 0; i < colChunks.size(); i++) {
ChunkRef cr = colChunks.get(i);
int segV = FbsLayout.createSegmentsVector(fbb, new long[]{cr.segIdx()});
flats[i] = FbsLayout.createFbsLayout(fbb, FLAT, cr.rowCount(), 0, 0, segV);
colRows += cr.rowCount();
}
int childV = FbsLayout.createChildrenVector(fbb, flats);
int dataChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, colRows, 0, childV, 0);
colLayouts[c] = zoned(fbb, zoneMaps.get(colName), dataChunked, colRows);
if (totalRows == 0) {
totalRows = colRows;
}
}
}

int rootChildV = FbsLayout.createChildrenVector(fbb, colLayouts);
int rootLayout = FbsLayout.createFbsLayout(fbb, STRUCT, totalRows, 0, rootChildV, 0);
FbsLayout.finishFbsLayoutBuffer(fbb, rootLayout);
return fbb.dataSegment().asByteBuffer().order(ByteOrder.LITTLE_ENDIAN);
}

/// Wraps a column's data layout in a zoned layout when a zone-map was emitted for it (`zm`
/// not `null`); otherwise returns the data layout unchanged.
private static int zoned(FbsBuilder fbb, ZoneMapRef zm, int dataLayout, long colRows) {
if (zm == null) {
return dataLayout;
}
int zonesSegV = FbsLayout.createSegmentsVector(fbb, new long[]{zm.zonesSegIdx()});
int zonesFlat = FbsLayout.createFbsLayout(fbb, FLAT, zm.nZones(), 0, 0, zonesSegV);
int childV = FbsLayout.createChildrenVector(fbb, new int[]{dataLayout, zonesFlat});
int metaV = FbsLayout.createMetadataVector(fbb, zm.metadata());
return FbsLayout.createFbsLayout(fbb, ZONED, colRows, metaV, childV, 0);
}

/// A column's dict layout: one dictionary is the column's layout; several are a chunked layout
/// of them.
private static int dictLayout(FbsBuilder fbb, List<DictColRef> refs) {
int[] dictLayouts = new int[refs.size()];
for (int i = 0; i < dictLayouts.length; i++) {
dictLayouts[i] = dictColLayout(fbb, refs.get(i));
}
int dictLayout = dictLayouts[0];
if (dictLayouts.length > 1) {
dictLayout = FbsLayout.createFbsLayout(fbb, CHUNKED, rows(refs), 0,
FbsLayout.createChildrenVector(fbb, dictLayouts), 0);
}
return dictLayout;
}

private static long rows(List<DictColRef> refs) {
long rows = 0;
for (DictColRef ref : refs) {
rows += ref.totalRows();
}
return rows;
}

private static int dictColLayout(FbsBuilder fbb, DictColRef ref) {
// Build codes Chunked layout: children are one Flat per original chunk
int numChunks = ref.codesSegIdxes().size();
int[] codesFlats = new int[numChunks];
long totalCodesRows = 0;
for (int j = 0; j < numChunks; j++) {
long rowCount = ref.codesRowCounts().get(j);
int segV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.codesSegIdxes().get(j)});
codesFlats[j] = FbsLayout.createFbsLayout(fbb, FLAT, rowCount, 0, 0, segV);
totalCodesRows += rowCount;
}
int codesChildV = FbsLayout.createChildrenVector(fbb, codesFlats);
int codesChunked = FbsLayout.createFbsLayout(fbb, CHUNKED, totalCodesRows, 0, codesChildV, 0);

// Build values Flat layout
int valSegV = FbsLayout.createSegmentsVector(fbb, new long[]{ref.valuesSegIdx()});
int valuesFlat = FbsLayout.createFbsLayout(fbb, FLAT, ref.valuesLen(), 0, 0, valSegV);

// DictLayoutMetadata proto (matches Rust): field 1 = codes_ptype (PType varint)
byte[] metaBytes = dictLayoutMetadata(DictColumnState.CODES_PTYPE, ref.nullableCodes());
int metaVec = metaBytes.length > 0 ? FbsLayout.createMetadataVector(fbb, metaBytes) : 0;

// Dict layout: child[0]=values, child[1]=codes (matches Rust DictLayout child order)
int[] dictChildren = {valuesFlat, codesChunked};
int dictChildV = FbsLayout.createChildrenVector(fbb, dictChildren);
return FbsLayout.createFbsLayout(fbb, DICT, totalCodesRows, metaVec, dictChildV, 0);
}

/// Rust's `DictLayoutMetadata`: field 1 `codes_ptype`, field 2 `is_nullable_codes`, field 3
/// `all_values_referenced`. Rust's writer always sets fields 2 and 3 (field 3 as `false`: its
/// `DictLayout::new` never claims every value is referenced), so the metadata matches
/// vortex-jni's byte for byte.
private static byte[] dictLayoutMetadata(PType codePType, boolean nullableCodes) {
int ordinal = codePType.ordinal();
// Fields 2 and 3, wire type 0 (varint): tags (2<<3)|0 = 0x10 and (3<<3)|0 = 0x18
byte[] flags = {0x10, (byte) (nullableCodes ? 1 : 0), 0x18, 0};
if (ordinal == 0) {
// Proto3 omits default values; U8 ordinal=0 is the default
return flags;
}
// Field 1, wire type 0 (varint): tag = (1<<3)|0 = 0x08
return new byte[]{0x08, (byte) ordinal, flags[0], flags[1], flags[2], flags[3]};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import io.github.dfa1.vortex.writer.encode.ComparableValues;
import io.github.dfa1.vortex.writer.encode.DateTimePartsData;
import io.github.dfa1.vortex.writer.encode.NullableData;
import io.github.dfa1.vortex.writer.encode.RowCount;

import java.lang.reflect.Array;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -97,7 +98,7 @@ static Object head(Object data, int rows) {
/// @param from how many leading rows to drop, less than the batch's row count
/// @return the remaining rows
static Object tail(Object data, int from) {
return concat(List.of(new Piece(data, from, (int) VortexWriter.arrayLength(data) - from, 0)));
return concat(List.of(new Piece(data, from, RowCount.of(data) - from, 0)));
}

/// Whether sizing a batch of `data` reads its values, as for strings and binaries -- the
Expand Down
Loading
Loading