Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- `EncodeContext` has a new last component, `trace`, so code calling its canonical constructor passes `EncodeTrace.NONE`. ([#601](https://github.com/dfa1/vortex-java/issues/601))
- **Breaking:** `ArrayStats` moves from `io.github.dfa1.vortex.reader` to `io.github.dfa1.vortex.core.model`, and `ArrayStats.fromFbs` becomes `ArrayStatsParser.parse`. ([#664](https://github.com/dfa1/vortex-java/issues/664))
- Writes are about 6% faster: a segment is summed only for the legacy `vortex.stats` zone map, the one that records the sum. ([#712](https://github.com/dfa1/vortex-java/pull/712))

### Removed
- `CompressorBuilder#seed(long)` of the `vortex-fsst` module: training takes Rust's fixed sample, not a seeded one. ([#583](https://github.com/dfa1/vortex-java/issues/583))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ final class DictColumnState {
private final boolean binary;
private final PType ptype;
private final boolean nullable;
// Only the legacy vortex.stats zone map records a sum: under vortex.zoned it is never read.
private final boolean chunkSums;
// First-seen value -> code map (keys are boxed primitives, String, or ByteBuffer — see varBinKey).
private final Map<Object, Integer> valueToCode = new LinkedHashMap<>();
// Hot-path side index for the primitive path: raw value bits -> code + 1 (0 == empty), open
Expand All @@ -84,8 +86,9 @@ final class DictColumnState {
// Rust's `dict_bytes()`: a 16-byte view per entry plus the bytes of each value too long to inline.
private long dictBytes;

DictColumnState(DType dtype) {
DictColumnState(DType dtype, boolean chunkSums) {
this.dtype = dtype;
this.chunkSums = chunkSums;
this.binary = dtype instanceof DType.Binary;
this.varBin = binary || dtype instanceof DType.Utf8;
this.ptype = dtype instanceof DType.Primitive p ? p.ptype() : null;
Expand Down Expand Up @@ -233,8 +236,11 @@ int ingestDictChunk(Object data) {
values = prefix(values, consumed);
}
MinMax mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values);
record(codes, validity, consumed, mm != null ? mm.min() : null, mm != null ? mm.max() : null,
PrimitiveEncodingEncoder.sumStat(ptype, values));
ProtoScalarValue sum = null;
if (chunkSums) {
sum = PrimitiveEncodingEncoder.sumStat(ptype, values);
}
record(codes, validity, consumed, mm != null ? mm.min() : null, mm != null ? mm.max() : null, sum);
return consumed;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,11 @@ interface ChunkAppender {
this.dictColRefs = new LinkedHashMap<>();
}

/// Whether the legacy `vortex.stats` zone map is written, the only reader of the chunk sums.
private boolean legacyZoneMap() {
return options.enableZoneMaps() && !repartitionCodes;
}

/// Decides from a column's first batch whether it becomes a candidate.
///
/// Candidate detection inspects raw primitive/String/byte[] arrays. Nullable columns (carried as
Expand Down Expand Up @@ -107,7 +112,7 @@ void detect(ColumnName colName, DType colDtype, Object data) {
validity != null ? Arrays.copyOf(validity, probeRows) : null);
}
if (candidate) {
dictStates.put(colName, new DictColumnState(colDtype));
dictStates.put(colName, new DictColumnState(colDtype, legacyZoneMap()));
}
}

Expand Down Expand Up @@ -170,7 +175,7 @@ private void ingestChunk(ColumnName colName, DType colDtype, Object data) {
return;
}
dictRuns.computeIfAbsent(colName, _ -> new ArrayList<>()).add(state);
dictStates.put(colName, new DictColumnState(colDtype));
dictStates.put(colName, new DictColumnState(colDtype, legacyZoneMap()));
rest = consumed == 0 ? rest : Repartitioner.tail(rest, consumed);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ final class SegmentEncoder {
private static final List<EncodingId.WellKnown> WELL_KNOWN_IDS = List.of(EncodingId.WellKnown.values());

private final WriteOptions options;
// Only the legacy vortex.stats zone map records a segment's sum: under vortex.zoned it is never read.
private final boolean segmentSums;
// a copy, so a caller's list changed after create() cannot race the encoding tasks
private final List<EncodingEncoder> encodings;
// whether the writer was created with DEFAULT_CODECS rather than a caller's encoder list
Expand Down Expand Up @@ -103,8 +105,9 @@ final class SegmentEncoder {

/// @throws IllegalArgumentException if a [ColumnEncoding] names a column outside `columns` or an
/// encoding this writer cannot produce
SegmentEncoder(WriteOptions options, List<EncodingEncoder> encodings, List<ColumnName> columns) {
SegmentEncoder(WriteOptions options, List<EncodingEncoder> encodings, List<ColumnName> columns, boolean segmentSums) {
this.options = options;
this.segmentSums = segmentSums;
this.defaultCodecs = encodings == DEFAULT_CODECS;
this.encodings = List.copyOf(encodings);
this.defaultRegistry = buildRegistry(this.encodings);
Expand Down Expand Up @@ -201,7 +204,11 @@ EncodedSegment encode(EncodeTrace trace, DType dtype, Object data, EncodingEncod
min = fallback != null ? fallback.min() : null;
max = fallback != null ? fallback.max() : null;
}
SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount);
ProtoScalarValue sum = null;
if (segmentSums) {
sum = ZoneMapStatCodec.columnSum(dtype, data);
}
SegmentStats stats = new SegmentStats(min, max, sum, nullCount);
// the stats the flat segment's root node carries are Rust's, not the zone map's
result = result.withRootStats(dtype, min, max, RowCount.of(data), nullCount);
if (event.shouldCommit()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,9 @@ private VortexWriter(
this.dtypeSerializer = new DTypeFbsSerializer(extensionRegistry);
validateExtensions(schema);
this.zonedLayout = options.enableZoneMaps() && editionHasZonedLayout(options.editions());
this.segmentEncoder = new SegmentEncoder(options, encodings, schema.fieldNames());
// a segment's sum feeds only the legacy vortex.stats zone map
this.segmentEncoder = new SegmentEncoder(options, encodings, schema.fieldNames(),
options.enableZoneMaps() && !zonedLayout);
this.sink = new SegmentSink(channel, segmentEncoder, options.executor());
this.globalDict = new GlobalDictWriter(options, segmentEncoder, sink, zonedLayout, this::appendChunk);
this.zoneMaps = new ZoneMapWriter(schema, options, zonedLayout, segmentEncoder, sink);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ void ingestThenReconstruct_everyPtype_roundTripsInItsCarrier(PType ptype) {
case F64 -> new double[]{7, 8, 0, 7, 9};
};
boolean[] validity = {true, true, false, true, true};
var sut = new DictColumnState(new DType.Primitive(ptype, true));
var sut = new DictColumnState(new DType.Primitive(ptype, true), true);

// When
int ingested = sut.ingestDictChunk(new NullableData(values, validity));
Expand Down Expand Up @@ -129,7 +129,7 @@ void isVarBinDictCandidate_binaryComparesContentNotIdentity() {
@Test
void ingestDictChunk_binary_dedupsByContentAndReconstructs() {
// Given — a nullable Binary column whose equal values arrive as distinct byte[] instances
var sut = new DictColumnState(new DType.Binary(true));
var sut = new DictColumnState(new DType.Binary(true), true);
byte[][] values = {{1, 2}, {3}, null, {1, 2}, {3}};
boolean[] validity = {true, true, false, true, true};

Expand All @@ -155,7 +155,7 @@ void ingestDictChunk_assignsCodesFirstSeen_nullIncluded() {
// at its first appearance. The dominant 0.0 arrives after a rare value: a frequency ranking
// (the old remap) would make it code 0 and push null to the end, so the codes reaching the
// cascade differed from Rust's.
var sut = new DictColumnState(new DType.Primitive(PType.F64, true));
var sut = new DictColumnState(new DType.Primitive(PType.F64, true), true);
double[] values = {1.75, 0.0, 0.0, 0.0, 0.0, -1.75};
boolean[] validity = {true, true, false, true, false, true};

Expand All @@ -173,7 +173,7 @@ void ingestDictChunk_assignsCodesFirstSeen_nullIncluded() {
@Test
void ingestDictChunk_stopsAtTheNullEntryThatNoLongerFits() {
// Given — a dictionary filled to u16::MAX entries; the next chunk's first row is a null
var sut = new DictColumnState(new DType.Primitive(PType.I64, true));
var sut = new DictColumnState(new DType.Primitive(PType.I64, true), true);
long[] full = new long[DictColumnState.MAX_ENTRIES];
for (int i = 0; i < full.length; i++) {
full[i] = i;
Expand All @@ -194,7 +194,7 @@ void ingestDictChunk_stopsAtTheNullEntryThatNoLongerFits() {
@Test
void ingestDictChunk_stopsAtTheValueThatNoLongerFitsAndKeepsThePrefixStats() {
// Given — a dictionary filled to u16::MAX entries
var sut = new DictColumnState(new DType.Primitive(PType.I64, false));
var sut = new DictColumnState(new DType.Primitive(PType.I64, false), true);
long[] full = new long[DictColumnState.MAX_ENTRIES];
for (int i = 0; i < full.length; i++) {
full[i] = i;
Expand Down Expand Up @@ -262,7 +262,7 @@ void readPrimitiveElement_returnsElementAtIndex(Object data, PType ptype, int in
@Test
void ingestDictChunkReportsPerChunkStringMinMax() {
// Given
DictColumnState sut = new DictColumnState(new DType.Utf8(true));
DictColumnState sut = new DictColumnState(new DType.Utf8(true), true);
String emoji = new String(Character.toChars(0x1F600));

// When
Expand Down Expand Up @@ -312,7 +312,7 @@ private static String[] repeated(int distinct) {
@Test
void ingestVarBinChunk_stopsAtTheEntryThatNoLongerFits() {
// Given — a dictionary filled to u16::MAX entries
DictColumnState sut = new DictColumnState(new DType.Utf8(false));
DictColumnState sut = new DictColumnState(new DType.Utf8(false), true);
String[] full = new String[DictColumnState.MAX_ENTRIES];
for (int i = 0; i < full.length; i++) {
full[i] = "v" + i;
Expand All @@ -331,7 +331,7 @@ void ingestVarBinChunk_stopsAtTheEntryThatNoLongerFits() {
@Test
void ingestVarBinChunk_countsViewsAndOutOfLineBytesAgainstTheByteLimit() {
// Given — 20-byte values cost a 16-byte view plus themselves: 36 bytes, so 29127 fit in 1 MiB
DictColumnState sut = new DictColumnState(new DType.Utf8(false));
DictColumnState sut = new DictColumnState(new DType.Utf8(false), true);
String[] values = new String[40_000];
for (int i = 0; i < values.length; i++) {
values[i] = String.format("%020d", i);
Expand All @@ -347,7 +347,7 @@ void ingestVarBinChunk_countsViewsAndOutOfLineBytesAgainstTheByteLimit() {
@Test
void ingestVarBinChunk_admitsTheFirstEntryWhateverItsSize() {
// Given — one value past the byte limit on its own
DictColumnState sut = new DictColumnState(new DType.Utf8(false));
DictColumnState sut = new DictColumnState(new DType.Utf8(false), true);
String huge = "x".repeat((int) DictColumnState.MAX_DICT_BYTES + 1);

// When
Expand All @@ -360,7 +360,7 @@ void ingestVarBinChunk_admitsTheFirstEntryWhateverItsSize() {
@Test
void ingestVarBinChunk_aFullDictionaryTakesNothingNew() {
// Given
DictColumnState sut = new DictColumnState(new DType.Utf8(false));
DictColumnState sut = new DictColumnState(new DType.Utf8(false), true);
sut.ingestVarBinChunk(new String[]{"x".repeat((int) DictColumnState.MAX_DICT_BYTES + 1)});

// When
Expand Down
Loading