diff --git a/CHANGELOG.md b/CHANGELOG.md index a69dc73b5..1cac3757f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - `EncodeContext` has a new last component, `trace`, so code calling its canonical constructor passes `EncodeTrace.NONE`. ([#601](https://github.com/dfa1/vortex-java/issues/601)) - **Breaking:** `ArrayStats` moves from `io.github.dfa1.vortex.reader` to `io.github.dfa1.vortex.core.model`, and `ArrayStats.fromFbs` becomes `ArrayStatsParser.parse`. ([#664](https://github.com/dfa1/vortex-java/issues/664)) +- Writes are about 6% faster: a segment is summed only for the legacy `vortex.stats` zone map, the one that records the sum. ([#712](https://github.com/dfa1/vortex-java/pull/712)) ### Removed - `CompressorBuilder#seed(long)` of the `vortex-fsst` module: training takes Rust's fixed sample, not a seeded one. ([#583](https://github.com/dfa1/vortex-java/issues/583)) diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java index 3ec25bee8..6666c1e4e 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/DictColumnState.java @@ -58,6 +58,8 @@ final class DictColumnState { private final boolean binary; private final PType ptype; private final boolean nullable; + // Only the legacy vortex.stats zone map records a sum: under vortex.zoned it is never read. + private final boolean chunkSums; // First-seen value -> code map (keys are boxed primitives, String, or ByteBuffer — see varBinKey). private final Map valueToCode = new LinkedHashMap<>(); // Hot-path side index for the primitive path: raw value bits -> code + 1 (0 == empty), open @@ -84,8 +86,9 @@ final class DictColumnState { // Rust's `dict_bytes()`: a 16-byte view per entry plus the bytes of each value too long to inline. private long dictBytes; - DictColumnState(DType dtype) { + DictColumnState(DType dtype, boolean chunkSums) { this.dtype = dtype; + this.chunkSums = chunkSums; this.binary = dtype instanceof DType.Binary; this.varBin = binary || dtype instanceof DType.Utf8; this.ptype = dtype instanceof DType.Primitive p ? p.ptype() : null; @@ -233,8 +236,11 @@ int ingestDictChunk(Object data) { values = prefix(values, consumed); } MinMax mm = PrimitiveEncodingEncoder.minMaxStats(ptype, values); - record(codes, validity, consumed, mm != null ? mm.min() : null, mm != null ? mm.max() : null, - PrimitiveEncodingEncoder.sumStat(ptype, values)); + ProtoScalarValue sum = null; + if (chunkSums) { + sum = PrimitiveEncodingEncoder.sumStat(ptype, values); + } + record(codes, validity, consumed, mm != null ? mm.min() : null, mm != null ? mm.max() : null, sum); return consumed; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/GlobalDictWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/GlobalDictWriter.java index efba4662e..b291ad97f 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/GlobalDictWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/GlobalDictWriter.java @@ -67,6 +67,11 @@ interface ChunkAppender { this.dictColRefs = new LinkedHashMap<>(); } + /// Whether the legacy `vortex.stats` zone map is written, the only reader of the chunk sums. + private boolean legacyZoneMap() { + return options.enableZoneMaps() && !repartitionCodes; + } + /// Decides from a column's first batch whether it becomes a candidate. /// /// Candidate detection inspects raw primitive/String/byte[] arrays. Nullable columns (carried as @@ -107,7 +112,7 @@ void detect(ColumnName colName, DType colDtype, Object data) { validity != null ? Arrays.copyOf(validity, probeRows) : null); } if (candidate) { - dictStates.put(colName, new DictColumnState(colDtype)); + dictStates.put(colName, new DictColumnState(colDtype, legacyZoneMap())); } } @@ -170,7 +175,7 @@ private void ingestChunk(ColumnName colName, DType colDtype, Object data) { return; } dictRuns.computeIfAbsent(colName, _ -> new ArrayList<>()).add(state); - dictStates.put(colName, new DictColumnState(colDtype)); + dictStates.put(colName, new DictColumnState(colDtype, legacyZoneMap())); rest = consumed == 0 ? rest : Repartitioner.tail(rest, consumed); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentEncoder.java index 24b0483ed..977b25a44 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/SegmentEncoder.java @@ -73,6 +73,8 @@ final class SegmentEncoder { private static final List WELL_KNOWN_IDS = List.of(EncodingId.WellKnown.values()); private final WriteOptions options; + // Only the legacy vortex.stats zone map records a segment's sum: under vortex.zoned it is never read. + private final boolean segmentSums; // a copy, so a caller's list changed after create() cannot race the encoding tasks private final List encodings; // whether the writer was created with DEFAULT_CODECS rather than a caller's encoder list @@ -103,8 +105,9 @@ final class SegmentEncoder { /// @throws IllegalArgumentException if a [ColumnEncoding] names a column outside `columns` or an /// encoding this writer cannot produce - SegmentEncoder(WriteOptions options, List encodings, List columns) { + SegmentEncoder(WriteOptions options, List encodings, List columns, boolean segmentSums) { this.options = options; + this.segmentSums = segmentSums; this.defaultCodecs = encodings == DEFAULT_CODECS; this.encodings = List.copyOf(encodings); this.defaultRegistry = buildRegistry(this.encodings); @@ -201,7 +204,11 @@ EncodedSegment encode(EncodeTrace trace, DType dtype, Object data, EncodingEncod min = fallback != null ? fallback.min() : null; max = fallback != null ? fallback.max() : null; } - SegmentStats stats = new SegmentStats(min, max, ZoneMapStatCodec.columnSum(dtype, data), nullCount); + ProtoScalarValue sum = null; + if (segmentSums) { + sum = ZoneMapStatCodec.columnSum(dtype, data); + } + SegmentStats stats = new SegmentStats(min, max, sum, nullCount); // the stats the flat segment's root node carries are Rust's, not the zone map's result = result.withRootStats(dtype, min, max, RowCount.of(data), nullCount); if (event.shouldCommit()) { diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index e144b64bc..dd38f046a 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -96,7 +96,9 @@ private VortexWriter( this.dtypeSerializer = new DTypeFbsSerializer(extensionRegistry); validateExtensions(schema); this.zonedLayout = options.enableZoneMaps() && editionHasZonedLayout(options.editions()); - this.segmentEncoder = new SegmentEncoder(options, encodings, schema.fieldNames()); + // a segment's sum feeds only the legacy vortex.stats zone map + this.segmentEncoder = new SegmentEncoder(options, encodings, schema.fieldNames(), + options.enableZoneMaps() && !zonedLayout); this.sink = new SegmentSink(channel, segmentEncoder, options.executor()); this.globalDict = new GlobalDictWriter(options, segmentEncoder, sink, zonedLayout, this::appendChunk); this.zoneMaps = new ZoneMapWriter(schema, options, zonedLayout, segmentEncoder, sink); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java index 553a10aed..0fd25ac22 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/DictColumnStateTest.java @@ -39,7 +39,7 @@ void ingestThenReconstruct_everyPtype_roundTripsInItsCarrier(PType ptype) { case F64 -> new double[]{7, 8, 0, 7, 9}; }; boolean[] validity = {true, true, false, true, true}; - var sut = new DictColumnState(new DType.Primitive(ptype, true)); + var sut = new DictColumnState(new DType.Primitive(ptype, true), true); // When int ingested = sut.ingestDictChunk(new NullableData(values, validity)); @@ -129,7 +129,7 @@ void isVarBinDictCandidate_binaryComparesContentNotIdentity() { @Test void ingestDictChunk_binary_dedupsByContentAndReconstructs() { // Given — a nullable Binary column whose equal values arrive as distinct byte[] instances - var sut = new DictColumnState(new DType.Binary(true)); + var sut = new DictColumnState(new DType.Binary(true), true); byte[][] values = {{1, 2}, {3}, null, {1, 2}, {3}}; boolean[] validity = {true, true, false, true, true}; @@ -155,7 +155,7 @@ void ingestDictChunk_assignsCodesFirstSeen_nullIncluded() { // at its first appearance. The dominant 0.0 arrives after a rare value: a frequency ranking // (the old remap) would make it code 0 and push null to the end, so the codes reaching the // cascade differed from Rust's. - var sut = new DictColumnState(new DType.Primitive(PType.F64, true)); + var sut = new DictColumnState(new DType.Primitive(PType.F64, true), true); double[] values = {1.75, 0.0, 0.0, 0.0, 0.0, -1.75}; boolean[] validity = {true, true, false, true, false, true}; @@ -173,7 +173,7 @@ void ingestDictChunk_assignsCodesFirstSeen_nullIncluded() { @Test void ingestDictChunk_stopsAtTheNullEntryThatNoLongerFits() { // Given — a dictionary filled to u16::MAX entries; the next chunk's first row is a null - var sut = new DictColumnState(new DType.Primitive(PType.I64, true)); + var sut = new DictColumnState(new DType.Primitive(PType.I64, true), true); long[] full = new long[DictColumnState.MAX_ENTRIES]; for (int i = 0; i < full.length; i++) { full[i] = i; @@ -194,7 +194,7 @@ void ingestDictChunk_stopsAtTheNullEntryThatNoLongerFits() { @Test void ingestDictChunk_stopsAtTheValueThatNoLongerFitsAndKeepsThePrefixStats() { // Given — a dictionary filled to u16::MAX entries - var sut = new DictColumnState(new DType.Primitive(PType.I64, false)); + var sut = new DictColumnState(new DType.Primitive(PType.I64, false), true); long[] full = new long[DictColumnState.MAX_ENTRIES]; for (int i = 0; i < full.length; i++) { full[i] = i; @@ -262,7 +262,7 @@ void readPrimitiveElement_returnsElementAtIndex(Object data, PType ptype, int in @Test void ingestDictChunkReportsPerChunkStringMinMax() { // Given - DictColumnState sut = new DictColumnState(new DType.Utf8(true)); + DictColumnState sut = new DictColumnState(new DType.Utf8(true), true); String emoji = new String(Character.toChars(0x1F600)); // When @@ -312,7 +312,7 @@ private static String[] repeated(int distinct) { @Test void ingestVarBinChunk_stopsAtTheEntryThatNoLongerFits() { // Given — a dictionary filled to u16::MAX entries - DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + DictColumnState sut = new DictColumnState(new DType.Utf8(false), true); String[] full = new String[DictColumnState.MAX_ENTRIES]; for (int i = 0; i < full.length; i++) { full[i] = "v" + i; @@ -331,7 +331,7 @@ void ingestVarBinChunk_stopsAtTheEntryThatNoLongerFits() { @Test void ingestVarBinChunk_countsViewsAndOutOfLineBytesAgainstTheByteLimit() { // Given — 20-byte values cost a 16-byte view plus themselves: 36 bytes, so 29127 fit in 1 MiB - DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + DictColumnState sut = new DictColumnState(new DType.Utf8(false), true); String[] values = new String[40_000]; for (int i = 0; i < values.length; i++) { values[i] = String.format("%020d", i); @@ -347,7 +347,7 @@ void ingestVarBinChunk_countsViewsAndOutOfLineBytesAgainstTheByteLimit() { @Test void ingestVarBinChunk_admitsTheFirstEntryWhateverItsSize() { // Given — one value past the byte limit on its own - DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + DictColumnState sut = new DictColumnState(new DType.Utf8(false), true); String huge = "x".repeat((int) DictColumnState.MAX_DICT_BYTES + 1); // When @@ -360,7 +360,7 @@ void ingestVarBinChunk_admitsTheFirstEntryWhateverItsSize() { @Test void ingestVarBinChunk_aFullDictionaryTakesNothingNew() { // Given - DictColumnState sut = new DictColumnState(new DType.Utf8(false)); + DictColumnState sut = new DictColumnState(new DType.Utf8(false), true); sut.ingestVarBinChunk(new String[]{"x".repeat((int) DictColumnState.MAX_DICT_BYTES + 1)}); // When