Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- The array stats on a flat segment's root node are the ones Rust keeps: integers under `vortex.primitive` and `fastlanes.bitpacked` carry min, max, `is_constant`, `null_count` and the uncompressed size, and every other encoding carries none, so a Java-written string column reports no flat `min`/`max`, as a Rust-written one. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- The child nodes of a flat segment carry the array stats Rust keeps: a sequence its sorted flags, a nullable integer's validity bitmap its true count, and patch values `is_constant = false`. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- The zone table of a column holds Rust's shape: no validity or mask on a column without nulls, and a constant bool stays a `vortex.bool` until the constant is smaller. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- Text columns stay `vortex.varbinview`, Rust's canonical form, unless a scheme is smaller, and a `vortex.varbin` is only chosen for binary. ([#583](https://github.com/dfa1/vortex-java/issues/583))
- The compact preset writes text columns as zstd frames of 8192 values, as Rust does, so a row-range scan decompresses only the frames it overlaps. ([#636](https://github.com/dfa1/vortex-java/issues/636))
- A nullable decimal column keeps its validity in its own encoding, or has none when every row is valid, instead of a `vortex.masked` wrapper, and a column of equal decimals is one constant, as Rust writes them: decimal zone-stats tables are smaller. ([#646](https://github.com/dfa1/vortex-java/issues/646))
- `VortexAggregates` summarizes a `U64` column exactly: `MIN`, `MAX` and `SUM` compare and add unsigned instead of failing past 2^63. ([#665](https://github.com/dfa1/vortex-java/issues/665))
Expand Down
3 changes: 2 additions & 1 deletion docs/compatibility.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,8 @@ patches carry `is_constant = false`, and the `vortex.bool` validity bitmap of a
when every row is valid (or none is) and otherwise `min`, `max`, `sum` (the valid-row count) and `null_count`.
`NodeStatsInteropIntegrationTest` compares every flat root, and the whole tree of the numeric columns, against
vortex-jni. The string columns' children (FSST lengths and codes, whose values depend on the trained symbol
table) and the zone-table segments of string columns, which wait for Rust's `vortex.varbinview` canonical form, are not covered yet
table) are not covered, nor is the zone table of a column whose few bounds Rust's trained table compresses and ours does not.
Text is `vortex.varbinview` unless a scheme beats its size (`vortex.varbin` is a binary-only scheme, as in Rust), and that node carries `is_constant`, `null_count` and the size of its views
([#583](https://github.com/dfa1/vortex-java/issues/583)).

### Decode shape
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,10 @@ void javaWritesTheRootStatsJniWrites(@TempDir Path tmp) throws Exception {
for (int c : new int[]{0, 1, 2, 3, 4, 8, 9, 10, 11, 12}) {
assertThat(resultColumns.get(c)).as("column %d", c).isEqualTo(expectedColumns.get(c));
}
// the string columns' zone tables wait for Rust's varbinview canonical form (#583)
// the zone table of `high` holds three bounds, which Rust's trained symbol table compresses and ours
// does not: the string children depend on the trained table, which is not ported byte for byte
for (int c = 0; c < expectedColumns.size(); c++) {
if (c == 5 || c == 6 || c == 13) {
if (c == 6) {
continue;
}
assertThat(zoneTable(resultColumns.get(c))).as("zone table of column %d", c)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,16 @@ private static Boolean isConstant(ProtoScalarValue min, ProtoScalarValue max, lo
return min != null && max != null && Arrays.equals(min.encode(), max.encode());
}

/// The stats of a canonical text array, a `vortex.varbinview`: the flag, the null count and the size
/// of its views and out-of-line bytes.
///
/// @param nullCount the number of null rows
/// @param viewBytes the exact size of the canonical array in bytes
/// @return the node's stats
static NodeStats ofView(long nullCount, long viewBytes) {
return new NodeStats(null, null, notConstant(nullCount), null, null, nullCount, viewBytes, null);
}

private static Boolean notConstant(long nullCount) {
if (nullCount > 0) {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ public final class CascadingCompressor {
private static final DType OFFSETS = DTypes.I64;

private static final EncodingEncoder CANONICAL_PRIMITIVE = new PrimitiveEncodingEncoder();
private static final EncodingEncoder CANONICAL_VARBINVIEW = new VarBinViewEncodingEncoder();
private static final EncodingEncoder CANONICAL_BOOL = new BoolEncodingEncoder();
/// Bytes of a `vortex.constant` bool: Rust keeps a bool array whose packed bits are no larger.
private static final int CONSTANT_BOOL_BYTES = 2;
Expand Down Expand Up @@ -214,6 +215,14 @@ private static boolean eligible(EncodingEncoder enc, DType dtype, Object data, E
if (!enc.accepts(dtype) || ctx.excluded().contains(enc.encodingId())) {
return false;
}
// Rust's canonical text is `vortex.varbinview`, the baseline every scheme must beat, and
// its `VarBinScheme` re-encodes binary only
if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) {
EncodingId id = enc.encodingId();
if (id.equals(EncodingId.VORTEX_VARBINVIEW) || id.equals(EncodingId.VORTEX_VARBIN) && dtype instanceof DType.Utf8) {
return false;
}
}
return !(data instanceof NullableData) || enc.acceptsNullable(dtype);
}

Expand Down Expand Up @@ -278,6 +287,13 @@ private static NodeStats nodeStats(DType dtype, Object data, EncodingId root) {
}
return CachedNodeStats.ofBool(bools.length, SimdOperationsSupport.preferred().allEqual(bools));
}
if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) {
if (!root.equals(EncodingId.VORTEX_VARBINVIEW)) {
return NodeStats.NONE;
}
long nulls = data instanceof NullableData nullable ? nullable.nullCount() : 0L;
return CachedNodeStats.ofView(nulls, viewBytes(data, Integer.MAX_VALUE, 0L));
}
if (!(dtype instanceof DType.Primitive)) {
return NodeStats.NONE;
}
Expand Down Expand Up @@ -330,7 +346,7 @@ private EncodeResult encodeArray(DType dtype, Object data, EncodeContext ctx) {
// practice).
if (dtype instanceof DType.Utf8 || dtype instanceof DType.Binary) {
return competeAndEncode(dtype, new ArrayAndStats(dtype, data, StatsOptions.defaults()), ctx,
sampleSize -> Long.MAX_VALUE);
sampleSize -> viewBytes(data, sampleSize, ctx.sampleSeed()));
}

// Remaining non-primitives (extension types, List, ...): find the accepting encoding and
Expand Down Expand Up @@ -509,6 +525,21 @@ && totalBytes(result) >= uncompressedBytes(dtype, data)) {
return result;
}

/// The size of the canonical text array of the `sampleSize` rows a trial encodes: Rust's
/// `vortex.varbinview` is a 16-byte view per row plus the bytes of each value too long to be
/// inlined in it.
private static long viewBytes(Object data, int sampleSize, long seed) {
Object sample = sampleSize < dataLength(data) ? stratifiedSample(data, sampleSize, seed) : data;
if (sample instanceof NullableData nullable) {
sample = nullable.values();
}
long total = 0;
for (byte[] row : VarBinBytes.toByteArrays(sample)) {
total += 16 + (row.length > 12 ? row.length : 0);
}
return total;
}

private static long totalBytes(EncodeResult result) {
long total = 0;
for (EncodedBuffer buffer : result.encodedBuffers()) {
Expand Down Expand Up @@ -767,6 +798,10 @@ private EncodingEncoder findPrimitiveEncoding(DType dtype, Object data, Set<Enco
return enc;
}
}
if ((dtype instanceof DType.Utf8 || dtype instanceof DType.Binary)
&& !excluded.contains(EncodingId.VORTEX_VARBINVIEW)) {
return CANONICAL_VARBINVIEW;
}
// Fall through to any accepting encoding (still honoring exclusions so that
// spliceResult's notApplicable retry rotates to the next candidate).
for (EncodingEncoder enc : encodings) {
Expand All @@ -782,7 +817,7 @@ private EncodingEncoder findPrimitiveEncoding(DType dtype, Object data, Set<Enco
return switch (dtype) {
case DType.Primitive _ -> CANONICAL_PRIMITIVE;
case DType.Bool _ -> CANONICAL_BOOL;
case DType.Utf8 _, DType.Binary _ -> CANONICAL_VARBIN;
case DType.Utf8 _, DType.Binary _ -> CANONICAL_VARBINVIEW;
case DType.Decimal _ -> CANONICAL_DECIMAL;
case DType.Extension _ -> CANONICAL_EXTENSION;
default -> throw new UnsupportedOperationException("no encoder for dtype: " + dtype);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
for (int b = 0; b < dataBufs.length; b++) {
dataBufs[b] = arena.allocate(bufferSizes.get(b));
}
MemorySegment viewsBuf = arena.allocate(n > 0 ? (long) n * VIEW_SIZE : 1);
MemorySegment viewsBuf = arena.allocate((long) n * VIEW_SIZE, VIEW_SIZE);

int bufferIndex = 0;
long dataOffset = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,45 @@ void nullableFloatValidity_isABoolChildCarryingTheValidCount() {

}

@Nested
class TextCanonical {

@Test
void incompressibleUtf8_staysTheCanonicalVarBinView() {
// Given — distinct random 8-byte strings: no scheme beats the 16-byte views
Random random = new Random(3);
String[] values = new String[2048];
for (int i = 0; i < values.length; i++) {
values[i] = Long.toString(random.nextLong() & 0xffffffffL, 36);
}
CascadingCompressor sut = new CascadingCompressor(ALL_CODECS);

// When
EncodeResult result = sut.encode(DTypes.UTF8, values, ctx(0));

// Then — Rust's `VarBinScheme` re-encodes binary only
assertThat(result.rootNode().encodingId()).isEqualTo(EncodingId.VORTEX_VARBINVIEW);
}

@Test
void varBinViewRoot_carriesTheFlagNullCountAndViewSize() {
// Given
Random random = new Random(3);
String[] values = new String[2048];
for (int i = 0; i < values.length; i++) {
values[i] = Long.toString(random.nextLong() & 0xffffffffL, 36);
}
CascadingCompressor sut = new CascadingCompressor(ALL_CODECS);

// When
EncodeResult result = sut.encode(DTypes.UTF8, values, ctx(0));

// Then — a 16-byte view per row, values of up to 12 bytes inline
assertThat(result.rootNode().stats())
.isEqualTo(new NodeStats(null, null, false, null, null, 0L, 2048L * 16, null));
}
}

@Nested
class RoundTrip {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,19 @@ void accepts_binary_true() {
assertThat(result).isTrue();
}

@Test
void encode_noRows_hasAnEmptyViewsBuffer() {
// Given
VarBinViewEncodingEncoder sut = new VarBinViewEncodingEncoder();

// When
EncodeResult result = sut.encode(DTypes.UTF8, new String[0], EncodeTestHelper.testCtx());

// Then — Rust checks the views buffer against the row count, so an empty array has none
assertThat(result.encodedBuffers()).hasSize(1);
assertThat(result.encodedBuffers().getFirst().data().byteSize()).isZero();
}

@Test
void accepts_primitive_false() {
// Given
Expand Down
Loading