diff --git a/AGENTS.md b/AGENTS.md index 34077a325..63cc7856c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ Split work into PRs that each stand alone on `main`. scripts/simd-benchmark.sh # auto-vectorized vs Vector API kernels, table of speedups (also a CI job) scripts/hydrate-raincloud-corpus.sh --max-mb 200 # hydrate real-world conformance corpus (#205), then: ./mvnw verify -pl integration -am -Dvortex.it.excludedGroups= -Dit.test="RaincloudConformanceIntegrationTest" -java --add-opens java.base/java.nio=ALL-UNNAMED -cp "$(cat integration/target/test-classpath.txt)" scripts/RaincloudCompare.java [slug ...] # size/speed vs Rust and vortex-jni per slug (classpath: docs/compatibility.md) +VORTEX_PYTHON=/bin/python java --add-opens java.base/java.nio=ALL-UNNAMED -cp "$(cat integration/target/test-classpath.txt)" scripts/RaincloudCompare.java [slug ...] # size/speed vs Rust (default + compact) per slug (setup: docs/compatibility.md) JAZZER_FUZZ=1 JAZZER_MAX_DURATION=5m ./mvnw test -pl fuzz -am -Dtest='VortexReaderFuzzTest#scanMalformedFile' -Dsurefire.failIfNoSpecifiedTests=false # fuzz one target (ADR 0020) JAZZER_FUZZ=1 JAZZER_MAX_DURATION=5m ./mvnw verify -pl integration -am -Dvortex.it.excludedGroups= -Dit.test='WriteReadFuzzIntegrationTest#jniReadsJavaWrite' -Dtest=none -Dsurefire.failIfNoSpecifiedTests=false -Dfailsafe.failIfNoSpecifiedTests=false ./mvnw test -pl fuzz -am -Dtest=SeedCorpusGenerator -Dsurefire.failIfNoSpecifiedTests=false -Dvortex.fuzz.regenerateSeeds=true # rewrite per-encoding seeds diff --git a/docs/compatibility.md b/docs/compatibility.md index d236e19b1..730596633 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -101,21 +101,26 @@ git clone https://github.com/clflushopt/tpchgen-rs tpcgen-rs && git -C tpcgen-rs ``` The conformance test proves values match; `scripts/RaincloudCompare.java` reports what they cost. -Per hydrated slug it prints the Parquet size, two Rust sizes (Raincloud's file, from the -`vortex-data` its release pins, and that file rewritten by vortex-jni at the Rust version -vortex-java mirrors), the Java size with default and compact write options, the import time, -and the full-scan time of the Rust file and of Java's, flagging a Java file over 2% larger than -vortex-jni's. It reports and never fails: size is information, not a reason to diverge from -Rust. vortex-jni rewrites the batches its scan yields, not the Parquet row groups Java imports, -so its chunk boundaries can differ slightly. It needs vortex-jni and Arrow, so it runs with the -`integration` module's test class path and Arrow's JVM flags: +Per hydrated slug it prints the Parquet size, Raincloud's `.vortex` (from the `vortex-data` its +release pins), that file rewritten by vortex-jni, Python `vortex-data` at the Rust version +vortex-java mirrors writing the Parquet with its default and compact options, and vortex-java +importing it with both; then the import and rewrite times and the full-scan time of Raincloud's +file and of Java's. It flags a Java file over 2% larger than Rust's, default (`BIGGER`) or +compact (`BIGGER_COMPACT`), and never fails: size is information, not a reason to diverge from +Rust. The Python writer is the baseline: vortex-jni has no compact option +([vortex-data/vortex#11374](https://github.com/vortex-data/vortex/issues/11374)), and its rewrite +takes the batches its scan yields, which vary between runs, and its size with them (by 6% on +`openpowerlifting`), so `jni` is for reference only. Without `VORTEX_PYTHON` there is no +baseline and nothing is flagged. It needs vortex-jni and Arrow, so it runs with the `integration` module's test +class path and Arrow's JVM flags: ```bash +python3 -m venv /tmp/vx && /tmp/vx/bin/pip install pyarrow vortex-data==0.91.0 # = vortex-jni.version ./mvnw -q -pl integration -am -DskipTests package dependency:build-classpath \ -Dmdep.includeScope=test -Dmdep.outputFile=target/test-classpath.txt -java --enable-native-access=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED \ - --sun-misc-unsafe-memory-access=allow -cp "$(cat integration/target/test-classpath.txt)" \ - scripts/RaincloudCompare.java [slug ...] # no slug: the last manifest +VORTEX_PYTHON=/tmp/vx/bin/python java --enable-native-access=ALL-UNNAMED \ + --add-opens java.base/java.nio=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow \ + -cp "$(cat integration/target/test-classpath.txt)" scripts/RaincloudCompare.java [slug ...] ``` Per-slug status lives in `integration/src/test/resources/raincloud/expected-status.csv` diff --git a/scripts/RaincloudCompare.java b/scripts/RaincloudCompare.java index 3d9d31a92..3e36ce07d 100644 --- a/scripts/RaincloudCompare.java +++ b/scripts/RaincloudCompare.java @@ -14,18 +14,25 @@ import org.apache.arrow.vector.ipc.ArrowReader; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.concurrent.ForkJoinPool; /// Size and speed oracle for the Raincloud corpus (issue #205): compares, per slug, vortex-java -/// re-importing the slug's Parquet sibling against two Rust writers of the same data: +/// importing the slug's Parquet sibling, with default and compact write options, against Rust +/// writing the same data: /// -/// - `rust`: Raincloud's `.vortex`, written by the Python `vortex-data` its release pins +/// - `raincloud`: Raincloud's `.vortex`, written by the Python `vortex-data` its release pins /// - `jni`: that file rewritten by vortex-jni, the Rust version vortex-java mirrors (`vortex-jni.version`) +/// - `rust`, `rust_compact`: Python `vortex-data` at that same Rust version writing the Parquet +/// sibling with `VortexWriteOptions.default()` and `.compact()`, the baseline both comparisons use; +/// vortex-jni has no compact option (vortex-data/vortex#11374). Only with `VORTEX_PYTHON` set to +/// a Python that has it (`pip install pyarrow vortex-data==`), else `-` /// /// `RaincloudConformanceIntegrationTest` proves the values match; this reports what they cost. /// It never fails on a difference: size is information, not a reason to diverge from Rust @@ -36,18 +43,19 @@ /// ``` /// ./mvnw -q -pl integration -am -DskipTests package dependency:build-classpath \ /// -Dmdep.includeScope=test -Dmdep.outputFile=target/test-classpath.txt -/// java --enable-native-access=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED \ -/// --sun-misc-unsafe-memory-access=allow -cp "$(cat integration/target/test-classpath.txt)" \ -/// scripts/RaincloudCompare.java [slug ...] +/// VORTEX_PYTHON=/bin/python java --enable-native-access=ALL-UNNAMED \ +/// --add-opens java.base/java.nio=ALL-UNNAMED --sun-misc-unsafe-memory-access=allow \ +/// -cp "$(cat integration/target/test-classpath.txt)" scripts/RaincloudCompare.java [slug ...] /// ``` /// /// With no slugs, compares every slug of the last manifest written by -/// `scripts/hydrate-raincloud-corpus.sh`. Prints one TSV row per slug: sizes in bytes, the -/// import time with default and compact write options (as `vortex import` and `vortex import -/// --compact`), vortex-jni's rewrite time, and the best of three full scans of the Rust file and -/// of Java's. vortex-jni writes the batches it scans from the Rust file, not the Parquet row -/// groups Java imports, so its chunk boundaries can differ slightly. `flag` is `BIGGER` when -/// Java's file is over 2% larger than vortex-jni's on a file over 1 MB. +/// `scripts/hydrate-raincloud-corpus.sh`. Prints one TSV row per slug: sizes in bytes, Java's +/// size against Rust's (`rust` and `rust_compact`), the import +/// time with default and compact write options (as `vortex import` and `vortex import --compact`), +/// vortex-jni's rewrite time, and the best of three full scans of Raincloud's file and of Java's. +/// `jni` is for reference only: vortex-jni writes the batches it scans from Raincloud's file, which +/// vary between runs, and its size with them (by 6% on `openpowerlifting`). `flag` names each Java file over 2% larger than its Rust +/// counterpart on a file over 1 MB: `BIGGER` for the default write, `BIGGER_COMPACT` for compact. void main(String[] args) throws IOException { Path cacheRoot = Path.of(env("RAINCLOUD_HOME", System.getProperty("user.home") + "/.cache/raincloud")); Path manifest = Path.of(env("RAINCLOUD_CORPUS_MANIFEST", cacheRoot.resolve("corpus-manifest.tsv").toString())); @@ -69,59 +77,84 @@ void main(String[] args) throws IOException { fail("no slugs to compare (empty manifest?)"); } + String python = env("VORTEX_PYTHON", ""); + if (!python.isEmpty()) { + String jniVersion = jniVersion(); + String rustVersion = pythonVortexVersion(python); + if (!rustVersion.equals(jniVersion)) { + System.err.println("warning: " + python + " has vortex-data " + rustVersion + ", vortex-jni is " + jniVersion); + } + } NativeLoader.loadJni(); Session session = Session.create(); BufferAllocator allocator = ArrowAllocation.rootAllocator(); Path scratch = Files.createTempDirectory("raincloud-compare"); try { - IO.println("slug\tparquet\trust\tjni\tjava\tjava_compact\tjava_vs_jni\tjava_vs_rust" - + "\timport_ms\tcompact_ms\tjni_ms\trust_read_ms\tjava_read_ms\tflag"); + IO.println("slug\tparquet\traincloud\tjni\trust\trust_compact\tjava\tjava_compact\tjava_vs_rust" + + "\tcompact_vs_rust\timport_ms\tcompact_ms\tjni_ms\traincloud_read_ms\tjava_read_ms\tflag"); for (String slug : slugs) { - Path rust = outputs.resolve(slug).resolve("vortex").resolve(slug + ".vortex"); + Path raincloud = outputs.resolve(slug).resolve("vortex").resolve(slug + ".vortex"); Path parquet = outputs.resolve(slug).resolve("parquet").resolve(slug + ".parquet"); - if (!Files.exists(rust) || !Files.exists(parquet)) { + if (!Files.exists(raincloud) || !Files.exists(parquet)) { System.err.println("skip " + slug + ": not hydrated under " + outputs + " (run scripts/hydrate-raincloud-corpus.sh " + slug + ")"); continue; } - IO.println(compare(slug, parquet, rust, scratch, session, allocator)); + IO.println(compare(slug, parquet, raincloud, scratch, session, allocator, python)); } } finally { Files.deleteIfExists(scratch); } } -String compare(String slug, Path parquet, Path rust, Path scratch, Session session, BufferAllocator allocator) - throws IOException { +String compare(String slug, Path parquet, Path raincloud, Path scratch, Session session, BufferAllocator allocator, + String python) throws IOException { Path java = scratch.resolve(slug + ".vortex"); Path compact = scratch.resolve(slug + ".compact.vortex"); Path jni = scratch.resolve(slug + ".jni.vortex"); + Path rust = scratch.resolve(slug + ".rust.vortex"); + Path rustCompact = scratch.resolve(slug + ".rust.compact.vortex"); try { Duration importTime = importParquet(parquet, java, false); Duration compactTime = importParquet(parquet, compact, true); - Duration jniTime = jniRewrite(rust, jni, session, allocator); - Duration rustRead = bestScan(rust); + Duration jniTime = jniRewrite(raincloud, jni, session, allocator); + boolean withRust = !python.isEmpty() && pythonWrite(python, parquet, rust, rustCompact); + Duration raincloudRead = bestScan(raincloud); Duration javaRead = bestScan(java); - long rustBytes = Files.size(rust); long jniBytes = Files.size(jni); long javaBytes = Files.size(java); - double vsJni = (double) javaBytes / jniBytes - 1; - double vsRust = (double) javaBytes / rustBytes - 1; - String flag = ""; - if (vsJni > 0.02 && jniBytes > 1 << 20) { - flag = "BIGGER"; + long compactBytes = Files.size(compact); + // vortex-jni's rewrite is no baseline: its scan's batches vary between runs, and so its size + long rustBytes = withRust ? Files.size(rust) : -1; + long rustCompactBytes = withRust ? Files.size(rustCompact) : -1; + double vsRust = withRust ? (double) javaBytes / rustBytes - 1 : Double.NaN; + double compactVsRust = withRust ? (double) compactBytes / rustCompactBytes - 1 : Double.NaN; + List flags = new ArrayList<>(); + if (vsRust > 0.02 && rustBytes > 1 << 20) { + flags.add("BIGGER"); + } + if (compactVsRust > 0.02 && rustCompactBytes > 1 << 20) { + flags.add("BIGGER_COMPACT"); } - return String.format(Locale.ROOT, "%s\t%d\t%d\t%d\t%d\t%d\t%+.1f%%\t%+.1f%%\t%d\t%d\t%d\t%d\t%d\t%s", - slug, Files.size(parquet), rustBytes, jniBytes, javaBytes, Files.size(compact), vsJni * 100, - vsRust * 100, importTime.toMillis(), compactTime.toMillis(), jniTime.toMillis(), - rustRead.toMillis(), javaRead.toMillis(), flag); + return String.join("\t", slug, Long.toString(Files.size(parquet)), Long.toString(Files.size(raincloud)), + Long.toString(jniBytes), withRust ? Long.toString(rustBytes) : "-", + withRust ? Long.toString(rustCompactBytes) : "-", Long.toString(javaBytes), Long.toString(compactBytes), + withRust ? percent(vsRust) : "-", withRust ? percent(compactVsRust) : "-", Long.toString(importTime.toMillis()), + Long.toString(compactTime.toMillis()), Long.toString(jniTime.toMillis()), + Long.toString(raincloudRead.toMillis()), Long.toString(javaRead.toMillis()), String.join(",", flags)); } finally { Files.deleteIfExists(java); Files.deleteIfExists(compact); Files.deleteIfExists(jni); + Files.deleteIfExists(rust); + Files.deleteIfExists(rustCompact); } } +String percent(double ratio) { + return String.format(Locale.ROOT, "%+.1f%%", ratio * 100); +} + /// Imports as `vortex import [--compact]` does: default options on the common pool. Duration importParquet(Path parquet, Path vortex, boolean compact) throws IOException { ImportOptions defaults = ImportOptions.defaults(); @@ -174,6 +207,51 @@ Duration bestScan(Path vortex) throws IOException { return best; } +/// Writes `parquet` with Python `vortex-data`, default and compact options; `false` when it fails. +boolean pythonWrite(String python, Path parquet, Path target, Path compactTarget) throws IOException { + String program = """ + import sys, pyarrow.parquet as pq, vortex.io as io + table = pq.read_table(sys.argv[1]) + io.VortexWriteOptions.default().write(table, sys.argv[2]) + io.VortexWriteOptions.compact().write(table, sys.argv[3]) + """; + Process process = new ProcessBuilder(python, "-I", "-c", program, parquet.toString(), target.toString(), + compactTarget.toString()).redirectOutput(ProcessBuilder.Redirect.DISCARD).start(); + String error = new String(process.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); + try { + if (process.waitFor() != 0) { + System.err.println("warning: " + python + " could not write " + parquet + ": " + error.strip()); + return false; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + return true; +} + +/// The `vortex-data` version of `python`, or `?` when it has none. +String pythonVortexVersion(String python) throws IOException { + Process process = new ProcessBuilder(python, "-I", "-c", + "import importlib.metadata as m; print(m.version('vortex-data'))").start(); + String version = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8).strip(); + if (version.isEmpty()) { + return "?"; + } + return version; +} + +/// The vortex-jni version on the class path, read off its jar's file name. +String jniVersion() { + for (String entry : System.getProperty("java.class.path").split(java.io.File.pathSeparator)) { + String name = Path.of(entry).getFileName().toString(); + if (name.startsWith("vortex-jni-") && name.endsWith(".jar")) { + return name.substring("vortex-jni-".length(), name.length() - ".jar".length()); + } + } + return "?"; +} + String env(String name, String fallback) { String value = System.getenv(name); if (value == null || value.isBlank()) {