From ce0e90d2bc58197cde6bdbc6e38b92030b76c020 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 20 Nov 2026 16:51:50 -0500 Subject: [PATCH 01/23] Merge the size groups into one File Sizes group on bench The Compression Size and SQL File Size groups showed the same metric, file size per format, from two suites. One File Sizes group now charts the compress-bench datasets and three SQL scopes: ClickBench, TPC-H at SF10, and TPC-DS at SF1. Old Compression Size links redirect to it, and the homepage card reads the group by its new or old name. The summary compares each format with zstd Parquet on the compress-bench datasets and on TPC-DS. Each format uses its newest commit that has zstd Parquet. The wide-table and sparse-wide families count as one dataset each. A format measured on only some datasets says how many, has no total, and ranks after the complete formats. Parquet codec variants, such as parquet-snappy, draw in Parquet's color with a dashed line. Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 17 +- docs/bench/architecture/read-path.md | 2 +- src/bench/components/SummaryCard.test.tsx | 36 +++ src/bench/components/SummaryCard.tsx | 15 +- src/bench/lib/benchmark-docs.test.ts | 4 +- src/bench/lib/benchmark-docs.ts | 4 +- src/bench/lib/chart-format.test.ts | 13 + src/bench/lib/chart-format.ts | 30 ++- src/bench/lib/compression-suite.ts | 28 +- src/bench/lib/descriptions.test.ts | 5 +- src/bench/lib/descriptions.ts | 11 +- src/bench/lib/file-sizes.ts | 86 ++++++ src/bench/lib/group-charts.test.ts | 53 ++-- src/bench/lib/groups.test.ts | 95 ++++--- src/bench/lib/headline.test.ts | 22 +- src/bench/lib/queries.test.ts | 8 +- src/bench/lib/queries.ts | 308 ++++++++++++---------- src/bench/lib/site-files.test.ts | 2 + src/bench/lib/site-files.ts | 5 +- src/bench/lib/slug.test.ts | 8 +- src/bench/lib/slug.ts | 23 +- src/bench/lib/summary.test.ts | 147 +++++++++-- src/bench/lib/summary.ts | 242 +++++++++++------ src/lib/bench-summary.ts | 33 ++- tests/unit/bench-summary.test.ts | 30 ++- tests/unit/fixtures/bench-groups.json | 6 +- 26 files changed, 869 insertions(+), 364 deletions(-) create mode 100644 src/bench/lib/file-sizes.ts diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index e27c06ee..17e7ab07 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -64,6 +64,9 @@ column-by-column; an unknown `kind` or a missing dimension field fails the inges `compression_size` records include `uncompressed_bytes`, the Arrow memory size after the source Parquet file is decoded. The column is nullable so historical rows remain valid. +`compress-bench` also writes two size-only formats, with no `compression_time` records: +`parquet-snappy` (Parquet with Snappy; `parquet` is zstd level 3) and `vortex-compact` (Vortex +with the compact strategy). Each is one write per dataset that is not timed. `string-bench` shares the two compression tables with `compress-bench`. Its records use the dataset `strings/`, the string encoder (`onpair-12`, `fsst`, ...) as `dataset_variant`, @@ -73,11 +76,17 @@ rows in the `String Encoding` groups and compares each encoder with `fsst`, so t `fsst` label are part of the contract. `file_size` records are not written by `vortex-bench`: the monorepo's -`scripts/capture-file-sizes.py` writes one per Vortex file a SQL benchmark leaves on disk, with +`scripts/capture-file-sizes.py` writes one per data file a SQL benchmark leaves on disk, with `benchmark` (the `vx-bench` subcommand), `scale_factor`, `format`, `file` (the path below the format -directory) and `value_bytes`. The read service charts each benchmark and scale factor in the -`SQL File Size` group, one line per format summed over its files. CI reads them back as the develop -baseline for the PR comment's file-size section. +directory) and `value_bytes`. A Parquet file's `format` names its codec, which the script reads +from the file's footer: `parquet` for zstd, and `parquet-` (`parquet-snappy`, for example) +for any other. CI reads the records back as the develop baseline for the PR comment's file-size +section. + +The read service charts both tables in the `File Sizes` group, one line per format. It charts every +current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed +over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its +summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. `random_access_time` records include `open_mode`. The value is `cached` for a reused accessor or `reopen` when each timed take includes a new accessor. Historical rows use `cached`. diff --git a/docs/bench/architecture/read-path.md b/docs/bench/architecture/read-path.md index c7c1afac..fb9dae4b 100644 --- a/docs/bench/architecture/read-path.md +++ b/docs/bench/architecture/read-path.md @@ -120,7 +120,7 @@ the v3 Axum server (so the frontend is generation-agnostic). The landing-page order is the curated `GROUP_ORDER` in `queries.ts`: ``` -Compression, Compression Size, Random Access, Clickbench, +Compression, File Sizes, Random Access, Clickbench, TPC-H (NVMe/S3) SF=1, SF=10, SF=100 (alternating), TPC-DS (NVMe) SF=1, Statistical and Population Genetics, Polar Signals Profiling, diff --git a/src/bench/components/SummaryCard.test.tsx b/src/bench/components/SummaryCard.test.tsx index da04c1d1..08c851c0 100644 --- a/src/bench/components/SummaryCard.test.tsx +++ b/src/bench/components/SummaryCard.test.tsx @@ -152,6 +152,8 @@ describe('SummaryCard', () => { maxRatio: 1, compressionRatio: 4, totalBytes: null, + measured: 1, + total: 1, }, ], explanation: 'e', @@ -160,6 +162,30 @@ describe('SummaryCard', () => { expect(html).toContain('class="compression-size-heading compression-size-total-unit">'); }); + it('says how many datasets a partly measured size ranking covers', () => { + const ranking = (name: string, measured: number) => ({ + name, + minRatio: 1, + ratio: 1, + maxRatio: 1, + compressionRatio: null, + totalBytes: measured === 2 ? 10 : null, + measured, + total: 2, + }); + const html = render({ + type: 'compressionSize', + title: 'File Sizes Summary', + rankings: [ranking('parquet', 2), ranking('vortex-compact', 1)], + explanation: 'e', + }); + expect(html).toContain('title="vortex-compact · measured on 1 of 2 datasets"'); + expect(html).toContain(''); + expect(html).toContain(', measured on 1 of 2 datasets'); + // A complete ranking carries no coverage. + expect(html.match(/measured on/g)).toHaveLength(2); + }); + it('renders nothing for a compression card with no rankings', () => { expect(render({ type: 'compression', title: 't', rankings: [], explanation: 'e' })).toBe(''); }); @@ -176,6 +202,8 @@ describe('SummaryCard', () => { maxRatio: 1.55, compressionRatio: 8.25, totalBytes: 1_603_980_000, + measured: 1, + total: 1, }, { name: 'parquet', @@ -184,6 +212,8 @@ describe('SummaryCard', () => { maxRatio: 1, compressionRatio: 4.5, totalBytes: 1_428_150_000, + measured: 1, + total: 1, }, { name: 'lance', @@ -192,6 +222,8 @@ describe('SummaryCard', () => { maxRatio: 1.3, compressionRatio: null, totalBytes: 2_500_000, + measured: 1, + total: 1, }, ], explanation: 'higher is better', @@ -311,6 +343,8 @@ describe('SummaryCard', () => { maxRatio: 1, compressionRatio: 8, totalBytes: 100, + measured: 1, + total: 1, }, { name: 'lance', @@ -319,6 +353,8 @@ describe('SummaryCard', () => { maxRatio: 1, compressionRatio: 2, totalBytes: 200, + measured: 1, + total: 1, }, ], explanation: 'e', diff --git a/src/bench/components/SummaryCard.tsx b/src/bench/components/SummaryCard.tsx index 0ad86a9d..56797912 100644 --- a/src/bench/components/SummaryCard.tsx +++ b/src/bench/components/SummaryCard.tsx @@ -144,14 +144,27 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
{summary.rankings.map((item, idx) => { const label = displayFormat(item.name); + // A format measured on only some datasets has its means over those alone, so + // the row says how many, and it ranks after the complete ones. + const partial = item.measured < item.total; + const coverage = `measured on ${item.measured} of ${item.total} datasets`; return (
#{idx + 1} - + {label} + {partial ? ( + <> + + {`, ${coverage}`} + + ) : null} {item.totalBytes === null || totalUnit === null ? ( diff --git a/src/bench/lib/benchmark-docs.test.ts b/src/bench/lib/benchmark-docs.test.ts index e94a4009..9d94e5ce 100644 --- a/src/bench/lib/benchmark-docs.test.ts +++ b/src/bench/lib/benchmark-docs.test.ts @@ -40,8 +40,8 @@ describe('groupDocPath', () => { it('maps the three non-query families to their benchmark crate', () => { expect(groupDocPath({ k: 'CompressionTimeGroup' })).toBe('benchmarks/compress-bench/README.md'); - // Two measurement kinds of one benchmark, so one doc. - expect(groupDocPath({ k: 'CompressionSizeGroup' })).toBe('benchmarks/compress-bench/README.md'); + // File Sizes is mostly compress-bench's datasets, so it shares compress-bench's doc. + expect(groupDocPath({ k: 'FileSizeGroup' })).toBe('benchmarks/compress-bench/README.md'); expect(groupDocPath({ k: 'CompressionTimeGroup', suite: 'strings' })).toBe( 'benchmarks/string-bench/README.md', ); diff --git a/src/bench/lib/benchmark-docs.ts b/src/bench/lib/benchmark-docs.ts index 8fd75d70..068c61c7 100644 --- a/src/bench/lib/benchmark-docs.ts +++ b/src/bench/lib/benchmark-docs.ts @@ -83,12 +83,14 @@ export function groupDocPath(key: GroupKey): string | null { case 'CompressionTimeGroup': case 'CompressionSizeGroup': return key.suite === 'strings' ? STRING_BENCH_DOC : COMPRESS_BENCH_DOC; + // Most of File Sizes' charts are `compress-bench` datasets, and its doc covers them. + case 'FileSizeGroup': + return COMPRESS_BENCH_DOC; case 'RandomAccessGroup': return RANDOM_ACCESS_BENCH_DOC; case 'QueryGroup': return QUERY_DOCS[key.dataset] ?? null; case 'VectorSearchGroup': - case 'FileSizeGroup': return null; } } diff --git a/src/bench/lib/chart-format.test.ts b/src/bench/lib/chart-format.test.ts index 18c4d7d2..9f580635 100644 --- a/src/bench/lib/chart-format.test.ts +++ b/src/bench/lib/chart-format.test.ts @@ -139,6 +139,19 @@ describe('seriesStyle', () => { } }); + it("draws a Parquet codec variant dashed, in Parquet's color, thinner, behind Parquet", () => { + for (const theme of ['dark', 'light'] as const) { + const zstd = seriesStyle('parquet', { format: 'parquet' }, theme); + const snappy = seriesStyle('parquet-snappy', { format: 'parquet-snappy' }, theme); + expect(snappy.color).toBe(zstd.color); + expect(snappy.dash).toEqual([6, 4]); + expect(snappy.width).toBeLessThan(zstd.width); + } + const order = (format: string) => seriesOrder(format, { format }); + expect(order('parquet')).toBeLessThan(order('parquet-snappy')); + expect(order('parquet-snappy')).toBeLessThan(order('vortex-compact')); + }); + it('draws a cold random-access run dashed and a hot one solid', () => { const cold = seriesStyle('parquet:cold', { format: 'parquet' }, 'dark'); const hot = seriesStyle('parquet:hot', { format: 'parquet' }, 'dark'); diff --git a/src/bench/lib/chart-format.ts b/src/bench/lib/chart-format.ts index 07020635..cc2f1211 100644 --- a/src/bench/lib/chart-format.ts +++ b/src/bench/lib/chart-format.ts @@ -160,6 +160,16 @@ const FORMAT: Record = { 'arrow-ipc': ARROW, }; +/** + * The format a codec variant is drawn as: Parquet with a codec other than zstd + * (`parquet-snappy`, `parquet-uncompressed`, as the size tables name them) is + * Parquet's teal, dashed, so it reads as a kind of Parquet without a color of + * its own. Size charts have no engine, so the dash cannot be read as duckdb's. + */ +function codecVariantOf(format: string): string | undefined { + return format.startsWith('parquet-') ? 'parquet' : undefined; +} + /** The catch-all style for an otherwise-unknown series: a neutral gray with * 3:1 contrast on both cards. */ const FALLBACK_COLOR = '#8a8a87'; @@ -190,12 +200,19 @@ export function seriesStyle( theme: ChartTheme, ): SeriesStyle { const { engine, format } = seriesDims(name, meta); - const dash = engine === 'duckdb' || name.endsWith(':cold') ? [...DASH] : []; - const style = format ? FORMAT[format] : undefined; + const variantOf = format === undefined ? undefined : codecVariantOf(format); + const dash = + engine === 'duckdb' || name.endsWith(':cold') || variantOf !== undefined ? [...DASH] : []; + const base = variantOf ?? format; + const style = base ? FORMAT[base] : undefined; if (!style) { return { color: FALLBACK_COLOR, width: MUTED_WIDTH, dash }; } - return { color: style.color[theme], width: style.width, dash }; + return { + color: style.color[theme], + width: variantOf === undefined ? style.width : MUTED_WIDTH, + dash, + }; } /** Format draw priority, front (low) to back (high) -- Chart.js renders a lower @@ -231,7 +248,12 @@ export function seriesOrder( meta: { engine?: string; format?: string } | null | undefined, ): number { const { engine, format } = seriesDims(name, meta); - const formatRank = rankIn(FORMAT_ORDER, format, FORMAT_ORDER_FALLBACK); + // A codec variant sits just behind its format. + const variantOf = format === undefined ? undefined : codecVariantOf(format); + const formatRank = + variantOf === undefined + ? rankIn(FORMAT_ORDER, format, FORMAT_ORDER_FALLBACK) + : rankIn(FORMAT_ORDER, variantOf, FORMAT_ORDER_FALLBACK) + 0.5; const engineRank = rankIn(ENGINE_ORDER, engine, 0); return formatRank * 10 + engineRank; } diff --git a/src/bench/lib/compression-suite.ts b/src/bench/lib/compression-suite.ts index e9919e4d..66a60825 100644 --- a/src/bench/lib/compression-suite.ts +++ b/src/bench/lib/compression-suite.ts @@ -15,10 +15,11 @@ export const STRING_DATASET_PREFIX = 'strings/'; export const STRING_DATASET_PATTERN = `${STRING_DATASET_PREFIX}%`; /** - * `LIKE` patterns for the `compress-bench` datasets the compression summaries leave out: the + * `LIKE` patterns for the `compress-bench` datasets the compression time summary leaves out: the * `wide table cols=… chunks=… rows=…` and `sparse-wide-*` suites. They are pathological cases * with many variants each, so in a geomean over all datasets they would be heavily - * over-represented. They still chart in the group. Both patterns are anchored at the start of the + * over-represented. They still chart in the group. The File Sizes summary counts each suite as + * one dataset instead (`SIZE_SUMMARY_FAMILIES`). Both patterns are anchored at the start of the * name, as `compress-bench` writes it, so a dataset that only mentions them is not dropped. */ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ @@ -27,25 +28,38 @@ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * `LIKE` patterns for the datasets the compression size summary also leaves out, on top of - * `SUMMARY_EXCLUDED_DATASET_PATTERNS`. They still chart in the group, and the compression time - * summary keeps them. + * `LIKE` patterns for the `compress-bench` datasets the File Sizes summary leaves out. They still + * chart in the group, and the compression time summary keeps them (minus + * `SUMMARY_EXCLUDED_DATASET_PATTERNS`). * * - `tpc-h l_comment chunked` writes the same column as `tpc-h l_comment canonical`, so their * files are the same size. Only the input to the encoder differs, which matters for time but * not for size. The size summary counts the column once, as `canonical`. + * - The wide tables with 50 chunks write the same files as the ones with 1 chunk, for the same + * reason. The size summary counts each column count once, as `chunks=1`. * - `statpopgen`, which joined `compress-bench` on 2026-11-06, is left out of the size summary. * The pattern also matches the older `statpopgen/100000` rows, which have no Parquet size and * so were never in the summary. * - * Both names are exact. `_` is a `LIKE` wildcard, but no other dataset name matches. + * `_` is a `LIKE` wildcard, but no other dataset name matches these patterns. */ export const SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ - ...SUMMARY_EXCLUDED_DATASET_PATTERNS, 'tpc-h l_comment chunked', + 'wide table cols=% chunks=50 rows=%', 'statpopgen', ]; +/** + * Synthetic `compress-bench` families that the File Sizes summary counts as one dataset each: + * the geometric mean of the family's ratios is one entry in the summary's geometric mean. A + * family has several datasets of one kind, so counted one by one they would outweigh the real + * datasets. Each family is the datasets whose names start with its prefix. + */ +export const SIZE_SUMMARY_FAMILIES: readonly { name: string; prefix: string }[] = [ + { name: 'wide table', prefix: 'wide table ' }, + { name: 'sparse-wide', prefix: 'sparse-wide-' }, +]; + export function compressionSuite(dataset: string): CompressionSuite { return dataset.startsWith(STRING_DATASET_PREFIX) ? 'strings' : 'compress'; } diff --git a/src/bench/lib/descriptions.test.ts b/src/bench/lib/descriptions.test.ts index 7cba09bd..48760a8b 100644 --- a/src/bench/lib/descriptions.test.ts +++ b/src/bench/lib/descriptions.test.ts @@ -20,8 +20,9 @@ describe('groupDescription', () => { expect(groupDescription('Compression')).toBe( 'Measures write and scan throughput for Vortex, Parquet, and Lance files', ); - expect(groupDescription('Compression Size')).toBe( - 'Compares compressed file sizes and compression ratios across different encoding strategies', + expect(groupDescription('File Sizes')).toBe( + 'Compares file sizes and compression ratios across formats, on the compression datasets ' + + 'and on the files the ClickBench, TPC-H, and TPC-DS benchmarks write', ); expect(groupDescription('Clickbench')).toBe( "ClickHouse's analytical benchmark suite testing real-world query patterns on web " + diff --git a/src/bench/lib/descriptions.ts b/src/bench/lib/descriptions.ts index 2aaa6de0..bf0c6d39 100644 --- a/src/bench/lib/descriptions.ts +++ b/src/bench/lib/descriptions.ts @@ -46,20 +46,15 @@ function staticDescription(name: string): string | null { return 'Tests performance of selecting arbitrary row indices from a file on S3 storage'; case 'Compression': return 'Measures write and scan throughput for Vortex, Parquet, and Lance files'; - case 'Compression Size': + case 'File Sizes': return ( - 'Compares compressed file sizes and compression ratios across different encoding ' + - 'strategies' + 'Compares file sizes and compression ratios across formats, on the compression ' + + 'datasets and on the files the ClickBench, TPC-H, and TPC-DS benchmarks write' ); case 'String Encoding': return 'Measures Vortex file write and scan throughput for each string encoder'; case 'String Encoding Size': return 'Compares Vortex file sizes for each string encoder'; - case 'SQL File Size': - return ( - 'Tracks the total size of the Vortex files each SQL benchmark writes, per benchmark and ' + - 'scale factor' - ); case 'Clickbench': return ( "ClickHouse's analytical benchmark suite testing real-world query patterns on web " + diff --git a/src/bench/lib/file-sizes.ts b/src/bench/lib/file-sizes.ts new file mode 100644 index 00000000..e4340f22 --- /dev/null +++ b/src/bench/lib/file-sizes.ts @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +/** + * Which charts the File Sizes group holds. It joins two tables: `compression_sizes`, the + * size of each `compress-bench` dataset in each format, and `file_sizes`, the files that the + * SQL suites write. Both are one dataset's size per format per commit. + */ + +import { compressionSuite } from './compression-suite'; + +/** The File Sizes group's display name. */ +export const FILE_SIZES_GROUP_NAME = 'File Sizes'; + +/** One SQL suite's data at one scale factor, as `file_sizes` spells them. */ +export interface FileSizeScope { + benchmark: string; + scale_factor: string; +} + +/** + * The SQL suites' data the group charts, in this order. The other scopes in `file_sizes` stay + * in the table, for the PR benchmark comments, but the group leaves them out. TPC-DS is at + * SF1 because develop writes it on every commit, with Vortex Compact and DuckDB; it writes SF10 + * only nightly, with Parquet and Vortex alone. + */ +export const FILE_SIZE_SCOPES: readonly FileSizeScope[] = [ + { benchmark: 'clickbench', scale_factor: '1.0' }, + { benchmark: 'tpch', scale_factor: '10.0' }, + { benchmark: 'tpcds', scale_factor: '1.0' }, +]; + +/** + * The SQL scopes the summary ranks: TPC-DS alone. + * + * - ClickBench measures the Parquet files it downloads, which are not zstd, so it has no + * baseline to compare with. + * - TPC-H comes from the same generator as the `compress-bench` TPC-H tables (lineitem, orders, + * and partsupp), which the summary already ranks. Ranking both would count TPC-H twice. + */ +export const FILE_SIZE_SUMMARY_SCOPES: readonly FileSizeScope[] = [ + { benchmark: 'tpcds', scale_factor: '1.0' }, +]; + +/** Whether the group charts this SQL scope. */ +export function isFileSizeScope(benchmark: string, scaleFactor: string): boolean { + return FILE_SIZE_SCOPES.some( + (scope) => scope.benchmark === benchmark && scope.scale_factor === scaleFactor, + ); +} + +/** + * `compression_sizes` charts the group leaves out. Each entry is a `dataset` and + * `dataset_variant`, exactly. + * + * - The rows that the SQL suites wrote before `file_sizes` existed: Vortex sizes only, the last + * on 2026-05-28. `file_sizes` holds those suites' sizes now. + * - The 10-column wide tables, which `compress-bench` stopped writing on 2025-10-07. + * + * None of them has a Parquet size, so the summary, which pairs every size with Parquet's, never + * counts them either. + */ +const HIDDEN_SIZE_DATASETS: readonly (readonly [string, string | null])[] = [ + ['appian', null], + ['clickbench', null], + ['fineweb', null], + ['polarsignals', '1000000'], + ['statpopgen', '100000'], + ['tpcds', null], + ['tpch', null], + ['tpch', '10'], + ['tpch', '100'], + ['wide table cols=10 chunks=1 rows=1000', null], + ['wide table cols=10 chunks=50 rows=1000', null], +]; + +/** Whether the group charts this `compression_sizes` dataset: a current `compress-bench` one. */ +export function isFileSizesDataset(dataset: string, datasetVariant: string | null): boolean { + return ( + compressionSuite(dataset) === 'compress' && + !HIDDEN_SIZE_DATASETS.some( + ([hiddenDataset, hiddenVariant]) => + hiddenDataset === dataset && hiddenVariant === datasetVariant, + ) + ); +} diff --git a/src/bench/lib/group-charts.test.ts b/src/bench/lib/group-charts.test.ts index 519c21eb..11a76600 100644 --- a/src/bench/lib/group-charts.test.ts +++ b/src/bench/lib/group-charts.test.ts @@ -473,27 +473,27 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () [measurementId, sha, dataset, variant, valueBytes], ); } - // File sizes: TPC-H SF=1 has two files per format on C0, E and F (compact only from E); - // SF=10 has one file on F. - for (const [sha, scaleFactor, format, file, valueBytes] of [ - [C0, '1.0', 'vortex-file-compressed', 'a.vortex', 100], - [C0, '1.0', 'vortex-file-compressed', 'b.vortex', 10], - [E, '1.0', 'vortex-file-compressed', 'a.vortex', 101], - [E, '1.0', 'vortex-file-compressed', 'b.vortex', 11], - [E, '1.0', 'vortex-compact', 'a.vortex', 80], - [E, '1.0', 'vortex-compact', 'b.vortex', 8], - [F, '1.0', 'vortex-file-compressed', 'a.vortex', 102], - [F, '1.0', 'vortex-file-compressed', 'b.vortex', 12], - [F, '1.0', 'vortex-compact', 'a.vortex', 79], - [F, '1.0', 'vortex-compact', 'b.vortex', 7], - [F, '10.0', 'vortex-file-compressed', 'a.vortex', 1000], + // File sizes: TPC-DS SF=1 has two files per format on C0, E and F (compact only from E); + // TPC-H SF=10 has one file on F. + for (const [sha, benchmark, scaleFactor, format, file, valueBytes] of [ + [C0, 'tpcds', '1.0', 'vortex-file-compressed', 'a.vortex', 100], + [C0, 'tpcds', '1.0', 'vortex-file-compressed', 'b.vortex', 10], + [E, 'tpcds', '1.0', 'vortex-file-compressed', 'a.vortex', 101], + [E, 'tpcds', '1.0', 'vortex-file-compressed', 'b.vortex', 11], + [E, 'tpcds', '1.0', 'vortex-compact', 'a.vortex', 80], + [E, 'tpcds', '1.0', 'vortex-compact', 'b.vortex', 8], + [F, 'tpcds', '1.0', 'vortex-file-compressed', 'a.vortex', 102], + [F, 'tpcds', '1.0', 'vortex-file-compressed', 'b.vortex', 12], + [F, 'tpcds', '1.0', 'vortex-compact', 'a.vortex', 79], + [F, 'tpcds', '1.0', 'vortex-compact', 'b.vortex', 7], + [F, 'tpch', '10.0', 'vortex-file-compressed', 'a.vortex', 1000], ] as const) { measurementId += 1; await getPool().query( `INSERT INTO file_sizes (measurement_id, commit_sha, benchmark, scale_factor, format, file, value_bytes) - VALUES ($1, $2, 'tpch', $3, $4, $5, $6)`, - [measurementId, sha, scaleFactor, format, file, valueBytes], + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + [measurementId, sha, benchmark, scaleFactor, format, file, valueBytes], ); } }); @@ -584,24 +584,21 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () })), ], [ - { k: 'CompressionSizeGroup' }, + // The File Sizes group mixes the two tables: `compression_sizes` charts, a NULL variant + // between two non-NULL ones, and `file_sizes` charts. + { k: 'FileSizeGroup' }, [ + { k: 'FileSize', benchmark: 'tpcds', scale_factor: '1.0' }, { k: 'CompressionSize', dataset: 'sx', dataset_variant: 'v1' }, { k: 'CompressionSize', dataset: 'sx', dataset_variant: null }, - { k: 'CompressionSize', dataset: 'sy', dataset_variant: 'v2' }, - ], - ], - [ - { k: 'FileSizeGroup' }, - [ - { k: 'FileSize', benchmark: 'tpch', scale_factor: '1.0' }, { k: 'FileSize', benchmark: 'tpch', scale_factor: '10.0' }, + { k: 'CompressionSize', dataset: 'sy', dataset_variant: 'v2' }, ], ], ]; it.each(['1', '2', '3', '4', 'all'])( - 'matches chartPayload for random-access, vector-search, compression-size and file-size groups at ?n=%s', + 'matches chartPayload for random-access, vector-search, and file-size groups at ?n=%s', async (n) => { const window = commitWindow(n); for (const [group, keys] of OTHER_FAMILIES) { @@ -644,11 +641,11 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () }); it('sums each file-size chart over its files, one line per format', async () => { - const [sf1, sf10] = await groupChartPayloads( - ...defined(OTHER_FAMILIES[3]), + const [sf1, , , sf10] = await groupChartPayloads( + ...defined(OTHER_FAMILIES[2]), parseCommitWindow('all'), ); - expect(sf1?.display_name).toBe('tpch sf=1.0'); + expect(sf1?.display_name).toBe('tpcds sf=1.0'); expect(sf1?.unit_kind).toBe('bytes'); expect(sf1?.series).toEqual({ 'vortex-compact': [null, null, null, null, 88, 86], diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index 7d3784cd..c6e0a9cf 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -74,7 +74,7 @@ describe.skipIf(!databaseAvailable)( // Random-access sections precede Clickbench in GROUP_ORDER. expect(groups.map((g) => g.name)).toEqual([ 'Compression', - 'Compression Size', + 'File Sizes', 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', @@ -175,7 +175,7 @@ describe.skipIf(!databaseAvailable)( it('computes compression-size rankings for Vortex, Parquet, Lance, and Arrow IPC', async () => { const groups = await collectGroups(); const summary = expectDefined( - groups.find((g) => g.name === 'Compression Size')?.summary, + groups.find((g) => g.name === 'File Sizes')?.summary, 'compression size summary', ); if (summary.type !== 'compressionSize') { @@ -251,7 +251,7 @@ describe.skipIf(!databaseAvailable)( const body = (await res.json()) as GroupsResponse; expect(body.groups.map((g) => g.name)).toEqual([ 'Compression', - 'Compression Size', + 'File Sizes', 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', @@ -566,7 +566,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg const groups = await collectGroups(); expect(groups.map((group) => group.name)).toEqual([ 'Compression', - 'Compression Size', + 'File Sizes', 'String Encoding', 'String Encoding Size', ]); @@ -626,23 +626,29 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg ]); }); - it('discovers one file-size chart per benchmark and scale factor, with a summary', async () => { + it('charts the listed SQL scopes, and ranks TPC-DS against zstd Parquet', async () => { const develop = 'a'.repeat(40); const nightly = 'b'.repeat(40); await insertCommit(develop, '2026-04-23T12:00:00Z'); await insertCommit(nightly, '2026-04-24T03:00:00Z'); - // TPC-H at SF 1 and 10 from develop, and SF 100 from nightly, the newest commit. Clickbench - // has no scale-factor directory, so it is recorded as 1.0. + // The group charts ClickBench, TPC-H SF10, and TPC-DS SF1. It leaves out TPC-H SF1 and + // SF100 and Appian. The newest commit, a nightly run, writes Parquet and Snappy Parquet + // for TPC-DS but no Vortex. ClickBench has no scale-factor directory, so it is recorded as + // 1.0. for (const [sha, benchmark, scaleFactor, format, file, bytes] of [ [develop, 'tpch', '1.0', 'vortex-file-compressed', 'lineitem.vortex', 300], - [develop, 'tpch', '1.0', 'vortex-file-compressed', 'orders.vortex', 100], - [develop, 'tpch', '1.0', 'vortex-compact', 'lineitem.vortex', 150], - [develop, 'tpch', '1.0', 'vortex-compact', 'orders.vortex', 50], [develop, 'tpch', '10.0', 'vortex-file-compressed', 'lineitem.vortex', 4000], - [develop, 'tpch', '10.0', 'vortex-compact', 'lineitem.vortex', 1000], + [develop, 'tpch', '10.0', 'parquet', 'lineitem.parquet', 2000], [nightly, 'tpch', '100', 'vortex-file-compressed', 'lineitem.vortex', 40000], [develop, 'clickbench', '1.0', 'vortex-file-compressed', 'hits_0.vortex', 1000], - [develop, 'tpcds', '1.0', 'vortex-file-compressed', 'store_sales.vortex', 500], + [develop, 'tpcds', '1.0', 'vortex-file-compressed', 'store_sales.vortex', 300], + [develop, 'tpcds', '1.0', 'vortex-file-compressed', 'item.vortex', 100], + [develop, 'tpcds', '1.0', 'vortex-compact', 'store_sales.vortex', 150], + [develop, 'tpcds', '1.0', 'vortex-compact', 'item.vortex', 50], + [develop, 'tpcds', '1.0', 'parquet', 'store_sales.parquet', 350], + [develop, 'tpcds', '1.0', 'parquet', 'item.parquet', 50], + [nightly, 'tpcds', '1.0', 'parquet', 'store_sales.parquet', 400], + [nightly, 'tpcds', '1.0', 'parquet-snappy', 'store_sales.parquet', 600], [develop, 'appian', '1.0', 'vortex-file-compressed', 'data.vortex', 70], ] as const) { await getPool().query( @@ -654,40 +660,33 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg } const groups = await collectGroups(); - expect(groups.map((group) => group.name)).toEqual(['SQL File Size']); + expect(groups.map((group) => group.name)).toEqual(['File Sizes']); const [group] = groups; expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'FileSizeGroup' }); - // Clickbench, TPC-H and TPC-DS lead; the rest follow by name. TPC-H always names its scale - // factor, while Clickbench and Appian have none. expect(defined(group).charts.map((chart) => chart.name)).toEqual([ 'clickbench', - 'tpch sf=1.0', 'tpch sf=10.0', - 'tpch sf=100', 'tpcds sf=1.0', - 'appian', ]); - expect(chartKeyFromSlug(defined(defined(group).charts[3]).slug)).toEqual({ + expect(chartKeyFromSlug(defined(defined(group).charts[2]).slug)).toEqual({ k: 'FileSize', - benchmark: 'tpch', - scale_factor: '100', + benchmark: 'tpcds', + scale_factor: '1.0', }); - // Compact is half the size at SF 1 and a quarter at SF 10. Totals cover the scopes both - // formats have: 200 + 1000 against 400 + 4000. - expect(defined(group).summary).toMatchObject({ + // Only TPC-DS is ranked. Each format uses its own newest commit that has zstd Parquet: + // Snappy the nightly one, and the Vortex formats the develop one. + const summary = defined(group).summary; + expect(summary).toMatchObject({ type: 'compressionSize', - baseline: 'vortex', + title: 'File Sizes Summary', rankings: [ - { - name: 'vortex-compact', - minRatio: 0.25, - ratio: expect.closeTo(0.5 ** 1.5), - maxRatio: 0.5, - totalBytes: 1200, - }, - { name: 'vortex-file-compressed', ratio: 1, compressionRatio: null, totalBytes: 4400 }, + { name: 'vortex-compact', ratio: 0.5, totalBytes: 200, measured: 1, total: 1 }, + { name: 'parquet', ratio: 1, totalBytes: 400, measured: 1, total: 1 }, + { name: 'vortex-file-compressed', ratio: 1, compressionRatio: null, totalBytes: 400 }, + { name: 'parquet-snappy', ratio: 1.5, totalBytes: 600, measured: 1, total: 1 }, ], }); + expect(summary).not.toHaveProperty('baseline'); }); it('ranks random access over every chart, not the alphabetically first one', async () => { @@ -855,7 +854,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(timeByKey.get('decode:parquet')?.ratio).toBeCloseTo(1.0, 6); const size = expectDefined( - await collectGroupSummary({ k: 'CompressionSizeGroup' }), + await collectGroupSummary({ k: 'FileSizeGroup' }), 'compression-size summary', ); if (size.type !== 'compressionSize') { @@ -869,14 +868,20 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(defined(size.rankings[1]).ratio).toBeCloseTo(1.0, 6); }); - it('leaves the wide-table and sparse-wide datasets out of the compression summaries', async () => { + it('leaves the synthetic datasets out of the time summary, and counts each family once in the size summary', async () => { const sha = 'e'.repeat(40); await insertCommit(sha, '2026-04-23T12:00:00Z'); await insertCompTimePair(sha, 'encode', 1_000, 3_000); await insertCompSizePair(sha, 1_000, 4_000); - for (const dataset of ['wide table cols=1000 chunks=50 rows=1000', 'sparse-wide-nested-runs']) { + for (const [dataset, vortexBytes, parquetBytes] of [ + ['wide table cols=1000 chunks=1 rows=1000', 9_000, 1_000], + // The same file as `chunks=1`, so the size summary leaves it out. + ['wide table cols=1000 chunks=50 rows=1000', 1, 1_000], + ['sparse-wide-nested-runs', 1_000, 4_000], + ['sparse-wide-flat-runs', 4_000, 1_000], + ] as const) { await insertCompTimePair(sha, 'encode', 9_000, 1_000, dataset); - await insertCompSizePair(sha, 9_000, 1_000, dataset); + await insertCompSizePair(sha, vortexBytes, parquetBytes, dataset); } // The patterns match only at the start of the name, so this dataset stays in. await insertCompTimePair(sha, 'encode', 1_000, 3_000, 'taxi-sparse-wide'); @@ -898,19 +903,23 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(vortexTime.ratio).toBeCloseTo(3.0, 6); const size = expectDefined( - await collectGroupSummary({ k: 'CompressionSizeGroup' }), - 'compression-size summary', + await collectGroupSummary({ k: 'FileSizeGroup' }), + 'file size summary', ); if (size.type !== 'compressionSize') { throw new Error(`expected compressionSize summary, got ${size.type}`); } const vortexSize = expectDefined( size.rankings.find((ranking) => ranking.name === 'vortex-file-compressed'), - 'Vortex compression-size ranking', + 'Vortex file size ranking', ); + // Four entries: tpch-lineitem 0.25, taxi-sparse-wide 1, the wide tables 9, and the + // sparse-wide family sqrt(0.25 * 4) = 1. + expect(vortexSize.ratio).toBeCloseTo(2.25 ** 0.25, 6); + expect([vortexSize.measured, vortexSize.total]).toEqual([4, 4]); expect(vortexSize.minRatio).toBeCloseTo(0.25, 6); - expect(vortexSize.ratio).toBeCloseTo(0.5, 6); - expect(vortexSize.maxRatio).toBeCloseTo(1.0, 6); + expect(vortexSize.maxRatio).toBeCloseTo(9, 6); + expect(vortexSize.totalBytes).toBe(1_000 + 1_000 + 9_000 + 1_000 + 4_000); }); it('selects one commit when the latest timestamps collide', async () => { @@ -940,7 +949,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(vortexTime.ratio).toBeCloseTo(4.0, 6); const size = expectDefined( - await collectGroupSummary({ k: 'CompressionSizeGroup' }), + await collectGroupSummary({ k: 'FileSizeGroup' }), 'compression-size summary', ); if (size.type !== 'compressionSize') { diff --git a/src/bench/lib/headline.test.ts b/src/bench/lib/headline.test.ts index e9897649..1f446152 100644 --- a/src/bench/lib/headline.test.ts +++ b/src/bench/lib/headline.test.ts @@ -186,7 +186,16 @@ describe('groupHeadline', () => { title: 's', explanation: 'e', rankings: [ - { name: VX, minRatio: ratio, ratio, maxRatio: ratio, compressionRatio: 5, totalBytes: 1 }, + { + name: VX, + minRatio: ratio, + ratio, + maxRatio: ratio, + compressionRatio: 5, + totalBytes: 1, + measured: 1, + total: 1, + }, ], }); expect(h && headlineText(h)).toBe(expected); @@ -202,7 +211,16 @@ describe('groupHeadline', () => { explanation: 'e', baseline: 'vortex', rankings: [ - { name: VX, minRatio: 1, ratio: 1, maxRatio: 1, compressionRatio: null, totalBytes: 1 }, + { + name: VX, + minRatio: 1, + ratio: 1, + maxRatio: 1, + compressionRatio: null, + totalBytes: 1, + measured: 1, + total: 1, + }, ], }), ).toBeNull(); diff --git a/src/bench/lib/queries.test.ts b/src/bench/lib/queries.test.ts index 777d7dad..74e7ac1e 100644 --- a/src/bench/lib/queries.test.ts +++ b/src/bench/lib/queries.test.ts @@ -371,21 +371,19 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => { ]); }); - it('puts compression and SQL file size before random access and string encoding last', () => { + it('puts compression and file sizes before random access and string encoding last', () => { const sorted = [ 'String Encoding Size', 'Clickbench', 'cohere-large-10m / partitioned', 'String Encoding', 'Random Access (NVMe)', - 'SQL File Size', - 'Compression Size', + 'File Sizes', 'Compression', ].sort(compareGroupSortKey); expect(sorted).toEqual([ 'Compression', - 'Compression Size', - 'SQL File Size', + 'File Sizes', 'Random Access (NVMe)', 'Clickbench', 'cohere-large-10m / partitioned', diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index de6d0a3c..20279506 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -42,6 +42,12 @@ import { STRING_DATASET_PREFIX, } from './compression-suite'; import { groupDescription } from './descriptions'; +import { + FILE_SIZE_SCOPES, + FILE_SIZES_GROUP_NAME, + isFileSizeScope, + isFileSizesDataset, +} from './file-sizes'; import type { FilterUniverse } from './chart-format'; import { type ChartKind, compareCodeUnits, FAMILIES, type GroupKind } from './families'; import { chartKeyFromSlug, chartKeyToSlug, groupKeyFromSlug, groupKeyToSlug } from './slug'; @@ -782,9 +788,7 @@ function chartInGroup(group: GroupKey, key: ChartKey): boolean { key.k === 'CompressionTime' && compressionSuite(key.dataset) === (group.suite ?? 'compress') ); case 'CompressionSizeGroup': - return ( - key.k === 'CompressionSize' && compressionSuite(key.dataset) === (group.suite ?? 'compress') - ); + return key.k === 'CompressionSize' && compressionSuite(key.dataset) === group.suite; case 'RandomAccessGroup': return key.k === 'RandomAccess' && randomAccessStorage(key.dataset) === group.storage; case 'VectorSearchGroup': @@ -792,7 +796,10 @@ function chartInGroup(group: GroupKey, key: ChartKey): boolean { key.k === 'VectorSearch' && key.dataset === group.dataset && key.layout === group.layout ); case 'FileSizeGroup': - return key.k === 'FileSize'; + return ( + (key.k === 'FileSize' && isFileSizeScope(key.benchmark, key.scale_factor)) || + (key.k === 'CompressionSize' && isFileSizesDataset(key.dataset, key.dataset_variant)) + ); } } @@ -877,6 +884,9 @@ function queryChartEarliest(pins: (alias: string) => string): string { /** One per-chart column of a [`groupRowsQuery`] block: its name in `ch`, element type, values. */ type ChartColumn = readonly [name: string, type: string, values: readonly unknown[]]; +/** The key of a chart from either compression table. */ +type CompressionChartKey = Extract; + /** * One row per (chart, fact row in the window), plus one row with NULL fact * columns for a chart with no rows there. `ord` is the chart's 1-based position @@ -934,6 +944,48 @@ function groupRowsQuery( ${rows} ) d ON true`); }; + /** + * The blocks for compression charts, each at its 1-based position in `charts`. + * `dataset_variant` varies per chart. `IS NOT DISTINCT FROM ch.dataset_variant` would plan as + * if it matched one row and rescan the window per fact row, so the charts split by whether the + * variant is NULL and each block pins it sargably, as `sargableDimEq` does. + */ + const compressionBlocks = ( + kind: CompressionChartKey['k'], + charts: readonly (readonly [CompressionChartKey, number])[], + ): void => { + const time = kind === 'CompressionTime'; + const table = time ? 'compression_times' : 'compression_sizes'; + const value = time ? 'value_ns' : 'value_bytes'; + const fields = time ? ['format', 'op'] : ['format']; + for (const nullVariant of [true, false]) { + const picked = charts.filter(([key]) => (key.dataset_variant === null) === nullVariant); + if (picked.length === 0) { + continue; + } + const columns: ChartColumn[] = [['dataset', 'text', picked.map(([key]) => key.dataset)]]; + if (!nullVariant) { + columns.push(['dataset_variant', 'text', picked.map(([key]) => key.dataset_variant)]); + } + const pins = (alias: string): string => + `${alias}.dataset = ch.dataset + AND ${alias}.dataset_variant ${nullVariant ? 'IS NULL' : '= ch.dataset_variant'}`; + block( + fields, + picked.map(([, position]) => position), + columns, + `SELECT MIN(c2.timestamp) + FROM ${table} f2 + JOIN ${BRANCH_COMMITS} c2 ON c2.commit_sha = f2.commit_sha + WHERE ${pins('f2')}`, + `SELECT f.commit_sha, + ${fields.map((field) => `f.${field}`).join(', ')}, f.${value}::float8 AS value + FROM ${table} f + JOIN ${BRANCH_COMMITS} c USING (commit_sha) + WHERE ${pins('f')}${factWindowFilter(params, window)}`, + ); + } + }; switch (group.k) { case 'QueryGroup': { const keys = chartsOf(group, charts, 'QueryMeasurement'); @@ -957,50 +1009,20 @@ function groupRowsQuery( ); break; } - case 'CompressionTimeGroup': + case 'CompressionTimeGroup': { + const keys = chartsOf(group, charts, 'CompressionTime'); + compressionBlocks( + 'CompressionTime', + keys.map((key, i) => [key, i + 1]), + ); + break; + } case 'CompressionSizeGroup': { - const time = group.k === 'CompressionTimeGroup'; - const table = time ? 'compression_times' : 'compression_sizes'; - const value = time ? 'value_ns' : 'value_bytes'; - const fields = time ? ['format', 'op'] : ['format']; - const keys = chartsOf(group, charts, time ? 'CompressionTime' : 'CompressionSize'); - // `dataset_variant` varies per chart. `IS NOT DISTINCT FROM ch.dataset_variant` would plan - // as if it matched one row and rescan the window per fact row, so split the charts by - // whether the variant is NULL and pin each block sargably, as `sargableDimEq` does. - for (const nullVariant of [true, false]) { - const positions: number[] = []; - const picked: typeof keys = []; - keys.forEach((key, i) => { - if ((key.dataset_variant === null) === nullVariant) { - positions.push(i + 1); - picked.push(key); - } - }); - if (picked.length === 0) { - continue; - } - const columns: ChartColumn[] = [['dataset', 'text', picked.map((k) => k.dataset)]]; - if (!nullVariant) { - columns.push(['dataset_variant', 'text', picked.map((k) => k.dataset_variant)]); - } - const pins = (alias: string): string => - `${alias}.dataset = ch.dataset - AND ${alias}.dataset_variant ${nullVariant ? 'IS NULL' : '= ch.dataset_variant'}`; - block( - fields, - positions, - columns, - `SELECT MIN(c2.timestamp) - FROM ${table} f2 - JOIN ${BRANCH_COMMITS} c2 ON c2.commit_sha = f2.commit_sha - WHERE ${pins('f2')}`, - `SELECT f.commit_sha, - ${fields.map((field) => `f.${field}`).join(', ')}, f.${value}::float8 AS value - FROM ${table} f - JOIN ${BRANCH_COMMITS} c USING (commit_sha) - WHERE ${pins('f')}${factWindowFilter(params, window)}`, - ); - } + const keys = chartsOf(group, charts, 'CompressionSize'); + compressionBlocks( + 'CompressionSize', + keys.map((key, i) => [key, i + 1]), + ); break; } case 'RandomAccessGroup': { @@ -1048,15 +1070,32 @@ function groupRowsQuery( break; } case 'FileSizeGroup': { - const keys = chartsOf(group, charts, 'FileSize'); + // The group holds two families. Both give one size per format, so their blocks share + // columns. + const sizes: [CompressionChartKey, number][] = []; + const files: [Extract, number][] = []; + charts.forEach((key, i) => { + if (!chartInGroup(group, key)) { + throw new Error(`chart ${chartKeyToSlug(key)} is not in group ${groupKeyToSlug(group)}`); + } + if (key.k === 'CompressionSize') { + sizes.push([key, i + 1]); + } else if (key.k === 'FileSize') { + files.push([key, i + 1]); + } + }); + compressionBlocks('CompressionSize', sizes); + if (files.length === 0) { + break; + } const pins = (alias: string): string => `${alias}.benchmark = ch.benchmark AND ${alias}.scale_factor = ch.scale_factor`; block( ['format'], - keys.map((_, i) => i + 1), + files.map(([, position]) => position), [ - ['benchmark', 'text', keys.map((k) => k.benchmark)], - ['scale_factor', 'text', keys.map((k) => k.scale_factor)], + ['benchmark', 'text', files.map(([k]) => k.benchmark)], + ['scale_factor', 'text', files.map(([k]) => k.scale_factor)], ], `SELECT MIN(c2.timestamp) FROM file_sizes f2 @@ -1248,8 +1287,7 @@ export interface GroupChartsResponse { */ const GROUP_ORDER: readonly string[] = [ 'Compression', - 'Compression Size', - 'SQL File Size', + 'File Sizes', 'Random Access (NVMe)', 'Random Access (S3)', 'Clickbench', @@ -1531,46 +1569,60 @@ async function collectQueryGroups(): Promise { type DatasetVariantRow = { dataset: string; dataset_variant: string | null }; -/** Group display names per compression suite and family. */ -const COMPRESSION_GROUP_NAMES: Record< - 'CompressionTimeGroup' | 'CompressionSizeGroup', - Record -> = { +/** + * Group display names per compression suite and family. `compress-bench`'s sizes are in the File + * Sizes group, so the size family names only `string-bench`'s group. + */ +const COMPRESSION_GROUP_NAMES = { CompressionTimeGroup: { compress: 'Compression', strings: 'String Encoding' }, - CompressionSizeGroup: { compress: 'Compression Size', strings: 'String Encoding Size' }, -}; + CompressionSizeGroup: { strings: 'String Encoding Size' }, +} as const; + +/** A compression `(dataset, dataset_variant)` row's chart link, named as its chart is. */ +function compressionChartLink( + row: DatasetVariantRow, + chartKind: CompressionChartKey['k'], +): ChartLink { + const dataset = + compressionSuite(row.dataset) === 'strings' + ? row.dataset.slice(STRING_DATASET_PREFIX.length) + : row.dataset; + return { + name: row.dataset_variant !== null ? `${dataset}/${row.dataset_variant}` : dataset, + slug: chartKeyToSlug({ + k: chartKind, + dataset: row.dataset, + dataset_variant: row.dataset_variant, + }), + }; +} /** - * Split one compression table's `(dataset, dataset_variant)` rows into the `compress-bench` - * group and the `string-bench` group, dropping a group with no rows. String charts drop the - * shared `strings/` prefix from their names. + * Split one compression table's `(dataset, dataset_variant)` rows into the family's suite + * groups, dropping a group with no rows. String charts drop the shared `strings/` prefix from + * their names. */ function compressionGroups( rows: readonly DatasetVariantRow[], groupKind: 'CompressionTimeGroup' | 'CompressionSizeGroup', ): Group[] { const chartKind = groupKind === 'CompressionTimeGroup' ? 'CompressionTime' : 'CompressionSize'; + const names: Partial> = COMPRESSION_GROUP_NAMES[groupKind]; const suites: CompressionSuite[] = ['compress', 'strings']; return suites.flatMap((suite) => { + const name = names[suite]; + if (name === undefined) { + return []; + } const charts: ChartLink[] = rows .filter((row) => compressionSuite(row.dataset) === suite) - .map((row) => { - const dataset = - suite === 'strings' ? row.dataset.slice(STRING_DATASET_PREFIX.length) : row.dataset; - return { - name: row.dataset_variant !== null ? `${dataset}/${row.dataset_variant}` : dataset, - slug: chartKeyToSlug({ - k: chartKind, - dataset: row.dataset, - dataset_variant: row.dataset_variant, - }), - }; - }); + .map((row) => compressionChartLink(row, chartKind)); if (charts.length === 0) { return []; } - const key: GroupKey = suite === 'strings' ? { k: groupKind, suite } : { k: groupKind }; - return [{ name: COMPRESSION_GROUP_NAMES[groupKind][suite], slug: groupKeyToSlug(key), charts }]; + const key: GroupKey = + suite === 'strings' ? { k: groupKind, suite } : { k: 'CompressionTimeGroup' }; + return [{ name, slug: groupKeyToSlug(key), charts }]; }); } @@ -1586,16 +1638,20 @@ async function collectCompressionTimeGroup(): Promise { return compressionGroups(rows, 'CompressionTimeGroup'); } -/** The `Compression Size` and `String Encoding Size` groups, or `[]` if the fact table is empty. */ -async function collectCompressionSizeGroup(): Promise { +/** Every `(dataset, dataset_variant)` in `compression_sizes`. */ +async function compressionSizeDatasets(): Promise { const text = ` SELECT dataset, dataset_variant FROM compression_sizes GROUP BY dataset, dataset_variant ORDER BY dataset, dataset_variant NULLS FIRST `; - const rows = (await getPool().query(text)).rows; - return compressionGroups(rows, 'CompressionSizeGroup'); + return (await getPool().query(text)).rows; +} + +/** The `String Encoding Size` group, or `[]` if `string-bench` has no sizes. */ +async function collectCompressionSizeGroup(): Promise { + return compressionGroups(await compressionSizeDatasets(), 'CompressionSizeGroup'); } /** One random-access group per storage backend, omitting empty sections. */ @@ -1673,67 +1729,47 @@ async function collectVectorSearchGroups(): Promise { type FileSizeScopeRow = { benchmark: string; scale_factor: string }; -/** Benchmarks whose file-size charts lead the group, in this order; the rest follow by name. */ -const FILE_SIZE_CHART_ORDER: readonly string[] = ['clickbench', 'tpch', 'tpcds']; - -/** A benchmark's position in the file-size group: its `FILE_SIZE_CHART_ORDER` index, else after. */ -function fileSizeChartRank(benchmark: string): number { - const pos = FILE_SIZE_CHART_ORDER.indexOf(benchmark); - return pos === -1 ? FILE_SIZE_CHART_ORDER.length : pos; -} - /** - * The `SQL File Size` group: one chart per benchmark and scale factor, or `[]` if the table is - * empty. Charts order by [`FILE_SIZE_CHART_ORDER`], then by benchmark name, then by numeric - * scale factor, so `10.0` follows `1.0`. + * The File Sizes group, or `[]` if neither table has a chart for it: the SQL suites' scopes in + * [`FILE_SIZE_SCOPES`] that have rows, in that order, then every `compress-bench` dataset in + * `compression_sizes` by name. */ async function collectFileSizeGroup(): Promise { - // A recursive skip scan of `idx_file_sizes_baseline`: one descent per scope instead of a scan - // of every file of every commit. - const text = ` - WITH RECURSIVE scopes AS ( - (SELECT f.benchmark, f.scale_factor - FROM file_sizes f - ORDER BY f.benchmark, f.scale_factor - LIMIT 1) - UNION ALL - SELECT nxt.benchmark, nxt.scale_factor - FROM scopes s - CROSS JOIN LATERAL ( - SELECT f.benchmark, f.scale_factor - FROM file_sizes f - WHERE (f.benchmark, f.scale_factor) > (s.benchmark, s.scale_factor) - ORDER BY f.benchmark, f.scale_factor - LIMIT 1 - ) nxt - ) - SELECT benchmark, scale_factor FROM scopes + // One index probe of `idx_file_sizes_baseline` per listed scope. + const scopesText = ` + SELECT s.benchmark, s.scale_factor + FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS s(benchmark, scale_factor, ord) + WHERE EXISTS ( + SELECT 1 + FROM file_sizes f + WHERE f.benchmark = s.benchmark + AND f.scale_factor = s.scale_factor) + ORDER BY s.ord `; - const rows = (await getPool().query(text)).rows; - if (rows.length === 0) { + const [scopes, datasets] = await Promise.all([ + getPool().query(scopesText, [ + FILE_SIZE_SCOPES.map((scope) => scope.benchmark), + FILE_SIZE_SCOPES.map((scope) => scope.scale_factor), + ]), + compressionSizeDatasets(), + ]); + const charts: ChartLink[] = [ + ...scopes.rows.map((row) => ({ + name: fileSizeChartName(row.benchmark, row.scale_factor), + slug: chartKeyToSlug({ + k: 'FileSize', + benchmark: row.benchmark, + scale_factor: row.scale_factor, + }), + })), + ...datasets + .filter((row) => isFileSizesDataset(row.dataset, row.dataset_variant)) + .map((row) => compressionChartLink(row, 'CompressionSize')), + ]; + if (charts.length === 0) { return []; } - rows.sort( - (a, b) => - fileSizeChartRank(a.benchmark) - fileSizeChartRank(b.benchmark) || - compareCodeUnits(a.benchmark, b.benchmark) || - Number(a.scale_factor) - Number(b.scale_factor) || - compareCodeUnits(a.scale_factor, b.scale_factor), - ); - return [ - { - name: 'SQL File Size', - slug: groupKeyToSlug({ k: 'FileSizeGroup' }), - charts: rows.map((row) => ({ - name: fileSizeChartName(row.benchmark, row.scale_factor), - slug: chartKeyToSlug({ - k: 'FileSize', - benchmark: row.benchmark, - scale_factor: row.scale_factor, - }), - })), - }, - ]; + return [{ name: FILE_SIZES_GROUP_NAME, slug: groupKeyToSlug({ k: 'FileSizeGroup' }), charts }]; } /** Dispatch to the discovery pass for one family, the TS analogue of the Rust diff --git a/src/bench/lib/site-files.test.ts b/src/bench/lib/site-files.test.ts index cfb6e131..147edf69 100644 --- a/src/bench/lib/site-files.test.ts +++ b/src/bench/lib/site-files.test.ts @@ -136,6 +136,8 @@ describe('summaryLines', () => { maxRatio: 1.549, compressionRatio: 5.744, totalBytes: 1_603_980_000, + measured: 1, + total: 1, }, ], }).at(-1), diff --git a/src/bench/lib/site-files.ts b/src/bench/lib/site-files.ts index 8a5dd95b..5ca40b56 100644 --- a/src/bench/lib/site-files.ts +++ b/src/bench/lib/site-files.ts @@ -323,7 +323,10 @@ export function summaryLines(summary: Summary): string[] { (item.compressionRatio === null ? '' : `, ${formatCompressionSizeRatio(item.compressionRatio)} smaller than Arrow in memory`) + - (item.totalBytes === null ? '' : `, ${formatBytes(item.totalBytes)} in total`), + (item.totalBytes === null ? '' : `, ${formatBytes(item.totalBytes)} in total`) + + (item.measured < item.total + ? `, measured on ${item.measured} of ${item.total} datasets` + : ''), ), ]; } diff --git a/src/bench/lib/slug.test.ts b/src/bench/lib/slug.test.ts index 5f168d30..012f3c02 100644 --- a/src/bench/lib/slug.test.ts +++ b/src/bench/lib/slug.test.ts @@ -31,12 +31,12 @@ const CHART_KEYS: ChartKey[] = [ const GROUP_KEYS: GroupKey[] = [ { k: 'QueryGroup', dataset: 'tpch', dataset_variant: null, scale_factor: '1', storage: 'nvme' }, { k: 'CompressionTimeGroup' }, - { k: 'CompressionSizeGroup' }, { k: 'CompressionTimeGroup', suite: 'strings' }, { k: 'CompressionSizeGroup', suite: 'strings' }, { k: 'RandomAccessGroup', storage: 'nvme' }, { k: 'RandomAccessGroup', storage: 's3' }, { k: 'VectorSearchGroup', dataset: 'cohere', layout: 'partitioned' }, + { k: 'FileSizeGroup' }, ]; describe('chart slug round-trip', () => { @@ -105,6 +105,12 @@ describe('group slug round-trip', () => { expect(groupKeyFromSlug(legacy)).toEqual({ k: 'RandomAccessGroup', storage: 'nvme' }); }); + it('resolves the Compression Size group slug to File Sizes, which holds its charts', () => { + const legacy = 'csg.eyJrIjoiQ29tcHJlc3Npb25TaXplR3JvdXAifQ'; + expect(groupKeyFromSlug(legacy)).toEqual({ k: 'FileSizeGroup' }); + expect(groupKeyToSlug(groupKeyFromSlug(legacy))).toBe('fsg.eyJrIjoiRmlsZVNpemVHcm91cCJ9'); + }); + it.each(['disk', '', null, 1])('rejects invalid random-access storage %s', (storage) => { const slug = 'rag.' + diff --git a/src/bench/lib/slug.ts b/src/bench/lib/slug.ts index 89d6a97d..800bccfc 100644 --- a/src/bench/lib/slug.ts +++ b/src/bench/lib/slug.ts @@ -33,7 +33,7 @@ export type ChartKey = | { k: 'CompressionSize'; dataset: string; dataset_variant: string | null } | { k: 'RandomAccess'; dataset: string } | { k: 'VectorSearch'; dataset: string; layout: string; threshold: number } - // One SQL benchmark's data at one scale factor; its series are the Vortex formats. + // One SQL benchmark's data at one scale factor; its series are the formats it writes. | { k: 'FileSize'; benchmark: string; scale_factor: string }; /** Strongly-typed group key parsed from a slug. Discriminated on `k`. */ @@ -47,9 +47,11 @@ export type GroupKey = } // `suite` is absent for the `compress-bench` group, so its slug predates the split. | { k: 'CompressionTimeGroup'; suite?: 'strings' } - | { k: 'CompressionSizeGroup'; suite?: 'strings' } + // Only `string-bench`'s sizes. `compress-bench`'s sizes are in the File Sizes group. + | { k: 'CompressionSizeGroup'; suite: 'strings' } | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string } + // The File Sizes group: `compress-bench`'s sizes and the SQL suites' file sizes. | { k: 'FileSizeGroup' }; const CHART_KINDS: readonly ChartKind[] = [ @@ -254,9 +256,11 @@ export function groupKeyToSlug(key: GroupKey): string { ordered = { k: key.k, storage: key.storage }; break; case 'CompressionTimeGroup': - case 'CompressionSizeGroup': ordered = key.suite === undefined ? { k: key.k } : { k: key.k, suite: key.suite }; break; + case 'CompressionSizeGroup': + ordered = { k: key.k, suite: key.suite }; + break; case 'FileSizeGroup': ordered = { k: key.k }; break; @@ -294,8 +298,7 @@ export function groupKeyFromSlug(slug: string): GroupKey { } return { k: kind, storage }; } - case 'CompressionTimeGroup': - case 'CompressionSizeGroup': { + case 'CompressionTimeGroup': { if (obj.suite === undefined) { return { k: kind }; } @@ -304,6 +307,16 @@ export function groupKeyFromSlug(slug: string): GroupKey { } return { k: kind, suite: 'strings' }; } + case 'CompressionSizeGroup': { + // The `Compression Size` group's links resolve to File Sizes, which holds its charts now. + if (obj.suite === undefined) { + return { k: 'FileSizeGroup' }; + } + if (obj.suite !== 'strings') { + throw new Error('compression group suite must be strings when present'); + } + return { k: kind, suite: 'strings' }; + } case 'FileSizeGroup': return { k: kind }; } diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index d053b1f4..fa6ad451 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -20,6 +20,9 @@ const GROUP_KINDS = FAMILIES.map((family) => family.groupKind); describe('compression summaries', () => { beforeEach(() => { query.mockReset(); + // The File Sizes summary also reads the SQL suites' sizes; a test that sets no rows for that + // second query gets none. + query.mockResolvedValue({ rows: [] }); }); it('uses geometric mean size ratios', async () => { @@ -60,7 +63,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -82,12 +85,17 @@ describe('compression summaries', () => { // Both formats have both datasets, so each total sums both. expect(byFormat.get('vortex-file-compressed')?.totalBytes).toBe(901); expect(byFormat.get('parquet')?.totalBytes).toBe(104); + expect(summary.title).toBe('File Sizes Summary'); expect(summary.explanation).toBe( - 'Total compressed size, and geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', + 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', ); + expect(summary.rankings.map((ranking) => [ranking.measured, ranking.total])).toEqual([ + [2, 2], + [2, 2], + ]); }); - it('sums totals over the datasets every format has', async () => { + it('totals every dataset, for the formats measured on all of them', async () => { const row = (format: string, dataset: string, valueBytes: number, datasetVariant = null) => ({ format, dataset, @@ -102,23 +110,104 @@ describe('compression summaries', () => { row('vortex-file-compressed', 'b', 1_000), row('parquet', 'a', 20), row('parquet', 'b', 2_000), - // Lance has no size for dataset b, so no total counts b. - row('lance', 'a', 30), + // Lance has no size for dataset b, so it has no total, and it ranks after the formats + // measured on both datasets although its ratio is the lowest. + row('lance', 'a', 3), ], }); - const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } - const totals = new Map(summary.rankings.map((ranking) => [ranking.name, ranking.totalBytes])); - expect(totals).toEqual( - new Map([ - ['vortex-file-compressed', 10], - ['parquet', 20], - ['lance', 30], + expect( + summary.rankings.map((ranking) => [ + ranking.name, + ranking.totalBytes, + ranking.measured, + ranking.total, ]), - ); + ).toEqual([ + ['vortex-file-compressed', 1_010, 2, 2], + ['parquet', 2_020, 2, 2], + ['lance', null, 1, 2], + ]); + }); + + it('counts each synthetic family as one dataset', async () => { + const row = (format: string, dataset: string, valueBytes: number) => ({ + format, + dataset, + datasetVariant: null, + valueBytes, + baselineBytes: 100, + uncompressedBytes: null, + }); + query.mockResolvedValueOnce({ + rows: [ + row('vortex-file-compressed', 'taxi', 400), + // Three wide tables and two sparse-wide shapes: one entry each, their geometric means + // 1 and 0.25. + row('vortex-file-compressed', 'wide table cols=100 chunks=1 rows=1000', 50), + row('vortex-file-compressed', 'wide table cols=1000 chunks=1 rows=1000', 100), + row('vortex-file-compressed', 'wide table cols=10000 chunks=1 rows=1000', 200), + row('vortex-file-compressed', 'sparse-wide-flat-runs', 5), + row('vortex-file-compressed', 'sparse-wide-lists-runs', 125), + ], + }); + + const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + if (summary === null || summary.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + const vortex = defined(summary.rankings[0]); + // The cube root of 4 * 1 * 0.25 is 1. Counted one by one, the six datasets give 2^(1/6). + expect(vortex.ratio).toBeCloseTo(1, 6); + expect([vortex.measured, vortex.total]).toEqual([3, 3]); + // The range is over single datasets, and the total over every one of them. + expect(vortex.minRatio).toBeCloseTo(0.05, 6); + expect(vortex.maxRatio).toBeCloseTo(4, 6); + expect(vortex.totalBytes).toBe(880); + }); + + it("ranks the SQL suites' totals with the compress-bench sizes", async () => { + const row = (format: string, dataset: string, valueBytes: number, baselineBytes: number) => ({ + format, + dataset, + datasetVariant: dataset === 'tpcds' ? 'sf=1.0' : null, + valueBytes, + baselineBytes, + uncompressedBytes: null, + }); + query.mockResolvedValueOnce({ + rows: [row('vortex-file-compressed', 'taxi', 50, 100), row('parquet', 'taxi', 100, 100)], + }); + query.mockResolvedValueOnce({ + rows: [ + row('vortex-file-compressed', 'tpcds', 200, 100), + row('vortex-compact', 'tpcds', 50, 100), + row('parquet', 'tpcds', 100, 100), + ], + }); + + const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + if (summary === null || summary.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + expect(query.mock.calls[1]?.[1]).toEqual([['tpcds'], ['1.0'], 'parquet']); + expect( + summary.rankings.map((ranking) => [ + ranking.name, + ranking.ratio, + ranking.measured, + ranking.totalBytes, + ]), + ).toEqual([ + ['parquet', 1, 2, 200], + ['vortex-file-compressed', 1, 2, 250], + // Vortex Compact has a size for TPC-DS alone, so it ranks after the complete formats. + ['vortex-compact', 0.5, 1, null], + ]); }); it('gives no total when the formats share no dataset', async () => { @@ -143,7 +232,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -192,18 +281,24 @@ describe('compression summaries', () => { rows: [ { format: 'vortex-file-compressed', + dataset: 'a', + datasetVariant: null, valueBytes: 200, baselineBytes: 100, uncompressedBytes: 2_000, }, { format: 'parquet', + dataset: 'a', + datasetVariant: null, valueBytes: 100, baselineBytes: 100, uncompressedBytes: 500, }, { format: 'lance', + dataset: 'a', + datasetVariant: null, valueBytes: 300, baselineBytes: 100, uncompressedBytes: null, @@ -211,7 +306,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -288,9 +383,10 @@ describe('compression summaries', () => { query.mockResolvedValue({ rows: [] }); await collectGroupSummary({ k: 'CompressionTimeGroup' }); - await collectGroupSummary({ k: 'CompressionSizeGroup' }); + await collectGroupSummary({ k: 'FileSizeGroup' }); - expect(query).toHaveBeenCalledTimes(2); + // The File Sizes summary reads the SQL suites' sizes in a third query. + expect(query).toHaveBeenCalledTimes(3); const timingParams = [ ['vortex-file-compressed', 'parquet', 'lance'], ['vortex-file-compressed', 'parquet'], @@ -299,14 +395,21 @@ describe('compression summaries', () => { ['wide table %', 'sparse-wide-%'], ]; const sizeParams = [ - ['vortex-file-compressed', 'parquet', 'lance', 'arrow-ipc'], - ['vortex-file-compressed', 'parquet'], + [ + 'vortex-file-compressed', + 'parquet', + 'parquet-snappy', + 'vortex-compact', + 'lance', + 'arrow-ipc', + ], + ['vortex-file-compressed', 'parquet', 'parquet-snappy', 'vortex-compact'], 'vortex-file-compressed', 'parquet', 'arrow-ipc', - ['wide table %', 'sparse-wide-%', 'tpc-h l_comment chunked', 'statpopgen'], + ['tpc-h l_comment chunked', 'wide table cols=% chunks=50 rows=%', 'statpopgen'], ]; - const calls = query.mock.calls as Array<[string, unknown[]]>; + const calls = (query.mock.calls as Array<[string, unknown[]]>).slice(0, 2); expect(defined(calls[0])[1]).toEqual(timingParams); expect(defined(calls[1])[1]).toEqual(sizeParams); for (const [text] of calls) { @@ -750,7 +853,7 @@ describe('timing summaries (shared ranking model)', () => { storage: 'nvme', }, { k: 'CompressionTimeGroup' }, - { k: 'CompressionSizeGroup' }, + { k: 'CompressionSizeGroup', suite: 'strings' }, { k: 'RandomAccessGroup', storage: 'nvme' }, { k: 'VectorSearchGroup', dataset: 'd', layout: 'l' }, { k: 'FileSizeGroup' }, diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index 43edd969..15f02896 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -41,8 +41,10 @@ import { getPool } from './db'; import { randomAccessStorage, type RandomAccessStorage } from './random-access'; import { compareCodeUnits } from './families'; import type { GroupKey } from './slug'; +import { FILE_SIZE_SUMMARY_SCOPES } from './file-sizes'; import { SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS, + SIZE_SUMMARY_FAMILIES, STRING_DATASET_PATTERN, SUMMARY_EXCLUDED_DATASET_PATTERNS, } from './compression-suite'; @@ -58,6 +60,13 @@ const COMPRESSION_SHARED_SNAPSHOT_FORMATS = ['vortex-file-compressed', 'parquet' const COMPRESSION_INDEPENDENT_SNAPSHOT_FORMATS = ['lance']; const ARROW_IPC_FORMAT = 'arrow-ipc'; +/** + * Size-only formats that `compress-bench` writes beside the timed ones on every run, so they + * share the Vortex snapshot too. `parquet-snappy` is Parquet with Snappy; `parquet` is zstd. + * `vortex-compact` is Vortex with the compact strategy, as the SQL suites write it. + */ +const SIZE_ONLY_SHARED_SNAPSHOT_FORMATS = ['parquet-snappy', 'vortex-compact']; + const COMPRESSION_SNAPSHOT_ANCHOR = 'vortex-file-compressed'; const COMPRESSION_BASELINE = 'parquet'; @@ -82,13 +91,10 @@ function compressionSizeSummaryQueryParams(): [ string, string[], ] { + const shared = [...COMPRESSION_SHARED_SNAPSHOT_FORMATS, ...SIZE_ONLY_SHARED_SNAPSHOT_FORMATS]; return [ - [ - ...COMPRESSION_SHARED_SNAPSHOT_FORMATS, - ...COMPRESSION_INDEPENDENT_SNAPSHOT_FORMATS, - ARROW_IPC_FORMAT, - ], - COMPRESSION_SHARED_SNAPSHOT_FORMATS, + [...shared, ...COMPRESSION_INDEPENDENT_SNAPSHOT_FORMATS, ARROW_IPC_FORMAT], + shared, COMPRESSION_SNAPSHOT_ANCHOR, COMPRESSION_BASELINE, ARROW_IPC_FORMAT, @@ -132,7 +138,7 @@ interface CompressionRanking { throughputGbS?: number; } -/** One format in the compression size summary. */ +/** One format in a size summary. */ interface CompressionSizeRanking { /** On-disk format. */ name: string; @@ -145,10 +151,15 @@ interface CompressionSizeRanking { /** Geomean ratio of Arrow memory size to the format size. */ compressionRatio: number | null; /** - * Sum of this format's sizes, in bytes, over the datasets that every format in the summary - * has, so the totals compare directly. `null` when the formats share no dataset. + * Sum of this format's sizes, in bytes, over a set of datasets that is the same for every + * format, so the totals compare directly. See [`SizeTotals`]. `null` when the format has no + * total. */ totalBytes: number | null; + /** Datasets this format has a size for, counting a family once ([`sizeBucket`]). */ + measured: number; + /** Datasets in the summary, counted as `measured` is: the denominator it is out of. */ + total: number; } /** @@ -201,15 +212,13 @@ export function collectGroupSummary(key: GroupKey): Promise { case 'CompressionTimeGroup': return key.suite === 'strings' ? collectStringEncodingSummary() : collectCompressionSummary(); case 'CompressionSizeGroup': - return key.suite === 'strings' - ? collectStringEncodingSizeSummary() - : collectCompressionSizeSummary(); + return collectStringEncodingSizeSummary(); case 'RandomAccessGroup': return collectRandomAccessSummary(key.storage); case 'VectorSearchGroup': return collectVectorSearchSummary(key.dataset, key.layout); case 'FileSizeGroup': - return collectFileSizeSummary(); + return collectFileSizesSummary(); } } @@ -711,20 +720,39 @@ async function compressionSamples(): Promise { return (await getPool().query(text, compressionSummaryQueryParams())).rows; } -async function collectCompressionSizeSummary(): Promise { - const rankings = compressionSizeRankings(await compressionSizeSamples()); +/** + * The `File Sizes` rollup: each format's sizes against Parquet's (zstd) over the + * `compress-bench` datasets ([`compressionSizeSamples`]) and the SQL scopes in + * [`FILE_SIZE_SUMMARY_SCOPES`] ([`fileSizeSamples`]). Each synthetic family in + * [`SIZE_SUMMARY_FAMILIES`] counts as one dataset. Only `compress-bench` records Arrow memory + * sizes, so a compression ratio covers its datasets alone. The suites write different formats, so + * each ranking says how many datasets it covers. + */ +async function collectFileSizesSummary(): Promise { + const [sizes, files] = await Promise.all([compressionSizeSamples(), fileSizeSamples()]); + const rankings = compressionSizeRankings([...sizes, ...files], 'complete'); if (rankings.length === 0) { return null; } return { type: 'compressionSize', - title: 'Compression Size Summary', + title: 'File Sizes Summary', rankings, explanation: - 'Total compressed size, and geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', + 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', }; } +/** + * Which datasets a size summary's totals cover: + * - `shared`: the datasets that every ranked format has. Every format gets a total. + * - `complete`: every dataset in the summary. Only a format measured on all of them gets a + * total. The File Sizes summary uses it: its SQL scopes record DuckDB and Vortex Compact, + * which `compress-bench` does not write, and `compress-bench` records Arrow IPC, which the SQL + * suites do not, so no dataset is shared by every format. + */ +type SizeTotals = 'shared' | 'complete'; + /** One size paired with the baseline's size for the same dataset and commit. */ interface CompressionSizeSample { format: string; @@ -736,28 +764,62 @@ interface CompressionSizeSample { } /** - * Rank each format by its size ratios to the baseline and its compression ratio to Arrow. The - * order is by compression ratio once every format has one, and by baseline ratio until then. + * The summary entry a dataset counts toward: its family's name for a dataset in a + * [`SIZE_SUMMARY_FAMILIES`] family, and the dataset itself otherwise. + */ +function sizeBucket(dataset: string, datasetVariant: string | null): string { + const family = SIZE_SUMMARY_FAMILIES.find((candidate) => dataset.startsWith(candidate.prefix)); + return family === undefined + ? JSON.stringify([dataset, datasetVariant]) + : JSON.stringify([family.name]); +} + +/** Geometric mean of each bucket's geometric mean: one entry per bucket. */ +function bucketGeoMean(byBucket: ReadonlyMap): number | null { + return geoMean([...byBucket.values()].flatMap((values) => geoMean(values) ?? [])); +} + +/** + * Rank each format by its size ratios to the baseline and its compression ratio to Arrow. + * + * Each dataset is one entry in the geometric means, except that a family counts once + * ([`sizeBucket`]). `minRatio` and `maxRatio` are over single datasets. `measured` and `total` + * count buckets. A format measured on every bucket ranks above one that is not, because their + * means are over different datasets. Within each tier, the order is by compression ratio once + * every format in the summary has one, and by baseline ratio until then. */ -function compressionSizeRankings(rows: readonly CompressionSizeSample[]): CompressionSizeRanking[] { +function compressionSizeRankings( + rows: readonly CompressionSizeSample[], + totals: SizeTotals = 'shared', +): CompressionSizeRanking[] { const grouped = new Map< string, - { sizeRatios: number[]; compressionRatios: number[]; bytesByDataset: Map } + { + sizeRatios: Map; + compressionRatios: Map; + bytesByDataset: Map; + } >(); + const buckets = new Set(); for (const row of rows) { const ratios = grouped.get(row.format) ?? { - sizeRatios: [], - compressionRatios: [], + sizeRatios: new Map(), + compressionRatios: new Map(), bytesByDataset: new Map(), }; + const bucket = sizeBucket(row.dataset, row.datasetVariant); if ( row.valueBytes > 0 && Number.isFinite(row.valueBytes) && row.baselineBytes > 0 && Number.isFinite(row.baselineBytes) ) { - ratios.sizeRatios.push(row.valueBytes / row.baselineBytes); + ratios.sizeRatios.set(bucket, [ + ...(ratios.sizeRatios.get(bucket) ?? []), + row.valueBytes / row.baselineBytes, + ]); ratios.bytesByDataset.set(JSON.stringify([row.dataset, row.datasetVariant]), row.valueBytes); + buckets.add(bucket); } if ( row.uncompressedBytes !== null && @@ -766,23 +828,29 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre row.valueBytes > 0 && Number.isFinite(row.valueBytes) ) { - ratios.compressionRatios.push(row.uncompressedBytes / row.valueBytes); + ratios.compressionRatios.set(bucket, [ + ...(ratios.compressionRatios.get(bucket) ?? []), + row.uncompressedBytes / row.valueBytes, + ]); } grouped.set(row.format, ratios); } // Lance and Arrow IPC come from their own snapshots, so a format can lack a dataset that the - // others have. Totals use only the datasets every ranked format has. - const ranked = [...grouped.values()].filter((ratios) => ratios.sizeRatios.length > 0); - const commonDatasets = [...(ranked[0]?.bytesByDataset.keys() ?? [])].filter((dataset) => - ranked.every((ratios) => ratios.bytesByDataset.has(dataset)), - ); + // others have. Totals cover one set of datasets for every format (`SizeTotals`). + const ranked = [...grouped.values()].filter((ratios) => ratios.sizeRatios.size > 0); + const commonDatasets = + totals === 'shared' + ? [...(ranked[0]?.bytesByDataset.keys() ?? [])].filter((dataset) => + ranked.every((ratios) => ratios.bytesByDataset.has(dataset)), + ) + : [...new Set(ranked.flatMap((ratios) => [...ratios.bytesByDataset.keys()]))]; const rankings: CompressionSizeRanking[] = []; for (const [name, ratios] of grouped) { - const ratio = geoMean(ratios.sizeRatios); + const ratio = bucketGeoMean(ratios.sizeRatios); if (ratio !== null) { let minRatio = Infinity; let maxRatio = -Infinity; - for (const sizeRatio of ratios.sizeRatios) { + for (const sizeRatio of [...ratios.sizeRatios.values()].flat()) { minRatio = Math.min(minRatio, sizeRatio); maxRatio = Math.max(maxRatio, sizeRatio); } @@ -791,19 +859,27 @@ function compressionSizeRankings(rows: readonly CompressionSizeSample[]): Compre minRatio, ratio, maxRatio, - compressionRatio: geoMean(ratios.compressionRatios), + compressionRatio: bucketGeoMean(ratios.compressionRatios), totalBytes: - commonDatasets.length === 0 + commonDatasets.length === 0 || + !commonDatasets.every((dataset) => ratios.bytesByDataset.has(dataset)) ? null : commonDatasets.reduce( (sum, dataset) => sum + (ratios.bytesByDataset.get(dataset) ?? 0), 0, ), + measured: ratios.sizeRatios.size, + total: buckets.size, }); } } const hasCompleteArrowRatios = rankings.every((ranking) => ranking.compressionRatio !== null); + const partial = (ranking: CompressionSizeRanking) => (ranking.measured < ranking.total ? 1 : 0); rankings.sort((a, b) => { + const tier = partial(a) - partial(b); + if (tier !== 0) { + return tier; + } if (!hasCompleteArrowRatios) { return a.ratio - b.ratio || compareCodeUnits(a.name, b.name); } @@ -1023,64 +1099,73 @@ async function collectStringEncodingSizeSummary(): Promise { }; } -/** The format every other SQL benchmark file format is compared with. */ -const FILE_SIZE_BASELINE = 'vortex-file-compressed'; - /** - * The `SQL File Size` rollup: each Vortex format's total file size against - * `vortex-file-compressed`, per benchmark scope (benchmark and scale factor) at that scope's - * newest commit with a baseline. Scopes come from nightly as well as develop runs, so each one - * uses its own newest commit. File sizes record no Arrow size, so the Arrow ratio is empty. + * Size samples for the SQL scopes in [`FILE_SIZE_SUMMARY_SCOPES`]: each format's total over a + * scope's files, with Parquet's total from the same commit. Each format uses its own newest + * commit that also has Parquet, as Lance does in the `compress-bench` samples, so a format that a + * suite records less often than every run still counts. Scopes and formats come from a skip scan + * of `idx_file_sizes_baseline`. `latest` walks the read branch's commits newest first and probes + * that index, so it stops at the newest commit with both formats. */ -async function collectFileSizeSummary(): Promise { - // `scopes` is a skip scan of `idx_file_sizes_baseline`. `latest` walks the read branch's - // commits newest first and probes that index, so it stops at the newest commit with a baseline. +async function fileSizeSamples(): Promise { const text = ` - WITH RECURSIVE scopes AS ( - (SELECT f.benchmark, f.scale_factor + WITH RECURSIVE scope_formats AS ( + (SELECT f.benchmark, f.scale_factor, f.format FROM file_sizes f - ORDER BY f.benchmark, f.scale_factor + ORDER BY f.benchmark, f.scale_factor, f.format LIMIT 1) UNION ALL - SELECT nxt.benchmark, nxt.scale_factor - FROM scopes s + SELECT nxt.benchmark, nxt.scale_factor, nxt.format + FROM scope_formats p CROSS JOIN LATERAL ( - SELECT f.benchmark, f.scale_factor + SELECT f.benchmark, f.scale_factor, f.format FROM file_sizes f - WHERE (f.benchmark, f.scale_factor) > (s.benchmark, s.scale_factor) - ORDER BY f.benchmark, f.scale_factor + WHERE (f.benchmark, f.scale_factor, f.format) > (p.benchmark, p.scale_factor, p.format) + ORDER BY f.benchmark, f.scale_factor, f.format LIMIT 1 ) nxt ), latest AS ( - SELECT s.benchmark, s.scale_factor, l.commit_sha - FROM scopes s + SELECT t.benchmark, t.scale_factor, t.format, l.commit_sha + FROM scope_formats t + JOIN unnest($1::text[], $2::text[]) AS s(benchmark, scale_factor) + ON s.benchmark = t.benchmark + AND s.scale_factor = t.scale_factor CROSS JOIN LATERAL ( SELECT c.commit_sha FROM ${BRANCH_COMMITS} c WHERE EXISTS ( SELECT 1 FROM file_sizes f - WHERE f.benchmark = s.benchmark - AND f.scale_factor = s.scale_factor - AND f.format = $1 + WHERE f.benchmark = t.benchmark + AND f.scale_factor = t.scale_factor + AND f.format = t.format + AND f.commit_sha = c.commit_sha) + AND EXISTS ( + SELECT 1 + FROM file_sizes f + WHERE f.benchmark = t.benchmark + AND f.scale_factor = t.scale_factor + AND f.format = $3 AND f.commit_sha = c.commit_sha) ORDER BY c.timestamp DESC, c.commit_sha DESC LIMIT 1 ) l ), totals AS ( - SELECT f.benchmark, f.scale_factor, f.format, SUM(f.value_bytes)::float8 AS bytes - FROM file_sizes f - JOIN latest l - ON l.benchmark = f.benchmark - AND l.scale_factor = f.scale_factor - AND l.commit_sha = f.commit_sha - GROUP BY f.benchmark, f.scale_factor, f.format + SELECT l.benchmark, l.scale_factor, l.format, f.format AS measured, + SUM(f.value_bytes)::float8 AS bytes + FROM latest l + JOIN file_sizes f + ON f.benchmark = l.benchmark + AND f.scale_factor = l.scale_factor + AND f.commit_sha = l.commit_sha + AND f.format IN (l.format, $3) + GROUP BY l.benchmark, l.scale_factor, l.format, f.format ) SELECT t.format AS format, t.benchmark AS dataset, - t.scale_factor AS "datasetVariant", + 'sf=' || t.scale_factor AS "datasetVariant", t.bytes AS "valueBytes", b.bytes AS "baselineBytes", NULL::float8 AS "uncompressedBytes" @@ -1088,25 +1173,20 @@ async function collectFileSizeSummary(): Promise { JOIN totals b ON b.benchmark = t.benchmark AND b.scale_factor = t.scale_factor - AND b.format = $1 - WHERE t.bytes > 0 + AND b.format = t.format + AND b.measured = $3 + WHERE t.measured = t.format + AND t.bytes > 0 AND b.bytes > 0 ORDER BY t.format, t.benchmark, t.scale_factor `; - const rows = (await getPool().query(text, [FILE_SIZE_BASELINE])).rows; - const rankings = compressionSizeRankings(rows); - if (rankings.length === 0) { - return null; - } - return { - type: 'compressionSize', - title: 'SQL File Size Summary', - rankings, - explanation: - 'Total size of the Vortex files each SQL benchmark writes, and geometric means of sizes versus vortex (lower is better)', - // The card names formats by their display names, which call `vortex-file-compressed` `vortex`. - baseline: 'vortex', - }; + return ( + await getPool().query(text, [ + FILE_SIZE_SUMMARY_SCOPES.map((scope) => scope.benchmark), + FILE_SIZE_SUMMARY_SCOPES.map((scope) => scope.scale_factor), + COMPRESSION_BASELINE, + ]) + ).rows; } async function collectQuerySummary( diff --git a/src/lib/bench-summary.ts b/src/lib/bench-summary.ts index 8889aac8..480842a3 100644 --- a/src/lib/bench-summary.ts +++ b/src/lib/bench-summary.ts @@ -99,6 +99,11 @@ type Source = { kind: Panel["kind"]; better: Panel["better"]; group: string; + /** + * The group's earlier names. The page reads production bench while it + * builds, so a deploy that renames a group still reads the old name. + */ + formerGroups?: readonly string[]; /** The rankings that list the systems, from the group's summary. */ rankings: (summary: Ranking) => unknown; /** The ranking's name for a system's format. */ @@ -107,6 +112,14 @@ type Source = { where?: Ranking; /** The ranking's field that holds the score. */ score: "score" | "ratio"; + /** + * Show a system's score even when bench measured it on only part of the + * workload. The file size summary covers the SQL suites' files, which + * have no Lance or Arrow IPC copies, so those two are partial there. A + * size ratio is to Parquet's size on the same dataset, so it stays + * comparable over the datasets the system has. + */ + partialScores?: true; /** The systems the panel shows. */ systems: readonly string[]; /** @@ -203,14 +216,22 @@ const SOURCES: (Source | Panel)[] = [ unit: "×", kind: "size", better: "lower", - group: "Compression Size", + group: "File Sizes", + formerGroups: ["Compression Size"], rankings: (s) => s.rankings, name: (f) => f, score: "ratio", + partialScores: true, systems: WITH_ARROW } ]; +// A source's group in bench's `/api/groups`, by its name or an earlier one. +const findGroup = (groups: unknown[], source: Source) => + [source.group, ...(source.formerGroups ?? [])] + .map((name) => groups.find((g) => isObject(g) && g.name === name)) + .find((g) => g !== undefined); + const isObject = (x: unknown): x is Ranking => typeof x === "object" && x !== null && !Array.isArray(x); const positive = (x: unknown): x is number => @@ -224,8 +245,9 @@ const measured = (r: Row) => r.score !== null && !(r.measured !== undefined && r.total !== undefined && r.measured < r.total); -// Bench's coverage passes through: results.js shows a system measured -// on only part of the workload as "Not measured". +// Bench's coverage passes through, unless the source shows partial scores: +// results.js shows a system measured on only part of the workload as +// "Not measured". function rowFor(source: Source, rankings: Ranking[], format: string) { const name = source.name(format); const where = Object.entries(source.where ?? {}); @@ -234,6 +256,7 @@ function rowFor(source: Source, rankings: Ranking[], format: string) { ); if (!r) return { score: null }; const score = r[source.score]; + if (source.partialScores) return { score: positive(score) ? score : null }; const measured = count(r.measured); const total = count(r.total); return { @@ -340,7 +363,7 @@ function panelFor( groups: unknown[], charts: Record ): Panel | null { - const group = groups.find((g) => isObject(g) && g.name === source.group); + const group = findGroup(groups, source); const summary = isObject(group) ? group.summary : undefined; const list = isObject(summary) ? source.rankings(summary) : undefined; if (!Array.isArray(list)) return null; @@ -473,7 +496,7 @@ export async function getBenchSummary( const slugs: [string, string][] = []; for (const source of SOURCES) { if (!("times" in source) || !source.times) continue; - const group = groups.find((g) => isObject(g) && g.name === source.group); + const group = findGroup(groups, source); const slug = isObject(group) ? group.slug : undefined; if (typeof slug !== "string" || !/^[\w.-]+$/.test(slug)) return fail(`group ${source.group} has no usable slug`); diff --git a/tests/unit/bench-summary.test.ts b/tests/unit/bench-summary.test.ts index 33a45681..fb4c0a20 100644 --- a/tests/unit/bench-summary.test.ts +++ b/tests/unit/bench-summary.test.ts @@ -342,7 +342,7 @@ describe("panelsFrom", () => { it("drops a score that is not a positive number", () => { const gs = groups(); - const parquet = group(gs, "Compression Size").summary.rankings?.find( + const parquet = group(gs, "File Sizes").summary.rankings?.find( (r) => r.name === "parquet" ); assert.ok(parquet); @@ -353,10 +353,36 @@ describe("panelsFrom", () => { }); }); + it("keeps a file size score bench measured on part of the datasets", () => { + // The SQL suites write no Lance or Arrow IPC files, so bench's file size + // summary has those two on part of its datasets. A size ratio is to + // Parquet on the same dataset, so the card keeps it. + const gs = groups(); + const lance = group(gs, "File Sizes").summary.rankings?.find( + (r) => r.name === "lance" + ); + assert.ok(lance); + lance.measured = 12; + lance.total = 13; + assert.deepEqual(panelsFrom(gs, charts())?.[6]?.rows[2], { + system: "Lance", + score: 1.9876533869960349 + }); + }); + + it("reads the file size panel from the group's earlier name", () => { + // A deploy that renames a bench group builds against production bench, + // which still serves the old name. + const gs = groups(); + const sizes = group(gs, "File Sizes"); + sizes.name = "Compression Size"; + assert.deepEqual(panelsFrom(gs, charts()), panelsFrom(groups(), charts())); + }); + it("gives null when a group is missing, so the card never shows part of the table", () => { for (const name of [ "Compression", - "Compression Size", + "File Sizes", "Random Access (NVMe)", "Clickbench", "TPC-H (NVMe) (SF=10)" diff --git a/tests/unit/fixtures/bench-groups.json b/tests/unit/fixtures/bench-groups.json index 6b4a1582..64386d41 100644 --- a/tests/unit/fixtures/bench-groups.json +++ b/tests/unit/fixtures/bench-groups.json @@ -48,11 +48,11 @@ } }, { - "name": "Compression Size", - "slug": "csg.eyJrIjoiQ29tcHJlc3Npb25TaXplR3JvdXAifQ", + "name": "File Sizes", + "slug": "fsg.eyJrIjoiRmlsZVNpemVHcm91cCJ9", "summary": { "type": "compressionSize", - "title": "Compression Size Summary", + "title": "File Sizes Summary", "rankings": [ { "name": "parquet", From 17784d1d85347cd6b0fca5e73828a307cfd86e97 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 20 Nov 2026 17:01:33 -0500 Subject: [PATCH 02/23] Keep the SQL File Sizes group beside File Sizes on bench File Sizes moves to the Compression Size group key, so old Compression Size links open it. SQL File Sizes comes back on its own key: it charts every SQL benchmark and scale factor and ranks formats against Vortex, as before. Each group's info tooltip says how the two differ. Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 4 + docs/bench/architecture/read-path.md | 2 +- src/bench/lib/benchmark-docs.test.ts | 4 +- src/bench/lib/benchmark-docs.ts | 4 +- src/bench/lib/descriptions.test.ts | 9 +- src/bench/lib/descriptions.ts | 10 +- src/bench/lib/file-sizes.ts | 6 + src/bench/lib/group-charts.test.ts | 10 +- src/bench/lib/groups.test.ts | 42 +++++- src/bench/lib/queries.ts | 200 ++++++++++++++++++-------- src/bench/lib/slug.test.ts | 8 +- src/bench/lib/slug.ts | 23 +-- src/bench/lib/summary.test.ts | 14 +- src/bench/lib/summary.ts | 93 +++++++++++- tests/unit/fixtures/bench-groups.json | 2 +- 15 files changed, 318 insertions(+), 113 deletions(-) diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index 17e7ab07..b3528433 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -88,6 +88,10 @@ current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. +The `SQL File Sizes` group charts every `file_sizes` scope, at every scale factor, and leaves out +`compression_sizes`. Its summary compares each format with `vortex-file-compressed`, per scope at +that scope's newest commit with a Vortex size. + `random_access_time` records include `open_mode`. The value is `cached` for a reused accessor or `reopen` when each timed take includes a new accessor. Historical rows use `cached`. The `reopen` mode does not clear the OS page cache. diff --git a/docs/bench/architecture/read-path.md b/docs/bench/architecture/read-path.md index fb9dae4b..65895507 100644 --- a/docs/bench/architecture/read-path.md +++ b/docs/bench/architecture/read-path.md @@ -120,7 +120,7 @@ the v3 Axum server (so the frontend is generation-agnostic). The landing-page order is the curated `GROUP_ORDER` in `queries.ts`: ``` -Compression, File Sizes, Random Access, Clickbench, +Compression, File Sizes, SQL File Sizes, Random Access, Clickbench, TPC-H (NVMe/S3) SF=1, SF=10, SF=100 (alternating), TPC-DS (NVMe) SF=1, Statistical and Population Genetics, Polar Signals Profiling, diff --git a/src/bench/lib/benchmark-docs.test.ts b/src/bench/lib/benchmark-docs.test.ts index 9d94e5ce..e94a4009 100644 --- a/src/bench/lib/benchmark-docs.test.ts +++ b/src/bench/lib/benchmark-docs.test.ts @@ -40,8 +40,8 @@ describe('groupDocPath', () => { it('maps the three non-query families to their benchmark crate', () => { expect(groupDocPath({ k: 'CompressionTimeGroup' })).toBe('benchmarks/compress-bench/README.md'); - // File Sizes is mostly compress-bench's datasets, so it shares compress-bench's doc. - expect(groupDocPath({ k: 'FileSizeGroup' })).toBe('benchmarks/compress-bench/README.md'); + // Two measurement kinds of one benchmark, so one doc. + expect(groupDocPath({ k: 'CompressionSizeGroup' })).toBe('benchmarks/compress-bench/README.md'); expect(groupDocPath({ k: 'CompressionTimeGroup', suite: 'strings' })).toBe( 'benchmarks/string-bench/README.md', ); diff --git a/src/bench/lib/benchmark-docs.ts b/src/bench/lib/benchmark-docs.ts index 068c61c7..8fd75d70 100644 --- a/src/bench/lib/benchmark-docs.ts +++ b/src/bench/lib/benchmark-docs.ts @@ -83,14 +83,12 @@ export function groupDocPath(key: GroupKey): string | null { case 'CompressionTimeGroup': case 'CompressionSizeGroup': return key.suite === 'strings' ? STRING_BENCH_DOC : COMPRESS_BENCH_DOC; - // Most of File Sizes' charts are `compress-bench` datasets, and its doc covers them. - case 'FileSizeGroup': - return COMPRESS_BENCH_DOC; case 'RandomAccessGroup': return RANDOM_ACCESS_BENCH_DOC; case 'QueryGroup': return QUERY_DOCS[key.dataset] ?? null; case 'VectorSearchGroup': + case 'FileSizeGroup': return null; } } diff --git a/src/bench/lib/descriptions.test.ts b/src/bench/lib/descriptions.test.ts index 48760a8b..06d66a5f 100644 --- a/src/bench/lib/descriptions.test.ts +++ b/src/bench/lib/descriptions.test.ts @@ -21,8 +21,13 @@ describe('groupDescription', () => { 'Measures write and scan throughput for Vortex, Parquet, and Lance files', ); expect(groupDescription('File Sizes')).toBe( - 'Compares file sizes and compression ratios across formats, on the compression datasets ' + - 'and on the files the ClickBench, TPC-H, and TPC-DS benchmarks write', + 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + + 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS', + ); + expect(groupDescription('SQL File Sizes')).toBe( + 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + + 'Vortex. Unlike File Sizes, it leaves out the compression datasets and does not rank ' + + 'formats against Parquet', ); expect(groupDescription('Clickbench')).toBe( "ClickHouse's analytical benchmark suite testing real-world query patterns on web " + diff --git a/src/bench/lib/descriptions.ts b/src/bench/lib/descriptions.ts index bf0c6d39..5fa09581 100644 --- a/src/bench/lib/descriptions.ts +++ b/src/bench/lib/descriptions.ts @@ -48,8 +48,14 @@ function staticDescription(name: string): string | null { return 'Measures write and scan throughput for Vortex, Parquet, and Lance files'; case 'File Sizes': return ( - 'Compares file sizes and compression ratios across formats, on the compression ' + - 'datasets and on the files the ClickBench, TPC-H, and TPC-DS benchmarks write' + 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + + 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS' + ); + case 'SQL File Sizes': + return ( + 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + + 'Vortex. Unlike File Sizes, it leaves out the compression datasets and does not rank ' + + 'formats against Parquet' ); case 'String Encoding': return 'Measures Vortex file write and scan throughput for each string encoder'; diff --git a/src/bench/lib/file-sizes.ts b/src/bench/lib/file-sizes.ts index e4340f22..302f3469 100644 --- a/src/bench/lib/file-sizes.ts +++ b/src/bench/lib/file-sizes.ts @@ -12,6 +12,12 @@ import { compressionSuite } from './compression-suite'; /** The File Sizes group's display name. */ export const FILE_SIZES_GROUP_NAME = 'File Sizes'; +/** + * The SQL File Sizes group's display name. That group charts every SQL benchmark and scale factor + * in `file_sizes`, and its summary compares the formats with Vortex. + */ +export const SQL_FILE_SIZES_GROUP_NAME = 'SQL File Sizes'; + /** One SQL suite's data at one scale factor, as `file_sizes` spells them. */ export interface FileSizeScope { benchmark: string; diff --git a/src/bench/lib/group-charts.test.ts b/src/bench/lib/group-charts.test.ts index 11a76600..c8a6ead7 100644 --- a/src/bench/lib/group-charts.test.ts +++ b/src/bench/lib/group-charts.test.ts @@ -586,7 +586,7 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () [ // The File Sizes group mixes the two tables: `compression_sizes` charts, a NULL variant // between two non-NULL ones, and `file_sizes` charts. - { k: 'FileSizeGroup' }, + { k: 'CompressionSizeGroup' }, [ { k: 'FileSize', benchmark: 'tpcds', scale_factor: '1.0' }, { k: 'CompressionSize', dataset: 'sx', dataset_variant: 'v1' }, @@ -595,6 +595,14 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () { k: 'CompressionSize', dataset: 'sy', dataset_variant: 'v2' }, ], ], + [ + // The SQL File Sizes group holds `file_sizes` charts alone, at every scale factor. + { k: 'FileSizeGroup' }, + [ + { k: 'FileSize', benchmark: 'tpch', scale_factor: '10.0' }, + { k: 'FileSize', benchmark: 'tpcds', scale_factor: '1.0' }, + ], + ], ]; it.each(['1', '2', '3', '4', 'all'])( diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index c6e0a9cf..ce996dd4 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -660,9 +660,9 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg } const groups = await collectGroups(); - expect(groups.map((group) => group.name)).toEqual(['File Sizes']); - const [group] = groups; - expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'FileSizeGroup' }); + expect(groups.map((group) => group.name)).toEqual(['File Sizes', 'SQL File Sizes']); + const [group, sql] = groups; + expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'CompressionSizeGroup' }); expect(defined(group).charts.map((chart) => chart.name)).toEqual([ 'clickbench', 'tpch sf=10.0', @@ -687,6 +687,36 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg ], }); expect(summary).not.toHaveProperty('baseline'); + + // The SQL File Sizes group charts every scope, and ranks each format against Vortex at each + // scope's newest commit with a Vortex size. TPC-DS's newest commit with Vortex is the + // develop one, so the nightly Snappy file is not in its totals. + expect(groupKeyFromSlug(defined(sql).slug)).toEqual({ k: 'FileSizeGroup' }); + expect(defined(sql).charts.map((chart) => chart.name)).toEqual([ + 'clickbench', + 'tpch sf=1.0', + 'tpch sf=10.0', + 'tpch sf=100', + 'tpcds sf=1.0', + 'appian', + ]); + expect(defined(sql).summary).toMatchObject({ + type: 'compressionSize', + title: 'SQL File Sizes Summary', + baseline: 'vortex', + }); + const sqlSummary = defined(defined(sql).summary); + if (sqlSummary.type !== 'compressionSize') { + throw new Error(`expected a compressionSize summary, got ${sqlSummary.type}`); + } + expect( + Object.fromEntries(sqlSummary.rankings.map((ranking) => [ranking.name, ranking.ratio])), + ).toEqual({ + 'vortex-file-compressed': 1, + 'vortex-compact': 0.5, + // TPC-H SF10's Parquet is half its Vortex size, and TPC-DS's is the same size. + parquet: expect.closeTo(Math.sqrt(0.5 * 1), 12), + }); }); it('ranks random access over every chart, not the alphabetically first one', async () => { @@ -854,7 +884,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(timeByKey.get('decode:parquet')?.ratio).toBeCloseTo(1.0, 6); const size = expectDefined( - await collectGroupSummary({ k: 'FileSizeGroup' }), + await collectGroupSummary({ k: 'CompressionSizeGroup' }), 'compression-size summary', ); if (size.type !== 'compressionSize') { @@ -903,7 +933,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(vortexTime.ratio).toBeCloseTo(3.0, 6); const size = expectDefined( - await collectGroupSummary({ k: 'FileSizeGroup' }), + await collectGroupSummary({ k: 'CompressionSizeGroup' }), 'file size summary', ); if (size.type !== 'compressionSize') { @@ -949,7 +979,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(vortexTime.ratio).toBeCloseTo(4.0, 6); const size = expectDefined( - await collectGroupSummary({ k: 'FileSizeGroup' }), + await collectGroupSummary({ k: 'CompressionSizeGroup' }), 'compression-size summary', ); if (size.type !== 'compressionSize') { diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index 20279506..d00c53cc 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -47,6 +47,7 @@ import { FILE_SIZES_GROUP_NAME, isFileSizeScope, isFileSizesDataset, + SQL_FILE_SIZES_GROUP_NAME, } from './file-sizes'; import type { FilterUniverse } from './chart-format'; import { type ChartKind, compareCodeUnits, FAMILIES, type GroupKind } from './families'; @@ -788,7 +789,13 @@ function chartInGroup(group: GroupKey, key: ChartKey): boolean { key.k === 'CompressionTime' && compressionSuite(key.dataset) === (group.suite ?? 'compress') ); case 'CompressionSizeGroup': - return key.k === 'CompressionSize' && compressionSuite(key.dataset) === group.suite; + if (group.suite === 'strings') { + return key.k === 'CompressionSize' && compressionSuite(key.dataset) === 'strings'; + } + return ( + (key.k === 'FileSize' && isFileSizeScope(key.benchmark, key.scale_factor)) || + (key.k === 'CompressionSize' && isFileSizesDataset(key.dataset, key.dataset_variant)) + ); case 'RandomAccessGroup': return key.k === 'RandomAccess' && randomAccessStorage(key.dataset) === group.storage; case 'VectorSearchGroup': @@ -796,10 +803,7 @@ function chartInGroup(group: GroupKey, key: ChartKey): boolean { key.k === 'VectorSearch' && key.dataset === group.dataset && key.layout === group.layout ); case 'FileSizeGroup': - return ( - (key.k === 'FileSize' && isFileSizeScope(key.benchmark, key.scale_factor)) || - (key.k === 'CompressionSize' && isFileSizesDataset(key.dataset, key.dataset_variant)) - ); + return key.k === 'FileSize'; } } @@ -886,6 +890,7 @@ type ChartColumn = readonly [name: string, type: string, values: readonly unknow /** The key of a chart from either compression table. */ type CompressionChartKey = Extract; +type FileSizeChartKey = Extract; /** * One row per (chart, fact row in the window), plus one row with NULL fact @@ -986,6 +991,32 @@ function groupRowsQuery( ); } }; + /** One block for `file_sizes` charts: each chart's size per format, summed over its files. */ + const fileSizeBlocks = (files: readonly [FileSizeChartKey, number][]): void => { + if (files.length === 0) { + return; + } + const pins = (alias: string): string => + `${alias}.benchmark = ch.benchmark AND ${alias}.scale_factor = ch.scale_factor`; + block( + ['format'], + files.map(([, position]) => position), + [ + ['benchmark', 'text', files.map(([k]) => k.benchmark)], + ['scale_factor', 'text', files.map(([k]) => k.scale_factor)], + ], + `SELECT MIN(c2.timestamp) + FROM file_sizes f2 + JOIN ${BRANCH_COMMITS} c2 ON c2.commit_sha = f2.commit_sha + WHERE ${pins('f2')}`, + `SELECT f.commit_sha, + f.format, SUM(f.value_bytes)::float8 AS value + FROM file_sizes f + JOIN ${BRANCH_COMMITS} c USING (commit_sha) + WHERE ${pins('f')}${factWindowFilter(params, window)} + GROUP BY f.commit_sha, f.format`, + ); + }; switch (group.k) { case 'QueryGroup': { const keys = chartsOf(group, charts, 'QueryMeasurement'); @@ -1018,11 +1049,22 @@ function groupRowsQuery( break; } case 'CompressionSizeGroup': { - const keys = chartsOf(group, charts, 'CompressionSize'); - compressionBlocks( - 'CompressionSize', - keys.map((key, i) => [key, i + 1]), - ); + // The File Sizes group holds two families. Both give one size per format, so their blocks + // share columns. The String Encoding Size group holds `compression_sizes` charts alone. + const sizes: [CompressionChartKey, number][] = []; + const files: [FileSizeChartKey, number][] = []; + charts.forEach((key, i) => { + if (!chartInGroup(group, key)) { + throw new Error(`chart ${chartKeyToSlug(key)} is not in group ${groupKeyToSlug(group)}`); + } + if (key.k === 'CompressionSize') { + sizes.push([key, i + 1]); + } else if (key.k === 'FileSize') { + files.push([key, i + 1]); + } + }); + compressionBlocks('CompressionSize', sizes); + fileSizeBlocks(files); break; } case 'RandomAccessGroup': { @@ -1070,44 +1112,7 @@ function groupRowsQuery( break; } case 'FileSizeGroup': { - // The group holds two families. Both give one size per format, so their blocks share - // columns. - const sizes: [CompressionChartKey, number][] = []; - const files: [Extract, number][] = []; - charts.forEach((key, i) => { - if (!chartInGroup(group, key)) { - throw new Error(`chart ${chartKeyToSlug(key)} is not in group ${groupKeyToSlug(group)}`); - } - if (key.k === 'CompressionSize') { - sizes.push([key, i + 1]); - } else if (key.k === 'FileSize') { - files.push([key, i + 1]); - } - }); - compressionBlocks('CompressionSize', sizes); - if (files.length === 0) { - break; - } - const pins = (alias: string): string => - `${alias}.benchmark = ch.benchmark AND ${alias}.scale_factor = ch.scale_factor`; - block( - ['format'], - files.map(([, position]) => position), - [ - ['benchmark', 'text', files.map(([k]) => k.benchmark)], - ['scale_factor', 'text', files.map(([k]) => k.scale_factor)], - ], - `SELECT MIN(c2.timestamp) - FROM file_sizes f2 - JOIN ${BRANCH_COMMITS} c2 ON c2.commit_sha = f2.commit_sha - WHERE ${pins('f2')}`, - `SELECT f.commit_sha, - f.format, SUM(f.value_bytes)::float8 AS value - FROM file_sizes f - JOIN ${BRANCH_COMMITS} c USING (commit_sha) - WHERE ${pins('f')}${factWindowFilter(params, window)} - GROUP BY f.commit_sha, f.format`, - ); + fileSizeBlocks(chartsOf(group, charts, 'FileSize').map((key, i) => [key, i + 1])); break; } } @@ -1288,6 +1293,7 @@ export interface GroupChartsResponse { const GROUP_ORDER: readonly string[] = [ 'Compression', 'File Sizes', + 'SQL File Sizes', 'Random Access (NVMe)', 'Random Access (S3)', 'Clickbench', @@ -1570,8 +1576,9 @@ async function collectQueryGroups(): Promise { type DatasetVariantRow = { dataset: string; dataset_variant: string | null }; /** - * Group display names per compression suite and family. `compress-bench`'s sizes are in the File - * Sizes group, so the size family names only `string-bench`'s group. + * Group display names per compression suite and family. The size family's group without a suite + * is the File Sizes group, which [`collectFileSizesGroup`] builds, so the size family names only + * `string-bench`'s group here. */ const COMPRESSION_GROUP_NAMES = { CompressionTimeGroup: { compress: 'Compression', strings: 'String Encoding' }, @@ -1649,9 +1656,16 @@ async function compressionSizeDatasets(): Promise { return (await getPool().query(text)).rows; } -/** The `String Encoding Size` group, or `[]` if `string-bench` has no sizes. */ +/** + * The File Sizes group and the `String Encoding Size` group, each left out when it has no + * charts. + */ async function collectCompressionSizeGroup(): Promise { - return compressionGroups(await compressionSizeDatasets(), 'CompressionSizeGroup'); + const datasets = await compressionSizeDatasets(); + return [ + ...(await collectFileSizesGroup(datasets)), + ...compressionGroups(datasets, 'CompressionSizeGroup'), + ]; } /** One random-access group per storage backend, omitting empty sections. */ @@ -1731,10 +1745,10 @@ type FileSizeScopeRow = { benchmark: string; scale_factor: string }; /** * The File Sizes group, or `[]` if neither table has a chart for it: the SQL suites' scopes in - * [`FILE_SIZE_SCOPES`] that have rows, in that order, then every `compress-bench` dataset in - * `compression_sizes` by name. + * [`FILE_SIZE_SCOPES`] that have rows, in that order, then the `compress-bench` datasets of + * `datasets`, the rows of `compression_sizes`, by name. */ -async function collectFileSizeGroup(): Promise { +async function collectFileSizesGroup(datasets: readonly DatasetVariantRow[]): Promise { // One index probe of `idx_file_sizes_baseline` per listed scope. const scopesText = ` SELECT s.benchmark, s.scale_factor @@ -1746,12 +1760,9 @@ async function collectFileSizeGroup(): Promise { AND f.scale_factor = s.scale_factor) ORDER BY s.ord `; - const [scopes, datasets] = await Promise.all([ - getPool().query(scopesText, [ - FILE_SIZE_SCOPES.map((scope) => scope.benchmark), - FILE_SIZE_SCOPES.map((scope) => scope.scale_factor), - ]), - compressionSizeDatasets(), + const scopes = await getPool().query(scopesText, [ + FILE_SIZE_SCOPES.map((scope) => scope.benchmark), + FILE_SIZE_SCOPES.map((scope) => scope.scale_factor), ]); const charts: ChartLink[] = [ ...scopes.rows.map((row) => ({ @@ -1769,7 +1780,72 @@ async function collectFileSizeGroup(): Promise { if (charts.length === 0) { return []; } - return [{ name: FILE_SIZES_GROUP_NAME, slug: groupKeyToSlug({ k: 'FileSizeGroup' }), charts }]; + return [ + { name: FILE_SIZES_GROUP_NAME, slug: groupKeyToSlug({ k: 'CompressionSizeGroup' }), charts }, + ]; +} + +/** Benchmarks whose file-size charts lead the group, in this order; the rest follow by name. */ +const FILE_SIZE_CHART_ORDER: readonly string[] = ['clickbench', 'tpch', 'tpcds']; + +/** A benchmark's position in the file-size group: its `FILE_SIZE_CHART_ORDER` index, else after. */ +function fileSizeChartRank(benchmark: string): number { + const pos = FILE_SIZE_CHART_ORDER.indexOf(benchmark); + return pos === -1 ? FILE_SIZE_CHART_ORDER.length : pos; +} + +/** + * The SQL File Sizes group: one chart per benchmark and scale factor in `file_sizes`, or `[]` if + * the table is empty. Charts order by [`FILE_SIZE_CHART_ORDER`], then by benchmark name, then by numeric + * scale factor, so `10.0` follows `1.0`. + */ +async function collectFileSizeGroup(): Promise { + // A recursive skip scan of `idx_file_sizes_baseline`: one descent per scope instead of a scan + // of every file of every commit. + const text = ` + WITH RECURSIVE scopes AS ( + (SELECT f.benchmark, f.scale_factor + FROM file_sizes f + ORDER BY f.benchmark, f.scale_factor + LIMIT 1) + UNION ALL + SELECT nxt.benchmark, nxt.scale_factor + FROM scopes s + CROSS JOIN LATERAL ( + SELECT f.benchmark, f.scale_factor + FROM file_sizes f + WHERE (f.benchmark, f.scale_factor) > (s.benchmark, s.scale_factor) + ORDER BY f.benchmark, f.scale_factor + LIMIT 1 + ) nxt + ) + SELECT benchmark, scale_factor FROM scopes + `; + const rows = (await getPool().query(text)).rows; + if (rows.length === 0) { + return []; + } + rows.sort( + (a, b) => + fileSizeChartRank(a.benchmark) - fileSizeChartRank(b.benchmark) || + compareCodeUnits(a.benchmark, b.benchmark) || + Number(a.scale_factor) - Number(b.scale_factor) || + compareCodeUnits(a.scale_factor, b.scale_factor), + ); + return [ + { + name: SQL_FILE_SIZES_GROUP_NAME, + slug: groupKeyToSlug({ k: 'FileSizeGroup' }), + charts: rows.map((row) => ({ + name: fileSizeChartName(row.benchmark, row.scale_factor), + slug: chartKeyToSlug({ + k: 'FileSize', + benchmark: row.benchmark, + scale_factor: row.scale_factor, + }), + })), + }, + ]; } /** Dispatch to the discovery pass for one family, the TS analogue of the Rust diff --git a/src/bench/lib/slug.test.ts b/src/bench/lib/slug.test.ts index 012f3c02..5f168d30 100644 --- a/src/bench/lib/slug.test.ts +++ b/src/bench/lib/slug.test.ts @@ -31,12 +31,12 @@ const CHART_KEYS: ChartKey[] = [ const GROUP_KEYS: GroupKey[] = [ { k: 'QueryGroup', dataset: 'tpch', dataset_variant: null, scale_factor: '1', storage: 'nvme' }, { k: 'CompressionTimeGroup' }, + { k: 'CompressionSizeGroup' }, { k: 'CompressionTimeGroup', suite: 'strings' }, { k: 'CompressionSizeGroup', suite: 'strings' }, { k: 'RandomAccessGroup', storage: 'nvme' }, { k: 'RandomAccessGroup', storage: 's3' }, { k: 'VectorSearchGroup', dataset: 'cohere', layout: 'partitioned' }, - { k: 'FileSizeGroup' }, ]; describe('chart slug round-trip', () => { @@ -105,12 +105,6 @@ describe('group slug round-trip', () => { expect(groupKeyFromSlug(legacy)).toEqual({ k: 'RandomAccessGroup', storage: 'nvme' }); }); - it('resolves the Compression Size group slug to File Sizes, which holds its charts', () => { - const legacy = 'csg.eyJrIjoiQ29tcHJlc3Npb25TaXplR3JvdXAifQ'; - expect(groupKeyFromSlug(legacy)).toEqual({ k: 'FileSizeGroup' }); - expect(groupKeyToSlug(groupKeyFromSlug(legacy))).toBe('fsg.eyJrIjoiRmlsZVNpemVHcm91cCJ9'); - }); - it.each(['disk', '', null, 1])('rejects invalid random-access storage %s', (storage) => { const slug = 'rag.' + diff --git a/src/bench/lib/slug.ts b/src/bench/lib/slug.ts index 800bccfc..76436682 100644 --- a/src/bench/lib/slug.ts +++ b/src/bench/lib/slug.ts @@ -47,11 +47,11 @@ export type GroupKey = } // `suite` is absent for the `compress-bench` group, so its slug predates the split. | { k: 'CompressionTimeGroup'; suite?: 'strings' } - // Only `string-bench`'s sizes. `compress-bench`'s sizes are in the File Sizes group. - | { k: 'CompressionSizeGroup'; suite: 'strings' } + // Without `suite`, the File Sizes group: `compress-bench`'s sizes and three SQL scopes. + | { k: 'CompressionSizeGroup'; suite?: 'strings' } | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string } - // The File Sizes group: `compress-bench`'s sizes and the SQL suites' file sizes. + // The SQL File Sizes group: every SQL benchmark and scale factor in `file_sizes`. | { k: 'FileSizeGroup' }; const CHART_KINDS: readonly ChartKind[] = [ @@ -256,10 +256,8 @@ export function groupKeyToSlug(key: GroupKey): string { ordered = { k: key.k, storage: key.storage }; break; case 'CompressionTimeGroup': - ordered = key.suite === undefined ? { k: key.k } : { k: key.k, suite: key.suite }; - break; case 'CompressionSizeGroup': - ordered = { k: key.k, suite: key.suite }; + ordered = key.suite === undefined ? { k: key.k } : { k: key.k, suite: key.suite }; break; case 'FileSizeGroup': ordered = { k: key.k }; @@ -298,19 +296,10 @@ export function groupKeyFromSlug(slug: string): GroupKey { } return { k: kind, storage }; } - case 'CompressionTimeGroup': { - if (obj.suite === undefined) { - return { k: kind }; - } - if (obj.suite !== 'strings') { - throw new Error('compression group suite must be strings when present'); - } - return { k: kind, suite: 'strings' }; - } + case 'CompressionTimeGroup': case 'CompressionSizeGroup': { - // The `Compression Size` group's links resolve to File Sizes, which holds its charts now. if (obj.suite === undefined) { - return { k: 'FileSizeGroup' }; + return { k: kind }; } if (obj.suite !== 'strings') { throw new Error('compression group suite must be strings when present'); diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index fa6ad451..c5004b02 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -63,7 +63,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -116,7 +116,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -156,7 +156,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -190,7 +190,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -232,7 +232,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -306,7 +306,7 @@ describe('compression summaries', () => { ], }); - const summary = await collectGroupSummary({ k: 'FileSizeGroup' }); + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); if (summary === null || summary.type !== 'compressionSize') { throw new Error('expected a compressionSize summary'); } @@ -383,7 +383,7 @@ describe('compression summaries', () => { query.mockResolvedValue({ rows: [] }); await collectGroupSummary({ k: 'CompressionTimeGroup' }); - await collectGroupSummary({ k: 'FileSizeGroup' }); + await collectGroupSummary({ k: 'CompressionSizeGroup' }); // The File Sizes summary reads the SQL suites' sizes in a third query. expect(query).toHaveBeenCalledTimes(3); diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index 15f02896..a23ec214 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -212,13 +212,15 @@ export function collectGroupSummary(key: GroupKey): Promise { case 'CompressionTimeGroup': return key.suite === 'strings' ? collectStringEncodingSummary() : collectCompressionSummary(); case 'CompressionSizeGroup': - return collectStringEncodingSizeSummary(); + return key.suite === 'strings' + ? collectStringEncodingSizeSummary() + : collectFileSizesSummary(); case 'RandomAccessGroup': return collectRandomAccessSummary(key.storage); case 'VectorSearchGroup': return collectVectorSearchSummary(key.dataset, key.layout); case 'FileSizeGroup': - return collectFileSizesSummary(); + return collectFileSizeSummary(); } } @@ -720,6 +722,93 @@ async function compressionSamples(): Promise { return (await getPool().query(text, compressionSummaryQueryParams())).rows; } +/** The format every other SQL benchmark file format is compared with. */ +const FILE_SIZE_BASELINE = 'vortex-file-compressed'; + +/** + * The SQL File Sizes rollup: each format's total file size against `vortex-file-compressed`, per + * benchmark scope (benchmark and scale factor) at that scope's newest commit with a baseline. + * Unlike the File Sizes summary, it covers every scope in `file_sizes` and leaves out + * `compress-bench`. Scopes come from nightly as well as develop runs, so each one uses its own + * newest commit. File sizes record no Arrow size, so the Arrow ratio is empty. + */ +async function collectFileSizeSummary(): Promise { + // `scopes` is a skip scan of `idx_file_sizes_baseline`. `latest` walks the read branch's + // commits newest first and probes that index, so it stops at the newest commit with a baseline. + const text = ` + WITH RECURSIVE scopes AS ( + (SELECT f.benchmark, f.scale_factor + FROM file_sizes f + ORDER BY f.benchmark, f.scale_factor + LIMIT 1) + UNION ALL + SELECT nxt.benchmark, nxt.scale_factor + FROM scopes s + CROSS JOIN LATERAL ( + SELECT f.benchmark, f.scale_factor + FROM file_sizes f + WHERE (f.benchmark, f.scale_factor) > (s.benchmark, s.scale_factor) + ORDER BY f.benchmark, f.scale_factor + LIMIT 1 + ) nxt + ), + latest AS ( + SELECT s.benchmark, s.scale_factor, l.commit_sha + FROM scopes s + CROSS JOIN LATERAL ( + SELECT c.commit_sha + FROM ${BRANCH_COMMITS} c + WHERE EXISTS ( + SELECT 1 + FROM file_sizes f + WHERE f.benchmark = s.benchmark + AND f.scale_factor = s.scale_factor + AND f.format = $1 + AND f.commit_sha = c.commit_sha) + ORDER BY c.timestamp DESC, c.commit_sha DESC + LIMIT 1 + ) l + ), + totals AS ( + SELECT f.benchmark, f.scale_factor, f.format, SUM(f.value_bytes)::float8 AS bytes + FROM file_sizes f + JOIN latest l + ON l.benchmark = f.benchmark + AND l.scale_factor = f.scale_factor + AND l.commit_sha = f.commit_sha + GROUP BY f.benchmark, f.scale_factor, f.format + ) + SELECT t.format AS format, + t.benchmark AS dataset, + t.scale_factor AS "datasetVariant", + t.bytes AS "valueBytes", + b.bytes AS "baselineBytes", + NULL::float8 AS "uncompressedBytes" + FROM totals t + JOIN totals b + ON b.benchmark = t.benchmark + AND b.scale_factor = t.scale_factor + AND b.format = $1 + WHERE t.bytes > 0 + AND b.bytes > 0 + ORDER BY t.format, t.benchmark, t.scale_factor + `; + const rows = (await getPool().query(text, [FILE_SIZE_BASELINE])).rows; + const rankings = compressionSizeRankings(rows); + if (rankings.length === 0) { + return null; + } + return { + type: 'compressionSize', + title: 'SQL File Sizes Summary', + rankings, + explanation: + 'Total size of the files each SQL benchmark writes at every scale factor, and geometric means of sizes versus vortex (lower is better)', + // The card names formats by their display names, which call `vortex-file-compressed` `vortex`. + baseline: 'vortex', + }; +} + /** * The `File Sizes` rollup: each format's sizes against Parquet's (zstd) over the * `compress-bench` datasets ([`compressionSizeSamples`]) and the SQL scopes in diff --git a/tests/unit/fixtures/bench-groups.json b/tests/unit/fixtures/bench-groups.json index 64386d41..bcc6f54c 100644 --- a/tests/unit/fixtures/bench-groups.json +++ b/tests/unit/fixtures/bench-groups.json @@ -49,7 +49,7 @@ }, { "name": "File Sizes", - "slug": "fsg.eyJrIjoiRmlsZVNpemVHcm91cCJ9", + "slug": "csg.eyJrIjoiQ29tcHJlc3Npb25TaXplR3JvdXAifQ", "summary": { "type": "compressionSize", "title": "File Sizes Summary", From cca5c97f5b99c31e8551581a3e73667f891b8ef0 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 20 Nov 2026 17:25:29 -0500 Subject: [PATCH 03/23] Note the TPC-DS size-only formats in the bench contract Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index b3528433..b98805db 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -87,6 +87,8 @@ The read service charts both tables in the `File Sizes` group, one line per form current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. +On develop, TPC-DS SF1 also writes Arrow IPC, Lance, and Snappy Parquet after its queries, for +their sizes only, so that every format `compress-bench` measures has a TPC-DS size too. The `SQL File Sizes` group charts every `file_sizes` scope, at every scale factor, and leaves out `compression_sizes`. Its summary compares each format with `vortex-file-compressed`, per scope at From 8d90c34f0e7436cfd63e061f915f2ed1bb9cad60 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 20 Nov 2026 17:47:55 -0500 Subject: [PATCH 04/23] Rank a query series that skips a query below complete ones on bench A series with no time for a query now ranks below every series that ran all of them. The missing-series penalty alone scores a skipped query at 2x the best time, so a series could rank first by skipping the slowest query. Lance skips TPC-DS q72, which it cannot run. Snappy Parquet joins TPC-DS's queries, so a codec variant is now dotted, and dash-dotted on DuckDB, to keep it apart from DuckDB's dashed Parquet. Signed-off-by: Will Manning --- CLAUDE.md | 2 +- docs/bench/CONTRACT.md | 6 ++++-- src/bench/lib/chart-format.test.ts | 15 ++++++++++++-- src/bench/lib/chart-format.ts | 21 ++++++++++++++----- src/bench/lib/site-files.ts | 1 + src/bench/lib/summary.test.ts | 33 ++++++++++++++++++++++++++++++ src/bench/lib/summary.ts | 6 ++++++ 7 files changed, 74 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 10dcf6de..cd072464 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,7 +83,7 @@ The previous site generations (v2 and v3) are documented in [`docs/bench/legacy. - **Don't refetch on every scope change.** Once a chart's payload is in memory, pan/zoom/slider/range-strip all rebuild in place via the in-memory LTTB pass on the cached payload ([`src/bench/components/chart/controller.ts`](src/bench/components/chart/controller.ts)). The single exception is the latest-100 to full-history zoom-out path: when the user first zooms past the materialized window, the chart lazy-fetches `?n=all` once and replaces the payload in place. - **Group docs links mirror the monorepo's `Benchmark::doc_path`.** [`src/bench/lib/benchmark-docs.ts`](src/bench/lib/benchmark-docs.ts) maps each SQL dataset name to the benchmark's explainer doc in `vortex-data/vortex`, which is the same path that the monorepo's PR benchmark comment uses. The producer does not put `doc_path` on the wire, and upstream does not enforce a path convention, so update the explicit mapping when a suite is added or its doc moves. Groups without a matching explainer render no docs link. Do not guess a path or link to an unrelated general guide. - **Every group kind has a summary, by default.** [`src/bench/lib/summary.ts`](src/bench/lib/summary.ts) has no allowlist gate: a suite that lands in one of the six fact tables gets a rollup card from its first ingest. The three timing families (query, random access, vector search) rank through one `rankSeries` model, so a new suite in any of them needs no summary code at all. `collectGroupSummary`'s exhaustive switch makes a missing arm for a seventh fact table a compile error rather than a silently blank card. Do NOT reintroduce a per-dataset allowlist; the v2-era one is what left `spatialbench`, `fineweb`, `gharchive`, `appian`, `public-bi`, `clickbench-sorted`, and every vector-search group with no card. -- **A summary ranks the whole group, never one chart.** Random access is the cautionary case: the producer emits `dataset` as `{dataset}/{pattern}`, so the group holds ~nine charts, and the old summary published the alphabetically first chart's raw times under the group-wide title "Random Access Performance" — reporting `lance` at 352us when it is over 1ms on most of the other charts. Sum chart medians by dataset, then rank the dataset totals. Include the legacy taxi chart in the taxi total. Apply a penalty when a series lacks one dataset total. Report dataset coverage through `measured`/`total`. The secondary runtime is the arithmetic mean of complete dataset totals. +- **A summary ranks the whole group, never one chart.** Random access is the cautionary case: the producer emits `dataset` as `{dataset}/{pattern}`, so the group holds ~nine charts, and the old summary published the alphabetically first chart's raw times under the group-wide title "Random Access Performance" — reporting `lance` at 352us when it is over 1ms on most of the other charts. Sum chart medians by dataset, then rank the dataset totals. Include the legacy taxi chart in the taxi total. Apply a penalty when a series lacks one dataset total. Report dataset coverage through `measured`/`total`. The secondary runtime is the arithmetic mean of complete dataset totals. In a query summary, the penalty is not enough: a series that misses a query ranks below every series that ran them all, so a series cannot rank first by failing or skipping the hardest query (Lance skips TPC-DS q72). - **Don't write a server-side classifier for live ingest.** The emitter produces structured records directly. Classifying loose name strings at read time was the v2-era weakness every later generation existed to escape; it belongs nowhere in the live pipeline. - **Every root-relative bench URL goes through `benchPath()`.** A link, a fetch, an icon or a selector that names one of bench's paths must work on bench.vortex.dev (from the root) and everywhere else (under `/bench`); see "Repository layout". A path read from `location.pathname` already carries the base. diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index b98805db..707000e8 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -87,8 +87,10 @@ The read service charts both tables in the `File Sizes` group, one line per form current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. -On develop, TPC-DS SF1 also writes Arrow IPC, Lance, and Snappy Parquet after its queries, for -their sizes only, so that every format `compress-bench` measures has a TPC-DS size too. +On develop, TPC-DS SF1 has every format `compress-bench` measures. It queries Snappy Parquet +(`parquet-snappy`) on DataFusion and DuckDB and Lance on DataFusion, and writes Arrow IPC after its +queries, for its size only. Lance skips q72, which it cannot run; the query summary ranks a series +that misses a query below every series that ran them all. The `SQL File Sizes` group charts every `file_sizes` scope, at every scale factor, and leaves out `compression_sizes`. Its summary compares each format with `vortex-file-compressed`, per scope at diff --git a/src/bench/lib/chart-format.test.ts b/src/bench/lib/chart-format.test.ts index 9f580635..44343c66 100644 --- a/src/bench/lib/chart-format.test.ts +++ b/src/bench/lib/chart-format.test.ts @@ -139,12 +139,12 @@ describe('seriesStyle', () => { } }); - it("draws a Parquet codec variant dashed, in Parquet's color, thinner, behind Parquet", () => { + it("draws a Parquet codec variant dotted, in Parquet's color, thinner, behind Parquet", () => { for (const theme of ['dark', 'light'] as const) { const zstd = seriesStyle('parquet', { format: 'parquet' }, theme); const snappy = seriesStyle('parquet-snappy', { format: 'parquet-snappy' }, theme); expect(snappy.color).toBe(zstd.color); - expect(snappy.dash).toEqual([6, 4]); + expect(snappy.dash).toEqual([2, 3]); expect(snappy.width).toBeLessThan(zstd.width); } const order = (format: string) => seriesOrder(format, { format }); @@ -152,6 +152,17 @@ describe('seriesStyle', () => { expect(order('parquet-snappy')).toBeLessThan(order('vortex-compact')); }); + it("keeps a query chart's four Parquet lines apart", () => { + const styles = ['parquet', 'parquet-snappy'].flatMap((format) => + ['datafusion', 'duckdb'].map((engine) => + seriesStyle(`${engine}:${format}`, { engine, format }, 'dark'), + ), + ); + const looks = new Set(styles.map((style) => `${style.width}/${style.dash.join(',')}`)); + expect(looks.size).toBe(4); + expect(seriesStyle('duckdb:parquet-snappy', null, 'dark').dash).toEqual([6, 3, 2, 3]); + }); + it('draws a cold random-access run dashed and a hot one solid', () => { const cold = seriesStyle('parquet:cold', { format: 'parquet' }, 'dark'); const hot = seriesStyle('parquet:hot', { format: 'parquet' }, 'dark'); diff --git a/src/bench/lib/chart-format.ts b/src/bench/lib/chart-format.ts index cc2f1211..3d42b518 100644 --- a/src/bench/lib/chart-format.ts +++ b/src/bench/lib/chart-format.ts @@ -85,7 +85,8 @@ export const MAX_VISIBLE_POINTS = 500; // purple, DuckDB green, and Arrow orange. The color means the format and // nothing else, so a format looks the same on every engine: the line style // shows the engine instead. A duckdb run is dashed, as is a cold random-access -// run (the two never share a chart). Widths carry the importance tiers. +// run (the two never share a chart). A Parquet codec variant is dotted, or +// dash-dotted on duckdb. Widths carry the importance tiers. // // The palette has one set of colors per card theme, because one set cannot // stay legible on both cards: a color with 3:1 contrast against both the @@ -113,6 +114,10 @@ export interface SeriesStyle { /** The dash of a duckdb run and of a cold random-access run. */ const DASH: readonly number[] = [6, 4]; +/** The dash of a Parquet codec variant, which a query chart can hold on both + * engines beside zstd Parquet: dotted, and dash-dotted on duckdb. */ +const DOT: readonly number[] = [2, 3]; +const DASH_DOT: readonly number[] = [6, 3, 2, 3]; /** Line widths per importance tier, reinforcing the color hierarchy. */ const HERO_WIDTH = 2.6; @@ -162,9 +167,10 @@ const FORMAT: Record = { /** * The format a codec variant is drawn as: Parquet with a codec other than zstd - * (`parquet-snappy`, `parquet-uncompressed`, as the size tables name them) is - * Parquet's teal, dashed, so it reads as a kind of Parquet without a color of - * its own. Size charts have no engine, so the dash cannot be read as duckdb's. + * (`parquet-snappy`, `parquet-uncompressed`, as the size tables and TPC-DS's + * queries name them) is Parquet's teal, thinner and dotted, so it reads as a + * kind of Parquet without a color of its own. Its dots differ from duckdb's + * dash, so a query chart's four Parquet lines stay apart. */ function codecVariantOf(format: string): string | undefined { return format.startsWith('parquet-') ? 'parquet' : undefined; @@ -201,8 +207,13 @@ export function seriesStyle( ): SeriesStyle { const { engine, format } = seriesDims(name, meta); const variantOf = format === undefined ? undefined : codecVariantOf(format); + const duckdbOrCold = engine === 'duckdb' || name.endsWith(':cold'); const dash = - engine === 'duckdb' || name.endsWith(':cold') || variantOf !== undefined ? [...DASH] : []; + variantOf !== undefined + ? [...(engine === 'duckdb' ? DASH_DOT : DOT)] + : duckdbOrCold + ? [...DASH] + : []; const base = variantOf ?? format; const style = base ? FORMAT[base] : undefined; if (!style) { diff --git a/src/bench/lib/site-files.ts b/src/bench/lib/site-files.ts index 5ca40b56..685edcc2 100644 --- a/src/bench/lib/site-files.ts +++ b/src/bench/lib/site-files.ts @@ -191,6 +191,7 @@ const FORMATS: ReadonlyMap = new Map([ ['vortex-file-compressed', 'Vortex'], ['vortex-compact', 'Vortex (compact)'], ['parquet', 'Parquet'], + ['parquet-snappy', 'Parquet (Snappy)'], ['lance', 'Lance'], ['arrow-ipc', 'Arrow IPC'], ['arrow', 'Arrow'], diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index c5004b02..f9cdab4e 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -820,6 +820,39 @@ describe('timing summaries (shared ranking model)', () => { }, ); + it('ranks a series that misses a query below every series that ran them all', async () => { + // `partial` is twice as fast on the queries it runs and skips the slowest one. The penalty + // scores the skipped query at 2x the best, so its score, 2^(1/3), beats `complete`'s, 2^(2/3). + query.mockResolvedValueOnce({ + rows: [ + { query_idx: 1, series: 'lance:lance', value_ns: 1_000_000 }, + { query_idx: 1, series: 'datafusion:parquet', value_ns: 2_000_000 }, + { query_idx: 2, series: 'lance:lance', value_ns: 1_000_000 }, + { query_idx: 2, series: 'datafusion:parquet', value_ns: 2_000_000 }, + { query_idx: 3, series: 'datafusion:parquet', value_ns: 10_000_000 }, + ], + }); + + const summary = await collectGroupSummary({ + k: 'QueryGroup', + dataset: 'tpcds', + dataset_variant: null, + scale_factor: '1', + storage: 'nvme', + }); + if (summary === null || summary.type !== 'queryBenchmark') { + throw new Error('expected a queryBenchmark summary'); + } + + expect(summary.rankings.map((ranking) => ranking.name)).toEqual([ + 'datafusion:parquet', + 'lance:lance', + ]); + const [complete, partial] = summary.rankings; + expect(defined(partial).score).toBeLessThan(defined(complete).score); + expect(defined(partial)).toMatchObject({ measured: 2, total: 3 }); + }); + it('summarizes every query group, with no dataset allowlist', async () => { // `spatialbench` (and every other suite outside the retired v2 five) used to // fall through to `null` and render no card at all. diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index a23ec214..3804a52b 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -1462,6 +1462,12 @@ async function collectQuerySummary( if (rankings.length === 0) { return null; } + // A series that has no time for a query, because it fails it or skips it (Lance skips TPC-DS + // q72), ranks below every series that ran all of them. The penalty alone makes a missing query + // cost only 2x the best time on it, so a series could rank first by skipping the slowest query. + // Its score and its (measured/total) coverage still show. The sort is stable, so each part keeps + // its score order. + rankings.sort((a, b) => Number(a.measured < a.total) - Number(b.measured < b.total)); return { type: 'queryBenchmark', title: 'Performance Summary', From b9b9a1f6bd908c0e016d4e5700034d944af7b40a Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 20 Nov 2026 18:19:16 -0500 Subject: [PATCH 05/23] Block Vercel's preview Toolbar in Lighthouse runs Vercel adds its Toolbar to preview deployments, and the proxy's x-vercel-skip-toolbar header does not stop it. The Toolbar loads from vercel.live and Pusher, and its third-party cookies and issues cut Best Practices to 0.78 on every audited page. The site loads neither host, so Lighthouse now blocks both. Signed-off-by: Will Manning --- lighthouserc.cjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lighthouserc.cjs b/lighthouserc.cjs index 1476e21b..ea7e0ffe 100644 --- a/lighthouserc.cjs +++ b/lighthouserc.cjs @@ -145,7 +145,13 @@ module.exports = { settings: { preset: "desktop", throttlingMethod: "simulate", - skipAudits: ["uses-http2"] + skipAudits: ["uses-http2"], + // Vercel adds its Toolbar to preview deployments, even with the + // proxy's `x-vercel-skip-toolbar` header. It loads from vercel.live + // (and Pusher, for live comments), and its third-party cookies and + // issues cut Best Practices to 0.78 on every page from 2026-11-20. + // The site never loads either host, so block them. + blockedUrlPatterns: ["*vercel.live*", "*pusher.com*"] } }, assert: { From 27097f8284a86da9c026f87ae10c7d872d32991d Mon Sep 17 00:00:00 2001 From: Will Manning Date: Fri, 20 Nov 2026 21:34:05 -0500 Subject: [PATCH 06/23] Explain bench's two Parquet variants on the methodology page Parquet is zstd level 3, the size and throughput baseline. Parquet (Snappy) is Parquet as DuckDB writes it by default, which TPC-DS SF1 queries and the compression benchmark sizes. ClickBench and FineWeb query their published files as Parquet, and ClickBench's are Snappy. The page also says that a query summary ranks a series that missed a query below the complete ones, as Lance on TPC-DS q72. The contract now names file sizes by format directory, not by codec. Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 18 ++++++++++-------- src/bench/lib/methodology.ts | 24 +++++++++++++++++++----- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index 707000e8..1d818ebd 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -65,8 +65,9 @@ column-by-column; an unknown `kind` or a missing dimension field fails the inges `compression_size` records include `uncompressed_bytes`, the Arrow memory size after the source Parquet file is decoded. The column is nullable so historical rows remain valid. `compress-bench` also writes two size-only formats, with no `compression_time` records: -`parquet-snappy` (Parquet with Snappy; `parquet` is zstd level 3) and `vortex-compact` (Vortex -with the compact strategy). Each is one write per dataset that is not timed. +`parquet-snappy` (Parquet as DuckDB writes it, with Snappy; `parquet` is zstd level 3) and +`vortex-compact` (Vortex with the compact strategy). Each is one write per dataset that is not +timed. `string-bench` shares the two compression tables with `compress-bench`. Its records use the dataset `strings/`, the string encoder (`onpair-12`, `fsst`, ...) as `dataset_variant`, @@ -78,17 +79,18 @@ rows in the `String Encoding` groups and compares each encoder with `fsst`, so t `file_size` records are not written by `vortex-bench`: the monorepo's `scripts/capture-file-sizes.py` writes one per data file a SQL benchmark leaves on disk, with `benchmark` (the `vx-bench` subcommand), `scale_factor`, `format`, `file` (the path below the format -directory) and `value_bytes`. A Parquet file's `format` names its codec, which the script reads -from the file's footer: `parquet` for zstd, and `parquet-` (`parquet-snappy`, for example) -for any other. CI reads the records back as the develop baseline for the PR comment's file-size -section. +directory) and `value_bytes`. A file's `format` is the name of its format directory, not its +codec. `parquet` is the suite's base Parquet: zstd level 3 where the benchmarks write it, and the +dataset's published files where the suite prescribes them, as ClickBench's are Snappy. +`parquet-snappy` is Parquet as the DuckDB CLI writes it, with Snappy, from the base Parquet. CI +reads the records back as the develop baseline for the PR comment's file-size section. The read service charts both tables in the `File Sizes` group, one line per format. It charts every current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. -On develop, TPC-DS SF1 has every format `compress-bench` measures. It queries Snappy Parquet -(`parquet-snappy`) on DataFusion and DuckDB and Lance on DataFusion, and writes Arrow IPC after its +On develop, TPC-DS SF1 has every format `compress-bench` measures. It queries DuckDB's Snappy +Parquet (`parquet-snappy`) on DataFusion and DuckDB and Lance on DataFusion, and writes Arrow IPC after its queries, for its size only. Lance skips q72, which it cannot run; the query summary ranks a series that misses a query below every series that ran them all. diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 86497577..5598d921 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -11,8 +11,11 @@ * `bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py`, the runners in * `bench-orchestrator/bench_orchestrator/cli.py`, `scripts/random-access-split.py` * and `scripts/compress-split.py`, `scripts/setup-benchmark.sh`, and the versions - * in `Cargo.toml`, `Cargo.lock` and `vortex-duckdb/build.rs`. Update this file - * when those change. + * in `Cargo.toml`, `Cargo.lock` and `vortex-duckdb/build.rs`. The Parquet + * variants come from vortex-data/vortex#11720: `parquet_writer_properties()` in + * `vortex-bench/src/utils/parquet.rs`, `vortex-bench/src/parquet_copies.rs`, and + * the DuckDB version in `.github/actions/setup-duckdb/action.yml`. Update this + * file when those change. * The scoring is this site's own (`summary.ts`). */ @@ -68,9 +71,17 @@ export const METHODOLOGY: readonly MethodologySection[] = [ text: [ 'Queries run on DataFusion 55 and DuckDB 2.0. The formats are Vortex (its default ' + 'and compact write strategies), Apache Parquet, Lance 13, Arrow IPC, DuckDB’s own ' + - 'format, and, for random access, LitData. Not every suite runs every format. The ' + - 'compression and file size benchmarks write Parquet with parquet-rs 59 and zstd at ' + - 'level 3. ClickBench reads its published Parquet files.', + 'format, and, for random access, LitData. Not every suite runs every format.', + 'Parquet has two variants, one for size and one for speed. Parquet is zstd at level ' + + '3, written with parquet-rs 59. It is the baseline of the throughput and file size ' + + 'scores. Parquet (Snappy) is Parquet as DuckDB writes it by default: Snappy, with ' + + 'DuckDB’s row groups, dictionaries, statistics, and Bloom filters. The DuckDB 1.5 ' + + 'command line tool writes it from the same rows. The compression benchmark records ' + + 'its size, and TPC-DS at scale factor 1 queries it.', + 'Every SQL suite’s Parquet is zstd Parquet, except in ClickBench and FineWeb, which ' + + 'query their published Parquet files as Parquet. ClickBench prescribes its files, ' + + 'which are Snappy, so its Parquet results are not zstd Parquet and are not DuckDB’s ' + + 'Parquet (Snappy) either.', ], }, { @@ -96,6 +107,9 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'In both, a query or dataset a series did not run counts as twice that series’ own ' + 'slowest time, and at least twice the fastest series’ time, so a series cannot ' + 'rank well by skipping work. Each summary says how many a series ran.', + 'A query suite also ranks a series that missed a query below every series that ran ' + + 'them all. Lance cannot run TPC-DS query 72, so it ranks below the complete series ' + + 'on TPC-DS at scale factor 1.', `Write and scan throughput scores are ${SCORES.throughput}`, `File size scores are ${SCORES.size}`, 'Total is the sum of a format’s file sizes over the datasets that every format has.', From 86a487c10ddde6ad20d29564448331415a965bed Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 08:11:28 -0500 Subject: [PATCH 07/23] Name DuckDB 2.0 as the writer of Parquet (Snappy) on bench vortex-data/vortex#11720 now installs the DuckDB 2.0.0 CLI, the version the queries run on. Signed-off-by: Will Manning --- src/bench/lib/methodology.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 5598d921..1160d4c3 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -75,7 +75,7 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'Parquet has two variants, one for size and one for speed. Parquet is zstd at level ' + '3, written with parquet-rs 59. It is the baseline of the throughput and file size ' + 'scores. Parquet (Snappy) is Parquet as DuckDB writes it by default: Snappy, with ' + - 'DuckDB’s row groups, dictionaries, statistics, and Bloom filters. The DuckDB 1.5 ' + + 'DuckDB’s row groups, dictionaries, statistics, and Bloom filters. The DuckDB 2.0 ' + 'command line tool writes it from the same rows. The compression benchmark records ' + 'its size, and TPC-DS at scale factor 1 queries it.', 'Every SQL suite’s Parquet is zstd Parquet, except in ClickBench and FineWeb, which ' + From c0cccc5a6e0a0315c2e88e54251702944b89e111 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 15:15:25 -0500 Subject: [PATCH 08/23] Explain how bench's Parquet is tuned on the methodology page A new section gives each Parquet writer setting and the test behind it: zstd level, dictionary size, page size and page index, Bloom filters, and row groups. Parquet (Snappy) is now the same Parquet with Snappy in place of zstd, rather than Parquet as DuckDB writes it, and TPC-DS at scale factor 1 no longer queries it. Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 11 +++---- src/bench/lib/methodology.ts | 61 +++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index 1d818ebd..0dd1751b 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -65,7 +65,8 @@ column-by-column; an unknown `kind` or a missing dimension field fails the inges `compression_size` records include `uncompressed_bytes`, the Arrow memory size after the source Parquet file is decoded. The column is nullable so historical rows remain valid. `compress-bench` also writes two size-only formats, with no `compression_time` records: -`parquet-snappy` (Parquet as DuckDB writes it, with Snappy; `parquet` is zstd level 3) and +`parquet-snappy` (Parquet with Snappy, and otherwise the same writer settings as `parquet`, +which is zstd level 3) and `vortex-compact` (Vortex with the compact strategy). Each is one write per dataset that is not timed. @@ -81,17 +82,15 @@ rows in the `String Encoding` groups and compares each encoder with `fsst`, so t `benchmark` (the `vx-bench` subcommand), `scale_factor`, `format`, `file` (the path below the format directory) and `value_bytes`. A file's `format` is the name of its format directory, not its codec. `parquet` is the suite's base Parquet: zstd level 3 where the benchmarks write it, and the -dataset's published files where the suite prescribes them, as ClickBench's are Snappy. -`parquet-snappy` is Parquet as the DuckDB CLI writes it, with Snappy, from the base Parquet. CI +dataset's published files where the suite prescribes them, as ClickBench's are Snappy. CI reads the records back as the develop baseline for the PR comment's file-size section. The read service charts both tables in the `File Sizes` group, one line per format. It charts every current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. -On develop, TPC-DS SF1 has every format `compress-bench` measures. It queries DuckDB's Snappy -Parquet (`parquet-snappy`) on DataFusion and DuckDB and Lance on DataFusion, and writes Arrow IPC after its -queries, for its size only. Lance skips q72, which it cannot run; the query summary ranks a series +On develop, TPC-DS SF1 records the sizes of Lance and Arrow IPC beside Parquet and Vortex. It +queries Lance on DataFusion, and writes Arrow IPC after its queries, for its size only. Lance skips q72, which it cannot run; the query summary ranks a series that misses a query below every series that ran them all. The `SQL File Sizes` group charts every `file_sizes` scope, at every scale factor, and leaves out diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 1160d4c3..98507acf 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -12,10 +12,10 @@ * `bench-orchestrator/bench_orchestrator/cli.py`, `scripts/random-access-split.py` * and `scripts/compress-split.py`, `scripts/setup-benchmark.sh`, and the versions * in `Cargo.toml`, `Cargo.lock` and `vortex-duckdb/build.rs`. The Parquet - * variants come from vortex-data/vortex#11720: `parquet_writer_properties()` in - * `vortex-bench/src/utils/parquet.rs`, `vortex-bench/src/parquet_copies.rs`, and - * the DuckDB version in `.github/actions/setup-duckdb/action.yml`. Update this - * file when those change. + * settings come from vortex-data/vortex#11720: `parquet_writer_properties()` in + * `vortex-bench/src/utils/parquet.rs`, and `snappy_parquet_size()` in + * `benchmarks/compress-bench/src/parquet.rs`. The tuning results in "How Parquet + * is written" are that PR's. Update this file when those change. * The scoring is this site's own (`summary.ts`). */ @@ -72,16 +72,53 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'Queries run on DataFusion 55 and DuckDB 2.0. The formats are Vortex (its default ' + 'and compact write strategies), Apache Parquet, Lance 13, Arrow IPC, DuckDB’s own ' + 'format, and, for random access, LitData. Not every suite runs every format.', - 'Parquet has two variants, one for size and one for speed. Parquet is zstd at level ' + - '3, written with parquet-rs 59. It is the baseline of the throughput and file size ' + - 'scores. Parquet (Snappy) is Parquet as DuckDB writes it by default: Snappy, with ' + - 'DuckDB’s row groups, dictionaries, statistics, and Bloom filters. The DuckDB 2.0 ' + - 'command line tool writes it from the same rows. The compression benchmark records ' + - 'its size, and TPC-DS at scale factor 1 queries it.', + 'Parquet has two variants, which differ only in their codec. Parquet is zstd at level ' + + '3, and it is the baseline of the throughput and file size scores. Parquet (Snappy) ' + + 'is the same Parquet with Snappy, the default codec of DuckDB, Spark, and PyArrow. ' + + 'Only the compression benchmark writes it, for its size. Both are written with ' + + 'parquet-rs 59, except SpatialBench’s Parquet, which its generator writes with ' + + 'parquet-rs 56.', 'Every SQL suite’s Parquet is zstd Parquet, except in ClickBench and FineWeb, which ' + 'query their published Parquet files as Parquet. ClickBench prescribes its files, ' + - 'which are Snappy, so its Parquet results are not zstd Parquet and are not DuckDB’s ' + - 'Parquet (Snappy) either.', + 'which are Snappy, so its Parquet results are not zstd Parquet and do not use the ' + + 'settings below.', + ], + }, + { + title: 'How Parquet is written', + text: [ + 'The zstd Parquet is tuned to be good at everything bench measures: file size, scans, ' + + 'and random access. Every suite that generates Parquet writes it with one set of ' + + 'settings, and only the row group size depends on the storage. Each setting below ' + + 'was tested on its own, on TPC-H at scale factor 10 with both engines and on the ' + + 'random access datasets, on a 16-core Apple M-series Mac with local NVMe. Two runs ' + + 'with the same settings differed by up to 2% on DataFusion and 6% on DuckDB, so ' + + 'smaller differences are noise.', + 'Compression is zstd at level 3, the default of DuckDB and parquet-java. Level 1 made ' + + 'the files 7.5% larger, and level 9 made them 1.6% smaller. Neither changed query ' + + 'times by more than noise.', + 'A column chunk’s dictionary holds at most 64 KiB. When it outgrows that, the rest of ' + + 'the chunk is written without one. parquet-rs’s default of 1 MiB keeps dictionaries ' + + 'for columns with many distinct values, such as keys and comments. With 1 MiB ' + + 'dictionaries, the TPC-H files were 22% larger, DataFusion was 8% slower, and lookups ' + + 'were up to 2.8 times slower. DuckDB makes the choice once per column chunk: it keeps ' + + 'a dictionary when the chunk has at most one distinct value for every five rows.', + 'Data pages hold at most 20,000 rows and 1 MiB, the defaults of parquet-rs. ' + + 'parquet-java uses the same row limit, which it chose so that the page index has ' + + 'enough pages to skip. Each file has a page index, which records each page’s ' + + 'minimum, maximum, and position. With it, a lookup reads and decodes only the pages ' + + 'that hold its rows. Against pages limited by size alone, the row limit made lookups ' + + 'up to 4 times faster, at the same file size and with query times within noise. ' + + 'Leaving out the page index saved no space. DuckDB 2.0 writes no page index, and its ' + + 'column chunks are usually one page.', + 'The files have no Bloom filters. Filters on every column made the TPC-H files 14% ' + + 'larger and made no query faster.', + 'Row groups hold 122,880 rows, DuckDB’s default, in files that queries read from ' + + 'local NVMe. They hold 1,048,576 rows, parquet-rs’s default, in files read from S3 ' + + 'and in the random access files. On the benchmark machines, the smaller row groups ' + + 'made TPC-H at scale factor 10 28% to 38% faster on NVMe. On S3, the larger ones made ' + + 'DataFusion 14% to 21% faster and DuckDB 3% to 8% slower. The page index lets a ' + + 'lookup skip pages inside a row group, so random access keeps parquet-rs’s default.', ], }, { From fc1dc1c26c53269768b21f2bdd9d1ba1bbc6714b Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 15:28:56 -0500 Subject: [PATCH 09/23] Measure a size family only when a format has all of it, and bump the cache A format with one wide table or sparse-wide dataset counted the whole family as measured, so it could rank in the complete tier against formats measured on every member. A family now counts as measured only when the format has every dataset of it that any format has. The File Sizes group reuses the Compression Size slug with another name, other charts, and another summary, so the group payload cache moves to v16. Signed-off-by: Will Manning --- src/bench/lib/data-cache.test.ts | 8 ++++---- src/bench/lib/data-cache.ts | 2 +- src/bench/lib/summary.test.ts | 33 ++++++++++++++++++++++++++++++++ src/bench/lib/summary.ts | 17 ++++++++++------ 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/bench/lib/data-cache.test.ts b/src/bench/lib/data-cache.test.ts index 42f624d2..b8fd7587 100644 --- a/src/bench/lib/data-cache.test.ts +++ b/src/bench/lib/data-cache.test.ts @@ -69,10 +69,10 @@ describe('data-cache wrappers', () => { expect(BENCH_DATA_TAG).toBe('bench-data'); expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400); expect(cacheCalls.map((call) => call.keyParts)).toEqual([ - ['data-cache:group-charts:v15:n100'], - ['data-cache:chart-payload:v15:n100'], - ['data-cache:groups:v15'], - ['data-cache:filter-universe:v15'], + ['data-cache:group-charts:v16:n100'], + ['data-cache:chart-payload:v16:n100'], + ['data-cache:groups:v16'], + ['data-cache:filter-universe:v16'], ]); for (const call of cacheCalls) { expect(call.options.tags).toEqual([BENCH_DATA_TAG]); diff --git a/src/bench/lib/data-cache.ts b/src/bench/lib/data-cache.ts index cfc6d3da..600f38a5 100644 --- a/src/bench/lib/data-cache.ts +++ b/src/bench/lib/data-cache.ts @@ -59,7 +59,7 @@ const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_ * computed until the next ingest. * It is independent of the producer-facing benchmark schema version. */ -const GROUP_PAYLOAD_CACHE_VERSION = 'v15'; +const GROUP_PAYLOAD_CACHE_VERSION = 'v16'; // The default last-100 group bundle, keyed by group slug. The slug is the cache // key (an `unstable_cache` argument), so one wrapper covers every group. A diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index f9cdab4e..64cefc02 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -170,6 +170,39 @@ describe('compression summaries', () => { expect(vortex.totalBytes).toBe(880); }); + it('measures a family only for a format that has all of its datasets', async () => { + const row = (format: string, dataset: string, valueBytes: number) => ({ + format, + dataset, + datasetVariant: null, + valueBytes, + baselineBytes: 100, + uncompressedBytes: null, + }); + query.mockResolvedValueOnce({ + rows: [ + row('vortex-file-compressed', 'taxi', 50), + row('vortex-file-compressed', 'sparse-wide-flat-runs', 50), + row('vortex-file-compressed', 'sparse-wide-lists-runs', 50), + // Lance has one sparse-wide shape, and the smallest ratio. It has not measured the family, + // so it ranks after the format that has. + row('lance', 'taxi', 10), + row('lance', 'sparse-wide-flat-runs', 10), + ], + }); + + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + if (summary === null || summary.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + expect( + summary.rankings.map((ranking) => [ranking.name, ranking.measured, ranking.total]), + ).toEqual([ + ['vortex-file-compressed', 2, 2], + ['lance', 1, 2], + ]); + }); + it("ranks the SQL suites' totals with the compress-bench sizes", async () => { const row = (format: string, dataset: string, valueBytes: number, baselineBytes: number) => ({ format, diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index e26edabb..46abaa4d 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -874,7 +874,8 @@ function bucketGeoMean(byBucket: ReadonlyMap): number * * Each dataset is one entry in the geometric means, except that a family counts once * ([`sizeBucket`]). `minRatio` and `maxRatio` are over single datasets. `measured` and `total` - * count buckets. A format measured on every bucket ranks above one that is not, because their + * count buckets, and a format measures a family's bucket only when it has every dataset of the + * family that any format has. A format measured on every bucket ranks above one that is not, because their * means are over different datasets. Within each tier, the order is by compression ratio once * every format in the summary has one, and by baseline ratio until then. */ @@ -890,7 +891,8 @@ function compressionSizeRankings( bytesByDataset: Map; } >(); - const buckets = new Set(); + // The datasets in each bucket, over every format. + const bucketDatasets = new Map>(); for (const row of rows) { const ratios = grouped.get(row.format) ?? { sizeRatios: new Map(), @@ -908,8 +910,9 @@ function compressionSizeRankings( ...(ratios.sizeRatios.get(bucket) ?? []), row.valueBytes / row.baselineBytes, ]); - ratios.bytesByDataset.set(JSON.stringify([row.dataset, row.datasetVariant]), row.valueBytes); - buckets.add(bucket); + const dataset = JSON.stringify([row.dataset, row.datasetVariant]); + ratios.bytesByDataset.set(dataset, row.valueBytes); + bucketDatasets.set(bucket, (bucketDatasets.get(bucket) ?? new Set()).add(dataset)); } if ( row.uncompressedBytes !== null && @@ -958,8 +961,10 @@ function compressionSizeRankings( (sum, dataset) => sum + (ratios.bytesByDataset.get(dataset) ?? 0), 0, ), - measured: ratios.sizeRatios.size, - total: buckets.size, + measured: [...bucketDatasets.values()].filter((datasets) => + [...datasets].every((dataset) => ratios.bytesByDataset.has(dataset)), + ).length, + total: bucketDatasets.size, }); } } From f5eb6dbbaf4aeb40c58c7664e88b6270f6eb7fe7 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 15:36:53 -0500 Subject: [PATCH 10/23] Rename File Sizes to File Sizes & Compression Ratios, and leave DuckDB's file out of its ranking TPC-DS records DuckDB's database file, which is a query engine's storage rather than a file format, so the summary no longer ranks it. The group still charts it. The contract and methodology also say that TPC-DS SF1 records Snappy Parquet's size. Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 14 +++++++----- docs/bench/architecture/read-path.md | 2 +- src/bench/components/SummaryCard.test.tsx | 2 +- src/bench/lib/compression-suite.ts | 6 ++--- src/bench/lib/descriptions.test.ts | 4 ++-- src/bench/lib/descriptions.ts | 6 ++--- src/bench/lib/file-sizes.ts | 6 ++--- src/bench/lib/group-charts.test.ts | 2 +- src/bench/lib/groups.test.ts | 15 ++++++++----- src/bench/lib/methodology.ts | 6 +++-- src/bench/lib/queries.test.ts | 4 ++-- src/bench/lib/queries.ts | 10 ++++----- src/bench/lib/slug.ts | 2 +- src/bench/lib/summary.test.ts | 27 ++++++++++++++++++++--- src/bench/lib/summary.ts | 24 ++++++++++++++------ src/lib/bench-summary.ts | 2 +- tests/unit/bench-summary.test.ts | 18 ++++++++------- tests/unit/fixtures/bench-groups.json | 4 ++-- 18 files changed, 97 insertions(+), 57 deletions(-) diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index 0dd1751b..b022b187 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -85,12 +85,14 @@ codec. `parquet` is the suite's base Parquet: zstd level 3 where the benchmarks dataset's published files where the suite prescribes them, as ClickBench's are Snappy. CI reads the records back as the develop baseline for the PR comment's file-size section. -The read service charts both tables in the `File Sizes` group, one line per format. It charts every -current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed -over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its -summary compares each format with zstd Parquet on the `compress-bench` datasets and on TPC-DS. -On develop, TPC-DS SF1 records the sizes of Lance and Arrow IPC beside Parquet and Vortex. It -queries Lance on DataFusion, and writes Arrow IPC after its queries, for its size only. Lance skips q72, which it cannot run; the query summary ranks a series +The read service charts both tables in the `File Sizes & Compression Ratios` group, one line per +format. It charts every current `compress-bench` dataset in `compression_sizes`, and three +`file_sizes` scopes, each summed over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 +(`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the +`compress-bench` datasets and on TPC-DS. It leaves out `duckdb`, DuckDB's database file, which +the group still charts. On develop, TPC-DS SF1 records the sizes of Lance, Arrow IPC, and Snappy +Parquet (`parquet-snappy`) beside Parquet and Vortex. It queries Lance on DataFusion, and writes +Arrow IPC and Snappy Parquet after its queries, for their sizes only. Lance skips q72, which it cannot run; the query summary ranks a series that misses a query below every series that ran them all. The `SQL File Sizes` group charts every `file_sizes` scope, at every scale factor, and leaves out diff --git a/docs/bench/architecture/read-path.md b/docs/bench/architecture/read-path.md index 65895507..f5fc6734 100644 --- a/docs/bench/architecture/read-path.md +++ b/docs/bench/architecture/read-path.md @@ -120,7 +120,7 @@ the v3 Axum server (so the frontend is generation-agnostic). The landing-page order is the curated `GROUP_ORDER` in `queries.ts`: ``` -Compression, File Sizes, SQL File Sizes, Random Access, Clickbench, +Compression, File Sizes & Compression Ratios, SQL File Sizes, Random Access, Clickbench, TPC-H (NVMe/S3) SF=1, SF=10, SF=100 (alternating), TPC-DS (NVMe) SF=1, Statistical and Population Genetics, Polar Signals Profiling, diff --git a/src/bench/components/SummaryCard.test.tsx b/src/bench/components/SummaryCard.test.tsx index 08c851c0..54511d95 100644 --- a/src/bench/components/SummaryCard.test.tsx +++ b/src/bench/components/SummaryCard.test.tsx @@ -175,7 +175,7 @@ describe('SummaryCard', () => { }); const html = render({ type: 'compressionSize', - title: 'File Sizes Summary', + title: 'File Sizes & Compression Ratios Summary', rankings: [ranking('parquet', 2), ranking('vortex-compact', 1)], explanation: 'e', }); diff --git a/src/bench/lib/compression-suite.ts b/src/bench/lib/compression-suite.ts index 66a60825..05dace6c 100644 --- a/src/bench/lib/compression-suite.ts +++ b/src/bench/lib/compression-suite.ts @@ -18,7 +18,7 @@ export const STRING_DATASET_PATTERN = `${STRING_DATASET_PREFIX}%`; * `LIKE` patterns for the `compress-bench` datasets the compression time summary leaves out: the * `wide table cols=… chunks=… rows=…` and `sparse-wide-*` suites. They are pathological cases * with many variants each, so in a geomean over all datasets they would be heavily - * over-represented. They still chart in the group. The File Sizes summary counts each suite as + * over-represented. They still chart in the group. The File Sizes & Compression Ratios summary counts each suite as * one dataset instead (`SIZE_SUMMARY_FAMILIES`). Both patterns are anchored at the start of the * name, as `compress-bench` writes it, so a dataset that only mentions them is not dropped. */ @@ -28,7 +28,7 @@ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * `LIKE` patterns for the `compress-bench` datasets the File Sizes summary leaves out. They still + * `LIKE` patterns for the `compress-bench` datasets the File Sizes & Compression Ratios summary leaves out. They still * chart in the group, and the compression time summary keeps them (minus * `SUMMARY_EXCLUDED_DATASET_PATTERNS`). * @@ -50,7 +50,7 @@ export const SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * Synthetic `compress-bench` families that the File Sizes summary counts as one dataset each: + * Synthetic `compress-bench` families that the File Sizes & Compression Ratios summary counts as one dataset each: * the geometric mean of the family's ratios is one entry in the summary's geometric mean. A * family has several datasets of one kind, so counted one by one they would outweigh the real * datasets. Each family is the datasets whose names start with its prefix. diff --git a/src/bench/lib/descriptions.test.ts b/src/bench/lib/descriptions.test.ts index 06d66a5f..501dd438 100644 --- a/src/bench/lib/descriptions.test.ts +++ b/src/bench/lib/descriptions.test.ts @@ -20,13 +20,13 @@ describe('groupDescription', () => { expect(groupDescription('Compression')).toBe( 'Measures write and scan throughput for Vortex, Parquet, and Lance files', ); - expect(groupDescription('File Sizes')).toBe( + expect(groupDescription('File Sizes & Compression Ratios')).toBe( 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS', ); expect(groupDescription('SQL File Sizes')).toBe( 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + - 'Vortex. Unlike File Sizes, it leaves out the compression datasets and does not rank ' + + 'Vortex. Unlike File Sizes & Compression Ratios, it leaves out the compression datasets and does not rank ' + 'formats against Parquet', ); expect(groupDescription('Clickbench')).toBe( diff --git a/src/bench/lib/descriptions.ts b/src/bench/lib/descriptions.ts index 5fa09581..f1b2281e 100644 --- a/src/bench/lib/descriptions.ts +++ b/src/bench/lib/descriptions.ts @@ -46,7 +46,7 @@ function staticDescription(name: string): string | null { return 'Tests performance of selecting arbitrary row indices from a file on S3 storage'; case 'Compression': return 'Measures write and scan throughput for Vortex, Parquet, and Lance files'; - case 'File Sizes': + case 'File Sizes & Compression Ratios': return ( 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS' @@ -54,8 +54,8 @@ function staticDescription(name: string): string | null { case 'SQL File Sizes': return ( 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + - 'Vortex. Unlike File Sizes, it leaves out the compression datasets and does not rank ' + - 'formats against Parquet' + 'Vortex. Unlike File Sizes & Compression Ratios, it leaves out the compression ' + + 'datasets and does not rank formats against Parquet' ); case 'String Encoding': return 'Measures Vortex file write and scan throughput for each string encoder'; diff --git a/src/bench/lib/file-sizes.ts b/src/bench/lib/file-sizes.ts index 302f3469..b52258c2 100644 --- a/src/bench/lib/file-sizes.ts +++ b/src/bench/lib/file-sizes.ts @@ -2,15 +2,15 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors /** - * Which charts the File Sizes group holds. It joins two tables: `compression_sizes`, the + * Which charts the File Sizes & Compression Ratios group holds. It joins two tables: `compression_sizes`, the * size of each `compress-bench` dataset in each format, and `file_sizes`, the files that the * SQL suites write. Both are one dataset's size per format per commit. */ import { compressionSuite } from './compression-suite'; -/** The File Sizes group's display name. */ -export const FILE_SIZES_GROUP_NAME = 'File Sizes'; +/** The File Sizes & Compression Ratios group's display name. */ +export const FILE_SIZES_GROUP_NAME = 'File Sizes & Compression Ratios'; /** * The SQL File Sizes group's display name. That group charts every SQL benchmark and scale factor diff --git a/src/bench/lib/group-charts.test.ts b/src/bench/lib/group-charts.test.ts index c8a6ead7..dd394856 100644 --- a/src/bench/lib/group-charts.test.ts +++ b/src/bench/lib/group-charts.test.ts @@ -584,7 +584,7 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () })), ], [ - // The File Sizes group mixes the two tables: `compression_sizes` charts, a NULL variant + // The File Sizes & Compression Ratios group mixes the two tables: `compression_sizes` charts, a NULL variant // between two non-NULL ones, and `file_sizes` charts. { k: 'CompressionSizeGroup' }, [ diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index edeb650a..e1f221f2 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -75,7 +75,7 @@ describe.skipIf(!databaseAvailable)( // Random-access sections precede Clickbench in GROUP_ORDER. expect(groups.map((g) => g.name)).toEqual([ 'Compression', - 'File Sizes', + 'File Sizes & Compression Ratios', 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', @@ -176,7 +176,7 @@ describe.skipIf(!databaseAvailable)( it('computes compression-size rankings for Vortex, Parquet, Lance, and Arrow IPC', async () => { const groups = await collectGroups(); const summary = expectDefined( - groups.find((g) => g.name === 'File Sizes')?.summary, + groups.find((g) => g.name === 'File Sizes & Compression Ratios')?.summary, 'compression size summary', ); if (summary.type !== 'compressionSize') { @@ -252,7 +252,7 @@ describe.skipIf(!databaseAvailable)( const body = (await res.json()) as GroupsResponse; expect(body.groups.map((g) => g.name)).toEqual([ 'Compression', - 'File Sizes', + 'File Sizes & Compression Ratios', 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', @@ -567,7 +567,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg const groups = await collectGroups(); expect(groups.map((group) => group.name)).toEqual([ 'Compression', - 'File Sizes', + 'File Sizes & Compression Ratios', 'String Encoding', 'String Encoding Size', ]); @@ -661,7 +661,10 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg } const groups = await collectGroups(); - expect(groups.map((group) => group.name)).toEqual(['File Sizes', 'SQL File Sizes']); + expect(groups.map((group) => group.name)).toEqual([ + 'File Sizes & Compression Ratios', + 'SQL File Sizes', + ]); const [group, sql] = groups; expect(groupKeyFromSlug(defined(group).slug)).toEqual({ k: 'CompressionSizeGroup' }); expect(defined(group).charts.map((chart) => chart.name)).toEqual([ @@ -679,7 +682,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg const summary = defined(group).summary; expect(summary).toMatchObject({ type: 'compressionSize', - title: 'File Sizes Summary', + title: 'File Sizes & Compression Ratios Summary', rankings: [ { name: 'vortex-compact', ratio: 0.5, totalBytes: 200, measured: 1, total: 1 }, { name: 'parquet', ratio: 1, totalBytes: 400, measured: 1, total: 1 }, diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index eb570c63..49bdf592 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -77,7 +77,8 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'Parquet has two variants, which differ only in their codec. Parquet is zstd at level ' + '3, and it is the baseline of the throughput and file size scores. Parquet (Snappy) ' + 'is the same Parquet with Snappy, the default codec of DuckDB, Spark, and PyArrow. ' + - 'Only the compression benchmark writes it, for its size. Both are written with ' + + 'Only the compression benchmark and TPC-DS at scale factor 1 write it, for its ' + + 'size. Both are written with ' + 'parquet-rs 59, except SpatialBench’s Parquet, which its generator writes with ' + 'parquet-rs 56.', 'Every SQL suite’s Parquet is zstd Parquet, except in ClickBench and FineWeb, which ' + @@ -155,7 +156,8 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'The compression summaries leave out the wide table and sparse-wide datasets, which ' + 'have many variants each. The file size summary also leaves out statpopgen, and counts ' + 'the TPC-H l_comment column once, because its canonical and chunked files are the ' + - 'same size. These datasets still have charts.', + 'same size. It does not rank DuckDB’s database file, which is a query engine’s ' + + 'storage, not a file format. These datasets and DuckDB’s file still have charts.', 'The string encoding benchmark compares its encoders with FSST instead of Parquet, so ' + 'its throughput and size scores divide by FSST’s.', 'Each summary uses each series’ newest result on develop.', diff --git a/src/bench/lib/queries.test.ts b/src/bench/lib/queries.test.ts index 74e7ac1e..9ae68cce 100644 --- a/src/bench/lib/queries.test.ts +++ b/src/bench/lib/queries.test.ts @@ -378,12 +378,12 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => { 'cohere-large-10m / partitioned', 'String Encoding', 'Random Access (NVMe)', - 'File Sizes', + 'File Sizes & Compression Ratios', 'Compression', ].sort(compareGroupSortKey); expect(sorted).toEqual([ 'Compression', - 'File Sizes', + 'File Sizes & Compression Ratios', 'Random Access (NVMe)', 'Clickbench', 'cohere-large-10m / partitioned', diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index 7ad1a888..4653af5f 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -1055,7 +1055,7 @@ function groupRowsQuery( break; } case 'CompressionSizeGroup': { - // The File Sizes group holds two families. Both give one size per format, so their blocks + // The File Sizes & Compression Ratios group holds two families. Both give one size per format, so their blocks // share columns. The String Encoding Size group holds `compression_sizes` charts alone. const sizes: [CompressionChartKey, number][] = []; const files: [FileSizeChartKey, number][] = []; @@ -1298,7 +1298,7 @@ export interface GroupChartsResponse { */ const GROUP_ORDER: readonly string[] = [ 'Compression', - 'File Sizes', + 'File Sizes & Compression Ratios', 'SQL File Sizes', 'Random Access (NVMe)', 'Random Access (S3)', @@ -1583,7 +1583,7 @@ type DatasetVariantRow = { dataset: string; dataset_variant: string | null }; /** * Group display names per compression suite and family. The size family's group without a suite - * is the File Sizes group, which [`collectFileSizesGroup`] builds, so the size family names only + * is the File Sizes & Compression Ratios group, which [`collectFileSizesGroup`] builds, so the size family names only * `string-bench`'s group here. */ const COMPRESSION_GROUP_NAMES = { @@ -1663,7 +1663,7 @@ async function compressionSizeDatasets(): Promise { } /** - * The File Sizes group and the `String Encoding Size` group, each left out when it has no + * The File Sizes & Compression Ratios group and the `String Encoding Size` group, each left out when it has no * charts. */ async function collectCompressionSizeGroup(): Promise { @@ -1750,7 +1750,7 @@ async function collectVectorSearchGroups(): Promise { type FileSizeScopeRow = { benchmark: string; scale_factor: string }; /** - * The File Sizes group, or `[]` if neither table has a chart for it: the SQL suites' scopes in + * The File Sizes & Compression Ratios group, or `[]` if neither table has a chart for it: the SQL suites' scopes in * [`FILE_SIZE_SCOPES`] that have rows, in that order, then the `compress-bench` datasets of * `datasets`, the rows of `compression_sizes`, by name. */ diff --git a/src/bench/lib/slug.ts b/src/bench/lib/slug.ts index 76436682..97be93a1 100644 --- a/src/bench/lib/slug.ts +++ b/src/bench/lib/slug.ts @@ -47,7 +47,7 @@ export type GroupKey = } // `suite` is absent for the `compress-bench` group, so its slug predates the split. | { k: 'CompressionTimeGroup'; suite?: 'strings' } - // Without `suite`, the File Sizes group: `compress-bench`'s sizes and three SQL scopes. + // Without `suite`, the File Sizes & Compression Ratios group: `compress-bench`'s sizes and three SQL scopes. | { k: 'CompressionSizeGroup'; suite?: 'strings' } | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string } diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index 64cefc02..545977e3 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -20,7 +20,7 @@ const GROUP_KINDS = FAMILIES.map((family) => family.groupKind); describe('compression summaries', () => { beforeEach(() => { query.mockReset(); - // The File Sizes summary also reads the SQL suites' sizes; a test that sets no rows for that + // The File Sizes & Compression Ratios summary also reads the SQL suites' sizes; a test that sets no rows for that // second query gets none. query.mockResolvedValue({ rows: [] }); }); @@ -85,7 +85,7 @@ describe('compression summaries', () => { // Both formats have both datasets, so each total sums both. expect(byFormat.get('vortex-file-compressed')?.totalBytes).toBe(901); expect(byFormat.get('parquet')?.totalBytes).toBe(104); - expect(summary.title).toBe('File Sizes Summary'); + expect(summary.title).toBe('File Sizes & Compression Ratios Summary'); expect(summary.explanation).toBe( 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', ); @@ -243,6 +243,27 @@ describe('compression summaries', () => { ]); }); + it("leaves DuckDB's database file out of the ranking", async () => { + const row = (format: string, dataset: string, valueBytes: number) => ({ + format, + dataset, + datasetVariant: dataset === 'tpcds' ? 'sf=1.0' : null, + valueBytes, + baselineBytes: 100, + uncompressedBytes: null, + }); + query.mockResolvedValueOnce({ rows: [row('parquet', 'taxi', 100)] }); + query.mockResolvedValueOnce({ + rows: [row('parquet', 'tpcds', 100), row('duckdb', 'tpcds', 20)], + }); + + const summary = await collectGroupSummary({ k: 'CompressionSizeGroup' }); + if (summary === null || summary.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + expect(summary.rankings.map((ranking) => ranking.name)).toEqual(['parquet']); + }); + it('gives no total when the formats share no dataset', async () => { query.mockResolvedValueOnce({ rows: [ @@ -418,7 +439,7 @@ describe('compression summaries', () => { await collectGroupSummary({ k: 'CompressionTimeGroup' }); await collectGroupSummary({ k: 'CompressionSizeGroup' }); - // The File Sizes summary reads the SQL suites' sizes in a third query. + // The File Sizes & Compression Ratios summary reads the SQL suites' sizes in a third query. expect(query).toHaveBeenCalledTimes(3); const timingParams = [ ['vortex-file-compressed', 'parquet', 'lance'], diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index 46abaa4d..c0d0eab2 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -729,7 +729,7 @@ const FILE_SIZE_BASELINE = 'vortex-file-compressed'; /** * The SQL File Sizes rollup: each format's total file size against `vortex-file-compressed`, per * benchmark scope (benchmark and scale factor) at that scope's newest commit with a baseline. - * Unlike the File Sizes summary, it covers every scope in `file_sizes` and leaves out + * Unlike the File Sizes & Compression Ratios summary, it covers every scope in `file_sizes` and leaves out * `compress-bench`. Scopes come from nightly as well as develop runs, so each one uses its own * newest commit. File sizes record no Arrow size, so the Arrow ratio is empty. */ @@ -811,7 +811,7 @@ async function collectFileSizeSummary(): Promise { } /** - * The `File Sizes` rollup: each format's sizes against Parquet's (zstd) over the + * The `File Sizes & Compression Ratios` rollup: each format's sizes against Parquet's (zstd) over the * `compress-bench` datasets ([`compressionSizeSamples`]) and the SQL scopes in * [`FILE_SIZE_SUMMARY_SCOPES`] ([`fileSizeSamples`]). Each synthetic family in * [`SIZE_SUMMARY_FAMILIES`] counts as one dataset. Only `compress-bench` records Arrow memory @@ -820,26 +820,36 @@ async function collectFileSizeSummary(): Promise { */ async function collectFileSizesSummary(): Promise { const [sizes, files] = await Promise.all([compressionSizeSamples(), fileSizeSamples()]); - const rankings = compressionSizeRankings([...sizes, ...files], 'complete'); + const rankings = compressionSizeRankings( + [...sizes, ...files].filter((sample) => !SIZE_SUMMARY_EXCLUDED_FORMATS.includes(sample.format)), + 'complete', + ); if (rankings.length === 0) { return null; } return { type: 'compressionSize', - title: 'File Sizes Summary', + title: 'File Sizes & Compression Ratios Summary', rankings, explanation: 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', }; } +/** + * Formats the File Sizes & Compression Ratios summary leaves out. They still chart in the group. + * DuckDB's database file (`duckdb`) is a query engine's storage, not a file format to compare, + * and only TPC-DS records it. + */ +const SIZE_SUMMARY_EXCLUDED_FORMATS: readonly string[] = ['duckdb']; + /** * Which datasets a size summary's totals cover: * - `shared`: the datasets that every ranked format has. Every format gets a total. * - `complete`: every dataset in the summary. Only a format measured on all of them gets a - * total. The File Sizes summary uses it: its SQL scopes record DuckDB and Vortex Compact, - * which `compress-bench` does not write, and `compress-bench` records Arrow IPC, which the SQL - * suites do not, so no dataset is shared by every format. + * total. The File Sizes & Compression Ratios summary uses it, because its two sources record + * formats at different times and some formats only on some datasets, so a format can lack a + * dataset that the others have. */ type SizeTotals = 'shared' | 'complete'; diff --git a/src/lib/bench-summary.ts b/src/lib/bench-summary.ts index 480842a3..5e130bb9 100644 --- a/src/lib/bench-summary.ts +++ b/src/lib/bench-summary.ts @@ -216,7 +216,7 @@ const SOURCES: (Source | Panel)[] = [ unit: "×", kind: "size", better: "lower", - group: "File Sizes", + group: "File Sizes & Compression Ratios", formerGroups: ["Compression Size"], rankings: (s) => s.rankings, name: (f) => f, diff --git a/tests/unit/bench-summary.test.ts b/tests/unit/bench-summary.test.ts index fb4c0a20..67a08001 100644 --- a/tests/unit/bench-summary.test.ts +++ b/tests/unit/bench-summary.test.ts @@ -342,9 +342,10 @@ describe("panelsFrom", () => { it("drops a score that is not a positive number", () => { const gs = groups(); - const parquet = group(gs, "File Sizes").summary.rankings?.find( - (r) => r.name === "parquet" - ); + const parquet = group( + gs, + "File Sizes & Compression Ratios" + ).summary.rankings?.find((r) => r.name === "parquet"); assert.ok(parquet); parquet.ratio = "1"; assert.deepEqual(panelsFrom(gs, charts())?.[6]?.rows[1], { @@ -358,9 +359,10 @@ describe("panelsFrom", () => { // summary has those two on part of its datasets. A size ratio is to // Parquet on the same dataset, so the card keeps it. const gs = groups(); - const lance = group(gs, "File Sizes").summary.rankings?.find( - (r) => r.name === "lance" - ); + const lance = group( + gs, + "File Sizes & Compression Ratios" + ).summary.rankings?.find((r) => r.name === "lance"); assert.ok(lance); lance.measured = 12; lance.total = 13; @@ -374,7 +376,7 @@ describe("panelsFrom", () => { // A deploy that renames a bench group builds against production bench, // which still serves the old name. const gs = groups(); - const sizes = group(gs, "File Sizes"); + const sizes = group(gs, "File Sizes & Compression Ratios"); sizes.name = "Compression Size"; assert.deepEqual(panelsFrom(gs, charts()), panelsFrom(groups(), charts())); }); @@ -382,7 +384,7 @@ describe("panelsFrom", () => { it("gives null when a group is missing, so the card never shows part of the table", () => { for (const name of [ "Compression", - "File Sizes", + "File Sizes & Compression Ratios", "Random Access (NVMe)", "Clickbench", "TPC-H (NVMe) (SF=10)" diff --git a/tests/unit/fixtures/bench-groups.json b/tests/unit/fixtures/bench-groups.json index bcc6f54c..b85bdc0f 100644 --- a/tests/unit/fixtures/bench-groups.json +++ b/tests/unit/fixtures/bench-groups.json @@ -48,11 +48,11 @@ } }, { - "name": "File Sizes", + "name": "File Sizes & Compression Ratios", "slug": "csg.eyJrIjoiQ29tcHJlc3Npb25TaXplR3JvdXAifQ", "summary": { "type": "compressionSize", - "title": "File Sizes Summary", + "title": "File Sizes & Compression Ratios Summary", "rankings": [ { "name": "parquet", From 3021f81aa52cf0a167211ee144b1372dd948701f Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 15:42:11 -0500 Subject: [PATCH 11/23] Rename the size group to Compression & File Size The new name keeps more continuity with the Compression Size group it replaces. Signed-off-by: Will Manning --- docs/bench/CONTRACT.md | 2 +- docs/bench/architecture/read-path.md | 2 +- src/bench/components/SummaryCard.test.tsx | 2 +- src/bench/lib/compression-suite.ts | 6 +++--- src/bench/lib/descriptions.test.ts | 4 ++-- src/bench/lib/descriptions.ts | 4 ++-- src/bench/lib/file-sizes.ts | 6 +++--- src/bench/lib/group-charts.test.ts | 2 +- src/bench/lib/groups.test.ts | 12 ++++++------ src/bench/lib/queries.test.ts | 4 ++-- src/bench/lib/queries.ts | 10 +++++----- src/bench/lib/slug.ts | 2 +- src/bench/lib/summary.test.ts | 6 +++--- src/bench/lib/summary.ts | 10 +++++----- src/lib/bench-summary.ts | 2 +- tests/unit/bench-summary.test.ts | 18 ++++++++---------- tests/unit/fixtures/bench-groups.json | 4 ++-- 17 files changed, 47 insertions(+), 49 deletions(-) diff --git a/docs/bench/CONTRACT.md b/docs/bench/CONTRACT.md index b022b187..9a943d1b 100644 --- a/docs/bench/CONTRACT.md +++ b/docs/bench/CONTRACT.md @@ -85,7 +85,7 @@ codec. `parquet` is the suite's base Parquet: zstd level 3 where the benchmarks dataset's published files where the suite prescribes them, as ClickBench's are Snappy. CI reads the records back as the develop baseline for the PR comment's file-size section. -The read service charts both tables in the `File Sizes & Compression Ratios` group, one line per +The read service charts both tables in the `Compression & File Size` group, one line per format. It charts every current `compress-bench` dataset in `compression_sizes`, and three `file_sizes` scopes, each summed over its files: ClickBench, TPC-H at SF10, and TPC-DS at SF1 (`src/bench/lib/file-sizes.ts`). Its summary compares each format with zstd Parquet on the diff --git a/docs/bench/architecture/read-path.md b/docs/bench/architecture/read-path.md index f5fc6734..eef4a306 100644 --- a/docs/bench/architecture/read-path.md +++ b/docs/bench/architecture/read-path.md @@ -120,7 +120,7 @@ the v3 Axum server (so the frontend is generation-agnostic). The landing-page order is the curated `GROUP_ORDER` in `queries.ts`: ``` -Compression, File Sizes & Compression Ratios, SQL File Sizes, Random Access, Clickbench, +Compression, Compression & File Size, SQL File Sizes, Random Access, Clickbench, TPC-H (NVMe/S3) SF=1, SF=10, SF=100 (alternating), TPC-DS (NVMe) SF=1, Statistical and Population Genetics, Polar Signals Profiling, diff --git a/src/bench/components/SummaryCard.test.tsx b/src/bench/components/SummaryCard.test.tsx index 54511d95..671a6687 100644 --- a/src/bench/components/SummaryCard.test.tsx +++ b/src/bench/components/SummaryCard.test.tsx @@ -175,7 +175,7 @@ describe('SummaryCard', () => { }); const html = render({ type: 'compressionSize', - title: 'File Sizes & Compression Ratios Summary', + title: 'Compression & File Size Summary', rankings: [ranking('parquet', 2), ranking('vortex-compact', 1)], explanation: 'e', }); diff --git a/src/bench/lib/compression-suite.ts b/src/bench/lib/compression-suite.ts index 05dace6c..c842bee6 100644 --- a/src/bench/lib/compression-suite.ts +++ b/src/bench/lib/compression-suite.ts @@ -18,7 +18,7 @@ export const STRING_DATASET_PATTERN = `${STRING_DATASET_PREFIX}%`; * `LIKE` patterns for the `compress-bench` datasets the compression time summary leaves out: the * `wide table cols=… chunks=… rows=…` and `sparse-wide-*` suites. They are pathological cases * with many variants each, so in a geomean over all datasets they would be heavily - * over-represented. They still chart in the group. The File Sizes & Compression Ratios summary counts each suite as + * over-represented. They still chart in the group. The Compression & File Size summary counts each suite as * one dataset instead (`SIZE_SUMMARY_FAMILIES`). Both patterns are anchored at the start of the * name, as `compress-bench` writes it, so a dataset that only mentions them is not dropped. */ @@ -28,7 +28,7 @@ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * `LIKE` patterns for the `compress-bench` datasets the File Sizes & Compression Ratios summary leaves out. They still + * `LIKE` patterns for the `compress-bench` datasets the Compression & File Size summary leaves out. They still * chart in the group, and the compression time summary keeps them (minus * `SUMMARY_EXCLUDED_DATASET_PATTERNS`). * @@ -50,7 +50,7 @@ export const SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * Synthetic `compress-bench` families that the File Sizes & Compression Ratios summary counts as one dataset each: + * Synthetic `compress-bench` families that the Compression & File Size summary counts as one dataset each: * the geometric mean of the family's ratios is one entry in the summary's geometric mean. A * family has several datasets of one kind, so counted one by one they would outweigh the real * datasets. Each family is the datasets whose names start with its prefix. diff --git a/src/bench/lib/descriptions.test.ts b/src/bench/lib/descriptions.test.ts index 501dd438..a2de9568 100644 --- a/src/bench/lib/descriptions.test.ts +++ b/src/bench/lib/descriptions.test.ts @@ -20,13 +20,13 @@ describe('groupDescription', () => { expect(groupDescription('Compression')).toBe( 'Measures write and scan throughput for Vortex, Parquet, and Lance files', ); - expect(groupDescription('File Sizes & Compression Ratios')).toBe( + expect(groupDescription('Compression & File Size')).toBe( 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS', ); expect(groupDescription('SQL File Sizes')).toBe( 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + - 'Vortex. Unlike File Sizes & Compression Ratios, it leaves out the compression datasets and does not rank ' + + 'Vortex. Unlike Compression & File Size, it leaves out the compression datasets and does not rank ' + 'formats against Parquet', ); expect(groupDescription('Clickbench')).toBe( diff --git a/src/bench/lib/descriptions.ts b/src/bench/lib/descriptions.ts index f1b2281e..9dcc1e6e 100644 --- a/src/bench/lib/descriptions.ts +++ b/src/bench/lib/descriptions.ts @@ -46,7 +46,7 @@ function staticDescription(name: string): string | null { return 'Tests performance of selecting arbitrary row indices from a file on S3 storage'; case 'Compression': return 'Measures write and scan throughput for Vortex, Parquet, and Lance files'; - case 'File Sizes & Compression Ratios': + case 'Compression & File Size': return ( 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS' @@ -54,7 +54,7 @@ function staticDescription(name: string): string | null { case 'SQL File Sizes': return ( 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + - 'Vortex. Unlike File Sizes & Compression Ratios, it leaves out the compression ' + + 'Vortex. Unlike Compression & File Size, it leaves out the compression ' + 'datasets and does not rank formats against Parquet' ); case 'String Encoding': diff --git a/src/bench/lib/file-sizes.ts b/src/bench/lib/file-sizes.ts index b52258c2..be666f8e 100644 --- a/src/bench/lib/file-sizes.ts +++ b/src/bench/lib/file-sizes.ts @@ -2,15 +2,15 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors /** - * Which charts the File Sizes & Compression Ratios group holds. It joins two tables: `compression_sizes`, the + * Which charts the Compression & File Size group holds. It joins two tables: `compression_sizes`, the * size of each `compress-bench` dataset in each format, and `file_sizes`, the files that the * SQL suites write. Both are one dataset's size per format per commit. */ import { compressionSuite } from './compression-suite'; -/** The File Sizes & Compression Ratios group's display name. */ -export const FILE_SIZES_GROUP_NAME = 'File Sizes & Compression Ratios'; +/** The Compression & File Size group's display name. */ +export const FILE_SIZES_GROUP_NAME = 'Compression & File Size'; /** * The SQL File Sizes group's display name. That group charts every SQL benchmark and scale factor diff --git a/src/bench/lib/group-charts.test.ts b/src/bench/lib/group-charts.test.ts index dd394856..c5025f2f 100644 --- a/src/bench/lib/group-charts.test.ts +++ b/src/bench/lib/group-charts.test.ts @@ -584,7 +584,7 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () })), ], [ - // The File Sizes & Compression Ratios group mixes the two tables: `compression_sizes` charts, a NULL variant + // The Compression & File Size group mixes the two tables: `compression_sizes` charts, a NULL variant // between two non-NULL ones, and `file_sizes` charts. { k: 'CompressionSizeGroup' }, [ diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index e1f221f2..6ae94fb7 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -75,7 +75,7 @@ describe.skipIf(!databaseAvailable)( // Random-access sections precede Clickbench in GROUP_ORDER. expect(groups.map((g) => g.name)).toEqual([ 'Compression', - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', @@ -176,7 +176,7 @@ describe.skipIf(!databaseAvailable)( it('computes compression-size rankings for Vortex, Parquet, Lance, and Arrow IPC', async () => { const groups = await collectGroups(); const summary = expectDefined( - groups.find((g) => g.name === 'File Sizes & Compression Ratios')?.summary, + groups.find((g) => g.name === 'Compression & File Size')?.summary, 'compression size summary', ); if (summary.type !== 'compressionSize') { @@ -252,7 +252,7 @@ describe.skipIf(!databaseAvailable)( const body = (await res.json()) as GroupsResponse; expect(body.groups.map((g) => g.name)).toEqual([ 'Compression', - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'Random Access (NVMe)', 'TPC-H (NVMe) (SF=1)', 'cohere-large-10m / partitioned', @@ -567,7 +567,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg const groups = await collectGroups(); expect(groups.map((group) => group.name)).toEqual([ 'Compression', - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'String Encoding', 'String Encoding Size', ]); @@ -662,7 +662,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg const groups = await collectGroups(); expect(groups.map((group) => group.name)).toEqual([ - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'SQL File Sizes', ]); const [group, sql] = groups; @@ -682,7 +682,7 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg const summary = defined(group).summary; expect(summary).toMatchObject({ type: 'compressionSize', - title: 'File Sizes & Compression Ratios Summary', + title: 'Compression & File Size Summary', rankings: [ { name: 'vortex-compact', ratio: 0.5, totalBytes: 200, measured: 1, total: 1 }, { name: 'parquet', ratio: 1, totalBytes: 400, measured: 1, total: 1 }, diff --git a/src/bench/lib/queries.test.ts b/src/bench/lib/queries.test.ts index 9ae68cce..62aec078 100644 --- a/src/bench/lib/queries.test.ts +++ b/src/bench/lib/queries.test.ts @@ -378,12 +378,12 @@ describe('compareGroupSortKey canonical group ordering (no DB)', () => { 'cohere-large-10m / partitioned', 'String Encoding', 'Random Access (NVMe)', - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'Compression', ].sort(compareGroupSortKey); expect(sorted).toEqual([ 'Compression', - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'Random Access (NVMe)', 'Clickbench', 'cohere-large-10m / partitioned', diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index 4653af5f..663b1bc0 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -1055,7 +1055,7 @@ function groupRowsQuery( break; } case 'CompressionSizeGroup': { - // The File Sizes & Compression Ratios group holds two families. Both give one size per format, so their blocks + // The Compression & File Size group holds two families. Both give one size per format, so their blocks // share columns. The String Encoding Size group holds `compression_sizes` charts alone. const sizes: [CompressionChartKey, number][] = []; const files: [FileSizeChartKey, number][] = []; @@ -1298,7 +1298,7 @@ export interface GroupChartsResponse { */ const GROUP_ORDER: readonly string[] = [ 'Compression', - 'File Sizes & Compression Ratios', + 'Compression & File Size', 'SQL File Sizes', 'Random Access (NVMe)', 'Random Access (S3)', @@ -1583,7 +1583,7 @@ type DatasetVariantRow = { dataset: string; dataset_variant: string | null }; /** * Group display names per compression suite and family. The size family's group without a suite - * is the File Sizes & Compression Ratios group, which [`collectFileSizesGroup`] builds, so the size family names only + * is the Compression & File Size group, which [`collectFileSizesGroup`] builds, so the size family names only * `string-bench`'s group here. */ const COMPRESSION_GROUP_NAMES = { @@ -1663,7 +1663,7 @@ async function compressionSizeDatasets(): Promise { } /** - * The File Sizes & Compression Ratios group and the `String Encoding Size` group, each left out when it has no + * The Compression & File Size group and the `String Encoding Size` group, each left out when it has no * charts. */ async function collectCompressionSizeGroup(): Promise { @@ -1750,7 +1750,7 @@ async function collectVectorSearchGroups(): Promise { type FileSizeScopeRow = { benchmark: string; scale_factor: string }; /** - * The File Sizes & Compression Ratios group, or `[]` if neither table has a chart for it: the SQL suites' scopes in + * The Compression & File Size group, or `[]` if neither table has a chart for it: the SQL suites' scopes in * [`FILE_SIZE_SCOPES`] that have rows, in that order, then the `compress-bench` datasets of * `datasets`, the rows of `compression_sizes`, by name. */ diff --git a/src/bench/lib/slug.ts b/src/bench/lib/slug.ts index 97be93a1..43381c36 100644 --- a/src/bench/lib/slug.ts +++ b/src/bench/lib/slug.ts @@ -47,7 +47,7 @@ export type GroupKey = } // `suite` is absent for the `compress-bench` group, so its slug predates the split. | { k: 'CompressionTimeGroup'; suite?: 'strings' } - // Without `suite`, the File Sizes & Compression Ratios group: `compress-bench`'s sizes and three SQL scopes. + // Without `suite`, the Compression & File Size group: `compress-bench`'s sizes and three SQL scopes. | { k: 'CompressionSizeGroup'; suite?: 'strings' } | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string } diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index 545977e3..f6e1c090 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -20,7 +20,7 @@ const GROUP_KINDS = FAMILIES.map((family) => family.groupKind); describe('compression summaries', () => { beforeEach(() => { query.mockReset(); - // The File Sizes & Compression Ratios summary also reads the SQL suites' sizes; a test that sets no rows for that + // The Compression & File Size summary also reads the SQL suites' sizes; a test that sets no rows for that // second query gets none. query.mockResolvedValue({ rows: [] }); }); @@ -85,7 +85,7 @@ describe('compression summaries', () => { // Both formats have both datasets, so each total sums both. expect(byFormat.get('vortex-file-compressed')?.totalBytes).toBe(901); expect(byFormat.get('parquet')?.totalBytes).toBe(104); - expect(summary.title).toBe('File Sizes & Compression Ratios Summary'); + expect(summary.title).toBe('Compression & File Size Summary'); expect(summary.explanation).toBe( 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', ); @@ -439,7 +439,7 @@ describe('compression summaries', () => { await collectGroupSummary({ k: 'CompressionTimeGroup' }); await collectGroupSummary({ k: 'CompressionSizeGroup' }); - // The File Sizes & Compression Ratios summary reads the SQL suites' sizes in a third query. + // The Compression & File Size summary reads the SQL suites' sizes in a third query. expect(query).toHaveBeenCalledTimes(3); const timingParams = [ ['vortex-file-compressed', 'parquet', 'lance'], diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index c0d0eab2..8bda10e3 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -729,7 +729,7 @@ const FILE_SIZE_BASELINE = 'vortex-file-compressed'; /** * The SQL File Sizes rollup: each format's total file size against `vortex-file-compressed`, per * benchmark scope (benchmark and scale factor) at that scope's newest commit with a baseline. - * Unlike the File Sizes & Compression Ratios summary, it covers every scope in `file_sizes` and leaves out + * Unlike the Compression & File Size summary, it covers every scope in `file_sizes` and leaves out * `compress-bench`. Scopes come from nightly as well as develop runs, so each one uses its own * newest commit. File sizes record no Arrow size, so the Arrow ratio is empty. */ @@ -811,7 +811,7 @@ async function collectFileSizeSummary(): Promise { } /** - * The `File Sizes & Compression Ratios` rollup: each format's sizes against Parquet's (zstd) over the + * The `Compression & File Size` rollup: each format's sizes against Parquet's (zstd) over the * `compress-bench` datasets ([`compressionSizeSamples`]) and the SQL scopes in * [`FILE_SIZE_SUMMARY_SCOPES`] ([`fileSizeSamples`]). Each synthetic family in * [`SIZE_SUMMARY_FAMILIES`] counts as one dataset. Only `compress-bench` records Arrow memory @@ -829,7 +829,7 @@ async function collectFileSizesSummary(): Promise { } return { type: 'compressionSize', - title: 'File Sizes & Compression Ratios Summary', + title: 'Compression & File Size Summary', rankings, explanation: 'Total file size, and geometric means of file sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', @@ -837,7 +837,7 @@ async function collectFileSizesSummary(): Promise { } /** - * Formats the File Sizes & Compression Ratios summary leaves out. They still chart in the group. + * Formats the Compression & File Size summary leaves out. They still chart in the group. * DuckDB's database file (`duckdb`) is a query engine's storage, not a file format to compare, * and only TPC-DS records it. */ @@ -847,7 +847,7 @@ const SIZE_SUMMARY_EXCLUDED_FORMATS: readonly string[] = ['duckdb']; * Which datasets a size summary's totals cover: * - `shared`: the datasets that every ranked format has. Every format gets a total. * - `complete`: every dataset in the summary. Only a format measured on all of them gets a - * total. The File Sizes & Compression Ratios summary uses it, because its two sources record + * total. The Compression & File Size summary uses it, because its two sources record * formats at different times and some formats only on some datasets, so a format can lack a * dataset that the others have. */ diff --git a/src/lib/bench-summary.ts b/src/lib/bench-summary.ts index 5e130bb9..d14f8be8 100644 --- a/src/lib/bench-summary.ts +++ b/src/lib/bench-summary.ts @@ -216,7 +216,7 @@ const SOURCES: (Source | Panel)[] = [ unit: "×", kind: "size", better: "lower", - group: "File Sizes & Compression Ratios", + group: "Compression & File Size", formerGroups: ["Compression Size"], rankings: (s) => s.rankings, name: (f) => f, diff --git a/tests/unit/bench-summary.test.ts b/tests/unit/bench-summary.test.ts index 67a08001..a17b65f8 100644 --- a/tests/unit/bench-summary.test.ts +++ b/tests/unit/bench-summary.test.ts @@ -342,10 +342,9 @@ describe("panelsFrom", () => { it("drops a score that is not a positive number", () => { const gs = groups(); - const parquet = group( - gs, - "File Sizes & Compression Ratios" - ).summary.rankings?.find((r) => r.name === "parquet"); + const parquet = group(gs, "Compression & File Size").summary.rankings?.find( + (r) => r.name === "parquet" + ); assert.ok(parquet); parquet.ratio = "1"; assert.deepEqual(panelsFrom(gs, charts())?.[6]?.rows[1], { @@ -359,10 +358,9 @@ describe("panelsFrom", () => { // summary has those two on part of its datasets. A size ratio is to // Parquet on the same dataset, so the card keeps it. const gs = groups(); - const lance = group( - gs, - "File Sizes & Compression Ratios" - ).summary.rankings?.find((r) => r.name === "lance"); + const lance = group(gs, "Compression & File Size").summary.rankings?.find( + (r) => r.name === "lance" + ); assert.ok(lance); lance.measured = 12; lance.total = 13; @@ -376,7 +374,7 @@ describe("panelsFrom", () => { // A deploy that renames a bench group builds against production bench, // which still serves the old name. const gs = groups(); - const sizes = group(gs, "File Sizes & Compression Ratios"); + const sizes = group(gs, "Compression & File Size"); sizes.name = "Compression Size"; assert.deepEqual(panelsFrom(gs, charts()), panelsFrom(groups(), charts())); }); @@ -384,7 +382,7 @@ describe("panelsFrom", () => { it("gives null when a group is missing, so the card never shows part of the table", () => { for (const name of [ "Compression", - "File Sizes & Compression Ratios", + "Compression & File Size", "Random Access (NVMe)", "Clickbench", "TPC-H (NVMe) (SF=10)" diff --git a/tests/unit/fixtures/bench-groups.json b/tests/unit/fixtures/bench-groups.json index b85bdc0f..634b9487 100644 --- a/tests/unit/fixtures/bench-groups.json +++ b/tests/unit/fixtures/bench-groups.json @@ -48,11 +48,11 @@ } }, { - "name": "File Sizes & Compression Ratios", + "name": "Compression & File Size", "slug": "csg.eyJrIjoiQ29tcHJlc3Npb25TaXplR3JvdXAifQ", "summary": { "type": "compressionSize", - "title": "File Sizes & Compression Ratios Summary", + "title": "Compression & File Size Summary", "rankings": [ { "name": "parquet", From 49ac480059719c4287262ab330167fc495bac626 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 15:43:34 -0500 Subject: [PATCH 12/23] Keep the renamed size groups' anchors, and describe the size ranking Old #compression-size and #sql-file-size links land on the renamed groups. The group tooltip says which scopes it charts and which it ranks. The methodology page explains the Compression & File Size summary's totals and how it counts the wide-table suites. Signed-off-by: Will Manning --- src/bench/lib/anchor.test.ts | 2 ++ src/bench/lib/anchor.ts | 7 +++++-- src/bench/lib/compression-suite.ts | 18 +++++++++--------- src/bench/lib/descriptions.test.ts | 9 +++++---- src/bench/lib/descriptions.ts | 5 +++-- src/bench/lib/file-sizes.ts | 4 ++-- src/bench/lib/group-charts.test.ts | 4 ++-- src/bench/lib/methodology.ts | 11 ++++++++--- src/bench/lib/queries.ts | 19 ++++++++++--------- src/bench/lib/slug.ts | 3 ++- src/bench/lib/summary.test.ts | 4 ++-- 11 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/bench/lib/anchor.test.ts b/src/bench/lib/anchor.test.ts index 73c91e17..a91d291d 100644 --- a/src/bench/lib/anchor.test.ts +++ b/src/bench/lib/anchor.test.ts @@ -39,6 +39,8 @@ describe('groupAnchors', () => { it('keeps the anchor a renamed group had, so shared links still land on it', () => { expect(groupAnchor('Polar Signals Profiling')).toBe('polarsignals-profiling'); + expect(groupAnchor('Compression & File Size')).toBe('compression-size'); + expect(groupAnchor('SQL File Sizes')).toBe('sql-file-size'); }); it('keeps the full production group list collision-free', () => { diff --git a/src/bench/lib/anchor.ts b/src/bench/lib/anchor.ts index f9fd9c37..e9995972 100644 --- a/src/bench/lib/anchor.ts +++ b/src/bench/lib/anchor.ts @@ -13,11 +13,14 @@ /** * Anchors that stay as they were when a group's name changed, so links that - * readers already shared still land on the group. The group was named - * "PolarSignals Profiling" until its owners' spelling replaced it. + * readers already shared still land on the group. The groups were named + * "PolarSignals Profiling" (until its owners' spelling replaced it), + * "Compression Size", and "SQL File Size". */ const KEPT_ANCHORS: Readonly> = { 'Polar Signals Profiling': 'polarsignals-profiling', + 'Compression & File Size': 'compression-size', + 'SQL File Sizes': 'sql-file-size', }; /** diff --git a/src/bench/lib/compression-suite.ts b/src/bench/lib/compression-suite.ts index c842bee6..e5e40977 100644 --- a/src/bench/lib/compression-suite.ts +++ b/src/bench/lib/compression-suite.ts @@ -18,9 +18,9 @@ export const STRING_DATASET_PATTERN = `${STRING_DATASET_PREFIX}%`; * `LIKE` patterns for the `compress-bench` datasets the compression time summary leaves out: the * `wide table cols=… chunks=… rows=…` and `sparse-wide-*` suites. They are pathological cases * with many variants each, so in a geomean over all datasets they would be heavily - * over-represented. They still chart in the group. The Compression & File Size summary counts each suite as - * one dataset instead (`SIZE_SUMMARY_FAMILIES`). Both patterns are anchored at the start of the - * name, as `compress-bench` writes it, so a dataset that only mentions them is not dropped. + * over-represented. They still chart in the group. The Compression & File Size summary counts each + * suite as one dataset instead (`SIZE_SUMMARY_FAMILIES`). Both patterns are anchored at the start + * of the name, as `compress-bench` writes it, so a dataset that only mentions them is not dropped. */ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ 'wide table %', @@ -28,8 +28,8 @@ export const SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * `LIKE` patterns for the `compress-bench` datasets the Compression & File Size summary leaves out. They still - * chart in the group, and the compression time summary keeps them (minus + * `LIKE` patterns for the `compress-bench` datasets the Compression & File Size summary leaves out. + * They still chart in the group, and the compression time summary keeps them (minus * `SUMMARY_EXCLUDED_DATASET_PATTERNS`). * * - `tpc-h l_comment chunked` writes the same column as `tpc-h l_comment canonical`, so their @@ -50,10 +50,10 @@ export const SIZE_SUMMARY_EXCLUDED_DATASET_PATTERNS: readonly string[] = [ ]; /** - * Synthetic `compress-bench` families that the Compression & File Size summary counts as one dataset each: - * the geometric mean of the family's ratios is one entry in the summary's geometric mean. A - * family has several datasets of one kind, so counted one by one they would outweigh the real - * datasets. Each family is the datasets whose names start with its prefix. + * Synthetic `compress-bench` families that the Compression & File Size summary counts as one + * dataset each: the geometric mean of the family's ratios is one entry in the summary's geometric + * mean. A family has several datasets of one kind, so counted one by one they would outweigh the + * real datasets. Each family is the datasets whose names start with its prefix. */ export const SIZE_SUMMARY_FAMILIES: readonly { name: string; prefix: string }[] = [ { name: 'wide table', prefix: 'wide table ' }, diff --git a/src/bench/lib/descriptions.test.ts b/src/bench/lib/descriptions.test.ts index a2de9568..f4075024 100644 --- a/src/bench/lib/descriptions.test.ts +++ b/src/bench/lib/descriptions.test.ts @@ -21,13 +21,14 @@ describe('groupDescription', () => { 'Measures write and scan throughput for Vortex, Parquet, and Lance files', ); expect(groupDescription('Compression & File Size')).toBe( - 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + - 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS', + 'Compares file sizes and compression ratios across formats against zstd Parquet. It ' + + 'charts the compression datasets and one scale factor each of ClickBench, TPC-H, and ' + + 'TPC-DS, and ranks formats on the compression datasets and TPC-DS', ); expect(groupDescription('SQL File Sizes')).toBe( 'Tracks the size of the files each SQL benchmark writes, at every scale factor, against ' + - 'Vortex. Unlike Compression & File Size, it leaves out the compression datasets and does not rank ' + - 'formats against Parquet', + 'Vortex. Unlike Compression & File Size, it leaves out the compression datasets and does ' + + 'not rank formats against Parquet', ); expect(groupDescription('Clickbench')).toBe( "ClickHouse's analytical benchmark suite testing real-world query patterns on web " + diff --git a/src/bench/lib/descriptions.ts b/src/bench/lib/descriptions.ts index 9dcc1e6e..a4466ad9 100644 --- a/src/bench/lib/descriptions.ts +++ b/src/bench/lib/descriptions.ts @@ -48,8 +48,9 @@ function staticDescription(name: string): string | null { return 'Measures write and scan throughput for Vortex, Parquet, and Lance files'; case 'Compression & File Size': return ( - 'Compares file sizes and compression ratios across formats against zstd Parquet, on the ' + - 'compression datasets and on one scale factor each of ClickBench, TPC-H, and TPC-DS' + 'Compares file sizes and compression ratios across formats against zstd Parquet. It ' + + 'charts the compression datasets and one scale factor each of ClickBench, TPC-H, and ' + + 'TPC-DS, and ranks formats on the compression datasets and TPC-DS' ); case 'SQL File Sizes': return ( diff --git a/src/bench/lib/file-sizes.ts b/src/bench/lib/file-sizes.ts index be666f8e..c617117c 100644 --- a/src/bench/lib/file-sizes.ts +++ b/src/bench/lib/file-sizes.ts @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors /** - * Which charts the Compression & File Size group holds. It joins two tables: `compression_sizes`, the - * size of each `compress-bench` dataset in each format, and `file_sizes`, the files that the + * Which charts the Compression & File Size group holds. It joins two tables: `compression_sizes`, + * the size of each `compress-bench` dataset in each format, and `file_sizes`, the files that the * SQL suites write. Both are one dataset's size per format per commit. */ diff --git a/src/bench/lib/group-charts.test.ts b/src/bench/lib/group-charts.test.ts index c5025f2f..fb1a67fd 100644 --- a/src/bench/lib/group-charts.test.ts +++ b/src/bench/lib/group-charts.test.ts @@ -584,8 +584,8 @@ describe.skipIf(!databaseAvailable)('groupChartPayloads at the window edges', () })), ], [ - // The Compression & File Size group mixes the two tables: `compression_sizes` charts, a NULL variant - // between two non-NULL ones, and `file_sizes` charts. + // The Compression & File Size group mixes the two tables: `compression_sizes` charts, a NULL + // variant between two non-NULL ones, and `file_sizes` charts. { k: 'CompressionSizeGroup' }, [ { k: 'FileSize', benchmark: 'tpcds', scale_factor: '1.0' }, diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 49bdf592..12cd8d75 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -152,9 +152,14 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'on TPC-DS at scale factor 1.', `Write and scan throughput scores are ${SCORES.throughput}`, `File size scores are ${SCORES.size}`, - 'Total is the sum of a format’s file sizes over the datasets that every format has.', - 'The compression summaries leave out the wide table and sparse-wide datasets, which ' + - 'have many variants each. The file size summary also leaves out statpopgen, and counts ' + + 'Total is the sum of a format’s file sizes over the datasets that every format has. ' + + 'The Compression & File Size summary instead gives a total only to a format that has ' + + 'every dataset in the summary, and ranks a format that lacks a dataset after the ' + + 'formats that have them all.', + 'The compression time summary leaves out the wide table and sparse-wide datasets, ' + + 'which have many variants each. The Compression & File Size summary counts each of ' + + 'those two suites as one dataset, the geometric mean of its ratios. It also leaves ' + + 'out statpopgen, and counts ' + 'the TPC-H l_comment column once, because its canonical and chunked files are the ' + 'same size. It does not rank DuckDB’s database file, which is a query engine’s ' + 'storage, not a file format. These datasets and DuckDB’s file still have charts.', diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index 663b1bc0..975edcff 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -1055,8 +1055,9 @@ function groupRowsQuery( break; } case 'CompressionSizeGroup': { - // The Compression & File Size group holds two families. Both give one size per format, so their blocks - // share columns. The String Encoding Size group holds `compression_sizes` charts alone. + // The Compression & File Size group holds two families. Both give one size per format, so + // their blocks share columns. The String Encoding Size group holds `compression_sizes` charts + // alone. const sizes: [CompressionChartKey, number][] = []; const files: [FileSizeChartKey, number][] = []; charts.forEach((key, i) => { @@ -1583,8 +1584,8 @@ type DatasetVariantRow = { dataset: string; dataset_variant: string | null }; /** * Group display names per compression suite and family. The size family's group without a suite - * is the Compression & File Size group, which [`collectFileSizesGroup`] builds, so the size family names only - * `string-bench`'s group here. + * is the Compression & File Size group, which [`collectFileSizesGroup`] builds, so the size family + * names only `string-bench`'s group here. */ const COMPRESSION_GROUP_NAMES = { CompressionTimeGroup: { compress: 'Compression', strings: 'String Encoding' }, @@ -1663,8 +1664,8 @@ async function compressionSizeDatasets(): Promise { } /** - * The Compression & File Size group and the `String Encoding Size` group, each left out when it has no - * charts. + * The Compression & File Size group and the `String Encoding Size` group, each left out when it has + * no charts. */ async function collectCompressionSizeGroup(): Promise { const datasets = await compressionSizeDatasets(); @@ -1750,9 +1751,9 @@ async function collectVectorSearchGroups(): Promise { type FileSizeScopeRow = { benchmark: string; scale_factor: string }; /** - * The Compression & File Size group, or `[]` if neither table has a chart for it: the SQL suites' scopes in - * [`FILE_SIZE_SCOPES`] that have rows, in that order, then the `compress-bench` datasets of - * `datasets`, the rows of `compression_sizes`, by name. + * The Compression & File Size group, or `[]` if neither table has a chart for it: the SQL suites' + * scopes in [`FILE_SIZE_SCOPES`] that have rows, in that order, then the `compress-bench` datasets + * of `datasets`, the rows of `compression_sizes`, by name. */ async function collectFileSizesGroup(datasets: readonly DatasetVariantRow[]): Promise { // One index probe of `idx_file_sizes_baseline` per listed scope. diff --git a/src/bench/lib/slug.ts b/src/bench/lib/slug.ts index 43381c36..b0eaf207 100644 --- a/src/bench/lib/slug.ts +++ b/src/bench/lib/slug.ts @@ -47,7 +47,8 @@ export type GroupKey = } // `suite` is absent for the `compress-bench` group, so its slug predates the split. | { k: 'CompressionTimeGroup'; suite?: 'strings' } - // Without `suite`, the Compression & File Size group: `compress-bench`'s sizes and three SQL scopes. + // Without `suite`, the Compression & File Size group: `compress-bench`'s sizes and three SQL + // scopes. | { k: 'CompressionSizeGroup'; suite?: 'strings' } | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string } diff --git a/src/bench/lib/summary.test.ts b/src/bench/lib/summary.test.ts index f6e1c090..df2a3568 100644 --- a/src/bench/lib/summary.test.ts +++ b/src/bench/lib/summary.test.ts @@ -20,8 +20,8 @@ const GROUP_KINDS = FAMILIES.map((family) => family.groupKind); describe('compression summaries', () => { beforeEach(() => { query.mockReset(); - // The Compression & File Size summary also reads the SQL suites' sizes; a test that sets no rows for that - // second query gets none. + // The Compression & File Size summary also reads the SQL suites' sizes; a test that sets no + // rows for that second query gets none. query.mockResolvedValue({ rows: [] }); }); From bc7eae74e70fdd45dcc3d4344cb0b05be14fc4d1 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 16:27:55 -0500 Subject: [PATCH 13/23] Label zstd Parquet as parquet-zstd in the Compression & File Size group The group charts Snappy Parquet beside zstd Parquet, so its legends, tooltips, toolbar chips, and summary card name the zstd files parquet-zstd, and the card's baseline header reads "vs parquet-zstd". ClickBench's chart keeps parquet, because it charts the published Snappy files. The series keys stay parquet, so filters, colors, and URLs are unchanged. A compression chart's permalink page uses the same label; a file-size permalink page does not, because the SQL File Sizes group shares those charts. The group payload cache moves to v17. Signed-off-by: Will Manning --- src/app/(bench)/bench/chart/[slug]/page.tsx | 14 +++++++++-- src/bench/components/Chart.tsx | 13 ++++++++++- src/bench/components/GroupBody.tsx | 8 +++++-- src/bench/components/GroupSection.tsx | 3 ++- src/bench/components/GroupToolbar.tsx | 26 ++++++++++++++++++--- src/bench/components/SummaryCard.test.tsx | 26 +++++++++++++++++++-- src/bench/components/SummaryCard.tsx | 21 +++++++++++++---- src/bench/components/chart/config.ts | 5 ++-- src/bench/components/chart/controller.ts | 8 ++++++- src/bench/components/chart/tooltip.ts | 2 +- src/bench/components/chart/types.ts | 4 +++- src/bench/lib/chart-format.test.ts | 21 +++++++++++++++++ src/bench/lib/chart-format.ts | 26 +++++++++++++++++---- src/bench/lib/data-cache.test.ts | 8 +++---- src/bench/lib/data-cache.ts | 2 +- src/bench/lib/groups.test.ts | 7 ++++++ src/bench/lib/queries.ts | 26 ++++++++++++++++++--- src/bench/lib/summary.test.ts | 2 +- src/bench/lib/summary.ts | 2 +- 19 files changed, 188 insertions(+), 36 deletions(-) diff --git a/src/app/(bench)/bench/chart/[slug]/page.tsx b/src/app/(bench)/bench/chart/[slug]/page.tsx index 165bd954..39f2648b 100644 --- a/src/app/(bench)/bench/chart/[slug]/page.tsx +++ b/src/app/(bench)/bench/chart/[slug]/page.tsx @@ -14,6 +14,7 @@ import { chartUnitLabel, parseFilterCsv, singleSearchParam, + ZSTD_PARQUET_LABELS, } from '@/bench/lib/chart-format'; import { cachedDefaultChartPayload, cachedFilterUniverse } from '@/bench/lib/data-cache'; import { NOT_FOUND_METADATA } from '@/bench/lib/not-found-metadata'; @@ -176,8 +177,9 @@ export default async function ChartPage({ // well-formed slugs the chart payload and the filter universe are // independent queries run in parallel (getChart is React-cache'd, so the // generateMetadata dedupe is unaffected), matching the landing page. + let key: ChartKey; try { - chartKeyFromSlug(slug); + key = chartKeyFromSlug(slug); } catch { notFound(); } @@ -211,7 +213,15 @@ export default async function ChartPage({ {' · '} {seriesCount} series · {commitCount} commit{commitCount !== 1 ? 's' : ''}

- + diff --git a/src/bench/components/Chart.tsx b/src/bench/components/Chart.tsx index 575e8d50..b63a17a6 100644 --- a/src/bench/components/Chart.tsx +++ b/src/bench/components/Chart.tsx @@ -12,6 +12,7 @@ import { chartAriaLabel, chartPriority, LAZY_HYDRATION_ROOT_MARGIN, + type SeriesLabels, throttle, ZOOM_THROTTLE_MS, } from '@/bench/lib/chart-format'; @@ -75,13 +76,22 @@ export interface ChartIslandProps { groupSlug?: string; /** Server-fetched payload on the permalink page (skips the initial fetch). */ initialPayload?: ChartResponse; + /** Display names the chart gives some of its formats (see `SeriesLabels`). */ + seriesLabels?: SeriesLabels | undefined; } /** * One interactive chart card. Renders the full card chrome (the v3 * `chart_card` markup) and drives Chart.js through a [`ChartController`]. */ -export function Chart({ slug, name, index, groupSlug, initialPayload }: ChartIslandProps) { +export function Chart({ + slug, + name, + index, + groupSlug, + initialPayload, + seriesLabels, +}: ChartIslandProps) { const base = useBenchBase(); const cardRef = useRef(null); const canvasRef = useRef(null); @@ -192,6 +202,7 @@ export function Chart({ slug, name, index, groupSlug, initialPayload }: ChartIsl stripWindow: stripWindowRef.current, }), { setY, setLoading, setError, setRetryable, setConstructed, setLatestValues }, + seriesLabels, ); controllerRef.current = controller; if (initialPayload) { diff --git a/src/bench/components/GroupBody.tsx b/src/bench/components/GroupBody.tsx index ea2d69b2..816ef994 100644 --- a/src/bench/components/GroupBody.tsx +++ b/src/bench/components/GroupBody.tsx @@ -8,7 +8,7 @@ import { useEffect, useLayoutEffect, useRef, useState } from 'react'; import { useBenchBase } from '@/bench/components/BenchBase'; import { Chart } from '@/bench/components/Chart'; import { GroupToolbar } from '@/bench/components/GroupToolbar'; -import { chartPriority, type FilterUniverse } from '@/bench/lib/chart-format'; +import { chartPriority, type FilterUniverse, type SeriesLabels } from '@/bench/lib/chart-format'; import { abortGroupBundle, ensureGroupBundle } from '@/bench/lib/chart-store'; import { noteGroupBodyClosed, @@ -30,6 +30,8 @@ export interface GroupBodyProps { universe: FilterUniverse; /** The filter choices the page URL gives this group (`?group=…&hide=…`). */ initialFilter: GroupFilterChoices; + /** Display names the toolbar gives some formats (see `SeriesLabels`). */ + seriesLabels?: SeriesLabels | undefined; } /** @@ -58,6 +60,7 @@ export function GroupBody({ startIndex, universe, initialFilter, + seriesLabels, }: GroupBodyProps) { const base = useBenchBase(); const ref = useRef(null); @@ -120,7 +123,7 @@ export function GroupBody({
{open && ( <> - +
{charts.map((link, i) => ( ))}
diff --git a/src/bench/components/GroupSection.tsx b/src/bench/components/GroupSection.tsx index 9acdbf49..3feb79e6 100644 --- a/src/bench/components/GroupSection.tsx +++ b/src/bench/components/GroupSection.tsx @@ -130,13 +130,14 @@ export function GroupSection({
- + ); diff --git a/src/bench/components/GroupToolbar.tsx b/src/bench/components/GroupToolbar.tsx index fc3334b9..3bfbf3c2 100644 --- a/src/bench/components/GroupToolbar.tsx +++ b/src/bench/components/GroupToolbar.tsx @@ -6,7 +6,7 @@ import { useCallback, useEffect, useId, useRef, useState, useSyncExternalStore } from 'react'; import { FilterIcon } from '@/bench/components/icons'; -import { DEFAULT_VISIBLE, type FilterUniverse } from '@/bench/lib/chart-format'; +import { DEFAULT_VISIBLE, type FilterUniverse, type SeriesLabels } from '@/bench/lib/chart-format'; import { applyGroupMacro, clearGroupFilterRow, @@ -51,9 +51,12 @@ import { returnFocusToTrigger } from '@/bench/lib/panel-focus'; export function GroupToolbar({ groupSlug, universe, + seriesLabels, }: { groupSlug: string; universe: FilterUniverse; + /** Display names the chips give some formats (see `SeriesLabels`). */ + seriesLabels?: SeriesLabels | undefined; }) { const [open, setOpen] = useState(false); const panelId = useId(); @@ -179,6 +182,7 @@ export function GroupToolbar({ groupSlug={groupSlug} snapshot={snapshot} globalSnapshot={globalSnapshot} + seriesLabels={seriesLabels} />
Series @@ -207,7 +211,7 @@ export function GroupToolbar({ aria-pressed={isActive} onClick={() => toggleGroupSeries(groupSlug, label)} > - {label} + {relabel(label, seriesLabels)} ); })} @@ -262,6 +266,7 @@ function MacroRow({ groupSlug, snapshot, globalSnapshot, + seriesLabels, }: { label: string; dim: 'engine' | 'format'; @@ -269,6 +274,7 @@ function MacroRow({ groupSlug: string; snapshot: ReturnType; globalSnapshot: ReturnType; + seriesLabels?: SeriesLabels | undefined; }) { return (
@@ -295,10 +301,24 @@ function MacroRow({ aria-pressed={isActive} onClick={() => applyGroupMacro(groupSlug, dim, value)} > - {value} + {relabel(value, seriesLabels)} ); })}
); } + +/** + * A chip's text: the series label with each segment the group renames (see `SeriesLabels`) + * replaced. The chips otherwise show the real labels, so this is not `displaySeriesLabel`. + */ +function relabel(label: string, labels: SeriesLabels | undefined): string { + if (labels === undefined) { + return label; + } + return label + .split(':') + .map((segment) => labels[segment] ?? segment) + .join(':'); +} diff --git a/src/bench/components/SummaryCard.test.tsx b/src/bench/components/SummaryCard.test.tsx index 671a6687..65e1265e 100644 --- a/src/bench/components/SummaryCard.test.tsx +++ b/src/bench/components/SummaryCard.test.tsx @@ -5,13 +5,35 @@ import { renderToStaticMarkup } from 'react-dom/server'; import { describe, expect, it } from 'vitest'; import { SummaryCard } from '@/bench/components/SummaryCard'; +import { type SeriesLabels, ZSTD_PARQUET_LABELS } from '@/bench/lib/chart-format'; import type { Summary } from '@/bench/lib/summary'; -function render(summary?: Summary): string { - return renderToStaticMarkup(); +function render(summary?: Summary, seriesLabels?: SeriesLabels): string { + return renderToStaticMarkup(); } describe('SummaryCard', () => { + it('names zstd Parquet as the group labels it', () => { + const ranking = { + name: 'parquet', + minRatio: 1, + ratio: 1, + maxRatio: 1, + compressionRatio: 4.5, + totalBytes: 100, + measured: 1, + total: 1, + }; + const html = render( + { type: 'compressionSize', title: 't', rankings: [ranking], explanation: 'e' }, + ZSTD_PARQUET_LABELS, + ); + expect(html).toContain('>vs parquet-zstd<'); + expect(html).toContain('>parquet-zstd<'); + expect(html).toContain('parquet-zstd: minimum'); + expect(html).not.toContain('>parquet<'); + }); + it('renders nothing when there is no summary', () => { expect(render(undefined)).toBe(''); }); diff --git a/src/bench/components/SummaryCard.tsx b/src/bench/components/SummaryCard.tsx index 56797912..1d96a4f6 100644 --- a/src/bench/components/SummaryCard.tsx +++ b/src/bench/components/SummaryCard.tsx @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -import { displayFormat, displaySeriesLabel } from '@/bench/lib/chart-format'; +import { displayFormat, displaySeriesLabel, type SeriesLabels } from '@/bench/lib/chart-format'; import { byteUnit, formatBytesIn, @@ -61,9 +61,18 @@ function runtimeText(item: SeriesRanking): string { * its `
`, so it shows whether the group is open or collapsed. It is * server-rendered, so it shows before hydration and with JavaScript off too. * + * `seriesLabels` are the group's display names for some formats (see `SeriesLabels`). The + * compression-size panels use them for their rows and for the Parquet baseline's header. + * * Returns `null` when there is no summary or the variant has no content. */ -export function SummaryCard({ summary }: { summary?: Summary | undefined }) { +export function SummaryCard({ + summary, + seriesLabels, +}: { + summary?: Summary | undefined; + seriesLabels?: SeriesLabels | undefined; +}) { if (summary === undefined) { return null; } @@ -143,7 +152,7 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
{summary.rankings.map((item, idx) => { - const label = displayFormat(item.name); + const label = displayFormat(item.name, seriesLabels); // A format measured on only some datasets has its means over those alone, so // the row says how many, and it ranks after the complete ones. const partial = item.measured < item.total; @@ -189,7 +198,9 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
- vs {summary.baseline ?? 'Parquet'} + + vs {summary.baseline ?? seriesLabels?.parquet ?? 'Parquet'} + Min GeoMean Max @@ -205,7 +216,7 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) { key={item.name} > - {`${displayFormat(item.name)}: minimum ${minRatio}, geometric mean ${meanRatio}, maximum ${maxRatio}`} + {`${displayFormat(item.name, seriesLabels)}: minimum ${minRatio}, geometric mean ${meanRatio}, maximum ${maxRatio}`}