From 7fbe334515a4119701d5e7dcac2d531d16cac059 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 10:09:57 -0500 Subject: [PATCH 1/4] Call each Vortex scan vortex on bench, not vortex-file-compressed develop now runs Vortex's other scans as formats such as vortex-file-compressed [file, v1]. The display rename matched only the bare format, so those series showed the legacy name. They now read vortex [file, v1] on charts and summaries, and Vortex (file, v1) in llms.txt. Signed-off-by: Will Manning --- src/bench/lib/chart-format.test.ts | 8 ++++++++ src/bench/lib/chart-format.ts | 15 ++++++++++++++- src/bench/lib/site-files.test.ts | 3 +++ src/bench/lib/site-files.ts | 23 ++++++++++++++++++----- 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/bench/lib/chart-format.test.ts b/src/bench/lib/chart-format.test.ts index 18c4d7d2..b51ad451 100644 --- a/src/bench/lib/chart-format.test.ts +++ b/src/bench/lib/chart-format.test.ts @@ -529,6 +529,14 @@ describe('displayFormat / displaySeriesLabel', () => { expect(displayFormat('parquet')).toBe('parquet'); }); + it('renames vortex-file-compressed in the name of a Vortex scan', () => { + expect(displayFormat('vortex-file-compressed [file, v1]')).toBe('vortex [file, v1]'); + expect(displaySeriesLabel('datafusion:vortex-file-compressed [datasource, v1]')).toBe( + 'datafusion:vortex [datasource, v1]', + ); + expect(displayFormat('vortex-file-compressed-x')).toBe('vortex-file-compressed-x'); + }); + it('rewrites the vortex token in any colon-delimited label', () => { expect(displaySeriesLabel('datafusion:vortex-file-compressed')).toBe('datafusion:vortex'); expect(displaySeriesLabel('duckdb:vortex-file-compressed')).toBe('duckdb:vortex'); diff --git a/src/bench/lib/chart-format.ts b/src/bench/lib/chart-format.ts index 07020635..80f3aceb 100644 --- a/src/bench/lib/chart-format.ts +++ b/src/bench/lib/chart-format.ts @@ -240,15 +240,28 @@ export function seriesOrder( // Small formatting helpers. // --------------------------------------------------------------------------- +/** Vortex's on-disk format as the data stores it. */ +const VORTEX_FORMAT = 'vortex-file-compressed'; + /** * The user-facing display name for a format string. Vortex's on-disk format is * stored as `vortex-file-compressed` everywhere in the data (series keys, filter * allowlists, `series_meta`), but it is shown to users as plain `vortex`. This is * a presentation-only rename: callers pass the real format to the data layer and * route it through here ONLY when rendering. All other formats pass through. + * + * A Vortex scan other than the suite's default is the format followed by its + * integration and scan engine, as `vortex-file-compressed [file, v1]`, which reads + * `vortex [file, v1]`. */ export function displayFormat(format: string): string { - return format === 'vortex-file-compressed' ? 'vortex' : format; + if (format === VORTEX_FORMAT) { + return 'vortex'; + } + if (format.startsWith(`${VORTEX_FORMAT} [`)) { + return `vortex${format.slice(VORTEX_FORMAT.length)}`; + } + return format; } /** diff --git a/src/bench/lib/site-files.test.ts b/src/bench/lib/site-files.test.ts index cfb6e131..46f569f6 100644 --- a/src/bench/lib/site-files.test.ts +++ b/src/bench/lib/site-files.test.ts @@ -82,6 +82,9 @@ describe('seriesLabel', () => { expect(seriesLabel('duckdb:duckdb')).toBe("DuckDB on DuckDB's own format"); expect(seriesLabel('arrow-ipc')).toBe('Arrow IPC'); expect(seriesLabel('onpair-12')).toBe('onpair-12'); + expect(seriesLabel(`datafusion:${VX} [datasource, v1]`)).toBe( + 'DataFusion on Vortex (datasource, v1)', + ); }); }); diff --git a/src/bench/lib/site-files.ts b/src/bench/lib/site-files.ts index 8a5dd95b..f1286d36 100644 --- a/src/bench/lib/site-files.ts +++ b/src/bench/lib/site-files.ts @@ -198,6 +198,19 @@ const FORMATS: ReadonlyMap = new Map([ ['fsst', 'FSST'], ]); +/** + * A format in words. A Vortex scan other than the suite's default, as + * `vortex-file-compressed [file, v1]`, reads "Vortex (file, v1)". A name this + * table does not know stays as bench spells it. + */ +function formatLabel(format: string): string { + const scan = /^(.+) \[(.+)\]$/.exec(format); + if (scan?.[1] !== undefined && scan[2] !== undefined) { + return `${FORMATS.get(scan[1]) ?? scan[1]} (${scan[2]})`; + } + return FORMATS.get(format) ?? format; +} + /** * A series in words: `engine:format` as "DataFusion on Parquet", a format alone * by its name. A name this table does not know stays as bench spells it. @@ -205,9 +218,9 @@ const FORMATS: ReadonlyMap = new Map([ export function seriesLabel(name: string): string { const [first = '', second] = name.split(':'); if (second === undefined) { - return FORMATS.get(first) ?? first; + return formatLabel(first); } - return `${ENGINES.get(first) ?? first} on ${FORMATS.get(second) ?? second}`; + return `${ENGINES.get(first) ?? first} on ${formatLabel(second)}`; } const OPERATIONS: ReadonlyMap = new Map([ @@ -226,13 +239,13 @@ export function chartSeriesLabel(kind: ChartKind, name: string): string { case 'QueryMeasurement': return seriesLabel(name); case 'CompressionTime': - return `${FORMATS.get(format) ?? format} ${OPERATIONS.get(detail) ?? detail}`; + return `${formatLabel(format)} ${OPERATIONS.get(detail) ?? detail}`; case 'RandomAccess': - return `${FORMATS.get(format) ?? format} (${detail})`; + return `${formatLabel(format)} (${detail})`; case 'CompressionSize': case 'VectorSearch': case 'FileSize': - return FORMATS.get(name) ?? name; + return formatLabel(name); } } From c1064ef2d6631cf5257c0f787e4c807f1e55eeee Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 10:30:01 -0500 Subject: [PATCH 2/4] Show only each suite's default Vortex scan on bench The SQL benchmarks record Vortex's other scans as the format followed by the integration and scan engine, as `vortex-file-compressed [file, v1]`. bench drew each as its own series and ranked it in the summary cards. The pull request reports compare those scans with their develop baseline in bench's database, so the rows stay there, but bench now leaves them out of the charts, the summaries, and the Format filter. This replaces the previous commit, which only renamed them. Signed-off-by: Will Manning --- src/bench/lib/chart-format.test.ts | 12 ++++++------ src/bench/lib/chart-format.ts | 27 +++++++++++++-------------- src/bench/lib/groups.test.ts | 27 +++++++++++++++++++++++++++ src/bench/lib/methodology.ts | 2 ++ src/bench/lib/queries.ts | 15 ++++++++++++--- src/bench/lib/site-files.test.ts | 3 --- src/bench/lib/site-files.ts | 23 +++++------------------ src/bench/lib/summary.ts | 6 +++++- 8 files changed, 70 insertions(+), 45 deletions(-) diff --git a/src/bench/lib/chart-format.test.ts b/src/bench/lib/chart-format.test.ts index b51ad451..9e588872 100644 --- a/src/bench/lib/chart-format.test.ts +++ b/src/bench/lib/chart-format.test.ts @@ -17,6 +17,7 @@ import { displayFormat, displaySeriesLabel, escapeHtml, + isOtherScan, FETCH_TIMEOUT_MS, firstLine, formatDisplayValue, @@ -529,12 +530,11 @@ describe('displayFormat / displaySeriesLabel', () => { expect(displayFormat('parquet')).toBe('parquet'); }); - it('renames vortex-file-compressed in the name of a Vortex scan', () => { - expect(displayFormat('vortex-file-compressed [file, v1]')).toBe('vortex [file, v1]'); - expect(displaySeriesLabel('datafusion:vortex-file-compressed [datasource, v1]')).toBe( - 'datafusion:vortex [datasource, v1]', - ); - expect(displayFormat('vortex-file-compressed-x')).toBe('vortex-file-compressed-x'); + it('tells the other Vortex scans from the formats bench shows', () => { + expect(isOtherScan('vortex-file-compressed [file, v1]')).toBe(true); + expect(isOtherScan('vortex-file-compressed [datasource, v1]')).toBe(true); + expect(isOtherScan('vortex-file-compressed')).toBe(false); + expect(isOtherScan('parquet-snappy')).toBe(false); }); it('rewrites the vortex token in any colon-delimited label', () => { diff --git a/src/bench/lib/chart-format.ts b/src/bench/lib/chart-format.ts index 80f3aceb..4101c8d8 100644 --- a/src/bench/lib/chart-format.ts +++ b/src/bench/lib/chart-format.ts @@ -240,28 +240,27 @@ export function seriesOrder( // Small formatting helpers. // --------------------------------------------------------------------------- -/** Vortex's on-disk format as the data stores it. */ -const VORTEX_FORMAT = 'vortex-file-compressed'; - /** * The user-facing display name for a format string. Vortex's on-disk format is * stored as `vortex-file-compressed` everywhere in the data (series keys, filter * allowlists, `series_meta`), but it is shown to users as plain `vortex`. This is * a presentation-only rename: callers pass the real format to the data layer and * route it through here ONLY when rendering. All other formats pass through. - * - * A Vortex scan other than the suite's default is the format followed by its - * integration and scan engine, as `vortex-file-compressed [file, v1]`, which reads - * `vortex [file, v1]`. */ export function displayFormat(format: string): string { - if (format === VORTEX_FORMAT) { - return 'vortex'; - } - if (format.startsWith(`${VORTEX_FORMAT} [`)) { - return `vortex${format.slice(VORTEX_FORMAT.length)}`; - } - return format; + return format === 'vortex-file-compressed' ? 'vortex' : format; +} + +/** + * Whether `format` names one of Vortex's other scans. The SQL benchmarks run + * each suite's default scan under the plain format, and every other scan under + * the format followed by its integration and scan engine, as + * `vortex-file-compressed [datasource, v1]`. The pull request reports compare + * every scan with its develop baseline in bench's database, so those rows stay + * there, but bench shows only the default scan, as `vortex`. + */ +export function isOtherScan(format: string): boolean { + return / \[[^\]]+\]$/.test(format); } /** diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index 7d3784cd..29325546 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -16,6 +16,7 @@ vi.mock('next/cache', () => ({ })); import { + collectFilterUniverse, collectGroup, collectGroupCharts, collectGroups, @@ -998,6 +999,32 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(defined(summary.rankings[0]).ratio).toBeCloseTo(2.0, 6); }); + it('shows only the default Vortex scan, not the scans the PR reports compare', async () => { + const sha = 'f'.repeat(40); + await insertCommit(sha, '2026-04-23T12:00:00Z'); + await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed', 100_000); + await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed [file, v1]', 50_000); + await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed [datasource, v1]', 60_000); + await insertQuery(sha, 1, 'datafusion', 'parquet', 200_000); + + const key = { + k: 'QueryGroup', + dataset: 'tpch', + dataset_variant: null, + scale_factor: '1', + storage: 'nvme', + } as const; + const shown = ['datafusion:parquet', 'datafusion:vortex-file-compressed']; + const summary = await collectGroupSummary(key); + if (summary?.type !== 'queryBenchmark') { + throw new Error(`expected queryBenchmark summary, got ${summary?.type}`); + } + expect(summary.rankings.map((r) => r.name).sort()).toEqual(shown); + const bundle = await collectGroupCharts(key, parseCommitWindow(null)); + expect(Object.keys(defined(bundle?.charts[0]).series).sort()).toEqual(shown); + expect((await collectFilterUniverse()).formats).toEqual(['parquet', 'vortex-file-compressed']); + }); + it('applies the 300_000 penalty floor so a missing query flips the ranking', async () => { // All present runtimes are below the 300_000 ns floor, so the missing-query // penalty is max(maxRuntime, 300_000) * 2 = 600_000 for both series (the diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 86497577..f7063e3d 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -71,6 +71,8 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'format, and, for random access, LitData. Not every suite runs every format. The ' + 'compression and file size benchmarks write Parquet with parquet-rs 59 and zstd at ' + 'level 3. ClickBench reads its published Parquet files.', + 'Vortex has more than one way to scan a file in each query engine. bench shows each ' + + 'suite’s default scan. The benchmark reports on pull requests also compare the others.', ], }, { diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index de6d0a3c..b8873525 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -42,7 +42,7 @@ import { STRING_DATASET_PREFIX, } from './compression-suite'; import { groupDescription } from './descriptions'; -import type { FilterUniverse } from './chart-format'; +import { type FilterUniverse, isOtherScan } from './chart-format'; import { type ChartKind, compareCodeUnits, FAMILIES, type GroupKind } from './families'; import { chartKeyFromSlug, chartKeyToSlug, groupKeyFromSlug, groupKeyToSlug } from './slug'; import type { ChartKey, GroupKey } from './slug'; @@ -402,13 +402,19 @@ export function seriesFor( } } -/** Thread one fact row onto the seeded x-axis. A row off the seeded commits is dropped. */ +/** + * Thread one fact row onto the seeded x-axis. A row off the seeded commits is dropped, and so + * is a row of one of Vortex's other scans (`isOtherScan`). + */ function recordRow(acc: SeriesAccumulator, kind: ChartKind, row: FactRow): void { const idx = acc.commitIdx(row.commit_sha); if (idx === undefined) { return; } const { key, engine, format } = seriesFor(kind, row); + if (format !== undefined && isOtherScan(format)) { + return; + } acc.record(key, idx, row.value); acc.tag(key, engine, format); } @@ -1973,6 +1979,9 @@ export async function collectFilterUniverse(): Promise { ]); return { engines: engines.rows.map((r) => r.value).sort(compareCodeUnits), - formats: formats.rows.map((r) => r.value).sort(compareCodeUnits), + formats: formats.rows + .map((r) => r.value) + .filter((format) => !isOtherScan(format)) + .sort(compareCodeUnits), }; } diff --git a/src/bench/lib/site-files.test.ts b/src/bench/lib/site-files.test.ts index 46f569f6..cfb6e131 100644 --- a/src/bench/lib/site-files.test.ts +++ b/src/bench/lib/site-files.test.ts @@ -82,9 +82,6 @@ describe('seriesLabel', () => { expect(seriesLabel('duckdb:duckdb')).toBe("DuckDB on DuckDB's own format"); expect(seriesLabel('arrow-ipc')).toBe('Arrow IPC'); expect(seriesLabel('onpair-12')).toBe('onpair-12'); - expect(seriesLabel(`datafusion:${VX} [datasource, v1]`)).toBe( - 'DataFusion on Vortex (datasource, v1)', - ); }); }); diff --git a/src/bench/lib/site-files.ts b/src/bench/lib/site-files.ts index f1286d36..8a5dd95b 100644 --- a/src/bench/lib/site-files.ts +++ b/src/bench/lib/site-files.ts @@ -198,19 +198,6 @@ const FORMATS: ReadonlyMap = new Map([ ['fsst', 'FSST'], ]); -/** - * A format in words. A Vortex scan other than the suite's default, as - * `vortex-file-compressed [file, v1]`, reads "Vortex (file, v1)". A name this - * table does not know stays as bench spells it. - */ -function formatLabel(format: string): string { - const scan = /^(.+) \[(.+)\]$/.exec(format); - if (scan?.[1] !== undefined && scan[2] !== undefined) { - return `${FORMATS.get(scan[1]) ?? scan[1]} (${scan[2]})`; - } - return FORMATS.get(format) ?? format; -} - /** * A series in words: `engine:format` as "DataFusion on Parquet", a format alone * by its name. A name this table does not know stays as bench spells it. @@ -218,9 +205,9 @@ function formatLabel(format: string): string { export function seriesLabel(name: string): string { const [first = '', second] = name.split(':'); if (second === undefined) { - return formatLabel(first); + return FORMATS.get(first) ?? first; } - return `${ENGINES.get(first) ?? first} on ${formatLabel(second)}`; + return `${ENGINES.get(first) ?? first} on ${FORMATS.get(second) ?? second}`; } const OPERATIONS: ReadonlyMap = new Map([ @@ -239,13 +226,13 @@ export function chartSeriesLabel(kind: ChartKind, name: string): string { case 'QueryMeasurement': return seriesLabel(name); case 'CompressionTime': - return `${formatLabel(format)} ${OPERATIONS.get(detail) ?? detail}`; + return `${FORMATS.get(format) ?? format} ${OPERATIONS.get(detail) ?? detail}`; case 'RandomAccess': - return `${formatLabel(format)} (${detail})`; + return `${FORMATS.get(format) ?? format} (${detail})`; case 'CompressionSize': case 'VectorSearch': case 'FileSize': - return formatLabel(name); + return FORMATS.get(name) ?? name; } } diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index 43edd969..db42ff55 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -37,6 +37,7 @@ */ import { BRANCH_COMMITS, factOnReadBranch, onReadBranch } from './branch'; +import { isOtherScan } from './chart-format'; import { getPool } from './db'; import { randomAccessStorage, type RandomAccessStorage } from './random-access'; import { compareCodeUnits } from './families'; @@ -1284,8 +1285,11 @@ async function collectQuerySummary( await getPool().query<{ query_idx: number; series: string; value_ns: number }>(text, params) ).rows; + // `series` is `engine:format`, and no engine name holds a colon. const rankings = rankSeries( - rows.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })), + rows + .filter((row) => !isOtherScan(row.series.slice(row.series.indexOf(':') + 1))) + .map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })), (a, b) => a - b, QUERY_PENALTY_FLOOR_NS, 2, From 4d0311848a1c3ee438271b95fa51abce40968e8c Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 10:44:14 -0500 Subject: [PATCH 3/4] Key bench's cached reads to a new version so the scan filter takes effect The Data Cache outlives a deploy. With the same keys, a new deployment kept serving the group summaries, chart payloads, and filter universe that the old code computed, so the other Vortex scans still showed. The chart payload and filter universe keys now carry the version too. Signed-off-by: Will Manning --- src/bench/lib/data-cache.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/bench/lib/data-cache.ts b/src/bench/lib/data-cache.ts index f5fa5ebc..cfc6d3da 100644 --- a/src/bench/lib/data-cache.ts +++ b/src/bench/lib/data-cache.ts @@ -50,14 +50,16 @@ export const DATA_CACHE_BACKSTOP_SECONDS = 86400; const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_SECONDS }; /** - * Version of cached group payloads, including their embedded summary wire shape. + * Version of every cached read below: group payloads with their embedded + * summaries, chart payloads, and the filter universe. * This value must change when a deployment cannot read the preceding shape, and * when the same data would give a different payload, as after a change to a - * summary query. The Data Cache outlives a deploy, so without a new key the new - * code can serve payloads the old code computed until the next ingest. + * summary query or to the series a chart shows. The Data Cache outlives a + * deploy, so without a new key the new code can serve payloads the old code + * computed until the next ingest. * It is independent of the producer-facing benchmark schema version. */ -const GROUP_PAYLOAD_CACHE_VERSION = 'v14'; +const GROUP_PAYLOAD_CACHE_VERSION = 'v15'; // The default last-100 group bundle, keyed by group slug. The slug is the cache // key (an `unstable_cache` argument), so one wrapper covers every group. A @@ -78,7 +80,7 @@ const groupChartsCached = unstable_cache( const chartPayloadCached = unstable_cache( async (slug: string): Promise => chartPayload(chartKeyFromSlug(slug), { kind: 'last', n: DEFAULT_COMMIT_WINDOW }), - ['data-cache:chart-payload:n100'], + [`data-cache:chart-payload:${GROUP_PAYLOAD_CACHE_VERSION}:n100`], CACHE_OPTIONS, ); @@ -93,7 +95,7 @@ const groupsCached = unstable_cache( const filterUniverseCached = unstable_cache( async (): Promise => collectFilterUniverse(), - ['data-cache:filter-universe'], + [`data-cache:filter-universe:${GROUP_PAYLOAD_CACHE_VERSION}`], CACHE_OPTIONS, ); From f09216d97b6d23a17a5b626ddcb1af1142d67129 Mon Sep 17 00:00:00 2001 From: Will Manning Date: Sat, 21 Nov 2026 10:44:43 -0500 Subject: [PATCH 4/4] Pin bench's v15 cache keys in the data cache test Signed-off-by: Will Manning --- src/bench/lib/data-cache.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/bench/lib/data-cache.test.ts b/src/bench/lib/data-cache.test.ts index 884865fd..42f624d2 100644 --- a/src/bench/lib/data-cache.test.ts +++ b/src/bench/lib/data-cache.test.ts @@ -68,10 +68,12 @@ describe('data-cache wrappers', () => { it('tags every wrapper with the shared bench-data tag and the backstop TTL', () => { expect(BENCH_DATA_TAG).toBe('bench-data'); expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([ - 'data-cache:group-charts:v14:n100', + expect(cacheCalls.map((call) => call.keyParts)).toEqual([ + ['data-cache:group-charts:v15:n100'], + ['data-cache:chart-payload:v15:n100'], + ['data-cache:groups:v15'], + ['data-cache:filter-universe:v15'], ]); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v14']); for (const call of cacheCalls) { expect(call.options.tags).toEqual([BENCH_DATA_TAG]); expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS);