diff --git a/src/bench/lib/chart-format.test.ts b/src/bench/lib/chart-format.test.ts index 18c4d7d..9e58887 100644 --- a/src/bench/lib/chart-format.test.ts +++ b/src/bench/lib/chart-format.test.ts @@ -17,6 +17,7 @@ import { displayFormat, displaySeriesLabel, escapeHtml, + isOtherScan, FETCH_TIMEOUT_MS, firstLine, formatDisplayValue, @@ -529,6 +530,13 @@ describe('displayFormat / displaySeriesLabel', () => { expect(displayFormat('parquet')).toBe('parquet'); }); + it('tells the other Vortex scans from the formats bench shows', () => { + expect(isOtherScan('vortex-file-compressed [file, v1]')).toBe(true); + expect(isOtherScan('vortex-file-compressed [datasource, v1]')).toBe(true); + expect(isOtherScan('vortex-file-compressed')).toBe(false); + expect(isOtherScan('parquet-snappy')).toBe(false); + }); + it('rewrites the vortex token in any colon-delimited label', () => { expect(displaySeriesLabel('datafusion:vortex-file-compressed')).toBe('datafusion:vortex'); expect(displaySeriesLabel('duckdb:vortex-file-compressed')).toBe('duckdb:vortex'); diff --git a/src/bench/lib/chart-format.ts b/src/bench/lib/chart-format.ts index 0702063..4101c8d 100644 --- a/src/bench/lib/chart-format.ts +++ b/src/bench/lib/chart-format.ts @@ -251,6 +251,18 @@ export function displayFormat(format: string): string { return format === 'vortex-file-compressed' ? 'vortex' : format; } +/** + * Whether `format` names one of Vortex's other scans. The SQL benchmarks run + * each suite's default scan under the plain format, and every other scan under + * the format followed by its integration and scan engine, as + * `vortex-file-compressed [datasource, v1]`. The pull request reports compare + * every scan with its develop baseline in bench's database, so those rows stay + * there, but bench shows only the default scan, as `vortex`. + */ +export function isOtherScan(format: string): boolean { + return / \[[^\]]+\]$/.test(format); +} + /** * The display version of a colon-delimited series label (`engine:format` or the * compression-time `format:op`), rewriting each segment via [`displayFormat`] so diff --git a/src/bench/lib/data-cache.test.ts b/src/bench/lib/data-cache.test.ts index 884865f..42f624d 100644 --- a/src/bench/lib/data-cache.test.ts +++ b/src/bench/lib/data-cache.test.ts @@ -68,10 +68,12 @@ describe('data-cache wrappers', () => { it('tags every wrapper with the shared bench-data tag and the backstop TTL', () => { expect(BENCH_DATA_TAG).toBe('bench-data'); expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([ - 'data-cache:group-charts:v14:n100', + expect(cacheCalls.map((call) => call.keyParts)).toEqual([ + ['data-cache:group-charts:v15:n100'], + ['data-cache:chart-payload:v15:n100'], + ['data-cache:groups:v15'], + ['data-cache:filter-universe:v15'], ]); - expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v14']); for (const call of cacheCalls) { expect(call.options.tags).toEqual([BENCH_DATA_TAG]); expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS); diff --git a/src/bench/lib/data-cache.ts b/src/bench/lib/data-cache.ts index f5fa5eb..cfc6d3d 100644 --- a/src/bench/lib/data-cache.ts +++ b/src/bench/lib/data-cache.ts @@ -50,14 +50,16 @@ export const DATA_CACHE_BACKSTOP_SECONDS = 86400; const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_SECONDS }; /** - * Version of cached group payloads, including their embedded summary wire shape. + * Version of every cached read below: group payloads with their embedded + * summaries, chart payloads, and the filter universe. * This value must change when a deployment cannot read the preceding shape, and * when the same data would give a different payload, as after a change to a - * summary query. The Data Cache outlives a deploy, so without a new key the new - * code can serve payloads the old code computed until the next ingest. + * summary query or to the series a chart shows. The Data Cache outlives a + * deploy, so without a new key the new code can serve payloads the old code + * computed until the next ingest. * It is independent of the producer-facing benchmark schema version. */ -const GROUP_PAYLOAD_CACHE_VERSION = 'v14'; +const GROUP_PAYLOAD_CACHE_VERSION = 'v15'; // The default last-100 group bundle, keyed by group slug. The slug is the cache // key (an `unstable_cache` argument), so one wrapper covers every group. A @@ -78,7 +80,7 @@ const groupChartsCached = unstable_cache( const chartPayloadCached = unstable_cache( async (slug: string): Promise => chartPayload(chartKeyFromSlug(slug), { kind: 'last', n: DEFAULT_COMMIT_WINDOW }), - ['data-cache:chart-payload:n100'], + [`data-cache:chart-payload:${GROUP_PAYLOAD_CACHE_VERSION}:n100`], CACHE_OPTIONS, ); @@ -93,7 +95,7 @@ const groupsCached = unstable_cache( const filterUniverseCached = unstable_cache( async (): Promise => collectFilterUniverse(), - ['data-cache:filter-universe'], + [`data-cache:filter-universe:${GROUP_PAYLOAD_CACHE_VERSION}`], CACHE_OPTIONS, ); diff --git a/src/bench/lib/groups.test.ts b/src/bench/lib/groups.test.ts index 7d3784c..2932554 100644 --- a/src/bench/lib/groups.test.ts +++ b/src/bench/lib/groups.test.ts @@ -16,6 +16,7 @@ vi.mock('next/cache', () => ({ })); import { + collectFilterUniverse, collectGroup, collectGroupCharts, collectGroups, @@ -998,6 +999,32 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg expect(defined(summary.rankings[0]).ratio).toBeCloseTo(2.0, 6); }); + it('shows only the default Vortex scan, not the scans the PR reports compare', async () => { + const sha = 'f'.repeat(40); + await insertCommit(sha, '2026-04-23T12:00:00Z'); + await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed', 100_000); + await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed [file, v1]', 50_000); + await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed [datasource, v1]', 60_000); + await insertQuery(sha, 1, 'datafusion', 'parquet', 200_000); + + const key = { + k: 'QueryGroup', + dataset: 'tpch', + dataset_variant: null, + scale_factor: '1', + storage: 'nvme', + } as const; + const shown = ['datafusion:parquet', 'datafusion:vortex-file-compressed']; + const summary = await collectGroupSummary(key); + if (summary?.type !== 'queryBenchmark') { + throw new Error(`expected queryBenchmark summary, got ${summary?.type}`); + } + expect(summary.rankings.map((r) => r.name).sort()).toEqual(shown); + const bundle = await collectGroupCharts(key, parseCommitWindow(null)); + expect(Object.keys(defined(bundle?.charts[0]).series).sort()).toEqual(shown); + expect((await collectFilterUniverse()).formats).toEqual(['parquet', 'vortex-file-compressed']); + }); + it('applies the 300_000 penalty floor so a missing query flips the ranking', async () => { // All present runtimes are below the 300_000 ns floor, so the missing-query // penalty is max(maxRuntime, 300_000) * 2 = 600_000 for both series (the diff --git a/src/bench/lib/methodology.ts b/src/bench/lib/methodology.ts index 8649757..f7063e3 100644 --- a/src/bench/lib/methodology.ts +++ b/src/bench/lib/methodology.ts @@ -71,6 +71,8 @@ export const METHODOLOGY: readonly MethodologySection[] = [ 'format, and, for random access, LitData. Not every suite runs every format. The ' + 'compression and file size benchmarks write Parquet with parquet-rs 59 and zstd at ' + 'level 3. ClickBench reads its published Parquet files.', + 'Vortex has more than one way to scan a file in each query engine. bench shows each ' + + 'suite’s default scan. The benchmark reports on pull requests also compare the others.', ], }, { diff --git a/src/bench/lib/queries.ts b/src/bench/lib/queries.ts index de6d0a3..b887352 100644 --- a/src/bench/lib/queries.ts +++ b/src/bench/lib/queries.ts @@ -42,7 +42,7 @@ import { STRING_DATASET_PREFIX, } from './compression-suite'; import { groupDescription } from './descriptions'; -import type { FilterUniverse } from './chart-format'; +import { type FilterUniverse, isOtherScan } from './chart-format'; import { type ChartKind, compareCodeUnits, FAMILIES, type GroupKind } from './families'; import { chartKeyFromSlug, chartKeyToSlug, groupKeyFromSlug, groupKeyToSlug } from './slug'; import type { ChartKey, GroupKey } from './slug'; @@ -402,13 +402,19 @@ export function seriesFor( } } -/** Thread one fact row onto the seeded x-axis. A row off the seeded commits is dropped. */ +/** + * Thread one fact row onto the seeded x-axis. A row off the seeded commits is dropped, and so + * is a row of one of Vortex's other scans (`isOtherScan`). + */ function recordRow(acc: SeriesAccumulator, kind: ChartKind, row: FactRow): void { const idx = acc.commitIdx(row.commit_sha); if (idx === undefined) { return; } const { key, engine, format } = seriesFor(kind, row); + if (format !== undefined && isOtherScan(format)) { + return; + } acc.record(key, idx, row.value); acc.tag(key, engine, format); } @@ -1973,6 +1979,9 @@ export async function collectFilterUniverse(): Promise { ]); return { engines: engines.rows.map((r) => r.value).sort(compareCodeUnits), - formats: formats.rows.map((r) => r.value).sort(compareCodeUnits), + formats: formats.rows + .map((r) => r.value) + .filter((format) => !isOtherScan(format)) + .sort(compareCodeUnits), }; } diff --git a/src/bench/lib/summary.ts b/src/bench/lib/summary.ts index 43edd96..db42ff5 100644 --- a/src/bench/lib/summary.ts +++ b/src/bench/lib/summary.ts @@ -37,6 +37,7 @@ */ import { BRANCH_COMMITS, factOnReadBranch, onReadBranch } from './branch'; +import { isOtherScan } from './chart-format'; import { getPool } from './db'; import { randomAccessStorage, type RandomAccessStorage } from './random-access'; import { compareCodeUnits } from './families'; @@ -1284,8 +1285,11 @@ async function collectQuerySummary( await getPool().query<{ query_idx: number; series: string; value_ns: number }>(text, params) ).rows; + // `series` is `engine:format`, and no engine name holds a colon. const rankings = rankSeries( - rows.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })), + rows + .filter((row) => !isOtherScan(row.series.slice(row.series.indexOf(':') + 1))) + .map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })), (a, b) => a - b, QUERY_PENALTY_FLOOR_NS, 2,