Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/bench/lib/chart-format.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
displayFormat,
displaySeriesLabel,
escapeHtml,
isOtherScan,
FETCH_TIMEOUT_MS,
firstLine,
formatDisplayValue,
Expand Down Expand Up @@ -529,6 +530,13 @@ describe('displayFormat / displaySeriesLabel', () => {
expect(displayFormat('parquet')).toBe('parquet');
});

it('tells the other Vortex scans from the formats bench shows', () => {
expect(isOtherScan('vortex-file-compressed [file, v1]')).toBe(true);
expect(isOtherScan('vortex-file-compressed [datasource, v1]')).toBe(true);
expect(isOtherScan('vortex-file-compressed')).toBe(false);
expect(isOtherScan('parquet-snappy')).toBe(false);
});

it('rewrites the vortex token in any colon-delimited label', () => {
expect(displaySeriesLabel('datafusion:vortex-file-compressed')).toBe('datafusion:vortex');
expect(displaySeriesLabel('duckdb:vortex-file-compressed')).toBe('duckdb:vortex');
Expand Down
12 changes: 12 additions & 0 deletions src/bench/lib/chart-format.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,18 @@ export function displayFormat(format: string): string {
return format === 'vortex-file-compressed' ? 'vortex' : format;
}

/**
* Whether `format` names one of Vortex's other scans. The SQL benchmarks run
* each suite's default scan under the plain format, and every other scan under
* the format followed by its integration and scan engine, as
* `vortex-file-compressed [datasource, v1]`. The pull request reports compare
* every scan with its develop baseline in bench's database, so those rows stay
* there, but bench shows only the default scan, as `vortex`.
*/
export function isOtherScan(format: string): boolean {
return / \[[^\]]+\]$/.test(format);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Restrict the scan predicate to Vortex formats

isOtherScan currently classifies every format ending in a bracketed qualifier as an alternate Vortex scan, even though the documented convention is specifically vortex-file-compressed [...]. Because this predicate is used by recordRow for all chart families and by collectFilterUniverse, any legitimate non-Vortex format such as parquet [custom] is silently removed from charts and filter chips; require the vortex-file-compressed [ prefix before matching the suffix.

Useful? React with 👍 / 👎.

}

/**
* The display version of a colon-delimited series label (`engine:format` or the
* compression-time `format:op`), rewriting each segment via [`displayFormat`] so
Expand Down
8 changes: 5 additions & 3 deletions src/bench/lib/data-cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,12 @@ describe('data-cache wrappers', () => {
it('tags every wrapper with the shared bench-data tag and the backstop TTL', () => {
expect(BENCH_DATA_TAG).toBe('bench-data');
expect(DATA_CACHE_BACKSTOP_SECONDS).toBe(86400);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual([
'data-cache:group-charts:v14:n100',
expect(cacheCalls.map((call) => call.keyParts)).toEqual([
['data-cache:group-charts:v15:n100'],
['data-cache:chart-payload:v15:n100'],
['data-cache:groups:v15'],
['data-cache:filter-universe:v15'],
]);
expect(cacheCalls.map((call) => call.keyParts)).toContainEqual(['data-cache:groups:v14']);
for (const call of cacheCalls) {
expect(call.options.tags).toEqual([BENCH_DATA_TAG]);
expect(call.options.revalidate).toBe(DATA_CACHE_BACKSTOP_SECONDS);
Expand Down
14 changes: 8 additions & 6 deletions src/bench/lib/data-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,16 @@ export const DATA_CACHE_BACKSTOP_SECONDS = 86400;
const CACHE_OPTIONS = { tags: [BENCH_DATA_TAG], revalidate: DATA_CACHE_BACKSTOP_SECONDS };

/**
* Version of cached group payloads, including their embedded summary wire shape.
* Version of every cached read below: group payloads with their embedded
* summaries, chart payloads, and the filter universe.
* This value must change when a deployment cannot read the preceding shape, and
* when the same data would give a different payload, as after a change to a
* summary query. The Data Cache outlives a deploy, so without a new key the new
* code can serve payloads the old code computed until the next ingest.
* summary query or to the series a chart shows. The Data Cache outlives a
* deploy, so without a new key the new code can serve payloads the old code
* computed until the next ingest.
* It is independent of the producer-facing benchmark schema version.
*/
const GROUP_PAYLOAD_CACHE_VERSION = 'v14';
const GROUP_PAYLOAD_CACHE_VERSION = 'v15';

// The default last-100 group bundle, keyed by group slug. The slug is the cache
// key (an `unstable_cache` argument), so one wrapper covers every group. A
Expand All @@ -78,7 +80,7 @@ const groupChartsCached = unstable_cache(
const chartPayloadCached = unstable_cache(
async (slug: string): Promise<ChartResponse | null> =>
chartPayload(chartKeyFromSlug(slug), { kind: 'last', n: DEFAULT_COMMIT_WINDOW }),
['data-cache:chart-payload:n100'],
[`data-cache:chart-payload:${GROUP_PAYLOAD_CACHE_VERSION}:n100`],
CACHE_OPTIONS,
);

Expand All @@ -93,7 +95,7 @@ const groupsCached = unstable_cache(

const filterUniverseCached = unstable_cache(
async (): Promise<FilterUniverse> => collectFilterUniverse(),
['data-cache:filter-universe'],
[`data-cache:filter-universe:${GROUP_PAYLOAD_CACHE_VERSION}`],
CACHE_OPTIONS,
);

Expand Down
27 changes: 27 additions & 0 deletions src/bench/lib/groups.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ vi.mock('next/cache', () => ({
}));

import {
collectFilterUniverse,
collectGroup,
collectGroupCharts,
collectGroups,
Expand Down Expand Up @@ -998,6 +999,32 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg
expect(defined(summary.rankings[0]).ratio).toBeCloseTo(2.0, 6);
});

it('shows only the default Vortex scan, not the scans the PR reports compare', async () => {
const sha = 'f'.repeat(40);
await insertCommit(sha, '2026-04-23T12:00:00Z');
await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed', 100_000);
await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed [file, v1]', 50_000);
await insertQuery(sha, 1, 'datafusion', 'vortex-file-compressed [datasource, v1]', 60_000);
await insertQuery(sha, 1, 'datafusion', 'parquet', 200_000);

const key = {
k: 'QueryGroup',
dataset: 'tpch',
dataset_variant: null,
scale_factor: '1',
storage: 'nvme',
} as const;
const shown = ['datafusion:parquet', 'datafusion:vortex-file-compressed'];
const summary = await collectGroupSummary(key);
if (summary?.type !== 'queryBenchmark') {
throw new Error(`expected queryBenchmark summary, got ${summary?.type}`);
}
expect(summary.rankings.map((r) => r.name).sort()).toEqual(shown);
const bundle = await collectGroupCharts(key, parseCommitWindow(null));
expect(Object.keys(defined(bundle?.charts[0]).series).sort()).toEqual(shown);
expect((await collectFilterUniverse()).formats).toEqual(['parquet', 'vortex-file-compressed']);
});

it('applies the 300_000 penalty floor so a missing query flips the ranking', async () => {
// All present runtimes are below the 300_000 ns floor, so the missing-query
// penalty is max(maxRuntime, 300_000) * 2 = 600_000 for both series (the
Expand Down
2 changes: 2 additions & 0 deletions src/bench/lib/methodology.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ export const METHODOLOGY: readonly MethodologySection[] = [
'format, and, for random access, LitData. Not every suite runs every format. The ' +
'compression and file size benchmarks write Parquet with parquet-rs 59 and zstd at ' +
'level 3. ClickBench reads its published Parquet files.',
'Vortex has more than one way to scan a file in each query engine. bench shows each ' +
'suite’s default scan. The benchmark reports on pull requests also compare the others.',
],
},
{
Expand Down
15 changes: 12 additions & 3 deletions src/bench/lib/queries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import {
STRING_DATASET_PREFIX,
} from './compression-suite';
import { groupDescription } from './descriptions';
import type { FilterUniverse } from './chart-format';
import { type FilterUniverse, isOtherScan } from './chart-format';
import { type ChartKind, compareCodeUnits, FAMILIES, type GroupKind } from './families';
import { chartKeyFromSlug, chartKeyToSlug, groupKeyFromSlug, groupKeyToSlug } from './slug';
import type { ChartKey, GroupKey } from './slug';
Expand Down Expand Up @@ -402,13 +402,19 @@ export function seriesFor(
}
}

/** Thread one fact row onto the seeded x-axis. A row off the seeded commits is dropped. */
/**
* Thread one fact row onto the seeded x-axis. A row off the seeded commits is dropped, and so
* is a row of one of Vortex's other scans (`isOtherScan`).
*/
function recordRow(acc: SeriesAccumulator, kind: ChartKind, row: FactRow): void {
const idx = acc.commitIdx(row.commit_sha);
if (idx === undefined) {
return;
}
const { key, engine, format } = seriesFor(kind, row);
if (format !== undefined && isOtherScan(format)) {
return;
Comment on lines +415 to +416

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Drop charts that contain only hidden scans

When a discovered query has rows only for an alternate Vortex scan—for example after the default scan fails or is not emitted—the seed/total_commits queries still count those rows, but recordRow drops every one of them here. Both chartPayload and groupChartPayloads consequently return a non-null payload with an empty series object, so collectGroupCharts retains and renders a blank chart rather than omitting it; exclude hidden formats when determining chart existence or return null when filtering leaves no series.

Useful? React with 👍 / 👎.

}
acc.record(key, idx, row.value);
acc.tag(key, engine, format);
}
Expand Down Expand Up @@ -1973,6 +1979,9 @@ export async function collectFilterUniverse(): Promise<FilterUniverse> {
]);
return {
engines: engines.rows.map((r) => r.value).sort(compareCodeUnits),
formats: formats.rows.map((r) => r.value).sort(compareCodeUnits),
formats: formats.rows
.map((r) => r.value)
.filter((format) => !isOtherScan(format))
.sort(compareCodeUnits),
};
}
6 changes: 5 additions & 1 deletion src/bench/lib/summary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
*/

import { BRANCH_COMMITS, factOnReadBranch, onReadBranch } from './branch';
import { isOtherScan } from './chart-format';
import { getPool } from './db';
import { randomAccessStorage, type RandomAccessStorage } from './random-access';
import { compareCodeUnits } from './families';
Expand Down Expand Up @@ -1284,8 +1285,11 @@ async function collectQuerySummary(
await getPool().query<{ query_idx: number; series: string; value_ns: number }>(text, params)
).rows;

// `series` is `engine:format`, and no engine name holds a colon.
const rankings = rankSeries(
rows.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })),
rows
.filter((row) => !isOtherScan(row.series.slice(row.series.indexOf(':') + 1)))
.map((row) => ({ series: row.series, bucket: row.query_idx, value: row.value_ns })),
(a, b) => a - b,
QUERY_PENALTY_FLOOR_NS,
2,
Expand Down
Loading