lib: put node:bench behind an --experimental-bench flag

Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode
PR-URL: #65920
Reviewed-By: Jordan Harband <ljharb@gmail.com>
Reviewed-By: Rafael Gonzaga <rafael.nunu@hotmail.com>
Reviewed-By: Vinícius Lourenço Claro Cardoso <contact@viniciusl.com.br>
Reviewed-By: Xuguang Mei <meixuguang@gmail.com>
Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
Reviewed-By: Filip Skokan <panva.ip@gmail.com>

Author: jasnell

benchmark/_node-bench.js

@@ -53,6 +53,7 @@ function runBenchmark(binary, file, options) {
   const args = [
     ...options.nodeArgs,
     '--no-warnings',
+    '--experimental-bench',
     '--bench',
     '--bench-reporter=json',
     '--bench-samples=1',

doc/api/bench.md

@@ -12,7 +12,8 @@ added: REPLACEME
 
 The `node:bench` module supports defining and running JavaScript benchmarks in
 the current process, and running one benchmark file in a fresh child process.
-To access it:
+The module is only available when Node.js is started with the
+`--experimental-bench` flag and can only be imported with the `node:` scheme:
 
 ```mjs
 import { bench, suite } from 'node:bench';
@@ -22,8 +23,6 @@ import { bench, suite } from 'node:bench';
 const { bench, suite } = require('node:bench');
 ```
 
-This module is only available under the `node:` scheme.
-
 ## Example benchmark
 
 Save the following as `benchmark.mjs`:
@@ -57,7 +56,7 @@ suite('URL', () => {
 Run the benchmark from the command line:
 
 ```console
-node --bench benchmark.mjs
+node --experimental-bench --bench benchmark.mjs
 ```
 
 Benchmarks are executed serially in declaration order. Declared benchmarks are
@@ -183,8 +182,8 @@ declarations and a second call to `run()` is an error.
 The `--bench` flag runs one or more explicit benchmark files or glob patterns:
 
 ```console
-node --bench benchmark.mjs
-node --bench --bench-reporter=json 'benchmarks/**/*.js'
+node --experimental-bench --bench benchmark.mjs
+node --experimental-bench --bench --bench-reporter=json 'benchmarks/**/*.js'
 ```
 
 Files are sorted and executed serially. The default

doc/api/cli.md

@@ -482,10 +482,13 @@ Starts the Node.js command-line benchmark runner. At least one explicit file or
 glob pattern is required:
 
 ```console
-node --bench benchmark.mjs
-node --bench 'benchmarks/**/*.js'
+node --experimental-bench --bench benchmark.mjs
+node --experimental-bench --bench 'benchmarks/**/*.js'
 ```
 
+The `--experimental-bench` flag is required to use this flag or any other
+`--bench-*` option.
+
 Quote glob patterns to prevent expansion by the shell. Matching files are
 sorted and executed serially. By default, each file runs in a separate child
 process. Benchmark files declare benchmarks using `node:bench`; they must not
@@ -1210,6 +1213,16 @@ changes:
 
 Enable experimental import support for `.node` addons.
 
+### `--experimental-bench`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Enable the experimental `node:bench` module and command-line benchmark runner.
+
 ### `--experimental-config-file=path`, `--experimental-config-file`
 
 <!-- YAML
@@ -4062,6 +4075,7 @@ one is included in the list below.
 * `--entry-url`
 * `--experimental-abortcontroller`
 * `--experimental-addon-modules`
+* `--experimental-bench`
 * `--experimental-detect-module`
 * `--experimental-dtls`
 * `--experimental-eventsource`

doc/node-config-schema.json

@@ -45,6 +45,63 @@
         "$schema": {
           "type": "string"
         },
+        "bench": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [],
+          "properties": {
+            "bench": {
+              "type": "boolean",
+              "description": "launch benchmark runner on startup"
+            },
+            "bench-isolation": {
+              "type": "string",
+              "description": "configures the type of benchmark isolation used in the benchmark runner"
+            },
+            "bench-name-pattern": {
+              "type": "string",
+              "description": "run benchmarks whose name matches this regular expression"
+            },
+            "bench-reporter": {
+              "oneOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "minItems": 1,
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "description": "report benchmark output using the given reporter"
+            },
+            "bench-reporter-destination": {
+              "oneOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "minItems": 1,
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "description": "report the given benchmark reporter to the given destination"
+            },
+            "bench-samples": {
+              "type": "number",
+              "description": "specify the number of measured benchmark samples"
+            },
+            "bench-warmup": {
+              "type": "number",
+              "description": "specify the number of unreported benchmark warmup samples"
+            }
+          }
+        },
         "nodeOptions": {
           "additionalProperties": false,
           "required": [],
@@ -119,6 +176,52 @@
               "type": "boolean",
               "description": "Improve AsyncLocalStorage performance with AsyncContextFrame"
             },
+            "bench-isolation": {
+              "type": "string",
+              "description": "configures the type of benchmark isolation used in the benchmark runner"
+            },
+            "bench-name-pattern": {
+              "type": "string",
+              "description": "run benchmarks whose name matches this regular expression"
+            },
+            "bench-reporter": {
+              "oneOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "minItems": 1,
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "description": "report benchmark output using the given reporter"
+            },
+            "bench-reporter-destination": {
+              "oneOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "array",
+                  "minItems": 1,
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              ],
+              "description": "report the given benchmark reporter to the given destination"
+            },
+            "bench-samples": {
+              "type": "number",
+              "description": "specify the number of measured benchmark samples"
+            },
+            "bench-warmup": {
+              "type": "number",
+              "description": "specify the number of unreported benchmark warmup samples"
+            },
             "conditions": {
               "oneOf": [
                 {
@@ -209,6 +312,10 @@
               "type": "boolean",
               "description": "experimental import support for addons"
             },
+            "experimental-bench": {
+              "type": "boolean",
+              "description": "experimental node:bench module and benchmark runner"
+            },
             "experimental-detect-module": {
               "type": "boolean",
               "description": "when ambiguous modules fail to evaluate because they contain ES module syntax, try again to evaluate them as ES modules"

doc/node.1

@@ -296,9 +296,11 @@ Error: Access to this API has been restricted
 Starts the Node.js command-line benchmark runner. At least one explicit file or
 glob pattern is required:
 .Bd -literal
-node --bench benchmark.mjs
-node --bench 'benchmarks/**/*.js'
+node --experimental-bench --bench benchmark.mjs
+node --experimental-bench --bench 'benchmarks/**/*.js'
 .Ed
+The \fB--experimental-bench\fR flag is required to use this flag or any other
+\fB--bench-*\fR option.
 Quote glob patterns to prevent expansion by the shell. Matching files are
 sorted and executed serially. By default, each file runs in a separate child
 process. Benchmark files declare benchmarks using \fBnode:bench\fR; they must not
@@ -677,6 +679,9 @@ It is possible to run code containing inline types unless the
 .It Fl -experimental-addon-modules
 Enable experimental import support for \fB.node\fR addons.
 .
+.It Fl -experimental-bench
+Enable the experimental \fBnode:bench\fR module and command-line benchmark runner.
+.
 .It Fl -experimental-config-file Ns = Ns Ar path , Fl -experimental-config-file
 If present, Node.js will look for a configuration file at the specified path.
 If the path is not specified, Node.js will look for a \fBnode.config.json\fR file
@@ -2098,6 +2103,8 @@ one is included in the list below.
 .It
 \fB--experimental-addon-modules\fR
 .It
+\fB--experimental-bench\fR
+.It
 \fB--experimental-detect-module\fR
 .It
 \fB--experimental-dtls\fR

lib/internal/bench_runner/cli.js

@@ -106,7 +106,11 @@ const kEventTypes = new SafeSet([
   'bench:diagnostic',
   'bench:summary',
 ]);
-const kFilterArgs = ['--bench', '--experimental-default-config-file'];
+const kFilterArgs = [
+  '--bench',
+  '--experimental-bench',
+  '--experimental-default-config-file',
+];
 const kFilterArgValues = [
   '--bench-isolation',
   '--bench-name-pattern',
@@ -741,7 +745,8 @@ function getInheritedChildArgs() {
 function getChildArgs(path, options) {
   const args = options.execArgv === undefined ?
     getInheritedChildArgs() : ArrayPrototypeSlice(options.execArgv);
-  ArrayPrototypePush(args, '--bench', '--bench-isolation=none');
+  ArrayPrototypePush(
+    args, '--experimental-bench', '--bench', '--bench-isolation=none');
   if (options.namePatternSource.length > 0) {
     ArrayPrototypePush(
       args, `--bench-name-pattern=${options.namePatternSource}`);

lib/internal/bootstrap/realm.js

@@ -137,6 +137,8 @@ const schemelessBlockList = new SafeSet([
 ]);
 // Modules that will only be enabled at run time.
 const experimentalModuleList = new SafeSet([
+  'bench',
+  'bench/reporters',
   'dtls',
   'ffi',
   'quic',

lib/internal/process/pre_execution.js

@@ -114,6 +114,7 @@ function prepareExecution(options) {
   setupNetworkInspection();
   setupNavigator();
   setupWarningHandler();
+  setupBench();
   setupFFI();
   setupSQLite();
   setupStreamIter();
@@ -391,6 +392,16 @@ function setupNavigator() {
   defineReplaceableLazyAttribute(globalThis, 'internal/navigator', ['navigator'], false);
 }
 
+function setupBench() {
+  if (!getOptionValue('--experimental-bench')) {
+    return;
+  }
+
+  const { BuiltinModule } = require('internal/bootstrap/realm');
+  BuiltinModule.allowRequireByUsers('bench');
+  BuiltinModule.allowRequireByUsers('bench/reporters');
+}
+
 function setupFFI() {
   if (!getOptionValue('--experimental-ffi')) {
     return;

src/node.cc

@@ -725,6 +725,14 @@ void ResetStdio() {
 #endif  // __POSIX__
 }
 
+// Validates the benchmark runner options of the global (per-process) options
+// once every option source has been parsed. See
+// EnvironmentOptions::CheckBenchOptions().
+static void CheckGlobalBenchOptions(std::vector<std::string>* errors) {
+  Mutex::ScopedLock lock(per_process::cli_options_mutex);
+  per_process::cli_options->per_isolate->per_env->CheckBenchOptions(errors);
+}
+
 static ExitCode ProcessGlobalArgsInternal(std::vector<std::string>* args,
                                           std::vector<std::string>* exec_args,
                                           std::vector<std::string>* errors,
@@ -834,8 +842,16 @@ int ProcessGlobalArgs(std::vector<std::string>* args,
                       std::vector<std::string>* exec_args,
                       std::vector<std::string>* errors,
                       OptionEnvvarSettings settings) {
-  return static_cast<int>(
-      ProcessGlobalArgsInternal(args, exec_args, errors, settings));
+  const ExitCode exit_code =
+      ProcessGlobalArgsInternal(args, exec_args, errors, settings);
+  if (exit_code != ExitCode::kNoFailure) return static_cast<int>(exit_code);
+  // Embedders parse every option source in a single pass, so the benchmark
+  // options can be validated right away.
+  CheckGlobalBenchOptions(errors);
+  if (!errors->empty()) {
+    return static_cast<int>(ExitCode::kInvalidCommandLineArgument);
+  }
+  return static_cast<int>(ExitCode::kNoFailure);
 }
 
 static std::atomic_bool init_called{false};
@@ -1016,6 +1032,11 @@ static ExitCode InitializeNodeWithArgsInternal(
     if (exit_code != ExitCode::kNoFailure) return exit_code;
   }
 
+  // Every option source has now been parsed, so cross-source option
+  // constraints can finally be validated.
+  CheckGlobalBenchOptions(errors);
+  if (!errors->empty()) return ExitCode::kInvalidCommandLineArgument;
+
   // Set the process.title immediately after processing argv if --title is set.
   if (!per_process::cli_options->title.empty())
     uv_set_process_title(per_process::cli_options->title.c_str());

src/node_builtins.cc

@@ -158,15 +158,17 @@ BuiltinLoader::BuiltinCategories BuiltinLoader::GetBuiltinCategories() const {
 #if !HAVE_FFI
         "internal/ffi-shared-buffer", "internal/ffi/fast-api",
 #endif                  // !HAVE_FFI
-        "dtls",         // Experimental.
-        "ffi",          // Experimental.
-        "quic",         // Experimental.
-        "sqlite",       // Experimental.
-        "stream/iter",  // Experimental.
-        "sys",          // Deprecated.
-        "vfs",          // Experimental.
-        "wasi",         // Experimental.
-        "zlib/iter",    // Experimental.
+        "bench",        // Experimental.
+        "bench/reporters",  // Experimental.
+        "dtls",             // Experimental.
+        "ffi",              // Experimental.
+        "quic",             // Experimental.
+        "sqlite",           // Experimental.
+        "stream/iter",      // Experimental.
+        "sys",              // Deprecated.
+        "vfs",              // Experimental.
+        "wasi",             // Experimental.
+        "zlib/iter",        // Experimental.
 #if !HAVE_SQLITE
         "internal/webstorage",  // Experimental.
         "internal/inspector/webstorage",