From f1adf9c14890c446d26a501a986469ee843a2479 Mon Sep 17 00:00:00 2001 From: zackees Date: Mon, 13 Jul 2026 18:59:28 -0700 Subject: [PATCH] feat(cli): add post-ready test command hooks Closes #200 --- README.md | 23 +++ crates/fastled-cli/src/cli.rs | 28 +++ crates/fastled-cli/src/commands.rs | 71 +++++++- crates/fastled-cli/src/lib.rs | 1 + crates/fastled-cli/src/test_mode.rs | 263 ++++++++++++++++++++++++++++ 5 files changed, 381 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b01c0720..eb64c03b 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,29 @@ elapsed. Exit codes are `0` for success, the ready timeout. An interrupted run exits `130`. The viewer and local server are stopped automatically. +To run trusted host checks after the first rendered frame, repeat +`--test-cmd` options. They execute sequentially from the sketch directory, +and the test succeeds only after both the viewer schedule and all commands +finish: + +```bash +fastled examples/Blink --test --test-log=out/run.log \ + --test-cmd="python scripts/check_ready.py" \ + --test-cmd="python scripts/check_runtime_state.py" +``` + +Commands use `/bin/sh -c` on Unix and `%COMSPEC% /D /S /C` (normally +`cmd.exe`) on Windows, with null stdin and no extra console window. Pipes, +redirection, `&&`, and quoting follow the platform shell; invoke +`powershell -NoProfile -Command ...` explicitly for PowerShell syntax. +Command stdout and stderr are drained and mirrored to the terminal. With +`--test-log`, timestamped stream markers are appended, such as +`[fastled-test-cmd 0 stdout] ...` and `[fastled-test-cmd 0] exit=0`. +Commands are trusted input and inherit the user's environment and filesystem +permissions. A spawn failure or nonzero command exit returns `1`; the total +timeout (`124`) and interrupt (`130`) also terminate the active command and +its descendants. + ## Features ### Browser Compatibility diff --git a/crates/fastled-cli/src/cli.rs b/crates/fastled-cli/src/cli.rs index f6cd50d5..dca93b48 100644 --- a/crates/fastled-cli/src/cli.rs +++ b/crates/fastled-cli/src/cli.rs @@ -189,6 +189,15 @@ pub(crate) struct Cli { )] pub(crate) test_ready_timeout_secs: f64, + /// Run a trusted host command after the first rendered frame. May be repeated. + #[arg( + long, + requires = "test", + value_name = "COMMAND", + help_heading = "Production testing" + )] + pub(crate) test_cmd: Vec, + /// Use the latest tagged FastLED release when initialising examples with --init. /// Defaults to `master`; tagged releases older than the meson migration cannot be built. #[arg(long)] @@ -341,6 +350,25 @@ mod tests { assert_eq!(cli.test_screenshot, Some(PathBuf::from("out-{n:03}.png"))); assert_eq!(cli.test_log, Some(PathBuf::from("run.log"))); assert!(cli.test_exit_on_error); + assert!(cli.test_cmd.is_empty()); + } + + #[test] + fn repeated_test_commands_preserve_declaration_order() { + let cli = Cli::parse_from([ + "fastled", + "sketch", + "--test", + "--test-cmd=first", + "--test-cmd", + "second", + ]); + assert_eq!(cli.test_cmd, vec!["first", "second"]); + } + + #[test] + fn test_commands_require_master_switch() { + assert!(Cli::try_parse_from(["fastled", "sketch", "--test-cmd=echo hi"]).is_err()); } #[test] diff --git a/crates/fastled-cli/src/commands.rs b/crates/fastled-cli/src/commands.rs index 9e76c629..4c17897e 100644 --- a/crates/fastled-cli/src/commands.rs +++ b/crates/fastled-cli/src/commands.rs @@ -355,6 +355,11 @@ pub(crate) fn compile_and_test(dir: &str, cli: &Cli) -> ExitCode { let mut ready = false; let mut page_error = false; let mut saved = HashSet::new(); + let mut viewer_done: Option = None; + let mut commands_done = plan.commands.is_empty(); + let (command_tx, mut command_rx) = tokio::sync::mpsc::channel(256); + let mut command_tx = Some(command_tx); + let mut _command_task: Option = None; loop { tokio::select! { @@ -385,6 +390,19 @@ pub(crate) fn compile_and_test(dir: &str, cli: &Cli) -> ExitCode { if !write_test_log(&mut log_file, "[fastled-test] ready") { return test_exit(TestOutcome::Failure); } + if !plan.commands.is_empty() { + let Some(command_sender) = command_tx.take() else { + eprintln!("fastled: command runner was already started"); + return test_exit(TestOutcome::Failure); + }; + _command_task = Some(CommandTaskGuard(tokio::spawn( + test_mode::run_test_commands( + plan.commands.clone(), + NormalizedPath::new(&sketch_dir), + command_sender, + ), + ))); + } } } Some(server::TestEvent::ViewerLog(line)) => { @@ -413,11 +431,12 @@ pub(crate) fn compile_and_test(dir: &str, cli: &Cli) -> ExitCode { ); return test_exit(TestOutcome::Failure); } - return test_exit(if plan.exit_on_error && page_error { - TestOutcome::PageError - } else { - TestOutcome::Success - }); + viewer_done = Some(code); + if commands_done { + return test_exit(if plan.exit_on_error && page_error { + TestOutcome::PageError + } else { TestOutcome::Success }); + } } None => { eprintln!("fastled: test event channel closed unexpectedly"); @@ -425,11 +444,53 @@ pub(crate) fn compile_and_test(dir: &str, cli: &Cli) -> ExitCode { } } } + command_event = command_rx.recv(), if !commands_done => { + match command_event { + Some(test_mode::TestCommandEvent::Start { index }) => { + let marker = format!("[fastled-test-cmd {index}] start"); + println!("{marker}"); + if !write_test_log(&mut log_file, &marker) { return test_exit(TestOutcome::Failure); } + } + Some(test_mode::TestCommandEvent::Output { index, stream, line }) => { + let stream_name = match stream { test_mode::CommandStream::Stdout => "stdout", test_mode::CommandStream::Stderr => "stderr" }; + let marker = format!("[fastled-test-cmd {index} {stream_name}] {line}"); + match stream { test_mode::CommandStream::Stdout => println!("{line}"), test_mode::CommandStream::Stderr => eprintln!("{line}") } + if !write_test_log(&mut log_file, &marker) { return test_exit(TestOutcome::Failure); } + } + Some(test_mode::TestCommandEvent::Exit { index, code }) => { + let marker = format!("[fastled-test-cmd {index}] exit={code}"); + println!("{marker}"); + if !write_test_log(&mut log_file, &marker) { return test_exit(TestOutcome::Failure); } + } + Some(test_mode::TestCommandEvent::Done(result)) => { + commands_done = true; + if let Err(message) = result { + eprintln!("fastled: {message}"); + return test_exit(TestOutcome::Failure); + } + if viewer_done.is_some() { + return test_exit(if plan.exit_on_error && page_error { TestOutcome::PageError } else { TestOutcome::Success }); + } + } + None => { + eprintln!("fastled: command task ended without a completion result"); + return test_exit(TestOutcome::Failure); + } + } + } } } }) } +struct CommandTaskGuard(tokio::task::JoinHandle<()>); + +impl Drop for CommandTaskGuard { + fn drop(&mut self) { + self.0.abort(); + } +} + fn test_exit(outcome: TestOutcome) -> ExitCode { ExitCode::from(outcome.exit_code()) } diff --git a/crates/fastled-cli/src/lib.rs b/crates/fastled-cli/src/lib.rs index cdb14821..25da374f 100644 --- a/crates/fastled-cli/src/lib.rs +++ b/crates/fastled-cli/src/lib.rs @@ -234,6 +234,7 @@ mod tests { test_exit_on_error: false, test_timeout_secs: 120.0, test_ready_timeout_secs: 15.0, + test_cmd: Vec::new(), latest: false, branch: None, commit: None, diff --git a/crates/fastled-cli/src/test_mode.rs b/crates/fastled-cli/src/test_mode.rs index c9bcd2a9..4d4a282c 100644 --- a/crates/fastled-cli/src/test_mode.rs +++ b/crates/fastled-cli/src/test_mode.rs @@ -1,5 +1,8 @@ +use std::io::{BufRead, BufReader}; +use std::path::Path; use std::process::Command; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::mpsc; use crate::cli::Cli; use crate::path::NormalizedPath; @@ -41,6 +44,163 @@ pub(crate) struct TestPlan { pub(crate) screenshots: Vec<(String, NormalizedPath)>, pub(crate) log_path: Option, pub(crate) exit_on_error: bool, + pub(crate) commands: Vec, +} + +#[derive(Debug)] +pub(crate) enum TestCommandEvent { + Start { + index: usize, + }, + Output { + index: usize, + stream: CommandStream, + line: String, + }, + Exit { + index: usize, + code: i32, + }, + Done(Result<(), String>), +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CommandStream { + Stdout, + Stderr, +} + +pub(crate) fn shell_command(command: &str, sketch_dir: &Path) -> Command { + #[cfg(windows)] + let mut process = { + let shell = std::env::var_os("COMSPEC").unwrap_or_else(|| "cmd.exe".into()); + let mut c = Command::new(shell); + c.args(["/D", "/S", "/C", command]); + c + }; + #[cfg(not(windows))] + let mut process = { + let mut c = Command::new("/bin/sh"); + c.args(["-c", command]); + c + }; + process.current_dir(sketch_dir); + process +} + +pub(crate) async fn run_test_commands( + commands: Vec, + sketch_dir: NormalizedPath, + tx: mpsc::Sender, +) { + let result = run_test_commands_inner(commands, sketch_dir, &tx).await; + let _ = tx.send(TestCommandEvent::Done(result)).await; +} + +async fn run_test_commands_inner( + commands: Vec, + sketch_dir: NormalizedPath, + tx: &mpsc::Sender, +) -> Result<(), String> { + for (index, command_text) in commands.iter().enumerate() { + if tx.send(TestCommandEvent::Start { index }).await.is_err() { + return Ok(()); + } + let mut command = shell_command(command_text, sketch_dir.as_path()); + let group = running_process::ContainedProcessGroup::with_originator("FASTLED_TEST_CMD") + .map_err(|e| format!("could not create command process group: {e}"))?; + let mut child = group + .spawn( + &mut command, + running_process::SpawnStdio { + stdin: running_process::StdioSource::Null, + stdout: running_process::StdioSource::Pipe, + stderr: running_process::StdioSource::Pipe, + drain_timeout: Some(std::time::Duration::from_secs(2)), + show_console: false, + }, + ) + .map_err(|e| format!("could not spawn test command {index}: {e}"))?; + let (output_tx, mut output_rx) = mpsc::channel::<(CommandStream, String)>(256); + let mut readers = 0usize; + if let Some(stdout) = child.stdout.take() { + readers += 1; + spawn_reader(stdout, CommandStream::Stdout, output_tx.clone()); + } + if let Some(stderr) = child.stderr.take() { + readers += 1; + spawn_reader(stderr, CommandStream::Stderr, output_tx.clone()); + } + drop(output_tx); + let mut exit = None; + let mut closed_readers = 0usize; + let mut drain_deadline = None; + while exit.is_none() || closed_readers < readers { + if drain_deadline.is_some_and(|deadline| tokio::time::Instant::now() >= deadline) { + break; + } + tokio::select! { + item = output_rx.recv() => match item { + Some((stream, line)) => { + if tx.send(TestCommandEvent::Output { index, stream, line }).await.is_err() { return Ok(()); } + } + None => { closed_readers = readers; } + }, + _ = tokio::time::sleep(std::time::Duration::from_millis(10)), if exit.is_none() => { + if let Some(code) = child.try_wait().map_err(|e| format!("test command {index} wait failed: {e}"))? { + exit = Some(code); + drain_deadline = Some(tokio::time::Instant::now() + std::time::Duration::from_secs(2)); + } + }, + _ = tokio::time::sleep(std::time::Duration::from_millis(10)), if exit.is_some() => {} + } + if exit.is_some() && closed_readers < readers && output_rx.is_closed() { + break; + } + } + let code = exit.unwrap_or_else(|| child.try_wait().ok().flatten().unwrap_or(1)); + if tx + .send(TestCommandEvent::Exit { index, code }) + .await + .is_err() + { + return Ok(()); + } + if code != 0 { + return Err(format!("test command {index} exited with code {code}")); + } + } + Ok(()) +} + +fn spawn_reader( + reader: R, + stream: CommandStream, + tx: mpsc::Sender<(CommandStream, String)>, +) { + std::thread::spawn(move || { + let mut reader = BufReader::new(reader); + let mut bytes = Vec::new(); + loop { + bytes.clear(); + let Ok(count) = reader.read_until(b'\n', &mut bytes) else { + break; + }; + if count == 0 { + break; + } + while bytes + .last() + .is_some_and(|byte| *byte == b'\n' || *byte == b'\r') + { + bytes.pop(); + } + let line = String::from_utf8_lossy(&bytes).into_owned(); + if tx.blocking_send((stream, line)).is_err() { + break; + } + } + }); } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -108,6 +268,10 @@ pub(crate) fn build_test_plan(cli: &Cli) -> Result { None => None, }; + if cli.test_cmd.iter().any(|command| command.trim().is_empty()) { + return Err("--test-cmd values must not be empty or whitespace-only".to_string()); + } + if matches!(cli.test_count, Some(0)) { return Err("--test-count must be greater than zero".to_string()); } @@ -167,6 +331,7 @@ pub(crate) fn build_test_plan(cli: &Cli) -> Result { screenshots, log_path: cli.test_log.as_deref().map(NormalizedPath::new), exit_on_error: cli.test_exit_on_error, + commands: cli.test_cmd.clone(), }) } @@ -361,6 +526,14 @@ mod tests { assert!(build_test_plan(&huge_count).is_err()); } + #[test] + fn empty_test_commands_are_rejected() { + for value in ["", " ", "\t"] { + let cli = Cli::parse_from(["fastled", "sketch", "--test", "--test-cmd", value]); + assert!(build_test_plan(&cli).is_err(), "accepted {value:?}"); + } + } + #[tokio::test] async fn contained_command_is_stopped_at_the_hard_deadline() { #[cfg(windows)] @@ -382,4 +555,94 @@ mod tests { assert_eq!(result, TimedCommandResult::TimedOut); assert!(started.elapsed() < Duration::from_secs(1)); } + + #[tokio::test] + async fn issue_200_commands_run_sequentially_and_capture_both_streams() { + let temp = tempfile::tempdir().unwrap(); + #[cfg(windows)] + let commands = vec![ + "echo A>order.txt".to_string(), + "echo B>>order.txt & echo out & echo err 1>&2".to_string(), + ]; + #[cfg(not(windows))] + let commands = vec![ + "printf A >> order.txt".to_string(), + "printf B >> order.txt; printf out; printf err >&2".to_string(), + ]; + let (tx, mut rx) = mpsc::channel(256); + let task = tokio::spawn(run_test_commands( + commands, + NormalizedPath::new(temp.path()), + tx, + )); + let mut outputs = Vec::new(); + while let Some(event) = rx.recv().await { + let done = matches!(event, TestCommandEvent::Done(_)); + if let TestCommandEvent::Output { stream, line, .. } = event { + outputs.push((stream, line)); + } + if done { + break; + } + } + task.await.unwrap(); + let order = std::fs::read_to_string(temp.path().join("order.txt")).unwrap(); + assert_eq!(order.split_whitespace().collect::(), "AB"); + assert!(outputs + .iter() + .any(|(stream, line)| *stream == CommandStream::Stdout && line.contains("out"))); + assert!(outputs + .iter() + .any(|(stream, line)| *stream == CommandStream::Stderr && line.contains("err"))); + } + + #[tokio::test] + async fn issue_200_nonzero_command_stops_the_sequence() { + let temp = tempfile::tempdir().unwrap(); + #[cfg(windows)] + let commands = vec![ + "exit /b 7".to_string(), + "echo SHOULD_NOT_RUN > later.txt".to_string(), + ]; + #[cfg(not(windows))] + let commands = vec![ + "exit 7".to_string(), + "echo SHOULD_NOT_RUN > later.txt".to_string(), + ]; + let (tx, mut rx) = mpsc::channel(256); + tokio::spawn(run_test_commands( + commands, + NormalizedPath::new(temp.path()), + tx, + )); + let mut result = None; + while let Some(event) = rx.recv().await { + if let TestCommandEvent::Done(value) = event { + result = Some(value); + break; + } + } + assert!(result.unwrap().is_err()); + assert!(!temp.path().join("later.txt").exists()); + } + + #[tokio::test] + async fn issue_200_cancelling_runner_kills_the_contained_shell() { + let temp = tempfile::tempdir().unwrap(); + #[cfg(windows)] + let command = "ping -n 20 127.0.0.1 >NUL & echo LATE > late.txt"; + #[cfg(not(windows))] + let command = "sleep 2; echo LATE > late.txt"; + let (tx, _rx) = mpsc::channel(256); + let task = tokio::spawn(run_test_commands( + vec![command.to_string()], + NormalizedPath::new(temp.path()), + tx, + )); + tokio::time::sleep(Duration::from_millis(100)).await; + task.abort(); + let _ = task.await; + tokio::time::sleep(Duration::from_millis(250)).await; + assert!(!temp.path().join("late.txt").exists()); + } }