From 5c82cea51e9f0adc312e1231a18fb18cba6cf2c7 Mon Sep 17 00:00:00 2001 From: zackees Date: Sun, 31 May 2026 09:45:26 -0700 Subject: [PATCH] feat: restore DWARF source resolution + unblock end-to-end Windows compile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the legacy `debug_symbols.py` DWARF resolver to Rust and wires it into the embedded HTTP server, then fixes every blocker found while running `fastled --just-compile` against the master FastLED on Windows. End-to-end status after this change: `--init`, `--just-compile --quick`, `--just-compile --release`, `--just-compile --debug`, and `--serve-dir` all succeed on Windows (verified 538 KB quick / 2.3 MB debug `.wasm`). Features - crates/fastled-cli/src/debug_symbols.rs — new module: DwarfPrefixConfig (loaded from `build_flags.toml` `[dwarf]`), DebugSymbolConfig with named source roots (sketch/fastled/emsdk + `headers` alias), and DebugSymbolResolver with lexical traversal guard + `dwarfsource/...` unwrapping. 6 unit tests covering prefix stripping, traversal rejection, file-not-found, and prefix overrides. Refs #85. - crates/fastled-cli/src/server.rs — new `POST /dwarfsource` and `GET /debug/source-roots` axum handlers behind a shared `Arc>>`. 4 new integration tests. - crates/fastled-cli/src/lib.rs — populates the resolver from `BuildResult.{fastled_dir, emsdk_root}` after each successful build in the watch loop. - crates/fastled-cli/src/path.rs — new NormalizedPath wrapper that strips Windows `\?\` long-path prefix on construction, lexically resolves `.`/`..`, and case-folds for stable comparison on Windows/macOS. `wasm_build::normalize_path` now delegates to it. - dylints/ban_std_pathbuf/ — new sub-crate (nightly-2026-03-26, dylint port from zackees/zccache) that denies `std::path::PathBuf` outside an explicit per-file allowlist, with the current 14 PathBuf-using files pre-allowlisted. Refs #117. Windows compile-flow fixes - install.rs (#111): accept the published emscripten manifest schema where version entries live as siblings of `latest` (the current win/x86_64/manifest.json shape), not just the historical nested `{ "versions": { ... } }` form. Adds parse_platform_manifest with 3 unit tests for both shapes plus the failure paths. - wasm_build.rs (#113): write meson cross-files with single-quoted string literals (`'path'`) instead of `{:?}` debug-quoted form (`"path"`). Meson's INI parser rejects double quotes on Windows. - wasm_build.rs (#114): NormalizedPath strips the `\?\` UNC prefix that `fs::canonicalize` returns on Windows. Python's `open()` and meson's `DirectoryLock` can't handle that prefix; the open fails with OSError errno 22, meson's win32 lock handler silently swallows it, and crashes in `msvcrt.locking(self.lockfile.fileno(), …)` with `AttributeError: 'NoneType' object has no attribute 'fileno'`. - lib.rs + wasm_build.rs (#115): `--init` now defaults to `master` (not the latest tagged release — pre-meson tags have no `meson.build`) and always writes the resolved ref into `/fastled.json`. `resolve_fastled_dir` reads that ref back so the build path uses the same FastLED checkout the sketch was scaffolded against. - wasm_build.rs (#116): EMCC_CORES is now `available_parallelism().min(32)` instead of a hard-coded 128. Adds `run_em_link_with_retries`: on Windows, em++ link retries up to 6× because each completed sysroot library is cached to disk, so the retries converge through the flaky emcc subprocess crashes that appear under high contention. Tests/lint - 119/119 cargo lib tests pass (was 108; added 10 DWARF + 3 manifest + 2 meson-quote + 6 NormalizedPath, removed 3 in-place tests). - `bash lint` clean. Refs #85 #111 #113 #114 #115 #116 #117 Co-Authored-By: Claude Opus 4.7 --- .gitignore | 1 + crates/fastled-cli/src/debug_symbols.rs | 449 ++++++++++++++++++++ crates/fastled-cli/src/install.rs | 83 +++- crates/fastled-cli/src/lib.rs | 100 ++++- crates/fastled-cli/src/path.rs | 268 ++++++++++++ crates/fastled-cli/src/server.rs | 230 +++++++++- crates/fastled-cli/src/wasm_build.rs | 125 +++++- dylints/ban_std_pathbuf/Cargo.toml | 21 + dylints/ban_std_pathbuf/README.md | 35 ++ dylints/ban_std_pathbuf/rust-toolchain.toml | 3 + dylints/ban_std_pathbuf/src/allowlist.txt | 28 ++ dylints/ban_std_pathbuf/src/lib.rs | 149 +++++++ 12 files changed, 1446 insertions(+), 46 deletions(-) create mode 100644 crates/fastled-cli/src/debug_symbols.rs create mode 100644 crates/fastled-cli/src/path.rs create mode 100644 dylints/ban_std_pathbuf/Cargo.toml create mode 100644 dylints/ban_std_pathbuf/README.md create mode 100644 dylints/ban_std_pathbuf/rust-toolchain.toml create mode 100644 dylints/ban_std_pathbuf/src/allowlist.txt create mode 100644 dylints/ban_std_pathbuf/src/lib.rs diff --git a/.gitignore b/.gitignore index d3c78c3e..32c4985c 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,4 @@ vscode-plugin /src/fastled/bin/ .clud/loop/ .claude/worktrees/ +.claude/scheduled_tasks.lock diff --git a/crates/fastled-cli/src/debug_symbols.rs b/crates/fastled-cli/src/debug_symbols.rs new file mode 100644 index 00000000..00f3fa96 --- /dev/null +++ b/crates/fastled-cli/src/debug_symbols.rs @@ -0,0 +1,449 @@ +//! DWARF source-path resolution for the in-browser debugger. +//! +//! When a sketch is compiled in debug mode, emscripten embeds DWARF entries +//! that point at the original source files (sketch, FastLED library, emsdk +//! headers). The browser's devtools then issue `POST /dwarfsource` requests +//! to fetch the actual text by path. This module converts those request paths +//! into real on-disk locations, with traversal guards so that only files under +//! the configured roots are exposed. +//! +//! This is the Rust port of the legacy Python `debug_symbols.py` module that +//! used to back the Flask-based debug routes. + +use std::path::{Component, Path, PathBuf}; + +use serde::Deserialize; + +pub const DEFAULT_FASTLED_PREFIX: &str = "fastledsource"; +pub const DEFAULT_SKETCH_PREFIX: &str = "sketchsource"; +pub const DEFAULT_DWARF_PREFIX: &str = "dwarfsource"; + +/// Configurable DWARF path prefixes loaded from +/// `/platforms/wasm/compiler/build_flags.toml`. +#[derive(Debug, Clone, Deserialize)] +pub struct DwarfPrefixConfig { + #[serde(default = "default_fastled_prefix")] + pub fastled_prefix: String, + #[serde(default = "default_sketch_prefix")] + pub sketch_prefix: String, + #[serde(default = "default_dwarf_prefix")] + pub dwarf_prefix: String, +} + +fn default_fastled_prefix() -> String { + DEFAULT_FASTLED_PREFIX.to_string() +} + +fn default_sketch_prefix() -> String { + DEFAULT_SKETCH_PREFIX.to_string() +} + +fn default_dwarf_prefix() -> String { + DEFAULT_DWARF_PREFIX.to_string() +} + +impl Default for DwarfPrefixConfig { + fn default() -> Self { + Self { + fastled_prefix: default_fastled_prefix(), + sketch_prefix: default_sketch_prefix(), + dwarf_prefix: default_dwarf_prefix(), + } + } +} + +#[derive(Debug, Clone)] +pub struct DebugSymbolConfig { + pub sketch_dir: PathBuf, + pub fastled_dir: Option, + pub emsdk_path: Option, + pub prefixes: DwarfPrefixConfig, +} + +impl DebugSymbolConfig { + /// The set of named source roots that paths can resolve into. + /// + /// Order matters: longer / more-specific prefixes should win, but every + /// configured prefix is matched literally so ordering only matters for + /// equal-prefix ambiguities (none currently). + pub fn source_roots(&self) -> Vec<(String, PathBuf)> { + let mut roots: Vec<(String, PathBuf)> = Vec::new(); + let sketch = canonicalize_or(&self.sketch_dir); + roots.push((self.prefixes.sketch_prefix.clone(), sketch)); + if let Some(fastled_dir) = &self.fastled_dir { + let src = canonicalize_or(&fastled_dir.join("src")); + roots.push((self.prefixes.fastled_prefix.clone(), src.clone())); + // `headers` matches what the legacy resolver exposed so that DWARF + // entries that point at `headers/...` (resolved relative to the + // FastLED include root) keep working. + roots.push(("headers".to_string(), src)); + } + if let Some(emsdk_path) = &self.emsdk_path { + roots.push(("emsdk".to_string(), canonicalize_or(emsdk_path))); + } + roots + } +} + +fn canonicalize_or(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +/// Load DWARF prefix overrides from +/// `/platforms/wasm/compiler/build_flags.toml` if present. +/// +/// Falls back to defaults when the file or the `[dwarf]` table is absent so +/// callers can rely on this for unconditional setup. +pub fn load_debug_symbol_config( + sketch_dir: PathBuf, + fastled_dir: Option, + emsdk_path: Option, +) -> DebugSymbolConfig { + let prefixes = fastled_dir + .as_ref() + .and_then(|root| read_dwarf_prefixes(root)) + .unwrap_or_default(); + + DebugSymbolConfig { + sketch_dir, + fastled_dir, + emsdk_path, + prefixes, + } +} + +fn read_dwarf_prefixes(fastled_dir: &Path) -> Option { + #[derive(Deserialize)] + struct BuildFlagsToml { + dwarf: Option, + } + + let candidate = fastled_dir + .join("src") + .join("platforms") + .join("wasm") + .join("compiler") + .join("build_flags.toml"); + let text = std::fs::read_to_string(&candidate).ok()?; + let parsed: BuildFlagsToml = toml::from_str(&text).ok()?; + parsed.dwarf +} + +/// Resolves browser-issued DWARF paths to absolute file paths on disk. +#[derive(Debug, Clone)] +pub struct DebugSymbolResolver { + config: DebugSymbolConfig, +} + +impl DebugSymbolResolver { + pub fn new(config: DebugSymbolConfig) -> Self { + Self { config } + } + + pub fn config(&self) -> &DebugSymbolConfig { + &self.config + } + + fn known_prefixes(&self) -> [&str; 3] { + [ + self.config.prefixes.fastled_prefix.as_str(), + self.config.prefixes.sketch_prefix.as_str(), + self.config.prefixes.dwarf_prefix.as_str(), + ] + } + + /// Strip leading garbage from a DWARF-issued path so that what remains + /// starts with one of the configured prefixes (e.g. + /// `/build/fastledsource/FastLED.h` → `fastledsource/FastLED.h`). + /// Returns `None` if no prefix is present. + pub fn prune_path(&self, request_path: &str) -> Option { + let normalized = normalize_input_path(request_path); + let prefixes = self.known_prefixes(); + + for prefix in &prefixes { + if normalized == *prefix { + return Some(normalized); + } + let with_slash = format!("{prefix}/"); + if normalized.starts_with(&with_slash) { + return Some(normalized); + } + } + + let parts: Vec<&str> = normalized.split('/').filter(|p| !p.is_empty()).collect(); + let mut prefix_index: Option = None; + for (idx, part) in parts.iter().enumerate() { + if prefixes.contains(part) { + prefix_index = Some(idx); + } + } + let idx = prefix_index?; + let result = parts[idx..].join("/"); + Some(result) + } + + /// Map a DWARF request path to a real file. Errors: + /// - `Invalid`: traversal attempts, missing prefix, mismatched root + /// - `NotFound`: prefix and root matched but the file doesn't exist + pub fn resolve(&self, request_path: &str, check_exists: bool) -> Result { + if request_path.split(['/', '\\']).any(|seg| seg == "..") { + return Err(ResolveError::Invalid(format!( + "invalid path: {request_path}" + ))); + } + + let pruned = self + .prune_path(request_path) + .ok_or_else(|| ResolveError::Invalid(format!("invalid path: {request_path}")))?; + + let mut normalized = pruned.replace('\\', "/"); + normalized = normalized.trim_start_matches('/').to_string(); + + // `dwarfsource` is a sibling-prefix that wraps the named-root form, + // e.g. `dwarfsource/emsdk/upstream/.../cache.h`. Strip the outer + // prefix and let the rest match the actual root. + let dwarf_prefix_with_slash = format!("{}/", self.config.prefixes.dwarf_prefix); + if normalized.starts_with(&dwarf_prefix_with_slash) { + normalized = normalized[dwarf_prefix_with_slash.len()..].to_string(); + } else if normalized == self.config.prefixes.dwarf_prefix { + return Err(ResolveError::Invalid(format!( + "invalid path: {request_path}" + ))); + } + + for (prefix, root) in self.config.source_roots() { + let target = if normalized == prefix { + root.clone() + } else { + let with_slash = format!("{prefix}/"); + if let Some(suffix) = normalized.strip_prefix(&with_slash) { + root.join(suffix) + } else { + continue; + } + }; + + return finalize_target(&target, &root, check_exists, request_path); + } + + // Last-ditch fallback: try the path as if it were already relative to + // FastLED's include root. + if let Some(fastled_dir) = &self.config.fastled_dir { + let fastled_root = canonicalize_or(&fastled_dir.join("src")); + let target = fastled_root.join(&normalized); + if let Ok(resolved) = + finalize_target(&target, &fastled_root, check_exists, request_path) + { + return Ok(resolved); + } + } + + Err(ResolveError::Invalid(format!( + "invalid path: {request_path}" + ))) + } +} + +fn finalize_target( + target: &Path, + root: &Path, + check_exists: bool, + request_path: &str, +) -> Result { + let resolved_target = lexically_normalize(target); + let resolved_root = lexically_normalize(root); + if !is_within(&resolved_target, &resolved_root) { + return Err(ResolveError::Invalid(format!( + "invalid path: {request_path}" + ))); + } + if check_exists && !resolved_target.exists() { + return Err(ResolveError::NotFound(format!( + "could not find path {}", + resolved_target.display() + ))); + } + Ok(resolved_target) +} + +fn lexically_normalize(path: &Path) -> PathBuf { + let mut out = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => { + out.pop(); + } + Component::CurDir => {} + other => out.push(other.as_os_str()), + } + } + if let Ok(canonical) = path.canonicalize() { + return canonical; + } + out +} + +fn is_within(path: &Path, root: &Path) -> bool { + path.starts_with(root) +} + +fn normalize_input_path(request_path: &str) -> String { + request_path + .trim() + .replace('\\', "/") + .trim_start_matches('/') + .to_string() +} + +#[derive(Debug)] +pub enum ResolveError { + Invalid(String), + NotFound(String), +} + +impl std::fmt::Display for ResolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ResolveError::Invalid(msg) | ResolveError::NotFound(msg) => f.write_str(msg), + } + } +} + +impl std::error::Error for ResolveError {} + +/// Best-effort guess at the emscripten install root, used to expose the +/// `emsdk` source prefix when DWARF entries point at emsdk headers. +pub fn guess_emsdk_path() -> Option { + if let Some(emsdk) = std::env::var_os("EMSDK") { + return Some(PathBuf::from(emsdk)); + } + if let Some(install_dir) = std::env::var_os("FASTLED_EMSCRIPTEN_DIR").map(PathBuf::from) { + return install_dir.parent().map(Path::to_path_buf); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + fn setup_dirs() -> (TempDir, PathBuf, PathBuf, PathBuf) { + let tmp = TempDir::new().unwrap(); + let sketch_dir = tmp.path().join("sketch"); + let fastled_dir = tmp.path().join("FastLED"); + let emsdk_dir = tmp.path().join("emsdk"); + + fs::create_dir_all(sketch_dir.join("src")).unwrap(); + fs::create_dir_all(fastled_dir.join("src")).unwrap(); + fs::create_dir_all(emsdk_dir.join("upstream").join("emscripten")).unwrap(); + + fs::write(sketch_dir.join("src").join("demo.ino"), "sketch").unwrap(); + fs::write(fastled_dir.join("src").join("FastLED.h"), "fastled").unwrap(); + fs::write( + emsdk_dir + .join("upstream") + .join("emscripten") + .join("cache.h"), + "emsdk", + ) + .unwrap(); + + (tmp, sketch_dir, fastled_dir, emsdk_dir) + } + + #[test] + fn resolves_sketch_fastled_and_emsdk_prefixes() { + let (_tmp, sketch_dir, fastled_dir, emsdk_dir) = setup_dirs(); + let resolver = DebugSymbolResolver::new(load_debug_symbol_config( + sketch_dir.clone(), + Some(fastled_dir.clone()), + Some(emsdk_dir.clone()), + )); + + let sketch = resolver.resolve("sketchsource/src/demo.ino", true).unwrap(); + assert!(sketch.ends_with("demo.ino")); + + let fastled = resolver.resolve("fastledsource/FastLED.h", true).unwrap(); + assert!(fastled.ends_with("FastLED.h")); + + let emsdk = resolver + .resolve("dwarfsource/emsdk/upstream/emscripten/cache.h", true) + .unwrap(); + assert!(emsdk.ends_with("cache.h")); + } + + #[test] + fn rejects_traversal_attempts() { + let (_tmp, sketch_dir, _, _) = setup_dirs(); + let resolver = DebugSymbolResolver::new(load_debug_symbol_config(sketch_dir, None, None)); + let err = resolver + .resolve("dwarfsource/../../secret.txt", true) + .unwrap_err(); + match err { + ResolveError::Invalid(_) => {} + _ => panic!("expected Invalid"), + } + } + + #[test] + fn missing_file_returns_not_found() { + let (_tmp, sketch_dir, _, _) = setup_dirs(); + let resolver = DebugSymbolResolver::new(load_debug_symbol_config(sketch_dir, None, None)); + let err = resolver + .resolve("sketchsource/missing.cpp", true) + .unwrap_err(); + match err { + ResolveError::NotFound(_) => {} + _ => panic!("expected NotFound"), + } + } + + #[test] + fn prune_strips_leading_directories_before_prefix() { + let (_tmp, sketch_dir, fastled_dir, _) = setup_dirs(); + let resolver = DebugSymbolResolver::new(load_debug_symbol_config( + sketch_dir, + Some(fastled_dir), + None, + )); + let pruned = resolver + .prune_path("/build/wasm/fastledsource/FastLED.h") + .unwrap(); + assert_eq!(pruned, "fastledsource/FastLED.h"); + } + + #[test] + fn invalid_path_with_no_prefix_returns_invalid() { + let (_tmp, sketch_dir, _, _) = setup_dirs(); + let resolver = DebugSymbolResolver::new(load_debug_symbol_config(sketch_dir, None, None)); + let err = resolver.resolve("nothing/to/see/here.h", true).unwrap_err(); + match err { + ResolveError::Invalid(_) => {} + _ => panic!("expected Invalid"), + } + } + + #[test] + fn build_flags_toml_overrides_prefixes() { + let (_tmp, sketch_dir, fastled_dir, _) = setup_dirs(); + let compiler_dir = fastled_dir.join("src/platforms/wasm/compiler"); + fs::create_dir_all(&compiler_dir).unwrap(); + fs::write( + compiler_dir.join("build_flags.toml"), + r#" +[dwarf] +fastled_prefix = "fl_src" +sketch_prefix = "fl_sketch" +dwarf_prefix = "fl_dwarf" +"#, + ) + .unwrap(); + + let config = load_debug_symbol_config(sketch_dir, Some(fastled_dir), None); + assert_eq!(config.prefixes.fastled_prefix, "fl_src"); + assert_eq!(config.prefixes.sketch_prefix, "fl_sketch"); + assert_eq!(config.prefixes.dwarf_prefix, "fl_dwarf"); + } +} diff --git a/crates/fastled-cli/src/install.rs b/crates/fastled-cli/src/install.rs index 4a3744a7..6c29b111 100644 --- a/crates/fastled-cli/src/install.rs +++ b/crates/fastled-cli/src/install.rs @@ -78,8 +78,47 @@ fn fetch_manifest(url: &str) -> Result { .error_for_status() .with_context(|| format!("manifest fetch returned error for {url}"))?; let text = response.text().context("read manifest body")?; - serde_json::from_str::(&text) - .with_context(|| format!("parse manifest JSON from {url}")) + parse_platform_manifest(&text).with_context(|| format!("parse manifest JSON from {url}")) +} + +/// Parse a platform manifest, accepting both the current schema (`versions` +/// sub-map) and the historical layout where version keys live at the top +/// level next to `latest`. The two formats coexist on the assets server +/// today, so the CLI has to handle both. +fn parse_platform_manifest(text: &str) -> Result { + if let Ok(parsed) = serde_json::from_str::(text) { + return Ok(parsed); + } + + let value: serde_json::Value = serde_json::from_str(text)?; + let object = value + .as_object() + .ok_or_else(|| anyhow::anyhow!("manifest is not a JSON object"))?; + + let latest = object + .get("latest") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| anyhow::anyhow!("manifest is missing required field `latest`"))? + .to_string(); + + let mut versions = BTreeMap::new(); + for (key, value) in object { + if key == "latest" { + continue; + } + let info: VersionInfo = serde_json::from_value(value.clone()) + .with_context(|| format!("parse version entry `{key}`"))?; + versions.insert(key.clone(), info); + } + + if versions.is_empty() { + bail!("manifest does not declare any versions"); + } + if !versions.contains_key(&latest) { + bail!("manifest `latest`={latest} has no matching version entry"); + } + + Ok(PlatformManifest { latest, versions }) } /// Download each part to `parts_dir`, verify its SHA256, then concatenate the @@ -1177,4 +1216,44 @@ mod tests { assert_eq!(parts.len(), 1); assert!(parts[0].href.ends_with(".part-aa")); } + + /// Regression for issue #111: the win/x86_64 manifest publishes version + /// entries as siblings of `latest` rather than under a `versions` map. + /// The CLI must accept that legacy shape. + #[test] + fn parses_legacy_flat_manifest_shape() { + let text = r#"{ + "latest": "4.0.19", + "4.0.19": { + "href": "https://example.com/em-4.0.19.tar.zst", + "sha256": "b19c2e35b863eb17866034f917d7957514645e179e9d22800729b0dcbb2aa2e2" + } + }"#; + let manifest = parse_platform_manifest(text).expect("parse legacy manifest"); + assert_eq!(manifest.latest, "4.0.19"); + let entry = manifest + .versions + .get("4.0.19") + .expect("entry for latest version"); + assert_eq!(entry.sha256.len(), 64); + assert!(entry.href.ends_with(".tar.zst")); + } + + #[test] + fn legacy_manifest_with_no_versions_errors() { + let text = r#"{ "latest": "4.0.19" }"#; + assert!(parse_platform_manifest(text).is_err()); + } + + #[test] + fn legacy_manifest_with_unknown_latest_errors() { + let text = r#"{ + "latest": "9.9.9", + "4.0.19": { + "href": "https://example.com/em.tar.zst", + "sha256": "abcdef" + } + }"#; + assert!(parse_platform_manifest(text).is_err()); + } } diff --git a/crates/fastled-cli/src/lib.rs b/crates/fastled-cli/src/lib.rs index 9b765a53..3e35372e 100644 --- a/crates/fastled-cli/src/lib.rs +++ b/crates/fastled-cli/src/lib.rs @@ -4,12 +4,15 @@ use clap::Parser; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::ExitCode; +use std::sync::{Arc, RwLock}; mod archive; mod build; +pub mod debug_symbols; pub mod frontend; pub mod install; mod keyboard; +pub mod path; pub mod project; pub mod runtime; mod server; @@ -188,6 +191,7 @@ fn run_native_compile_streaming( sketch_dir: &Path, force_clean: bool, tx: &tokio::sync::broadcast::Sender, + debug_symbols: &server::DebugSymbolHandle, ) -> bool { if force_clean { purge_fastled_cache(cli.fastled_path.as_deref()); @@ -204,6 +208,14 @@ fn run_native_compile_streaming( match build::run_build(&request) { Ok(result) => { emit_build_result_logs(&result, tx); + if result.success { + update_debug_symbol_resolver( + debug_symbols, + sketch_dir, + result.fastled_dir.as_deref(), + result.emsdk_root.as_deref(), + ); + } result.success } Err(err) => { @@ -217,6 +229,25 @@ fn run_native_compile_streaming( } } +fn update_debug_symbol_resolver( + handle: &server::DebugSymbolHandle, + sketch_dir: &Path, + fastled_dir: Option<&Path>, + emsdk_root: Option<&Path>, +) { + let resolver = + debug_symbols::DebugSymbolResolver::new(debug_symbols::load_debug_symbol_config( + sketch_dir.to_path_buf(), + fastled_dir.map(Path::to_path_buf), + emsdk_root + .map(Path::to_path_buf) + .or_else(debug_symbols::guess_emsdk_path), + )); + if let Ok(mut guard) = handle.write() { + *guard = Some(resolver); + } +} + // --------------------------------------------------------------------------- // Compile + serve + watch (replaces Flask-based flow) // --------------------------------------------------------------------------- @@ -252,13 +283,23 @@ fn compile_and_serve(dir: &str, cli: &Cli) -> ExitCode { // Broadcast channel for SSE streaming to browser. let (tx, _rx) = tokio::sync::broadcast::channel::(256); + // Shared DWARF source resolver populated after the first successful build. + let debug_symbols: server::DebugSymbolHandle = Arc::new(RwLock::new(None)); + // Write initial compiling status for polling fallback. write_build_status(&output_dir, "compiling", "Compiling..."); let rt = tokio::runtime::Runtime::new().expect("failed to create tokio runtime"); rt.block_on(async { // Start the Rust HTTP server (background tokio task). - let addr = match server::start_server(output_dir.clone(), 0, Some(tx.clone())).await { + let addr = match server::start_server( + output_dir.clone(), + 0, + Some(tx.clone()), + debug_symbols.clone(), + ) + .await + { Ok(a) => a, Err(e) => { eprintln!("fastled: failed to start server: {e}"); @@ -283,7 +324,8 @@ fn compile_and_serve(dir: &str, cli: &Cli) -> ExitCode { r#"{"type":"status","status":"compiling","message":"Compiling..."}"#, ); - let success = run_native_compile_streaming(cli, &sketch_dir, cli.purge, &tx); + let success = + run_native_compile_streaming(cli, &sketch_dir, cli.purge, &tx, &debug_symbols); if success { write_build_status(&output_dir, "success", "Done"); @@ -333,7 +375,8 @@ fn compile_and_serve(dir: &str, cli: &Cli) -> ExitCode { r#"{"type":"status","status":"compiling","message":"Recompiling..."}"#, ); - let success = run_native_compile_streaming(cli, &sketch_dir, false, &tx); + let success = + run_native_compile_streaming(cli, &sketch_dir, false, &tx, &debug_symbols); if success { println!("Recompilation successful!"); write_build_status(&output_dir, "success", "Done"); @@ -409,7 +452,8 @@ struct Cli { #[arg(long)] no_https: bool, - /// Use the latest release when initialising examples with --init (default behaviour). + /// Use the latest tagged FastLED release when initialising examples with --init. + /// Defaults to `master`; tagged releases older than the meson migration cannot be built. #[arg(long)] latest: bool, @@ -459,9 +503,19 @@ fn validate_init_ref_flags(cli: &Cli) -> Result<(), &'static str> { fn requested_init_ref(cli: &Cli) -> Option<&str> { if cli.latest { + // `--latest` opts into the most recent tagged FastLED release. The + // build path only supports refs that include `meson.build`, so users + // who pass `--latest` are responsible for a release new enough to + // ship the meson backend. None + } else if let Some(explicit) = cli.commit.as_deref().or(cli.branch.as_deref()) { + Some(explicit) } else { - cli.commit.as_deref().or(cli.branch.as_deref()) + // Default to `master`. Tagged releases before the meson migration + // (≤ 3.10.x) do not contain `meson.build`, so the unconditional + // "latest release" default produced sketches that the build path + // could not compile. + Some("master") } } @@ -778,11 +832,13 @@ fn run_native_init(cli: &Cli, example: Option<&str>) -> ExitCode { } } }; + // Pin the local repo so subsequent builds use the same checkout. + let local_ref = local_repo.to_string_lossy().into_owned(); return match project::init_example_from_repo( &local_repo, &selected_example, &output_dir, - None, + Some(local_ref.as_str()), ) { Ok(out) => { println!("Project initialized at {}", out.display()); @@ -838,19 +894,22 @@ fn run_native_init(cli: &Cli, example: Option<&str>) -> ExitCode { selected_example ); + // Always pin the resolved ref into `fastled.json` so that a later + // `fastled --just-compile ` builds against the same FastLED + // checkout the sketch was scaffolded from. Without this, the build + // path defaulted to `master`, which broke compiles when the latest + // release used different header layouts than master. match project::init_example_from_repo( &repo_root, &selected_example, &output_dir, - ref_name.as_ref().map(|_| resolved_ref.as_str()), + Some(resolved_ref.as_str()), ) { Ok(out) => { - if ref_name.is_some() { - println!( - "Saved ref '{resolved_ref}' to {}", - out.join("fastled.json").display() - ); - } + println!( + "Saved ref '{resolved_ref}' to {}", + out.join("fastled.json").display() + ); println!("Project initialized at {}", out.display()); println!("\nInitialized FastLED project in {}", out.display()); println!("Use 'fastled {}' to compile the project.", out.display()); @@ -918,13 +977,14 @@ fn serve_directory(dir: &str) -> ExitCode { let rt = tokio::runtime::Runtime::new().expect("failed to create tokio runtime"); rt.block_on(async { - let addr = match server::start_server(path.clone(), 0, None).await { - Ok(a) => a, - Err(e) => { - eprintln!("fastled: failed to start server: {e}"); - return ExitCode::FAILURE; - } - }; + let addr = + match server::start_server(path.clone(), 0, None, Arc::new(RwLock::new(None))).await { + Ok(a) => a, + Err(e) => { + eprintln!("fastled: failed to start server: {e}"); + return ExitCode::FAILURE; + } + }; let url = format!("http://{addr}"); println!("Serving {dir} at {url}"); diff --git a/crates/fastled-cli/src/path.rs b/crates/fastled-cli/src/path.rs new file mode 100644 index 00000000..14ed7b76 --- /dev/null +++ b/crates/fastled-cli/src/path.rs @@ -0,0 +1,268 @@ +//! Cross-platform path utilities. +//! +//! `NormalizedPath` is a thin wrapper around `PathBuf` whose constructor +//! enforces a normalized representation: +//! - Lexically resolves `.` and `..` (no filesystem access, no symlink walk). +//! - Strips the Windows long-path prefix (`\\?\`) that +//! [`std::fs::canonicalize`] adds. Many external tools (meson, Python's +//! `open()`, emcc) cannot handle that prefix. +//! - Builds a case-insensitive comparison key on Windows / macOS for stable +//! equality and hashing across spellings. +//! +//! This is a focused port of `zccache_core::path::NormalizedPath` (see +//! https://github.com/zackees/zccache/blob/main/crates/zccache/src/core/path.rs). +//! It exists to give `fastled-cli` a single boundary type for any path that +//! crosses into an external process, so regressions like #114 cannot recur. + +use std::cmp::Ordering; +use std::ffi::OsStr; +use std::hash::{Hash, Hasher}; +use std::ops::Deref; +use std::path::{Component, Path, PathBuf}; + +/// A normalized, platform-aware path. +#[derive(Debug, Clone)] +pub struct NormalizedPath { + path: PathBuf, + case_key: Option, +} + +impl NormalizedPath { + /// Build a normalized path from anything convertible to `Path`. + pub fn new(path: impl AsRef) -> Self { + let normalized = normalize(path.as_ref()); + let case_key = if cfg!(windows) || cfg!(target_os = "macos") { + Some(normalize_for_key(&normalized)) + } else { + None + }; + Self { + path: normalized, + case_key, + } + } + + /// Borrow as a `Path`. + #[must_use] + pub fn as_path(&self) -> &Path { + &self.path + } + + /// Consume into an owned `PathBuf`. Use this only when handing to an API + /// that requires `PathBuf` — at that point you've crossed the boundary + /// where the wrapper's invariants stop being checked. + #[must_use] + pub fn into_path_buf(self) -> PathBuf { + self.path + } + + /// Case-insensitive comparison key. Populated on Windows / macOS. + #[must_use] + pub fn case_key(&self) -> Option<&str> { + self.case_key.as_deref() + } + + /// Build a new normalized path by joining a segment. + #[must_use] + pub fn join(&self, segment: impl AsRef) -> Self { + Self::new(self.path.join(segment)) + } + + /// Display the normalized path as a string. Round-trips through + /// `String::from(p.display().to_string())`; not lossless for non-UTF8 + /// paths but matches what we already do everywhere else. + #[must_use] + pub fn display_string(&self) -> String { + self.path.display().to_string() + } +} + +impl PartialEq for NormalizedPath { + fn eq(&self, other: &Self) -> bool { + normalize_for_key(&self.path) == normalize_for_key(&other.path) + } +} + +impl Eq for NormalizedPath {} + +impl Hash for NormalizedPath { + fn hash(&self, state: &mut H) { + normalize_for_key(&self.path).hash(state); + } +} + +impl PartialOrd for NormalizedPath { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for NormalizedPath { + fn cmp(&self, other: &Self) -> Ordering { + normalize_for_key(&self.path).cmp(&normalize_for_key(&other.path)) + } +} + +impl AsRef for NormalizedPath { + fn as_ref(&self) -> &Path { + self.as_path() + } +} + +impl AsRef for NormalizedPath { + fn as_ref(&self) -> &OsStr { + self.as_path().as_os_str() + } +} + +impl Deref for NormalizedPath { + type Target = Path; + + fn deref(&self) -> &Self::Target { + self.as_path() + } +} + +impl From for NormalizedPath { + fn from(path: PathBuf) -> Self { + Self::new(path) + } +} + +impl From<&Path> for NormalizedPath { + fn from(path: &Path) -> Self { + Self::new(path) + } +} + +impl From<&str> for NormalizedPath { + fn from(path: &str) -> Self { + Self::new(path) + } +} + +impl From for NormalizedPath { + fn from(path: String) -> Self { + Self::new(path) + } +} + +/// Lexically normalize a path: resolve `.` / `..` and strip the Windows +/// long-path prefix. Does not touch the filesystem. +#[must_use] +pub fn normalize(path: &Path) -> PathBuf { + let stripped = strip_windows_long_path_prefix(path); + let mut components = Vec::new(); + for component in stripped.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + if let Some(Component::Normal(_)) = components.last() { + components.pop(); + } else { + components.push(component); + } + } + other => components.push(other), + } + } + components.iter().collect() +} + +/// Strip a leading `\\?\` (and `\\?\UNC\` → `\\`) from a Windows long-path +/// form. No-op on non-Windows and on paths that don't carry the prefix. +fn strip_windows_long_path_prefix(path: &Path) -> PathBuf { + if !cfg!(windows) { + return path.to_path_buf(); + } + let s = path.to_string_lossy(); + if let Some(rest) = s.strip_prefix(r"\\?\UNC\") { + return PathBuf::from(format!(r"\\{rest}")); + } + if let Some(rest) = s.strip_prefix(r"\\?\") { + return PathBuf::from(rest.to_string()); + } + path.to_path_buf() +} + +/// Stable string key for a path: separator normalization, long-path prefix +/// removal, case folding on case-insensitive filesystems. +#[must_use] +pub fn normalize_for_key(path: &Path) -> String { + let normalized = normalize(path); + + #[cfg(windows)] + { + let mut s = normalized.to_string_lossy().replace('\\', "/"); + if let Some(stripped) = s.strip_prefix("//?/") { + s = stripped.to_string(); + } + s.make_ascii_lowercase(); + s + } + #[cfg(target_os = "macos")] + { + normalized.to_string_lossy().to_lowercase() + } + #[cfg(not(any(windows, target_os = "macos")))] + { + normalized.to_string_lossy().into_owned() + } +} + +/// Canonicalize via the filesystem, then normalize. Falls back to the input +/// path if canonicalization fails (e.g. the path doesn't exist yet). +#[must_use] +pub fn canonicalize_normalized(path: &Path) -> NormalizedPath { + match std::fs::canonicalize(path) { + Ok(p) => NormalizedPath::new(p), + Err(_) => NormalizedPath::new(path), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_strips_curdir_and_parentdir() { + assert_eq!(normalize(Path::new("a/./b")), PathBuf::from("a/b")); + assert_eq!(normalize(Path::new("a/b/../c")), PathBuf::from("a/c")); + } + + #[cfg(windows)] + #[test] + fn normalize_strips_windows_long_path_prefix() { + let normalized = normalize(Path::new(r"\\?\C:\Users\me\repo")); + assert_eq!(normalized, PathBuf::from(r"C:\Users\me\repo")); + } + + #[cfg(windows)] + #[test] + fn normalize_handles_unc_shares() { + let normalized = normalize(Path::new(r"\\?\UNC\server\share\dir")); + assert_eq!(normalized, PathBuf::from(r"\\server\share\dir")); + } + + #[cfg(windows)] + #[test] + fn normalize_for_key_folds_case_and_separators() { + let a = normalize_for_key(Path::new(r"\\?\C:\Work\src\..\src\main.cpp")); + let b = normalize_for_key(Path::new("c:/work/src/main.cpp")); + assert_eq!(a, b); + } + + #[test] + fn normalized_path_equality_uses_normalized_form() { + let a = NormalizedPath::new("a/./b/c"); + let b = NormalizedPath::new("a/b/c"); + assert_eq!(a, b); + } + + #[test] + fn normalized_path_join_returns_normalized() { + let base = NormalizedPath::new("/tmp/work"); + let joined = base.join("./sub/../target"); + assert_eq!(joined.as_path(), Path::new("/tmp/work/target")); + } +} diff --git a/crates/fastled-cli/src/server.rs b/crates/fastled-cli/src/server.rs index be4b1d24..5895a572 100644 --- a/crates/fastled-cli/src/server.rs +++ b/crates/fastled-cli/src/server.rs @@ -8,20 +8,32 @@ use std::convert::Infallible; use std::net::SocketAddr; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use axum::extract::State; use axum::http::{header, HeaderValue, Method, StatusCode}; use axum::response::sse::{Event, KeepAlive, Sse}; use axum::response::{Html, IntoResponse, Response}; -use axum::routing::get; -use axum::Router; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use serde::Deserialize; +use serde_json::json; use tokio::sync::broadcast; use tokio_stream::wrappers::BroadcastStream; use tokio_stream::StreamExt; use tower_http::cors::{Any, CorsLayer}; use tower_http::set_header::SetResponseHeaderLayer; +use crate::debug_symbols::{DebugSymbolResolver, ResolveError}; + +/// Shared, mutable handle to a DWARF source resolver. +/// +/// The resolver is unknown when the server starts (we haven't run a build +/// yet) and is filled in after the first successful build. Wrapping it in +/// `Arc>>` lets the watch loop swap it in without bouncing +/// the server. +pub type DebugSymbolHandle = Arc>>; + // --------------------------------------------------------------------------- // Loading page (shown while compilation is in progress) // --------------------------------------------------------------------------- @@ -102,6 +114,9 @@ struct AppState { /// Broadcast channel for SSE build streaming. `None` when serving a /// static directory (no compilation happening). build_tx: Option>, + /// Shared resolver for DWARF source paths. Empty until a successful + /// build populates it. + debug_symbols: DebugSymbolHandle, } // --------------------------------------------------------------------------- @@ -174,6 +189,87 @@ async fn build_stream(State(state): State) -> Response { .into_response() } +// --------------------------------------------------------------------------- +// DWARF debug source handlers +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +struct DwarfSourceRequest { + path: String, +} + +/// `POST /dwarfsource` — given `{"path": ""}`, return the source +/// text for the file the path maps to. Empty/missing payload, or a path that +/// escapes the configured roots, returns 400. Unknown files return 404. +async fn dwarf_source( + State(state): State, + Json(payload): Json, +) -> Response { + let resolver = match state.debug_symbols.read() { + Ok(guard) => guard.clone(), + Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR).into_response(), + }; + + let Some(resolver) = resolver else { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "debug source resolver unavailable"})), + ) + .into_response(); + }; + + let request_path = payload.path.trim(); + if request_path.is_empty() { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"error": "missing path"})), + ) + .into_response(); + } + + match resolver.resolve(request_path, true) { + Ok(file) => match tokio::fs::read(&file).await { + Ok(bytes) => ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/plain; charset=utf-8")], + bytes, + ) + .into_response(), + Err(err) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": err.to_string()})), + ) + .into_response(), + }, + Err(ResolveError::NotFound(msg)) => { + (StatusCode::NOT_FOUND, Json(json!({ "error": msg }))).into_response() + } + Err(ResolveError::Invalid(msg)) => { + (StatusCode::BAD_REQUEST, Json(json!({ "error": msg }))).into_response() + } + } +} + +/// `GET /debug/source-roots` — return the named source roots the resolver +/// knows about. Useful for tooling that wants to verify configuration. +async fn debug_source_roots(State(state): State) -> Response { + let resolver = match state.debug_symbols.read() { + Ok(guard) => guard.clone(), + Err(_) => return (StatusCode::INTERNAL_SERVER_ERROR).into_response(), + }; + + let roots = match resolver { + Some(resolver) => resolver + .config() + .source_roots() + .into_iter() + .map(|(prefix, path)| json!({"prefix": prefix, "path": path.display().to_string()})) + .collect::>(), + None => Vec::new(), + }; + Json(json!({ "roots": roots })).into_response() +} + fn mime_for_path(path: &str) -> &'static str { match path.rsplit('.').next().unwrap_or("") { "html" => "text/html; charset=utf-8", @@ -200,20 +296,25 @@ fn mime_for_path(path: &str) -> &'static str { /// Start the HTTP server in a background tokio task. /// /// Returns the actual address the server bound to (useful when port 0 is -/// requested for automatic assignment). +/// requested for automatic assignment). `debug_symbols` is shared with the +/// caller so a build can populate the resolver after the server starts. pub async fn start_server( serve_dir: PathBuf, port: u16, build_tx: Option>, + debug_symbols: DebugSymbolHandle, ) -> anyhow::Result { let state = AppState { serve_dir: Arc::new(serve_dir), build_tx, + debug_symbols, }; let app = Router::new() .route("/", get(serve_index)) .route("/build-stream", get(build_stream)) + .route("/dwarfsource", post(dwarf_source)) + .route("/debug/source-roots", get(debug_source_roots)) .route("/{*path}", get(serve_file)) .layer(SetResponseHeaderLayer::overriding( axum::http::HeaderName::from_static("cross-origin-embedder-policy"), @@ -260,10 +361,14 @@ mod tests { use super::*; use std::fs; + fn empty_handle() -> DebugSymbolHandle { + Arc::new(RwLock::new(None)) + } + /// Helper: create a temp dir, start the server, return (addr, dir). async fn setup_server() -> (SocketAddr, tempfile::TempDir) { let dir = tempfile::tempdir().unwrap(); - let addr = start_server(dir.path().to_path_buf(), 0, None) + let addr = start_server(dir.path().to_path_buf(), 0, None, empty_handle()) .await .unwrap(); // Give the server a moment to bind. @@ -411,9 +516,14 @@ mod tests { async fn test_sse_endpoint_streams_events() { let dir = tempfile::tempdir().unwrap(); let (tx, _rx) = broadcast::channel::(16); - let addr = start_server(dir.path().to_path_buf(), 0, Some(tx.clone())) - .await - .unwrap(); + let addr = start_server( + dir.path().to_path_buf(), + 0, + Some(tx.clone()), + empty_handle(), + ) + .await + .unwrap(); tokio::time::sleep(std::time::Duration::from_millis(50)).await; let url = format!("http://{addr}/build-stream"); @@ -477,4 +587,108 @@ mod tests { "loading page should connect to /build-stream" ); } + + // ------------------------------------------------------------------ + // DWARF source endpoint tests + // ------------------------------------------------------------------ + + fn json_body(value: serde_json::Value) -> String { + value.to_string() + } + + async fn post_json(addr: SocketAddr, path: &str, body: serde_json::Value) -> reqwest::Response { + reqwest::Client::new() + .post(format!("http://{addr}{path}")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(json_body(body)) + .send() + .await + .unwrap() + } + + #[tokio::test] + async fn dwarfsource_without_resolver_returns_400() { + let (addr, _dir) = setup_server().await; + let resp = post_json( + addr, + "/dwarfsource", + serde_json::json!({"path": "sketchsource/foo.ino"}), + ) + .await; + assert_eq!(resp.status(), 400); + } + + #[tokio::test] + async fn debug_source_roots_empty_without_resolver() { + let (addr, _dir) = setup_server().await; + let resp = reqwest::get(format!("http://{addr}/debug/source-roots")) + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body: serde_json::Value = serde_json::from_str(&resp.text().await.unwrap()).unwrap(); + assert!(body["roots"].as_array().unwrap().is_empty()); + } + + #[tokio::test] + async fn dwarfsource_returns_resolved_file() { + use crate::debug_symbols::{load_debug_symbol_config, DebugSymbolResolver}; + + let dir = tempfile::tempdir().unwrap(); + let sketch_dir = dir.path().join("sketch"); + fs::create_dir_all(sketch_dir.join("src")).unwrap(); + let sketch_file = sketch_dir.join("src").join("demo.ino"); + fs::write(&sketch_file, "void setup() {}").unwrap(); + + let resolver = DebugSymbolResolver::new(load_debug_symbol_config(sketch_dir, None, None)); + let handle: DebugSymbolHandle = Arc::new(RwLock::new(Some(resolver))); + + let addr = start_server(dir.path().to_path_buf(), 0, None, handle.clone()) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let resp = post_json( + addr, + "/dwarfsource", + serde_json::json!({"path": "sketchsource/src/demo.ino"}), + ) + .await; + assert_eq!(resp.status(), 200); + let body = resp.text().await.unwrap(); + assert!(body.contains("void setup()")); + + let resp = reqwest::get(format!("http://{addr}/debug/source-roots")) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_str(&resp.text().await.unwrap()).unwrap(); + let roots = body["roots"].as_array().unwrap(); + assert!(!roots.is_empty()); + assert!(roots + .iter() + .any(|r| r["prefix"].as_str() == Some("sketchsource"))); + } + + #[tokio::test] + async fn dwarfsource_rejects_traversal() { + use crate::debug_symbols::{load_debug_symbol_config, DebugSymbolResolver}; + + let dir = tempfile::tempdir().unwrap(); + let sketch_dir = dir.path().join("sketch"); + fs::create_dir_all(&sketch_dir).unwrap(); + let resolver = DebugSymbolResolver::new(load_debug_symbol_config(sketch_dir, None, None)); + let handle: DebugSymbolHandle = Arc::new(RwLock::new(Some(resolver))); + + let addr = start_server(dir.path().to_path_buf(), 0, None, handle) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let resp = post_json( + addr, + "/dwarfsource", + serde_json::json!({"path": "sketchsource/../escape.txt"}), + ) + .await; + assert_eq!(resp.status(), 400); + } } diff --git a/crates/fastled-cli/src/wasm_build.rs b/crates/fastled-cli/src/wasm_build.rs index 69b309c1..b9ca1020 100644 --- a/crates/fastled-cli/src/wasm_build.rs +++ b/crates/fastled-cli/src/wasm_build.rs @@ -49,6 +49,13 @@ pub struct BuildResult { pub sketch_time_secs: f64, pub strategy: String, pub output: String, + /// Resolved FastLED checkout used for this build. Surfaces upward so that + /// the embedded HTTP server can wire DWARF source resolution against it. + pub fastled_dir: Option, + /// Resolved emscripten install root (the parent of `emscripten/`), if + /// known. Used to expose `emsdk/` source paths for DWARF entries that + /// point at emsdk headers. + pub emsdk_root: Option, } #[derive(Debug, Clone)] @@ -165,7 +172,15 @@ fn build_env(tools: &ToolPaths) -> Vec<(String, String)> { ("EMCC_SKIP_SANITY_CHECK".to_string(), "1".to_string()), ]; if cfg!(windows) { - env.push(("EMCC_CORES".to_string(), "128".to_string())); + // Match the host's logical core count so emscripten's parallel + // sub-builds (libc, compiler_rt, libcxx) don't spawn 128 simultaneous + // emcc subprocesses, which on Windows can crash with access + // violations under contention. Clamp to a reasonable maximum. + let cores = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8) + .min(32); + env.push(("EMCC_CORES".to_string(), cores.to_string())); } env } @@ -188,8 +203,11 @@ fn run_status(mut command: Command, label: &str) -> Result<()> { Ok(()) } +/// Canonicalize and lexically normalize, stripping the Windows `\\?\` +/// long-path prefix that breaks external tools (meson, Python, emcc). +/// See `crate::path::canonicalize_normalized` for details. fn normalize_path(path: &Path) -> PathBuf { - fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) + crate::path::canonicalize_normalized(path).into_path_buf() } fn resolve_example_name(sketch_dir: &Path, fastled_dir: &Path) -> (String, PathBuf, bool) { @@ -380,15 +398,18 @@ fn write_native_cross_file(fastled_dir: &Path, tools: &ToolPaths) -> Result String { + let escaped = value.replace('\'', "\\'"); + format!("'{escaped}'") +} + fn ensure_meson_configured( fastled_dir: &Path, tools: &ToolPaths, @@ -742,16 +770,47 @@ fn link_wasm( args.extend(get_link_flags(fastled_dir, mode)?); println!("[WASM] Linking final WASM module..."); - let mut command = command_with_env(&tools.python, tools); - command - .current_dir(fastled_dir) - .arg(&tools.empp) - .args(&args); - run_status(command, "em++ wasm link")?; + run_em_link_with_retries(fastled_dir, tools, &args)?; copy_linked_output(sketch_cache_dir, output_js)?; Ok(()) } +/// On Windows, emscripten builds its sysroot libraries (libc, compiler_rt, +/// libc++…) from source the first time we link. Individual `emcc` +/// subprocesses occasionally crash mid-build with a non-deterministic +/// access-violation-style exit code; the surrounding em++ link then aborts. +/// Because each completed library is cached to disk, retrying advances the +/// build by one library each time. A small retry loop lets the link +/// converge without exposing the upstream flakiness to users. See #116. +fn run_em_link_with_retries( + fastled_dir: &Path, + tools: &ToolPaths, + args: &[String], +) -> Result<()> { + let max_attempts = if cfg!(windows) { 6 } else { 1 }; + let mut last_err: Option = None; + for attempt in 1..=max_attempts { + let mut command = command_with_env(&tools.python, tools); + command + .current_dir(fastled_dir) + .arg(&tools.empp) + .args(args); + match run_status(command, "em++ wasm link") { + Ok(()) => return Ok(()), + Err(err) => { + if attempt < max_attempts { + println!( + "[WASM] em++ link attempt {attempt}/{max_attempts} failed; \ + retrying (sysroot libs cache progressively): {err:#}" + ); + } + last_err = Some(err); + } + } + } + Err(last_err.unwrap_or_else(|| anyhow::anyhow!("em++ wasm link failed"))) +} + fn copy_linked_output(sketch_cache_dir: &Path, output_js: &Path) -> Result<()> { let output_dir = output_js .parent() @@ -812,7 +871,19 @@ fn resolve_fastled_dir(request: &BuildRequest) -> Result<(PathBuf, Option/fastled.json` by `fastled --init` + // so the build always uses the same FastLED checkout the sketch was + // scaffolded against (master example layout drifts from older releases). + // Falls back to `master` only when no pin is recorded. + let pinned_ref = crate::project::read_fastled_json_ref(&request.sketch_dir); + if let Some(local_path) = pinned_ref + .as_deref() + .filter(|s| Path::new(s).join("library.json").is_file()) + { + return Ok((normalize_path(Path::new(local_path)), None)); + } + let ref_arg = pinned_ref.as_deref().or(Some("master")); + let repo = install::ensure_fastled_repo(ref_arg)?; let cache_root = repo .parent() .map(Path::to_path_buf) @@ -871,6 +942,10 @@ pub fn run_build(request: &BuildRequest) -> Result { let tools = resolve_tool_paths(&emscripten_install)?; let (fastled_dir, _cleanup) = resolve_fastled_dir(request)?; let fastled_dir = normalize_path(&fastled_dir); + let emsdk_root = emscripten_install + .parent() + .map(Path::to_path_buf) + .or_else(|| Some(emscripten_install.clone())); let (example_name, example_dir, _is_in_tree) = resolve_example_name(&normalize_path(&request.sketch_dir), &fastled_dir); @@ -936,6 +1011,8 @@ pub fn run_build(request: &BuildRequest) -> Result { sketch_time_secs: sketch_start.elapsed().as_secs_f64(), strategy, output: "Native Rust WASM build successful".to_string(), + fastled_dir: Some(fastled_dir.clone()), + emsdk_root: emsdk_root.clone(), }), Err(err) => Ok(BuildResult { success: false, @@ -944,6 +1021,8 @@ pub fn run_build(request: &BuildRequest) -> Result { sketch_time_secs: sketch_start.elapsed().as_secs_f64(), strategy, output: format!("Native Rust WASM build failed: {err:#}"), + fastled_dir: Some(fastled_dir.clone()), + emsdk_root: emsdk_root.clone(), }), } } @@ -962,6 +1041,20 @@ mod tests { assert!(in_tree); } + #[test] + fn meson_quote_wraps_in_single_quotes() { + assert_eq!(meson_quote("simple"), "'simple'"); + assert_eq!( + meson_quote("C:/path/with spaces/em.py"), + "'C:/path/with spaces/em.py'" + ); + } + + #[test] + fn meson_quote_escapes_embedded_apostrophes() { + assert_eq!(meson_quote("it's"), r"'it\'s'"); + } + #[test] fn resolve_example_maps_external_to_examples_leaf() { let root = PathBuf::from("/tmp/FastLED"); diff --git a/dylints/ban_std_pathbuf/Cargo.toml b/dylints/ban_std_pathbuf/Cargo.toml new file mode 100644 index 00000000..84edd841 --- /dev/null +++ b/dylints/ban_std_pathbuf/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "ban_std_pathbuf" +version = "0.1.0" +description = "Ban std::path::PathBuf outside the legacy allowlist (ported from zccache)" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# `5.0.0` on crates.io predates the 2026-03-18 rustc_session layout change +# used by newer nightlies / Rust 1.94-era toolchains. Track the upstream +# compatibility fix until a newer crates.io release is available. +dylint_linting = { git = "https://github.com/trailofbits/dylint", rev = "4bd91ce7729b74c7ee5664bbb588f7baf30b4a09" } + +[dev-dependencies] +dylint_testing = { git = "https://github.com/trailofbits/dylint", rev = "4bd91ce7729b74c7ee5664bbb588f7baf30b4a09" } + +[package.metadata.rust-analyzer] +rustc_private = true diff --git a/dylints/ban_std_pathbuf/README.md b/dylints/ban_std_pathbuf/README.md new file mode 100644 index 00000000..acf3c6f2 --- /dev/null +++ b/dylints/ban_std_pathbuf/README.md @@ -0,0 +1,35 @@ +# ban_std_pathbuf + +A dylint that bans `std::path::PathBuf` outside an explicit per-file +allowlist (`src/allowlist.txt`). Migrated call sites should use +`fastled_cli::path::NormalizedPath` instead, which strips Windows +long-path prefixes (`\\?\`) and applies case-insensitive comparison on +case-insensitive filesystems. + +## Why + +Issue #114 hit a Windows-only crash where `fs::canonicalize` produced +`\\?\C:\...` paths that meson and Python's `open()` couldn't handle. +Wrapping every boundary path in `NormalizedPath` prevents that class of +regression from coming back into a different module. + +## Layout + +- `src/lib.rs` — the lint pass itself (ported from zccache). +- `src/allowlist.txt` — file suffixes that may still use raw `PathBuf`. + +## Toolchain + +Builds against `nightly-2026-03-26` (see `rust-toolchain.toml`). The main +workspace stays on stable; this sub-crate is intentionally not in the +workspace. + +## Future work + +Add a `cargo dylint` driver script and wire it into `bash lint` so CI +enforces the lint. For now this directory is the lint source; running it +is manual until the driver lands. + +## Reference + +Ported from . diff --git a/dylints/ban_std_pathbuf/rust-toolchain.toml b/dylints/ban_std_pathbuf/rust-toolchain.toml new file mode 100644 index 00000000..3b6ccbe8 --- /dev/null +++ b/dylints/ban_std_pathbuf/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-03-26" +components = ["llvm-tools-preview", "rust-src", "rustc-dev"] diff --git a/dylints/ban_std_pathbuf/src/allowlist.txt b/dylints/ban_std_pathbuf/src/allowlist.txt new file mode 100644 index 00000000..153b954e --- /dev/null +++ b/dylints/ban_std_pathbuf/src/allowlist.txt @@ -0,0 +1,28 @@ +# Files that may still use raw `std::path::PathBuf`. +# +# Entries are matched as path suffixes (forward-slashes), so +# `crates/fastled-cli/src/foo.rs` matches both the absolute path the rustc +# source map emits and the relative form in diagnostics. +# +# New files should default to `fastled_cli::path::NormalizedPath`. Remove a +# line from this list when its file has finished migrating. + +# Wrapper itself: defines NormalizedPath in terms of PathBuf. Permanent. +crates/fastled-cli/src/path.rs + +# Migrated boundary type already wraps the canonicalize result. +crates/fastled-cli/src/wasm_build.rs + +# Legacy modules — migrate progressively, then strike the line. +crates/fastled-cli/src/lib.rs +crates/fastled-cli/src/debug_symbols.rs +crates/fastled-cli/src/server.rs +crates/fastled-cli/src/install.rs +crates/fastled-cli/src/watcher.rs +crates/fastled-cli/src/viewer.rs +crates/fastled-cli/src/runtime.rs +crates/fastled-cli/src/project.rs +crates/fastled-cli/src/main.rs +crates/fastled-cli/src/tauri_viewer.rs +crates/fastled-cli/src/frontend.rs +crates/fastled-cli/src/archive.rs diff --git a/dylints/ban_std_pathbuf/src/lib.rs b/dylints/ban_std_pathbuf/src/lib.rs new file mode 100644 index 00000000..0b54fab3 --- /dev/null +++ b/dylints/ban_std_pathbuf/src/lib.rs @@ -0,0 +1,149 @@ +#![feature(rustc_private)] + +extern crate rustc_errors; +extern crate rustc_hir; +extern crate rustc_span; + +use rustc_errors::DiagDecorator; +use rustc_hir::{def::Res, AmbigArg, Expr, ExprKind, Ty, TyKind}; +use rustc_lint::{LateContext, LateLintPass, LintContext}; +use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; + +dylint_linting::declare_late_lint! { + /// ### What it does + /// + /// Bans `std::path::PathBuf` outside the explicit legacy allowlist. + /// + /// ### Why is this bad? + /// + /// Raw `PathBuf` values do not carry fastled-cli's normalization + /// invariant. Windows long-path canonicalize() output (`\\?\C:\...`) + /// has crashed external tools we hand paths to (meson, Python's + /// `open()`, emcc) — see issue #114. Use + /// `fastled_cli::path::NormalizedPath` at every boundary instead. + /// + /// ### Known problems + /// + /// The workspace still has legacy `PathBuf` call sites. Those files + /// are temporarily allowlisted; remove entries from + /// `src/allowlist.txt` as they migrate. + /// + /// ### Example + /// + /// ```rust + /// use std::path::PathBuf; + /// let path = PathBuf::from("src/foo.rs"); + /// ``` + /// + /// Use instead: + /// + /// ```rust + /// use fastled_cli::path::NormalizedPath; + /// let path = NormalizedPath::new("src/foo.rs"); + /// ``` + pub BAN_STD_PATHBUF, + Deny, + "ban std::path::PathBuf outside the legacy allowlist" +} + +const PATHBUF_DEF_PATH: &[&str] = &["std", "path", "PathBuf"]; +const ALLOWLIST: &str = include_str!("allowlist.txt"); + +impl<'tcx> LateLintPass<'tcx> for BanStdPathbuf { + fn check_ty(&mut self, cx: &LateContext<'tcx>, ty: &'tcx Ty<'tcx, AmbigArg>) { + if is_allowlisted(cx, ty.span) { + return; + } + + if let TyKind::Path(qpath) = ty.kind { + let res = cx.qpath_res(&qpath, ty.hir_id); + if res_is_pathbuf(cx, res) { + emit_lint(cx, ty.span); + } + } + } + + fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'tcx>) { + if is_allowlisted(cx, expr.span) { + return; + } + + if let ExprKind::Path(qpath) = expr.kind { + let res = cx.qpath_res(&qpath, expr.hir_id); + if res_is_pathbuf_assoc(cx, res) { + emit_lint(cx, expr.span); + } + } + } +} + +fn emit_lint(cx: &LateContext<'_>, span: rustc_span::Span) { + cx.opt_span_lint( + BAN_STD_PATHBUF, + Some(span), + DiagDecorator(|diag| { + diag.primary_message( + "use fastled_cli::path::NormalizedPath instead of std::path::PathBuf", + ); + }), + ); +} + +fn is_allowlisted(cx: &LateContext<'_>, span: rustc_span::Span) -> bool { + let filename = match cx.sess().source_map().span_to_filename(span) { + FileName::Real(real_filename) => real_filename + .local_path() + .map(|path| path.to_string_lossy().into_owned()) + .unwrap_or_else(|| { + real_filename + .path(RemapPathScopeComponents::DIAGNOSTICS) + .to_string_lossy() + .into_owned() + }), + filename => filename + .display(RemapPathScopeComponents::DIAGNOSTICS) + .to_string(), + }; + let normalized = filename.replace('\\', "/"); + ALLOWLIST + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .any(|allowed| normalized.ends_with(allowed)) +} + +fn res_is_pathbuf(cx: &LateContext<'_>, res: Res) -> bool { + match res { + Res::Def(_, def_id) => def_path_starts_with(cx, def_id, PATHBUF_DEF_PATH), + _ => false, + } +} + +fn res_is_pathbuf_assoc(cx: &LateContext<'_>, res: Res) -> bool { + match res { + Res::Def(_, def_id) => { + let def_path = cx.get_def_path(def_id); + def_path.len() > PATHBUF_DEF_PATH.len() + && def_path + .iter() + .take(PATHBUF_DEF_PATH.len()) + .zip(PATHBUF_DEF_PATH.iter()) + .all(|(actual, expected)| *actual == Symbol::intern(expected)) + } + _ => false, + } +} + +fn def_path_starts_with( + cx: &LateContext<'_>, + def_id: rustc_hir::def_id::DefId, + prefix: &[&str], +) -> bool { + let def_path = cx.get_def_path(def_id); + def_path.len() >= prefix.len() + && def_path + .iter() + .take(prefix.len()) + .zip(prefix.iter()) + .all(|(actual, expected)| *actual == Symbol::intern(expected)) +}