From b07a7643af2399c59edf80ca838db7d04fc583fc Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 18:31:40 -0700 Subject: [PATCH 1/5] chore(open_browser): drop redundant wait_for_server HTTP probe The Rust CLI prints its listening URL to stdout once the socket is bound. spawn_http_server() already parses that URL via regex and uses it. The follow-up HTTP probe through httpx duplicated that gate without adding information; the fallback path keeps a 1s sleep for the case where the URL line wasn't seen in time. Removes one httpx callsite. --- src/fastled/open_browser.py | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/src/fastled/open_browser.py b/src/fastled/open_browser.py index 0230fa4a..9841802b 100644 --- a/src/fastled/open_browser.py +++ b/src/fastled/open_browser.py @@ -139,25 +139,6 @@ def cleanup_if_alive(): atexit.register(cleanup_if_alive) -def wait_for_server(port: int, timeout: int = 15) -> None: - """Wait for the HTTP server to start.""" - import httpx - - future_time = time.time() + timeout - hosts = ["localhost", "127.0.0.1"] - while future_time > time.time(): - for host in hosts: - try: - url = f"http://{host}:{port}" - response = httpx.get(url, timeout=1) - if response.status_code < 600: - return - except httpx.HTTPError: - continue - time.sleep(0.1) - raise TimeoutError("Could not connect to server") - - def spawn_http_server( fastled_js: Path, port: int | None = None, @@ -198,10 +179,10 @@ def spawn_http_server( break if actual_port is None: - # Fallback: wait a bit and hope the server is up + # The Rust CLI prints its listening URL once the socket is bound, so the + # URL parse above already gates "server is up". Only the fallback path + # (URL line not seen yet) needs a small grace period. time.sleep(1.0) - else: - wait_for_server(actual_port, timeout=15) if open_browser: if actual_port: From 90fb5edec750536621d686a4fcffd7ac6e22140d Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 18:40:36 -0700 Subject: [PATCH 2/5] chore(rust): own emscripten install in the Rust CLI (refs #59) The Python compile path used to download and extract the emscripten toolchain itself (httpx + pyzstd, via toolchain/emscripten_archive.py). That now lives in Rust: - crates/fastled-cli/src/install.rs (new) - ensure_emscripten_installed() fetches the platform manifest, downloads multipart parts with SHA-256 verification, extracts the .tar.zst, and writes the .emscripten config into ~/.fastled/toolchains/emscripten/{platform}/{arch}/{version}/. Uses ctcb-manifest (workspace dep) for the manifest schema and the existing archive.rs primitives for download/sha256/extract/config. - crates/fastled-cli/src/main.rs - compile_and_serve() now calls ensure_emscripten_installed() once, before invoking Python, and exports FASTLED_EMSCRIPTEN_DIR for the child process to consume. - crates/fastled-cli/src/archive.rs - dropped #![allow(dead_code)] since the module is now wired into the compile flow. Python: - src/fastled/toolchain/emscripten.py - _get_clang_tool_chain_emscripten_dir() reads FASTLED_EMSCRIPTEN_DIR first; ensure_clang_tool_chain_emscripten() is now a thin lookup (the install is upstream). The clang_tool_chain PyPI fallback was removed. - src/fastled/toolchain/emscripten_archive.py - deleted (Rust owns it). - pyproject.toml - dropped clang-tool-chain runtime dep. Net: Python runtime deps now httpx, fasteners, zcmds_win32 (Win-only). Lint clean, 118 unit tests pass. --- Cargo.lock | 137 ++++++++++- Cargo.toml | 3 + crates/fastled-cli/Cargo.toml | 3 + crates/fastled-cli/src/archive.rs | 4 - crates/fastled-cli/src/install.rs | 175 +++++++++++++ crates/fastled-cli/src/main.rs | 14 ++ pyproject.toml | 1 - src/fastled/toolchain/emscripten.py | 93 ++----- src/fastled/toolchain/emscripten_archive.py | 259 -------------------- 9 files changed, 347 insertions(+), 342 deletions(-) create mode 100644 crates/fastled-cli/src/install.rs delete mode 100644 src/fastled/toolchain/emscripten_archive.py diff --git a/Cargo.lock b/Cargo.lock index fb2bfb88..9b099cf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -716,6 +716,30 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ctcb-core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d666d682b69add1b1bea317cb3184018f21a0ff95f3a9d1b44887bd5b01961" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "ctcb-manifest" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97c47fc9cd2017787aad53c7a11fb8529291cadb369dc15ad041eb81fbc06221" +dependencies = [ + "anyhow", + "ctcb-core", + "serde", + "serde_json", +] + [[package]] name = "ctor" version = "0.2.9" @@ -832,13 +856,34 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys", + "dirs-sys 0.5.0", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.4.6", + "windows-sys 0.48.0", ] [[package]] @@ -849,7 +894,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.5.2", "windows-sys 0.61.2", ] @@ -1014,6 +1059,9 @@ dependencies = [ "axum", "clap", "crossterm", + "ctcb-manifest", + "dirs 5.0.1", + "flate2", "notify", "reqwest 0.12.28", "serde", @@ -3369,6 +3417,17 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -4316,7 +4375,7 @@ dependencies = [ "anyhow", "bytes", "cookie", - "dirs", + "dirs 6.0.0", "dunce", "embed_plist", "getrandom 0.3.4", @@ -4366,7 +4425,7 @@ checksum = "4bbc990d1dbf57a8e1c7fa2327f2a614d8b757805603c1b9ba5c81bade09fd4d" dependencies = [ "anyhow", "cargo_toml", - "dirs", + "dirs 6.0.0", "glob", "heck 0.5.0", "json-patch", @@ -4896,7 +4955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5e85aa143ceb072062fc4d6356c1b520a51d636e7bc8e77ec94be3608e5e80c" dependencies = [ "crossbeam-channel", - "dirs", + "dirs 6.0.0", "libappindicator", "muda", "objc2", @@ -5557,6 +5616,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5608,6 +5676,21 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5665,6 +5748,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5683,6 +5772,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5701,6 +5796,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5731,6 +5832,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5749,6 +5856,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5767,6 +5880,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5785,6 +5904,12 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -5935,7 +6060,7 @@ dependencies = [ "block2", "cookie", "crossbeam-channel", - "dirs", + "dirs 6.0.0", "dom_query", "dpi", "dunce", diff --git a/Cargo.toml b/Cargo.toml index 1187266c..1be7bf9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,12 @@ reqwest = { version = "0.12", features = ["blocking"] } zip = "2" tar = "0.4" zstd = "0.13" +flate2 = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" anyhow = "1" +ctcb-manifest = "0.4.1" +dirs = "5" strsim = "0.11" crossterm = "0.28" tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "fs", "signal"] } diff --git a/crates/fastled-cli/Cargo.toml b/crates/fastled-cli/Cargo.toml index 7a2cc0b2..5ab3a9c0 100644 --- a/crates/fastled-cli/Cargo.toml +++ b/crates/fastled-cli/Cargo.toml @@ -20,6 +20,7 @@ reqwest = { workspace = true } zip = { workspace = true } tar = { workspace = true } zstd = { workspace = true } +flate2 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } anyhow = { workspace = true } @@ -29,6 +30,8 @@ tokio = { workspace = true } axum = { workspace = true } tower-http = { workspace = true } tokio-stream = { workspace = true } +ctcb-manifest = { workspace = true } +dirs = { workspace = true } tempfile = "3" [package.metadata.binstall] diff --git a/crates/fastled-cli/src/archive.rs b/crates/fastled-cli/src/archive.rs index 82121fcc..aac88e63 100644 --- a/crates/fastled-cli/src/archive.rs +++ b/crates/fastled-cli/src/archive.rs @@ -7,10 +7,6 @@ //! * `.zip` extraction (general-purpose) //! * Writing the `.emscripten` config file after installation -// This module is library code — not yet wired into the CLI entry point. -// Suppress dead-code lints until a later integration phase. -#![allow(dead_code)] - use std::fs::{self, File}; use std::io::{BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; diff --git a/crates/fastled-cli/src/install.rs b/crates/fastled-cli/src/install.rs new file mode 100644 index 00000000..60e34678 --- /dev/null +++ b/crates/fastled-cli/src/install.rs @@ -0,0 +1,175 @@ +//! Toolchain install entry points driven from the Rust CLI. +//! +//! Mirrors `src/fastled/toolchain/emscripten_archive.py` so the Python side +//! no longer needs `httpx` / `pyzstd` to materialise the emscripten toolchain. +//! Public entry points are intended to be called once at the top of the +//! compile flow; results are cached on disk via a `done.txt` marker. + +use std::fs; +use std::io::{BufReader, BufWriter, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use ctcb_manifest::{PartRef, PlatformManifest}; + +use crate::archive; + +const EMSCRIPTEN_MANIFEST_BASE_URL: &str = + "https://raw.githubusercontent.com/zackees/clang-tool-chain-bins/main/assets/emscripten"; + +fn fastled_root() -> Result { + let home = dirs::home_dir().context("cannot resolve home directory")?; + Ok(home.join(".fastled")) +} + +fn detect_platform_arch() -> (&'static str, &'static str) { + let platform = if cfg!(target_os = "windows") { + "win" + } else if cfg!(target_os = "macos") { + "darwin" + } else { + "linux" + }; + let arch = if cfg!(target_arch = "x86_64") { + "x86_64" + } else if cfg!(target_arch = "aarch64") { + "arm64" + } else { + std::env::consts::ARCH + }; + (platform, arch) +} + +/// Fetch and parse the platform manifest JSON. +fn fetch_manifest(url: &str) -> Result { + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(60)) + .redirect(reqwest::redirect::Policy::limited(10)) + .build() + .context("failed to build HTTP client for manifest fetch")?; + let response = client + .get(url) + .send() + .with_context(|| format!("GET {url} failed"))? + .error_for_status() + .with_context(|| format!("manifest fetch returned error for {url}"))?; + let text = response.text().context("read manifest body")?; + serde_json::from_str::(&text) + .with_context(|| format!("parse manifest JSON from {url}")) +} + +/// Download each part to `parts_dir`, verify its SHA256, then concatenate the +/// parts into `merged_path`. Existing parts whose checksum still matches are +/// reused (no re-download). +fn download_multipart(parts: &[PartRef], parts_dir: &Path, merged_path: &Path) -> Result<()> { + fs::create_dir_all(parts_dir) + .with_context(|| format!("create parts dir {}", parts_dir.display()))?; + let merged_file = fs::File::create(merged_path) + .with_context(|| format!("create merged archive {}", merged_path.display()))?; + let mut merged_writer = BufWriter::new(merged_file); + + for (index, part) in parts.iter().enumerate() { + let part_path = parts_dir.join(format!("part-{:02}", index)); + if !part_path.exists() { + archive::download(&part.href, &part_path) + .with_context(|| format!("download part {}", part.href))?; + } + if !archive::verify_sha256(&part_path, &part.sha256)? { + let actual = archive::sha256_file(&part_path).unwrap_or_default(); + let _ = fs::remove_file(&part_path); + bail!( + "checksum mismatch for {}: got {}, expected {}", + part.href, + actual, + part.sha256 + ); + } + let mut part_reader = BufReader::new( + fs::File::open(&part_path) + .with_context(|| format!("open part {}", part_path.display()))?, + ); + std::io::copy(&mut part_reader, &mut merged_writer).with_context(|| { + format!( + "concatenate part {} into {}", + part_path.display(), + merged_path.display() + ) + })?; + } + merged_writer.flush()?; + Ok(()) +} + +/// Ensure the emscripten toolchain is installed at +/// `~/.fastled/toolchains/emscripten/{platform}/{arch}/{version}/`. +/// Returns the install directory path. +/// +/// The layout intentionally matches the Python implementation so a previously +/// Python-installed toolchain is picked up without re-downloading. +pub fn ensure_emscripten_installed() -> Result { + let (platform, arch) = detect_platform_arch(); + let root = fastled_root()?; + let install_base = root + .join("toolchains") + .join("emscripten") + .join(platform) + .join(arch); + let cache_dir = root.join("toolchains").join("archives"); + fs::create_dir_all(&install_base)?; + fs::create_dir_all(&cache_dir)?; + + let manifest_url = format!("{EMSCRIPTEN_MANIFEST_BASE_URL}/{platform}/{arch}/manifest.json"); + let manifest = fetch_manifest(&manifest_url)?; + let version = manifest.latest.clone(); + let install_dir = install_base.join(&version); + let done_file = install_dir.join("done.txt"); + if done_file.exists() { + return Ok(install_dir); + } + + let version_info = manifest + .versions + .get(&version) + .with_context(|| format!("manifest has no entry for version {version}"))?; + + let archive_path = cache_dir.join(format!("emscripten-{platform}-{arch}-{version}.tar.zst")); + let parts_subdir = cache_dir.join(format!("emscripten-{platform}-{arch}-{version}.parts")); + + if !archive_path.exists() { + let parts = version_info + .parts + .as_ref() + .with_context(|| format!("manifest version {version} has no parts"))?; + download_multipart(parts, &parts_subdir, &archive_path)?; + } + + if !archive::verify_sha256(&archive_path, &version_info.sha256)? { + let actual = archive::sha256_file(&archive_path).unwrap_or_default(); + let _ = fs::remove_file(&archive_path); + bail!( + "archive checksum mismatch for {}: got {}, expected {}", + version_info.href, + actual, + version_info.sha256 + ); + } + + // Atomic-ish install via staging directory. + let staging = install_base.join(format!("{version}.staging")); + if staging.exists() { + fs::remove_dir_all(&staging)?; + } + fs::create_dir_all(&staging)?; + archive::extract_tar_zst(&archive_path, &staging)?; + + fs::write(staging.join("done.txt"), "ok\n")?; + + if install_dir.exists() { + fs::remove_dir_all(&install_dir)?; + } + fs::rename(&staging, &install_dir)?; + + archive::write_emscripten_config(&install_dir, "node")?; + + Ok(install_dir) +} diff --git a/crates/fastled-cli/src/main.rs b/crates/fastled-cli/src/main.rs index a50c5fdf..cb5caa1f 100644 --- a/crates/fastled-cli/src/main.rs +++ b/crates/fastled-cli/src/main.rs @@ -5,6 +5,7 @@ use std::process::{Command, ExitCode, Stdio}; mod archive; mod build; +mod install; mod keyboard; mod project; mod server; @@ -164,6 +165,19 @@ fn compile_and_serve(dir: &str, cli: &Cli, mode: ViewerMode) -> ExitCode { return ExitCode::FAILURE; } + // Ensure the emscripten toolchain is installed before invoking Python. + // Sets FASTLED_EMSCRIPTEN_DIR so the Python compile path skips its own + // download logic and uses the Rust-installed directory. + match install::ensure_emscripten_installed() { + Ok(install_dir) => { + std::env::set_var("FASTLED_EMSCRIPTEN_DIR", &install_dir); + } + Err(e) => { + eprintln!("fastled: emscripten toolchain install failed: {e:#}"); + return ExitCode::FAILURE; + } + } + let output_dir = sketch_dir.join("fastled_js"); if let Err(e) = std::fs::create_dir_all(&output_dir) { eprintln!( diff --git a/pyproject.toml b/pyproject.toml index 4c2dc481..d2e38952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ classifiers = ["Programming Language :: Python :: 3"] dependencies = [ "httpx>=0.28.1", "fasteners>=0.20", - "clang-tool-chain>=1.2.6", "zcmds_win32; sys_platform == 'win32'", ] diff --git a/src/fastled/toolchain/emscripten.py b/src/fastled/toolchain/emscripten.py index ca054f51..d9dad44c 100644 --- a/src/fastled/toolchain/emscripten.py +++ b/src/fastled/toolchain/emscripten.py @@ -19,7 +19,6 @@ from pathlib import Path from fastled.interrupts import handle_keyboard_interrupt -from fastled.toolchain.emscripten_archive import install_emscripten_from_archive from fastled.types import BuildMode _clang_tool_chain_emscripten_dir_cache: Path | None | str = "UNSET" @@ -45,20 +44,24 @@ def _resolve_example_name( def _get_clang_tool_chain_emscripten_dir() -> Path | None: - """Get the clang-tool-chain Emscripten installation directory (cached).""" + """Resolve the Emscripten installation directory (cached). + + Source of truth: ``FASTLED_EMSCRIPTEN_DIR`` set by the Rust CLI before + invoking Python — Rust owns the actual download/install via ``install.rs``. + Falls back to a directory search under ``CLANG_TOOL_CHAIN_DOWNLOAD_PATH`` + (or ``~/.clang-tool-chain``) so an existing Python-installed toolchain is + still picked up when Python is invoked without the Rust pre-step. + """ global _clang_tool_chain_emscripten_dir_cache if _clang_tool_chain_emscripten_dir_cache != "UNSET": return _clang_tool_chain_emscripten_dir_cache # type: ignore[return-value] - try: - internal_dir = install_emscripten_from_archive() - if (internal_dir / "emscripten" / "emcc.py").exists(): - _clang_tool_chain_emscripten_dir_cache = internal_dir - return internal_dir - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - except Exception: - pass + rust_install = os.environ.get("FASTLED_EMSCRIPTEN_DIR") + if rust_install: + candidate = Path(rust_install) + if (candidate / "emscripten" / "emcc.py").exists(): + _clang_tool_chain_emscripten_dir_cache = candidate + return candidate env_path = os.environ.get("CLANG_TOOL_CHAIN_DOWNLOAD_PATH") base_dir = Path(env_path) if env_path else Path.home() / ".clang-tool-chain" @@ -87,69 +90,15 @@ def _get_clang_tool_chain_emscripten_dir() -> Path | None: return None -def _get_platform_arch() -> tuple[str, str]: - """Get the current platform and architecture strings for clang-tool-chain.""" - import struct - - if sys.platform == "win32": - plat = "win" - elif sys.platform == "darwin": - plat = "darwin" - else: - plat = "linux" - - # Detect architecture from pointer size and platform hints - is_64bit = struct.calcsize("P") * 8 == 64 - _uname = getattr(os, "uname", None) - if _uname is not None: - machine = _uname().machine.lower() - elif is_64bit: - machine = "amd64" - else: - machine = "x86" - - if machine in ("x86_64", "amd64"): - arch = "x86_64" - elif machine in ("arm64", "aarch64"): - arch = "arm64" - else: - arch = machine - - return plat, arch - - def ensure_clang_tool_chain_emscripten() -> Path | None: - """Ensure clang-tool-chain Emscripten is installed.""" - existing_dir = _get_clang_tool_chain_emscripten_dir() - if existing_dir: - return existing_dir - - try: - import inspect - - from clang_tool_chain.installers.emscripten import ( # type: ignore[import-not-found] - ensure_emscripten_available, - get_emscripten_install_dir, - ) + """Return the resolved Emscripten installation directory. - sig = inspect.signature(ensure_emscripten_available) - num_params = len(sig.parameters) - if num_params == 0: - ensure_emscripten_available() # type: ignore[call-arg] - return get_emscripten_install_dir() # type: ignore[call-arg] - elif num_params == 2: - plat, arch = _get_platform_arch() - ensure_emscripten_available(plat, arch) # type: ignore[call-arg] - return get_emscripten_install_dir(plat, arch) # type: ignore[call-arg] - else: - return None - except ImportError: - return None - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - except Exception as e: - print(f"Warning: Failed to ensure clang-tool-chain Emscripten: {e}") - return None + Install is now driven by the Rust CLI (`crates/fastled-cli/src/install.rs`) + which sets ``FASTLED_EMSCRIPTEN_DIR`` before invoking Python. This function + is kept for backward compatibility with callers that just want the + resolved directory; the actual download has moved out of Python. + """ + return _get_clang_tool_chain_emscripten_dir() def _setup_emscripten_env(env: dict[str, str]) -> None: diff --git a/src/fastled/toolchain/emscripten_archive.py b/src/fastled/toolchain/emscripten_archive.py deleted file mode 100644 index 500004fc..00000000 --- a/src/fastled/toolchain/emscripten_archive.py +++ /dev/null @@ -1,259 +0,0 @@ -from __future__ import annotations - -import hashlib -import os -import shutil -import struct -import sys -import tarfile -from dataclasses import dataclass -from pathlib import Path - -import fasteners -import httpx -import pyzstd - -MANIFEST_BASE_URL = "https://raw.githubusercontent.com/zackees/clang-tool-chain-bins/main/assets/emscripten" -ARCHIVE_CACHE_DIR = Path.home() / ".fastled" / "toolchains" / "archives" -INSTALL_ROOT = Path.home() / ".fastled" / "toolchains" / "emscripten" - - -@dataclass(frozen=True) -class EmscriptenArchivePart: - href: str - sha256: str - - -@dataclass(frozen=True) -class EmscriptenArchive: - version: str - href: str - sha256: str - parts: tuple[EmscriptenArchivePart, ...] - - -def _platform_arch() -> tuple[str, str]: - if sys_platform := sys.platform: - if sys_platform == "win32": - platform = "win" - elif sys_platform == "darwin": - platform = "darwin" - else: - platform = "linux" - else: - platform = "linux" - - is_64bit = struct.calcsize("P") * 8 == 64 - uname = getattr(os, "uname", None) - if uname is not None: - machine = uname().machine.lower() - else: - machine = "amd64" if is_64bit else "x86" - - if machine in ("x86_64", "amd64"): - arch = "x86_64" - elif machine in ("arm64", "aarch64"): - arch = "arm64" - else: - arch = machine - return platform, arch - - -def _manifest_url(platform: str, arch: str) -> str: - return f"{MANIFEST_BASE_URL}/{platform}/{arch}/manifest.json" - - -def _parse_manifest(payload: dict[str, object]) -> EmscriptenArchive: - latest = payload.get("latest") - if not isinstance(latest, str) or not latest: - raise ValueError("Manifest missing latest version") - - versions = payload.get("versions") - if isinstance(versions, dict): - raw_entry = versions.get(latest) - else: - raw_entry = payload.get(latest) - - if not isinstance(raw_entry, dict): - raise ValueError(f"Manifest missing archive metadata for {latest}") - - href = raw_entry.get("href") - sha256 = raw_entry.get("sha256") - parts = raw_entry.get("parts") - if not isinstance(href, str) or not isinstance(sha256, str): - raise ValueError(f"Manifest entry for {latest} is invalid") - if not isinstance(parts, list) or not parts: - raise ValueError(f"Manifest entry for {latest} has no parts") - - parsed_parts: list[EmscriptenArchivePart] = [] - for index, part in enumerate(parts): - if not isinstance(part, dict): - raise ValueError(f"Manifest part {index} for {latest} is invalid") - part_href = part.get("href") - part_sha = part.get("sha256") - if not isinstance(part_href, str) or not isinstance(part_sha, str): - raise ValueError(f"Manifest part {index} for {latest} is missing fields") - parsed_parts.append(EmscriptenArchivePart(href=part_href, sha256=part_sha)) - - return EmscriptenArchive( - version=latest, - href=href, - sha256=sha256, - parts=tuple(parsed_parts), - ) - - -def _sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with open(path, "rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def _download_file(client: httpx.Client, url: str, destination: Path) -> None: - with client.stream("GET", url, follow_redirects=True) as response: - response.raise_for_status() - with open(destination, "wb") as handle: - for chunk in response.iter_bytes(): - handle.write(chunk) - - -def _write_emscripten_config(install_dir: Path) -> None: - node_path = shutil.which("node") or "node" - config = "\n".join( - [ - "# Emscripten configuration file", - "# Auto-generated by fastled", - "", - f"LLVM_ROOT = {str(install_dir / 'bin').replace(os.sep, '/')!r}", - f"BINARYEN_ROOT = {str(install_dir).replace(os.sep, '/')!r}", - f"NODE_JS = {node_path!r}", - "", - ] - ) - (install_dir / ".emscripten").write_text(config + "\n", encoding="utf-8") - - -def _write_ctc_metadata(install_dir: Path) -> None: - bin_dir = install_dir / "bin" - emscripten_dir = install_dir / "emscripten" - python_path = Path(sys.executable) - node_path = shutil.which("node") or "node" - suffix = ".exe" if sys.platform == "win32" else "" - emcc_script = emscripten_dir / "emcc.py" - empp_script = emscripten_dir / "em++.py" - wasm_ld = bin_dir / f"wasm-ld{suffix}" - - lines = [ - f"emscripten_dir={emscripten_dir}", - f"config_path={install_dir / '.emscripten'}", - f"bin_dir={bin_dir}", - f"node_path={node_path}", - f"python_path={python_path}", - f"emcc_script={emcc_script}", - f"empp_script={empp_script}", - ] - (install_dir / ".ctc-emcc-paths").write_text( - "\n".join(lines) + "\n", encoding="utf-8" - ) - (install_dir / ".ctc-wasmld-cache").write_text( - f"wasm_ld_path={wasm_ld}\n", encoding="utf-8" - ) - - -def _extract_archive(archive_path: Path, install_dir: Path) -> None: - staging_dir = install_dir.parent / f"{install_dir.name}.staging" - if staging_dir.exists(): - shutil.rmtree(staging_dir, ignore_errors=True) - staging_dir.mkdir(parents=True, exist_ok=True) - - with open(archive_path, "rb") as handle: - with pyzstd.ZstdFile(handle, "rb") as zstd_stream: - with tarfile.open(fileobj=zstd_stream, mode="r|") as tar: - tar.extractall(staging_dir, filter="data") - - children = [child for child in staging_dir.iterdir()] - if len(children) == 1 and children[0].is_dir(): - extracted_root = children[0] - for child in extracted_root.iterdir(): - shutil.move(str(child), staging_dir / child.name) - shutil.rmtree(extracted_root, ignore_errors=True) - - # Fix execute permissions on Unix (tarfile filter="data" may strip them). - if os.name != "nt": - for subdir in ("bin", "emscripten", "libexec"): - bin_dir = staging_dir / subdir - if bin_dir.is_dir(): - for p in bin_dir.rglob("*"): - if p.is_file(): - p.chmod(p.stat().st_mode | 0o111) - - (staging_dir / "done.txt").write_text("ok\n", encoding="utf-8") - - if install_dir.exists(): - shutil.rmtree(install_dir, ignore_errors=True) - staging_dir.replace(install_dir) - - # Write configs AFTER rename so paths reference the final install_dir, - # not the now-deleted staging_dir. - _write_emscripten_config(install_dir) - _write_ctc_metadata(install_dir) - - -def install_emscripten_from_archive(force: bool = False) -> Path: - platform, arch = _platform_arch() - install_base = INSTALL_ROOT / platform / arch - install_base.mkdir(parents=True, exist_ok=True) - ARCHIVE_CACHE_DIR.mkdir(parents=True, exist_ok=True) - - lock = fasteners.InterProcessLock(str(install_base / ".install.lock")) - with lock: - manifest_path = install_base / "manifest.json" - with httpx.Client(timeout=120) as client: - response = client.get(_manifest_url(platform, arch), follow_redirects=True) - response.raise_for_status() - manifest_path.write_text(response.text, encoding="utf-8") - archive = _parse_manifest(response.json()) - - install_dir = install_base / archive.version - done_file = install_dir / "done.txt" - if done_file.exists() and not force: - return install_dir - - archive_path = ( - ARCHIVE_CACHE_DIR - / f"emscripten-{platform}-{arch}-{archive.version}.tar.zst" - ) - if force and archive_path.exists(): - archive_path.unlink() - - if not archive_path.exists(): - part_dir = ( - ARCHIVE_CACHE_DIR - / f"emscripten-{platform}-{arch}-{archive.version}.parts" - ) - part_dir.mkdir(parents=True, exist_ok=True) - with open(archive_path, "wb") as merged: - for index, part in enumerate(archive.parts): - part_path = part_dir / f"part-{index:02d}" - if not part_path.exists(): - _download_file(client, part.href, part_path) - part_hash = _sha256_file(part_path) - if part_hash != part.sha256: - part_path.unlink(missing_ok=True) - raise ValueError( - f"Checksum mismatch for {part.href}: {part_hash} != {part.sha256}" - ) - with open(part_path, "rb") as handle: - shutil.copyfileobj(handle, merged) - - archive_hash = _sha256_file(archive_path) - if archive_hash != archive.sha256: - archive_path.unlink(missing_ok=True) - raise ValueError( - f"Archive checksum mismatch for {archive.href}: {archive_hash} != {archive.sha256}" - ) - - _extract_archive(archive_path, install_dir) - return install_dir From d3d13301fb32c7ff64d438968e2c62d6712aae4f Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 18:47:18 -0700 Subject: [PATCH 3/5] chore(rust): own esbuild install in the Rust CLI; drop fasteners (refs #59) The Python frontend bundling step used to download esbuild from npmjs itself (httpx + tarfile + fasteners InterProcessLock). That now lives in Rust: - crates/fastled-cli/src/install.rs - ensure_esbuild_installed() downloads the npm registry .tgz, extracts the single bin into ~/.fastled/toolchains/esbuild/{platform}/{arch}/{version}/, returns the path. - crates/fastled-cli/src/archive.rs - new extract_member_from_tgz() helper plucks a single file out of a gzipped tarball (needs flate2, added to workspace). - crates/fastled-cli/src/main.rs - compile_and_serve() now also calls ensure_esbuild_installed() and exports FASTLED_ESBUILD_PATH for the Python subprocess. Python: - src/fastled/frontend_esbuild.py - install_esbuild() removed; the module now resolves esbuild via FASTLED_ESBUILD_PATH (set by Rust) or shutil.which fallback. Drops httpx, tarfile, fasteners imports from this file. - src/fastled/toolchain/emscripten_archive.py was the only other fasteners consumer (already gone in the previous commit), so fasteners drops from pyproject.toml entirely. Net: runtime Python deps now httpx, zcmds_win32 (Win-only). Lint clean, 118 unit tests pass. --- crates/fastled-cli/src/archive.rs | 41 +++++++++++ crates/fastled-cli/src/install.rs | 89 ++++++++++++++++++++++++ crates/fastled-cli/src/main.rs | 15 +++- pyproject.toml | 1 - src/fastled/frontend_esbuild.py | 111 ++++++------------------------ 5 files changed, 164 insertions(+), 93 deletions(-) diff --git a/crates/fastled-cli/src/archive.rs b/crates/fastled-cli/src/archive.rs index aac88e63..5ba38e6f 100644 --- a/crates/fastled-cli/src/archive.rs +++ b/crates/fastled-cli/src/archive.rs @@ -186,6 +186,47 @@ pub fn extract_zip(archive: &Path, dest: &Path) -> Result<()> { Ok(()) } +// --------------------------------------------------------------------------- +// Single-file extraction from a gzipped tar (.tgz) +// --------------------------------------------------------------------------- + +/// Extract a single member from a `.tgz` archive into `dest`. +/// +/// Used to pluck the esbuild binary out of an npm-style tarball (`package/...` +/// layout) without unpacking the whole archive. +/// +/// # Errors +/// Returns an error if the archive cannot be read or if `member` is not +/// present in the archive. +pub fn extract_member_from_tgz(archive: &Path, member: &str, dest: &Path) -> Result<()> { + let file = File::open(archive) + .with_context(|| format!("cannot open archive {}", archive.display()))?; + let gz = flate2::read::GzDecoder::new(BufReader::new(file)); + let mut tar_archive = tar::Archive::new(gz); + + for entry in tar_archive + .entries() + .with_context(|| format!("cannot iterate entries of {}", archive.display()))? + { + let mut entry = entry.with_context(|| format!("bad entry in {}", archive.display()))?; + let path = entry + .path() + .with_context(|| format!("bad entry path in {}", archive.display()))?; + if path.to_string_lossy() == member { + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("cannot create parent {}", parent.display()))?; + } + let mut out = + File::create(dest).with_context(|| format!("cannot create {}", dest.display()))?; + std::io::copy(&mut entry, &mut out) + .with_context(|| format!("cannot write {}", dest.display()))?; + return Ok(()); + } + } + anyhow::bail!("member {member} not found in {}", archive.display()); +} + // --------------------------------------------------------------------------- // .emscripten config generation // --------------------------------------------------------------------------- diff --git a/crates/fastled-cli/src/install.rs b/crates/fastled-cli/src/install.rs index 60e34678..867cdfac 100644 --- a/crates/fastled-cli/src/install.rs +++ b/crates/fastled-cli/src/install.rs @@ -17,6 +17,8 @@ use crate::archive; const EMSCRIPTEN_MANIFEST_BASE_URL: &str = "https://raw.githubusercontent.com/zackees/clang-tool-chain-bins/main/assets/emscripten"; +const ESBUILD_VERSION: &str = "0.28.0"; + fn fastled_root() -> Result { let home = dirs::home_dir().context("cannot resolve home directory")?; Ok(home.join(".fastled")) @@ -173,3 +175,90 @@ pub fn ensure_emscripten_installed() -> Result { Ok(install_dir) } + +// --------------------------------------------------------------------------- +// esbuild +// --------------------------------------------------------------------------- + +/// npm package platform-arch strings differ from the emscripten ones. +fn esbuild_platform_arch() -> Result<(&'static str, &'static str)> { + let platform = if cfg!(target_os = "windows") { + "win32" + } else if cfg!(target_os = "macos") { + "darwin" + } else { + "linux" + }; + let arch = if cfg!(target_arch = "x86_64") { + "x64" + } else if cfg!(target_arch = "aarch64") { + "arm64" + } else { + anyhow::bail!( + "unsupported architecture for esbuild: {}", + std::env::consts::ARCH + ); + }; + Ok((platform, arch)) +} + +/// Ensure the esbuild binary is installed at +/// `~/.fastled/toolchains/esbuild/{platform}/{arch}/{version}/`. +/// Returns the path to the executable. +/// +/// Layout matches the previous Python implementation +/// (`src/fastled/frontend_esbuild.py`). +pub fn ensure_esbuild_installed() -> Result { + let (platform, arch) = esbuild_platform_arch()?; + let version = ESBUILD_VERSION; + let root = fastled_root()?; + let install_dir = root + .join("toolchains") + .join("esbuild") + .join(platform) + .join(arch) + .join(version); + let exe_name = if cfg!(target_os = "windows") { + "esbuild.exe" + } else { + "esbuild" + }; + let esbuild_path = install_dir.join(exe_name); + let done_file = install_dir.join("done.txt"); + if done_file.exists() && esbuild_path.exists() { + return Ok(esbuild_path); + } + fs::create_dir_all(&install_dir)?; + + let archive_cache = root.join("toolchains").join("archives"); + fs::create_dir_all(&archive_cache)?; + let archive_path = archive_cache.join(format!("esbuild-{platform}-{arch}-{version}.tgz")); + if !archive_path.exists() { + let url = format!( + "https://registry.npmjs.org/@esbuild/{platform}-{arch}/-/{platform}-{arch}-{version}.tgz" + ); + archive::download(&url, &archive_path)?; + } + + let member = if cfg!(target_os = "windows") { + format!("package/{exe_name}") + } else { + "package/bin/esbuild".to_string() + }; + + if esbuild_path.exists() { + let _ = fs::remove_file(&esbuild_path); + } + archive::extract_member_from_tgz(&archive_path, &member, &esbuild_path)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&esbuild_path)?.permissions(); + perms.set_mode(perms.mode() | 0o111); + fs::set_permissions(&esbuild_path, perms)?; + } + + fs::write(&done_file, "ok\n")?; + Ok(esbuild_path) +} diff --git a/crates/fastled-cli/src/main.rs b/crates/fastled-cli/src/main.rs index cb5caa1f..5a94e031 100644 --- a/crates/fastled-cli/src/main.rs +++ b/crates/fastled-cli/src/main.rs @@ -165,9 +165,9 @@ fn compile_and_serve(dir: &str, cli: &Cli, mode: ViewerMode) -> ExitCode { return ExitCode::FAILURE; } - // Ensure the emscripten toolchain is installed before invoking Python. - // Sets FASTLED_EMSCRIPTEN_DIR so the Python compile path skips its own - // download logic and uses the Rust-installed directory. + // Ensure the emscripten + esbuild toolchains are installed before invoking + // Python. The env vars below let the Python compile path skip its own + // download logic and consume the Rust-installed directories. match install::ensure_emscripten_installed() { Ok(install_dir) => { std::env::set_var("FASTLED_EMSCRIPTEN_DIR", &install_dir); @@ -177,6 +177,15 @@ fn compile_and_serve(dir: &str, cli: &Cli, mode: ViewerMode) -> ExitCode { return ExitCode::FAILURE; } } + match install::ensure_esbuild_installed() { + Ok(esbuild_path) => { + std::env::set_var("FASTLED_ESBUILD_PATH", &esbuild_path); + } + Err(e) => { + eprintln!("fastled: esbuild install failed: {e:#}"); + return ExitCode::FAILURE; + } + } let output_dir = sketch_dir.join("fastled_js"); if let Err(e) = std::fs::create_dir_all(&output_dir) { diff --git a/pyproject.toml b/pyproject.toml index d2e38952..48b59b9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ license = { text = "BSD 3-Clause License" } classifiers = ["Programming Language :: Python :: 3"] dependencies = [ "httpx>=0.28.1", - "fasteners>=0.20", "zcmds_win32; sys_platform == 'win32'", ] diff --git a/src/fastled/frontend_esbuild.py b/src/fastled/frontend_esbuild.py index 2efb2dd6..426b4af7 100644 --- a/src/fastled/frontend_esbuild.py +++ b/src/fastled/frontend_esbuild.py @@ -1,101 +1,34 @@ from __future__ import annotations import hashlib -import platform +import os import shutil -import stat import subprocess -import sys -import tarfile from pathlib import Path -import fasteners -import httpx - -ESBUILD_VERSION = "0.28.0" -INSTALL_ROOT = Path.home() / ".fastled" / "toolchains" / "esbuild" -ARCHIVE_CACHE_DIR = Path.home() / ".fastled" / "toolchains" / "archives" FRONTEND_DIR = Path(__file__).resolve().parent / "frontend" -def _platform_arch() -> tuple[str, str]: - if sys.platform == "win32": - plat = "win32" - elif sys.platform == "darwin": - plat = "darwin" - else: - plat = "linux" - - machine = platform.machine().lower() - if machine in ("x86_64", "amd64"): - arch = "x64" - elif machine in ("arm64", "aarch64"): - arch = "arm64" - else: - raise RuntimeError(f"Unsupported architecture for esbuild: {machine}") - return plat, arch - - -def _package_name(platform: str, arch: str) -> str: - return f"@esbuild/{platform}-{arch}" - - -def _tarball_url(platform: str, arch: str) -> str: - package_name = _package_name(platform, arch) - package_tail = package_name.split("/", 1)[1] - return f"https://registry.npmjs.org/{package_name}/-/{package_tail}-{ESBUILD_VERSION}.tgz" - - -def install_esbuild(force: bool = False) -> Path: - platform, arch = _platform_arch() - install_dir = INSTALL_ROOT / platform / arch / ESBUILD_VERSION - install_dir.mkdir(parents=True, exist_ok=True) - lock = fasteners.InterProcessLock(str(install_dir / ".install.lock")) - - with lock: - exe_name = "esbuild.exe" if platform == "win32" else "esbuild" - esbuild_path = install_dir / exe_name - done_file = install_dir / "done.txt" - if done_file.exists() and esbuild_path.exists() and not force: - return esbuild_path - - ARCHIVE_CACHE_DIR.mkdir(parents=True, exist_ok=True) - archive_path = ( - ARCHIVE_CACHE_DIR / f"esbuild-{platform}-{arch}-{ESBUILD_VERSION}.tgz" - ) - if force and archive_path.exists(): - archive_path.unlink() - - if not archive_path.exists(): - response = httpx.get( - _tarball_url(platform, arch), follow_redirects=True, timeout=120 - ) - response.raise_for_status() - archive_path.write_bytes(response.content) - - if esbuild_path.exists(): - esbuild_path.unlink() - - with tarfile.open(archive_path, mode="r:gz") as tf: - member_name = ( - f"package/{exe_name}" if platform == "win32" else "package/bin/esbuild" - ) - member = tf.getmember(member_name) - extracted = tf.extractfile(member) - if extracted is None: - raise RuntimeError( - f"Could not extract {member_name} from {archive_path}" - ) - esbuild_path.write_bytes(extracted.read()) - - if platform != "win32": - current_mode = esbuild_path.stat().st_mode - esbuild_path.chmod( - current_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - ) - - done_file.write_text("ok\n", encoding="utf-8") - return esbuild_path +def _resolve_esbuild() -> Path: + """Return the path to the esbuild binary. + + The Rust CLI installs esbuild via ``crates/fastled-cli/src/install.rs`` + and exports the path through ``FASTLED_ESBUILD_PATH``. If that env var + is missing (e.g. Python invoked directly without the Rust pre-step), the + function falls back to the binary on ``PATH``. + """ + env_path = os.environ.get("FASTLED_ESBUILD_PATH") + if env_path: + candidate = Path(env_path) + if candidate.is_file(): + return candidate + located = shutil.which("esbuild") + if located: + return Path(located) + raise RuntimeError( + "esbuild binary not available. Ensure FASTLED_ESBUILD_PATH is set by " + "the Rust CLI or that esbuild is on PATH." + ) def _copy_tree(src: Path, dst: Path) -> None: @@ -129,7 +62,7 @@ def _compute_dir_hash(directory: Path) -> str: def _run_esbuild(source_dir: Path, args: list[str]) -> None: - esbuild = install_esbuild() + esbuild = _resolve_esbuild() result = subprocess.run( [ str(esbuild), From aee0d07ed7ad148a5d4b62d678f7317c1e52c286 Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 19:35:21 -0700 Subject: [PATCH 4/5] chore(rust): own FastLED repo download in the Rust CLI (refs #59, #63) Python's project_init.py used httpx to query the GitHub API for the latest release tag and to download the FastLED repo zip. That logic now lives in Rust: - crates/fastled-cli/src/install.rs - ensure_fastled_repo(ref: Option<&str>) - latest-release lookup via api.github.com (User-Agent header set) - resolve_fastled_ref: handles commit SHAs, tags, branches; HEAD-checks a tag URL before falling back to a branch URL (mirrors Python) - download via archive::download, extract via archive::extract_zip - cache at ~/.fastled/cache/fastled-{ref}/ (cheap library.json check on subsequent calls) - crates/fastled-cli/src/main.rs - two integration points: - --init pre-step: when --init is set, Rust calls ensure_fastled_repo (deriving the ref from --latest / --commit / --branch) and exports FASTLED_LOCAL_REPO_DIR before delegating to Python. - --internal-ensure-fastled-repo [REF] hidden flag: prints the local path to stdout for the Python-driven Api.project_init / Api.get_examples paths to consume via subprocess. Python: - src/fastled/project_init.py rewritten to use either FASTLED_LOCAL_REPO_DIR (preferred) or the Rust binary's --internal-ensure-fastled-repo flag. Removes the httpx, tempfile-based extract, and DownloadResult helpers that used to live here. _find_rust_fastled_cli() walks the workspace target/ tree so it returns the Rust binary, not the Python entry-point shim that shadows it in .venv/Scripts/. Lint clean. test_api.py::test_project_init now exercises the Rust path end-to-end and passes (21s). --- crates/fastled-cli/src/install.rs | 147 ++++++++++++++++ crates/fastled-cli/src/main.rs | 47 ++++++ src/fastled/project_init.py | 271 ++++++++++++------------------ 3 files changed, 298 insertions(+), 167 deletions(-) diff --git a/crates/fastled-cli/src/install.rs b/crates/fastled-cli/src/install.rs index 867cdfac..09f81344 100644 --- a/crates/fastled-cli/src/install.rs +++ b/crates/fastled-cli/src/install.rs @@ -19,6 +19,10 @@ const EMSCRIPTEN_MANIFEST_BASE_URL: &str = const ESBUILD_VERSION: &str = "0.28.0"; +const FASTLED_REPO: &str = "FastLED/FastLED"; +const FASTLED_LATEST_RELEASE_API: &str = + "https://api.github.com/repos/FastLED/FastLED/releases/latest"; + fn fastled_root() -> Result { let home = dirs::home_dir().context("cannot resolve home directory")?; Ok(home.join(".fastled")) @@ -262,3 +266,146 @@ pub fn ensure_esbuild_installed() -> Result { fs::write(&done_file, "ok\n")?; Ok(esbuild_path) } + +// --------------------------------------------------------------------------- +// FastLED repo download (used by --init) +// --------------------------------------------------------------------------- + +fn is_commit_sha(ref_str: &str) -> bool { + let n = ref_str.len(); + (7..=40).contains(&n) && ref_str.chars().all(|c| c.is_ascii_hexdigit()) +} + +/// Hit the GitHub API for the latest FastLED release tag. +/// Returns `None` on any failure so callers can fall back to `master`. +fn fetch_latest_release_tag() -> Option { + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::limited(10)) + .build() + .ok()?; + let resp = client + .get(FASTLED_LATEST_RELEASE_API) + .header("Accept", "application/vnd.github.v3+json") + .header("User-Agent", "fastled-cli") + .send() + .ok()?; + if !resp.status().is_success() { + return None; + } + let text = resp.text().ok()?; + let value: serde_json::Value = serde_json::from_str(&text).ok()?; + value + .get("tag_name") + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +fn head_check(url: &str) -> bool { + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::limited(10)) + .build(); + match client { + Ok(c) => c + .head(url) + .header("User-Agent", "fastled-cli") + .send() + .map(|r| r.status().is_success()) + .unwrap_or(false), + Err(_) => false, + } +} + +/// Resolve `ref` to `(display_name, archive_url)`. Mirrors +/// `project_init._resolve_ref` in Python. +fn resolve_fastled_ref(ref_opt: Option<&str>) -> (String, String) { + let archive_base = format!("https://github.com/{FASTLED_REPO}/archive"); + + match ref_opt { + None | Some("latest_release") => match fetch_latest_release_tag() { + Some(tag) => { + let url = format!("{archive_base}/refs/tags/{tag}.zip"); + (tag, url) + } + None => { + eprintln!( + "fastled: could not fetch latest FastLED release tag, falling back to master" + ); + let url = format!("{archive_base}/refs/heads/master.zip"); + ("master".to_string(), url) + } + }, + Some(r) if is_commit_sha(r) => { + let url = format!("{archive_base}/{r}.zip"); + (r.to_string(), url) + } + Some(r) => { + // Try as tag first, fall back to branch (mirrors Python). + let tag_url = format!("{archive_base}/refs/tags/{r}.zip"); + if head_check(&tag_url) { + (r.to_string(), tag_url) + } else { + let branch_url = format!("{archive_base}/refs/heads/{r}.zip"); + (r.to_string(), branch_url) + } + } + } +} + +/// Locate the root of an extracted FastLED archive (e.g. `FastLED-master`, +/// `FastLED-3.9.12`, `FastLED-`). +fn find_fastled_extract_root(dir: &Path) -> Result { + for entry in fs::read_dir(dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() && entry.file_name().to_string_lossy().starts_with("FastLED") + { + return Ok(entry.path()); + } + } + anyhow::bail!("no FastLED* directory found inside {}", dir.display()) +} + +/// Ensure the FastLED repo for `ref_opt` is downloaded and extracted under +/// `~/.fastled/cache/fastled-{ref}/`. Returns the resolved local repo root. +/// +/// Re-uses an existing extraction if `library.json` is already present, so +/// repeated calls are cheap. +pub fn ensure_fastled_repo(ref_opt: Option<&str>) -> Result { + let (ref_name, url) = resolve_fastled_ref(ref_opt); + let root = fastled_root()?; + let cache_base = root.join("cache"); + fs::create_dir_all(&cache_base)?; + let repo_dir = cache_base.join(format!("fastled-{ref_name}")); + + if repo_dir.join("library.json").is_file() { + return Ok(repo_dir); + } + + let archive_cache = cache_base.join("archives"); + fs::create_dir_all(&archive_cache)?; + let archive_path = archive_cache.join(format!("FastLED-{ref_name}.zip")); + if !archive_path.exists() { + archive::download(&url, &archive_path) + .with_context(|| format!("download FastLED archive from {url}"))?; + } + + // Extract to a staging dir so a partial extraction never poisons the final + // location. + let staging = cache_base.join(format!("fastled-{ref_name}.staging")); + if staging.exists() { + fs::remove_dir_all(&staging)?; + } + fs::create_dir_all(&staging)?; + archive::extract_zip(&archive_path, &staging)?; + + let extracted_root = find_fastled_extract_root(&staging)?; + + if repo_dir.exists() { + fs::remove_dir_all(&repo_dir)?; + } + fs::rename(&extracted_root, &repo_dir)?; + fs::remove_dir_all(&staging).ok(); + + Ok(repo_dir) +} diff --git a/crates/fastled-cli/src/main.rs b/crates/fastled-cli/src/main.rs index 5a94e031..f703240f 100644 --- a/crates/fastled-cli/src/main.rs +++ b/crates/fastled-cli/src/main.rs @@ -390,6 +390,13 @@ struct Cli { #[arg(long)] purge: bool, + /// Internal plumbing flag: ensure the FastLED repo for the given ref + /// (defaults to latest release) is downloaded and extracted, print the + /// local path to stdout, and exit. Used by the Python `Api.project_init` + /// path so the Python side never has to do an HTTP download. + #[arg(long, value_name = "REF", num_args = 0..=1, default_missing_value = "__latest__", hide = true)] + internal_ensure_fastled_repo: Option, + // Build mode (mutually exclusive). /// Build in debug mode. #[arg(long, conflicts_with_all = ["quick", "release"])] @@ -581,6 +588,26 @@ fn open_browser(url: &str) { fn main() -> ExitCode { let cli = Cli::parse(); + // Hidden plumbing for the Python side: download the FastLED repo and + // print the local path. No further work. + if let Some(ref ref_str) = cli.internal_ensure_fastled_repo { + let ref_opt = if ref_str == "__latest__" { + None + } else { + Some(ref_str.as_str()) + }; + return match install::ensure_fastled_repo(ref_opt) { + Ok(path) => { + println!("{}", path.display()); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("fastled: failed to fetch FastLED repo: {e:#}"); + ExitCode::FAILURE + } + }; + } + // Handle --serve-dir natively with the Rust HTTP server (no Python needed). if let Some(ref serve_dir) = cli.serve_dir { return serve_directory(serve_dir, &cli); @@ -607,6 +634,26 @@ fn main() -> ExitCode { } } + // For --init: pre-download the FastLED repo with Rust so the Python side + // doesn't need httpx. Set FASTLED_LOCAL_REPO_DIR for the subprocess. + if cli.init.is_some() { + // Resolve ref precedence: --latest > --commit > --branch > default. + let ref_opt: Option<&str> = if cli.latest { + None + } else { + cli.commit.as_deref().or(cli.branch.as_deref()) + }; + match install::ensure_fastled_repo(ref_opt) { + Ok(repo_dir) => { + std::env::set_var("FASTLED_LOCAL_REPO_DIR", &repo_dir); + } + Err(e) => { + eprintln!("fastled: failed to fetch FastLED repo: {e:#}"); + return ExitCode::FAILURE; + } + } + } + // --- Delegate to Python for everything else ------------------------------ // (--just-compile, --init, --install, etc.) diff --git a/src/fastled/project_init.py b/src/fastled/project_init.py index 89ac38e2..6c85eb8b 100644 --- a/src/fastled/project_init.py +++ b/src/fastled/project_init.py @@ -1,108 +1,120 @@ -import io import json -import re +import os import shutil -import tempfile -import zipfile +import subprocess +import sys from dataclasses import dataclass from pathlib import Path -import httpx - from fastled.interrupts import handle_keyboard_interrupt -from fastled.spinner import Spinner DEFAULT_EXAMPLE = "wasm" -GITHUB_REPO = "FastLED/FastLED" -GITHUB_RELEASES_API = f"https://api.github.com/repos/{GITHUB_REPO}/releases/latest" -# Sentinel value meaning "use the latest release" -REF_LATEST_RELEASE = "latest_release" +@dataclass +class _CachedRepo: + root: Path + ref_name: str + +def _find_rust_fastled_cli() -> Path | None: + """Locate the **Rust** fastled CLI binary, not the Python entry-point shim. -@dataclass -class DownloadResult: - """Result of downloading the FastLED repo from GitHub.""" + The Python `[project.scripts] fastled = ...` console-script sits in the + same directory as the interpreter and shadows the Rust binary in step 1 + of the open_browser lookup. We need the Rust one, so search the workspace + target/ tree first, then PATH. + """ + import shutil + import sys + + exe_name = "fastled.exe" if sys.platform == "win32" else "fastled" + + # Walk up to find a Cargo.toml workspace root and check target dirs. + current = Path(__file__).resolve().parent + for _ in range(10): + if (current / "Cargo.toml").is_file(): + for profile in ("release", "debug"): + candidate = current / "target" / profile / exe_name + if candidate.is_file(): + return candidate + target_dir = current / "target" + if target_dir.is_dir(): + for arch_dir in target_dir.iterdir(): + if arch_dir.is_dir() and not arch_dir.name.startswith("."): + for profile in ("release", "debug"): + candidate = arch_dir / profile / exe_name + if candidate.is_file(): + return candidate + break + parent = current.parent + if parent == current: + break + current = parent - content: bytes | None = None - error: Exception | None = None + found = shutil.which(exe_name) + if found: + return Path(found) + return None - @property - def ok(self) -> bool: - return self.content is not None +def _ensure_repo_via_rust(ref: str | None) -> Path: + """Invoke the Rust binary's hidden ``--internal-ensure-fastled-repo`` flag + to materialise the FastLED repo locally and return the path. -def _is_commit_sha(ref: str) -> bool: - """Check if ref looks like a git commit SHA (7-40 hex chars).""" - return bool(re.match(r"^[0-9a-fA-F]{7,40}$", ref)) + The Rust side owns the actual GitHub download. Python never performs the + HTTP request itself. + """ + cli = _find_rust_fastled_cli() + if cli is None: + raise RuntimeError( + "Could not locate the fastled CLI binary; cannot fetch FastLED repo." + ) + cmd: list[str] = [str(cli), "--internal-ensure-fastled-repo"] + if ref is not None: + cmd.append(ref) -def _get_latest_release_tag() -> str | None: - """Fetch the latest release tag from GitHub API. Returns None on failure.""" try: - response = httpx.get( - GITHUB_RELEASES_API, - follow_redirects=True, - timeout=10, - headers={"Accept": "application/vnd.github.v3+json"}, - ) - response.raise_for_status() - return response.json()["tag_name"] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) except KeyboardInterrupt as ki: handle_keyboard_interrupt(ki) - except Exception: - return None - - -def _build_archive_url(ref: str) -> str: - """Build the GitHub archive URL for a given ref. - - Args: - ref: A branch name, tag name, or commit SHA. - """ - base = f"https://github.com/{GITHUB_REPO}/archive" - if _is_commit_sha(ref): - return f"{base}/{ref}.zip" - # Could be a branch or tag — GitHub resolves both via this URL pattern - return f"{base}/refs/heads/{ref}.zip" + raise + if result.returncode != 0: + sys.stderr.write(result.stderr) + raise RuntimeError( + f"fastled --internal-ensure-fastled-repo failed (exit {result.returncode})" + ) -def _build_tag_archive_url(tag: str) -> str: - """Build the GitHub archive URL for a specific tag.""" - return f"https://github.com/{GITHUB_REPO}/archive/refs/tags/{tag}.zip" + repo_path_str = result.stdout.strip().splitlines()[-1] if result.stdout else "" + repo_path = Path(repo_path_str) + if not repo_path.is_dir(): + raise RuntimeError( + f"Rust binary returned non-existent repo path: {repo_path_str!r}" + ) + return repo_path -def _resolve_ref(ref: str | None) -> tuple[str, str]: - """Resolve a ref string into (display_name, archive_url). +def _get_local_fastled_repo(ref: str | None) -> _CachedRepo: + """Return a path to the locally-cached FastLED repo (and the resolved ref). - Returns: - Tuple of (ref_display_name, archive_url). - ref_display_name is the resolved ref (e.g. "3.9.12" for latest release). + Prefers ``FASTLED_LOCAL_REPO_DIR`` set by the Rust CLI's --init pre-step. + Falls back to invoking the Rust binary directly so direct Python API + consumers (Api.project_init, Api.get_examples) still work without httpx. """ - if ref is None or ref == REF_LATEST_RELEASE: - tag = _get_latest_release_tag() - if tag: - return tag, _build_tag_archive_url(tag) - # Fallback to master if we can't get latest release - print("Warning: Could not fetch latest release, falling back to master") - return "master", _build_archive_url("master") - - if _is_commit_sha(ref): - return ref, _build_archive_url(ref) - - # Try as a tag first, fall back to branch - tag_url = _build_tag_archive_url(ref) - try: - resp = httpx.head(tag_url, follow_redirects=True, timeout=10) - if resp.status_code == 200: - return ref, tag_url - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - except Exception: - pass + env_path = os.environ.get("FASTLED_LOCAL_REPO_DIR") + if env_path: + candidate = Path(env_path) + if candidate.is_dir(): + return _CachedRepo( + root=candidate, + ref_name=candidate.name.replace("fastled-", "", 1) or "master", + ) - return ref, _build_archive_url(ref) + repo_root = _ensure_repo_via_rust(ref) + ref_name = repo_root.name.replace("fastled-", "", 1) or (ref or "master") + return _CachedRepo(root=repo_root, ref_name=ref_name) # --- fastled.json persistence --- @@ -158,7 +170,7 @@ def _find_fastled_repo_via_library_json(start: Path) -> Path | None: return None -# --- Core download --- +# --- Core lookup helpers --- def _find_example_in_repo(repo_root: Path, example: str) -> Path | None: @@ -170,11 +182,9 @@ def _find_example_in_repo(repo_root: Path, example: str) -> Path | None: examples_dir = repo_root / "examples" if not examples_dir.exists(): return None - # Direct match: examples/{example}/ direct = examples_dir / example if direct.exists() and direct.is_dir(): return direct - # Nested match: examples/*/{example}/ for subdir in examples_dir.iterdir(): if subdir.is_dir(): nested = subdir / example @@ -183,36 +193,6 @@ def _find_example_in_repo(repo_root: Path, example: str) -> Path | None: return None -def _download_fastled_repo(ref: str | None = None) -> tuple[DownloadResult, str]: - """Download the FastLED repo zip from GitHub. - - Args: - ref: Git ref to download. None means latest release. - - Returns: - Tuple of (DownloadResult, resolved_ref_name). - """ - ref_name, url = _resolve_ref(ref) - try: - response = httpx.get(url, follow_redirects=True, timeout=60) - response.raise_for_status() - return DownloadResult(content=response.content), ref_name - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - raise - except Exception as e: - return DownloadResult(error=e), ref_name - - -def _find_repo_root(tmp_dir: Path) -> Path: - """Find the extracted FastLED repo root directory.""" - # Try common patterns: FastLED-master, FastLED-3.9.12, FastLED-, etc. - dirs = [d for d in tmp_dir.iterdir() if d.is_dir() and d.name.startswith("FastLED")] - if dirs: - return dirs[0] - raise FileNotFoundError("Failed to find FastLED directory in downloaded archive") - - def _collect_examples_from_dir(examples_dir: Path) -> list[str]: """Collect example names from an examples directory.""" if not examples_dir.exists(): @@ -224,7 +204,6 @@ def _collect_examples_from_dir(examples_dir: Path) -> list[str]: if ino_files: found.append(entry.name) else: - # Check nested directories (e.g., examples/Fx/FxSdCard/) for nested in entry.iterdir(): if nested.is_dir() and list(nested.glob("*.ino")): found.append(nested.name) @@ -237,28 +216,13 @@ def get_examples(ref: str | None = None) -> list[str]: Args: ref: Git ref to use. None means latest release. """ - # If we're inside the FastLED repo, use local examples local_repo = _find_fastled_repo_via_library_json(Path.cwd()) if local_repo is not None: print(f"Using local FastLED repo at {local_repo}") return _collect_examples_from_dir(local_repo / "examples") - print("Fetching examples from FastLED GitHub repo...") - with Spinner("Downloading FastLED repo..."): - result, _ = _download_fastled_repo(ref) - if not result.ok: - assert result.error is not None - raise result.error - - assert result.content is not None - tmp_dir = Path(tempfile.mkdtemp(prefix="fastled_examples_")) - try: - with zipfile.ZipFile(io.BytesIO(result.content)) as zf: - zf.extractall(tmp_dir) - repo_root = _find_repo_root(tmp_dir) - return _collect_examples_from_dir(repo_root / "examples") - finally: - shutil.rmtree(tmp_dir, ignore_errors=True) + cached = _get_local_fastled_repo(ref) + return _collect_examples_from_dir(cached.root / "examples") def _prompt_for_example(ref: str | None = None) -> str: @@ -266,7 +230,6 @@ def _prompt_for_example(ref: str | None = None) -> str: examples = get_examples(ref=ref) - # Find default example index (prefer DEFAULT_EXAMPLE if it exists) default_index = 0 if DEFAULT_EXAMPLE in examples: default_index = examples.index(DEFAULT_EXAMPLE) @@ -279,34 +242,24 @@ def _prompt_for_example(ref: str | None = None) -> str: ) if result is None: - # Fallback to DEFAULT_EXAMPLE if user cancelled return DEFAULT_EXAMPLE return result def project_init( - example: str | None = "PROMPT", # prompt for example + example: str | None = "PROMPT", outputdir: Path | None = None, ref: str | None = None, ) -> Path: - """Initialize a new FastLED project by downloading the example from GitHub. - - Args: - example: Example name, "PROMPT" to prompt user, or None for default. - outputdir: Output directory (defaults to "fastled"). - ref: Git ref to download. None means latest release. Use "master" for - master branch, a branch name, or a commit SHA. - """ + """Initialize a new FastLED project from a Rust-cached repo copy.""" outputdir = Path(outputdir) if outputdir is not None else Path("fastled") outputdir.mkdir(exist_ok=True, parents=True) - # Check if we're inside the FastLED repo — use local files instead local_repo = _find_fastled_repo_via_library_json(Path.cwd()) if local_repo is not None: return _init_from_local_repo(local_repo, example, outputdir) - # Check for saved ref in fastled.json (in the output directory or cwd) if ref is None: saved_ref = _read_fastled_json(Path.cwd()) or _read_fastled_json(outputdir) if saved_ref is not None: @@ -327,39 +280,23 @@ def project_init( ref_display = ref if ref else "latest release" print( - f"Initializing project with example '{example}' from GitHub repo ({ref_display})" + f"Initializing project with example '{example}' from FastLED repo ({ref_display})" ) - with Spinner(f"Downloading FastLED repo for '{example}'..."): - result, resolved_ref = _download_fastled_repo(ref) - - print() # New line after spinner - - if not result.ok: - assert result.error is not None - raise result.error - - assert result.content is not None - tmp_dir = Path(tempfile.mkdtemp(prefix="fastled_init_")) - try: - with zipfile.ZipFile(io.BytesIO(result.content)) as zf: - zf.extractall(tmp_dir) + cached = _get_local_fastled_repo(ref) + repo_root = cached.root + resolved_ref = cached.ref_name - repo_root = _find_repo_root(tmp_dir) - - example_src = _find_example_in_repo(repo_root, example) - if example_src is None: - raise FileNotFoundError(f"Example '{example}' not found in FastLED repo") + example_src = _find_example_in_repo(repo_root, example) + if example_src is None: + raise FileNotFoundError(f"Example '{example}' not found in FastLED repo") - dest = outputdir / example - dest.mkdir(parents=True, exist_ok=True) - shutil.copytree(example_src, dest, dirs_exist_ok=True) - finally: - shutil.rmtree(tmp_dir, ignore_errors=True) + dest = outputdir / example + dest.mkdir(parents=True, exist_ok=True) + shutil.copytree(example_src, dest, dirs_exist_ok=True) out = outputdir / example - # Save fastled.json for non-default refs if ref is not None: _write_fastled_json(out, resolved_ref) print(f"Saved ref '{resolved_ref}' to {out / 'fastled.json'}") From 0ab3a82096c6998553a8c6a98abd8d853f144dc2 Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 12 May 2026 19:39:39 -0700 Subject: [PATCH 5/5] chore(rust): own Chrome extension CRX download; drop httpx (closes #63) Last remaining Python httpx callsites moved into the Rust CLI: - crates/fastled-cli/src/install.rs - ensure_chrome_extension(id, name): GET the Chrome Web Store CRX with the proper User-Agent / Referer the endpoint expects, scan the body for the PK\x03\x04 (or PK\x05\x06) ZIP magic, strip the CRX header, extract the embedded zip into ~/.fastled/chrome-extensions/{name}/. Cached via manifest.json check. - crates/fastled-cli/src/main.rs - new hidden plumbing flag --internal-ensure-chrome-extension EXTENSION_ID NAME that prints the install path to stdout (mirrors the FastLED-repo flag). - src/fastled/playwright/chrome_extension_downloader.py - thin shim that subprocesses the Rust binary. ChromeExtensionDownloader class deleted (no external callers). - src/fastled/toolchain/emscripten.py - EmscriptenToolchain._download_fastled used httpx to grab FastLED master for native compile when no local repo was specified. Now delegates to project_init._ensure_repo_via_rust (the Rust binary handles the download) and copies the cached repo into the short-name target_dir/repo layout the build pipeline expects. pyproject.toml: drop httpx. Net runtime deps now: zcmds_win32 (Win-only). Lint clean, 119 unit tests pass, no `import httpx` left in src/. --- crates/fastled-cli/src/install.rs | 80 ++++++ crates/fastled-cli/src/main.rs | 27 ++ pyproject.toml | 1 - .../playwright/chrome_extension_downloader.py | 263 +++++------------- src/fastled/toolchain/emscripten.py | 45 +-- 5 files changed, 198 insertions(+), 218 deletions(-) diff --git a/crates/fastled-cli/src/install.rs b/crates/fastled-cli/src/install.rs index 09f81344..92048b78 100644 --- a/crates/fastled-cli/src/install.rs +++ b/crates/fastled-cli/src/install.rs @@ -23,6 +23,10 @@ const FASTLED_REPO: &str = "FastLED/FastLED"; const FASTLED_LATEST_RELEASE_API: &str = "https://api.github.com/repos/FastLED/FastLED/releases/latest"; +const CHROME_CRX_URL: &str = "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=114.0&acceptformat=crx2,crx3&x=id%3D{ID}%26uc"; +const CHROME_USER_AGENT: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"; + fn fastled_root() -> Result { let home = dirs::home_dir().context("cannot resolve home directory")?; Ok(home.join(".fastled")) @@ -409,3 +413,79 @@ pub fn ensure_fastled_repo(ref_opt: Option<&str>) -> Result { Ok(repo_dir) } + +// --------------------------------------------------------------------------- +// Chrome extension (CRX) download +// --------------------------------------------------------------------------- + +/// Download a CRX from the Chrome Web Store, strip the CRX header, extract +/// the embedded ZIP into ``~/.fastled/chrome-extensions/{name}/``, and return +/// the install path. +/// +/// Reuses the cached extension if `manifest.json` already exists at the +/// destination. +pub fn ensure_chrome_extension(extension_id: &str, name: &str) -> Result { + let root = fastled_root()?; + let install_dir = root.join("chrome-extensions").join(name); + if install_dir.join("manifest.json").is_file() { + return Ok(install_dir); + } + fs::create_dir_all(&install_dir) + .with_context(|| format!("create chrome extension dir {}", install_dir.display()))?; + + let crx_url = CHROME_CRX_URL.replace("{ID}", extension_id); + + let client = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(120)) + .redirect(reqwest::redirect::Policy::limited(10)) + .build() + .context("failed to build HTTP client for CRX download")?; + + let bytes = client + .get(&crx_url) + .header("User-Agent", CHROME_USER_AGENT) + .header("Referer", "https://chrome.google.com") + .send() + .with_context(|| format!("GET {crx_url} failed"))? + .error_for_status() + .with_context(|| format!("CRX download returned error for {crx_url}"))? + .bytes() + .context("read CRX body")?; + + // CRX file starts with a header; find the embedded ZIP magic. + let buf: &[u8] = &bytes; + let zip_start = find_zip_start(buf) + .with_context(|| format!("no ZIP magic found in CRX for {extension_id}"))?; + + // Write the ZIP portion to a temp file and extract. + let tmp_zip = tempfile::NamedTempFile::new().context("create temp zip for CRX extraction")?; + { + use std::io::Write; + let mut handle = tmp_zip.as_file(); + handle + .write_all(&buf[zip_start..]) + .context("write CRX zip portion")?; + handle.flush().ok(); + } + archive::extract_zip(tmp_zip.path(), &install_dir)?; + + if !install_dir.join("manifest.json").is_file() { + let _ = fs::remove_dir_all(&install_dir); + anyhow::bail!( + "extension extraction failed: manifest.json not found in {}", + install_dir.display() + ); + } + Ok(install_dir) +} + +fn find_zip_start(bytes: &[u8]) -> Option { + // PK\x03\x04 is a normal ZIP local file header; PK\x05\x06 is an empty + // central directory end-of-archive marker. Mirror the Python search. + let local = [0x50u8, 0x4B, 0x03, 0x04]; + let end = [0x50u8, 0x4B, 0x05, 0x06]; + bytes + .windows(4) + .position(|w| w == local) + .or_else(|| bytes.windows(4).position(|w| w == end)) +} diff --git a/crates/fastled-cli/src/main.rs b/crates/fastled-cli/src/main.rs index f703240f..bc5523fb 100644 --- a/crates/fastled-cli/src/main.rs +++ b/crates/fastled-cli/src/main.rs @@ -397,6 +397,12 @@ struct Cli { #[arg(long, value_name = "REF", num_args = 0..=1, default_missing_value = "__latest__", hide = true)] internal_ensure_fastled_repo: Option, + /// Internal plumbing flag: download a Chrome Web Store extension (by ID) + /// into ``~/.fastled/chrome-extensions//``, print the path to + /// stdout, and exit. Used by the Python playwright helpers. + #[arg(long, value_names = ["EXTENSION_ID", "NAME"], num_args = 2, hide = true)] + internal_ensure_chrome_extension: Vec, + // Build mode (mutually exclusive). /// Build in debug mode. #[arg(long, conflicts_with_all = ["quick", "release"])] @@ -608,6 +614,27 @@ fn main() -> ExitCode { }; } + // Hidden plumbing: download a Chrome Web Store extension and print its + // local install path. Used by the Python playwright helpers. + if !cli.internal_ensure_chrome_extension.is_empty() { + if cli.internal_ensure_chrome_extension.len() != 2 { + eprintln!("fastled: --internal-ensure-chrome-extension requires EXTENSION_ID NAME"); + return ExitCode::FAILURE; + } + let extension_id = &cli.internal_ensure_chrome_extension[0]; + let name = &cli.internal_ensure_chrome_extension[1]; + return match install::ensure_chrome_extension(extension_id, name) { + Ok(path) => { + println!("{}", path.display()); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("fastled: failed to fetch Chrome extension: {e:#}"); + ExitCode::FAILURE + } + }; + } + // Handle --serve-dir natively with the Rust HTTP server (no Python needed). if let Some(ref serve_dir) = cli.serve_dir { return serve_directory(serve_dir, &cli); diff --git a/pyproject.toml b/pyproject.toml index 48b59b9b..7840b05f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ keywords = ["template-python-cmd"] license = { text = "BSD 3-Clause License" } classifiers = ["Programming Language :: Python :: 3"] dependencies = [ - "httpx>=0.28.1", "zcmds_win32; sys_platform == 'win32'", ] diff --git a/src/fastled/playwright/chrome_extension_downloader.py b/src/fastled/playwright/chrome_extension_downloader.py index f0dd57c6..ae97fed9 100644 --- a/src/fastled/playwright/chrome_extension_downloader.py +++ b/src/fastled/playwright/chrome_extension_downloader.py @@ -1,213 +1,102 @@ -""" -Chrome extension downloader utility for FastLED WASM compiler. +"""Chrome Web Store extension downloader (thin Python shim). -This module provides functionality to download Chrome extensions from the -Chrome Web Store and prepare them for use with Playwright browser. +The actual CRX download + decode lives in `crates/fastled-cli/src/install.rs` +(`ensure_chrome_extension`) so the Python side no longer imports httpx. +This module just invokes the Rust binary via a hidden internal flag and +returns the local install path. """ -import os -import re -import shutil -import tempfile +import subprocess +import sys import warnings -import zipfile from pathlib import Path -import httpx - from fastled.interrupts import handle_keyboard_interrupt - -class ChromeExtensionDownloader: - """Downloads Chrome extensions from the Chrome Web Store.""" - - # Chrome Web Store CRX download URL - CRX_URL = "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=114.0&acceptformat=crx2,crx3&x=id%3D{extension_id}%26uc" - - # Modern user agent string - USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" - - def __init__(self, cache_dir: Path | None = None): - """Initialize the Chrome extension downloader. - - Args: - cache_dir: Directory to store downloaded extensions. - Defaults to ~/.fastled/chrome-extensions - """ - if cache_dir is None: - cache_dir = Path.home() / ".fastled" / "chrome-extensions" - - self.cache_dir = Path(cache_dir) - self.cache_dir.mkdir(parents=True, exist_ok=True) - - self.headers = { - "User-Agent": self.USER_AGENT, - "Referer": "https://chrome.google.com", - } - - def extract_extension_id(self, url: str) -> str: - """Extract extension ID from Chrome Web Store URL. - - Args: - url: Chrome Web Store URL - - Returns: - Extension ID string - - Raises: - ValueError: If URL is not a valid Chrome Web Store URL - """ - # Match new Chrome Web Store URLs (chromewebstore.google.com) - new_pattern = r"chromewebstore\.google\.com/detail/[^/]+/([a-z]{32})" - match = re.search(new_pattern, url) - - if match: - return match.group(1) - - # Match old Chrome Web Store URLs (chrome.google.com/webstore) - old_pattern = r"chrome\.google\.com/webstore/detail/[^/]+/([a-z]{32})" - match = re.search(old_pattern, url) - - if match: - return match.group(1) - - # Try direct extension ID - if re.match(r"^[a-z]{32}$", url): - return url - - raise ValueError(f"Invalid Chrome Web Store URL or extension ID: {url}") - - def download_crx(self, extension_id: str) -> bytes: - """Download CRX file from Chrome Web Store. - - Args: - extension_id: Chrome extension ID - - Returns: - CRX file content as bytes - - Raises: - httpx.RequestError: If download fails - """ - download_url = self.CRX_URL.format(extension_id=extension_id) - - with httpx.Client(follow_redirects=True) as client: - response = client.get(download_url, headers=self.headers) - response.raise_for_status() - - return response.content - - def extract_crx_to_directory(self, crx_content: bytes, extract_dir: Path) -> None: - """Extract CRX file content to a directory. - - CRX files are essentially ZIP files with a header that needs to be removed. - - Args: - crx_content: CRX file content as bytes - extract_dir: Directory to extract the extension to - """ - # CRX files have a header before the ZIP content - # We need to find the ZIP header (starts with 'PK') - zip_start = crx_content.find(b"PK\x03\x04") - if zip_start == -1: - zip_start = crx_content.find(b"PK\x05\x06") # Empty ZIP - - if zip_start == -1: - raise ValueError("Could not find ZIP header in CRX file") - - # Extract the ZIP portion - zip_content = crx_content[zip_start:] - - # Create temporary file to extract from - with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temp_zip: - temp_zip.write(zip_content) - temp_zip_path = temp_zip.name - - try: - # Extract the ZIP file - extract_dir.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(temp_zip_path, "r") as zip_ref: - zip_ref.extractall(extract_dir) - finally: - # Clean up temporary file - os.unlink(temp_zip_path) - - def get_extension_path( - self, url_or_id: str, extension_name: str | None = None - ) -> Path: - """Download and extract Chrome extension, returning the path to the extracted directory. - - Args: - url_or_id: Chrome Web Store URL or extension ID - extension_name: Optional name for the extension directory - - Returns: - Path to the extracted extension directory - """ - extension_id = self.extract_extension_id(url_or_id) - - if extension_name is None: - extension_name = extension_id - - extension_dir = self.cache_dir / extension_name - - # Check if extension is already downloaded and extracted - if extension_dir.exists() and (extension_dir / "manifest.json").exists(): - print(f"✅ Chrome extension already cached: {extension_dir}") - return extension_dir - - print(f"🔽 Downloading Chrome extension {extension_id}...") - - try: - # Download the CRX file - crx_content = self.download_crx(extension_id) - - # Clean up existing directory if it exists - if extension_dir.exists(): - shutil.rmtree(extension_dir) - - # Extract the CRX file - self.extract_crx_to_directory(crx_content, extension_dir) - - # Verify extraction worked - if not (extension_dir / "manifest.json").exists(): - raise ValueError("Extension extraction failed - no manifest.json found") - - print(f"✅ Chrome extension downloaded and extracted: {extension_dir}") - return extension_dir - - except KeyboardInterrupt as ki: - handle_keyboard_interrupt(ki) - raise - except Exception as e: - warnings.warn(f"Failed to download Chrome extension {extension_id}: {e}") - if extension_dir.exists(): - shutil.rmtree(extension_dir) - raise +CPP_DEVTOOLS_EXTENSION_ID = "pdcpmagijalfljmkmjngeonclgbbannb" +CPP_DEVTOOLS_EXTENSION_NAME = "cpp-devtools-support" + + +def _find_rust_fastled_cli() -> Path | None: + """Locate the Rust fastled binary, preferring target/ over the venv shim.""" + import shutil + + exe_name = "fastled.exe" if sys.platform == "win32" else "fastled" + + current = Path(__file__).resolve().parent + for _ in range(10): + if (current / "Cargo.toml").is_file(): + for profile in ("release", "debug"): + candidate = current / "target" / profile / exe_name + if candidate.is_file(): + return candidate + target_dir = current / "target" + if target_dir.is_dir(): + for arch_dir in target_dir.iterdir(): + if arch_dir.is_dir() and not arch_dir.name.startswith("."): + for profile in ("release", "debug"): + candidate = arch_dir / profile / exe_name + if candidate.is_file(): + return candidate + break + parent = current.parent + if parent == current: + break + current = parent + + found = shutil.which(exe_name) + if found: + return Path(found) + return None def download_cpp_devtools_extension() -> Path | None: - """Download the C++ DevTools Support (DWARF) extension. + """Ensure the C++ DevTools (DWARF) Chrome extension is installed locally. - Returns: - Path to the extracted extension directory, or None if download failed + Returns the install path, or ``None`` if the Rust binary is missing or + the download failed (a warning is emitted for caller-visibility). """ - # C++ DevTools Support (DWARF) extension - extension_url = "https://chromewebstore.google.com/detail/cc++-devtools-support-dwa/pdcpmagijalfljmkmjngeonclgbbannb" + cli = _find_rust_fastled_cli() + if cli is None: + warnings.warn( + "Could not locate the Rust fastled CLI; " + "Chrome extension cannot be installed." + ) + return None + + cmd = [ + str(cli), + "--internal-ensure-chrome-extension", + CPP_DEVTOOLS_EXTENSION_ID, + CPP_DEVTOOLS_EXTENSION_NAME, + ] try: - downloader = ChromeExtensionDownloader() - return downloader.get_extension_path(extension_url, "cpp-devtools-support") + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) except KeyboardInterrupt as ki: handle_keyboard_interrupt(ki) raise except Exception as e: - warnings.warn(f"Failed to download C++ DevTools Support extension: {e}") + warnings.warn(f"Failed to invoke fastled CLI for Chrome extension: {e}") + return None + + if result.returncode != 0: + sys.stderr.write(result.stderr) + warnings.warn( + "fastled --internal-ensure-chrome-extension failed " + f"(exit {result.returncode})" + ) + return None + + path_str = result.stdout.strip().splitlines()[-1] if result.stdout else "" + path = Path(path_str) + if not path.is_dir(): + warnings.warn(f"Rust binary returned non-existent extension path: {path_str!r}") return None + return path if __name__ == "__main__": - # Test the downloader with the C++ DevTools Support extension extension_path = download_cpp_devtools_extension() if extension_path: print(f"Extension downloaded to: {extension_path}") diff --git a/src/fastled/toolchain/emscripten.py b/src/fastled/toolchain/emscripten.py index d9dad44c..8c7d3333 100644 --- a/src/fastled/toolchain/emscripten.py +++ b/src/fastled/toolchain/emscripten.py @@ -324,41 +324,26 @@ def _find_compilers(self) -> CompilerPaths: CACHE_MAX_AGE_SECONDS = 24 * 60 * 60 # 24 hours def _download_fastled(self, target_dir: Path) -> Path: - """Download FastLED library from GitHub master branch.""" - import io - import zipfile - - import httpx - - print("Downloading FastLED library from GitHub master...") - response = httpx.get( - self.FASTLED_ARCHIVE_URL, follow_redirects=True, timeout=60 - ) - response.raise_for_status() - - with zipfile.ZipFile(io.BytesIO(response.content)) as zf: - zf.extractall(target_dir) + """Materialise the FastLED library at ``target_dir/repo``. + + Delegates the actual GitHub download to the Rust CLI's + ``--internal-ensure-fastled-repo`` flag (see + ``crates/fastled-cli/src/install.rs``) so the Python side no longer + needs httpx. The Rust binary caches under ``~/.fastled/cache/`` and + returns a path which we then copy into ``target_dir/repo`` (the + short-name move avoids Windows command-line length limits in + deeply-nested build paths). + """ + from fastled.project_init import _ensure_repo_via_rust - fastled_dir = target_dir / "FastLED-master" - if not fastled_dir.exists(): - dirs = [ - d - for d in target_dir.iterdir() - if d.is_dir() and d.name.startswith("FastLED") - ] - if dirs: - fastled_dir = dirs[0] - else: - raise FileNotFoundError("Failed to extract FastLED library") + print("Materialising FastLED library via fastled CLI (master)...") + repo_root = _ensure_repo_via_rust("master") - # Rename to a shorter name to avoid Windows path length limits. - # The deeply nested build paths (e.g. .build/meson-wasm-quick/...) - # combined with long directory names cause emar command lines to - # exceed Windows' CreateProcess limit ([WinError 206]). short_dir = target_dir / "repo" if short_dir.exists(): shutil.rmtree(short_dir, ignore_errors=True) - fastled_dir.rename(short_dir) + target_dir.mkdir(parents=True, exist_ok=True) + shutil.copytree(repo_root, short_dir, symlinks=True) return short_dir def _is_cache_fresh(self) -> bool: