diff --git a/crates/fbuild-toolchain/src/toolchain/README.md b/crates/fbuild-toolchain/src/toolchain/README.md index af633b1b0..7631ac4eb 100644 --- a/crates/fbuild-toolchain/src/toolchain/README.md +++ b/crates/fbuild-toolchain/src/toolchain/README.md @@ -10,6 +10,8 @@ Platform-specific toolchain management: download, cache, and provide tool paths - **`teensy_arm.rs`** -- Teensy-pinned ARM GCC 11.3.1 package from PlatformIO/PJRC - **`esp32.rs`** -- ESP32 RISC-V (`riscv32-esp-elf`) and Xtensa (`xtensa-esp-elf`) GCC from Espressif - **`esp32_metadata.rs`** -- Resolves ESP32 toolchain URLs from `tools.json` metadata packages +- **`esp_idf_install.rs`** -- Installs a whole pinned ESP-IDF release (source + tools + Python) and returns the `PATH`/env a delegated `idf.py` build needs: the venv's `bin` leads `PATH`, `IDF_PATH` and `IDF_PYTHON_ENV_PATH` are exported, and `IDF_TOOLS_PATH` deliberately is not (#1970) +- **`esp_idf_python.rs`** -- Pins the ESP-IDF v5.5.1 *Python environment*: a python-build-standalone CPython 3.12 per host plus the release's `requirements.core.txt` resolved against its constraints into one hash-locked file, installed with `pip --require-hashes --no-deps` (#1970) - **`esp_idf_release.rs`** -- Pinned ESP-IDF v5.5.1 *tool components* (Xtensa GCC, ULP binutils, ROM ELFs, OpenOCD) as fbuild packages, with an `Esp32S3Only` install profile; replaces `install.sh` provisioning (#1970) - **`esp_idf_release_profile.rs`** -- Installs a whole `IdfProfile` and describes the environment a delegated `idf.py` build needs (PATH entries, `ESP_ROM_ELF_DIR`, `OPENOCD_SCRIPTS`); `installed_bytes()` measures the installed tree in `du` terms (hardlinks counted once) so the 1.3 GB budget is checked against disk (#1970) - **`esp_idf_source.rs`** -- Pins the ESP-IDF v5.5.1 *source tree* as one fbuild package: the pinned commit's own archive plus all 23 gitlinks, each sha256-verified and filled by `SubmodulePlan`, so a delegated `idf.py` build has a real checkout with no `git` and no `install.sh` (#1970) diff --git a/crates/fbuild-toolchain/src/toolchain/esp_idf_install.rs b/crates/fbuild-toolchain/src/toolchain/esp_idf_install.rs new file mode 100644 index 000000000..05fea60d8 --- /dev/null +++ b/crates/fbuild-toolchain/src/toolchain/esp_idf_install.rs @@ -0,0 +1,261 @@ +//! Whole-release install of the pinned ESP-IDF (FastLED/fbuild#1970). +//! +//! The three fbuild packages a delegated `idf.py` build (#1971) needs are pinned +//! separately — tool archives ([`super::esp_idf_release`]), the source tree +//! ([`super::esp_idf_source`]) and the Python environment +//! ([`super::esp_idf_python`]) — because they are independent downloads with +//! independent cache keys. This module is the one place that installs all three +//! and answers the only question a build orchestrator should ask: *what +//! environment do I hand the subprocess?* +//! +//! Two properties matter and are both tested here: +//! +//! - **Ordering is part of the answer.** `idf.py` is a Python program whose +//! shebang is the pinned venv interpreter, and `idf.py` in turn re-executes +//! from `IDF_PATH`. So the venv's `bin/` must come *before* the tool `bin/` +//! directories on `PATH`; otherwise a host `python3` wins and ESP-IDF's own +//! dependency checker rejects an environment that is in fact correct. +//! - **The environment never mentions `IDF_TOOLS_PATH`.** That variable makes +//! `idf_tools.py` look for an `~/.espressif`-shaped tree, which is exactly the +//! side tree this design removes; the tool roots are exported directly as +//! `PATH` entries plus the two variables CMake reads (`ESP_ROM_ELF_DIR`, +//! `OPENOCD_SCRIPTS`). +//! +//! Nothing here builds anything. It is the input to #1971. + +use std::collections::HashSet; +use std::path::Path; + +use fbuild_core::Result; +use fbuild_core::path::NormalizedPath; + +use super::esp_idf_python::{EspIdfPython, IdfPythonEnvironment}; +use super::esp_idf_release::{IdfComponent, IdfProfile}; +use super::esp_idf_release_profile::{EspIdfRelease, IdfEnvironment, dir_bytes}; +use super::esp_idf_source::EspIdfSource; + +/// The three fbuild packages that make up one ESP-IDF release. +/// +/// A unit is listed separately from a component because the units fail +/// independently and are cached independently: a re-run after a failed Python +/// install must not re-download the 450 MB source tree, and this enum is what a +/// future `--report` would name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum IdfInstallUnit { + /// The pinned source checkout (`esp-idf-v5.5.1/source`). + Source, + /// Every tool archive the profile installs. + Tools, + /// The pinned interpreter and its locked requirements. + Python, +} + +impl IdfInstallUnit { + /// Every unit, in install order. + pub const ALL: [Self; 3] = [Self::Source, Self::Tools, Self::Python]; + + /// Cache key / log name of the unit. + pub const fn id(self) -> &'static str { + match self { + Self::Source => "source", + Self::Tools => "tools", + Self::Python => "python", + } + } +} + +/// A fully installed release: the tree, the tools and the interpreter. +#[derive(Debug, Clone)] +pub struct InstalledEspIdf { + source_root: NormalizedPath, + tools: IdfEnvironment, + python: IdfPythonEnvironment, +} + +impl InstalledEspIdf { + /// Assemble from already-installed pieces (used by tests and by callers that + /// resolve roots themselves, e.g. from a caller-supplied cache root). + pub fn from_parts( + source_root: NormalizedPath, + tools: IdfEnvironment, + python: IdfPythonEnvironment, + ) -> Self { + Self { + source_root, + tools, + python, + } + } + + /// Root of the pinned IDF source tree (`IDF_PATH`). + pub fn source_root(&self) -> &NormalizedPath { + &self.source_root + } + + /// The tool roots of the installed profile. + pub fn tools(&self) -> &IdfEnvironment { + &self.tools + } + + /// The Python environment `idf.py` must run in. + pub fn python(&self) -> &IdfPythonEnvironment { + &self.python + } + + /// `PATH` entries, most specific first. + /// + /// The venv's `bin` leads: `idf.py`'s shebang resolves `python` from here, + /// and ESP-IDF's startup check rejects the environment if a host interpreter + /// answers instead. Tool `bin/` directories follow in install order, which + /// keeps `xtensa-esp32-elf-gcc` (the profile's cross compiler) ahead of any + /// other tool that ships a same-named script. + pub fn path_dirs(&self) -> Vec { + let mut dirs = vec![self.python.bin_dir()]; + dirs.extend(self.tools.path_dirs()); + dirs + } + + /// Environment variables for a delegated build, in a stable order. + /// + /// `IDF_PATH` is exported rather than passed as `--idf-path` because + /// `idf.py` re-invokes `idf_tools.py` and CMake from child processes, and + /// children inherit the environment — not the parent's argv. + pub fn env_vars(&self) -> Vec<(&'static str, NormalizedPath)> { + let mut vars = vec![ + ("IDF_PATH", self.source_root.clone()), + ("IDF_PYTHON_ENV_PATH", self.python.env_dir().clone()), + ]; + vars.extend(self.tools.env_vars()); + vars + } + + /// Real bytes on disk of all three packages, `du`-style: symlinks are not + /// followed and a hardlinked file is counted once. + /// + /// One identity set spans the whole release because a future component may + /// hardlink an interpreter or a tool into another entry; counting the + /// entries separately would then over-report exactly like the per-component + /// walk did before #1976. + pub fn installed_bytes(&self) -> u64 { + let mut seen: HashSet = HashSet::new(); + let source = dir_bytes(self.source_root.as_path(), &mut seen); + let tools = self + .tools + .tool_roots() + .iter() + .map(|(_, root)| dir_bytes(root.as_path(), &mut seen)) + .sum::(); + let python = dir_bytes(self.python.env_dir().as_path(), &mut seen); + source + tools + python + } + + /// The units that are present on disk. + /// + /// Reported separately from [`Self::installed_bytes`] so a partial tree can + /// be named in an error instead of showing up as a suspiciously small byte + /// count. + pub fn missing_units(&self) -> Vec { + let mut missing = Vec::new(); + if !self.source_root.as_path().is_dir() { + missing.push(IdfInstallUnit::Source); + } + if self.tools.tool_roots().is_empty() { + missing.push(IdfInstallUnit::Tools); + } + if !self.python.python().as_path().is_file() { + missing.push(IdfInstallUnit::Python); + } + missing + } +} + +/// Installs one pinned ESP-IDF release: source + tools + Python. +pub struct EspIdfInstall { + profile: IdfProfile, + source: EspIdfSource, + release: EspIdfRelease, + python: EspIdfPython, +} + +impl EspIdfInstall { + /// Resolve every package against the default cache. + pub fn new(project_dir: &Path, profile: IdfProfile) -> Result { + Ok(Self { + profile, + source: EspIdfSource::new(project_dir)?, + release: EspIdfRelease::new(project_dir, profile)?, + python: EspIdfPython::new(project_dir)?, + }) + } + + /// Install into an explicit cache root (tests and the end-to-end proof). + /// + /// `#[cfg(test)]` on purpose, matching the sibling modules: an explicit root + /// is how the proof keeps ~1.6 GB of installed trees out of the user's real + /// cache. + #[cfg(test)] + pub(in crate::toolchain) fn with_cache_root( + project_dir: &Path, + cache_root: &Path, + profile: IdfProfile, + ) -> Result { + Ok(Self { + profile, + source: EspIdfSource::with_cache_root(project_dir, cache_root)?, + release: EspIdfRelease::with_cache_root(project_dir, cache_root, profile)?, + python: EspIdfPython::with_cache_root(project_dir, cache_root)?, + }) + } + + /// The profile this installer provisions. + pub fn profile(&self) -> IdfProfile { + self.profile + } + + /// True only when all three units are installed. + /// + /// Deliberately not `source && tools.is_installed() && python`: each of those + /// is stricter about *its* markers than a build needs, and this method is + /// the cheap pre-flight; [`Self::install`] is what repairs the difference. + pub fn is_installed(&self) -> bool { + self.source.is_installed() && self.release.is_installed() && self.python.is_installed() + } + + /// Install the three units in order and return the build environment. + /// + /// Order is `source -> tools -> python`, and it is the cheap-to-diagnose + /// order rather than the dependency order (none of the three depends on + /// another): the source tree is the one whose absence breaks the most + /// later steps, and the Python environment is the one whose install can + /// still fail after everything else succeeded (`pip` against PyPI). + /// + /// A unit already installed is skipped by its own package, so a second call + /// is a no-op and a failed call is repaired on re-run without re-downloading + /// what already landed. + pub async fn install(&self) -> Result { + let source_root = self.source.ensure_normalized().await?; + let tools = self.release.install().await?; + let python = self.python.install().await?; + Ok(InstalledEspIdf::from_parts(source_root, tools, python)) + } + + /// Real bytes of the three packages currently on disk, `du`-style. + /// + /// Each package is its own cache entry, so each is measured with its own + /// identity set; the units share no files by construction (separate keys + /// under `esp-idf-v5.5.1/`). + pub fn installed_bytes(&self) -> u64 { + let mut source_seen: HashSet = HashSet::new(); + let source = dir_bytes(self.source.install_path().as_path(), &mut source_seen); + source + self.release.installed_bytes() + self.python.installed_bytes() + } + + /// The tool components the profile installs (re-exported for reporting). + pub fn components(&self) -> &'static [IdfComponent] { + self.profile.components() + } +} + +#[cfg(test)] +#[path = "esp_idf_install_tests.rs"] +mod esp_idf_install_tests; diff --git a/crates/fbuild-toolchain/src/toolchain/esp_idf_install_tests.rs b/crates/fbuild-toolchain/src/toolchain/esp_idf_install_tests.rs new file mode 100644 index 000000000..054568140 --- /dev/null +++ b/crates/fbuild-toolchain/src/toolchain/esp_idf_install_tests.rs @@ -0,0 +1,278 @@ +//! Unit + end-to-end tests for the whole-release ESP-IDF installer (#1970). + +use super::*; +use crate::toolchain::esp_idf_release::IDF_RELEASE_VERSION; + +fn fabricated_source(base: &Path) -> NormalizedPath { + let root = NormalizedPath::new(base).join("source"); + std::fs::create_dir_all(root.join("tools").as_path()).unwrap(); + std::fs::write(root.join("tools").join("idf.py").as_path(), b"# idf\n").unwrap(); + root +} + +fn fabricated_tool(base: &Path, component: IdfComponent, with_bin: bool) -> NormalizedPath { + let root = NormalizedPath::new(base).join(component.id()); + std::fs::create_dir_all(root.as_path()).unwrap(); + if with_bin { + let bin = root.join("bin"); + std::fs::create_dir_all(bin.as_path()).unwrap(); + std::fs::write(bin.join("tool").as_path(), b"x").unwrap(); + } + root +} + +fn fabricated_envs(base: &Path) -> (IdfEnvironment, IdfPythonEnvironment) { + let tools = IdfEnvironment::from_roots(vec![ + ( + IdfComponent::XtensaEspElf, + fabricated_tool(base, IdfComponent::XtensaEspElf, true), + ), + ( + IdfComponent::Esp32UlpElf, + fabricated_tool(base, IdfComponent::Esp32UlpElf, true), + ), + ( + IdfComponent::EspRomElfs, + fabricated_tool(base, IdfComponent::EspRomElfs, false), + ), + ( + IdfComponent::OpenocdEsp32, + fabricated_tool(base, IdfComponent::OpenocdEsp32, true), + ), + ]); + // OpenOCD ships its script tree; `env_vars` only exports a path that exists. + let scripts = NormalizedPath::new(base) + .join(IdfComponent::OpenocdEsp32.id()) + .join("share") + .join("openocd") + .join("scripts"); + std::fs::create_dir_all(scripts.as_path()).unwrap(); + let env_dir = NormalizedPath::new(base).join("python_env").join("idf_env"); + std::fs::create_dir_all(env_dir.join("bin").as_path()).unwrap(); + std::fs::write(env_dir.join("bin").join("python").as_path(), b"#!/bin/sh\n").unwrap(); + (tools, IdfPythonEnvironment::new(env_dir)) +} + +#[test] +fn the_venv_bin_leads_path_then_every_tool_bin() { + let tmp = tempfile::TempDir::new().unwrap(); + let (tools, python) = fabricated_envs(tmp.path()); + let installed = + InstalledEspIdf::from_parts(fabricated_source(tmp.path()), tools, python.clone()); + let dirs = installed.path_dirs(); + assert_eq!(dirs.len(), 4, "venv bin + 3 tool bins: {dirs:?}"); + assert_eq!( + dirs[0], + python.bin_dir(), + "a host python answering before the venv breaks idf.py's startup check" + ); + assert_eq!( + installed + .tools() + .tool_root(IdfComponent::XtensaEspElf) + .unwrap() + .join("bin"), + dirs[1], + "the profile's cross compiler must be first among the tools" + ); +} + +#[test] +fn the_environment_names_idf_path_and_never_idf_tools_path() { + let tmp = tempfile::TempDir::new().unwrap(); + let (tools, python) = fabricated_envs(tmp.path()); + let source = fabricated_source(tmp.path()); + let installed = InstalledEspIdf::from_parts(source.clone(), tools, python.clone()); + let vars = installed.env_vars(); + let names: Vec<&str> = vars.iter().map(|(name, _)| *name).collect(); + assert_eq!( + names, + [ + "IDF_PATH", + "IDF_PYTHON_ENV_PATH", + "ESP_ROM_ELF_DIR", + "OPENOCD_SCRIPTS" + ] + ); + assert_eq!(vars[0].1, source); + assert_eq!(vars[1].1, *python.env_dir()); + assert!( + !names.contains(&"IDF_TOOLS_PATH"), + "that variable would point idf_tools.py back at an ~/.espressif side tree" + ); +} + +#[test] +fn installed_bytes_sums_every_unit_and_dedupes_hardlinks() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = fabricated_source(tmp.path()); + let (tools, python) = fabricated_envs(tmp.path()); + // 4096 bytes in the source tree, of which 2048 are a hardlink of a file + // inside the same tree: `du` counts it once. + std::fs::write(source.join("a.bin").as_path(), vec![0u8; 2048]).unwrap(); + std::fs::hard_link( + source.join("a.bin").as_path(), + source.join("a-link.bin").as_path(), + ) + .unwrap(); + let installed = InstalledEspIdf::from_parts(source, tools, python); + let measured = installed.installed_bytes(); + // 2048 (hardlink counted once) + 6 (`tools/idf.py`) + 10 (venv python stub) + // + 3 one-byte tool markers. + assert_eq!(measured, 2067, "measured {measured} B"); + assert!(installed.missing_units().is_empty()); +} + +#[test] +fn a_missing_unit_is_named_instead_of_reading_as_a_small_tree() { + let tmp = tempfile::TempDir::new().unwrap(); + let source = fabricated_source(tmp.path()); + let (tools, python) = fabricated_envs(tmp.path()); + let installed = InstalledEspIdf::from_parts(source, tools.clone(), python.clone()); + std::fs::remove_file(python.python().as_path()).unwrap(); + assert_eq!(installed.missing_units(), [IdfInstallUnit::Python]); + let only_tools = InstalledEspIdf::from_parts( + NormalizedPath::new(tmp.path()).join("nowhere"), + tools, + python, + ); + assert_eq!( + only_tools.missing_units(), + [IdfInstallUnit::Source, IdfInstallUnit::Python] + ); +} + +#[test] +fn an_empty_cache_reports_the_whole_release_missing() { + let tmp = tempfile::TempDir::new().unwrap(); + let install = + EspIdfInstall::with_cache_root(tmp.path(), tmp.path(), IdfProfile::Esp32S3Only).unwrap(); + assert_eq!(install.profile(), IdfProfile::Esp32S3Only); + assert_eq!(install.components().len(), 4); + assert!(!install.is_installed()); + assert_eq!(install.installed_bytes(), 0); + assert_eq!( + IdfInstallUnit::ALL.map(IdfInstallUnit::id), + ["source", "tools", "python"] + ); +} + +/// The whole release, for real: 24 source archives, 4 tool archives and the +/// pinned interpreter with its locked wheels (~450 MB of downloads, ~1.6 GB on +/// disk). Asserts the combination, not the pieces: ESP-IDF's own +/// `tools/check_python_dependencies.py` runs out of the *installed* source tree +/// with the *installed* interpreter, which is the same gate `idf.py` runs at +/// startup, and the installed cross compiler reports the pinned version. +/// +/// Run with: +/// `soldr cargo test -p fbuild-toolchain --lib -- --ignored esp_idf_release_install` +#[tokio::test] +#[ignore = "downloads the whole pinned ESP-IDF release (~450 MB) and installs ~1.6 GB"] +async fn esp_idf_release_install_provisions_the_whole_release() { + let dir = tempfile::TempDir::new().unwrap(); + let install = + EspIdfInstall::with_cache_root(dir.path(), dir.path(), IdfProfile::Esp32S3Only).unwrap(); + assert!(!install.is_installed()); + + let installed = install.install().await.unwrap(); + assert!(install.is_installed()); + assert!(installed.missing_units().is_empty()); + + // The tree is the pinned release, versioned by IDF's own CMake file. + let version = installed + .source_root() + .join("tools") + .join("cmake") + .join("version.cmake"); + assert!(version.as_path().is_file(), "missing {version}"); + + // The combination works: IDF's checker, installed python, installed + // requirements, installed constraints. + let checker = installed + .source_root() + .join("tools") + .join("check_python_dependencies.py"); + let requirements = installed + .source_root() + .join("tools") + .join("requirements") + .join("requirements.core.txt"); + let constraints = install.python.constraints_path().to_string(); + let py = installed.python().python().to_string(); + let output = fbuild_core::subprocess::run_command( + &[ + &py, + &checker.to_string(), + "-c", + &constraints, + "-r", + &requirements.to_string(), + ], + None, + None, + None, + ) + .await + .unwrap(); + assert!( + output.success(), + "ESP-IDF rejected the combined environment:\n{}\n{}", + output.stdout, + output.stderr + ); + assert!( + output.stdout.contains("requirements are satisfied"), + "unexpected checker output: {}", + output.stdout + ); + + // And the toolchain half is real, at the pinned version. + let gcc = installed + .tools() + .tool_root(IdfComponent::XtensaEspElf) + .unwrap() + .join("bin") + .join("xtensa-esp32-elf-gcc"); + let version_out = + fbuild_core::subprocess::run_command(&[&gcc.to_string(), "--version"], None, None, None) + .await + .unwrap(); + assert!( + version_out.stdout.contains("esp-14.2.0_20241119"), + "xtensa GCC is not the pinned release: {}", + version_out.stdout + ); + + // Report, then AC2's tools-only budget plus the aggregate total. + let tools_bytes = install.release.installed_bytes(); + let total = installed.installed_bytes(); + println!( + "{IDF_RELEASE_VERSION}: tools {tools_bytes} B, release total {total} B ({:.0} MB)", + total as f64 / 1e6 + ); + assert!( + tools_bytes <= 1_300_000_000, + "tools measured {tools_bytes} B" + ); + assert!( + total > 1_500_000_000, + "aggregate measurement looks truncated: {total} B" + ); + + // AC4: a second install of the whole release is a no-op. + const SENTINEL: &[u8] = b"keep me"; + let sentinel = installed.source_root().join("agent-sentinel"); + std::fs::write(sentinel.as_path(), SENTINEL).unwrap(); + let again = install.install().await.unwrap(); + assert!( + sentinel.as_path().is_file(), + "a second install re-extracted" + ); + assert_eq!(again.env_vars(), installed.env_vars()); + assert_eq!(again.path_dirs(), installed.path_dirs()); + assert_eq!( + again.installed_bytes(), + total + SENTINEL.len() as u64, + "a second install changed more than the sentinel" + ); +} diff --git a/crates/fbuild-toolchain/src/toolchain/mod.rs b/crates/fbuild-toolchain/src/toolchain/mod.rs index 1c483e683..2d0077485 100644 --- a/crates/fbuild-toolchain/src/toolchain/mod.rs +++ b/crates/fbuild-toolchain/src/toolchain/mod.rs @@ -7,6 +7,7 @@ pub mod clang; pub mod esp32; pub mod esp32_metadata; pub mod esp8266; +pub mod esp_idf_install; pub mod esp_idf_python; pub mod esp_idf_release; pub mod esp_idf_release_profile; @@ -23,6 +24,7 @@ pub use arm::ArmToolchain; pub use arm_gcc8::ArmGcc8Toolchain; pub use avr::AvrToolchain; pub use clang::{ClangComponent, ClangComponentKind}; +pub use esp_idf_install::{EspIdfInstall, IdfInstallUnit, InstalledEspIdf}; pub use esp_idf_python::{ EspIdfPython, IDF_PYTHON_ENV_NAME, IdfPythonEnvironment, PYTHON_INTERPRETERS, PYTHON_VERSION, PythonInterpreterPin, REQUIREMENTS_CONSTRAINTS, REQUIREMENTS_LOCK,