Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/fbuild-build-esp/src/esp32/espidf_artifacts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ pub(super) const INDEX_FILE: &str = "idf-artifacts.json";

/// The seven artifacts a delegated build must produce (FastLED/fbuild#1971).
#[derive(Debug, Clone)]
pub(super) struct IdfArtifacts {
pub(crate) struct IdfArtifacts {
/// Project name as IDF named the image (`<name>.bin`).
pub project: String,
pub app_bin: NormalizedPath,
Expand Down
14 changes: 13 additions & 1 deletion crates/fbuild-build-esp/src/esp32/espidf_delegate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,12 @@ use fbuild_config::PlatformIOConfig;
use fbuild_core::path::NormalizedPath;
use fbuild_core::subprocess::{CommandReport, run_command_reporting};
use fbuild_core::{FbuildError, Result};
use fbuild_toolchain::toolchain::{EspIdfInstall, IdfProfile, InstalledEspIdf};
use fbuild_toolchain::toolchain::{
EspIdfInstall, IDF_RELEASE_VERSION, IdfProfile, InstalledEspIdf,
};

use super::espidf_artifacts::{IdfArtifacts, discover_and_publish};
use super::orchestrator::helpers::{IdfCmakeProof, esp32_idf_cmake_stack_summary};

/// `platformio.ini` key that opts an environment into the delegated build.
///
Expand Down Expand Up @@ -402,6 +405,15 @@ pub(super) async fn delegate_build(
outcome.build.stdout_bytes
));

// #1972: a delegated build may report `integration=idf-cmake`, but only
// once its own artifacts prove it. `verified` fails the build rather than
// letting the run continue with a weaker label, so a build that produced
// no bin/ELF, no compile commands or no sdkconfig is never described as a
// CMake/component build.
let proof = IdfCmakeProof::verified(&outcome.artifacts, IDF_RELEASE_VERSION)?;
ctx.build_log
.push(esp32_idf_cmake_stack_summary(&proof, &chip));

let index = outcome.artifacts.write_index(&chip, &build_dir)?;
let message = format!(
"ESP-IDF delegated build ({chip}): {} app image, {} bytes of artifacts, index {index}",
Expand Down
171 changes: 167 additions & 4 deletions crates/fbuild-build-esp/src/esp32/orchestrator/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};

use fbuild_core::Result;
use fbuild_core::path::NormalizedPath;
use fbuild_core::{FbuildError, Result};
use sha2::{Digest, Sha256};

use super::super::espidf_artifacts::IdfArtifacts;

use super::super::espidf_delegate::EspBuildSystem;

Expand All @@ -21,22 +25,181 @@ use super::super::espidf_delegate::EspBuildSystem;
/// compiled with the Arduino core. Recording the mode beside the requested and
/// resolved frameworks keeps a build log from claiming IDF component coverage
/// it never produced (FastLED/fbuild#1664).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum Esp32Integration {
/// `framework = arduino`: the Arduino core only.
ArduinoCore,
/// `framework = arduino, espidf`: the Arduino core links the precompiled
/// ESP-IDF SDK. Not an ESP-IDF component build.
ArduinoOnIdf,
/// A delegated CMake/`idf.py` build (FastLED/fbuild#1971). The variant
/// carries [`IdfCmakeProof`], so no code can name this mode — and
/// `RequestedFrameworks::integration`, which has no build result to
/// inspect, cannot produce it at all — without the artifacts behind it
/// (FastLED/fbuild#1972).
IdfCmake(IdfCmakeProof),
}

impl Esp32Integration {
pub(super) fn label(self) -> &'static str {
pub(super) fn label(&self) -> &'static str {
match self {
Self::ArduinoCore => "arduino-core",
Self::ArduinoOnIdf => "arduino-on-idf",
Self::IdfCmake(_) => IdfCmakeProof::INTEGRATION_LABEL,
}
}
}

/// Proof that a delegated `idf.py` build actually produced an application.
///
/// `integration=idf-cmake` is a claim about what a build did, and the whole
/// point of recording it is that a claim must never outrun the artifacts
/// behind it (FastLED/fbuild#1972, #1730). The fields are private and
/// [`IdfCmakeProof::verified`] is the only constructor, so a value of this
/// type exists only after the app image and ELF were measured non-empty, the
/// compile database was seen to hold at least one entry, and the generated
/// `sdkconfig` was hashed. A build that cannot produce that proof is an error,
/// never a silent downgrade to a weaker label.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct IdfCmakeProof {
idf_release: String,
app_bin_bytes: u64,
elf_bytes: u64,
compile_db_entries: usize,
sdkconfig_sha256: String,
}

impl IdfCmakeProof {
/// The only spelling of the integration mode; [`Esp32Integration::label`]
/// reads it from here so the literal has a single owner.
pub(crate) const INTEGRATION_LABEL: &'static str = "idf-cmake";

/// Measure `artifacts` and accept the claim only if every proof element is
/// present: non-empty app image and ELF, a non-empty compile database and
/// a readable `sdkconfig` digest.
pub(crate) fn verified(artifacts: &IdfArtifacts, idf_release: &str) -> Result<Self> {
let app_bin_bytes = nonempty_bytes(&artifacts.app_bin, "app image")?;
let elf_bytes = nonempty_bytes(&artifacts.elf, "app ELF")?;
let compile_db_entries = compile_db_entry_count(&artifacts.compile_commands)?;
if compile_db_entries == 0 {
return Err(FbuildError::Other(format!(
"the delegated ESP-IDF build produced an empty compile database ({}): a CMake/component build compiles at least one translation unit, so {}={} cannot be claimed for it",
artifacts.compile_commands,
"integration",
Self::INTEGRATION_LABEL
)));
}
let sdkconfig_sha256 = sha256_file(&artifacts.sdkconfig, "generated sdkconfig")?;
Ok(Self {
idf_release: idf_release.to_string(),
app_bin_bytes,
elf_bytes,
compile_db_entries,
sdkconfig_sha256,
})
}

pub(crate) fn idf_release(&self) -> &str {
&self.idf_release
}

pub(crate) fn app_bin_bytes(&self) -> u64 {
self.app_bin_bytes
}

pub(crate) fn elf_bytes(&self) -> u64 {
self.elf_bytes
}

pub(crate) fn compile_db_entries(&self) -> usize {
self.compile_db_entries
}

pub(crate) fn sdkconfig_sha256(&self) -> &str {
&self.sdkconfig_sha256
}
}

/// The stack summary for a delegated build — the only summary line allowed to
/// name `integration=idf-cmake`, and it cannot be written without the proof.
/// The pinned IDF release is part of the line so a log says *what* it built
/// with, not merely that it was "espidf".
pub(crate) fn esp32_idf_cmake_stack_summary(proof: &IdfCmakeProof, chip: &str) -> String {
let integration = Esp32Integration::IdfCmake(proof.clone());
format!(
"ESP32 packages: build_system=espidf, chip={chip}, integration={}, idf={}; proof: app_bin_bytes={}, elf_bytes={}, compile_db_entries={}, sdkconfig_sha256={}",
integration.label(),
proof.idf_release(),
proof.app_bin_bytes(),
proof.elf_bytes(),
proof.compile_db_entries(),
proof.sdkconfig_sha256(),
)
}

/// A required artifact's byte count; zero bytes is a failed build, not proof.
fn nonempty_bytes(path: &NormalizedPath, label: &str) -> Result<u64> {
let bytes = std::fs::metadata(path.as_path())
.map_err(|error| {
FbuildError::Other(format!(
"the delegated ESP-IDF build's {label} {path} is unreadable ({error}); {}={} needs it as proof",
"integration",
IdfCmakeProof::INTEGRATION_LABEL
))
})?
.len();
if bytes == 0 {
return Err(FbuildError::Other(format!(
"the delegated ESP-IDF build's {label} {path} is empty; {}={} cannot be claimed for a build with no {label}",
"integration",
IdfCmakeProof::INTEGRATION_LABEL
)));
}
Ok(bytes)
}

/// How many translation units the delegated build's compile database names.
fn compile_db_entry_count(path: &NormalizedPath) -> Result<usize> {
let raw = std::fs::read(path.as_path()).map_err(|error| {
FbuildError::Other(format!(
"the delegated ESP-IDF build's compile database {path} is unreadable ({error})"
))
})?;
let entries: serde_json::Value = serde_json::from_slice(&raw).map_err(|error| {
FbuildError::Other(format!(
"the delegated ESP-IDF build's compile database {path} is not valid JSON ({error})"
))
})?;
entries.as_array().map(Vec::len).ok_or_else(|| {
FbuildError::Other(format!(
"the delegated ESP-IDF build's compile database {path} is not a JSON array of compile commands"
))
})
}

/// SHA-256 of a file, streamed so a large `sdkconfig` is not held in memory.
fn sha256_file(path: &NormalizedPath, label: &str) -> Result<String> {
use std::io::Read;

let mut file = std::fs::File::open(path.as_path()).map_err(|error| {
FbuildError::Other(format!(
"the delegated ESP-IDF build's {label} {path} is unreadable ({error})"
))
})?;
let mut hasher = Sha256::new();
let mut buffer = vec![0u8; 64 * 1024];
loop {
let read = file.read(&mut buffer).map_err(|error| {
FbuildError::Other(format!(
"reading the delegated ESP-IDF build's {label} {path} failed: {error}"
))
})?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
Ok(format!("{:x}", hasher.finalize()))
}

/// The frameworks an environment's `framework =` value requests.
Expand Down Expand Up @@ -98,7 +261,7 @@ pub(super) fn esp32_package_stack_summary(
.map_or("<platform manifest>", str::trim);
let integration = requested_frameworks(requested_framework)
.integration()
.map_or("unsupported-espidf", Esp32Integration::label);
.map_or("unsupported-espidf", |integration| integration.label());
let sdk_version = installed_framework
.bundled_esp_idf_version(mcu)
.unwrap_or_else(|| "unknown (bundled with framework)".to_string());
Expand Down
2 changes: 1 addition & 1 deletion crates/fbuild-build-esp/src/esp32/orchestrator/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ mod embed_stage;
mod fingerprint;
mod framework_library_cache;
mod framework_libs;
mod helpers;
pub(crate) mod helpers;
mod job_pool;
mod local_libs;
mod packages;
Expand Down
104 changes: 102 additions & 2 deletions crates/fbuild-build-esp/src/esp32/orchestrator/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,17 @@ use super::super::espidf_delegate::{BUILD_SYSTEM_KEY, EspBuildSystem};
use super::Esp32Orchestrator;
use super::cdc::{cdc_on_boot_enabled, is_esp32_project, warn_if_cdc_on_boot};
use super::helpers::{
Esp32Integration, apply_effective_define_flags, esp32_package_stack_summary,
reject_unsupported_esp_framework, reject_unsupported_sdkconfig_overlay, requested_frameworks,
Esp32Integration, IdfCmakeProof, apply_effective_define_flags, esp32_idf_cmake_stack_summary,
esp32_package_stack_summary, reject_unsupported_esp_framework,
reject_unsupported_sdkconfig_overlay, requested_frameworks,
};
use super::helpers::{
framework_failure_marker, framework_signature, record_failed_framework_lib,
should_skip_failed_framework_lib,
};
use crate::BuildOrchestrator;
use fbuild_core::Platform;
use fbuild_core::path::NormalizedPath;
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::Duration;
Expand Down Expand Up @@ -445,3 +447,101 @@ fn stack_summary_records_requested_framework_and_integration_mode() {
);
assert!(plain.contains("integration=arduino-core"), "{plain}");
}

/// A set of published delegated artifacts in a throwaway root, so the proof
/// gate can be exercised against real files instead of fabricated counts.
fn artifact_fixture(
app_bin: &[u8],
elf: &[u8],
compile_db: &str,
) -> (
tempfile::TempDir,
super::super::espidf_artifacts::IdfArtifacts,
) {
let root = tempfile::TempDir::new().unwrap();
let dir = NormalizedPath::new(root.path());
let write = |name: &str, bytes: &[u8]| {
let path = dir.join(name);
std::fs::write(path.as_path(), bytes).unwrap();
path
};
let artifacts = super::super::espidf_artifacts::IdfArtifacts {
project: "hello_world".to_string(),
app_bin: write("hello_world.bin", app_bin),
elf: write("hello_world.elf", elf),
map: write("hello_world.map", b"map"),
bootloader_bin: write("bootloader.bin", b"boot"),
partition_table_bin: write("partition-table.bin", b"pt"),
sdkconfig: write("sdkconfig", b"CONFIG_IDF_TARGET=\"esp32s3\"\n"),
compile_commands: write("compile_commands.json", compile_db.as_bytes()),
};
(root, artifacts)
}

const ONE_ENTRY_DB: &str =
r#"[{"directory":"/proj/build","file":"/proj/main/app_main.c","command":"cc app_main.c"}]"#;

/// The gate is the constructor: `integration=idf-cmake` cannot be produced
/// from a build that did not leave an app image, an ELF and a compile database
/// entry behind (FastLED/fbuild#1972).
#[test]
fn idf_cmake_proof_is_refused_without_artifacts() {
let valid_db = ONE_ENTRY_DB;
let (_root, complete) = artifact_fixture(b"\x7fELFbin", b"\x7fELFelf", valid_db);
let proof = IdfCmakeProof::verified(&complete, "v5.5.1").expect("complete artifacts prove it");
assert!(proof.app_bin_bytes() > 0, "app image bytes are recorded");
assert!(proof.elf_bytes() > 0, "ELF bytes are recorded");
assert_eq!(proof.compile_db_entries(), 1);
assert_eq!(proof.sdkconfig_sha256().len(), 64, "sdkconfig is digested");

// Each proof element on its own is enough to refuse the claim.
let (_root, empty_bin) = artifact_fixture(b"", b"\x7fELFelf", valid_db);
let refused =
IdfCmakeProof::verified(&empty_bin, "v5.5.1").expect_err("an empty app image is not proof");
assert!(refused.to_string().contains("empty"), "{refused}");

let (_root, empty_elf) = artifact_fixture(b"\x7fELFbin", b"", valid_db);
assert!(IdfCmakeProof::verified(&empty_elf, "v5.5.1").is_err());

let (_root, empty_db) = artifact_fixture(b"\x7fELFbin", b"\x7fELFelf", "[]");
let refused = IdfCmakeProof::verified(&empty_db, "v5.5.1")
.expect_err("zero compile commands is not proof");
assert!(
refused.to_string().contains("empty compile database"),
"{refused}"
);
}

/// The claimed mode and the pinned release are on the summary line, and the
/// proof numbers travel with it so a log can be checked against the artifacts.
#[test]
fn idf_cmake_summary_names_the_pinned_idf_and_proof() {
let (_root, complete) = artifact_fixture(b"\x7fELFbin", b"\x7fELFelf", ONE_ENTRY_DB);
let proof = IdfCmakeProof::verified(&complete, "v5.5.1").unwrap();
let summary = esp32_idf_cmake_stack_summary(&proof, "esp32s3");
assert!(summary.contains("integration=idf-cmake"), "{summary}");
assert!(summary.contains("idf=v5.5.1"), "{summary}");
assert!(summary.contains("chip=esp32s3"), "{summary}");
assert!(summary.contains("compile_db_entries=1"), "{summary}");
assert!(summary.contains("sdkconfig_sha256="), "{summary}");
}

/// The requested-framework path inspects `framework =` only, so it has no
/// build result to prove anything with and must never name `idf-cmake`.
#[test]
fn requested_frameworks_can_never_claim_idf_cmake() {
for raw in [
"arduino",
"espidf",
"arduino, espidf",
"",
"ESPIDF",
"espidf, espidf",
] {
let integration = requested_frameworks(raw).integration();
assert!(
!matches!(integration, Some(Esp32Integration::IdfCmake(_))),
"framework = {raw} must not name idf-cmake"
);
}
}
Loading