Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions scripts/host-testbench.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
# stop on most errors
set -e

# Prevent snap or desktop environment from breaking octave
unset GTK_PATH

function filesize() {
du -b "$1" | awk '{print $1}'
}
Expand Down
20 changes: 19 additions & 1 deletion src/audio/mfcc/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,10 +1,28 @@
# SPDX-License-Identifier: BSD-3-Clause

if(CONFIG_COMP_MFCC_PCAN)
if(NOT DEFINED TFLM_PATH)
if(DEFINED SOF_ROOT_SOURCE_DIRECTORY AND EXISTS "${SOF_ROOT_SOURCE_DIRECTORY}/../tflite-micro")
set(TFLM_PATH "${SOF_ROOT_SOURCE_DIRECTORY}/../tflite-micro")
elseif(DEFINED sof_top_dir AND EXISTS "${sof_top_dir}/../tflite-micro")
set(TFLM_PATH "${sof_top_dir}/../tflite-micro")
elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/../../../../tflite-micro")
set(TFLM_PATH "${CMAKE_CURRENT_LIST_DIR}/../../../../tflite-micro")
endif()
endif()
is_zephyr(zephyr)
if(zephyr)
zephyr_include_directories(${TFLM_PATH})
else()
target_include_directories(sof PRIVATE ${TFLM_PATH})
endif()
endif()

if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
add_subdirectory(llext ${PROJECT_BINARY_DIR}/mfcc_llext)
add_dependencies(app mfcc)
else()
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c mfcc_vad.c)
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c mfcc_vad.c mfcc_agc8.c)
if(CONFIG_IPC_MAJOR_4)
add_local_sources(sof mfcc_ipc4.c)
endif()
Expand Down
23 changes: 23 additions & 0 deletions src/audio/mfcc/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,26 @@ config COMP_MFCC
The characteristic of the audio features are defined in the binary
control blob. Directory tools/tune/mfcc contains a tool to create
the configurations.

if COMP_MFCC

config COMP_MFCC_PCAN
bool "Enable PCAN (Per-Channel AGC Normalization) in MFCC"
default n
select MATH_PCAN
help
Select to build MFCC with PCAN (Per-Channel Automatic Gain Control
Normalization) support on the Mel filterbank energies. PCAN performs
dynamic per-channel gain control and piecewise polynomial root
compression to suppress stationary noise and enhance transients,
producing normalized features suitable for direct int8 quantization
downstream (e.g. microWakeWord). The topology PCAN configuration
fields are ignored when this option is disabled.

config COMP_MFCC_DEBUG_TRACE
bool "Verbose per-process debug traces in MFCC"
default n
help
Enable MFCC per-process debug prints in the hot path.

endif # COMP_MFCC
104 changes: 89 additions & 15 deletions src/audio/mfcc/README.md
Original file line number Diff line number Diff line change
@@ -1,25 +1,99 @@
# MFCC Feature Extraction Architecture
# MFCC & PCAN Feature Extraction Architecture

This directory contains the Mel-Frequency Cepstral Coefficients (MFCC) feature extractor.
This directory contains the **Mel-Frequency Cepstral Coefficients (MFCC)** feature extractor and integrated **PCAN (Per-Channel AGC Normalization)** audio pre-processing component for Sound Open Firmware (SOF).

## Overview
---

MFCC extracts audio features commonly used as inputs for machine learning models, such as wake-word detection or speech recognition.
## 1. Overview

## Architecture Diagram
The MFCC module converts raw streaming time-domain PCM audio into compact acoustic feature representations (e.g. 13-bin cepstral coefficients, 40-bin or 80-bin Mel spectrograms) suitable for on-device machine learning models such as wake-word classifiers (TFLM, microWakeWord) and speech recognition engines (Whisper).

When enabled, **PCAN** applies per-channel adaptive dynamic gain control and piecewise root dynamic range compression across time to suppress stationary background noise and normalize channel energy before feature quantization.

### Output modes

One build supports all output modes, selected at runtime by the configuration blob:

| Mode | Blob settings | Output per hop |
|---|---|---|
| Cepstral coefficients | `num_ceps > 0` | int16 Q9.7 widened to int32 |
| Mel log spectra | `num_ceps == 0` | int32 Q9.23 |
| PCAN Mel | `num_ceps == 0`, `enable_pcan` (needs `CONFIG_COMP_MFCC_PCAN`) | int8 |
| Soft AGC Mel (AGC8) | `num_ceps == 0`, `enable_agc8` | int8 Q1.7 |

AGC8 is a light alternative to PCAN without the per-channel noise estimate and the `tflite-micro` dependency. A single gain is added to all Q9.23 Mel log values of a hop. The gain attacks instantly to keep the hop peak at or below +1.0, and releases towards a target with a slower step when the VAD reports speech. The values are clamped to -1.0 .. +1.0 and converted to int8 Q1.7. The blob fields `agc_target`, `agc_floor` (Q8.7) and `agc_release`, `agc_release_speech` (Q9.23 per hop) set the parameters, zero selects the default.

---

## 2. Architecture & Data Flow

```mermaid
graph LR
In[Audio Frame] --> Win[Windowing]
Win --> FFT[FFT]
FFT --> Mel[Mel Filterbank]
Mel --> DCT[DCT]
DCT --> Out[MFCC Output Features]
In[Audio PCM Frame] --> Win[Windowing: Hamming/Hann]
Win --> FFT[FFT: 512-pt]
FFT --> Mel[Mel Filterbank: 40/80 Bins]
Mel --> PCAN[PCAN Gain Normalization]
PCAN --> Log[Log Scaling / dB]
Log --> VAD[VAD / DTX Silence Gating]
Log --> AGC8[Soft AGC int8]
VAD --> AGC8
Log --> DCT[DCT: Cepstral Matrix]
DCT --> Lifter[Cepstral Lifter]
Lifter --> Out[MFCC Output Features]
```

---

## 3. Dependencies

| Dependency | Purpose |
|---|---|
| **West Manifest (`west.yml`)** | Manages workspace dependencies including `tflite-micro` |
| **`tflite-micro`** | Provides Google's upstream Apache-2.0 `microfrontend` PCAN library (`pcan_gain_control.c`, `noise_reduction.c`) |
| **GNU Octave ($\ge 5.0$)** | Required for topology configuration generation (`tune/setup_mfcc.m`) and reference vector export |
| **CMake ($\ge 3.13$)** | SOF build system |
| **Zephyr SDK / Xtensa XCC** | DSP cross-compilation toolchains for Cadence HiFi3 / HiFi4 SIMD kernels |
| **CMocka** | Host unit testing framework |

---

## 4. Build & Tuning Instructions

### 4.1 Synchronize Dependencies via West
```bash
west update
```

### 4.2 Build Host Unit Tests
```bash
# Configure unit tests
cmake -S sof -B build_ut -DBUILD_UNIT_TESTS=ON -DBUILD_UNIT_TESTS_HOST=ON -DINIT_CONFIG=unit_test_defconfig

# Build all math and MFCC tests
cmake --build build_ut --target pcan
ctest --test-dir build_ut -R "pcan|auditory|dct|matrix|window|fft" --output-on-failure
```

## Configuration and Scripts
### 4.3 Build Target DSP Firmware
```bash
west build -b intel_adsp_ace15_mtpm app
```

### 4.4 Generate Topology Configurations (Octave)
The tuning script generates binary configuration blobs and ALSA topology configurations:
```bash
cd src/audio/mfcc/tune
octave-cli --eval "setup_mfcc"
```
Available presets generated:
- `ceps13_compress_dtx.conf`: 13-bin cepstral features with VAD and DTX silence suppression.
- `mel80_compress.conf`: 80-bin linear Mel spectrogram features.
- `mel80_compress_dtx.conf`: 80-bin Mel spectrogram with DTX gating.
- `mel80_pcan_compress.conf`: 80-bin Mel spectrogram with Google PCAN dynamic AGC normalization.

---

## 5. Configuration Options

- **Kconfig**: Enables the MFCC component (`COMP_MFCC`) which selects necessary math libraries (`MATH_FFT`, `MATH_DCT`, `MATH_16BIT_MEL_FILTERBANK`, etc.). Depends on `COMP_MODULE_ADAPTER`.
- **CMakeLists.txt**: Compiles generic, common, and HIFI implementations (`mfcc_hifi3.c`, `mfcc_hifi4.c`). Provides support for Zephyr loadable extensions (`llext`).
- **mfcc.toml**: Specifies the topology configuration for the MFCC module (UUID, affinity, memory parameters, and pin formats).
- **Topology (.conf)**: Derived from `tools/topology/topology2/include/components/mfcc.conf`, configuring a `mfcc` widget object of type `effect` with UUID `73:a7:10:db:a4:1a:ea:4c:a2:1f:2d:57:a5:c9:82:eb`.
- **`CONFIG_COMP_MFCC`**: Enables the MFCC component and automatically selects required math libraries (`CONFIG_MATH_FFT`, `CONFIG_MATH_AUDITORY`, `CONFIG_MATH_DCT`, `CONFIG_MATH_PCAN`, `CONFIG_MATH_WINDOW`).
- **`CONFIG_COMP_MODULE_ADAPTER`**: Module adapter infrastructure for SOF processing components.
1 change: 1 addition & 0 deletions src/audio/mfcc/llext/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ sof_llext_build("mfcc"
../mfcc_generic.c
../mfcc_hifi3.c
../mfcc_hifi4.c
../mfcc_agc8.c
LIB openmodules
)
72 changes: 66 additions & 6 deletions src/audio/mfcc/mfcc.c
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,19 @@
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#if CONFIG_COMP_MFCC_DEBUG_TRACE
#include <zephyr/kernel.h>
#endif

LOG_MODULE_REGISTER(mfcc, CONFIG_SOF_LOG_LEVEL);

SOF_DEFINE_REG_UUID(mfcc);

#if CONFIG_COMP_MFCC_DEBUG_TRACE
static uint32_t last_mfcc_cycle;
static uint32_t mfcc_call_count;
#endif

/** \brief Source/sink API based source copy function map. */
struct mfcc_source_func_map {
uint8_t source_fmt;
Expand Down Expand Up @@ -161,6 +169,13 @@ static int mfcc_process(struct processing_module *mod,
size_t source_avail;
int frames;
int num_ceps;
#if CONFIG_COMP_MFCC_DEBUG_TRACE
uint32_t now = k_cycle_get_32();
uint32_t delta_cycles = now - last_mfcc_cycle;

last_mfcc_cycle = now;
mfcc_call_count++;
#endif

comp_dbg(dev, "start");

Expand All @@ -171,23 +186,47 @@ static int mfcc_process(struct processing_module *mod,
* can continue while the previous period is drained.
*/
pending = state->header_pending || state->out_remain > 0;
if (cd->config->compress_output && pending)
return mfcc_process_output(mod, cd, sources, sinks, 0, 0);
if (cd->config->compress_output && pending) {
int ret = mfcc_process_output(mod, cd, sources, sinks, 0, 0);

#if CONFIG_COMP_MFCC_DEBUG_TRACE
comp_info(dev, "[MFCC proc %u] delta=%u us (pending retry, ret=%d)",
mfcc_call_count, k_cyc_to_us_near32(delta_cycles), ret);
#endif
return ret;
}

source_avail = source_get_data_frames_available(sources[0]);
frames = MIN(source_avail, cd->max_frames);
if (!frames)
if (!frames) {
#if CONFIG_COMP_MFCC_DEBUG_TRACE
comp_info(dev, "[MFCC proc %u] delta=%u us (no frames, avail=%zu)",
mfcc_call_count, k_cyc_to_us_near32(delta_cycles), source_avail);
#endif
return 0;
}

/* Copy input audio from source to MFCC internal circular buffer */
cd->source_func(sources[0], &state->buf, &state->emph, frames, state->source_channel);

/* Run STFT and Mel/DCT processing */
num_ceps = mfcc_stft_process(mod, cd);
if (num_ceps < 0)
if (num_ceps < 0) {
#if CONFIG_COMP_MFCC_DEBUG_TRACE
comp_info(dev, "[MFCC proc %u] delta=%u us stft error %d",
mfcc_call_count, k_cyc_to_us_near32(delta_cycles), num_ceps);
#endif
return num_ceps;
}

int ret = mfcc_process_output(mod, cd, sources, sinks, num_ceps, frames);

return mfcc_process_output(mod, cd, sources, sinks, num_ceps, frames);
#if CONFIG_COMP_MFCC_DEBUG_TRACE
comp_info(dev, "[MFCC proc %u] delta=%u us avail=%zu frames=%d ceps=%d ret=%d",
mfcc_call_count, k_cyc_to_us_near32(delta_cycles),
source_avail, frames, num_ceps, ret);
#endif
return ret;
}

/**
Expand Down Expand Up @@ -243,12 +282,29 @@ static int mfcc_prepare(struct processing_module *mod,
* in the processing code, so require it to be at least that large.
*/
if (cd->config && data_size >= sizeof(struct sof_mfcc_config)) {
ret = mfcc_setup(mod, dev->frames + 4, audio_stream_get_rate(&sourceb->stream),
uint32_t src_rate = audio_stream_get_rate(&sourceb->stream);
int max_frames;

if (dev->frames < cd->config->frame_shift)
dev->frames = cd->config->frame_shift;

max_frames = dev->frames + 4;

ret = mfcc_setup(mod, max_frames, src_rate,
audio_stream_get_channels(&sourceb->stream));
if (ret < 0) {
comp_err(dev, "setup failed.");
return ret;
}

/* Set DP scheduling period from FFT hop cadence; sink rate is
* not yet propagated at prepare time so module_adapter cannot
* derive it from mod->sinks[].
*/
if (src_rate && cd->state.fft.fft_hop_size)
dev->period = (uint32_t)(1000000ULL *
cd->state.fft.fft_hop_size /
src_rate);
} else {
comp_err(dev, "configuration is missing.");
return -EINVAL;
Expand Down Expand Up @@ -308,6 +364,10 @@ static int mfcc_reset(struct processing_module *mod)

/* Reset to similar state as init() */
cd->source_func = NULL;
#if CONFIG_COMP_MFCC_DEBUG_TRACE
last_mfcc_cycle = 0;
mfcc_call_count = 0;
#endif
return 0;
}

Expand Down
71 changes: 71 additions & 0 deletions src/audio/mfcc/mfcc_agc8.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// SPDX-License-Identifier: BSD-3-Clause
//
// Copyright(c) 2026 Intel Corporation.
//
// Author: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>

/**
* \file mfcc_agc8.c
* \brief Soft AGC and int8 quantization of MFCC Mel log spectra.
*/

#include <sof/audio/mfcc/mfcc_agc8.h>
#include <sof/audio/format.h>
#include <user/mfcc.h>
#include <stdint.h>

#define MFCC_AGC8_CLIP_MAX_Q23 (1 << 23)
#define MFCC_AGC8_CLIP_MIN_Q23 (-(1 << 23))

void mfcc_agc8_init(struct mfcc_agc8 *agc, const struct sof_mfcc_config *config)
{
/* Blob target and floor are Q8.7, release steps are Q9.23 */
agc->target = config->agc_target ? (int32_t)config->agc_target << 16 :
MFCC_AGC8_TARGET_Q23;
agc->floor = config->agc_floor ? (int32_t)config->agc_floor << 16 :
MFCC_AGC8_FLOOR_Q23;
agc->release = config->agc_release ? config->agc_release :
MFCC_AGC8_RELEASE_Q23;
agc->release_speech = config->agc_release_speech ? config->agc_release_speech :
MFCC_AGC8_RELEASE_SPEECH_Q23;
agc->gain = agc->target;
}

void mfcc_agc8_process(struct mfcc_agc8 *agc, const int32_t *mel, int8_t *out, int n,
bool speech)
{
int32_t peak = mel[0];
int32_t gain;
int32_t v;
int i;

for (i = 1; i < n; i++) {
if (mel[i] > peak)
peak = mel[i];
}

/* Attack: instantly keep the hop peak at or below +1.0 */
if (agc->gain > MFCC_AGC8_CLIP_MAX_Q23 - peak)
agc->gain = MFCC_AGC8_CLIP_MAX_Q23 - peak;
if (agc->gain < agc->floor)
agc->gain = agc->floor;

gain = agc->gain;
for (i = 0; i < n; i++) {
v = mel[i] + gain;
if (v > MFCC_AGC8_CLIP_MAX_Q23)
v = MFCC_AGC8_CLIP_MAX_Q23;
else if (v < MFCC_AGC8_CLIP_MIN_Q23)
v = MFCC_AGC8_CLIP_MIN_Q23;

v = Q_SHIFT_RND(v, 23, 7);
out[i] = (int8_t)(v > INT8_MAX ? INT8_MAX : v);
}

/* Release towards target, with a slower step during speech */
if (agc->gain < agc->target) {
agc->gain += speech ? agc->release_speech : agc->release;
if (agc->gain > agc->target)
agc->gain = agc->target;
}
}
Loading
Loading