Skip to content

Sync final copy of local AR from MM #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
netmindz wants to merge 1 commit into
base: main
Choose a base branch
from
+101 −42
Open

Sync final copy of local AR from MM #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 91 additions & 35 deletions audio_reactive.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,9 @@ static bool audioSyncSequence = true; // if true, the receiv
static uint8_t audioSyncPurge = 1; // 0: process each packet (don't purge); 1: auto-purge old packets; 2: only process latest received packet (always purge)
static bool audioSyncBroadcast = false; // if true, use subnet broadcast for sending instead of multicast; receivers on multicast sockets will still receive broadcast packets
static bool udpSyncConnected = false; // UDP connection status -> true if connected to UDP sync
static volatile bool isOOM = false; // FFTask: not enough memory for buffers (audio processing failed to start)

#define NUM_GEQ_CHANNELS 16 // number of frequency channels. Don't change !!
#define NUM_GEQ_CHANNELS 16 // number of frequency channels. Don't change !!

// audioreactive variables
#ifdef ARDUINO_ARCH_ESP32
Expand Down Expand Up @@ -257,8 +258,8 @@ const float agcSampleSmooth[AGC_NUM_PRESETS] = { 1/12.f, 1/6.f, 1/16.f}; //
static AudioSource *audioSource = nullptr;
static uint8_t useInputFilter = 0; // enables low-cut filtering. Applies before FFT.

//WLEDMM add experimental settings
static uint8_t micLevelMethod = 0; // 0=old "floating" miclev, 1=new "freeze" mode, 2=fast freeze mode (mode 2 may not work for you)
//WLEDMM experimental settings
static uint8_t micLevelMethod = 1; // 0=old "floating" miclev, 1=new "freeze" mode, 2=fast freeze mode (mode 2 may not work for you)
#if defined(CONFIG_IDF_TARGET_ESP32S2) || defined(CONFIG_IDF_TARGET_ESP32C3)
static constexpr uint8_t averageByRMS = false; // false: use mean value, true: use RMS (root mean squared). use simpler method on slower MCUs.
#else
Expand Down Expand Up @@ -459,13 +460,13 @@ static bool alocateFFTBuffers(void) {
USER_PRINT(F("\nFree heap ")); USER_PRINTLN(ESP.getFreeHeap());
#endif

if (vReal) free(vReal); // should not happen
if (vImag) free(vImag); // should not happen
if ((vReal = (float*) calloc(samplesFFT, sizeof(float))) == nullptr) return false; // calloc or die
if ((vImag = (float*) calloc(samplesFFT, sizeof(float))) == nullptr) return false;
if (vReal) d_free(vReal); vReal = nullptr; // should not happen
if (vImag) d_free(vImag); vImag = nullptr; // should not happen
if ((vReal = (float*) d_calloc(samplesFFT, sizeof(float))) == nullptr) return false; // calloc or die
if ((vImag = (float*) d_calloc(samplesFFT, sizeof(float))) == nullptr) return false;
#ifdef FFT_MAJORPEAK_HUMAN_EAR
if (pinkFactors) free(pinkFactors);
if ((pinkFactors = (float*) calloc(samplesFFT, sizeof(float))) == nullptr) return false;
if (pinkFactors) p_free(pinkFactors);
if ((pinkFactors = (float*) p_calloc(samplesFFT, sizeof(float))) == nullptr) return false;
#endif

#ifdef SR_DEBUG
Expand All @@ -477,6 +478,27 @@ static bool alocateFFTBuffers(void) {
return(true); // success
}

// de-allocate FFT sample buffers from heap
static void destroyFFTBuffers(bool panicOOM) {
#ifdef FFT_MAJORPEAK_HUMAN_EAR
if (pinkFactors) p_free(pinkFactors); pinkFactors = nullptr;
#endif
if (vImag) d_free(vImag); vImag = nullptr;
if (vReal) d_free(vReal); vReal = nullptr;

if (panicOOM && !isOOM) { // notify user
isOOM = true;
errorFlag = ERR_LOW_MEM;
USER_PRINTLN("AR startup failed - out of memory!");
}
#ifdef SR_DEBUG
USER_PRINTLN("\ndestroyFFTBuffers() completed successfully.");
USER_PRINT(F("Free heap: ")); USER_PRINTLN(ESP.getFreeHeap());
USER_FLUSH();
#endif
}


// High-Pass "DC blocker" filter
// see https://www.dsprelated.com/freebooks/filters/DC_Blocker.html
static void runDCBlocker(uint_fast16_t numSamples, float *sampleBuffer) {
Expand All @@ -493,6 +515,22 @@ static void runDCBlocker(uint_fast16_t numSamples, float *sampleBuffer) {
}
}

//
// FFT runner - "return" from a task function causes crash. This wrapper keeps the task alive
//
void runFFTcode(void * parameter) __attribute__((noreturn,used));
void runFFTcode(void * parameter) {
bool firstFail = true; // prevents flood of warnings
do {
if (!disableSoundProcessing) {
FFTcode(parameter);
if (firstFail) {USER_PRINTLN(F("warning: unexpected exit of FFT main task."));}
firstFail = false;
} else firstFail = true; // re-enable warning message
vTaskDelay(1000); // if we arrive here, FFcode has returned due to OOM. Wait a bit, then try again.
} while (true);
}

//
// FFT main task
//
Expand All @@ -516,13 +554,18 @@ void FFTcode(void * parameter)
static float* oldSamples = nullptr; // previous 50% of samples
static bool haveOldSamples = false; // for sliding window FFT
bool usingOldSamples = false;
if (!oldSamples) oldSamples = (float*) calloc(samplesFFT_2, sizeof(float)); // allocate on first run
if (!oldSamples) { disableSoundProcessing = true; return; } // no memory -> die
if (!oldSamples) oldSamples = (float*) d_calloc(samplesFFT_2, sizeof(float)); // allocate on first run
if (!oldSamples) { disableSoundProcessing = true; haveOldSamples = false; destroyFFTBuffers(true); return; } // no memory -> die
#endif

bool success = true;
if ((vReal == nullptr) || (vImag == nullptr)) success = alocateFFTBuffers(); // allocate sample buffers on first run
if (success == false) { disableSoundProcessing = true; return; } // no memory -> die
if (success == false) {
// no memory -> clean up heap, then suspend
disableSoundProcessing = true;
destroyFFTBuffers(true);
return;
}

// create FFT object - we have to do if after allocating buffers
#if defined(FFT_LIB_REV) && FFT_LIB_REV > 0x19
Expand All @@ -532,7 +575,8 @@ void FFTcode(void * parameter)
// recommended version optimized by @softhack007 (API version 1.9)
#if defined(WLED_ENABLE_HUB75MATRIX) && defined(CONFIG_IDF_TARGET_ESP32)
static float* windowWeighingFactors = nullptr;
if (!windowWeighingFactors) windowWeighingFactors = (float*) calloc(samplesFFT, sizeof(float)); // cache for FFT windowing factors - use heap
if (!windowWeighingFactors) windowWeighingFactors = (float*) d_calloc(samplesFFT, sizeof(float)); // cache for FFT windowing factors - use heap
if (!windowWeighingFactors) { disableSoundProcessing = true; haveOldSamples = false; destroyFFTBuffers(true); return; } // alloc failed
#else
static float windowWeighingFactors[samplesFFT] = {0.0f}; // cache for FFT windowing factors - use global RAM
#endif
Expand Down Expand Up @@ -1874,7 +1918,11 @@ class AudioReactive : public Usermod {
} catch (...) {
packetSize = 0;
#ifdef ARDUINO_ARCH_ESP32
#if ESP_IDF_VERSION_MAJOR < 5
fftUdp.flush();
#else
fftUdp.clear();
#endif
#endif
DEBUG_PRINTLN(F("receiveAudioData: parsePacket out of memory exception caught!"));
USER_FLUSH();
Expand All @@ -1886,7 +1934,11 @@ class AudioReactive : public Usermod {

#ifdef ARDUINO_ARCH_ESP32
if ((packetSize > 0) && ((packetSize < 5) || (packetSize > UDPSOUND_MAX_PACKET))) {
#if ESP_IDF_VERSION_MAJOR < 5
fftUdp.flush();
#else
fftUdp.clear();
#endif
continue; // Skip invalid packets -> next iteration
}
#endif
Expand Down Expand Up @@ -1915,7 +1967,7 @@ class AudioReactive : public Usermod {
} while ((packetSize > 0) && ((packetsReceived < maxSamples) || (maxSamples == AR_UDP_FLUSH_ALL))); // repeat until we have read enough packets, or no more packets available

#if defined(WLED_DEBUG) || defined(SR_DEBUG)
if ((packetsReceived > 1) && haveFreshData) {DEBUGSR_PRINTF("AR UDP: dropped %d packets [%ums]\t%d maxDrop.\n", packetsReceived-1, millis() - last_UDPTime, maxSamples-1);} // for debugging
if ((packetsReceived > 1) && haveFreshData) {DEBUGSR_PRINTF("AR UDP: dropped %d packets [%lums]\t%d maxDrop.\n", packetsReceived-1, millis() - last_UDPTime, maxSamples-1);} // for debugging
#endif
return haveFreshData;
}
Expand All @@ -1935,6 +1987,7 @@ class AudioReactive : public Usermod {
void setup() override
{
disableSoundProcessing = true; // just to be sure
isOOM = false;
if (!initDone) {
// usermod exchangeable data
// we will assign all usermod exportable data here as pointers to original variables or arrays and allocate memory for pointers
Expand Down Expand Up @@ -2034,7 +2087,7 @@ class AudioReactive : public Usermod {
break;
#if !defined(CONFIG_IDF_TARGET_ESP32S2) && !defined(CONFIG_IDF_TARGET_ESP32C3)
case 5:
DEBUGSR_PRINT(F("AR: I2S PDM Microphone - ")); DEBUGSR_PRINTLN(F(I2S_PDM_MIC_CHANNEL_TEXT));
DEBUGSR_PRINT(F("AR: Generic PDM Microphone - ")); DEBUGSR_PRINTLN(F(I2S_PDM_MIC_CHANNEL_TEXT));
audioSource = new I2SSource(SAMPLE_RATE, BLOCK_SIZE, 1.0f/4.0f);
useInputFilter = 1; // PDM bandpass filter - this reduces the noise floor on SPM1423 from 5% Vpp (~380) down to 0.05% Vpp (~5)
delay(100);
Expand Down Expand Up @@ -2248,14 +2301,9 @@ class AudioReactive : public Usermod {
useNetworkAudio = true; // don't fall back to local audio in standard "receive mode"
}

// suspend local sound processing when "real time mode" is active (E131, UDP, ADALIGHT, ARTNET)
if ( (realtimeOverride == REALTIME_OVERRIDE_NONE) // please add other overrides here if needed
&&( (realtimeMode == REALTIME_MODE_GENERIC)
||(realtimeMode == REALTIME_MODE_E131)
||(realtimeMode == REALTIME_MODE_UDP)
||(realtimeMode == REALTIME_MODE_ADALIGHT)
||(realtimeMode == REALTIME_MODE_ARTNET) ) ) // please add other modes here if needed
{
// suspend local sound processing when "real time mode" is active (E131, UDP, ADALIGHT, ARTNET, DDP, DMX)
// exception: sound input is still needed when useMainSegmentOnly - other segments are still running with local input.
if ((realtimeMode != REALTIME_MODE_INACTIVE) && (realtimeOverride == REALTIME_OVERRIDE_NONE) && !useMainSegmentOnly) {
#ifdef WLED_DEBUG
if ((disableSoundProcessing == false) && (audioSyncEnabled < AUDIOSYNC_REC)) { // we just switched to "disabled"
DEBUG_PRINTLN("[AR userLoop] realtime mode active - audio processing suspended.");
Expand Down Expand Up @@ -2384,7 +2432,11 @@ class AudioReactive : public Usermod {
lastTime = millis();
} else {
#ifdef ARDUINO_ARCH_ESP32
#if ESP_IDF_VERSION_MAJOR < 5
fftUdp.flush(); // WLEDMM: Flush this if we haven't read it. Does not work on 8266.
#else
fftUdp.clear();
#endif
#endif
}
if (useNetworkAudio) {
Expand Down Expand Up @@ -2511,11 +2563,12 @@ class AudioReactive : public Usermod {
vTaskResume(FFT_Task);
connected(); // resume UDP
} else {
isOOM = false;
if (audioSource) // WLEDMM only create FFT task if we have a valid audio source
// xTaskCreatePinnedToCore(
// xTaskCreate( // no need to "pin" this task to core #0
xTaskCreateUniversal(
FFTcode, // Function to implement the task
runFFTcode, // Function to implement the task
"FFT", // Name of the task
3592, // Stack size in words // 3592 leaves 800-1024 bytes of task stack free
NULL, // Task input parameter
Expand Down Expand Up @@ -2664,15 +2717,17 @@ class AudioReactive : public Usermod {
#else // ESP32 only
} else {
// Analog or I2S digital input
if (audioSource && (audioSource->isInitialized())) {
if (audioSource && (audioSource->isInitialized()) && !isOOM) {
// audio source successfully configured
if (audioSource->getType() == AudioSource::Type_I2SAdc) {
infoArr.add(F("ADC analog"));
} else {
if (dmType != 51)
infoArr.add(F("I2S digital"));
if (dmType != 51) {
if (dmType == 5) infoArr.add(F("PDM digital"));
else infoArr.add(F("I2S digital"));
}
else
infoArr.add(F("legacy I2S PDM"));
infoArr.add(F("legacy PDM"));
}
// input level or "silence"
if (maxSample5sec > 1.0) {
Expand All @@ -2685,7 +2740,8 @@ class AudioReactive : public Usermod {
} else {
// error during audio source setup
infoArr.add(F("not initialized"));
if (dmType < 254) infoArr.add(F(" - check pin settings"));
if (isOOM) infoArr.add(F(" - out of memory"));
else if (dmType < 254) infoArr.add(F(" - check pin settings"));
}
}

Expand Down Expand Up @@ -3048,14 +3104,14 @@ class AudioReactive : public Usermod {
#endif
#if !defined(CONFIG_IDF_TARGET_ESP32S2) && !defined(CONFIG_IDF_TARGET_ESP32C3)
#if SR_DMTYPE==5
oappend(SET_F("addOption(dd,'Generic I2S PDM (⎌)',5);"));
oappend(SET_F("addOption(dd,'Generic PDM (⎌)',5);"));
#else
oappend(SET_F("addOption(dd,'Generic I2S PDM',5);"));
oappend(SET_F("addOption(dd,'Generic PDM',5);"));
#endif
#if SR_DMTYPE==51
oappend(SET_F("addOption(dd,'.Legacy I2S PDM ☾ (⎌)',51);"));
oappend(SET_F("addOption(dd,'.Legacy PDM ☾ (⎌)',51);"));
#else
oappend(SET_F("addOption(dd,'.Legacy I2S PDM ☾',51);"));
oappend(SET_F("addOption(dd,'.Legacy PDM ☾',51);"));
#endif
#endif
#if SR_DMTYPE==6
Expand Down Expand Up @@ -3094,8 +3150,8 @@ class AudioReactive : public Usermod {
//WLEDMM: experimental settings
oappend(SET_F("xx='experiments';")); // shortcut
oappend(SET_F("dd=addDropdown(ux,xx+':micLev');"));
oappend(SET_F("addOption(dd,'Floating (⎌)',0);"));
oappend(SET_F("addOption(dd,'Freeze',1);"));
oappend(SET_F("addOption(dd,'Floating',0);"));
oappend(SET_F("addOption(dd,'Freeze (⎌)',1);"));
oappend(SET_F("addOption(dd,'Fast Freeze',2);"));
oappend(SET_F("addInfo(ux+':'+xx+':micLev',1,'☾');"));

Expand Down
17 changes: 10 additions & 7 deletions audio_source.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,15 @@ constexpr i2s_port_t AR_I2S_PORT = I2S_NUM_0; // I2S port to use (do not c
// data type requested from the I2S driver - currently we always use 32bit
//#define I2S_USE_16BIT_SAMPLES // (experimental) define this to request 16bit - more efficient but possibly less compatible

#if defined(WLED_ENABLE_HUB75MATRIX) && defined(CONFIG_IDF_TARGET_ESP32)
// this is bitter, but necessary to survive
#define I2S_USE_16BIT_SAMPLES
#endif
//#if defined(WLED_ENABLE_HUB75MATRIX) && defined(CONFIG_IDF_TARGET_ESP32)
// // this is bitter, but necessary to survive // disabled - does not work!!
// #define I2S_USE_16BIT_SAMPLES
//#endif

#ifdef I2S_USE_16BIT_SAMPLES
#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(4, 2, 0)) // see https://github.com/MoonModules/WLED-MM/issues/333#issuecomment-3910779668
#error "16bit sampling currently does not work with newer framework versions. Please build without I2S_USE_16BIT_SAMPLES"
#endif
#define I2S_SAMPLE_RESOLUTION I2S_BITS_PER_SAMPLE_16BIT
#define I2S_datatype int16_t
#define I2S_unsigned_datatype uint16_t
Expand Down Expand Up @@ -332,11 +335,11 @@ class I2SSource : public AudioSource {
// S3: not supported; S2: supported; C3: not supported
_config.use_apll = false; // APLL not supported on this MCU
#endif
#if defined(ARDUINO_ARCH_ESP32) && !defined(CONFIG_IDF_TARGET_ESP32S3) && !defined(CONFIG_IDF_TARGET_ESP32S2) && !defined(CONFIG_IDF_TARGET_ESP32C3)
#if defined(CONFIG_IDF_TARGET_ESP32)
if (ESP.getChipRevision() == 0) _config.use_apll = false; // APLL is broken on ESP32 revision 0
#endif
#if defined(WLED_ENABLE_HUB75MATRIX)
_config.use_apll = false; // APLL needed for HUB75 DMA driver ?
#if defined(WLED_USE_ETHERNET) || defined(WLED_ENABLE_HUB75MATRIX)
_config.use_apll = false; // APLL is needed for Ethernet, and possibly for HUB75, too
#endif
#endif

Expand Down