WiP -Audio fastpath

package.json

@@ -1,6 +1,6 @@
 {
   "name": "wled",
-  "version": "0.14.0-b15.27",
+  "version": "0.14.0-b15.28",
   "description": "Tools for WLED project",
   "main": "tools/cdata.js",
   "directories": {

platformio.ini

@@ -216,7 +216,8 @@ upload_speed = 115200
 # ------------------------------------------------------------------------------
 lib_compat_mode = strict
 lib_deps =
-    fastled/FastLED @ 3.5.0
+    fastled/FastLED @ 3.6.0 ;; 3.6.0 was release recently
+    ;https://github.com/FastLED/FastLED.git#master @3.6.0+sha.23c67b7 ;; up to 50% faster - using a "known good" hash, so we get predictable builds
     IRremoteESP8266 @ 2.8.2
     ;;makuna/NeoPixelBus @ 2.7.5 ;; WLEDMM will be added in board specific sections
     ;;https://github.com/Aircoookie/ESPAsyncWebServer.git @ ~2.0.7
@@ -886,11 +887,14 @@ build_flags_S =
   -Wno-attributes -Wno-unused-variable -Wno-unused-function -Wno-deprecated-declarations ;disables some stupid warnings
   ;-D WLED_DISABLE_BROWNOUT_DET ; enable if you get "brownout detected" errors at startup 
   -D WLED_USE_MY_CONFIG
+  ; -D WLED_DISABLE_2D ;; un-comment to build a firmware without 2D matrix support
   ; -D WLED_USE_CIE_BRIGHTNESS_TABLE ;; experimental: use different color / brightness lookup table
   -D USERMOD_AUDIOREACTIVE
   -D UM_AUDIOREACTIVE_USE_NEW_FFT ; use latest (upstream) FFTLib, instead of older library modified by blazoncek. Slightly faster, more accurate, needs 2KB RAM extra   
   -D USERMOD_ARTIFX ; WLEDMM usermod
-  ; -D WLEDMM_PROTECT_SERVICE ;; WLEDMM experimental feature to prevent crashes when effects are drawing while async_tcp tries to modify segment or strip objects.
+  -D WLEDMM_FASTPATH ; WLEDMM experimental option. Reduces audio lag (latency), and allows for faster LED framerates
+  -D WLEDMM_PROTECT_SERVICE ;; WLEDMM experimental feature to prevent crashes when effects are drawing while async_tcp tries to modify segment or strip objects.
+  ; -D WLED_DEBUG_HEAP ;; WLEDMM enable heap debugging
   ; -D WLED_DISABLE_LOXONE
   ; -D WLED_DISABLE_ALEXA
   ; -D WLED_DISABLE_HUESYNC
@@ -998,12 +1002,15 @@ lib_deps = ${esp32_4MB_M_base.lib_deps} ${common_mm.lib_deps_XL}
 board = esp32dev
 upload_speed = 460800 ; or 921600
 platform = ${esp32.platformV4}
-platform_packages = ${esp32.platformV4_packages}
+platform_packages = 
+  ${esp32.platformV4_packages}
+  toolchain-xtensa-esp32 @ 8.4.0+2021r2-patch5 ; align main tools with riscV tools
 build_unflags = ${common.build_unflags}
 build_flags = ${common.build_flags} ${esp32.build_flagsV4} ${common_mm.build_flags_S}
   -Wno-misleading-indentation -Wno-format-truncation 
   -Wshadow=compatible-local ;; emit warning in case a local variable "shadows" another local one
   ;-Wstack-usage=2732       ;; warn if a function needs more that 30% of availeable stack ("stack usage might be unbounded", "stack usage is 2824 bytes")
+  ;-Wsuggest-attribute=const -Wsuggest-attribute=pure ;;  ask compiler for hints on attributes
   -D WLED_ENABLE_DMX_INPUT
 lib_deps = ${esp32.lib_depsV4} ${common_mm.lib_deps_S}
   https://github.com/someweisguy/esp_dmx.git#v3.0.2-beta  ;; for DMX_INPUT
@@ -1050,6 +1057,7 @@ build_flags =
 [Debug_Flags]
 build_flags = 
   -DDEBUG=1 ;; enable some debug assertions in the platform core
+  ; -D WLED_DEBUG_HEAP ;; WLEDMM enable heap debugging
   -D WLED_DEBUG ; lots of generic debug messages
   -D SR_DEBUG ; some extra debug messages from audioreactive
   ; -D MIC_LOGGER ; for sound input monitoring & debugging (use arduino serial plotter)
@@ -1118,6 +1126,7 @@ build_unflags = ${common.build_unflags}
   -D CORE_DEBUG_LEVEL=0
 build_flags = ${esp32_4MB_M_base.build_flags} 
   ${Debug_Flags.build_flags}
+  ; -D WLED_DEBUG_HEAP ;; WLEDMM enable heap debugging (needs newer framework versions)
   -D CORE_DEBUG_LEVEL=2   ;; 2=warning
   -D WLED_RELEASE_NAME=esp32_16MB_M_debug
 monitor_filters = esp32_exception_decoder
@@ -1331,16 +1340,14 @@ build_flags = ${esp32_4MB_V4_M_base.build_flags}
   -D WLED_RELEASE_NAME=esp32_4MB_V4_M
   -D WLED_WATCHDOG_TIMEOUT=0 #-D WLED_DISABLE_BROWNOUT_DET
   -D ARDUINO_USB_CDC_ON_BOOT=0 ; needed for arduino-esp32 >=2.0.4; avoids errors on startup
-  ; RAM:   [==        ]  24.8% (used 81316 bytes from 327680 bytes)
-  ; Flash: [==========]  97.2% (used 1528253 bytes from 1572864 bytes)
   -D WLED_DISABLE_LOXONE ; FLASH 1272 bytes
   -D WLED_DISABLE_ALEXA ;  RAM 116 bytes; FLASH 13524 bytes
   -D WLED_DISABLE_HUESYNC ;RAM 122 bytes; FLASH  6308 bytes
-  ; RAM:   [==        ]  24.7% (used 81076 bytes from 327680 bytes)
-  ; Flash: [==========]  95.8% (used 1506893 bytes from 1572864 bytes)
   ;-D WLED_DISABLE_MQTT ;   RAM 216 bytes; FLASH 16496 bytes
   -D WLED_DISABLE_INFRARED ;RAM 136 bytes; FLASH 24492 bytes  ;; softhack007 disabled to stay below 100% flash size
 lib_ignore = IRremoteESP8266 ; use with WLED_DISABLE_INFRARED for faster compilation
+;; RAM:   [==        ]  24.7% (used 80884 bytes from 327680 bytes)
+;; Flash: [==========]  99.0% (used 1556429 bytes from 1572864 bytes)
 
 ; compiled with ESP-IDF 4.4.1
 [env:esp32_16MB_V4_M]
@@ -1362,6 +1369,7 @@ build_unflags = ${common.build_unflags}
 build_flags = ${esp32_4MB_V4_M_base.build_flags}
   ${Debug_Flags.build_flags}
   -D CORE_DEBUG_LEVEL=4  ;; 0=none, 1=error, 2=warning, 3=info, 4=debug, 5=verbose
+  -D WLED_DEBUG_HEAP ;; WLEDMM enable heap debugging
   -D WLED_RELEASE_NAME=esp32_16MB_V4_M_debug
   -D WLED_WATCHDOG_TIMEOUT=0 #-D WLED_DISABLE_BROWNOUT_DET
   -D ARDUINO_USB_CDC_ON_BOOT=0 ; needed for arduino-esp32 >=2.0.4; avoids errors on startup
@@ -1395,6 +1403,44 @@ lib_ignore = IRremoteESP8266 ; use with WLED_DISABLE_INFRARED for faster compila
 ;; RAM:   [==        ]  24.3% (used 79524 bytes from 327680 bytes)
 ;; Flash: [========= ]  93.2% (used 1466389 bytes from 1572864 bytes)
 
+;; similar to 4MB_PSRAM_S, but optimized for WROVER-E (chip revision >= 3) that doesn't need any workarounds for PSRAM any more
+;; tl;dr: its faster on PSRAM. But it will not work on all boards.
+[env:esp32_4MB_PSRAM_REV3_S]
+extends = esp32_4MB_V4_S_base
+;board = esp32cam
+board = lolin_d32_pro
+;board_build.partitions = tools/WLED_ESP32_4MB_256KB_FS.csv   ;; Alternative for 4MB flash:   1.8MB firmware, 256KB filesystem (esptool erase_flash needed before changing)
+build_unflags = ${esp32_4MB_V4_S_base.build_unflags}
+  -DARDUINO_EVENT_RUNNING_CORE=1  ;; we want to run wifi on core0, so remove the standard flag
+  -mfix-esp32-psram-cache-issue    ;; this fix is not needed any more for revision 3
+  -mfix-esp32-psram-cache-strategy=memw ;; same as above
+  ;;-Os ;; if you want to be even faster - use together with "-O2" in build_flags. Enable alternative default_partitions (above) to have more program space.
+
+build_flags = ${esp32_4MB_V4_S_base.build_flags}
+  -DARDUINO_EVENT_RUNNING_CORE=0  ;; assign Wifi to core0, to have more CPU on core#1 (arduino loop)
+  -DARDUINO_RUNNING_CORE=1        ;; should be default, but does not hurt
+  ;-DCONFIG_MBEDTLS_DYNAMIC_BUFFER=1 ;; optional - seems to move more buffers into PSRAM
+  ;; enable the next line together with "-Os" in build_unflags (unfortunately --> 104.0%% Flash)
+  ;;-O2 -fno-jump-tables -fno-tree-switch-conversion -Wwrite-strings -fstrict-volatile-bitfields
+  -DBOARD_HAS_PSRAM -D WLED_USE_PSRAM_JSON ;; -D WLED_USE_PSRAM ;; WLED_USE_PSRAM causes major slow-down (slow LEDs) on some ESP32 boards
+  ;;-D CONFIG_ESP32_REV_MIN=3  ;;  disables PSRAM bug workarounds in the core, reducing the code size and improving overall performance.
+  -D WLED_RELEASE_NAME=esp32_4MB_PSRAM_REV3_S
+  -D WLED_WATCHDOG_TIMEOUT=0 #-D WLED_DISABLE_BROWNOUT_DET
+  -D ARDUINO_USB_CDC_ON_BOOT=0 ; needed for arduino-esp32 >=2.0.4; avoids errors on startup
+  -D WLED_DISABLE_LOXONE ;  FLASH 1272 bytes
+  -D WLED_DISABLE_HUESYNC ; RAM 122 bytes; FLASH  6308 bytes
+  -D WLED_DISABLE_ALEXA ;   RAM 116 bytes; FLASH 13524 bytes
+  -D WLED_DISABLE_MQTT ;    RAM 216 bytes; FLASH 16496 bytes
+  -D WLED_DISABLE_INFRARED ;RAM 136 bytes; FLASH 24492 bytes ewowi: disabled to stay below 100%
+  ;-D WLED_DISABLE_ADALIGHT  ;; To disable serial protocols (see upstream #3128)
+  ; -D WLED_ENABLE_DMX
+  ; -D WLED_DEBUG
+  ; -D SR_DEBUG
+  ; -D MIC_LOGGER
+lib_ignore = IRremoteESP8266 ; use with WLED_DISABLE_INFRARED for faster compilation
+;; RAM:   [==        ]  16.8% (used 55124 bytes from 327680 bytes)
+;; Flash: [========= ]  92.7% (used 1458729 bytes from 1572864 bytes)
+
 ;; PSRAM build env that only leaves 300Kb for filesystem (instead of 1MB), but adds 300kB for program space
 [env:esp32_4MB_PSRAM_M]
 extends = esp32_4MB_V4_M_base

usermods/artifx/arti_wled.h

@@ -214,9 +214,11 @@ float ARTI::arti_external_function(uint8_t function, float par1, float par2, flo
       case F_millis:
         return millis();
 
+#ifndef WLED_DISABLE_2D
       case F_jsonToPixels:
         SEGMENT.jsonToPixels(SEGMENT.name,(uint8_t)par1);
         return floatNull;
+#endif
 
       default: {}
     }

usermods/audioreactive/audio_reactive.h

@@ -254,7 +254,11 @@ constexpr SRate_t SAMPLE_RATE = 22050;        // Base sample rate in Hz - 22Khz
 //constexpr SRate_t SAMPLE_RATE = 16000;        // 16kHz - use if FFTtask takes more than 20ms. Physical sample time -> 32ms
 //constexpr SRate_t SAMPLE_RATE = 20480;        // Base sample rate in Hz - 20Khz is experimental.    Physical sample time -> 25ms
 //constexpr SRate_t SAMPLE_RATE = 10240;        // Base sample rate in Hz - previous default.         Physical sample time -> 50ms
+#ifndef WLEDMM_FASTPATH
 #define FFT_MIN_CYCLE 21                      // minimum time before FFT task is repeated. Use with 22Khz sampling
+#else
+#define FFT_MIN_CYCLE 15                      // reduce min time, to allow faster catch-up when I2S is lagging 
+#endif
 //#define FFT_MIN_CYCLE 30                      // Use with 16Khz sampling
 //#define FFT_MIN_CYCLE 23                      // minimum time before FFT task is repeated. Use with 20Khz sampling
 //#define FFT_MIN_CYCLE 46                      // minimum time before FFT task is repeated. Use with 10Khz sampling
@@ -794,7 +798,7 @@ static void detectSamplePeak(void) {
 
 #if 0
   // alternate detection, based on FFT_MajorPeak and FFT_Magnitude. Not much better...
-  if ((binNum > 0)  && (maxVol > 8) && (binNum < 10) && (sampleAgc > 127) && 
+  if ((binNum > 1)  && (maxVol > 8) && (binNum < 10) && (sampleAgc > 127) && 
       (FFT_MajorPeak > 50) && (FFT_MajorPeak < 250) && (FFT_Magnitude > (16.0f * (maxVol+42.0)) /*my_magnitude > 136.0f*16.0f*/) && 
       (millis() - timeOfPeak > 80)) {
     havePeak = true;
@@ -904,7 +908,6 @@ class AudioReactive : public Usermod {
 
     // variables  for UDP sound sync
     WiFiUDP fftUdp;               // UDP object for sound sync (from WiFi UDP, not Async UDP!)
-    uint8_t fftUdpBuffer[UDPSOUND_MAX_PACKET+1] = { 0 }; // static buffer for receiving
     unsigned long lastTime = 0;   // last time of running UDP Microphone Sync
     const uint16_t delayMs = 10;  // I don't want to sample too often and overload WLED
     uint16_t audioSyncPort= 11988;// default port for UDP sound sync
@@ -1134,8 +1137,13 @@ class AudioReactive : public Usermod {
     {
       float    sampleAdj;           // Gain adjusted sample value
       float    tmpSample;           // An interim sample variable used for calculatioins.
+#ifdef WLEDMM_FASTPATH
+      constexpr float weighting = 0.35f;  // slightly reduced filter strength, to reduce audio latency
+      constexpr float weighting2 = 0.25f;
+#else
       const float weighting = 0.2f; // Exponential filter weighting. Will be adjustable in a future release.
       const float weighting2 = 0.073f; // Exponential filter weighting, for rising signal (a bit more robust against spikes)
+#endif
       const int   AGC_preset = (soundAgc > 0)? (soundAgc-1): 0; // make sure the _compiler_ knows this value will not change while we are inside the function
       static bool isFrozen = false;
       static bool haveSilence = true;
@@ -1218,7 +1226,7 @@ class AudioReactive : public Usermod {
         sampleMax = sampleMax + 0.5f * (sampleReal - sampleMax);  // new peak - with some filtering
 #if 1
         // another simple way to detect samplePeak - cannot detect beats, but reacts on peak volume
-        if (((binNum < 11) || (maxVol < 1)) && (millis() - timeOfPeak > 80) && (sampleAvg > 1)) {
+        if (((binNum < 12) || ((maxVol < 1))) && (millis() - timeOfPeak > 80) && (sampleAvg > 1)) {
           samplePeak    = true;
           timeOfPeak    = millis();
           udpSamplePeak = true;
@@ -1371,9 +1379,10 @@ class AudioReactive : public Usermod {
       transmitData.FFT_Magnitude = my_magnitude;
       transmitData.FFT_MajorPeak = FFT_MajorPeak;
 
-      fftUdp.beginMulticastPacket();
-      fftUdp.write(reinterpret_cast<uint8_t *>(&transmitData), sizeof(transmitData));
-      fftUdp.endPacket();
+      if (fftUdp.beginMulticastPacket() != 0) { // beginMulticastPacket returns 0 in case of error
+        fftUdp.write(reinterpret_cast<uint8_t *>(&transmitData), sizeof(transmitData));
+        fftUdp.endPacket();
+      }
       return;
     } // transmitAudioData()
 
@@ -1441,9 +1450,24 @@ class AudioReactive : public Usermod {
       if (!udpSyncConnected) return false;
       bool haveFreshData = false;
 
-      size_t packetSize = fftUdp.parsePacket();
+      size_t packetSize = 0;
+      // WLEDMM use exception handler to catch out-of-memory errors
+      #if __cpp_exceptions
+        try{
+          packetSize = fftUdp.parsePacket();
+        } catch(...) {
+          packetSize = 0; // low heap memory -> discard packet.
+          fftUdp.flush();
+          DEBUG_PRINTLN(F("receiveAudioData: parsePacket out of memory exception caught!"));
+          USER_FLUSH();
+        }
+      #else
+        packetSize = fftUdp.parsePacket();
+      #endif
+
       if ((packetSize > 0) && ((packetSize < 5) || (packetSize > UDPSOUND_MAX_PACKET))) fftUdp.flush(); // discard invalid packets (too small or too big)
       if ((packetSize > 5) && (packetSize <= UDPSOUND_MAX_PACKET)) {
+        static uint8_t fftUdpBuffer[UDPSOUND_MAX_PACKET+1] = { 0 }; // static buffer for receiving, to reuse the same memory and avoid heap fragmentation
         //DEBUGSR_PRINTLN("Received UDP Sync Packet");
         fftUdp.read(fftUdpBuffer, packetSize);
 
@@ -1738,7 +1762,11 @@ class AudioReactive : public Usermod {
 
         // get AGC sensitivity and sound pressure
         static unsigned long lastEstimate = 0;
+#ifdef WLEDMM_FASTPATH
+        if (millis() - lastEstimate > 7) {
+#else
         if (millis() - lastEstimate > 12) {
+#endif
           lastEstimate = millis();
           agcSensitivity = getSensitivity();
           if (limiterOn)
@@ -1763,6 +1791,8 @@ class AudioReactive : public Usermod {
             have_new_sample = receiveAudioData();
             if (have_new_sample) last_UDPTime = millis();
             lastTime = millis();
+          } else {
+            fftUdp.flush(); // WLEDMM: Flush this if we haven't read it.
           }
           if (have_new_sample) syncVolumeSmth = volumeSmth;   // remember received sample
           else volumeSmth = syncVolumeSmth;                   // restore originally received sample for next run of dynamics limiter

usermods/audioreactive/audio_source.h

@@ -184,8 +184,13 @@ class I2SSource : public AudioSource {
 #if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(4, 2, 0)
         .communication_format = i2s_comm_format_t(I2S_COMM_FORMAT_STAND_I2S),
         //.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1,
+#ifdef WLEDMM_FASTPATH
+        .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2|ESP_INTR_FLAG_LEVEL3,  // seems to reduce noise
+        .dma_buf_count = 28,                                            // 160ms buffer (128 * dma_buf_count / sampleRate)
+#else
         .intr_alloc_flags = ESP_INTR_FLAG_LEVEL2,
         .dma_buf_count = 8,
+#endif
         .dma_buf_len = _blockSize,
         .use_apll = 0,
         //.fixed_mclk = 0,

usermods/usermod_v2_weather/usermod_v2_weather.h

@@ -35,7 +35,7 @@ uint16_t mode_2DWeather(void) {
       // currentTime = localTime;
     }
     else
-      color = ColorFromPalette(SEGPALETTE, map((uint8_t)weather_temps[x%100], 0, 40, 0, 255), 255, LINEARBLEND);
+      color = ColorFromPalette(SEGPALETTE, map((uint8_t)weather_temps[x%100], 0, 40, 1, 255), 255, LINEARBLEND);
 
     for (int y=0; y<SEGMENT.virtualHeight() * (weather_temps[x%100]-weather_minTemp)/(weather_maxTemp - weather_minTemp); y++) {
       SEGMENT.setPixelColorXY(x, SEGMENT.virtualHeight() - y, color);
@@ -58,7 +58,7 @@ uint16_t mode_2DWeather(void) {
   snprintf(tempString, 5, "%5.2f", currentTemp); // snprintf will prevent overflow
   // Serial.println();
 
-  CRGB color = ColorFromPalette(SEGPALETTE, map((uint8_t)currentTemp, 0, 40, 0, 255), 255, LINEARBLEND);
+  CRGB color = ColorFromPalette(SEGPALETTE, map((uint8_t)currentTemp, 0, 40, 1, 255), 255, LINEARBLEND);
   //really don't understand why this is not working if width < 16 (only works when Serial.println is uncommented ???)
   // uint16_t x = 0;
   // const uint16_t xSpace = (SEGMENT.virtualWidth()<16)?4:5;
@@ -78,17 +78,17 @@ uint16_t mode_2DWeather(void) {
   // // Serial.printf("  %d %d\n", x, xSpace);
 
  if (SEGMENT.virtualWidth() < 16) {
-    SEGMENT.drawCharacter(tempString[0], 0, -2, 5, 8, color);
-    SEGMENT.drawCharacter(tempString[1], 4, -2, 5, 8, color);
+    SEGMENT.drawCharacter(tempString[0], 0, -2, 5, 8, uint32_t(color), BLACK);
+    SEGMENT.drawCharacter(tempString[1], 4, -2, 5, 8, uint32_t(color), BLACK);
     SEGMENT.setPixelColorXY(8, 4, color);
-    SEGMENT.drawCharacter(tempString[3], 9, -2, 5, 8, color);
+    SEGMENT.drawCharacter(tempString[3], 9, -2, 5, 8, uint32_t(color), BLACK);
   }
   else {
-    SEGMENT.drawCharacter(tempString[0], 0, -2, 5, 8, color);
-    SEGMENT.drawCharacter(tempString[1], 5, -2, 5, 8, color);
+    SEGMENT.drawCharacter(tempString[0], 0, -2, 5, 8, uint32_t(color), BLACK);
+    SEGMENT.drawCharacter(tempString[1], 5, -2, 5, 8, uint32_t(color), BLACK);
     SEGMENT.setPixelColorXY(10, 4, color);
-    SEGMENT.drawCharacter(tempString[3], 12, -2, 5, 8, color);
-    SEGMENT.drawCharacter(tempString[4], 17, -2, 5, 8, color);
+    SEGMENT.drawCharacter(tempString[3], 12, -2, 5, 8, uint32_t(color), BLACK);
+    SEGMENT.drawCharacter(tempString[4], 17, -2, 5, 8, uint32_t(color), BLACK);
   }
 
   return FRAMETIME;