[RF] handle 64 bits values (except for UNO)

docs/prerequisites/board.md

@@ -33,7 +33,7 @@ The boards below need hardware modifications and electronic/hardware competencie
 |ESP32|X|X|X|X|not tested|X|X|
 |ESP8266|X|X|X|not tested|X|X|X|
 
-*Note that Pilight is only supported on ESP for the moment.*
+*Note that Pilight is only supported on ESP and Arduino UNO handle only 32bits values in our context.*
 
 ![boards](../img/OpenMQTTGateway_boards.png)
 

docs/upload/pio.md

@@ -135,3 +135,7 @@ uncommented.
 Added to that auto discovery box should be selected into your Home Assistant MQTT integration configuration.
 
 With an ESP if you did not set your network and mqtt parameters manualy you can now open the [web portal configuration](portal.md).
+
+::: warning Note
+simpleReceiving on Arduino boards doesn't accept 64 bits MQTT values, you can only send 32bits values from the MQTT broker.
+:::

main/User_config.h

@@ -269,14 +269,23 @@ uint8_t wifiProtocol = 0; // default mode, automatic selection
 // uncomment the line below to integrate msg value into the subject when receiving
 //#define valueAsASubject true
 
-//variables to avoid duplicates
-#define time_avoid_duplicate 3000 // if you want to avoid duplicate mqtt message received set this to > 0, the value is the time in milliseconds during which we don't publish duplicates
-
-#if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
-#  define JSON_MSG_BUFFER           512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
-#  define ARDUINOJSON_USE_LONG_LONG 1
+#if defined(ESP8266) || defined(ESP32)
+#  define JSON_MSG_BUFFER    512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
+#  define SIGNAL_SIZE_UL_ULL uint64_t
+#  define STRTO_UL_ULL       strtoull
+#elif defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
+#  define JSON_MSG_BUFFER    512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
+#  define SIGNAL_SIZE_UL_ULL uint64_t
+#  define STRTO_UL_ULL       strtoul
 #else // boards with smaller memory
-#  define JSON_MSG_BUFFER 64 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
+#  define JSON_MSG_BUFFER    64 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
+#  define SIGNAL_SIZE_UL_ULL uint32_t
+#  define STRTO_UL_ULL       strtoul
+#endif
+
+#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
+// variable to avoid duplicates
+#  define time_avoid_duplicate 3000 // if you want to avoid duplicate mqtt message received set this to > 0, the value is the time in milliseconds during which we don't publish duplicates
 #endif
 
 #define TimeBetweenReadingSYS        120 // time between (s) system readings (like memory)

main/ZgatewayIR.ino

@@ -122,7 +122,7 @@ void IRtoMQTT() {
 #  ifdef ESP32
     Log.trace(F("IR Task running on core :%d" CR), xPortGetCoreID());
 #  endif
-    IRdata.set("value", (unsigned long long)(results.value));
+    IRdata.set("value", (SIGNAL_SIZE_UL_ULL)(results.value));
     IRdata.set("protocol", (int)(results.decode_type));
     IRdata.set("bits", (int)(results.bits));
 #  if defined(ESP8266) || defined(ESP32) //resultToHexidecimal is only available with IRremoteESP8266
@@ -162,15 +162,15 @@ void IRtoMQTT() {
     Log.trace(F("raw redirected" CR));
 #  endif
     irrecv.resume(); // Receive the next value
-    unsigned long long MQTTvalue = IRdata.get<unsigned long long>("value");
+    SIGNAL_SIZE_UL_ULL MQTTvalue = IRdata.get<SIGNAL_SIZE_UL_ULL>("value");
     //trc(MQTTvalue);
     if ((pubIRunknownPrtcl == false && IRdata.get<int>("protocol") == -1)) { // don't publish unknown IR protocol
       Log.notice(F("--no pub unknwn prt--" CR));
-    } else if (!isAduplicate(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of IR -->MQTT
+    } else if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of IR -->MQTT
       Log.trace(F("Adv data IRtoMQTT" CR));
       pub(subjectIRtoMQTT, IRdata);
       Log.trace(F("Store val: %D" CR), MQTTvalue);
-      storeValue(MQTTvalue);
+      storeSignalValue(MQTTvalue);
       if (repeatIRwMQTT) {
         Log.trace(F("Pub. IR for rpt" CR));
         pubMQTT(subjectForwardMQTTtoIR, MQTTvalue);
@@ -281,7 +281,7 @@ void MQTTtoIR(char* topicOri, JsonObject& IRdata) {
 }
 #  endif
 
-bool sendIdentifiedProtocol(const char* protocol_name, unsigned long long data, const char* hex, unsigned int valueBITS, uint16_t valueRPT) {
+bool sendIdentifiedProtocol(const char* protocol_name, SIGNAL_SIZE_UL_ULL data, const char* hex, unsigned int valueBITS, uint16_t valueRPT) {
   uint8_t dataarray[valueBITS];
   if (hex) {
     const char* ptr = NULL;

main/ZgatewayRF.ino

@@ -38,7 +38,7 @@
 RCSwitch mySwitch = RCSwitch();
 
 #  ifdef ZmqttDiscovery
-void RFtoMQTTdiscovery(unsigned long MQTTvalue) { //on the fly switch creation from received RF values
+void RFtoMQTTdiscovery(SIGNAL_SIZE_UL_ULL MQTTvalue) { //on the fly switch creation from received RF values
   char val[11];
   sprintf(val, "%lu", MQTTvalue);
   Log.trace(F("switchRFDiscovery" CR));
@@ -78,20 +78,20 @@ void RFtoMQTT() {
 #  ifdef ESP32
     Log.trace(F("RF Task running on core :%d" CR), xPortGetCoreID());
 #  endif
-    RFdata.set("value", (unsigned long)mySwitch.getReceivedValue());
+    SIGNAL_SIZE_UL_ULL MQTTvalue = mySwitch.getReceivedValue();
+    RFdata.set("value", (SIGNAL_SIZE_UL_ULL)MQTTvalue);
     RFdata.set("protocol", (int)mySwitch.getReceivedProtocol());
     RFdata.set("length", (int)mySwitch.getReceivedBitlength());
     RFdata.set("delay", (int)mySwitch.getReceivedDelay());
     mySwitch.resetAvailable();
 
-    unsigned long MQTTvalue = RFdata.get<unsigned long>("value");
-    if (!isAduplicate(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
+    if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
 #  ifdef ZmqttDiscovery //component creation for HA
       RFtoMQTTdiscovery(MQTTvalue);
 #  endif
       pub(subjectRFtoMQTT, RFdata);
       Log.trace(F("Store val: %lu" CR), MQTTvalue);
-      storeValue(MQTTvalue);
+      storeSignalValue(MQTTvalue);
       if (repeatRFwMQTT) {
         Log.trace(F("Pub RF for rpt" CR));
         pub(subjectMQTTtoRF, RFdata);
@@ -107,7 +107,7 @@ void MQTTtoRF(char* topicOri, char* datacallback) {
   mySwitch.disableReceive();
   mySwitch.enableTransmit(RF_EMITTER_GPIO);
 #    endif
-  unsigned long data = strtoul(datacallback, NULL, 10); // we will not be able to pass values > 4294967295
+  SIGNAL_SIZE_UL_ULL data = STRTO_UL_ULL(datacallback, NULL, 10); // we will not be able to pass values > 4294967295 on Arduino boards
 
   // RF DATA ANALYSIS
   //We look into the subject to see if a special RF protocol is defined
@@ -171,7 +171,7 @@ void MQTTtoRF(char* topicOri, JsonObject& RFdata) { // json object decoding
 #    endif
   if (cmpToMainTopic(topicOri, subjectMQTTtoRF)) {
     Log.trace(F("MQTTtoRF json" CR));
-    unsigned long data = RFdata["value"];
+    SIGNAL_SIZE_UL_ULL data = RFdata["value"];
     if (data != 0) {
       int valuePRT = RFdata["protocol"] | 1;
       int valuePLSL = RFdata["delay"] | 350;

main/ZgatewaySRFB.ino

@@ -126,11 +126,11 @@ void _rfbDecode() {
     unsigned long MQTTvalue = (unsigned long)strtoul(val, NULL, 10);
     SRFBdata.set("value", (unsigned long)MQTTvalue);
 
-    if (!isAduplicate(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
+    if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
       Log.trace(F("Adv data SRFBtoMQTT" CR));
       pub(subjectSRFBtoMQTT, SRFBdata);
       Log.trace(F("Store val: %lu" CR), MQTTvalue);
-      storeValue(MQTTvalue);
+      storeSignalValue(MQTTvalue);
       if (repeatSRFBwMQTT) {
         Log.trace(F("Publish SRFB for rpt" CR));
         pub(subjectMQTTtoSRFB, SRFBdata);

main/ZgatewayWeatherStation.ino

@@ -54,7 +54,7 @@ void setupWeatherStation() {
 
 void sendWindSpeedData(byte id, float wind_speed, byte battery_status) {
   unsigned long MQTTvalue = 10000 + round(wind_speed);
-  if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
+  if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
     const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(3);
     StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
     JsonObject& RFdata = jsonBuffer.createObject();
@@ -63,13 +63,13 @@ void sendWindSpeedData(byte id, float wind_speed, byte battery_status) {
     RFdata.set("battery", bitRead(battery_status, 0) == 0 ? "OK" : "Low");
     pub(subjectRFtoMQTT, RFdata);
     Log.trace(F("Store wind speed val: %lu" CR), MQTTvalue);
-    storeValue(MQTTvalue);
+    storeSignalValue(MQTTvalue);
   }
 }
 
 void sendRainData(byte id, float rain_volume, byte battery_status) {
   unsigned long MQTTvalue = 11000 + round(rain_volume * 10.0);
-  if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
+  if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
     const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(3);
     StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
     JsonObject& RFdata = jsonBuffer.createObject();
@@ -78,13 +78,13 @@ void sendRainData(byte id, float rain_volume, byte battery_status) {
     RFdata.set("battery", bitRead(battery_status, 1) == 0 ? "OK" : "Low");
     pub(subjectRFtoMQTT, RFdata);
     Log.trace(F("Store rain_volume: %lu" CR), MQTTvalue);
-    storeValue(MQTTvalue);
+    storeSignalValue(MQTTvalue);
   }
 }
 
 void sendWindData(byte id, int wind_direction, float wind_gust, byte battery_status) {
   unsigned long MQTTvalue = 20000 + round(wind_gust * 10.0) + wind_direction;
-  if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
+  if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
     const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(4);
     StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
     JsonObject& RFdata = jsonBuffer.createObject();
@@ -94,13 +94,13 @@ void sendWindData(byte id, int wind_direction, float wind_gust, byte battery_sta
     RFdata.set("battery", bitRead(battery_status, 0) == 0 ? "OK" : "Low");
     pub(subjectRFtoMQTT, RFdata);
     Log.trace(F("Store wind data val: %lu" CR), MQTTvalue);
-    storeValue(MQTTvalue);
+    storeSignalValue(MQTTvalue);
   }
 }
 
 void sendTemperatureData(byte id, float temperature, int humidity, byte battery_status) {
   unsigned long MQTTvalue = 40000 + abs(round(temperature * 100.0)) + humidity;
-  if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
+  if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
     const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(4);
     StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
     JsonObject& RFdata = jsonBuffer.createObject();
@@ -110,7 +110,7 @@ void sendTemperatureData(byte id, float temperature, int humidity, byte battery_
     RFdata.set("battery", bitRead(battery_status, 0) == 0 ? "OK" : "Low");
     pub(subjectRFtoMQTT, RFdata);
     Log.trace(F("Store temp val: %lu" CR), MQTTvalue);
-    storeValue(MQTTvalue);
+    storeSignalValue(MQTTvalue);
   }
 }
 

main/main.ino

@@ -27,17 +27,27 @@
 */
 #include "User_config.h"
 
+// Macros and structure to enable the duplicates removing on the following gateways
+#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
 // array to store previous received RFs, IRs codes and their timestamps
+struct ReceivedSignal {
+  SIGNAL_SIZE_UL_ULL value;
+  uint32_t time;
+};
+#  if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
+#    define struct_size 12
+ReceivedSignal receivedSignal[struct_size] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
+#  else // boards with smaller memory
+#    define struct_size 4
+ReceivedSignal receivedSignal[struct_size] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}};
+#  endif
+#endif
+
 #if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
-#  define array_size 12
-unsigned long ReceivedSignal[array_size][2] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
 //Time used to wait for an interval before checking system measures
 unsigned long timer_sys_measures = 0;
-#else // boards with smaller memory
-#  define array_size 4
-unsigned long ReceivedSignal[array_size][2] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}};
+#  define ARDUINOJSON_USE_LONG_LONG 1
 #endif
-
 #include <ArduinoJson.h>
 #include <ArduinoLog.h>
 #include <PubSubClient.h>
@@ -222,7 +232,7 @@ void pub(char* topicori, JsonObject& data) {
     digitalWrite(led_receive, HIGH);
     String topic = String(mqtt_topic) + String(topicori);
 #ifdef valueAsASubject
-    unsigned long value = data["value"];
+    SIGNAL_SIZE_UL_ULL value = data["value"];
     if (value != 0) {
       topic = topic + "/" + String(value);
     }
@@ -246,11 +256,11 @@ void pub(char* topicori, JsonObject& data) {
 #  if defined(ESP8266)
       yield();
 #  endif
-      if (p.value.is<unsigned long>() && strcmp(p.key, "rssi") != 0) { //test rssi , bypass solution due to the fact that a int is considered as an unsigned long
+      if (p.value.is<SIGNAL_SIZE_UL_ULL>() && strcmp(p.key, "rssi") != 0) { //test rssi , bypass solution due to the fact that a int is considered as an SIGNAL_SIZE_UL_ULL
         if (strcmp(p.key, "value") == 0) { // if data is a value we don't integrate the name into the topic
-          pubMQTT(topic, p.value.as<unsigned long>());
+          pubMQTT(topic, p.value.as<SIGNAL_SIZE_UL_ULL>());
         } else { // if data is not a value we integrate the name into the topic
-          pubMQTT(topic + "/" + String(p.key), p.value.as<unsigned long>());
+          pubMQTT(topic + "/" + String(p.key), p.value.as<SIGNAL_SIZE_UL_ULL>());
         }
       } else if (p.value.is<int>()) {
         pubMQTT(topic + "/" + String(p.key), p.value.as<int>());
@@ -330,6 +340,12 @@ void pubMQTT(String topic, int payload) {
   client.publish((char*)topic.c_str(), val);
 }
 
+void pubMQTT(String topic, unsigned long long payload) {
+  char val[21];
+  sprintf(val, "%llu", payload);
+  client.publish((char*)topic.c_str(), val);
+}
+
 void pubMQTT(String topic, float payload) {
   char val[12];
   dtostrf(payload, 3, 1, val);
@@ -1300,67 +1316,71 @@ void stateMeasures() {
 }
 #endif
 
-void storeValue(unsigned long MQTTvalue) {
+#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
+/** 
+ * Store signal values from RF, IR, SRFB or Weather stations so as to avoid duplicates
+ */
+void storeSignalValue(SIGNAL_SIZE_UL_ULL MQTTvalue) {
   unsigned long now = millis();
   // find oldest value of the buffer
   int o = getMin();
   Log.trace(F("Min ind: %d" CR), o);
   // replace it by the new one
-  ReceivedSignal[o][0] = MQTTvalue;
-  ReceivedSignal[o][1] = now;
-  Log.trace(F("store code : %u / %u" CR), ReceivedSignal[o][0], ReceivedSignal[o][1]);
+  receivedSignal[o].value = MQTTvalue;
+  receivedSignal[o].time = now;
+  Log.trace(F("store code : %u / %u" CR), receivedSignal[o].value, receivedSignal[o].time);
   Log.trace(F("Col: val/timestamp" CR));
-  for (int i = 0; i < array_size; i++) {
-    Log.trace(F("mem code : %u / %u" CR), ReceivedSignal[i][0], ReceivedSignal[i][1]);
+  for (int i = 0; i < struct_size; i++) {
+    Log.trace(F("mem code : %u / %u" CR), receivedSignal[i].value, receivedSignal[i].time);
   }
 }
 
+/** 
+ * get oldest time index from the values array from RF, IR, SRFB or Weather stations so as to avoid duplicates
+ */
 int getMin() {
-  unsigned int minimum = ReceivedSignal[0][1];
+  unsigned int minimum = receivedSignal[0].time;
   int minindex = 0;
-  for (int i = 0; i < array_size; i++) {
-    if (ReceivedSignal[i][1] < minimum) {
-      minimum = ReceivedSignal[i][1];
+  for (int i = 0; i < struct_size; i++) {
+    if (receivedSignal[i].time < minimum) {
+      minimum = receivedSignal[i].time;
       minindex = i;
     }
   }
   return minindex;
 }
 
-bool isAduplicate(unsigned long value) {
+/** 
+ * Check if signal values from RF, IR, SRFB or Weather stations are duplicates
+ */
+bool isAduplicateSignal(SIGNAL_SIZE_UL_ULL value) {
   Log.trace(F("isAdupl?" CR));
-  // check if the value has been already sent during the last time_avoid_duplicate
-  for (int i = 0; i < array_size; i++) {
-    if (ReceivedSignal[i][0] == value) {
+  for (int i = 0; i < struct_size; i++) {
+    if (receivedSignal[i].value == value) {
       unsigned long now = millis();
-      if (now - ReceivedSignal[i][1] < time_avoid_duplicate) { // change
+      if (now - receivedSignal[i].time < time_avoid_duplicate) { // change
         Log.notice(F("no pub. dupl" CR));
         return true;
       }
     }
   }
   return false;
 }
+#endif
 
 void receivingMQTT(char* topicOri, char* datacallback) {
   StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
   JsonObject& jsondata = jsonBuffer.parseObject(datacallback);
 
   if (strstr(topicOri, subjectMultiGTWKey) != NULL) // storing received value so as to avoid publishing this value if it has been already sent by this or another OpenMQTTGateway
   {
-    unsigned long data = 0;
-#ifdef jsonPublishing
-    if (jsondata.success())
-      data = jsondata["value"];
-#endif
-
-#ifdef simplePublishing
-    data = strtoul(datacallback, NULL, 10); // we will not be able to pass values > 4294967295
-#endif
-
+#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
+    SIGNAL_SIZE_UL_ULL data = 0;
+    data = jsondata.success() ? jsondata["value"] : STRTO_UL_ULL(datacallback, NULL, 10);
     if (data != 0) {
-      storeValue(data);
+      storeSignalValue(data);
     }
+#endif
   }
 
   if (jsondata.success()) { // json object ok -> json decoding

platformio.ini

@@ -68,7 +68,7 @@ extra_configs =
 arduinojson = [email protected]
 arduinolog = https://github.com/1technophile/Arduino-Log.git#d13cd80
 pubsubclient = [email protected]
-rc-switch = https://github.com/1technophile/rc-switch.git#8fc6d06
+rc-switch = https://github.com/1technophile/rc-switch.git#385a7e0
 newremoteswitch = https://github.com/1technophile/NewRemoteSwitch.git#8eb980e
 ble = https://github.com/eos1d3/ESP32_BLE.git#9d6c1ed
 irremoteesp = [email protected]