Skip to content

Commit

Permalink
Renamings and optimize deduplication functions
Browse files Browse the repository at this point in the history
* rename TYPE_UL_ULL to SIGNAL_SIZE_UL_ULL
* build deduplication function only when the relevant gateways are built
* replace 2 dimensions array by a structure so as to handle the time with an uint32_t  when using an ESP (instead of a uint64_t)
* change storeValue and isAduplicate function names
* point to the rcswitch revision with the new 40 bits protocol
  • Loading branch information
1technophile committed Jul 18, 2020
1 parent d757d2f commit fd36729
Show file tree
Hide file tree
Showing 8 changed files with 94 additions and 75 deletions.
26 changes: 14 additions & 12 deletions main/User_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -269,21 +269,23 @@ uint8_t wifiProtocol = 0; // default mode, automatic selection
// uncomment the line below to integrate msg value into the subject when receiving
//#define valueAsASubject true

//variables to avoid duplicates
#define time_avoid_duplicate 3000 // if you want to avoid duplicate mqtt message received set this to > 0, the value is the time in milliseconds during which we don't publish duplicates

#if defined(ESP8266) || defined(ESP32)
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define TYPE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoull
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define SIGNAL_SIZE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoull
#elif defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define TYPE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoul
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define SIGNAL_SIZE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoul
#else // boards with smaller memory
# define JSON_MSG_BUFFER 64 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define TYPE_UL_ULL uint32_t
# define STRTO_UL_ULL strtoul
# define JSON_MSG_BUFFER 64 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define SIGNAL_SIZE_UL_ULL uint32_t
# define STRTO_UL_ULL strtoul
#endif

#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
// variable to avoid duplicates
# define time_avoid_duplicate 3000 // if you want to avoid duplicate mqtt message received set this to > 0, the value is the time in milliseconds during which we don't publish duplicates
#endif

#define TimeBetweenReadingSYS 120 // time between (s) system readings (like memory)
Expand Down
10 changes: 5 additions & 5 deletions main/ZgatewayIR.ino
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ void IRtoMQTT() {
# ifdef ESP32
Log.trace(F("IR Task running on core :%d" CR), xPortGetCoreID());
# endif
IRdata.set("value", (TYPE_UL_ULL)(results.value));
IRdata.set("value", (SIGNAL_SIZE_UL_ULL)(results.value));
IRdata.set("protocol", (int)(results.decode_type));
IRdata.set("bits", (int)(results.bits));
# if defined(ESP8266) || defined(ESP32) //resultToHexidecimal is only available with IRremoteESP8266
Expand Down Expand Up @@ -162,15 +162,15 @@ void IRtoMQTT() {
Log.trace(F("raw redirected" CR));
# endif
irrecv.resume(); // Receive the next value
TYPE_UL_ULL MQTTvalue = IRdata.get<TYPE_UL_ULL>("value");
SIGNAL_SIZE_UL_ULL MQTTvalue = IRdata.get<SIGNAL_SIZE_UL_ULL>("value");
//trc(MQTTvalue);
if ((pubIRunknownPrtcl == false && IRdata.get<int>("protocol") == -1)) { // don't publish unknown IR protocol
Log.notice(F("--no pub unknwn prt--" CR));
} else if (!isAduplicate(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of IR -->MQTT
} else if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of IR -->MQTT
Log.trace(F("Adv data IRtoMQTT" CR));
pub(subjectIRtoMQTT, IRdata);
Log.trace(F("Store val: %D" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
if (repeatIRwMQTT) {
Log.trace(F("Pub. IR for rpt" CR));
pubMQTT(subjectForwardMQTTtoIR, MQTTvalue);
Expand Down Expand Up @@ -281,7 +281,7 @@ void MQTTtoIR(char* topicOri, JsonObject& IRdata) {
}
# endif

bool sendIdentifiedProtocol(const char* protocol_name, TYPE_UL_ULL data, const char* hex, unsigned int valueBITS, uint16_t valueRPT) {
bool sendIdentifiedProtocol(const char* protocol_name, SIGNAL_SIZE_UL_ULL data, const char* hex, unsigned int valueBITS, uint16_t valueRPT) {
uint8_t dataarray[valueBITS];
if (hex) {
const char* ptr = NULL;
Expand Down
14 changes: 7 additions & 7 deletions main/ZgatewayRF.ino
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
RCSwitch mySwitch = RCSwitch();

# ifdef ZmqttDiscovery
void RFtoMQTTdiscovery(TYPE_UL_ULL MQTTvalue) { //on the fly switch creation from received RF values
void RFtoMQTTdiscovery(SIGNAL_SIZE_UL_ULL MQTTvalue) { //on the fly switch creation from received RF values
char val[11];
sprintf(val, "%lu", MQTTvalue);
Log.trace(F("switchRFDiscovery" CR));
Expand Down Expand Up @@ -78,20 +78,20 @@ void RFtoMQTT() {
# ifdef ESP32
Log.trace(F("RF Task running on core :%d" CR), xPortGetCoreID());
# endif
TYPE_UL_ULL MQTTvalue = mySwitch.getReceivedValue();
RFdata.set("value", (TYPE_UL_ULL)MQTTvalue);
SIGNAL_SIZE_UL_ULL MQTTvalue = mySwitch.getReceivedValue();
RFdata.set("value", (SIGNAL_SIZE_UL_ULL)MQTTvalue);
RFdata.set("protocol", (int)mySwitch.getReceivedProtocol());
RFdata.set("length", (int)mySwitch.getReceivedBitlength());
RFdata.set("delay", (int)mySwitch.getReceivedDelay());
mySwitch.resetAvailable();

if (!isAduplicate(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
# ifdef ZmqttDiscovery //component creation for HA
RFtoMQTTdiscovery(MQTTvalue);
# endif
pub(subjectRFtoMQTT, RFdata);
Log.trace(F("Store val: %lu" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
if (repeatRFwMQTT) {
Log.trace(F("Pub RF for rpt" CR));
pub(subjectMQTTtoRF, RFdata);
Expand All @@ -107,7 +107,7 @@ void MQTTtoRF(char* topicOri, char* datacallback) {
mySwitch.disableReceive();
mySwitch.enableTransmit(RF_EMITTER_GPIO);
# endif
TYPE_UL_ULL data = STRTO_UL_ULL(datacallback, NULL, 10); // we will not be able to pass values > 4294967295 on Arduino boards
SIGNAL_SIZE_UL_ULL data = STRTO_UL_ULL(datacallback, NULL, 10); // we will not be able to pass values > 4294967295 on Arduino boards

// RF DATA ANALYSIS
//We look into the subject to see if a special RF protocol is defined
Expand Down Expand Up @@ -171,7 +171,7 @@ void MQTTtoRF(char* topicOri, JsonObject& RFdata) { // json object decoding
# endif
if (cmpToMainTopic(topicOri, subjectMQTTtoRF)) {
Log.trace(F("MQTTtoRF json" CR));
TYPE_UL_ULL data = RFdata["value"];
SIGNAL_SIZE_UL_ULL data = RFdata["value"];
if (data != 0) {
int valuePRT = RFdata["protocol"] | 1;
int valuePLSL = RFdata["delay"] | 350;
Expand Down
4 changes: 2 additions & 2 deletions main/ZgatewaySRFB.ino
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,11 @@ void _rfbDecode() {
unsigned long MQTTvalue = (unsigned long)strtoul(val, NULL, 10);
SRFBdata.set("value", (unsigned long)MQTTvalue);

if (!isAduplicate(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
Log.trace(F("Adv data SRFBtoMQTT" CR));
pub(subjectSRFBtoMQTT, SRFBdata);
Log.trace(F("Store val: %lu" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
if (repeatSRFBwMQTT) {
Log.trace(F("Publish SRFB for rpt" CR));
pub(subjectMQTTtoSRFB, SRFBdata);
Expand Down
16 changes: 8 additions & 8 deletions main/ZgatewayWeatherStation.ino
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ void setupWeatherStation() {

void sendWindSpeedData(byte id, float wind_speed, byte battery_status) {
unsigned long MQTTvalue = 10000 + round(wind_speed);
if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(3);
StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
JsonObject& RFdata = jsonBuffer.createObject();
Expand All @@ -63,13 +63,13 @@ void sendWindSpeedData(byte id, float wind_speed, byte battery_status) {
RFdata.set("battery", bitRead(battery_status, 0) == 0 ? "OK" : "Low");
pub(subjectRFtoMQTT, RFdata);
Log.trace(F("Store wind speed val: %lu" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
}
}

void sendRainData(byte id, float rain_volume, byte battery_status) {
unsigned long MQTTvalue = 11000 + round(rain_volume * 10.0);
if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(3);
StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
JsonObject& RFdata = jsonBuffer.createObject();
Expand All @@ -78,13 +78,13 @@ void sendRainData(byte id, float rain_volume, byte battery_status) {
RFdata.set("battery", bitRead(battery_status, 1) == 0 ? "OK" : "Low");
pub(subjectRFtoMQTT, RFdata);
Log.trace(F("Store rain_volume: %lu" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
}
}

void sendWindData(byte id, int wind_direction, float wind_gust, byte battery_status) {
unsigned long MQTTvalue = 20000 + round(wind_gust * 10.0) + wind_direction;
if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(4);
StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
JsonObject& RFdata = jsonBuffer.createObject();
Expand All @@ -94,13 +94,13 @@ void sendWindData(byte id, int wind_direction, float wind_gust, byte battery_sta
RFdata.set("battery", bitRead(battery_status, 0) == 0 ? "OK" : "Low");
pub(subjectRFtoMQTT, RFdata);
Log.trace(F("Store wind data val: %lu" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
}
}

void sendTemperatureData(byte id, float temperature, int humidity, byte battery_status) {
unsigned long MQTTvalue = 40000 + abs(round(temperature * 100.0)) + humidity;
if (!isAduplicate(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
if (!isAduplicateSignal(MQTTvalue)) { // conditions to avoid duplications of RF -->MQTT
const int JSON_MSG_CALC_BUFFER = JSON_OBJECT_SIZE(4);
StaticJsonBuffer<JSON_MSG_CALC_BUFFER> jsonBuffer;
JsonObject& RFdata = jsonBuffer.createObject();
Expand All @@ -110,7 +110,7 @@ void sendTemperatureData(byte id, float temperature, int humidity, byte battery_
RFdata.set("battery", bitRead(battery_status, 0) == 0 ? "OK" : "Low");
pub(subjectRFtoMQTT, RFdata);
Log.trace(F("Store temp val: %lu" CR), MQTTvalue);
storeValue(MQTTvalue);
storeSignalValue(MQTTvalue);
}
}

Expand Down
79 changes: 48 additions & 31 deletions main/main.ino
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,25 @@
*/
#include "User_config.h"

// Macros and structure to enable the duplicates removing on the following gateways
#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
// array to store previous received RFs, IRs codes and their timestamps
#if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
# define array_size 12
unsigned long long ReceivedSignal[array_size][2] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
//Time used to wait for an interval before checking system measures
unsigned long timer_sys_measures = 0;
#else // boards with smaller memory
# define array_size 4
unsigned long ReceivedSignal[array_size][2] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}};
struct ReceivedSignal {
SIGNAL_SIZE_UL_ULL value;
uint32_t time;
};
# if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
# define struct_size 12
ReceivedSignal receivedSignal[struct_size] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}};
# else // boards with smaller memory
# define struct_size 4
ReceivedSignal receivedSignal[struct_size] = {{0, 0}, {0, 0}, {0, 0}, {0, 0}};
# endif
#endif

#if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
//Time used to wait for an interval before checking system measures
unsigned long timer_sys_measures = 0;
# define ARDUINOJSON_USE_LONG_LONG 1
#endif
#include <ArduinoJson.h>
Expand Down Expand Up @@ -225,7 +232,7 @@ void pub(char* topicori, JsonObject& data) {
digitalWrite(led_receive, HIGH);
String topic = String(mqtt_topic) + String(topicori);
#ifdef valueAsASubject
TYPE_UL_ULL value = data["value"];
SIGNAL_SIZE_UL_ULL value = data["value"];
if (value != 0) {
topic = topic + "/" + String(value);
}
Expand All @@ -249,11 +256,11 @@ void pub(char* topicori, JsonObject& data) {
# if defined(ESP8266)
yield();
# endif
if (p.value.is<TYPE_UL_ULL>() && strcmp(p.key, "rssi") != 0) { //test rssi , bypass solution due to the fact that a int is considered as an TYPE_UL_ULL
if (p.value.is<SIGNAL_SIZE_UL_ULL>() && strcmp(p.key, "rssi") != 0) { //test rssi , bypass solution due to the fact that a int is considered as an SIGNAL_SIZE_UL_ULL
if (strcmp(p.key, "value") == 0) { // if data is a value we don't integrate the name into the topic
pubMQTT(topic, p.value.as<TYPE_UL_ULL>());
pubMQTT(topic, p.value.as<SIGNAL_SIZE_UL_ULL>());
} else { // if data is not a value we integrate the name into the topic
pubMQTT(topic + "/" + String(p.key), p.value.as<TYPE_UL_ULL>());
pubMQTT(topic + "/" + String(p.key), p.value.as<SIGNAL_SIZE_UL_ULL>());
}
} else if (p.value.is<int>()) {
pubMQTT(topic + "/" + String(p.key), p.value.as<int>());
Expand Down Expand Up @@ -1309,61 +1316,71 @@ void stateMeasures() {
}
#endif

void storeValue(TYPE_UL_ULL MQTTvalue) {
#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
/**
* Store signal values from RF, IR, SRFB or Weather stations so as to avoid duplicates
*/
void storeSignalValue(SIGNAL_SIZE_UL_ULL MQTTvalue) {
unsigned long now = millis();
// find oldest value of the buffer
int o = getMin();
Log.trace(F("Min ind: %d" CR), o);
// replace it by the new one
ReceivedSignal[o][0] = MQTTvalue;
ReceivedSignal[o][1] = now;
Log.trace(F("store code : %u / %u" CR), ReceivedSignal[o][0], ReceivedSignal[o][1]);
receivedSignal[o].value = MQTTvalue;
receivedSignal[o].time = now;
Log.trace(F("store code : %u / %u" CR), receivedSignal[o].value, receivedSignal[o].time);
Log.trace(F("Col: val/timestamp" CR));
for (int i = 0; i < array_size; i++) {
Log.trace(F("mem code : %u / %u" CR), ReceivedSignal[i][0], ReceivedSignal[i][1]);
for (int i = 0; i < struct_size; i++) {
Log.trace(F("mem code : %u / %u" CR), receivedSignal[i].value, receivedSignal[i].time);
}
}

/**
* get oldest time index from the values array from RF, IR, SRFB or Weather stations so as to avoid duplicates
*/
int getMin() {
unsigned int minimum = ReceivedSignal[0][1];
unsigned int minimum = receivedSignal[0].time;
int minindex = 0;
for (int i = 0; i < array_size; i++) {
if (ReceivedSignal[i][1] < minimum) {
minimum = ReceivedSignal[i][1];
for (int i = 0; i < struct_size; i++) {
if (receivedSignal[i].time < minimum) {
minimum = receivedSignal[i].time;
minindex = i;
}
}
return minindex;
}

bool isAduplicate(TYPE_UL_ULL value) {
/**
* Check if signal values from RF, IR, SRFB or Weather stations are duplicates
*/
bool isAduplicateSignal(SIGNAL_SIZE_UL_ULL value) {
Log.trace(F("isAdupl?" CR));
// check if the value has been already sent during the last time_avoid_duplicate
for (int i = 0; i < array_size; i++) {
if (ReceivedSignal[i][0] == value) {
for (int i = 0; i < struct_size; i++) {
if (receivedSignal[i].value == value) {
unsigned long now = millis();
if (now - ReceivedSignal[i][1] < time_avoid_duplicate) { // change
if (now - receivedSignal[i].time < time_avoid_duplicate) { // change
Log.notice(F("no pub. dupl" CR));
return true;
}
}
}
return false;
}
#endif

void receivingMQTT(char* topicOri, char* datacallback) {
StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
JsonObject& jsondata = jsonBuffer.parseObject(datacallback);

if (strstr(topicOri, subjectMultiGTWKey) != NULL) // storing received value so as to avoid publishing this value if it has been already sent by this or another OpenMQTTGateway
{
TYPE_UL_ULL data = 0;

#if defined(ZgatewayRF) || defined(ZgatewayIR) || defined(ZgatewaySRFB) || defined(ZgatewaySRFB) || defined(ZgatewayWeatherStation)
SIGNAL_SIZE_UL_ULL data = 0;
data = jsondata.success() ? jsondata["value"] : STRTO_UL_ULL(datacallback, NULL, 10);

if (data != 0) {
storeValue(data);
storeSignalValue(data);
}
#endif
}

if (jsondata.success()) { // json object ok -> json decoding
Expand Down
2 changes: 1 addition & 1 deletion platformio.ini
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ extra_configs =
arduinojson = [email protected]
arduinolog = https://github.com/1technophile/Arduino-Log.git#d13cd80
pubsubclient = [email protected]
rc-switch = https://github.com/1technophile/rc-switch.git#8fc6d06
rc-switch = https://github.com/1technophile/rc-switch.git#385a7e0
newremoteswitch = https://github.com/1technophile/NewRemoteSwitch.git#8eb980e
ble = https://github.com/eos1d3/ESP32_BLE.git#9d6c1ed
irremoteesp = [email protected]
Expand Down
18 changes: 9 additions & 9 deletions test/Test_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -171,17 +171,17 @@ char gateway_name[parameters_size * 2] = Gateway_Name;
#endif

#if defined(ESP8266) || defined(ESP32)
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define TYPE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoull
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define SIGNAL_SIZE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoull
#elif defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define TYPE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoul
# define JSON_MSG_BUFFER 512 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define SIGNAL_SIZE_UL_ULL uint64_t
# define STRTO_UL_ULL strtoul
#else // boards with smaller memory
# define JSON_MSG_BUFFER 64 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define TYPE_UL_ULL uint32_t
# define STRTO_UL_ULL strtoul
# define JSON_MSG_BUFFER 64 // Json message max buffer size, don't put 1024 or higher it is causing unexpected behaviour on ESP8266
# define SIGNAL_SIZE_UL_ULL uint32_t
# define STRTO_UL_ULL strtoul
#endif

#define TimeBetweenReadingSYS 120000 // time between system readings (like memory)
Expand Down

0 comments on commit fd36729

Please sign in to comment.