Commit to provide secureMQTT with X.509 certs as an option in sender …

…class (universam1#451)

* Commit to provide secureMQTT with X.509 certs as an option in sender

This is pre-cursor work to AWS IoT Core integration option that required secure connection.

The main features here are:
1) Creation of a method, SenderClass::SendSecureMQTT which is a copy of SendMQTT class with parameters for secure connections
2) Modification of SenderClass::MQTTConnect to select WifiClient or WifiClientSecure object based on input parameters
3) Due to memory constraint I had to reduce JSON buffer to 256 from 1024 in sender.h. This seems to be sufficient from checks I did on ArduinoJSON test site for the standard messages we send.

I chose this approach to avoid changes to other areas of the code.

I will submit a seperate pull request with changes to Globals, iSpindel.cpp and WifiManager to enable AWS as a connection option

* Commit to provide secureMQTT with X.509 certs as an option in sender

This is pre-cursor work to AWS IoT Core integration option that required secure connection.

The main features here are:
1) Creation of a method, SenderClass::SendSecureMQTT which is a copy of SendMQTT class with parameters for secure connections
2) Modification of SenderClass::MQTTConnect to select WifiClient or WifiClientSecure object based on input parameters
3) Due to memory constraint I had to reduce JSON buffer to 256 from 1024 in sender.h. This seems to be sufficient from checks I did on ArduinoJSON test site for the standard messages we send.

I chose this approach to avoid changes to other areas of the code.

I will submit a seperate pull request with changes to Globals, iSpindel.cpp and WifiManager to enable AWS as a connection option

* Successful connection to AWS MQTT using secrets.h

You need to cut and paste the relevant cert and key data into secrets.h

MQTT publish reformatted to support AWS Thing Shadow JSON format

* add secrets.h to gitignore

* Clean Up UI as requested by universam1

* Remove AWS from core UI and add readme_aws

.gitignore

@@ -19,3 +19,4 @@ iSpindel/__vm/
 .cproject
 .project
 .settings
+pio/src/secrets.h

README_AWS.md

@@ -0,0 +1,15 @@
+*AWS IoT Conectivity*
+ADVANCED USERS ONLY
+
+Recently we have started to add support for AWS IOT Core using the Secure MQTT option.
+
+This option currently requires you to manually input your keys and certs into the secrets.h file, compile and upload yourself.
+We are looking to add the ability to upload a certificate in future.
+
+0. Configure AWS IoT core including creating your thing, certificates, policies and download your certs and keys
+1. Download the Repo
+2. Edit secrets.h to supply AWS Root Cert, Device Private Cert, Device Private Key
+3. Compile and upload to your iSpindel
+4. Configure using the config menu as normal with the AWSIOTMQTT option
+
+Note: Secure MQTT (required by AWS) using a lot more battery as the device has to be on longer during each send. It needs to sync to an NTP server and establish and encrypted connection - both of which require processing time and hence battery use on the ESP8266.

pio/lib/Globals/Globals.h

@@ -32,6 +32,7 @@ extern Ticker flasher;
 #define API_THINGSPEAK true
 #define API_BLYNK true
 #define API_BREWBLOX true
+#define API_AWSIOTMQTT true         //AWS
 
 //#define BLYNK_DEBUG
 //#define APP_DEBUG
@@ -97,6 +98,7 @@ extern Ticker flasher;
 #define DTTHINGSPEAK 11
 #define DTBLYNK 12
 #define DTBREWBLOX 13
+#define DTAWSIOTMQTT 14             //AWS
 
 // Number of seconds after reset during which a
 // subseqent reset will be considered a double reset.

pio/lib/Sender/Sender.cpp

@@ -10,10 +10,15 @@
 #include <PubSubClient.h>
 #include <ThingSpeak.h>
 #include <BlynkSimpleEsp8266.h> //https://github.com/blynkkk/blynk-library
+#include <ESP8266WiFi.h>
+#include <ESP8266HTTPClient.h>
+#include <ArduinoJson.h>
+#include <WiFiClientSecure.h>
 
 #define UBISERVER "industrial.api.ubidots.com"
 #define BLYNKSERVER "blynk-cloud.com"
 #define CONNTIMEOUT 2000
+#define TIMECHECK 172800
 
 SenderClass::SenderClass() {}
 
@@ -36,11 +41,54 @@ void SenderClass::add(String id, int32_t value)
 void SenderClass::stopclient()
 {
     _client.stop();
+    _secureClient.stop();
     delay(100); // allow gracefull session close
 }
+bool SenderClass::RTCSyncToNTP()
+{
+    CONSOLE(F("Starting NTP Sync: "));
+    time_t now = time(nullptr);
+    if (now < TIMECHECK) {
+        configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov");
+    }  
+    while (now < TIMECHECK) {
+        delay(500);
+        CONSOLE(".");
+        now = time(nullptr);
+    }
+    struct tm timeinfo;
+    gmtime_r(&now, &timeinfo);
+    CONSOLELN(""); CONSOLE(F("Current time set to: ")); CONSOLELN(asctime(&timeinfo));
+    _doc["time"] = time(nullptr);
+    if (now > TIMECHECK){
+        return true;
+    }
+    else {
+        return false;
+    }
+}
 
-bool SenderClass::mqttConnect(const String &server, uint16_t port, const String &name, const String &username, const String &password) {
-    _mqttClient.setClient(_client);
+bool SenderClass::mqttConnect(const String &server, uint16_t port, const String &name, const String &username, const String &password, const bool secure, const char CACert[], const char deviceCert[], const char deviceKey[]) {
+
+    if (secure) {
+        if (!RTCSyncToNTP()){
+            CONSOLELN(F("ERROR - Time failed to be set. Secure connection will fail."));
+        }
+        // Configure the secure client
+        BearSSL::X509List cert(CACert);
+        _secureClient.setTrustAnchors(&cert);
+        BearSSL::X509List client_crt(deviceCert);
+        BearSSL::PrivateKey key(deviceKey);
+        _secureClient.setClientRSACert(&client_crt, &key);
+        _secureClient.setBufferSizes(512, 512);
+        _secureClient.connect(server,port);
+        // Allocate the Secure client to PubSubClient
+        _mqttClient.setClient(_secureClient);
+    }
+    else {
+        // Allocate the noraml WiFi client to the PubSubClient
+        _mqttClient.setClient(_client);
+    }
     _mqttClient.setServer(server.c_str(), port);
     _mqttClient.setCallback([this](char *topic, byte *payload, unsigned int length) { this->mqttCallback(topic, payload, length); });
 
@@ -146,6 +194,25 @@ void SenderClass::mqttCallback(char *topic, byte *payload, unsigned int length)
     }
 }
 
+bool SenderClass::sendSecureMQTT(char CACert[], char deviceCert[], char deviceKey[], String server, uint16_t port, String name, String topic)    //AWS
+{
+    bool response = mqttConnect(server, port, name, "", "", true, CACert, deviceCert, deviceKey);
+
+    if (response)
+    {
+        String json;
+        serializeJson(_doc, json);
+        CONSOLELN("MQTT publish: " + topic);
+        CONSOLELN(("{\"state\":{\"reported\":" + json  + "}, \"key\":\"" + name + "\"}").c_str());
+        _mqttClient.publish(topic.c_str(), ("{\"state\":{\"reported\":" + json  + "}, \"key\":\"" + name + "\"}").c_str());
+    }
+    CONSOLELN(F("Closing MQTT connection"));
+    _mqttClient.disconnect();
+    stopclient();
+    return response;
+
+}
+
 String SenderClass::sendTCP(String server, uint16_t port)
 {
     int timeout = 0;
@@ -208,7 +275,6 @@ bool SenderClass::sendThingSpeak(String token, long Channel)
     return true;
     }
 
-
 bool SenderClass::sendGenericPost(String server, String uri, uint16_t port)
 {
     serializeJson(_doc, Serial);
@@ -506,10 +572,11 @@ bool SenderClass::sendTCONTROL(String server, uint16_t port)
     return true;
 }
 
-//Blynk HTTP was taking 2 seconds longer and did not show in the App
-//when device was connected, therefore best to use their API.
 bool SenderClass::sendBlynk(char* token)
 {
+    //Blynk HTTP was taking 2 seconds longer and did not show in the App
+    //when device was connected, therefore best to use their API.
+
     serializeJson(_doc, Serial);
 
     Blynk.config(token);

pio/lib/Sender/Sender.h

@@ -12,6 +12,7 @@
 #include <ESP8266HTTPClient.h>
 #include <ArduinoJson.h>
 #include <PubSubClient.h>
+#include <WiFiClientSecure.h>
 
 class SenderClass
 {
@@ -28,19 +29,22 @@ class SenderClass
   bool sendTCONTROL(String server, uint16_t port);
   bool sendBlynk(char* token);
   bool sendBrewblox(String server, uint16_t port, String topic, String username, String password, String name);
+  bool sendSecureMQTT(char CACert[], char deviceCert[], char deviceKey[], String server, uint16_t port, String name, String topic);    //AWS
   void add(String id, float value);
   void add(String id, String value);
   void add(String id, int32_t value);
   void add(String id, uint32_t value);
   void stopclient();
+  bool RTCSyncToNTP();
   void mqttCallback(char *topic, byte *payload, unsigned int length);
-  bool mqttConnect(const String &server, uint16_t port, const String &name, const String &username, const String &password);
+  bool mqttConnect(const String &server, uint16_t port, const String &name, const String &username, const String &password, const bool secure = false, const char CACert[] = "", const char deviceCert[] = "", const char deviceKey[] = "");
   // ~SenderClass();
 
 private:
   WiFiClient _client;
   PubSubClient _mqttClient;
-  StaticJsonDocument<1024> _doc;
+  StaticJsonDocument<256> _doc;
+  WiFiClientSecure _secureClient;
 };
 
 #endif

pio/lib/WiFiManagerKT/WiFiManagerKT.h

@@ -41,20 +41,21 @@ const char HTTP_STYLE[] PROGMEM = "<style>body,textarea,input,select{background:
 const char HTTP_SCRIPT[] PROGMEM = R"V0G0N(
 <script>
 var lAPI = [
-{"name":"Ubidots",    "token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"empty",      "token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"CraftBeerPi","token":0,"server":1,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"HTTP",       "token":1,"server":1,"uri":1,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"TControl",   "token":0,"server":1,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"FHEM",       "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"TCP",        "token":1,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"iSpindel.de","token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"InfluxDB",   "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":1,"username":1,"password":1,"job":0,"instance":0},
-{"name":"Prometheus", "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":1,"instance":1},
-{"name":"MQTT",       "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":1,"password":1,"job":0,"instance":0},
-{"name":"ThingSpeak", "token":1,"server":0,"uri":0,"port":0,"channel":1,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"Blynk",      "token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0},
-{"name":"Brewblox",   "token":0,"server":1,"uri":1,"port":1,"channel":0,"db":0,"username":1,"password":1,"job":0,"instance":0}];
+{"name":"Ubidots",    "token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"empty",      "token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"CraftBeerPi","token":0,"server":1,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"HTTP",       "token":1,"server":1,"uri":1,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"TControl",   "token":0,"server":1,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"FHEM",       "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"TCP",        "token":1,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"iSpindel.de","token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"InfluxDB",   "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":1,"username":1,"password":1,"job":0,"instance":0,"warning1":0},
+{"name":"Prometheus", "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":1,"instance":1,"warning1":0},
+{"name":"MQTT",       "token":0,"server":1,"uri":0,"port":1,"channel":0,"db":0,"username":1,"password":1,"job":0,"instance":0,"warning1":0},
+{"name":"ThingSpeak", "token":1,"server":0,"uri":0,"port":0,"channel":1,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"Blynk",      "token":1,"server":0,"uri":0,"port":0,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":0},
+{"name":"Brewblox",   "token":0,"server":1,"uri":1,"port":1,"channel":0,"db":0,"username":1,"password":1,"job":0,"instance":0,"warning1":0},
+{"name":"AWSIOTMQTT", "token":0,"server":1,"uri":1,"port":1,"channel":0,"db":0,"username":0,"password":0,"job":0,"instance":0,"warning1":1}];
 var $ = function (id) { return document.getElementById(id); };
 var labels = document.getElementsByTagName('LABEL');
 function set(id, show) {

pio/src/iSpindel.cpp

@@ -23,6 +23,7 @@ All rights reserverd by S.Lang <[email protected]>
 #include <ESP8266WiFi.h> //https://github.com/esp8266/Arduino
 #include <FS.h>          //this needs to be first
 #include "tinyexpr.h"
+#include "secrets.h"                          //AWS - Currently a file for Keys, Certs, etc - Need to make this a captured variable for iSpindle
 
 #include "Sender.h"
 // !DEBUG 1
@@ -360,6 +361,8 @@ bool startConfiguration()
   WiFiManagerParameter custom_tempscale("tempscale", "tempscale",
                                         String(my_tempscale).c_str(),
                                         5, TYPE_HIDDEN, WFM_NO_LABEL);
+  WiFiManagerParameter custom_warning1("warning1","WARNING! Secure MQTT has a big impact on battery usage.<BR>&nbsp;<BR>For AWS:<UL><LI>Name must be Thingname</LI><LI>Server must be Endpoint</LI><LI>Port must be 8883</LI><LI>Path/URI is Publish Topic</LI></UL>",
+                                        "<<<<< >>>>>",TKIDSIZE);
 
   wifiManager.addParameter(&custom_name);
   wifiManager.addParameter(&custom_sleep);
@@ -375,6 +378,7 @@ bool startConfiguration()
   wifiManager.addParameter(&api_list);
   wifiManager.addParameter(&custom_api);
 
+  wifiManager.addParameter(&custom_warning1);
   wifiManager.addParameter(&custom_token);
   wifiManager.addParameter(&custom_server);
   wifiManager.addParameter(&custom_port);
@@ -559,6 +563,22 @@ bool uploadData(uint8_t service)
   }
 #endif
 
+#ifdef API_AWSIOTMQTT             //AWS
+  if (service == DTAWSIOTMQTT)
+  {
+    sender.add("name", my_name);
+    sender.add("tilt", Tilt);
+    sender.add("temperature", scaleTemperature(Temperatur));
+    sender.add("battery", Volt);
+    sender.add("gravity", Gravity);
+    sender.add("interval", my_sleeptime);
+    sender.add("RSSI", WiFi.RSSI());
+    CONSOLELN("Calling AWSIOTMQTT Sender");
+    return sender.sendSecureMQTT(AWS_CERT_CA, AWS_CERT_CRT, AWS_CERT_PRIVATE, my_server, my_port, my_name, my_uri);
+    //AWS - NOTE - Need to replace secrets.h with the relevant parameters
+  }
+#endif
+
 #ifdef API_MQTT
   if (service == DTMQTT)
   {
@@ -1105,6 +1125,7 @@ bool isSafeMode(float _volt)
 bool connectBackupCredentials()
 {
   WiFi.disconnect();
+  WiFi.mode(WIFI_STA);  //suggestion that MQTT connection failures can happen if WIFI mode isn't STA.
   WiFi.begin(my_ssid.c_str(), my_psk.c_str());
   CONSOLELN(F("Rescued Wifi credentials"));
 

pio/src/secrets.h

@@ -0,0 +1,28 @@
+#include <pgmspace.h>
+
+// Amazon Root CA 1
+char AWS_CERT_CA[] PROGMEM = R"EOF(
+-----BEGIN CERTIFICATE-----
+MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF
+....
+rqXRfboQnoZsG4q5WTP468SQvvG5
+-----END CERTIFICATE-----
+)EOF";
+
+// Device Certificate
+char AWS_CERT_CRT[] PROGMEM = R"EOF(
+-----BEGIN CERTIFICATE-----
+MIIDWjCCAkKgAwIBAgIVAJWkLwCw4/J6kFdfGOc6Iy7oUdblMA0GCSqGSIb3DQEB
+...
+lO/ftnkLAtFdfdEYNRt4Pwnzz5dMzIy4BFjxIc9TEWUlQfYts+MKeXJZPN1R6Q==
+-----END CERTIFICATE-----
+)EOF";
+
+// Device Private Key
+char AWS_CERT_PRIVATE[] PROGMEM = R"KEY(
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAzSJSdu9I/rp7x2SoIDwor+fANnwc77OY58I6Ayd74abqUP0x
+...
+yWtqFU/uV4LPITfESKnVUUYK39kPRwvuxtDN0pq/mHfD5xnzkLMNPCk=
+-----END RSA PRIVATE KEY-----
+)KEY";