Merge pull request #1907 from aws/prasad/OTA_Suspend_Resume

OTA support for resume on MQTT re-connection and Suspend/Resume APIs

libraries/freertos_plus/aws/ota/include/aws_iot_ota_agent.h

@@ -106,6 +106,7 @@ typedef enum
     eOTA_AgentState_RequestingFileBlock,
     eOTA_AgentState_WaitingForFileBlock,
     eOTA_AgentState_ClosingFile,
+    eOTA_AgentState_Suspended,
     eOTA_AgentState_ShuttingDown,
     eOTA_AgentState_Stopped,
     eOTA_AgentState_All
@@ -128,6 +129,8 @@ typedef enum
     eOTA_AgentEvent_ReceivedFileBlock,
     eOTA_AgentEvent_RequestTimer,
     eOTA_AgentEvent_CloseFile,
+    eOTA_AgentEvent_Suspend,
+    eOTA_AgentEvent_Resume,
     eOTA_AgentEvent_UserAbort,
     eOTA_AgentEvent_Shutdown,
     eOTA_AgentEvent_Max
@@ -514,6 +517,7 @@ typedef struct
 #define kOTA_Err_SelfTestTimerFailed     0x2b000000UL     /*!< Attempt to start self-test timer faield. */
 #define kOTA_Err_EventQueueSendFailed    0x2c000000UL     /*!< Posting event message to the event queue failed. */
 #define kOTA_Err_InvalidDataProtocol     0x2d000000UL     /*!< Job does not have a valid protocol for data transfer. */
+#define kOTA_Err_OTAAgentStopped         0x2e000000UL     /*!< Returned when operations are performed that requires OTA Agent running & its stopped. */
 /* @[define_ota_err_codes] */
 
 /* @[define_ota_err_code_helpers] */
@@ -671,6 +675,24 @@ OTA_ImageState_t OTA_GetImageState( void );
  */
 OTA_Err_t OTA_CheckForUpdate( void );
 
+/* @brief Suspend OTA agent oeprations .
+ *
+ * @param[in]
+ *
+ * @return kOTA_Err_None if successful, otherwise an error code prefixed with 'kOTA_Err_' from the
+ * list above.
+ */
+OTA_Err_t OTA_Suspend( void );
+
+/* @brief Resume OTA agent oeprations .
+ *
+ * @param[in] pxConnection Update connection context.
+ *
+ * @return kOTA_Err_None if successful, otherwise an error code prefixed with 'kOTA_Err_' from the
+ * list above.
+ */
+OTA_Err_t OTA_Resume( void * pxConnection );
+
 /*---------------------------------------------------------------------------*/
 /*							Statistics API									 */
 /*---------------------------------------------------------------------------*/

libraries/freertos_plus/aws/ota/src/aws_iot_ota_agent.c

@@ -259,6 +259,8 @@ static OTA_Err_t prvRequestDataHandler( OTA_EventData_t * pxEventData );
 static OTA_Err_t prvShutdownHandler( OTA_EventData_t * pxEventData );
 static OTA_Err_t prvCloseFileHandler( OTA_EventData_t * pxEventData );
 static OTA_Err_t prvUserAbortHandler( OTA_EventData_t * pxEventData );
+static OTA_Err_t prvSuspendHandler( OTA_EventData_t * pxEventData );
+static OTA_Err_t prvResumeHandler( OTA_EventData_t * pxEventData );
 
 /* OTA default callback initializer. */
 
@@ -315,6 +317,8 @@ OTAStateTableEntry_t OTATransitionTable[] =
     { eOTA_AgentState_WaitingForFileBlock, eOTA_AgentEvent_RequestFileBlock,    prvRequestDataHandler, eOTA_AgentState_WaitingForFileBlock },
     { eOTA_AgentState_WaitingForFileBlock, eOTA_AgentEvent_RequestJobDocument,  prvRequestJobHandler,  eOTA_AgentState_WaitingForJob       },
     { eOTA_AgentState_WaitingForFileBlock, eOTA_AgentEvent_CloseFile,           prvCloseFileHandler,   eOTA_AgentState_WaitingForJob       },
+    { eOTA_AgentState_Suspended,           eOTA_AgentEvent_Resume,              prvResumeHandler,      eOTA_AgentState_RequestingJob       },
+    { eOTA_AgentState_All,                 eOTA_AgentEvent_Suspend,             prvSuspendHandler,     eOTA_AgentState_Suspended           },
     { eOTA_AgentState_All,                 eOTA_AgentEvent_UserAbort,           prvUserAbortHandler,   eOTA_AgentState_WaitingForJob       },
     { eOTA_AgentState_All,                 eOTA_AgentEvent_Shutdown,            prvShutdownHandler,    eOTA_AgentState_ShuttingDown        },
 };
@@ -329,6 +333,7 @@ const char * pcOTA_AgentState_Strings[ eOTA_AgentState_All ] =
     "RequestingFileBlock",
     "WaitingForFileBlock",
     "ClosingFile",
+    "Suspended",
     "ShuttingDown",
     "Stopped"
 };
@@ -344,6 +349,8 @@ const char * pcOTA_Event_Strings[ eOTA_AgentEvent_Max ] =
     "ReceivedFileBlock",
     "RequestTimer",
     "CloseFile",
+    "Suspend",
+    "Resume",
     "UserAbort",
     "Shutdown"
 };
@@ -1125,6 +1132,46 @@ static OTA_Err_t prvShutdownHandler( OTA_EventData_t * pxEventData )
     return kOTA_Err_None;
 }
 
+static OTA_Err_t prvSuspendHandler( OTA_EventData_t * pxEventData )
+{
+    DEFINE_OTA_METHOD_NAME( "prvSuspendHandler" );
+
+    ( void ) pxEventData;
+    OTA_Err_t xErr = kOTA_Err_None;
+
+    /* Log the state change to suspended state.*/
+    OTA_LOG_L1( "[%s] OTA Agent is suspended.\r\n", OTA_METHOD_NAME );
+
+    return xErr;
+}
+
+static OTA_Err_t prvResumeHandler( OTA_EventData_t * pxEventData )
+{
+    DEFINE_OTA_METHOD_NAME( "prvResumeHandler" )
+
+        ( void ) pxEventData;
+    OTA_Err_t xErr = kOTA_Err_None;
+
+    OTA_EventMsg_t xEventMsg = { 0 };
+
+    /*
+     * Update the connection handle before resuming the OTA process.
+     */
+
+    OTA_LOG_L2( "[%s] Updating the connection handle. %d\r\n", OTA_METHOD_NAME );
+
+    xOTA_Agent.pvConnectionContext = pxEventData;
+
+    /*
+     * Send signal to request job document.
+     */
+
+    xEventMsg.xEventId = eOTA_AgentEvent_RequestJobDocument;
+    OTA_SignalEvent( &xEventMsg );
+
+    return xErr;
+}
+
 /*
  * This is a private function only meant to be called by the OTA agent after the
  * currently running image that is in the self test phase rejects the update.
@@ -2946,6 +2993,73 @@ OTA_ImageState_t OTA_GetImageState( void )
     return xOTA_Agent.eImageState;
 }
 
+/*
+ * Suspend OTA Agent task.
+ */
+OTA_Err_t OTA_Suspend( void )
+{
+    DEFINE_OTA_METHOD_NAME( "OTA_Suspend" );
+
+    OTA_Err_t xErr = kOTA_Err_Uninitialized;
+    OTA_EventMsg_t xEventMsg = { 0 };
+
+    /* Stop the request timer. */
+    prvStopRequestTimer();
+
+    /* Check if OTA Agent is running. */
+    if( xOTA_Agent.eState != eOTA_AgentState_Stopped )
+    {
+        /*
+         * Send event to OTA agent task.
+         */
+        xEventMsg.xEventId = eOTA_AgentEvent_Suspend;
+        OTA_SignalEvent( &xEventMsg );
+
+        xErr = kOTA_Err_None;
+    }
+    else
+    {
+        OTA_LOG_L1( "[%s] Error: OTA Agent is not running, cannot suspend.\r\n", OTA_METHOD_NAME );
+
+        xErr = kOTA_Err_OTAAgentStopped;
+    }
+
+    return xErr;
+}
+
+/*
+ * Resume OTA Agent task.
+ */
+OTA_Err_t OTA_Resume( void * pxConnection )
+{
+    DEFINE_OTA_METHOD_NAME( "OTA_Resume" );
+
+    OTA_Err_t xErr = kOTA_Err_Uninitialized;
+    OTA_EventMsg_t xEventMsg = { 0 };
+
+    xEventMsg.pxEventData = pxConnection;
+
+    /* Check if OTA Agent is running. */
+    if( xOTA_Agent.eState != eOTA_AgentState_Stopped )
+    {
+        /*
+         * Send event to OTA agent task.
+         */
+        xEventMsg.xEventId = eOTA_AgentEvent_Resume;
+        OTA_SignalEvent( &xEventMsg );
+
+        xErr = kOTA_Err_None;
+    }
+    else
+    {
+        OTA_LOG_L1( "[%s] Error: OTA Agent is not running, cannot resume.\r\n", OTA_METHOD_NAME );
+
+        xErr = kOTA_Err_OTAAgentStopped;
+    }
+
+    return xErr;
+}
+
 /*-----------------------------------------------------------*/
 
 /* Provide access to private members for testing. */

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_back_to_back_downloads.py

@@ -28,6 +28,10 @@
 
 
 class OtaTestBackToBackDownloads(OtaTestCase):
+    """
+    This test creates 3 consecutive OTA updates. The device is expected to update 3 times in a row.
+    """
+
     is_positive = True
 
     def __buildAndOtaInputVersion(self, x, y, z):

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_back_to_back_switch_protocol.py

@@ -28,6 +28,14 @@
 
 
 class OtaTestBackToBackSwitchProtocol(OtaTestCase):
+    """
+    This test verifies that device is able to switch data transfer protocol between MQTT and HTTP.
+    It first creates an OTA update over MQTT but with invalid signature to ensure that update is
+    done via MQTT but device won't reboot. Then it creates another OTA update over HTTP with valid
+    signature, device should be able to switch to HTTP to finish the update. Then the same process
+    is repeated but with HTTP first, then MQTT.
+    """
+
     is_positive = True
 
     @classmethod

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_default_data_protocol.py

@@ -28,6 +28,16 @@
 
 
 class OtaTestDefaultDataProtocol(OtaTestCase):
+    """
+    When device supports multiple data transfer protocols, it has a default one to use when the
+    update is created with multiple data transfer protocols enabled. This test builds the initial
+    image with MQTT as the default protocol, and flash to the device. Then it builds another image
+    with HTTP as the default protocol, and uses it to create an OTA update with both MQTT and HTTP
+    enabled. The device is expected to finish the update with MQTT. Then another update is created,
+    the device is expected to finish this update with HTTP because the second image sets HTTP as
+    default protocol.
+    """
+
     is_positive = True
 
     @classmethod

tools/ota_e2e_tests/aws_ota_test/aws_ota_test_case_disconnect_cancel_update.py

@@ -0,0 +1,121 @@
+"""
+FreeRTOS
+Copyright (C) 2020 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+http://aws.amazon.com/freertos
+http://www.FreeRTOS.org
+
+"""
+import time
+from pathlib import Path
+import boto3
+from botocore.exceptions import ClientError
+import paho.mqtt.client as mqtt
+from .aws_ota_test_case import OtaTestCase
+from .aws_ota_test_result import OtaTestResult
+
+
+class OtaTestDisconnectCancelUpdate(OtaTestCase):
+    """
+    Tests that device can recover itself if the OTA job gets canceled while it is in suspend state.
+    It does the same thing as OtaTestDisconnectResume test except that after connecting to IoT core
+    which disconnects the device, it will cancel the OTA update. Then a new update will be created.
+    The device is expected to reconnect to the IoT core, abort current update, go back to waiting
+    state, and accept and finish the next update.
+    """
+
+    is_positive = True
+    connected_to_iot = False
+
+    def get_job_exec_status(self, update_id, thing_name):
+        iot_client = boto3.client('iot')
+        exec_status = 'QUEUED'
+        try:
+            response = iot_client.describe_job_execution(jobId=f'AFR_OTA-{update_id}', thingName=thing_name)
+            exec_status = response['execution']['status']
+        except ClientError:
+            print('Could not obtain job status yet')
+        return exec_status
+
+    def connect_to_iot(self, thing_name, certfile, keyfile):
+        def on_connect(client, userdata, flags, rc):
+            print('----------------------------------------\n'
+                  'Connected to IoT core. Device should now disconnect then reconnect.\n'
+                  '----------------------------------------')
+            self.connected_to_iot = True
+            mqtt_client.disconnect()
+            mqtt_client.loop_stop()
+
+        mqtt_client = mqtt.Client(client_id=thing_name)
+        mqtt_client.tls_set(certfile=certfile, keyfile=keyfile)
+        mqtt_client.on_connect = on_connect
+        mqtt_client.connect_async(self._otaAwsAgent.getAwsIotEndpoint(), 8883, 60)
+        mqtt_client.loop_start()
+
+    def run(self):
+        # Increase the version of the OTA image.
+        self._otaProject.setApplicationVersion(0, 9, 1)
+        # Build the OTA image.
+        self._otaProject.buildProject()
+        # Start an OTA Update.
+        otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
+
+        # Wait up to 3 minute until the job is in progress.
+        thing_name = self._otaAwsAgent._iotThing.thing_name
+        exec_status = 'QUEUED'
+        timeout = time.time() + 180
+        while exec_status == 'QUEUED' and time.time() < timeout:
+            exec_status = self.get_job_exec_status(otaUpdateId, thing_name)
+            time.sleep(1)
+        if exec_status == 'QUEUED':
+            return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
+                                 summary='Timeout waiting for OTA job status.')
+
+        # Connect to IoT core with same thing name, this should disconnect the device from IoT core.
+        # Note: currently there's no way to load the cert/key from memory, this is a limitation from
+        # the python ssl module. We have to write them to temporary files.
+        cert = Path('tmp_cert.pem')
+        cert.write_text(self._otaAwsAgent._iotThing.cert)
+        key = Path('tmp_key.pem')
+        key.write_text(self._otaAwsAgent._iotThing.prv_key)
+        self.connect_to_iot(thing_name, str(cert), str(key))
+
+        # Wait up to 10 second until we connect to the IoT core
+        timeout = time.time() + 10
+        while not self.connected_to_iot and time.time() < timeout:
+            time.sleep(1)
+
+        # Delete the temporary credential files.
+        cert.unlink()
+        key.unlink()
+
+        # Cancel the job, device should reconnect, and go back to waiting state.
+        if self.connected_to_iot:
+            iot_client = boto3.client('iot')
+            iot_client.cancel_job_execution(jobId=f'AFR_OTA-{otaUpdateId}', thingName=thing_name, force=True)
+
+            # Do another OTA update, this should succeed.
+            self._otaProject.setApplicationVersion(0, 9, 2)
+            self._otaProject.buildProject()
+            otaUpdateId = self._otaAwsAgent.quickCreateOtaUpdate(self._otaConfig, [self._protocol])
+            return self.getTestResultAfterOtaUpdateCompletion(otaUpdateId)
+        else:
+            return OtaTestResult(testName=self._name, result=OtaTestResult.ERROR,
+                                 summary='Could not disconnect the device from IoT core.')