Add support to opportunistic TLS

A lot of protocols provide the feature to upgrade their plain text
connections to an encrypted one, via some kind of "STARTTLS" command.

Add generic code to support this extension, and allow dissection of the
entire TLS handshake.

As examples, SMTP, POP, IMAP and FTP dissectors have been updated.

Since this feature requires to process more packets per flow, add the
possibility to disable it.

Fix some log messages.

Slight improvement on TCP sequence number tracking.

As a side effect, this commit fix also a memory leak found by
oss-fuzzer
```
==108966==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 22 byte(s) in 1 object(s) allocated from:
    #0 0x55f8b367a0be in malloc (/home/ivan/svnrepos/nDPI/fuzz/fuzz_ndpi_reader_with_main+0x5480be) (BuildId: 94debacb4a6784c30420ab748c8bf3cc59621063)
    #1 0x55f8b36e1345 in ndpi_malloc_wrapper /home/ivan/svnrepos/nDPI/example/reader_util.c:321:10
    #2 0x55f8b379c7d2 in ndpi_malloc /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:212:25
    #3 0x55f8b379cb18 in ndpi_strdup /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:279:13
    #4 0x55f8b386ce46 in processClientServerHello /home/ivan/svnrepos/nDPI/src/lib/protocols/tls.c:2153:34
    #5 0x55f8b385ebf7 in processTLSBlock /home/ivan/svnrepos/nDPI/src/lib/protocols/tls.c:867:5
    #6 0x55f8b39e708c in ndpi_extra_search_mail_smtp_tcp /home/ivan/svnrepos/nDPI/src/lib/protocols/mail_smtp.c:422:9
    #7 0x55f8b37e636c in ndpi_process_extra_packet /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:5884:9
    #8 0x55f8b37edc05 in ndpi_detection_process_packet /home/ivan/svnrepos/nDPI/src/lib/ndpi_main.c:6276:5
    #9 0x55f8b3701ffc in packet_processing /home/ivan/svnrepos/nDPI/example/reader_util.c:1619:31
    #10 0x55f8b36faf14 in ndpi_workflow_process_packet /home/ivan/svnrepos/nDPI/example/reader_util.c:2189:10
    #11 0x55f8b36b6a50 in LLVMFuzzerTestOneInput /home/ivan/svnrepos/nDPI/fuzz/fuzz_ndpi_reader.c:107:7

```
See: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=50765

example/reader_util.c

@@ -1214,6 +1214,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
           || is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_SMTPS)
           || is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_IMAPS)
           || is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_POPS)
+          || is_ndpi_proto(flow, NDPI_PROTOCOL_FTPS)
 	  || ((is_quic = is_ndpi_proto(flow, NDPI_PROTOCOL_QUIC)))
 	  ) {
     flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.tls_quic.ssl_version;

src/include/ndpi_api.h.in

@@ -1010,6 +1010,11 @@ extern "C" {
 			       lru_cache_type cache_type,
 			       struct ndpi_lru_cache_stats *stats);
 
+  int ndpi_set_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
+				 u_int16_t proto, int value);
+  int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
+				 u_int16_t proto);
+
   /**
    * Find a protocol id associated with a string automata
    *

src/include/ndpi_protocol_ids.h

@@ -339,6 +339,7 @@ typedef enum {
   NDPI_PROTOCOL_TIVOCONNECT           = 308,
   NDPI_PROTOCOL_KISMET                = 309,
   NDPI_PROTOCOL_FASTCGI               = 310,
+  NDPI_PROTOCOL_FTPS                  = 311,
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"

src/include/ndpi_typedefs.h

@@ -1208,6 +1208,11 @@ struct ndpi_detection_module_struct {
 
   /* *** If you add a new LRU cache, please update lru_cache_type above! *** */
 
+  int opportunistic_tls_smtp_enabled;
+  int opportunistic_tls_imap_enabled;
+  int opportunistic_tls_pop_enabled;
+  int opportunistic_tls_ftp_enabled;
+
   ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
 
   u_int8_t direction_detect_disable:1, /* disable internal detection of packet direction */ _pad:7;
@@ -1395,7 +1400,7 @@ struct ndpi_flow_struct {
         char *esni;
       } encrypted_sni;
       ndpi_cipher_weakness server_unsafe_cipher;
-    } tls_quic; /* Used also by DTLS and POPS/IMAPS/SMTPS */
+    } tls_quic; /* Used also by DTLS and POPS/IMAPS/SMTPS/FTPS */
 
     struct {
       char client_signature[48], server_signature[48];

src/lib/ndpi_main.c

@@ -1974,6 +1974,10 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
                           "FastCGI", NDPI_PROTOCOL_CATEGORY_NETWORK,
                           ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
                           ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+  ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_UNSAFE, NDPI_PROTOCOL_FTPS,
+			  "FTPS", NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main.c"
@@ -2759,6 +2763,11 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
     return(NULL);
   }
 
+  ndpi_str->opportunistic_tls_smtp_enabled = 1;
+  ndpi_str->opportunistic_tls_imap_enabled = 1;
+  ndpi_str->opportunistic_tls_pop_enabled = 1;
+  ndpi_str->opportunistic_tls_ftp_enabled = 1;
+
   ndpi_init_protocol_defaults(ndpi_str);
 
   if(ndpi_callback_init(ndpi_str)) {
@@ -4924,7 +4933,8 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) {
        flow_is_proto(flow, NDPI_PROTOCOL_DTLS) ||
        flow_is_proto(flow, NDPI_PROTOCOL_MAIL_SMTPS) ||
        flow_is_proto(flow, NDPI_PROTOCOL_MAIL_POPS) ||
-       flow_is_proto(flow, NDPI_PROTOCOL_MAIL_IMAPS)) {
+       flow_is_proto(flow, NDPI_PROTOCOL_MAIL_IMAPS) ||
+       flow_is_proto(flow, NDPI_PROTOCOL_FTPS)) {
       if(flow->protos.tls_quic.server_names)
 	ndpi_free(flow->protos.tls_quic.server_names);
 
@@ -5193,15 +5203,17 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
         }
       }
 
-      if((flow->next_tcp_seq_nr[0] == 0 && flow->next_tcp_seq_nr[1] == 0) ||
-	 (flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0)) {
+      if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
+	 (tcph->syn && flow->packet_counter == 0)) {
 	/* initialize tcp sequence counters */
 	/* the ack flag needs to be set to get valid sequence numbers from the other
 	 * direction. Usually it will catch the second packet syn+ack but it works
 	 * also for asymmetric traffic where it will use the first data packet
 	 *
 	 * if the syn flag is set add one to the sequence number,
 	 * otherwise use the payload length.
+	 *
+	 * If we receive multiple syn-ack (before any real data), keep the last one
 	 */
 	if(tcph->ack != 0) {
 	  flow->next_tcp_seq_nr[packet->packet_direction] =
@@ -6246,9 +6258,8 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
   u_int32_t num_calls = 0;
   ndpi_protocol ret = { flow->detected_protocol_stack[1], flow->detected_protocol_stack[0], flow->category, NULL };
 
-  if(ndpi_str->ndpi_log_level >= NDPI_LOG_TRACE)
-    NDPI_LOG(flow ? flow->detected_protocol_stack[0] : NDPI_PROTOCOL_UNKNOWN, ndpi_str, NDPI_LOG_TRACE,
-	     "START packet processing\n");
+  NDPI_LOG_DBG(ndpi_str, "[%d/%d] START packet processing\n",
+               flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
 
   if(flow == NULL)
     return(ret);
@@ -8889,3 +8900,51 @@ int ndpi_seen_flow_beginning(const struct ndpi_flow_struct *flow)
     return 0;
   return 1;
 }
+
+/* ******************************************************************** */
+
+int ndpi_set_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
+			       u_int16_t proto, int value)
+{
+  if(!ndpi_struct || (value != 0 && value != 1))
+    return -1;
+
+  switch(proto) {
+  case NDPI_PROTOCOL_MAIL_SMTP:
+    ndpi_struct->opportunistic_tls_smtp_enabled = value;
+    return 0;
+  case NDPI_PROTOCOL_MAIL_IMAP:
+    ndpi_struct->opportunistic_tls_imap_enabled = value;
+    return 0;
+  case NDPI_PROTOCOL_MAIL_POP:
+    ndpi_struct->opportunistic_tls_pop_enabled = value;
+    return 0;
+  case NDPI_PROTOCOL_FTP_CONTROL:
+    ndpi_struct->opportunistic_tls_ftp_enabled = value;
+    return 0;
+  default:
+    return -1;
+  }
+}
+
+/* ******************************************************************** */
+
+int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
+			       u_int16_t proto)
+{
+  if(!ndpi_struct)
+    return -1;
+
+  switch(proto) {
+  case NDPI_PROTOCOL_MAIL_SMTP:
+    return ndpi_struct->opportunistic_tls_smtp_enabled;
+  case NDPI_PROTOCOL_MAIL_IMAP:
+    return ndpi_struct->opportunistic_tls_imap_enabled;
+  case NDPI_PROTOCOL_MAIL_POP:
+    return ndpi_struct->opportunistic_tls_pop_enabled;
+  case NDPI_PROTOCOL_FTP_CONTROL:
+    return ndpi_struct->opportunistic_tls_ftp_enabled;
+  default:
+    return -1;
+  }
+}

src/lib/protocols/ftp_control.c

@@ -29,6 +29,9 @@
 
 // #define FTP_DEBUG
 
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
+
 /* *************************************************************** */
 
 static void ndpi_int_ftp_control_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
@@ -643,10 +646,21 @@ static void ndpi_check_ftp_control(struct ndpi_detection_module_struct *ndpi_str
 
 	if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0' &&
 	   flow->l4.tcp.ftp_imap_pop_smtp.auth_done == 0 &&
-	   flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) /* TODO: any values on dissecting TLS handshake? */
+	   flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 0) {
 	  flow->ftp_control_stage = 0;
-	else
+	} else if (flow->l4.tcp.ftp_imap_pop_smtp.auth_tls == 1 &&
+		   ndpi_struct->opportunistic_tls_ftp_enabled) {
+	  flow->host_server_name[0] = '\0'; /* Remove any data set by other dissectors (eg. SMTP) */
+	  /* Switch classification to FTPS */
+	  ndpi_set_detected_protocol(ndpi_struct, flow,
+				     NDPI_PROTOCOL_FTPS, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+	  NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+		       flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+	  /* We are done (in FTP dissector): delegating TLS... */
+	  switch_extra_dissection_to_tls(ndpi_struct, flow);
+	} else {
 	  ndpi_int_ftp_control_add_connection(ndpi_struct, flow);
+	}
       } else {
 	NDPI_LOG_DBG2(ndpi_struct, "The reply did not seem to belong to FTP_CONTROL, "
 		      "resetting the stage to 0\n");

src/lib/protocols/mail_imap.c

@@ -30,6 +30,9 @@
 
 /* #define IMAP_DEBUG 1*/
 
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
+
 static void ndpi_int_mail_imap_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow,
 					      u_int16_t protocol) {
   flow->guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN; /* Avoid IMAPS to be used s sub-protocol */
@@ -51,13 +54,6 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
   printf("%s() [%.*s]\n", __FUNCTION__, packet->payload_packet_len, packet->payload);
 #endif
 
-  if(flow->l4.tcp.mail_imap_starttls == 2) {
-    NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
-    NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MAIL_IMAP);
-    NDPI_DEL_PROTOCOL_FROM_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_TLS);
-    return;
-  }
-
   if(packet->payload_packet_len >= 4 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
     // the DONE command appears without a tag
     if(packet->payload_packet_len == 6 && ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
@@ -113,8 +109,17 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && (packet->payload[command_start + 1] == 'K' || packet->payload[command_start + 1] == 'k')
 	    && packet->payload[command_start + 2] == ' ') {
 	  flow->l4.tcp.mail_imap_stage += 1;
-	  if(flow->l4.tcp.mail_imap_starttls == 1)
-	    flow->l4.tcp.mail_imap_starttls = 2;
+	  if(flow->l4.tcp.mail_imap_starttls == 1) {
+	    NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
+	    ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS);
+	    if(ndpi_struct->opportunistic_tls_imap_enabled) {
+	      NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+			   flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+	      /* We are done (in IMAP dissector): delegating TLS... */
+	      switch_extra_dissection_to_tls(ndpi_struct, flow);
+	      return;
+	    }
+	  }
 	  saw_command = 1;
 	} else if((packet->payload[command_start] == 'U' || packet->payload[command_start] == 'u')
 		   && (packet->payload[command_start + 1] == 'I' || packet->payload[command_start + 1] == 'i')
@@ -126,7 +131,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && packet->payload[command_start + 2] == ' ') {
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  if(flow->l4.tcp.mail_imap_starttls == 1)
-	    flow->l4.tcp.mail_imap_starttls = 2;
+	    flow->l4.tcp.mail_imap_starttls = 0;
 	  saw_command = 1;
 	}
       }
@@ -156,7 +161,6 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	    && (packet->payload[command_start + 7] == 'S' || packet->payload[command_start + 7] == 's')) {
         flow->l4.tcp.mail_imap_stage += 1;
         flow->l4.tcp.mail_imap_starttls = 1;
-        ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS);
         saw_command = 1;
 	}
       }
@@ -242,7 +246,6 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
 	  flow->l4.tcp.mail_imap_stage += 1;
 	  /* Authenticate phase may have multiple messages. Ignore them since they are
 	     somehow encrypted anyway. */
-	  flow->l4.tcp.mail_imap_starttls = 2;
           ndpi_int_mail_imap_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_IMAPS);
 	  saw_command = 1;
 	}

src/lib/protocols/mail_pop.c

@@ -43,12 +43,16 @@
 #define POP_BIT_STLS		0x0400
 
 
+extern void switch_extra_dissection_to_tls(struct ndpi_detection_module_struct *ndpi_struct,
+					   struct ndpi_flow_struct *flow);
+
 static void ndpi_int_mail_pop_add_connection(struct ndpi_detection_module_struct
-					     *ndpi_struct, struct ndpi_flow_struct *flow) {
+					     *ndpi_struct, struct ndpi_flow_struct *flow,
+					     u_int16_t protocol) {
 
   NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n");
   flow->guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN; /* Avoid POP3S to be used s sub-protocol */
-  ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
+  ndpi_set_detected_protocol(ndpi_struct, flow, protocol, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
 }
 
 /* **************************************** */
@@ -142,6 +146,7 @@ static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_mod
 	       && (packet->payload[2] == 'L' || packet->payload[2] == 'l')
 	       && (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
       flow->l4.tcp.pop_command_bitmask |= POP_BIT_STLS;
+      flow->l4.tcp.mail_imap_starttls = 1;
       return 1;
     }
   }
@@ -168,6 +173,19 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
 	      && (packet->payload[3] == 'R' || packet->payload[3] == 'r')))) {
     // +OK or -ERR seen
     flow->l4.tcp.mail_pop_stage += 1;
+    if(packet->payload[0] == '+' && flow->l4.tcp.mail_imap_starttls == 1) {
+      NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
+      ndpi_int_mail_pop_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POPS);
+      if(ndpi_struct->opportunistic_tls_pop_enabled) {
+        NDPI_LOG_DBG(ndpi_struct, "Switching to [%d/%d]\n",
+		     flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
+	/* We are done (in POP dissector): delegating TLS... */
+	switch_extra_dissection_to_tls(ndpi_struct, flow);
+	return;
+      }
+    }
+    if(packet->payload[0] == '-' && flow->l4.tcp.mail_imap_starttls == 1)
+      flow->l4.tcp.mail_imap_starttls = 0;
   } else if(!ndpi_int_mail_pop_check_for_client_commands(ndpi_struct, flow)) {
     goto maybe_split_pop;
   }
@@ -189,7 +207,7 @@ void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
 
 	if((flow->l4.tcp.ftp_imap_pop_smtp.password[0] != '\0')
 	   || (flow->l4.tcp.mail_pop_stage > 3)) {
-	  ndpi_int_mail_pop_add_connection(ndpi_struct, flow);
+	  ndpi_int_mail_pop_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POP);
 	  if(flow->l4.tcp.ftp_imap_pop_smtp.password[0] == '\0')
 	    popInitExtraPacketProcessing(flow);
 	}