STUN: major code rework (#2116)

Try to have a faster classification, on first packet; use standard extra
dissection data path for sub-classification, metadata extraction and
monitoring.

STUN caches:
* use the proper confidence value
* lookup into the caches only once per flow, after having found a proper
STUN classification

Add identification of Telegram VoIP calls.

.github/workflows/cifuzz.yml

@@ -18,7 +18,7 @@ jobs:
         uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
         with:
           oss-fuzz-project-name: 'ndpi'
-          fuzz-seconds: 1200
+          fuzz-seconds: 1320
           dry-run: false
           sanitizer: ${{ matrix.sanitizer }}
       - name: Check Crash (fails when a crash is detected)

.gitignore

@@ -76,6 +76,8 @@
 /fuzz/fuzz_binaryfusefilter
 /fuzz/fuzz_tls_certificate
 /fuzz/fuzz_dga
+/fuzz/fuzz_is_stun_udp
+/fuzz/fuzz_is_stun_tcp
 /fuzz/fuzz_ds_cmsketch
 /fuzz/fuzz_gcrypt_light
 /fuzz/fuzz_gcrypt_aes
@@ -91,6 +93,8 @@
 /fuzz/fuzz_quic_get_crypto_data_seed_corpus.zip
 /fuzz/fuzz_config_seed_corpus.zip
 /fuzz/fuzz_community_id_seed_corpus.zip
+/fuzz/fuzz_is_stun_udp_seed_corpus.zip
+/fuzz/fuzz_is_stun_tcp_seed_corpus.zip
 /fuzz/fuzz_serialization_seed_corpus.zip
 /fuzz/fuzz_alg_ses_des_seed_corpus.zip
 /fuzz/fuzz_alg_bins_seed_corpus.zip

doc/protocols.rst

@@ -179,4 +179,13 @@ References: `Main site <https://ethereum.org/en/developers/docs/intro-to-ethereu
 
 Notes:
 
-- same as Bitcoin, not each crypto exchange is a mining, it could be a normal transaction, sending or receving or even blockchain exploration.
+- same as Bitcoin, not each crypto exchange is a mining, it could be a normal transaction, sending or receving or even blockchain exploration.
+
+
+.. _Proto 355:
+
+`NDPI_PROTOCOL_TELEGRAM_VOIP`
+============================
+Audio/video calls made using the Telegram app.
+
+References: `Wikipedia <https://en.wikipedia.org/wiki/Telegram_(software)/>`_.

fuzz/Makefile.am

@@ -1,4 +1,4 @@
-bin_PROGRAMS = fuzz_process_packet fuzz_ndpi_reader fuzz_ndpi_reader_alloc_fail fuzz_ndpi_reader_payload_analyzer fuzz_quic_get_crypto_data fuzz_config fuzz_community_id fuzz_serialization fuzz_tls_certificate fuzz_dga
+bin_PROGRAMS = fuzz_process_packet fuzz_ndpi_reader fuzz_ndpi_reader_alloc_fail fuzz_ndpi_reader_payload_analyzer fuzz_quic_get_crypto_data fuzz_config fuzz_community_id fuzz_serialization fuzz_tls_certificate fuzz_dga fuzz_is_stun_udp fuzz_is_stun_tcp
 #Alghoritms
 bin_PROGRAMS += fuzz_alg_bins fuzz_alg_hll fuzz_alg_hw_rsi_outliers_da fuzz_alg_jitter fuzz_alg_ses_des fuzz_alg_crc32_md5 fuzz_alg_bytestream
 #Data structures
@@ -412,6 +412,32 @@ fuzz_dga_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
     $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
     $(fuzz_dga_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
 
+fuzz_is_stun_udp_SOURCES = fuzz_is_stun.c fuzz_common_code.c
+fuzz_is_stun_udp_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
+fuzz_is_stun_udp_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
+fuzz_is_stun_udp_LDFLAGS = $(LIBS)
+if HAS_FUZZLDFLAGS
+fuzz_is_stun_udp_CFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_is_stun_udp_LDFLAGS += $(LIB_FUZZING_ENGINE)
+endif
+# force usage of CXX for linker
+fuzz_is_stun_udp_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+    $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
+    $(fuzz_is_stun_udp_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
+
+fuzz_is_stun_tcp_SOURCES = fuzz_is_stun.c fuzz_common_code.c
+fuzz_is_stun_tcp_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) -DSTUN_TCP
+fuzz_is_stun_tcp_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
+fuzz_is_stun_tcp_LDFLAGS = $(LIBS)
+if HAS_FUZZLDFLAGS
+fuzz_is_stun_tcp_CFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_is_stun_tcp_LDFLAGS += $(LIB_FUZZING_ENGINE)
+endif
+# force usage of CXX for linker
+fuzz_is_stun_tcp_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+    $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
+    $(fuzz_is_stun_tcp_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
+
 fuzz_gcrypt_light_SOURCES = fuzz_gcrypt_light.cpp fuzz_common_code.c
 fuzz_gcrypt_light_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) -I ../src/lib/third_party/include/
 fuzz_gcrypt_light_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) -I ../src/lib/third_party/include/
@@ -555,10 +581,17 @@ fuzz_serialization_seed_corpus.zip: $(files_corpus_fuzz_serialization)
 	zip -j fuzz_serialization_seed_corpus.zip $(files_corpus_fuzz_serialization)
 
 files_corpus_fuzz_community_id :=  $(wildcard corpus/fuzz_community_id/*)
-
 fuzz_community_id_seed_corpus.zip: $(files_corpus_fuzz_community_id)
 	zip -j fuzz_community_id_seed_corpus.zip $(files_corpus_fuzz_community_id)
 
+files_corpus_fuzz_is_stun_udp :=  $(wildcard corpus/fuzz_is_stun_udp/*)
+fuzz_is_stun_udp_seed_corpus.zip: $(files_corpus_fuzz_is_stun_udp)
+	zip -j fuzz_is_stun_udp_seed_corpus.zip $(files_corpus_fuzz_is_stun_udp)
+
+files_corpus_fuzz_is_stun_tcp :=  $(wildcard corpus/fuzz_is_stun_tcp/*)
+fuzz_is_stun_tcp_seed_corpus.zip: $(files_corpus_fuzz_is_stun_tcp)
+	zip -j fuzz_is_stun_tcp_seed_corpus.zip $(files_corpus_fuzz_is_stun_tcp)
+
 files_corpus_fuzz_ds_patricia :=  $(wildcard corpus/fuzz_ds_patricia/*)
 fuzz_ds_patricia_seed_corpus.zip: $(files_corpus_fuzz_ds_patricia)
 	zip -j fuzz_ds_patricia_seed_corpus.zip $(files_corpus_fuzz_ds_patricia)
@@ -639,7 +672,7 @@ files_corpus_fuzz_ds_domain_classify :=  $(wildcard corpus/fuzz_ds_domain_classi
 fuzz_ds_domain_classify_seed_corpus.zip: $(files_corpus_fuzz_ds_domain_classify)
 	zip -j fuzz_ds_domain_classify_seed_corpus.zip $(files_corpus_fuzz_ds_domain_classify)
 
-corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_config_seed_corpus.zip fuzz_ds_patricia_seed_corpus.zip fuzz_ds_ahocorasick_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_hw_rsi_outliers_da_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_ds_tree_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip
+corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_config_seed_corpus.zip fuzz_ds_patricia_seed_corpus.zip fuzz_ds_ahocorasick_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_hw_rsi_outliers_da_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_ds_tree_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_is_stun_udp_seed_corpus.zip fuzz_is_stun_tcp_seed_corpus.zip
 	cp corpus/fuzz_*seed_corpus.zip .
 
 #Create dictionaries exactly as expected by oss-fuzz.
@@ -672,6 +705,8 @@ distdir:
 		-o -path './corpus/fuzz_readerutils_workflow/*' \
 		-o -path './corpus/fuzz_readerutils_parseprotolist/*' \
 		-o -path './corpus/fuzz_config/*' \
+		-o -path './corpus/fuzz_is_stun_udp/*' \
+		-o -path './corpus/fuzz_is_stun_tcp/*' \
 		-o -path './corpus/fuzz_serialization/*' \
 		-o -path './corpus/fuzz_community_id/*' \
 		-o -path './corpus/fuzz_libinjection/*' \

fuzz/fuzz_is_stun.c

@@ -0,0 +1,38 @@
+#define NDPI_LIB_COMPILATION
+
+#include "ndpi_api.h"
+#include "fuzz_common_code.h"
+
+static struct ndpi_detection_module_struct *ndpi_struct = NULL;
+static struct ndpi_flow_struct ndpi_flow;
+#ifdef STUN_TCP
+struct ndpi_tcphdr tcph;
+#else
+struct ndpi_udphdr udph;
+#endif
+
+extern int is_stun(struct ndpi_detection_module_struct *ndpi_struct,
+                   struct ndpi_flow_struct *flow,
+                   u_int16_t *app_proto);
+
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  u_int16_t app_proto; /* unused */
+  struct ndpi_packet_struct *packet;
+
+  if (ndpi_struct == NULL) {
+    fuzz_init_detection_module(&ndpi_struct);
+  }
+
+  packet = &ndpi_struct->packet;
+  packet->payload = data;
+  packet->payload_packet_len = size;
+#ifndef STUN_TCP
+  packet->udp = &udph;
+#else
+  packet->tcp = &tcph;
+#endif
+
+  is_stun(ndpi_struct, &ndpi_flow, &app_proto);
+  return 0;
+}

src/include/ndpi_main.h

@@ -84,6 +84,9 @@ extern "C" {
 				  u_int16_t lower_detected_protocol,
 				  ndpi_confidence_t confidence);
 
+  void ndpi_reset_detected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
+				    struct ndpi_flow_struct *flow);
+
   void ndpi_set_detected_protocol_keeping_master(struct ndpi_detection_module_struct *ndpi_str,
 						 struct ndpi_flow_struct *flow,
 						 u_int16_t detected_protocol,

src/include/ndpi_protocol_ids.h

@@ -383,6 +383,7 @@ typedef enum {
   NDPI_PROTOCOL_CAN                   = 352,
   NDPI_PROTOCOL_PROTOBUF              = 353,
   NDPI_PROTOCOL_ETHEREUM              = 354, 
+  NDPI_PROTOCOL_TELEGRAM_VOIP         = 355,
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"

src/include/ndpi_typedefs.h

@@ -1533,7 +1533,7 @@ struct ndpi_flow_struct {
   } kerberos_buf;
 
   struct {
-    u_int8_t num_pkts, num_binding_requests, num_processed_pkts, maybe_dtls;
+    u_int8_t maybe_dtls;
   } stun;
 
   struct {

src/lib/ndpi_main.c

@@ -2176,6 +2176,11 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			        "ETHEREUM", NDPI_PROTOCOL_CATEGORY_CRYPTO_CURRENCY,
 			        ndpi_build_default_ports(ports_a, 30303, 0, 0, 0, 0) /* TCP */,
 			        ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+  ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 1 /* app proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TELEGRAM_VOIP,
+			  "TelegramVoip", NDPI_PROTOCOL_CATEGORY_VOIP,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main.c"
@@ -7082,14 +7087,6 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
   if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN)
     return(ret);
 
-  if((flow->guessed_protocol_id == NDPI_PROTOCOL_STUN) ||
-     (enable_guess &&
-      flow->stun.num_binding_requests > 0 &&
-      flow->stun.num_processed_pkts > 0)) {
-    ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL);
-    ret.app_protocol = flow->detected_protocol_stack[0];
-  }
-
   /* Check some caches */
 
   /* Does it looks like BitTorrent? */
@@ -8582,6 +8579,14 @@ void ndpi_set_detected_protocol(struct ndpi_detection_module_struct *ndpi_str, s
 
 /* ********************************************************************************* */
 
+void ndpi_reset_detected_protocol(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow) {
+  flow->detected_protocol_stack[1] = NDPI_PROTOCOL_UNKNOWN;
+  flow->detected_protocol_stack[0] = NDPI_PROTOCOL_UNKNOWN;
+  flow->confidence = NDPI_CONFIDENCE_UNKNOWN;
+}
+
+/* ********************************************************************************* */
+
 u_int16_t ndpi_get_flow_masterprotocol(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow) {
   return(flow->detected_protocol_stack[1]);
 }

src/lib/ndpi_utils.c

@@ -1373,16 +1373,6 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
     ndpi_serialize_end_of_block(serializer);
     break;
 
-  case NDPI_PROTOCOL_STUN:
-    ndpi_serialize_start_of_block(serializer, "stun");
-    ndpi_serialize_string_uint32(serializer, "num_pkts", flow->stun.num_pkts);
-    ndpi_serialize_string_uint32(serializer, "num_binding_requests",
-                                 flow->stun.num_binding_requests);
-    ndpi_serialize_string_uint32(serializer, "num_processed_pkts",
-                                 flow->stun.num_processed_pkts);
-    ndpi_serialize_end_of_block(serializer);
-    break;
-
   case NDPI_PROTOCOL_TELNET:
     ndpi_serialize_start_of_block(serializer, "telnet");
     ndpi_serialize_string_string(serializer, "username", flow->protos.telnet.username);

src/lib/protocols/rtp.c

@@ -282,19 +282,10 @@ static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
       } else {
         rtp_get_stream_type(payload[1] & 0x7F, &flow->flow_multimedia_type);
 
-	/* Previous pkts were STUN */
-        if(flow->stun.num_binding_requests > 0 ||
-           flow->stun.num_processed_pkts > 0) {
-          NDPI_LOG_INFO(ndpi_struct, "Found RTP (previous traffic was STUN)\n");
-          ndpi_set_detected_protocol(ndpi_struct, flow,
-                                     NDPI_PROTOCOL_RTP, NDPI_PROTOCOL_STUN,
-                                     NDPI_CONFIDENCE_DPI);
-        } else {
-          NDPI_LOG_INFO(ndpi_struct, "Found RTP\n");
-          ndpi_set_detected_protocol(ndpi_struct, flow,
-                                     NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_RTP,
-                                     NDPI_CONFIDENCE_DPI);
-	}
+        NDPI_LOG_INFO(ndpi_struct, "Found RTP\n");
+        ndpi_set_detected_protocol(ndpi_struct, flow,
+                                   NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_RTP,
+                                   NDPI_CONFIDENCE_DPI);
       }
       return;
     }