restapi logger: return array instead of dict - breaking changes (#313)

* convert to array
* be more generic /search

dnsutils/message.go

@@ -157,6 +157,7 @@ type TransformSuspicious struct {
 	UnallowedChars        bool    `json:"unallowed-chars" msgpack:"unallowed-chars"`
 	UncommonQtypes        bool    `json:"uncommon-qtypes" msgpack:"uncommon-qtypes"`
 	ExcessiveNumberLabels bool    `json:"excessive-number-labels" msgpack:"excessive-number-labels"`
+	Domain                string  `json:"domain,omitempty" msgpack:"-"`
 }
 
 type TransformPublicSuffix struct {

doc/swagger.yml

@@ -1,13 +1,13 @@
 openapi: 3.0.2
 info:
   title: Swagger for DNS-collector tool
-  version: 0.28.0
+  version: 0.32.0
   description: This is a swagger for the API of the DNS-collector.
   contact:
     email: [email protected]
   license:
     name: MIT
-    url: 'https://github.com/dmachard/go-dns-collector/blob/main/LICENSE'
+    url: 'https://github.com/dmachard/go-dnscollector/blob/main/LICENSE'
   x-logo:
     url: ''
 servers:
@@ -16,29 +16,19 @@ paths:
   /search:
     get:
       parameters:
-        - in: query
-          name: stream_id
+        - in: filter
+          name: filter
           schema:
             type: string
-          description: stream identity name
-        - in: query
-          name: query_ip
-          schema:
-            type: string
-          description: query ip to search
-        - in: query
-          name: query_name
-          schema:
-            type: string
-          description: query name to search
+          description: domain or address to search
       responses:
         '200':
-          description: Return list of domains founded
+          description: Return list of domains or addresses founded
           content:
             text/plain:
               schema:
                 type: string
-      summary: Return a list of domains
+      summary: Return a list of domains or addresses
   /streams:
     get:
       responses:
@@ -153,12 +143,12 @@ paths:
     get:
       responses:
         '200':
-          description: Rerurn suspicious domains list
+          description: Return suspicious domains list
           content:
             text/plain:
               schema:
                 type: string
       summary: Return suspicious domains list         
 security: []
 externalDocs:
-  url: 'https://github.com/dmachard/go-dns-collector'
+  url: 'https://github.com/dmachard/go-dnscollector'

loggers/restapi.go

@@ -37,6 +37,11 @@ type HitsUniq struct {
 	Suspicious     map[string]*dnsutils.TransformSuspicious
 }
 
+type KeyHit struct {
+	Key string `json:"key"`
+	Hit int    `json:"hit"`
+}
+
 type RestAPI struct {
 	done       chan bool
 	done_api   chan bool
@@ -255,7 +260,14 @@ func (s *RestAPI) GetTLDsHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.HitsUniq.PublicSuffixes)
+		// return as array
+		dataArray := []KeyHit{}
+		for tld, hit := range s.HitsUniq.PublicSuffixes {
+			dataArray = append(dataArray, KeyHit{Key: tld, Hit: hit})
+		}
+
+		// encode
+		json.NewEncoder(w).Encode(dataArray)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -274,7 +286,14 @@ func (s *RestAPI) GetClientsHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.HitsUniq.Clients)
+		// return as array
+		dataArray := []KeyHit{}
+		for address, hit := range s.HitsUniq.Clients {
+			dataArray = append(dataArray, KeyHit{Key: address, Hit: hit})
+		}
+
+		// encode
+		json.NewEncoder(w).Encode(dataArray)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -293,7 +312,14 @@ func (s *RestAPI) GetDomainsHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.HitsUniq.Domains)
+		// return as array
+		dataArray := []KeyHit{}
+		for domain, hit := range s.HitsUniq.Domains {
+			dataArray = append(dataArray, KeyHit{Key: domain, Hit: hit})
+		}
+
+		// encode
+		json.NewEncoder(w).Encode(dataArray)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -312,7 +338,15 @@ func (s *RestAPI) GetNxDomainsHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.HitsUniq.NxDomains)
+		// convert to array
+		dataArray := []KeyHit{}
+		for domain, hit := range s.HitsUniq.NxDomains {
+			dataArray = append(dataArray, KeyHit{Key: domain, Hit: hit})
+		}
+
+		// encode
+		json.NewEncoder(w).Encode(dataArray)
+
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -331,7 +365,14 @@ func (s *RestAPI) GetSfDomainsHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.HitsUniq.SfDomains)
+		// return as array
+		dataArray := []KeyHit{}
+		for domain, hit := range s.HitsUniq.SfDomains {
+			dataArray = append(dataArray, KeyHit{Key: domain, Hit: hit})
+		}
+
+		// encode
+		json.NewEncoder(w).Encode(dataArray)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -350,7 +391,15 @@ func (s *RestAPI) GetSuspiciousHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.HitsUniq.Suspicious)
+		// return as array
+		dataArray := []*dnsutils.TransformSuspicious{}
+		for domain, suspicious := range s.HitsUniq.Suspicious {
+			suspicious.Domain = domain
+			dataArray = append(dataArray, suspicious)
+		}
+
+		// encode
+		json.NewEncoder(w).Encode(dataArray)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -368,53 +417,39 @@ func (s *RestAPI) GetSearchHandler(w http.ResponseWriter, r *http.Request) {
 	switch r.Method {
 	case http.MethodGet:
 
-		streamId := r.URL.Query()["stream_id"]
-		queryIp := r.URL.Query()["query_ip"]
-		queryName := r.URL.Query()["query_name"]
-
-		if len(streamId) == 0 && len(queryIp) == 0 && len(queryName) == 0 {
+		filter := r.URL.Query()["filter"]
+		if len(filter) == 0 {
 			http.Error(w, "Arguments are missing", http.StatusBadRequest)
 		}
 
-		// search in a stream
-		if len(streamId) == 1 {
-			if _, exists := s.HitsStream.Streams[streamId[0]]; exists {
-				stream := s.HitsStream.Streams[streamId[0]]
-
-				if len(queryIp) == 1 && len(queryName) == 1 {
-					if _, exists := stream.Clients[queryIp[0]]; exists {
-						client := stream.Clients[queryIp[0]]
-						if _, domainExists := client.Hits[queryName[0]]; domainExists {
-							w.Header().Set("Content-Type", "application/text")
-							w.Write([]byte(strconv.Itoa(client.Hits[queryName[0]])))
-						} else {
-							http.Error(w, "{\"error\": \"Query Name not found\"}", http.StatusNotFound)
-						}
-					} else {
-						http.Error(w, "{\"error\": \"Query IP not found\"}", http.StatusNotFound)
-					}
+		dataArray := []KeyHit{}
 
-				} else if len(queryIp) == 1 {
-					if _, exists := stream.Clients[queryIp[0]]; exists {
-						w.Header().Set("Content-Type", "application/json")
-						json.NewEncoder(w).Encode(stream.Clients[queryIp[0]])
-					} else {
-						http.Error(w, "{\"error\": \"Query IP not found\"}", http.StatusNotFound)
-					}
+		// search by IP
+		for _, search := range s.HitsStream.Streams {
+			userHits, clientExists := search.Clients[filter[0]]
+			if clientExists {
+				for domain, hit := range userHits.Hits {
+					dataArray = append(dataArray, KeyHit{Key: domain, Hit: hit})
+				}
+			}
+		}
 
-				} else if len(queryName) == 1 {
-					if _, exists := stream.Domains[queryName[0]]; exists {
-						w.Header().Set("Content-Type", "application/json")
-						json.NewEncoder(w).Encode(stream.Domains[queryName[0]])
-					} else {
-						http.Error(w, "{\"error\": \"Query Name not found\"}", http.StatusNotFound)
+		// search by domain
+		if len(dataArray) == 0 {
+			for _, search := range s.HitsStream.Streams {
+				domainHists, domainExists := search.Domains[filter[0]]
+				if domainExists {
+					for addr, hit := range domainHists.Hits {
+						dataArray = append(dataArray, KeyHit{Key: addr, Hit: hit})
 					}
 				}
-
-			} else {
-				http.Error(w, "{\"error\": \"Stream ID not Found\"}", http.StatusNotFound)
 			}
 		}
+
+		// encode to json
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(dataArray)
+
 	default:
 		http.Error(w, "{\"error\": \"Method not allowed\"}", http.StatusMethodNotAllowed)
 	}
@@ -433,7 +468,13 @@ func (s *RestAPI) GetStreamsHandler(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodGet:
-		json.NewEncoder(w).Encode(s.Streams)
+
+		dataArray := []KeyHit{}
+		for stream, hit := range s.Streams {
+			dataArray = append(dataArray, KeyHit{Key: stream, Hit: hit})
+		}
+
+		json.NewEncoder(w).Encode(dataArray)
 	default:
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
 	}
@@ -555,13 +596,12 @@ func (s *RestAPI) ListenAndServe() {
 	mux.HandleFunc("/clients", s.GetClientsHandler)
 	mux.HandleFunc("/clients/top", s.GetTopClientsHandler)
 	mux.HandleFunc("/domains", s.GetDomainsHandler)
+	mux.HandleFunc("/domains/servfail", s.GetSfDomainsHandler)
 	mux.HandleFunc("/domains/top", s.GetTopDomainsHandler)
-	mux.HandleFunc("/domains/nx", s.GetNxDomainsHandler)
 	mux.HandleFunc("/domains/nx/top", s.GetTopNxDomainsHandler)
-	mux.HandleFunc("/domains/servfail", s.GetSfDomainsHandler)
 	mux.HandleFunc("/domains/servfail/top", s.GetTopSfDomainsHandler)
 	mux.HandleFunc("/suspicious", s.GetSuspiciousHandler)
-	mux.HandleFunc("/search", s.GetSearchHandler)
+	mux.HandleFunc("/search/address", s.GetSearchHandler)
 
 	var err error
 	var listener net.Listener