Refactor for TLS handlers

admin/handlers/go.mod

@@ -0,0 +1,3 @@
+module github.com/javuto/osctrl/admin/handlers
+
+go 1.12

admin/handlers/handlers.go

@@ -0,0 +1,10 @@
+package handlers
+
+const (
+	metricJSONReq   = "admin-json-req"
+	metricJSONErr   = "admin-json-err"
+	metricJSONOK    = "admin-json-ok"
+	metricHealthReq = "health-req"
+	metricHealthOK  = "health-ok"
+)
+

admin/main.go

@@ -256,8 +256,6 @@ func main() {
 	// Initialize service settings
 	log.Println("Loading service settings")
 	loadingSettings()
-	// multiple listeners channel
-	finish := make(chan bool)
 
 	// Start SAML Middleware if we are using SAML
 	if adminConfig.Auth == settings.AuthSAML {
@@ -416,6 +414,9 @@ func main() {
 		}
 	}()
 
+	// multiple listeners channel
+	finish := make(chan bool)
+
 	// Launch HTTP server for admin
 	go func() {
 		serviceAdmin := adminConfig.Listener + ":" + adminConfig.Port

deploy/provision.sh

@@ -631,8 +631,14 @@ DEST="$DEST_PATH" make install_cli
 
 # If we are in dev, create environment and enroll host
 if [[ "$MODE" == "dev" ]]; then
-  log "Creating environment for dev"
   __db_conf="$DEST_PATH/config/$DB_CONF"
+
+  # Create admin user
+  log "Creating admin user"
+  "$DEST_PATH"/osctrl-cli -D "$__db_conf" user add -u "$_ADMIN_USER" -p "$_ADMIN_PASS" -a -n Admin
+
+  # Create environment for dev
+  log "Creating environment for dev"
   __osquery_dev="$SOURCE_PATH/deploy/osquery/osquery-dev.json"
   __osctrl_crt="/etc/nginx/certs/osctrl.crt"
   "$DEST_PATH"/osctrl-cli -D "$__db_conf" environment add -n "dev" -host "$_T_HOST" -conf "$__osquery_dev" -crt "$__osctrl_crt"
@@ -649,16 +655,13 @@ if [[ "$MODE" == "dev" ]]; then
     sleep 1
   done
 
+  # Enroll host in environment
   if [[ "$ENROLL" == true ]]; then
     log "Adding host in dev environment"
     eval $( "$DEST_PATH"/osctrl-cli -D "$__db_conf" environment quick-add -n "dev" )
   fi
 fi
 
-# Create admin user
-log "Creating admin user"
-"$DEST_PATH"/osctrl-cli -D "$__db_conf" user add -u "$_ADMIN_USER" -p "$_ADMIN_PASS" -a -n Admin
-
 # Ascii art is always appreciated
 if [[ "$DISTRO" == "ubuntu" ]]; then
   set_motd_ubuntu "$SOURCE_PATH/deploy/motd-osctrl.sh"

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/jmpsec/osctrl/nodes v0.2.1
 	github.com/jmpsec/osctrl/queries v0.2.1
 	github.com/jmpsec/osctrl/settings v0.2.1
+	github.com/jmpsec/osctrl/tls/handlers v0.2.1
 	github.com/jmpsec/osctrl/types v0.2.1
 	github.com/jmpsec/osctrl/users v0.2.1
 	github.com/jmpsec/osctrl/utils v0.2.1
@@ -51,3 +52,5 @@ replace github.com/jmpsec/osctrl/users => ./users
 replace github.com/jmpsec/osctrl/utils => ./utils
 
 replace github.com/jmpsec/osctrl/logging => ./logging
+
+replace github.com/jmpsec/osctrl/tls/handlers => ./tls/handlers

logging/db.go

@@ -7,8 +7,8 @@ import (
 	"github.com/jinzhu/gorm"
 
 	"github.com/jmpsec/osctrl/backend"
-	"github.com/jmpsec/osctrl/types"
 	"github.com/jmpsec/osctrl/settings"
+	"github.com/jmpsec/osctrl/types"
 )
 
 const (
@@ -82,19 +82,28 @@ func CreateLoggerDB() (*LoggerDB, error) {
 func (logDB *LoggerDB) Settings(mgr *settings.Settings) {
 	log.Printf("Setting DB logging settings\n")
 	// Setting link for on-demand queries
-	v := settings.QueryLink
-	if err := mgr.SetString(v, settings.ServiceAdmin, settings.QueryResultLink, false); err != nil {
-		log.Printf("Error setting %s with %s - %v", v, settings.QueryResultLink, err)
+	if !mgr.IsValue(settings.ServiceAdmin, settings.QueryResultLink) {
+		if err := mgr.NewStringValue(settings.ServiceAdmin, settings.QueryResultLink, settings.QueryLink); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.QueryResultLink, err)
+		}
+	} else if err := mgr.SetString(settings.QueryLink, settings.ServiceAdmin, settings.QueryResultLink, false); err != nil {
+		log.Printf("Error setting %s with %s - %v", settings.QueryResultLink, settings.QueryLink, err)
 	}
-	v = settings.StatusLink
 	// Setting link for status logs
-	if err := mgr.SetString(v, settings.ServiceAdmin, settings.StatusLogsLink, false); err != nil {
-		log.Printf("Error setting %s with %s - %v", v, settings.StatusLogsLink, err)
+	if !mgr.IsValue(settings.ServiceAdmin, settings.StatusLogsLink) {
+		if err := mgr.NewStringValue(settings.ServiceAdmin, settings.StatusLogsLink, settings.StatusLink); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.DebugHTTP, err)
+		}
+	} else if err := mgr.SetString(settings.StatusLink, settings.ServiceAdmin, settings.StatusLogsLink, false); err != nil {
+		log.Printf("Error setting %s with %s - %v", settings.StatusLogsLink, settings.StatusLink, err)
 	}
-	v = settings.ResultsLink
 	// Setting link for result logs
-	if err := mgr.SetString(v, settings.ServiceAdmin, settings.ResultLogsLink, false); err != nil {
-		log.Printf("Error setting %s with %s - %v", v, settings.ResultLogsLink, err)
+	if !mgr.IsValue(settings.ServiceAdmin, settings.ResultLogsLink) {
+		if err := mgr.NewStringValue(settings.ServiceAdmin, settings.ResultLogsLink, settings.ResultsLink); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.DebugHTTP, err)
+		}
+	} else if err := mgr.SetString(settings.ResultsLink, settings.ServiceAdmin, settings.ResultLogsLink, false); err != nil {
+		log.Printf("Error setting %s with %s - %v", settings.ResultLogsLink, settings.ResultsLink, err)
 	}
 }
 

logging/dispatch.go

@@ -0,0 +1,66 @@
+package logging
+
+import (
+	"encoding/json"
+	"log"
+
+	"github.com/jmpsec/osctrl/nodes"
+	"github.com/jmpsec/osctrl/types"
+)
+
+// DispatchLogs - Helper to dispatch logs
+func (l *LoggerTLS) DispatchLogs(data []byte, uuid, logType, environment string, metadata nodes.NodeMetadata, debug bool) {
+	// Use metadata to update record
+	if err := l.Nodes.UpdateMetadataByUUID(uuid, metadata); err != nil {
+		log.Printf("error updating metadata %s", err)
+	}
+	// Send data to storage
+	// FIXME allow multiple types of logging
+	if debug {
+		log.Printf("dispatching logs to %s", l.Logging)
+	}
+	l.Log(
+		logType,
+		data,
+		environment,
+		uuid,
+		debug)
+	// Refresh last logging request
+	if logType == types.StatusLog {
+		err := l.Nodes.RefreshLastStatus(uuid)
+		if err != nil {
+			log.Printf("error refreshing last status %v", err)
+		}
+	}
+	if logType == types.ResultLog {
+		if err := l.Nodes.RefreshLastResult(uuid); err != nil {
+			log.Printf("error refreshing last result %v", err)
+		}
+	}
+}
+
+// DispatchQueries - Helper to dispatch queries
+func (l *LoggerTLS) DispatchQueries(queryData types.QueryWriteData, node nodes.OsqueryNode, debug bool) {
+	// Prepare data to send
+	data, err := json.Marshal(queryData)
+	if err != nil {
+		log.Printf("error preparing data %v", err)
+	}
+	// Refresh last query write request
+	if err := l.Nodes.RefreshLastQueryWrite(node.UUID); err != nil {
+		log.Printf("error refreshing last query write %v", err)
+	}
+	// Send data to storage
+	// FIXME allow multiple types of logging
+	if debug {
+		log.Printf("dispatching queries to %s", l.Logging)
+	}
+	l.QueryLog(
+		types.QueryLog,
+		data,
+		node.Environment,
+		node.UUID,
+		queryData.Name,
+		queryData.Status,
+		debug)
+}

logging/logging.go

@@ -1,6 +1,8 @@
 package logging
 
 import (
+	"github.com/jmpsec/osctrl/nodes"
+	"github.com/jmpsec/osctrl/queries"
 	"github.com/jmpsec/osctrl/settings"
 )
 
@@ -10,15 +12,19 @@ type LoggerTLS struct {
 	Graylog *LoggerGraylog
 	Splunk  *LoggerSplunk
 	Logging string
+	Nodes   *nodes.NodeManager
+	Queries *queries.Queries
 }
 
 // CreateLoggerTLS to instantiate a new logger for the TLS endpoint
-func CreateLoggerTLS(logging string, mgr *settings.Settings) (*LoggerTLS, error) {
+func CreateLoggerTLS(logging string, mgr *settings.Settings, nodes *nodes.NodeManager, queries *queries.Queries) (*LoggerTLS, error) {
 	l := &LoggerTLS{
 		DB:      &LoggerDB{},
 		Splunk:  &LoggerSplunk{},
 		Logging: logging,
 		Graylog: &LoggerGraylog{},
+		Nodes:   nodes,
+		Queries: queries,
 	}
 	switch logging {
 	case settings.LoggingSplunk:

logging/process.go

@@ -0,0 +1,88 @@
+package logging
+
+import (
+	"encoding/json"
+	"log"
+
+	"github.com/jmpsec/osctrl/nodes"
+	"github.com/jmpsec/osctrl/types"
+)
+
+// ProcessLogs - Helper to process logs
+func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipaddress string, debug bool) {
+	// Parse log to extract metadata
+	var logs []types.LogGenericData
+	err := json.Unmarshal(data, &logs)
+	if err != nil {
+		// FIXME metrics for this
+		log.Printf("error parsing log %s %v", string(data), err)
+	}
+	if debug {
+		log.Printf("parsing logs for metadata in %s:%s", logType, environment)
+	}
+	// Iterate through received messages to extract metadata
+	var uuids, hosts, names, users, osqueryusers, hashes, dhashes, osqueryversions []string
+	for _, l := range logs {
+		uuids = append(uuids, l.HostIdentifier)
+		hosts = append(hosts, l.Decorations.Hostname)
+		names = append(names, l.Decorations.LocalHostname)
+		users = append(users, l.Decorations.Username)
+		osqueryusers = append(osqueryusers, l.Decorations.OsqueryUser)
+		hashes = append(hashes, l.Decorations.ConfigHash)
+		dhashes = append(dhashes, l.Decorations.DaemonHash)
+		osqueryversions = append(osqueryversions, l.Version)
+	}
+	if debug {
+		log.Printf("metadata and dispatch for %s", uniq(uuids)[0])
+	}
+	// FIXME it only uses the first element from the []string that uniq returns
+	metadata := nodes.NodeMetadata{
+		IPAddress:      ipaddress,
+		Username:       uniq(users)[0],
+		OsqueryUser:    uniq(osqueryusers)[0],
+		Hostname:       uniq(hosts)[0],
+		Localname:      uniq(names)[0],
+		ConfigHash:     uniq(hashes)[0],
+		DaemonHash:     uniq(dhashes)[0],
+		OsqueryVersion: uniq(osqueryversions)[0],
+	}
+	// Dispatch logs and update metadata
+	l.DispatchLogs(data, uniq(uuids)[0], logType, environment, metadata, debug)
+}
+
+// ProcessLogQueryResult - Helper to process on-demand query result logs
+func (l *LoggerTLS) ProcessLogQueryResult(queries types.QueryWriteQueries, statuses types.QueryWriteStatuses, nodeKey string, environment string, debug bool) {
+	// Retrieve node
+	node, err := l.Nodes.GetByKey(nodeKey)
+	if err != nil {
+		log.Printf("error retrieving node %s", err)
+	}
+	// Tap into results so we can update internal metrics
+	for q, r := range queries {
+		// Dispatch query name, result and status
+		d := types.QueryWriteData{
+			Name:   q,
+			Result: r,
+			Status: statuses[q],
+		}
+		go l.DispatchQueries(d, node, debug)
+		// Update internal metrics per query
+		var err error
+		if statuses[q] != 0 {
+			err = l.Queries.IncError(q)
+		} else {
+			err = l.Queries.IncExecution(q)
+		}
+		if err != nil {
+			log.Printf("error updating query %s", err)
+		}
+		// Add a record for this query
+		if err := l.Queries.TrackExecution(q, node.UUID, statuses[q]); err != nil {
+			log.Printf("error adding query execution %s", err)
+		}
+		// Check if query is completed
+		if err := l.Queries.VerifyComplete(q); err != nil {
+			log.Printf("error verifying and completing query %s", err)
+		}
+	}
+}

logging/utils.go

@@ -0,0 +1,14 @@
+package logging
+
+// Helper to remove duplicates from array of strings
+func uniq(duplicated []string) []string {
+	keys := make(map[string]bool)
+	result := []string{}
+	for _, entry := range duplicated {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			result = append(result, entry)
+		}
+	}
+	return result
+}