Merge pull request #11 from jmpsec/osquery-upgrade

A lot of changes everywhere
jmpsec · Sep 16, 2019 · fda4258 · fda4258
2 parents 10b1e83 + e0ac952
commit fda4258
Show file tree

Hide file tree

Showing 38 changed files with 878 additions and 348 deletions.
diff --git a/cmd/admin/handlers-get.go b/cmd/admin/handlers-get.go
@@ -850,6 +850,8 @@ func nodeHandler(w http.ResponseWriter, r *http.Request) {
 	funcMap := template.FuncMap{
 		"pastTimeAgo":   pastTimeAgo,
 		"jsonRawIndent": jsonRawIndent,
+		"statusLogsLink": statusLogsLink,
+		"resultLogsLink": resultLogsLink,
 	}
 	// Prepare template
 	t, err := template.New("node.html").Funcs(funcMap).ParseFiles(

diff --git a/cmd/admin/json-carves.go b/cmd/admin/json-carves.go
@@ -91,8 +91,9 @@ func jsonCarvesHandler(w http.ResponseWriter, r *http.Request) {
 			status = queries.StatusComplete
 		}
 		progress := make(CarveProgress)
-		progress["total"] = q.Expected
-		progress["completed"] = q.Executions
+		progress["expected"] = q.Expected
+		progress["executions"] = q.Executions
+		progress["errors"] = q.Errors
 		data := make(CarveData)
 		data["path"] = q.Path
 		data["name"] = q.Name

diff --git a/cmd/admin/json-queries.go b/cmd/admin/json-queries.go
@@ -85,6 +85,7 @@ func jsonQueryHandler(w http.ResponseWriter, r *http.Request) {
 		data := make(QueryData)
 		data["query"] = q.Query
 		data["name"] = q.Name
+		data["link"] = queryResultLink(q.Name)
 		// Preparing query targets
 		ts, _ := queriesmgr.GetTargets(q.Name)
 		_ts := []QueryTarget{}

diff --git a/cmd/admin/main.go b/cmd/admin/main.go
@@ -31,7 +31,7 @@ const (
 	// Service name
 	serviceName string = projectName + "-" + settings.ServiceAdmin
 	// Service version
-	serviceVersion string = "0.1.6"
+	serviceVersion string = "0.1.7"
 	// Service description
 	serviceDescription string = "Admin service for osctrl"
 	// Application description
@@ -51,7 +51,7 @@ const (
 	// Default SAML configuration file
 	samlConfigurationFile string = "config/saml.json"
 	// osquery version to display tables
-	osqueryTablesVersion string = "3.3.2"
+	osqueryTablesVersion string = "4.0.1"
 	// JSON file with osquery tables data
 	osqueryTablesFile string = "data/" + osqueryTablesVersion + ".json"
 	// Static files folder
@@ -104,7 +104,8 @@ var validAuth = map[string]bool{
 	settings.AuthJSON:    true,
 }
 var validLogging = map[string]bool{
-	settings.LoggingDB: true,
+	settings.LoggingDB:     true,
+	settings.LoggingSplunk: true,
 }
 
 // Function to load the configuration file

diff --git a/cmd/admin/settings.go b/cmd/admin/settings.go
@@ -33,6 +33,28 @@ func loadingMetrics() {
 	}
 }
 
+// Function to load the logging settings
+func loadingLogging() {
+	// Check if logging settings for query results link is ready
+	if !settingsmgr.IsValue(settings.ServiceAdmin, settings.QueryResultLink) {
+		if err := settingsmgr.NewStringValue(settings.ServiceAdmin, settings.QueryResultLink, settings.QueryLink); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.QueryResultLink, err)
+		}
+	}
+	// Check if logging settings for status logs link is ready
+	if !settingsmgr.IsValue(settings.ServiceAdmin, settings.StatusLogsLink) {
+		if err := settingsmgr.NewStringValue(settings.ServiceAdmin, settings.StatusLogsLink, settings.StatusLink); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.DebugHTTP, err)
+		}
+	}
+	// Check if logging settings for result logs link is ready
+	if !settingsmgr.IsValue(settings.ServiceAdmin, settings.ResultLogsLink) {
+		if err := settingsmgr.NewStringValue(settings.ServiceAdmin, settings.ResultLogsLink, settings.ResultsLink); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.DebugHTTP, err)
+		}
+	}
+}
+
 // Function to load all settings for the service
 func loadingSettings() {
 	// Check if service settings for debug service is ready
@@ -71,6 +93,8 @@ func loadingSettings() {
 	}
 	// Metrics
 	loadingMetrics()
+	// Logging
+	loadingLogging()
 	// Write JSON config to settings
 	if err := settingsmgr.SetAllJSON(settings.ServiceAdmin, adminConfig.Listener, adminConfig.Port, adminConfig.Host, adminConfig.Auth, adminConfig.Logging); err != nil {
 		log.Fatalf("Failed to add JSON values to configuration: %v", err)

diff --git a/cmd/admin/templates/carves-details.html b/cmd/admin/templates/carves-details.html
@@ -22,7 +22,11 @@
           {{ with .Query }}
             <div class="card mt-2">
               <div class="card-header">
-                <i class="fa fas fa-server"></i> Carved files for {{ .Name }}
+                {{ if .Completed }}
+                  <i class="fas fa-flag-checkered"></i> [ <b>COMPLETED</b> ] - Carved files for {{ .Name }}
+                {{ else }}
+                  <i class="fas fa-hourglass-half"></i> [ <b>ACTIVE</b> ] - Carved files for {{ .Name }}
+                {{ end }}
                 <div class="card-header-actions">
                   <button class="btn btn-sm btn-outline-primary" data-tooltip="true"
                     data-placement="bottom" title="Refresh details" onclick="refreshCarveDetails();">
@@ -35,8 +39,8 @@
                   <thead>
                     <tr>
                       <th width="45%">Path to Carve</th>
-                      <th width="45%">Target</th>
-                      <th width="10%">Expected</th>
+                      <th width="40%">Target</th>
+                      <th width="15%">Expected / Status</th>
                     </tr>
                   </thead>
                   <tbody>
@@ -56,7 +60,11 @@
                           {{ end }}
                         </table>
                       </td>
-                      <td style="text-align: center;vertical-align: middle;">{{ .Expected }}</td>
+                      <td style="text-align: center;vertical-align: middle;">
+                        <span style="color:black;">{{ .Expected }}</span>/
+                        <b><span style="color:green;">{{ .Executions }}</span></b>/
+                        <b><span style="color:red;">{{ .Errors }}</span></b>
+                      </td>
                     </tr>
                     <tr>
                       <td colspan="4" style="font-size: 1.5em; font-family: monospace;">{{ .Query }}</td>

diff --git a/cmd/admin/templates/carves.html b/cmd/admin/templates/carves.html
@@ -150,8 +150,9 @@
               data: 'progress',
               render: function (data, type, row, meta) {
                 if (type === 'display') {
-                  return  '<b>'+data.total+'</b>/' +
-                          '<b><span style="color:green;">'+data.completed+'</span></b>';
+                  return  '<b>'+data.expected+'</b>/' +
+                          '<b><span style="color:green;">'+data.executions+'</span></b>/' +
+                          '<b><span style="color:red;">'+data.errors+'</span></b>';
                 } else {
                   return data;
                 }

diff --git a/cmd/admin/templates/components/page-sidebar.html b/cmd/admin/templates/components/page-sidebar.html
@@ -92,6 +92,9 @@
           {{if eq $e "opensuse"}}
             <i class="nav-icon fa fl-opensuse"></i>
           {{end}}
+          {{if eq $e "arch"}}
+            <i class="nav-icon fa fl-archlinux"></i>
+          {{end}}
           {{if eq $e "unknown"}}
             <i class="nav-icon fa fa-question-circle"></i>
           {{end}}
@@ -128,6 +131,9 @@
               {{if eq $e "opensuse"}}
                 <i class="nav-icon fa fl-opensuse"></i>
               {{end}}
+              {{if eq $e "arch"}}
+                <i class="nav-icon fa fl-archlinux"></i>
+              {{end}}
               {{if eq $e "unknown"}}
                 <i class="nav-icon fa fa-question-circle"></i>
               {{end}}
@@ -161,6 +167,9 @@
               {{if eq $e "freebsd"}}
                 <i class="nav-icon fa fl-freebsd"></i>
               {{end}}
+              {{if eq $e "arch"}}
+                <i class="nav-icon fa fl-archlinux"></i>
+              {{end}}
               {{if eq $e "opensuse"}}
                 <i class="nav-icon fa fl-opensuse"></i>
               {{end}}
@@ -197,6 +206,9 @@
               {{if eq $e "freebsd"}}
                 <i class="nav-icon fa fl-freebsd"></i>
               {{end}}
+              {{if eq $e "arch"}}
+                <i class="nav-icon fa fl-archlinux"></i>
+              {{end}}
               {{if eq $e "opensuse"}}
                 <i class="nav-icon fa fl-opensuse"></i>
               {{end}}

diff --git a/cmd/admin/templates/node.html b/cmd/admin/templates/node.html
@@ -59,14 +59,12 @@
                       <li class="nav-item">
                         <a class="nav-link" data-toggle="tab" href="#metadata" role="tab" aria-controls="metadata">Metadata</a>
                       </li>
-                    {{ if eq $template.Logs "db" }}
                       <li class="nav-item">
                         <a class="nav-link" data-toggle="tab" href="#status-logs" role="tab" aria-controls="status-logs">Status Logs</a>
                       </li>
                       <li class="nav-item">
                         <a class="nav-link" data-toggle="tab" href="#result-logs" role="tab" aria-controls="result-logs">Result Logs</a>
                       </li>
-                    {{ end }}
                     </ul>
 
                     <div class="tab-content">
@@ -122,6 +120,7 @@
                                     {{if eq .Platform "windows"}}<i class='fab fa-windows'></i> windows{{end}}
                                     {{if eq .Platform "freebsd"}}<i class='fl-freebsd'></i> freebsd{{end}}
                                     {{if eq .Platform "opensuse"}}<i class='fl-opensuse'></i> opensuse{{end}}
+                                    {{if eq .Platform "arch"}}<i class='fl-archlinux'></i> arch{{end}}
                                     {{if eq .Platform "unknown"}}<i class='fa fa-question-circle'></i> unknown{{end}}
                                   - {{ .PlatformVersion }}</p>
                               </div>
@@ -315,6 +314,19 @@
                           </div>
                         </div>
                       </div>
+                    {{ else }}
+                      <div class="tab-pane fade" id="status-logs" role="tabpanel">
+                        <div class="card mt-2">
+                          <div id="result-card-header" class="card-header">
+                            <i class="fas fa-stream"></i> See status logs for node {{ .UUID }}
+                          </div>
+                          <div id="status-table" class="card-body">
+                            <a href="{{ statusLogsLink .UUID }}" target="_blank">
+                              See status logs in {{ $template.Logs }}
+                            </a>
+                          </div>
+                        </div>
+                      </div>
                     {{ end }}
 
                     {{ if eq $template.Logs "db" }}
@@ -348,6 +360,19 @@
                           </div>
                         </div>
                       </div>
+                    {{ else }}
+                      <div class="tab-pane fade" id="result-logs" role="tabpanel">
+                        <div class="card mt-2">
+                          <div id="result-card-header" class="card-header">
+                            <i class="fas fa-stream"></i> See result logs for node {{ .UUID }}
+                          </div>
+                          <div id="results-table" class="card-body">
+                            <a href="{{ resultLogsLink .UUID }}" target="_blank">
+                              See result logs in {{ $template.Logs }}
+                            </a>
+                          </div>
+                        </div>
+                      </div>
                     {{ end }}
 
                     </div>
@@ -377,7 +402,6 @@
     <!-- custom JS -->
     <script src="/static/js/nodeactions.js"></script>
     <script src="/static/js/tables.js"></script>
-  {{ if eq .Logs "db" }}
     {{ with .Node }}
     <script type="text/javascript">
       // Highlight.js code element initialization
@@ -392,6 +416,7 @@
           hljs.highlightBlock(block);
         });
 
+      {{ if eq $template.Logs "db" }}
         // Handle datatable ajax error
         $.fn.dataTable.ext.errMode = function(settings, helpPage, message) {
           console.log(message);
@@ -480,8 +505,6 @@
             { width: '80%', targets: 2 }
           ]
         });
-        // Enable all tooltips
-        $('[data-tooltip="true"]').tooltip({trigger : 'hover'});
 
         // Display the number of seconds left and refresh for result logs
         var refreshSecondsResult = 60;
@@ -499,6 +522,7 @@
             tableResultLogs.ajax.reload();
           }
         },1000);
+      {{ end }}
 
         // Refresh sidebar stats
         beginStats();
@@ -514,10 +538,12 @@
         $("#carveModal").on('shown.bs.modal', function(){
           $(this).find('#carve').focus();
         });
+
+        // Enable all tooltips
+        $('[data-tooltip="true"]').tooltip({trigger : 'hover'});
       });
     </script>
     {{ end }}
-  {{ end }}
 
   </body>
 </html>
diff --git a/cmd/admin/templates/queries-logs.html b/cmd/admin/templates/queries-logs.html
@@ -20,7 +20,11 @@
           {{ with .Query }}
             <div class="card mt-2">
               <div class="card-header">
-                <i class="fa fas fa-server"></i> Results for {{ .Name }}
+                {{ if .Completed }}
+                  <i class="fas fa-flag-checkered"></i> [ <b>COMPLETED</b> ] - Results for {{ .Name }}
+                {{ else }}
+                  <i class="fas fa-hourglass-half"></i> [ <b>ACTIVE</b> ] - Results for {{ .Name }}
+                {{ end }}
                 <div class="card-header-actions">
                   <button class="btn btn-sm btn-outline-primary" data-tooltip="true"
                     data-placement="bottom" title="Refresh table" onclick="refreshTableNow('tableQueryLogs');">
@@ -34,7 +38,7 @@
                     <tr>
                       <th width="60%">Query</th>
                       <th width="25%">Target</th>
-                      <th width="15%">Expected</th>
+                      <th width="15%">Expected / Status</th>
                     </tr>
                   </thead>
                   <tbody>
@@ -54,7 +58,11 @@
                           {{ end }}
                         </table>
                       </td>
-                      <td style="text-align: center;vertical-align: middle;">{{ .Expected }}</td>
+                      <td style="text-align: center;vertical-align: middle;">
+                        <span style="color:black;">{{ .Expected }}</span>/
+                        <b><span style="color:green;">{{ .Executions }}</span></b>/
+                        <b><span style="color:red;">{{ .Errors }}</span></b>
+                      </td>
                     </tr>
                   </tbody>
                 </table>

diff --git a/cmd/admin/templates/queries.html b/cmd/admin/templates/queries.html
@@ -119,7 +119,7 @@
               data: 'query',
               render: function (data, type, row, meta) {
                 if (type === 'display') {
-                  return '<span style="font-family: monospace;"><a href="/query/logs/'+data.name+'">'+data.query+'</a></span>';
+                  return '<span style="font-family: monospace; font-size: 1.3em;"><a href="'+data.link+'">'+data.query+'</a></span>';
                 } else {
                   return data;
                 }

diff --git a/cmd/admin/templates/table.html b/cmd/admin/templates/table.html
@@ -158,6 +158,8 @@
                     return '<i class="fl-ubuntu-inverse"></i> ubuntu';
                   case "debian":
                     return '<i class="fl-debian"></i> debian';
+                  case "arch":
+                    return '<i class="fl-archlinux"></i> arch';
                   case "unknown":
                     return '<i class="fa fa-question-circle"></i> unknown';
                   case "darwin":

diff --git a/cmd/admin/utils.go b/cmd/admin/utils.go
@@ -85,23 +85,25 @@ func checkValidPlatform(platform string) bool {
 	return false
 }
 
-/*
 // Helper to remove backslashes from text
 func removeBackslash(rawString string) string {
 	return strings.Replace(rawString, "\\", " ", -1)
 }
 
 // Helper to generate a link to results for on-demand queries
-func resultsSearchLink(name string) string {
-		if adminConfig.Logging == settings.LoggingSplunk {
-			return strings.Replace(.LoggingCfg["search"], "{{NAME}}", removeBackslash(name), 1)
-		}
-	if adminConfig.Logging == settings.LoggingDB {
-		return "/query/logs/" + removeBackslash(name)
-	}
-	return ""
+func queryResultLink(name string) string {
+	return strings.Replace(settingsmgr.QueryResultLink(), "{{NAME}}", removeBackslash(name), 1)
+}
+
+// Helper to generate a link to results for status logs
+func statusLogsLink(uuid string) string {
+	return strings.Replace(settingsmgr.StatusLogsLink(), "{{UUID}}", removeBackslash(uuid), 1)
+}
+
+// Helper to generate a link to results for result logs
+func resultLogsLink(uuid string) string {
+	return strings.Replace(settingsmgr.ResultLogsLink(), "{{UUID}}", removeBackslash(uuid), 1)
 }
-*/
 
 // Helper to get a string based on the difference of two times
 func stringifyTime(seconds int) string {