Merge pull request #554 from jmpsec/api-fix-complete

Multiple changes to osctrl-api: Crash fix, show carve queries and better targets to list queries

admin/handlers/json-queries.go

@@ -11,8 +11,6 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
-const ()
-
 // Define targets to be used
 var (
 	QueryTargets = map[string]bool{

api/handlers/carves.go

@@ -66,6 +66,103 @@ func (h *HandlersApi) CarveShowHandler(w http.ResponseWriter, r *http.Request) {
 	h.Inc(metricAPICarvesOK)
 }
 
+// GET Handler to return carve queries in JSON by target and environment
+func (h *HandlersApi) CarveQueriesHandler(w http.ResponseWriter, r *http.Request) {
+	h.Inc(metricAPICarvesReq)
+	utils.DebugHTTPDump(r, h.Settings.DebugHTTP(settings.ServiceAPI, settings.NoEnvironmentID), false)
+	// Extract environment
+	envVar := r.PathValue("env")
+	if envVar == "" {
+		apiErrorResponse(w, "error with environment", http.StatusBadRequest, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Get environment
+	env, err := h.Envs.GetByUUID(envVar)
+	if err != nil {
+		apiErrorResponse(w, "error getting environment", http.StatusInternalServerError, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Get context data and check access
+	ctx := r.Context().Value(ContextKey(contextAPI)).(ContextValue)
+	if !h.Users.CheckPermissions(ctx[ctxUser], users.CarveLevel, env.UUID) {
+		apiErrorResponse(w, "no access", http.StatusForbidden, fmt.Errorf("attempt to use API by user %s", ctx[ctxUser]))
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Extract target
+	targetVar := r.PathValue("target")
+	if targetVar == "" {
+		apiErrorResponse(w, "error with target", http.StatusBadRequest, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Verify target
+	if !QueryTargets[targetVar] {
+		apiErrorResponse(w, "invalid target", http.StatusBadRequest, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Get carves
+	carves, err := h.Queries.GetCarves(targetVar, env.ID)
+	if err != nil {
+		apiErrorResponse(w, "error getting carve queries", http.StatusInternalServerError, err)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	if len(carves) == 0 {
+		apiErrorResponse(w, "no carve queries", http.StatusNotFound, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Serialize and serve JSON
+	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, carves)
+	h.Inc(metricAPICarvesOK)
+}
+
+// GET Handler to return carves in JSON by environment
+func (h *HandlersApi) CarveListHandler(w http.ResponseWriter, r *http.Request) {
+	h.Inc(metricAPICarvesReq)
+	utils.DebugHTTPDump(r, h.Settings.DebugHTTP(settings.ServiceAPI, settings.NoEnvironmentID), false)
+	// Extract environment
+	envVar := r.PathValue("env")
+	if envVar == "" {
+		apiErrorResponse(w, "error with environment", http.StatusBadRequest, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Get environment
+	env, err := h.Envs.GetByUUID(envVar)
+	if err != nil {
+		apiErrorResponse(w, "error getting environment", http.StatusInternalServerError, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Get context data and check access
+	ctx := r.Context().Value(ContextKey(contextAPI)).(ContextValue)
+	if !h.Users.CheckPermissions(ctx[ctxUser], users.CarveLevel, env.UUID) {
+		apiErrorResponse(w, "no access", http.StatusForbidden, fmt.Errorf("attempt to use API by user %s", ctx[ctxUser]))
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Get carves
+	carves, err := h.Carves.GetByEnv(env.ID)
+	if err != nil {
+		apiErrorResponse(w, "error getting carves", http.StatusInternalServerError, err)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	if len(carves) == 0 {
+		apiErrorResponse(w, "no carves", http.StatusNotFound, nil)
+		h.Inc(metricAPICarvesErr)
+		return
+	}
+	// Serialize and serve JSON
+	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, carves)
+	h.Inc(metricAPICarvesOK)
+}
+
 // POST Handler to run a carve
 func (h *HandlersApi) CarvesRunHandler(w http.ResponseWriter, r *http.Request) {
 	h.Inc(metricAPICarvesReq)
@@ -212,6 +309,13 @@ func (h *HandlersApi) CarvesActionHandler(w http.ResponseWriter, r *http.Request
 			return
 		}
 		msgReturn = fmt.Sprintf("carve %s expired successfully", nameVar)
+	case settings.CarveComplete:
+		if err := h.Queries.Complete(nameVar, env.ID); err != nil {
+			apiErrorResponse(w, "error completing carve", http.StatusInternalServerError, err)
+			h.Inc(metricAPICarvesErr)
+			return
+		}
+		msgReturn = fmt.Sprintf("carve %s completed successfully", nameVar)
 	}
 	// Return message as serialized response
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, types.ApiGenericResponse{Message: msgReturn})

api/handlers/queries.go

@@ -14,6 +14,19 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+var QueryTargets = map[string]bool{
+	queries.TargetAll:             true,
+	queries.TargetAllFull:         true,
+	queries.TargetActive:          true,
+	queries.TargetHiddenActive:    true,
+	queries.TargetCompleted:       true,
+	queries.TargetExpired:         true,
+	queries.TargetSaved:           true,
+	queries.TargetHiddenCompleted: true,
+	queries.TargetDeleted:         true,
+	queries.TargetHidden:          true,
+}
+
 // QueryShowHandler - GET Handler to return a single query in JSON
 func (h *HandlersApi) QueryShowHandler(w http.ResponseWriter, r *http.Request) {
 	h.Inc(metricAPIQueriesReq)
@@ -277,6 +290,13 @@ func (h *HandlersApi) QueriesActionHandler(w http.ResponseWriter, r *http.Reques
 			return
 		}
 		msgReturn = fmt.Sprintf("query %s expired successfully", nameVar)
+	case settings.QueryComplete:
+		if err := h.Queries.Complete(nameVar, env.ID); err != nil {
+			apiErrorResponse(w, "error completing query", http.StatusInternalServerError, err)
+			h.Inc(metricAPIQueriesErr)
+			return
+		}
+		msgReturn = fmt.Sprintf("query %s completed successfully", nameVar)
 	}
 	// Return message as serialized response
 	utils.HTTPResponse(w, utils.JSONApplicationUTF8, http.StatusOK, types.ApiGenericResponse{Message: msgReturn})
@@ -325,8 +345,8 @@ func (h *HandlersApi) AllQueriesShowHandler(w http.ResponseWriter, r *http.Reque
 	h.Inc(metricAPIQueriesOK)
 }
 
-// HiddenQueriesShowHandler - GET Handler to return hidden queries in JSON
-func (h *HandlersApi) HiddenQueriesShowHandler(w http.ResponseWriter, r *http.Request) {
+// QueryListHandler - GET Handler to return queries in JSON by target and environment
+func (h *HandlersApi) QueryListHandler(w http.ResponseWriter, r *http.Request) {
 	h.Inc(metricAPIQueriesReq)
 	utils.DebugHTTPDump(r, h.Settings.DebugHTTP(settings.ServiceAPI, settings.NoEnvironmentID), false)
 	// Extract environment
@@ -350,8 +370,21 @@ func (h *HandlersApi) HiddenQueriesShowHandler(w http.ResponseWriter, r *http.Re
 		h.Inc(metricAPIQueriesErr)
 		return
 	}
+	// Extract target
+	targetVar := r.PathValue("target")
+	if targetVar == "" {
+		apiErrorResponse(w, "error with target", http.StatusBadRequest, nil)
+		h.Inc(metricAPIQueriesErr)
+		return
+	}
+	// Verify target
+	if !QueryTargets[targetVar] {
+		apiErrorResponse(w, "invalid target", http.StatusBadRequest, nil)
+		h.Inc(metricAPIQueriesErr)
+		return
+	}
 	// Get queries
-	queries, err := h.Queries.GetQueries(queries.TargetHiddenCompleted, env.ID)
+	queries, err := h.Queries.GetQueries(targetVar, env.ID)
 	if err != nil {
 		apiErrorResponse(w, "error getting queries", http.StatusInternalServerError, err)
 		h.Inc(metricAPIQueriesErr)

api/main.go

@@ -570,13 +570,16 @@ func osctrlAPIService() {
 	muxAPI.Handle("POST "+_apiPath(apiNodesPath)+"/{env}/delete", handlerAuthCheck(http.HandlerFunc(handlersApi.DeleteNodeHandler)))
 	// API: queries by environment
 	muxAPI.Handle("GET "+_apiPath(apiQueriesPath)+"/{env}", handlerAuthCheck(http.HandlerFunc(handlersApi.AllQueriesShowHandler)))
+	muxAPI.Handle("GET "+_apiPath(apiQueriesPath)+"/{env}/list/{target}", handlerAuthCheck(http.HandlerFunc(handlersApi.QueryListHandler)))
 	muxAPI.Handle("POST "+_apiPath(apiQueriesPath)+"/{env}", handlerAuthCheck(http.HandlerFunc(handlersApi.QueriesRunHandler)))
 	muxAPI.Handle("GET "+_apiPath(apiQueriesPath)+"/{env}/{name}", handlerAuthCheck(http.HandlerFunc(handlersApi.QueryShowHandler)))
 	muxAPI.Handle("GET "+_apiPath(apiQueriesPath)+"/{env}/results/{name}", handlerAuthCheck(http.HandlerFunc(handlersApi.QueryResultsHandler)))
 	muxAPI.Handle("GET "+_apiPath(apiAllQueriesPath+"/{env}"), handlerAuthCheck(http.HandlerFunc(handlersApi.AllQueriesShowHandler)))
 	muxAPI.Handle("POST "+_apiPath(apiQueriesPath)+"/{env}/{action}/{name}", handlerAuthCheck(http.HandlerFunc(handlersApi.QueriesActionHandler)))
 	// API: carves by environment
 	muxAPI.Handle("GET "+_apiPath(apiCarvesPath)+"/{env}", handlerAuthCheck(http.HandlerFunc(handlersApi.CarveShowHandler)))
+	muxAPI.Handle("GET "+_apiPath(apiCarvesPath)+"/{env}/queries/{target}", handlerAuthCheck(http.HandlerFunc(handlersApi.CarveQueriesHandler)))
+	muxAPI.Handle("GET "+_apiPath(apiCarvesPath)+"/{env}/list", handlerAuthCheck(http.HandlerFunc(handlersApi.CarveListHandler)))
 	muxAPI.Handle("POST "+_apiPath(apiCarvesPath)+"/{env}", handlerAuthCheck(http.HandlerFunc(handlersApi.CarvesRunHandler)))
 	muxAPI.Handle("GET "+_apiPath(apiCarvesPath)+"/{env}/{name}", handlerAuthCheck(http.HandlerFunc(handlersApi.CarveShowHandler)))
 	muxAPI.Handle("POST "+_apiPath(apiCarvesPath)+"/{env}/{action}/{name}", handlerAuthCheck(http.HandlerFunc(handlersApi.CarvesActionHandler)))

cli/api-carve.go

@@ -6,15 +6,30 @@ import (
 	"strings"
 
 	"github.com/jmpsec/osctrl/carves"
+	"github.com/jmpsec/osctrl/queries"
 	"github.com/jmpsec/osctrl/settings"
 	"github.com/jmpsec/osctrl/types"
 	"github.com/rs/zerolog/log"
 )
 
+// GetCarveQueries to retrieve carves from osctrl
+func (api *OsctrlAPI) GetCarveQueries(target, env string) ([]queries.DistributedQuery, error) {
+	var qs []queries.DistributedQuery
+	reqURL := fmt.Sprintf("%s%s%s/%s/queries/%s", api.Configuration.URL, APIPath, APICarves, env, target)
+	rawCs, err := api.GetGeneric(reqURL, nil)
+	if err != nil {
+		return qs, fmt.Errorf("error api request - %v - %s", err, string(rawCs))
+	}
+	if err := json.Unmarshal(rawCs, &qs); err != nil {
+		return qs, fmt.Errorf("can not parse body - %v", err)
+	}
+	return qs, nil
+}
+
 // GetCarves to retrieve carves from osctrl
 func (api *OsctrlAPI) GetCarves(env string) ([]carves.CarvedFile, error) {
 	var cs []carves.CarvedFile
-	reqURL := fmt.Sprintf("%s%s%s/%s", api.Configuration.URL, APIPath, APICarves, env)
+	reqURL := fmt.Sprintf("%s%s%s/%s/list", api.Configuration.URL, APIPath, APICarves, env)
 	rawCs, err := api.GetGeneric(reqURL, nil)
 	if err != nil {
 		return cs, fmt.Errorf("error api request - %v - %s", err, string(rawCs))

cli/api-query.go

@@ -11,9 +11,9 @@ import (
 )
 
 // GetQueries to retrieve queries from osctrl
-func (api *OsctrlAPI) GetQueries(env string) ([]queries.DistributedQuery, error) {
+func (api *OsctrlAPI) GetQueries(target, env string) ([]queries.DistributedQuery, error) {
 	var qs []queries.DistributedQuery
-	reqURL := fmt.Sprintf("%s%s%s/%s", api.Configuration.URL, APIPath, APIQueries, env)
+	reqURL := fmt.Sprintf("%s%s%s/%s/list/%s", api.Configuration.URL, APIPath, APIQueries, env, target)
 	rawQs, err := api.GetGeneric(reqURL, nil)
 	if err != nil {
 		return qs, fmt.Errorf("error api request - %v - %s", err, string(rawQs))

cli/carve.go

@@ -47,19 +47,6 @@ func carveToData(c carves.CarvedFile, header []string) [][]string {
 
 func listCarves(c *cli.Context) error {
 	// Get values from flags
-	target := "all"
-	if c.Bool("all") {
-		target = "all"
-	}
-	if c.Bool("active") {
-		target = "active"
-	}
-	if c.Bool("completed") {
-		target = "completed"
-	}
-	if c.Bool("deleted") {
-		target = "deleted"
-	}
 	env := c.String("env")
 	if env == "" {
 		fmt.Println("❌ environment is required")
@@ -110,11 +97,96 @@ func listCarves(c *cli.Context) error {
 		table := tablewriter.NewWriter(os.Stdout)
 		table.SetHeader(header)
 		if len(cs) > 0 {
-			fmt.Printf("Existing %s queries (%d):\n", target, len(cs))
+			fmt.Printf("Existing carves (%d):\n", len(cs))
 			data := carvesToData(cs, nil)
 			table.AppendBulk(data)
 		} else {
-			fmt.Printf("No %s nodes\n", target)
+			fmt.Println("No carves")
+		}
+		table.Render()
+	}
+	return nil
+}
+
+func listCarveQueries(c *cli.Context) error {
+	// Get values from flags
+	target := "all"
+	if c.Bool("all") {
+		target = "all"
+	}
+	if c.Bool("active") {
+		target = "active"
+	}
+	if c.Bool("completed") {
+		target = "completed"
+	}
+	if c.Bool("deleted") {
+		target = "deleted"
+	}
+	if c.Bool("hidden") {
+		target = "hidden"
+	}
+	if c.Bool("expired") {
+		target = "expired"
+	}
+	env := c.String("env")
+	if env == "" {
+		fmt.Println("❌ environment is required")
+		os.Exit(1)
+	}
+	// Retrieve data
+	var qs []queries.DistributedQuery
+	if dbFlag {
+		e, err := envs.Get(env)
+		if err != nil {
+			return fmt.Errorf("❌ error env get - %s", err)
+		}
+		qs, err = queriesmgr.GetCarves(target, e.ID)
+		if err != nil {
+			return fmt.Errorf("❌ error get carve queries - %s", err)
+		}
+	} else if apiFlag {
+		qs, err = osctrlAPI.GetCarveQueries(target, env)
+		if err != nil {
+			return fmt.Errorf("❌ error get carve queries - %s", err)
+		}
+	}
+	header := []string{
+		"Name",
+		"Creator",
+		"Query",
+		"Type",
+		"Executions",
+		"Errors",
+		"Active",
+		"Hidden",
+		"Completed",
+		"Deleted",
+		"Expired",
+		"Expiration",
+	}
+	// Prepare output
+	if formatFlag == jsonFormat {
+		jsonRaw, err := json.Marshal(qs)
+		if err != nil {
+			return fmt.Errorf("❌ error json marshal - %s", err)
+		}
+		fmt.Println(string(jsonRaw))
+	} else if formatFlag == csvFormat {
+		data := queriesToData(qs, header)
+		w := csv.NewWriter(os.Stdout)
+		if err := w.WriteAll(data); err != nil {
+			return fmt.Errorf("❌ error csv writeall - %s", err)
+		}
+	} else if formatFlag == prettyFormat {
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetHeader(header)
+		if len(qs) > 0 {
+			fmt.Printf("Existing %s carve queries (%d):\n", target, len(qs))
+			data := queriesToData(qs, nil)
+			table.AppendBulk(data)
+		} else {
+			fmt.Printf("No %s carve queries\n", target)
 		}
 		table.Render()
 	}