From 2c7312c3bbb13e3de8f78a013842ce8d94d4b662 Mon Sep 17 00:00:00 2001 From: Srinandan Sridhar <13950006+srinandan@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:56:12 -0800 Subject: [PATCH] feat: adds support to import security profiles (#342) * feat: exports sec profiles to a folder * chore: pretty prints json in debug * chore: fixes description of list params * feat: adds support to import sec profiles * feat: adds support to compute secu profile scores * chore: fix linting errors * feat: disable all revisions export #340 * feat: remove the ability to export revisions #340 * chore: fix linting issues --- cmd/apidocs/import.go | 1 + cmd/org/export.go | 4 +- cmd/org/import.go | 10 + cmd/org/variables.go | 7 +- cmd/securityprofiles/compute.go | 89 ++++++ cmd/securityprofiles/export.go | 9 +- cmd/securityprofiles/import.go | 51 ++++ cmd/securityprofiles/list.go | 4 +- cmd/securityprofiles/securityprofiles.go | 2 + internal/apiclient/httpclient.go | 11 +- internal/client/apidocs/apidocs.go | 13 +- .../securityprofiles/securityprofiles.go | 265 ++++++++++++++++-- 12 files changed, 410 insertions(+), 56 deletions(-) create mode 100644 cmd/securityprofiles/compute.go create mode 100644 cmd/securityprofiles/import.go diff --git a/cmd/apidocs/import.go b/cmd/apidocs/import.go index 6e6aab53e..737e52627 100644 --- a/cmd/apidocs/import.go +++ b/cmd/apidocs/import.go @@ -16,6 +16,7 @@ package apidocs import ( "fmt" + "internal/apiclient" "internal/client/apidocs" diff --git a/cmd/org/export.go b/cmd/org/export.go index 2c278ab5d..037c563b6 100644 --- a/cmd/org/export.go +++ b/cmd/org/export.go @@ -176,7 +176,7 @@ var ExportCmd = &cobra.Command{ if orgs.GetAddOn("apiSecurityConfig") { clilog.Info.Println("Exporting API Security Configuration...") - if err = securityprofiles.Export(conn, folder, allRevisions); proceedOnError(err) != nil { + if err = securityprofiles.Export(conn, securityProfilesFolderName); proceedOnError(err) != nil { return err } } @@ -319,7 +319,7 @@ func createFolders() (err error) { if err = os.Mkdir(portalsFolderName, 0o755); err != nil { return err } - return nil + return os.Mkdir(securityProfilesFolderName, 0o755) } func exportKVMEntries(scope string, env string, listKVMBytes []byte) (err error) { diff --git a/cmd/org/import.go b/cmd/org/import.go index 629aa35e1..ea564a781 100644 --- a/cmd/org/import.go +++ b/cmd/org/import.go @@ -33,11 +33,14 @@ import ( "internal/client/envgroups" "internal/client/keystores" "internal/client/kvm" + "internal/client/orgs" "internal/client/products" "internal/client/references" "internal/client/sharedflows" "internal/client/targetservers" + "internal/client/securityprofiles" + "github.com/apigee/apigeecli/cmd/utils" "github.com/spf13/cobra" ) @@ -125,6 +128,13 @@ var ImportCmd = &cobra.Command{ } } + if orgs.GetAddOn("apiSecurityConfig") { + clilog.Info.Println("Importing Security Profile Configuration...") + if err = securityprofiles.Import(conn, path.Join(folder, securityProfilesFolderName)); err != nil { + return err + } + } + var envRespBody []byte if envRespBody, err = env.List(); err != nil { return err diff --git a/cmd/org/variables.go b/cmd/org/variables.go index 175fa499a..06b7ed2e9 100644 --- a/cmd/org/variables.go +++ b/cmd/org/variables.go @@ -30,9 +30,10 @@ const ( tracecfgFileName = "_tracecfg.json" referencesFileName = "references.json" - proxiesFolderName = "proxies" - sharedFlowsFolderName = "sharedflows" - portalsFolderName = "portals" + proxiesFolderName = "proxies" + sharedFlowsFolderName = "sharedflows" + portalsFolderName = "portals" + securityProfilesFolderName = "securityprofiles" ) var conn int diff --git a/cmd/securityprofiles/compute.go b/cmd/securityprofiles/compute.go new file mode 100644 index 000000000..1f050ae92 --- /dev/null +++ b/cmd/securityprofiles/compute.go @@ -0,0 +1,89 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package securityprofiles + +import ( + "fmt" + "time" + + "internal/apiclient" + "internal/client/securityprofiles" + + "github.com/spf13/cobra" +) + +// ComputeCmd to list catalog items +var ComputeCmd = &cobra.Command{ + Use: "compute", + Short: "Calculates scores for requested time range", + Long: "Calculates scores for requested time range for the specified security profile", + Args: func(cmd *cobra.Command, args []string) (err error) { + if startTime != "" { + if _, err = time.Parse(time.RFC3339, startTime); err != nil { + return fmt.Errorf("invalid format for startTime: %v", err) + } + } + if endTime != "" { + if _, err = time.Parse(time.RFC3339, endTime); err != nil { + return fmt.Errorf("invalid format for endTime: %v", err) + } + } + apiclient.SetApigeeEnv(environment) + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + if endTime == "" { + // if endTime is not set, then set current timestamp + endTime = time.Now().UTC().Format(time.RFC3339) + } + if startTime == "" { + // if startTime is not set, then set yesterday's timestamp + startTime = time.Now().AddDate(0, 0, -1).UTC().Format(time.RFC3339) + } + _, err = securityprofiles.Compute(name, startTime, endTime, + filters, pageSize, pageToken, full) + return + }, +} + +var ( + startTime, endTime string + filters []string + full bool +) + +func init() { + ComputeCmd.Flags().StringVarP(&name, "name", "n", + "", "Name of the security profile") + ComputeCmd.Flags().StringVarP(&environment, "env", "e", + "", "Apigee environment name") + + ComputeCmd.Flags().StringVarP(&startTime, "start-time", "", + "", "Inclusive start of the interval; default is 24 hours ago") + ComputeCmd.Flags().StringVarP(&endTime, "end-time", "", + "", "Exclusive end of the interval; default current timestamp") + + ComputeCmd.Flags().StringArrayVarP(&filters, "filters", "f", + nil, "Filters are used to filter scored components") + ComputeCmd.Flags().IntVarP(&pageSize, "page-size", "", + -1, "The maximum number of versions to return") + ComputeCmd.Flags().StringVarP(&pageToken, "page-token", "", + "", "A page token, received from a previous call") + ComputeCmd.Flags().BoolVarP(&full, "full", "", + false, "Generate all scores") + + _ = ComputeCmd.MarkFlagRequired("name") + _ = ComputeCmd.MarkFlagRequired("env") +} diff --git a/cmd/securityprofiles/export.go b/cmd/securityprofiles/export.go index 6ce904add..be155866b 100644 --- a/cmd/securityprofiles/export.go +++ b/cmd/securityprofiles/export.go @@ -39,14 +39,13 @@ var ExpCmd = &cobra.Command{ if err = apiclient.FolderExists(folder); err != nil { return err } - return securityprofiles.Export(conn, folder, allRevisions) + return securityprofiles.Export(conn, folder) }, } var ( - conn int - folder string - allRevisions bool + conn int + folder string ) func init() { @@ -54,6 +53,4 @@ func init() { 4, "Number of connections") ExpCmd.Flags().StringVarP(&folder, "folder", "f", "", "folder to export Security Profiles") - ExpCmd.Flags().BoolVarP(&allRevisions, "all", "", - false, "Export all proxy revisions") } diff --git a/cmd/securityprofiles/import.go b/cmd/securityprofiles/import.go new file mode 100644 index 000000000..4e83a800e --- /dev/null +++ b/cmd/securityprofiles/import.go @@ -0,0 +1,51 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package securityprofiles + +import ( + "fmt" + "os" + + "internal/apiclient" + + "internal/client/securityprofiles" + + "github.com/spf13/cobra" +) + +// ImpCmd to import sec profiles +var ImpCmd = &cobra.Command{ + Use: "import", + Short: "Import a folder containing Security Profiles", + Long: "Import a folder containing Security Profiles", + Args: func(cmd *cobra.Command, args []string) (err error) { + return apiclient.SetApigeeOrg(org) + }, + RunE: func(cmd *cobra.Command, args []string) (err error) { + if stat, err := os.Stat(folder); err == nil && !stat.IsDir() { + return fmt.Errorf("supplied path is not a folder") + } + return securityprofiles.Import(conn, folder) + }, +} + +func init() { + ImpCmd.Flags().StringVarP(&folder, "folder", "f", + "", "folder containing one or more security profiles") + ImpCmd.Flags().IntVarP(&conn, "conn", "c", + 4, "Number of connections") + + _ = ImpCmd.MarkFlagRequired("folder") +} diff --git a/cmd/securityprofiles/list.go b/cmd/securityprofiles/list.go index 029d1347b..95d020b5b 100644 --- a/cmd/securityprofiles/list.go +++ b/cmd/securityprofiles/list.go @@ -41,8 +41,8 @@ var ( ) func init() { - ListCmd.Flags().IntVarP(&pageSize, "pageSize", "", + ListCmd.Flags().IntVarP(&pageSize, "page-size", "", -1, "The maximum number of versions to return") - ListCmd.Flags().StringVarP(&pageToken, "pageToken", "", + ListCmd.Flags().StringVarP(&pageToken, "page-token", "", "", "A page token, received from a previous call") } diff --git a/cmd/securityprofiles/securityprofiles.go b/cmd/securityprofiles/securityprofiles.go index 99477f05e..e3ff2910f 100644 --- a/cmd/securityprofiles/securityprofiles.go +++ b/cmd/securityprofiles/securityprofiles.go @@ -41,6 +41,8 @@ func init() { Cmd.AddCommand(ListRevisionsCmd) Cmd.AddCommand(UpdateCmd) Cmd.AddCommand(ExpCmd) + Cmd.AddCommand(ImpCmd) + Cmd.AddCommand(ComputeCmd) _ = Cmd.MarkFlagRequired("org") } diff --git a/internal/apiclient/httpclient.go b/internal/apiclient/httpclient.go index d9cea7771..31c232a76 100644 --- a/internal/apiclient/httpclient.go +++ b/internal/apiclient/httpclient.go @@ -275,17 +275,14 @@ func HttpClient(params ...string) (respBody []byte, err error) { return nil, err } - if DryRun() { - return nil, nil - } - clilog.Debug.Println("Connecting to: ", params[0]) switch paramLen := len(params); paramLen { case 1: req, err = http.NewRequest(http.MethodGet, params[0], nil) case 2: - clilog.Debug.Println("Payload: ", params[1]) + payload, _ := PrettifyJSON([]byte(params[1])) + clilog.Debug.Println("Payload: ", string(payload)) req, err = http.NewRequest(http.MethodPost, params[0], bytes.NewBuffer([]byte(params[1]))) case 3: if req, err = getRequest(params); err != nil { @@ -313,6 +310,10 @@ func HttpClient(params ...string) (respBody []byte, err error) { clilog.Debug.Println("Content-Type : ", contentType) req.Header.Set("Content-Type", contentType) + if DryRun() { + return nil, nil + } + resp, err := ApigeeAPIClient.Do(req) if err != nil { clilog.Error.Println("error connecting: ", err) diff --git a/internal/client/apidocs/apidocs.go b/internal/client/apidocs/apidocs.go index f59e26366..f9db022ee 100644 --- a/internal/client/apidocs/apidocs.go +++ b/internal/client/apidocs/apidocs.go @@ -214,15 +214,14 @@ func Delete(siteid string, id string) (respBody []byte, err error) { func UpdateDocumentation(siteid string, id string, displayName string, openAPIDoc string, graphQLDoc string, endpointUri string, ) (respBody []byte, err error) { - var data map[string]interface{} // var payload string if openAPIDoc != "" { data = map[string]interface{}{ "oasDocumentation": map[string]interface{}{ "spec": map[string]interface{}{ - "displayName":displayName, - "contents":openAPIDoc, + "displayName": displayName, + "contents": openAPIDoc, }, }, } @@ -231,10 +230,10 @@ func UpdateDocumentation(siteid string, id string, displayName string, if graphQLDoc != "" { data = map[string]interface{}{ "graphqlDocumentation": map[string]interface{}{ - "endpointUri":endpointUri, + "endpointUri": endpointUri, "schema": map[string]interface{}{ - "displayName":displayName, - "contents":graphQLDoc, + "displayName": displayName, + "contents": graphQLDoc, }, }, } @@ -245,7 +244,7 @@ func UpdateDocumentation(siteid string, id string, displayName string, fmt.Printf("could not marshal json: %s\n", err) return } - payload := string(jsonData); + payload := string(jsonData) u, _ := url.Parse(apiclient.BaseURL) u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apidocs", id, "documentation") diff --git a/internal/client/securityprofiles/securityprofiles.go b/internal/client/securityprofiles/securityprofiles.go index f75ae15a1..a3ff31b48 100644 --- a/internal/client/securityprofiles/securityprofiles.go +++ b/internal/client/securityprofiles/securityprofiles.go @@ -18,8 +18,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/url" + "os" "path" + "path/filepath" + "regexp" "strconv" "strings" "sync" @@ -61,6 +65,58 @@ type category struct { Cors interface{} `json:"cors,omitempty"` } +type computeenvscore struct { + TimeRange timeInterval `json:"timeRange,omitempty"` + Filters []scoreFilter `json:"filters,omitempty"` + PageSize int `json:"pageSize,omitempty"` + PageToken string `json:"pageToken,omitempty"` +} + +type timeInterval struct { + StartTime string `json:"startTime,omitempty"` + EndTime string `json:"endTime,omitempty"` +} + +type scoreFilter struct { + ScorePath string `json:"scorePath,omitempty"` +} + +type scores struct { + Scores []score `json:"scores,omitempty"` + NextPageToken string `json:"nextPageToken,omitempty"` +} + +type score struct { + TimeRange timeInterval `json:"timeRange,omitempty"` + Component component `json:"component,omitempty"` + Subcomponents []component `json:"subcomponents,omitempty"` +} + +type component struct { + Score int `json:"score,omitempty"` + ScorePath string `json:"scorePath,omitempty"` + Recommendations []recommendation `json:"recommendations,omitempty"` + DataCaptureTime string `json:"dataCaptureTime,omitempty"` + CalculateTime string `json:"calculateTime,omitempty"` + DrilldownPaths []string `json:"drilldownPaths,omitempty"` +} + +type recommendation struct { + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + Impact int `json:"impact,omitempty"` + Actions []action `json:"actions,omitempty"` +} + +type action struct { + Description string `json:"description,omitempty"` + ActionContext actionContext `json:"actionContext,omitempty"` +} + +type actionContext struct { + DocumentationLink string `json:"documentationLink,omitempty"` +} + const maxPageSize = 50 // Create @@ -160,8 +216,69 @@ func Update(name string, content []byte) (respBody []byte, err error) { return respBody, err } +// Compute +func Compute(name string, startTime string, endTime string, filters []string, + pageSize int, pageToken string, full bool, +) (respBody []byte, err error) { + score := computeenvscore{} + if pageSize != -1 { + score.PageSize = pageSize + } + if pageToken != "" { + score.PageToken = pageToken + } + if startTime == "" || endTime == "" { + return nil, fmt.Errorf("startTime and endTime are mandatory") + } + score.TimeRange.StartTime = startTime + score.TimeRange.EndTime = endTime + + for _, f := range filters { + sf := scoreFilter{} + sf.ScorePath = f + score.Filters = append(score.Filters, sf) + } + + fullScores := scores{} + + for { + payload, err := json.Marshal(score) + if err != nil { + return nil, err + } + + u, _ := url.Parse(apiclient.BaseURL) + u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "securityProfiles", + name, "environments", apiclient.GetApigeeEnv()+":computeEnvironmentScores") + + if respBody, err = apiclient.HttpClient(u.String(), string(payload)); err != nil { + return nil, err + } + + if !full { + return respBody, err + } + + s := scores{} + if err = json.Unmarshal(respBody, &s); err != nil { + return nil, err + } + + fullScores.Scores = append(fullScores.Scores, s.Scores...) + score.PageToken = s.NextPageToken + + if s.NextPageToken == "" { + break + } + + } + + respBody, err = json.Marshal(fullScores) + return respBody, err +} + // Export -func Export(conn int, folder string, allRevisions bool) (err error) { +func Export(conn int, folder string) (err error) { apiclient.ClientPrintHttpResponse.Set(false) defer apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting()) @@ -207,7 +324,7 @@ func Export(conn int, folder string, allRevisions bool) (err error) { for i := 0; i < conn; i++ { fanOutWg.Add(1) - go exportWorker(&fanOutWg, workChan, folder, allRevisions, errChan) + go exportWorker(&fanOutWg, workChan, folder, errChan) } for _, i := range listsecprofiles.SecurityProfiles { @@ -226,46 +343,132 @@ func Export(conn int, folder string, allRevisions bool) (err error) { return nil } -func exportWorker(wg *sync.WaitGroup, workCh <-chan secprofile, folder string, allRevisions bool, errs chan<- error) { +func exportWorker(wg *sync.WaitGroup, workCh <-chan secprofile, folder string, errs chan<- error) { defer wg.Done() for { - var respBody []byte var err error - listsecprofiles := secprofiles{} - work, ok := <-workCh if !ok { return } - if allRevisions { - securityProfileName := work.Name[strings.LastIndex(work.Name, "/")+1:] - clilog.Info.Printf("Exporting all the revisions for Security Profile %s\n", securityProfileName) + if work.Name == "default" { + // do not export the default profile. this is set by the system + return + } - if respBody, err = ListRevisions(securityProfileName); err != nil { - errs <- err - } - err = json.Unmarshal(respBody, &listsecprofiles) - if err != nil { - errs <- err - } - for _, s := range listsecprofiles.SecurityProfiles { - payload, err := json.Marshal(s) - if err != nil { - errs <- err - } - payload, _ = apiclient.PrettifyJSON(payload) - apiclient.WriteByteArrayToFile(path.Join(folder, s.Name), false, payload) - } - } else { - clilog.Info.Printf("Exporting Security Profile %s\n", work.Name) - payload, err := json.Marshal(work) - if err != nil { - errs <- err + clilog.Info.Printf("Exporting Security Profile %s\n", work.Name) + payload, err := json.Marshal(work) + if err != nil { + errs <- err + } + payload, _ = apiclient.PrettifyJSON(payload) + apiclient.WriteByteArrayToFile(path.Join(folder, work.Name+".json"), false, payload) + } +} + +// Import +func Import(conn int, folder string) error { + var profiles []string + err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if filepath.Ext(path) != ".json" { + return nil + } + profiles = append(profiles, path) + return nil + }) + if err != nil { + return err + } + + if len(profiles) == 0 { + clilog.Warning.Println("No profiles were found in the folder") + return nil + } + + clilog.Debug.Printf("Found %d profiles in the folder\n", len(profiles)) + clilog.Debug.Printf("Importing security profiles with %d parallel connections\n", conn) + + jobChan := make(chan string) + errChan := make(chan error) + + fanOutWg := sync.WaitGroup{} + fanInWg := sync.WaitGroup{} + + errs := []string{} + fanInWg.Add(1) + go func() { + defer fanInWg.Done() + for { + newErr, ok := <-errChan + if !ok { + return } - payload, _ = apiclient.PrettifyJSON(payload) - apiclient.WriteByteArrayToFile(path.Join(folder, work.Name+".json"), false, payload) + errs = append(errs, newErr.Error()) + } + }() + + for i := 0; i < conn; i++ { + fanOutWg.Add(1) + go importProfile(&fanOutWg, jobChan, errChan) + } + + for _, profile := range profiles { + jobChan <- profile + } + close(jobChan) + fanOutWg.Wait() + close(errChan) + fanInWg.Wait() + + if len(errs) > 0 { + return errors.New(strings.Join(errs, "\n")) + } + return nil +} + +func importProfile(wg *sync.WaitGroup, jobs <-chan string, errs chan<- error) { + defer wg.Done() + for { + job, ok := <-jobs + if !ok { + return + } + content, err := readFile(job) + if err != nil { + errs <- err + continue + } + if _, err = Create(getSecurityProfileName(job), content); err != nil { + errs <- err + continue } } } + +func readFile(filePath string) (byteValue []byte, err error) { + userFile, err := os.Open(filePath) + if err != nil { + return nil, err + } + + defer userFile.Close() + + byteValue, err = io.ReadAll(userFile) + if err != nil { + return nil, err + } + return byteValue, err +} + +func getSecurityProfileName(job string) (name string) { + ver := regexp.MustCompile(`@\d+`) + return ver.ReplaceAllString(strings.TrimSuffix(filepath.Base(job), filepath.Ext(job)), "") +}