From 6d5c7cffc4b3613d6def8d38291ac060cb042f5e Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Thu, 6 Feb 2025 12:06:42 -0600 Subject: [PATCH 1/7] Updated Postman source logging --- pkg/sources/postman/postman.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index b2838ae6ab51..5624873799e6 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -307,7 +307,7 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu // scanCollection scans a collection and all its items, folders, and requests. // locally scoped Metadata is updated as we drill down into the collection. func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, collection Collection) { - ctx.Logger().V(2).Info("starting scanning collection", collection.Info.Name, "uuid", collection.Info.UID) + ctx.Logger().V(2).Info("starting to scan collection", "collection name", collection.Info.Name, "collection uuid", collection.Info.UID) metadata.CollectionInfo = collection.Info metadata.Type = COLLECTION_TYPE s.attemptToAddKeyword(collection.Info.Name) @@ -637,7 +637,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources. func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, variableData VariableData) { if len(variableData.KeyValues) == 0 { - ctx.Logger().V(2).Info("no variables to scan", "type", m.Type, "uuid", m.FullID) + ctx.Logger().V(2).Info("no variables to scan", "type", m.Type, "item uuid", m.FullID) return } @@ -673,12 +673,14 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources. func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) { if data == "" { + ctx.Logger().V(5).Info("Data string is empty", "workspace ID", metadata.WorkspaceUUID) return } if metadata.FieldType == "" { metadata.FieldType = metadata.Type } + ctx.Logger().V(5).Info("Generating chunk and passing it to the channel", "link", metadata.Link) chunksChan <- &sources.Chunk{ SourceType: s.Type(), SourceName: s.name, From d4bb2eb00082736aad7d696e13bb78dac1cb403c Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Wed, 19 Feb 2025 14:00:19 -0600 Subject: [PATCH 2/7] Added logging for enumeration --- pkg/sources/postman/postman.go | 1 + pkg/sources/postman/postman_client.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index 5624873799e6..cb891af1ebc7 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -210,6 +210,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . if err != nil { return fmt.Errorf("error enumerating postman workspaces: %w", err) } + ctx.Logger().Info("Enumerated workspaces", "count", len(workspaces)) for _, workspace := range workspaces { s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspace.ID), "") if err = s.scanWorkspace(ctx, chunksChan, workspace); err != nil { diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 0d2733797375..0682f9c68b12 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -7,6 +7,8 @@ import ( "net/http" "time" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" ) @@ -250,6 +252,7 @@ func (c *Client) getPostmanReq(url string, headers map[string]string) (*http.Res // EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal). // Consider adding additional flags to support filtering. func (c *Client) EnumerateWorkspaces() ([]Workspace, error) { + context.Background().Logger().Info("Enumerating workspaces") var workspaces []Workspace workspacesObj := struct { Workspaces []Workspace `json:"workspaces"` From 18f1fa87074790b4134eb29e26c2f647467b2b30 Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Thu, 20 Feb 2025 16:36:17 -0600 Subject: [PATCH 3/7] Got workspace enumeration to go with the additional step of making a request to every workspace listed --- pkg/sources/postman/postman.go | 3 ++- pkg/sources/postman/postman_client.go | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index cb891af1ebc7..fb405f7f90e6 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -185,6 +185,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . return fmt.Errorf("error getting workspace %s: %w", workspaceID, err) } s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspaceID), "") + ctx.Logger().V(3).Info("Scanning workspace from workspaces given", "workspace", workspaceID) if err = s.scanWorkspace(ctx, chunksChan, w); err != nil { return fmt.Errorf("error scanning workspace %s: %w", workspaceID, err) } @@ -210,7 +211,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . if err != nil { return fmt.Errorf("error enumerating postman workspaces: %w", err) } - ctx.Logger().Info("Enumerated workspaces", "count", len(workspaces)) + ctx.Logger().V(3).Info("enumerated workspaces", "workspaces", workspaces) for _, workspace := range workspaces { s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspace.ID), "") if err = s.scanWorkspace(ctx, chunksChan, workspace); err != nil { diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 0682f9c68b12..81cfd8103d43 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -276,6 +276,15 @@ func (c *Client) EnumerateWorkspaces() ([]Workspace, error) { return workspaces, err } + for i, workspace := range workspacesObj.Workspaces { + workspacesObj.Workspaces[i], err = c.GetWorkspace(workspace.ID) + if err != nil { + err = fmt.Errorf("could not get workspace during enumeration: %s (%s)", workspace.Name, workspace.ID) + return workspaces, err + } + context.Background().Logger().V(4).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) + } + return workspacesObj.Workspaces, nil } From cdb06a8ae0c1dc3b96d8abb957e41d1c875e346a Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Thu, 20 Feb 2025 17:22:07 -0600 Subject: [PATCH 4/7] Edited logging levels --- pkg/sources/postman/postman.go | 8 ++++---- pkg/sources/postman/postman_client.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index fb405f7f90e6..f877becbc44f 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -185,7 +185,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . return fmt.Errorf("error getting workspace %s: %w", workspaceID, err) } s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspaceID), "") - ctx.Logger().V(3).Info("Scanning workspace from workspaces given", "workspace", workspaceID) + ctx.Logger().V(2).Info("scanning workspace from workspaces given", "workspace", workspaceID) if err = s.scanWorkspace(ctx, chunksChan, w); err != nil { return fmt.Errorf("error scanning workspace %s: %w", workspaceID, err) } @@ -211,7 +211,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . if err != nil { return fmt.Errorf("error enumerating postman workspaces: %w", err) } - ctx.Logger().V(3).Info("enumerated workspaces", "workspaces", workspaces) + ctx.Logger().V(2).Info("enumerated workspaces", "workspaces", workspaces) for _, workspace := range workspaces { s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspace.ID), "") if err = s.scanWorkspace(ctx, chunksChan, workspace); err != nil { @@ -675,14 +675,14 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources. func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) { if data == "" { - ctx.Logger().V(5).Info("Data string is empty", "workspace ID", metadata.WorkspaceUUID) + ctx.Logger().V(3).Info("Data string is empty", "workspace ID", metadata.WorkspaceUUID) return } if metadata.FieldType == "" { metadata.FieldType = metadata.Type } - ctx.Logger().V(5).Info("Generating chunk and passing it to the channel", "link", metadata.Link) + ctx.Logger().V(3).Info("Generating chunk and passing it to the channel", "link", metadata.Link) chunksChan <- &sources.Chunk{ SourceType: s.Type(), SourceName: s.name, diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 81cfd8103d43..40595a5fb957 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -252,7 +252,7 @@ func (c *Client) getPostmanReq(url string, headers map[string]string) (*http.Res // EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal). // Consider adding additional flags to support filtering. func (c *Client) EnumerateWorkspaces() ([]Workspace, error) { - context.Background().Logger().Info("Enumerating workspaces") + context.Background().Logger().V(2).Info("enumerating workspaces") var workspaces []Workspace workspacesObj := struct { Workspaces []Workspace `json:"workspaces"` @@ -282,7 +282,7 @@ func (c *Client) EnumerateWorkspaces() ([]Workspace, error) { err = fmt.Errorf("could not get workspace during enumeration: %s (%s)", workspace.Name, workspace.ID) return workspaces, err } - context.Background().Logger().V(4).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) + context.Background().Logger().V(3).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) } return workspacesObj.Workspaces, nil From a78dd096ed4c2f8c17611cc855ecd3fbaefe039c Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Fri, 21 Feb 2025 11:11:28 -0600 Subject: [PATCH 5/7] Updated context parameter and logging snake case --- pkg/sources/postman/postman.go | 8 ++++---- pkg/sources/postman/postman_client.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index f877becbc44f..b97c1fadc177 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -207,7 +207,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . // Scan personal workspaces (from API token) if s.conn.Workspaces == nil && s.conn.Collections == nil && s.conn.Environments == nil && s.conn.GetToken() != "" { - workspaces, err := s.client.EnumerateWorkspaces() + workspaces, err := s.client.EnumerateWorkspaces(ctx) if err != nil { return fmt.Errorf("error enumerating postman workspaces: %w", err) } @@ -309,7 +309,7 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu // scanCollection scans a collection and all its items, folders, and requests. // locally scoped Metadata is updated as we drill down into the collection. func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, collection Collection) { - ctx.Logger().V(2).Info("starting to scan collection", "collection name", collection.Info.Name, "collection uuid", collection.Info.UID) + ctx.Logger().V(2).Info("starting to scan collection", "collection_name", collection.Info.Name, "collection_uuid", collection.Info.UID) metadata.CollectionInfo = collection.Info metadata.Type = COLLECTION_TYPE s.attemptToAddKeyword(collection.Info.Name) @@ -639,7 +639,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources. func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, variableData VariableData) { if len(variableData.KeyValues) == 0 { - ctx.Logger().V(2).Info("no variables to scan", "type", m.Type, "item uuid", m.FullID) + ctx.Logger().V(2).Info("no variables to scan", "type", m.Type, "item_uuid", m.FullID) return } @@ -675,7 +675,7 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources. func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) { if data == "" { - ctx.Logger().V(3).Info("Data string is empty", "workspace ID", metadata.WorkspaceUUID) + ctx.Logger().V(3).Info("Data string is empty", "workspace_id", metadata.WorkspaceUUID) return } if metadata.FieldType == "" { diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 40595a5fb957..38038c126f38 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -251,8 +251,8 @@ func (c *Client) getPostmanReq(url string, headers map[string]string) (*http.Res // EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal). // Consider adding additional flags to support filtering. -func (c *Client) EnumerateWorkspaces() ([]Workspace, error) { - context.Background().Logger().V(2).Info("enumerating workspaces") +func (c *Client) EnumerateWorkspaces(ctx context.Context) ([]Workspace, error) { + ctx.Logger().V(2).Info("enumerating workspaces") var workspaces []Workspace workspacesObj := struct { Workspaces []Workspace `json:"workspaces"` @@ -282,7 +282,7 @@ func (c *Client) EnumerateWorkspaces() ([]Workspace, error) { err = fmt.Errorf("could not get workspace during enumeration: %s (%s)", workspace.Name, workspace.ID) return workspaces, err } - context.Background().Logger().V(3).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) + ctx.Logger().V(3).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) } return workspacesObj.Workspaces, nil From d16fe2d1857b58e871ecdef9f313fb8141337240 Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Fri, 21 Feb 2025 12:17:43 -0600 Subject: [PATCH 6/7] When errors occur, return explicit null values instead of zero variable --- pkg/sources/postman/postman.go | 2 +- pkg/sources/postman/postman_client.go | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go index b97c1fadc177..6aa2ccacf8e0 100644 --- a/pkg/sources/postman/postman.go +++ b/pkg/sources/postman/postman.go @@ -180,7 +180,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ . // Scan workspaces for _, workspaceID := range s.conn.Workspaces { - w, err := s.client.GetWorkspace(workspaceID) + w, err := s.client.GetWorkspace(ctx, workspaceID) if err != nil { return fmt.Errorf("error getting workspace %s: %w", workspaceID, err) } diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 38038c126f38..083c78791220 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -253,7 +253,6 @@ func (c *Client) getPostmanReq(url string, headers map[string]string) (*http.Res // Consider adding additional flags to support filtering. func (c *Client) EnumerateWorkspaces(ctx context.Context) ([]Workspace, error) { ctx.Logger().V(2).Info("enumerating workspaces") - var workspaces []Workspace workspacesObj := struct { Workspaces []Workspace `json:"workspaces"` }{} @@ -261,26 +260,28 @@ func (c *Client) EnumerateWorkspaces(ctx context.Context) ([]Workspace, error) { r, err := c.getPostmanReq("https://api.getpostman.com/workspaces", nil) if err != nil { err = fmt.Errorf("could not get workspaces") - return workspaces, err + return nil, err } body, err := io.ReadAll(r.Body) if err != nil { err = fmt.Errorf("could not read response body for workspaces") - return workspaces, err + return nil, err } r.Body.Close() if err := json.Unmarshal([]byte(body), &workspacesObj); err != nil { err = fmt.Errorf("could not unmarshal workspaces JSON") - return workspaces, err + return nil, err } for i, workspace := range workspacesObj.Workspaces { - workspacesObj.Workspaces[i], err = c.GetWorkspace(workspace.ID) + tempWorkspace, err := c.GetWorkspace(ctx, workspace.ID) if err != nil { err = fmt.Errorf("could not get workspace during enumeration: %s (%s)", workspace.Name, workspace.ID) - return workspaces, err + return nil, err + } else { + workspacesObj.Workspaces[i] = tempWorkspace } ctx.Logger().V(3).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) } @@ -289,8 +290,8 @@ func (c *Client) EnumerateWorkspaces(ctx context.Context) ([]Workspace, error) { } // GetWorkspace returns the workspace for a given workspace -func (c *Client) GetWorkspace(workspaceUUID string) (Workspace, error) { - var workspace Workspace +func (c *Client) GetWorkspace(ctx context.Context, workspaceUUID string) (Workspace, error) { + ctx.Logger().V(2).Info("getting workspace", "workspace", workspaceUUID) obj := struct { Workspace Workspace `json:"workspace"` }{} @@ -299,19 +300,19 @@ func (c *Client) GetWorkspace(workspaceUUID string) (Workspace, error) { r, err := c.getPostmanReq(url, nil) if err != nil { err = fmt.Errorf("could not get workspace: %s", workspaceUUID) - return workspace, err + return Workspace{}, err } body, err := io.ReadAll(r.Body) if err != nil { err = fmt.Errorf("could not read response body for workspace: %s", workspaceUUID) - return workspace, err + return Workspace{}, err } r.Body.Close() if err := json.Unmarshal([]byte(body), &obj); err != nil { err = fmt.Errorf("could not unmarshal workspace JSON for workspace: %s", workspaceUUID) - return workspace, err + return Workspace{}, err } return obj.Workspace, nil From 30daef845c755aec2a6d6cba079006fb96b0b1ab Mon Sep 17 00:00:00 2001 From: Casey Tran Date: Fri, 21 Feb 2025 12:44:22 -0600 Subject: [PATCH 7/7] Updated returned error message --- pkg/sources/postman/postman_client.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go index 083c78791220..c939ac8be093 100644 --- a/pkg/sources/postman/postman_client.go +++ b/pkg/sources/postman/postman_client.go @@ -278,12 +278,11 @@ func (c *Client) EnumerateWorkspaces(ctx context.Context) ([]Workspace, error) { for i, workspace := range workspacesObj.Workspaces { tempWorkspace, err := c.GetWorkspace(ctx, workspace.ID) if err != nil { - err = fmt.Errorf("could not get workspace during enumeration: %s (%s)", workspace.Name, workspace.ID) - return nil, err - } else { - workspacesObj.Workspaces[i] = tempWorkspace + return nil, fmt.Errorf("could not get workspace %q (%s) during enumeration: %w", workspace.Name, workspace.ID, err) } - ctx.Logger().V(3).Info("individual workspace getting added to the array", "workspace", workspacesObj.Workspaces[i]) + workspacesObj.Workspaces[i] = tempWorkspace + + ctx.Logger().V(3).Info("individual workspace getting added to the slice", "workspace", workspacesObj.Workspaces[i]) } return workspacesObj.Workspaces, nil