Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update postman metadata #3852

Merged
merged 13 commits into from
Jan 30, 2025
Merged

Update postman metadata #3852

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3483447
Added more fields to the Postman source metadata
casey-tran Jan 17, 2025
2431d25
Updated metadata fields
casey-tran Jan 23, 2025
a013c66
Updated protobuf fields to accomodate unknown
casey-tran Jan 23, 2025
3976cb2
Modified some stuff
casey-tran Jan 24, 2025
f9cfd97
Added ugly check to reassign the enum value to the string
casey-tran Jan 24, 2025
1427264
Updated protobuf fields, added location data as chunks get made, and …
casey-tran Jan 25, 2025
101fc65
Took globals out of protobuf definition
casey-tran Jan 28, 2025
55e6d4f
Disabled body scanning
casey-tran Jan 29, 2025
db49d77
Reverted script tab change
casey-tran Jan 29, 2025
a1d1c90
Removed unused global constant
casey-tran Jan 29, 2025
5718f09
Merge branch 'main' into update-postman-metadata
casey-tran Jan 29, 2025
40e55b2
Removed body function
casey-tran Jan 29, 2025
9c6f2a1
Updated protobuf fields and names. Also added some commentary
casey-tran Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions pkg/output/plain.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,16 @@ func structToMap(obj any) (m map[string]map[string]any, err error) {
return
}
err = json.Unmarshal(data, &m)
// Due to PostmanLocationType protobuf field being an enum, we want to be able to assign the string value of the enum to the field without needing to create another Protobuf field.
// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID is filled since every secret in the Postman source
// should have a valid workspace ID and the 0 value is considered nil for integers.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I love this comment - it explains the problem as well as why you picked the solution you did.

if m["Postman"]["workspace_uuid"] != nil {
if m["Postman"]["location_type"] == nil {
m["Postman"]["location_type"] = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN.String()
} else {
m["Postman"]["location_type"] = obj.(*source_metadatapb.MetaData_Postman).Postman.LocationType.String()
}
}
return
}

Expand Down
705 changes: 408 additions & 297 deletions pkg/pb/source_metadatapb/source_metadata.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 1 addition & 5 deletions pkg/pb/source_metadatapb/source_metadata.pb.validate.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

108 changes: 64 additions & 44 deletions pkg/sources/postman/postman.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
const (
SourceType = sourcespb.SourceType_SOURCE_TYPE_POSTMAN
LINK_BASE_URL = "https://go.postman.co/"
GLOBAL_TYPE = "globals"
ENVIRONMENT_TYPE = "environment"
AUTH_TYPE = "authorization"
REQUEST_TYPE = "request"
Expand Down Expand Up @@ -147,7 +146,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
if err = json.Unmarshal(contents, &env); err != nil {
return err
}
s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentName: env.ID, fromLocal: true, Link: envPath}, env)
s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentID: env.ID, EnvironmentName: env.Name, fromLocal: true, Link: envPath, LocationType: source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE}, env)
}

// Scan local workspaces
Expand Down Expand Up @@ -230,7 +229,9 @@ func (s *Source) scanLocalWorkspace(ctx context.Context, chunksChan chan *source

for _, environment := range workspace.EnvironmentsRaw {
metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/environments/" + environment.ID + ".json"
metadata.LocationType = source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE
s.scanVariableData(ctx, chunksChan, metadata, environment)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}
for _, collection := range workspace.CollectionsRaw {
metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/collections/" + collection.Info.PostmanID + ".json"
Expand Down Expand Up @@ -265,13 +266,20 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
metadata.Link = LINK_BASE_URL + "environments/" + envID.UUID
metadata.FullID = envVars.ID
metadata.EnvironmentID = envID.UUID
metadata.EnvironmentName = envVars.Name

ctx.Logger().V(2).Info("scanning environment vars", "environment_uuid", metadata.FullID)
for _, word := range strings.Split(envVars.Name, " ") {
s.attemptToAddKeyword(word)
}

metadata.LocationType = source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE
s.scanVariableData(ctx, chunksChan, metadata, envVars)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
metadata.Type = ""
metadata.Link = ""
metadata.FullID = ""
metadata.EnvironmentID = ""
metadata.EnvironmentName = ""
ctx.Logger().V(2).Info("finished scanning environment vars", "environment_uuid", metadata.FullID)
}
ctx.Logger().V(2).Info("finished scanning environments")
Expand Down Expand Up @@ -305,11 +313,13 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
metadata.Link = LINK_BASE_URL + COLLECTION_TYPE + "/" + metadata.FullID
}

metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_VARIABLE
// variables must be scanned first before drilling down into the folders and events
// because we need to pick up the substitutions from the top level collection variables
s.scanVariableData(ctx, chunksChan, metadata, VariableData{
KeyValues: collection.Variables,
})
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN

for _, event := range collection.Events {
s.scanEvent(ctx, chunksChan, metadata, event)
Expand Down Expand Up @@ -345,6 +355,7 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c

// check if there are any requests in the folder
if item.Request.Method != "" {
metadata.FolderName = strings.Replace(metadata.FolderName, (" > " + item.Name), "", -1)
metadata.RequestID = item.ID
metadata.RequestName = item.Name
metadata.Type = REQUEST_TYPE
Expand All @@ -368,8 +379,16 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
s.scanEvent(ctx, chunksChan, metadata, event)
}

if metadata.RequestID != "" {
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_AUTHORIZATION
} else if metadata.FolderID != "" {
metadata.LocationType = source_metadatapb.PostmanLocationType_FOLDER_AUTHORIZATION
} else if metadata.CollectionInfo.UID != "" {
metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
}
// an auth all by its lonesome could be inherited to subfolders and requests
s.scanAuth(ctx, chunksChan, metadata, item.Auth, item.Request.URL)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, event Event) {
Expand All @@ -378,15 +397,24 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,

// Prep direct links. Ignore updating link if it's a local JSON file
if !metadata.fromLocal {
metadata.Link = LINK_BASE_URL + metadata.Type + "/" + metadata.FullID
metadata.Link = LINK_BASE_URL + (strings.Replace(metadata.Type, " > event", "", -1)) + "/" + metadata.FullID
if event.Listen == "prerequest" {
metadata.Link += "?tab=pre-request-scripts"
} else {
metadata.Link += "?tab=tests"
}
}

if strings.Contains(metadata.Type, REQUEST_TYPE) {
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_SCRIPT
} else if strings.Contains(metadata.Type, FOLDER_TYPE) {
metadata.LocationType = source_metadatapb.PostmanLocationType_FOLDER_SCRIPT
} else if strings.Contains(metadata.Type, COLLECTION_TYPE) {
metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
}

s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, data)), metadata)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, auth Auth, u URL) {
Expand Down Expand Up @@ -471,7 +499,16 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
s.attemptToAddKeyword(authData)

m.FieldType = AUTH_TYPE

if strings.Contains(m.Type, REQUEST_TYPE) {
m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_AUTHORIZATION
} else if strings.Contains(m.Type, FOLDER_TYPE) {
m.LocationType = source_metadatapb.PostmanLocationType_FOLDER_AUTHORIZATION
} else if strings.Contains(m.Type, COLLECTION_TYPE) {
m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, authData)), m)
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, r Request) {
Expand All @@ -484,66 +521,38 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
KeyValues: r.Header,
}
metadata.Type = originalType + " > header"
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
s.scanVariableData(ctx, chunksChan, metadata, vars)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if r.URL.Raw != "" {
metadata.Type = originalType + " > request URL (no query parameters)"
// Note: query parameters are handled separately
u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, u)), metadata)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if len(r.URL.Query) > 0 {
vars := VariableData{
KeyValues: r.URL.Query,
}
metadata.Type = originalType + " > GET parameters (query)"
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_QUERY_PARAMETER
s.scanVariableData(ctx, chunksChan, metadata, vars)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if r.Auth.Type != "" {
metadata.Type = originalType + " > request auth"
s.scanAuth(ctx, chunksChan, metadata, r.Auth, r.URL)
}

if r.Body.Mode != "" {
metadata.Type = originalType + " > body"
s.scanBody(ctx, chunksChan, metadata, r.Body)
}
}

func (s *Source) scanBody(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, b Body) {
if !m.fromLocal {
m.Link = m.Link + "?tab=body"
}
originalType := m.Type
switch b.Mode {
case "formdata":
m.Type = originalType + " > form data"
vars := VariableData{
KeyValues: b.FormData,
}
s.scanVariableData(ctx, chunksChan, m, vars)
case "urlencoded":
m.Type = originalType + " > url encoded"
vars := VariableData{
KeyValues: b.URLEncoded,
}
s.scanVariableData(ctx, chunksChan, m, vars)
case "raw", "graphql":
data := b.Raw
if b.Mode == "graphql" {
m.Type = originalType + " > graphql"
data = b.GraphQL.Query + " " + b.GraphQL.Variables
}
if b.Mode == "raw" {
m.Type = originalType + " > raw"
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
default:
break
}
// We would scan the body, but currently the body has different radio buttons that can be scanned but only the selected one is scanned. The unselected radio button options can still
// have secrets in them but will not be scanned. The selction of the radio button will also change the secret metadata for that particular scanning pass and can create confusion for
// the user as to the status of a secret. We will reimplement at some point.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a little unclear - does the API prevent us from scanning the body data from the unselected fields?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Postman API will need further exploration to answer that question Joe. As it is written in main currently, I'm not seeing unselected fields getting scanned.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Understood. I think it's still worth it to scan the body data that we can see, but if that confuses the user, then I understand punting it down the road. I'll just add that if the Postman API doesn't allow us to scan all body types now, it likely never will, so the existing implementation might be the only option.

}

func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, response Response) {
Expand All @@ -558,13 +567,17 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
KeyValues: response.Header,
}
m.Type = originalType + " > response header"
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
s.scanVariableData(ctx, chunksChan, m, vars)
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

// Body in a response is just a string
if response.Body != "" {
m.Type = originalType + " > response body"
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, response.Body)), m)
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if response.OriginalRequest.Method != "" {
Expand Down Expand Up @@ -600,14 +613,22 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
}

m.FieldType = m.Type + " variables"
switch m.FieldType {
case "request > GET parameters (query) variables":
m.Link = m.Link + "?tab=params"
case "request > header variables":
m.Link = m.Link + "?tab=headers"
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(values), m)
}

func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) {
if data == "" {
return
}
metadata.FieldType = metadata.Type
if metadata.FieldType == "" {
metadata.FieldType = metadata.Type
}

chunksChan <- &sources.Chunk{
SourceType: s.Type(),
Expand All @@ -630,8 +651,7 @@ func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, d
FolderId: metadata.FolderID,
FolderName: metadata.FolderName,
FieldType: metadata.FieldType,
FieldName: metadata.FieldName,
VariableType: metadata.VarType,
LocationType: metadata.LocationType,
},
},
},
Expand Down
9 changes: 5 additions & 4 deletions pkg/sources/postman/postman_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"io"
"net/http"
"time"

"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
)

const (
Expand Down Expand Up @@ -68,17 +70,16 @@ type Metadata struct {
EnvironmentID string
CollectionInfo Info
FolderID string // UUID of the folder (but not full ID)
FolderName string
FolderName string // Folder path if the item is nested under one or more folders
RequestID string // UUID of the request (but not full ID)
RequestName string
FullID string //full ID of the reference item (created_by + ID) OR just the UUID
Link string //direct link to the folder (could be .json file path)
Type string //folder, request, etc.
EnvironmentName string
VarType string
FieldName string
FieldType string
FieldType string // Path of the item type
fromLocal bool
LocationType source_metadatapb.PostmanLocationType // The distinct Postman location type that the item falls under
}

type Collection struct {
Expand Down
12 changes: 6 additions & 6 deletions pkg/sources/postman/substitution_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,14 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
s := &Source{
sub: NewSubstitution(),
}
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var1", "value1")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var2", "value2")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "", "value2")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token", "'{{continuation_token}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var1", "value1")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var2", "value2")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "", "value2")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token", "'{{continuation_token}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation

metadata := Metadata{
Type: GLOBAL_TYPE,
Type: ENVIRONMENT_TYPE,
}

testCases := []struct {
Expand Down
29 changes: 25 additions & 4 deletions proto/source_metadata.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,10 +296,11 @@ message AzureRepos {
}

message Postman {
reserved 4, 14, 15;
reserved "globals_id", "field_name", "variable_type";
string link = 1;
string workspace_uuid = 2;
string workspace_name = 3;
string globals_id = 4;
string collection_id = 5;
string collection_name = 6;
string environment_id = 7;
Expand All @@ -308,9 +309,29 @@ message Postman {
string request_name = 10;
string folder_id = 11;
string folder_name = 12;
string field_type = 13;
string field_name = 14;
string variable_type = 15;
Comment on lines -312 to -313
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reserve these numbers (https://protobuf.dev/best-practices/dos-donts/#reserve-tag-numbers)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done and for globals_id as well. I also reserved the names too.

string field_type = 13; // Do not use this field for transport. It is only used for output to STDOUT.
PostmanLocationType location_type = 16;
}

enum PostmanLocationType {
UNKNOWN_POSTMAN = 0;
REQUEST_QUERY_PARAMETER = 1;
REQUEST_AUTHORIZATION = 2;
REQUEST_HEADER = 3;
REQUEST_BODY_FORM_DATA = 4;
REQUEST_BODY_RAW = 5;
REQUEST_BODY_URL_ENCODED = 6;
REQUEST_BODY_GRAPHQL = 7;
REQUEST_SCRIPT = 8;
REQUEST_URL = 9;
ENVIRONMENT_VARIABLE = 10;
FOLDER_AUTHORIZATION = 11;
FOLDER_SCRIPT = 12;
COLLECTION_SCRIPT = 13;
COLLECTION_VARIABLE = 14;
COLLECTION_AUTHORIZATION = 15;
RESPONSE_BODY = 16;
RESPONSE_HEADER = 17;
}

message Vector {
Expand Down
Loading