Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update postman metadata #3852

Merged
merged 13 commits into from
Jan 30, 2025
Merged

Update postman metadata #3852

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3483447
Added more fields to the Postman source metadata
casey-tran Jan 17, 2025
2431d25
Updated metadata fields
casey-tran Jan 23, 2025
a013c66
Updated protobuf fields to accomodate unknown
casey-tran Jan 23, 2025
3976cb2
Modified some stuff
casey-tran Jan 24, 2025
f9cfd97
Added ugly check to reassign the enum value to the string
casey-tran Jan 24, 2025
1427264
Updated protobuf fields, added location data as chunks get made, and …
casey-tran Jan 25, 2025
101fc65
Took globals out of protobuf definition
casey-tran Jan 28, 2025
55e6d4f
Disabled body scanning
casey-tran Jan 29, 2025
db49d77
Reverted script tab change
casey-tran Jan 29, 2025
a1d1c90
Removed unused global constant
casey-tran Jan 29, 2025
5718f09
Merge branch 'main' into update-postman-metadata
casey-tran Jan 29, 2025
40e55b2
Removed body function
casey-tran Jan 29, 2025
9c6f2a1
Updated protobuf fields and names. Also added some commentary
casey-tran Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pkg/output/plain.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package output
import (
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -110,6 +111,9 @@ func structToMap(obj any) (m map[string]map[string]any, err error) {
return
}
err = json.Unmarshal(data, &m)
if reflect.TypeOf(obj) == reflect.TypeOf(&source_metadatapb.MetaData_Postman{}) {
m["Postman"]["location"] = obj.(*source_metadatapb.MetaData_Postman).Postman.Location.String()
}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably no need to reflect; just check the generated dictionary

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also pls add an explanatory (apologetic) comment

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

return
}

Expand Down
694 changes: 400 additions & 294 deletions pkg/pb/source_metadatapb/source_metadata.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 1 addition & 5 deletions pkg/pb/source_metadatapb/source_metadata.pb.validate.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

78 changes: 69 additions & 9 deletions pkg/sources/postman/postman.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
const (
SourceType = sourcespb.SourceType_SOURCE_TYPE_POSTMAN
LINK_BASE_URL = "https://go.postman.co/"
GLOBAL_TYPE = "globals"
ENVIRONMENT_TYPE = "environment"
AUTH_TYPE = "authorization"
REQUEST_TYPE = "request"
Expand Down Expand Up @@ -147,7 +146,7 @@
if err = json.Unmarshal(contents, &env); err != nil {
return err
}
s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentName: env.ID, fromLocal: true, Link: envPath}, env)
s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentID: env.ID, EnvironmentName: env.Name, fromLocal: true, Link: envPath, Location: source_metadatapb.PostmanLocation_ENVIRONMENT_VARIABLE}, env)
}

// Scan local workspaces
Expand Down Expand Up @@ -230,7 +229,9 @@

for _, environment := range workspace.EnvironmentsRaw {
metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/environments/" + environment.ID + ".json"
metadata.Location = source_metadatapb.PostmanLocation_ENVIRONMENT_VARIABLE
s.scanVariableData(ctx, chunksChan, metadata, environment)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}
for _, collection := range workspace.CollectionsRaw {
metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/collections/" + collection.Info.PostmanID + ".json"
Expand Down Expand Up @@ -265,14 +266,21 @@
metadata.Link = LINK_BASE_URL + "environments/" + envID.UUID
metadata.FullID = envVars.ID
metadata.EnvironmentID = envID.UUID
metadata.EnvironmentName = envVars.Name

ctx.Logger().V(2).Info("scanning environment vars", "environment_uuid", metadata.FullID)
for _, word := range strings.Split(envVars.Name, " ") {
s.attemptToAddKeyword(word)
}

metadata.Location = source_metadatapb.PostmanLocation_ENVIRONMENT_VARIABLE
s.scanVariableData(ctx, chunksChan, metadata, envVars)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
ctx.Logger().V(2).Info("finished scanning environment vars", "environment_uuid", metadata.FullID)
metadata.Type = ""
metadata.Link = ""
metadata.FullID = ""
metadata.EnvironmentID = ""
metadata.EnvironmentName = ""
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

log before or after, but not in the middle!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

}
ctx.Logger().V(2).Info("finished scanning environments")

Expand Down Expand Up @@ -305,11 +313,13 @@
metadata.Link = LINK_BASE_URL + COLLECTION_TYPE + "/" + metadata.FullID
}

metadata.Location = source_metadatapb.PostmanLocation_COLLECTION_VARIABLE
// variables must be scanned first before drilling down into the folders and events
// because we need to pick up the substitutions from the top level collection variables
s.scanVariableData(ctx, chunksChan, metadata, VariableData{
KeyValues: collection.Variables,
})
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN

for _, event := range collection.Events {
s.scanEvent(ctx, chunksChan, metadata, event)
Expand Down Expand Up @@ -368,8 +378,16 @@
s.scanEvent(ctx, chunksChan, metadata, event)
}

if metadata.RequestID != "" {
metadata.Location = source_metadatapb.PostmanLocation_REQUEST_AUTHORIZATION
} else if metadata.FolderID != "" {
metadata.Location = source_metadatapb.PostmanLocation_FOLDER_AUTHORIZATION
} else if metadata.CollectionInfo.UID != "" {
metadata.Location = source_metadatapb.PostmanLocation_COLLECTION_AUTHORIZATION
}
// an auth all by its lonesome could be inherited to subfolders and requests
s.scanAuth(ctx, chunksChan, metadata, item.Auth, item.Request.URL)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, event Event) {
Expand All @@ -378,15 +396,24 @@

// Prep direct links. Ignore updating link if it's a local JSON file
if !metadata.fromLocal {
metadata.Link = LINK_BASE_URL + metadata.Type + "/" + metadata.FullID
metadata.Link = LINK_BASE_URL + (strings.Replace(metadata.Type, " > event", "", -1)) + "/" + metadata.FullID
if event.Listen == "prerequest" {
metadata.Link += "?tab=pre-request-scripts"
} else {
metadata.Link += "?tab=tests"
}
}

if strings.Contains(metadata.Type, REQUEST_TYPE) {
metadata.Location = source_metadatapb.PostmanLocation_REQUEST_SCRIPT
} else if strings.Contains(metadata.Type, FOLDER_TYPE) {
metadata.Location = source_metadatapb.PostmanLocation_FOLDER_SCRIPT
} else if strings.Contains(metadata.Type, COLLECTION_TYPE) {
metadata.Location = source_metadatapb.PostmanLocation_COLLECTION_SCRIPT
}

s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, data)), metadata)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, auth Auth, u URL) {
Expand Down Expand Up @@ -471,7 +498,16 @@
s.attemptToAddKeyword(authData)

m.FieldType = AUTH_TYPE

if strings.Contains(m.Type, REQUEST_TYPE) {
m.Location = source_metadatapb.PostmanLocation_REQUEST_AUTHORIZATION
} else if strings.Contains(m.Type, FOLDER_TYPE) {
m.Location = source_metadatapb.PostmanLocation_FOLDER_AUTHORIZATION
} else if strings.Contains(m.Type, COLLECTION_TYPE) {
m.Location = source_metadatapb.PostmanLocation_COLLECTION_AUTHORIZATION
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, authData)), m)
m.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, r Request) {
Expand All @@ -484,36 +520,42 @@
KeyValues: r.Header,
}
metadata.Type = originalType + " > header"
metadata.Location = source_metadatapb.PostmanLocation_REQUEST_HEADER
s.scanVariableData(ctx, chunksChan, metadata, vars)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

if r.URL.Raw != "" {
metadata.Type = originalType + " > request URL (no query parameters)"
// Note: query parameters are handled separately
u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
metadata.Location = source_metadatapb.PostmanLocation_REQUEST_URL
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, u)), metadata)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

if len(r.URL.Query) > 0 {
vars := VariableData{
KeyValues: r.URL.Query,
}
metadata.Type = originalType + " > GET parameters (query)"
metadata.Location = source_metadatapb.PostmanLocation_REQUEST_QUERY_PARAMETER
s.scanVariableData(ctx, chunksChan, metadata, vars)
metadata.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

if r.Auth.Type != "" {
metadata.Type = originalType + " > request auth"
s.scanAuth(ctx, chunksChan, metadata, r.Auth, r.URL)
}

if r.Body.Mode != "" {
/*if r.Body.Mode != "" {
metadata.Type = originalType + " > body"
s.scanBody(ctx, chunksChan, metadata, r.Body)
}
}*/
}

func (s *Source) scanBody(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, b Body) {

Check failure on line 558 in pkg/sources/postman/postman.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

func `(*Source).scanBody` is unused (unused)
if !m.fromLocal {
m.Link = m.Link + "?tab=body"
}
Expand All @@ -524,23 +566,30 @@
vars := VariableData{
KeyValues: b.FormData,
}
m.Location = source_metadatapb.PostmanLocation_REQUEST_BODY_FORM_DATA
s.scanVariableData(ctx, chunksChan, m, vars)
m.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
case "urlencoded":
m.Type = originalType + " > url encoded"
vars := VariableData{
KeyValues: b.URLEncoded,
}
m.Location = source_metadatapb.PostmanLocation_REQUEST_BODY_URL_ENCODED
s.scanVariableData(ctx, chunksChan, m, vars)
m.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
case "raw", "graphql":
data := b.Raw
if b.Mode == "graphql" {
m.Type = originalType + " > graphql"
data = b.GraphQL.Query + " " + b.GraphQL.Variables
m.Location = source_metadatapb.PostmanLocation_REQUEST_BODY_GRAPHQL
}
if b.Mode == "raw" {
m.Type = originalType + " > raw"
m.Location = source_metadatapb.PostmanLocation_REQUEST_BODY_RAW
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
m.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
default:
break
}
Expand All @@ -558,13 +607,17 @@
KeyValues: response.Header,
}
m.Type = originalType + " > response header"
m.Location = source_metadatapb.PostmanLocation_RESPONSE_HEADER
s.scanVariableData(ctx, chunksChan, m, vars)
m.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

// Body in a response is just a string
if response.Body != "" {
m.Type = originalType + " > response body"
m.Location = source_metadatapb.PostmanLocation_RESPONSE_BODY
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, response.Body)), m)
m.Location = source_metadatapb.PostmanLocation_UNKNOWN_POSTMAN
}

if response.OriginalRequest.Method != "" {
Expand Down Expand Up @@ -600,14 +653,22 @@
}

m.FieldType = m.Type + " variables"
switch m.FieldType {
case "request > GET parameters (query) variables":
m.Link = m.Link + "?tab=params"
case "request > header variables":
m.Link = m.Link + "?tab=headers"
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(values), m)
}

func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) {
if data == "" {
return
}
metadata.FieldType = metadata.Type
if metadata.FieldType == "" {
metadata.FieldType = metadata.Type
}

chunksChan <- &sources.Chunk{
SourceType: s.Type(),
Expand All @@ -630,8 +691,7 @@
FolderId: metadata.FolderID,
FolderName: metadata.FolderName,
FieldType: metadata.FieldType,
FieldName: metadata.FieldName,
VariableType: metadata.VarType,
Location: metadata.Location,
},
},
},
Expand Down
5 changes: 3 additions & 2 deletions pkg/sources/postman/postman_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"io"
"net/http"
"time"

"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
)

const (
Expand Down Expand Up @@ -75,10 +77,9 @@ type Metadata struct {
Link string //direct link to the folder (could be .json file path)
Type string //folder, request, etc.
EnvironmentName string
VarType string
FieldName string
FieldType string
fromLocal bool
Location source_metadatapb.PostmanLocation
}

type Collection struct {
Expand Down
12 changes: 6 additions & 6 deletions pkg/sources/postman/substitution_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,14 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
s := &Source{
sub: NewSubstitution(),
}
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var1", "value1")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var2", "value2")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "", "value2")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token", "'{{continuation_token}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var1", "value1")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var2", "value2")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "", "value2")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token", "'{{continuation_token}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation

metadata := Metadata{
Type: GLOBAL_TYPE,
Type: ENVIRONMENT_TYPE,
}

testCases := []struct {
Expand Down
43 changes: 31 additions & 12 deletions proto/source_metadata.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,18 +299,37 @@ message Postman {
string link = 1;
string workspace_uuid = 2;
string workspace_name = 3;
string globals_id = 4;
string collection_id = 5;
string collection_name = 6;
string environment_id = 7;
string environment_name = 8;
string request_id = 9;
string request_name = 10;
string folder_id = 11;
string folder_name = 12;
string field_type = 13;
string field_name = 14;
string variable_type = 15;
Comment on lines -312 to -313
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reserve these numbers (https://protobuf.dev/best-practices/dos-donts/#reserve-tag-numbers)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done and for globals_id as well. I also reserved the names too.

string collection_id = 4;
string collection_name = 5;
string environment_id = 6;
string environment_name = 7;
string request_id = 8;
string request_name = 9;
string folder_id = 10;
string folder_name = 11;
string field_type = 12;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

leave a comment saying that this field is used for local output but not in transport

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

PostmanLocation location = 13;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rename to location_type

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

}

enum PostmanLocation {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rename to PostmanLocationType

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

UNKNOWN_POSTMAN = 0;
REQUEST_QUERY_PARAMETER = 1;
REQUEST_AUTHORIZATION = 2;
REQUEST_HEADER = 3;
REQUEST_BODY_FORM_DATA = 4;
REQUEST_BODY_RAW = 5;
REQUEST_BODY_URL_ENCODED = 6;
REQUEST_BODY_GRAPHQL = 7;
REQUEST_SCRIPT = 8;
REQUEST_URL = 9;
ENVIRONMENT_VARIABLE = 10;
FOLDER_AUTHORIZATION = 11;
FOLDER_SCRIPT = 12;
COLLECTION_SCRIPT = 13;
COLLECTION_VARIABLE = 14;
COLLECTION_AUTHORIZATION = 15;
RESPONSE_BODY = 16;
RESPONSE_HEADER = 17;
}

message Vector {
Expand Down
Loading