Display IDs (#191)

* Add DisplayID concept * Add display names for iam.Policy and sns.SNS * Add additional test case for missing display ID * Update static assets * Route53 zones have a Name field, not sns topics Co-authored-by: Cloudgrep <[email protected]>
Miracle-Engineering · Jul 8, 2022 · dec0ccb · dec0ccb
1 parent 07372de
commit dec0ccb
Show file tree

Hide file tree

Showing 24 changed files with 179 additions and 90 deletions.
diff --git a/fe/src/models/Resource.ts b/fe/src/models/Resource.ts
@@ -9,6 +9,7 @@ export interface Property {
 export interface Resource {
 	type: string;
 	id: string;
+	displayId: string;
 	region: string;
 	rawData: Object;
 	tags?: Tag[];

diff --git a/fe/src/pages/Insights/InsightTable.tsx b/fe/src/pages/Insights/InsightTable.tsx
@@ -195,7 +195,7 @@ const InsightTable: FC = () => {
 									{row.type}
 								</TableCell>
 								<TableCell sx={tableStyles.bodyRow} align="left">
-									{row.id}
+									{row.displayId || row.id}
 								</TableCell>
 								<TableCell sx={tableStyles.regionRow} align="left">
 									{row.region}

diff --git a/hack/awsgen/config/types.go b/hack/awsgen/config/types.go
@@ -70,6 +70,10 @@ type ListAPI struct {
 	// IDField points to the field within each resource struct that stores the ID of that resource
 	IDField Field `yaml:"id"`
 
+	// DisplayIDField points to the field within each resource struct that stores the display ID of that resource.
+	// The display ID should be an easily readable identifier, and is intended to be consumed by humans.
+	DisplayIDField Field `yaml:"displayId"`
+
 	// Tags configures where to look for tags on this resource.
 	// Required if the getTagsApi is not configured.
 	Tags *TagField `yaml:"tags"`

diff --git a/hack/awsgen/generator/service.go b/hack/awsgen/generator/service.go
@@ -54,11 +54,12 @@ func (g Generator) generateServiceRegister(service config.Service) (string, util
 		}
 
 		data.Types = append(data.Types, typeRegisterInfo{
-			ResourceName:  resourceName(service, typ),
-			FetchFuncName: fetchFuncName(service, typ),
-			IDField:       typ.ListAPI.IDField,
-			Global:        global,
-			Tags:          typ.ListAPI.Tags,
+			ResourceName:   resourceName(service, typ),
+			FetchFuncName:  fetchFuncName(service, typ),
+			IDField:        typ.ListAPI.IDField,
+			DisplayIDField: typ.ListAPI.DisplayIDField,
+			Global:         global,
+			Tags:           typ.ListAPI.Tags,
 		})
 
 		if !typ.GetTagsAPI.Tags.Zero() {
@@ -73,7 +74,8 @@ type typeRegisterInfo struct {
 	ResourceName  string
 	FetchFuncName string
 
-	IDField config.Field
-	Global  bool
-	Tags    *config.TagField
+	IDField        config.Field
+	DisplayIDField config.Field
+	Global         bool
+	Tags           *config.TagField
 }
diff --git a/hack/awsgen/template/templates/service-register.go.tmpl b/hack/awsgen/template/templates/service-register.go.tmpl
@@ -4,6 +4,9 @@ func (p *{{ .ProviderName }}) {{ .FuncName }}(mapping map[string]mapper) {
 		ServiceEndpointID: {{ $.EndpointID | quote }},
 		FetchFunc: p.{{ .FetchFuncName }},
 		IdField: {{ .IDField.Name | quote }},
+		{{- if (not .DisplayIDField.Zero) }}
+		DisplayIDField: {{ .DisplayIDField.Name | quote }},
+		{{- end }}
 		IsGlobal: {{ .Global }},
 		{{- if (not .Tags.Zero) }}
 		TagField: resourceconverter.TagField{

diff --git a/pkg/model/resource.go b/pkg/model/resource.go
@@ -10,6 +10,7 @@ import (
 //TODO store provider info in resource (needed when we can have more than one provider)
 type Resource struct {
 	Id        string         `json:"id" gorm:"primaryKey"`
+	DisplayId string         `json:"displayId"`
 	Region    string         `json:"region"`
 	Type      string         `json:"type"`
 	Tags      Tags           `json:"tags"`

diff --git a/pkg/provider/aws/config/iam.yaml b/pkg/provider/aws/config/iam.yaml
@@ -25,6 +25,7 @@ types:
           Scope: listPoliciesScope
       outputKey: Policies
       id: Arn
+      displayId: PolicyName
     getTagsApi:
       type: Policy
       call: ListPolicyTags

diff --git a/pkg/provider/aws/config/route53.yaml b/pkg/provider/aws/config/route53.yaml
@@ -35,6 +35,7 @@ types:
       id:
         name: Id
         pointer: true
+      displayId: Name
     getTagsApi:
       call: ListTagsForResources
       type: HostedZone

diff --git a/pkg/provider/aws/config/sns.yaml b/pkg/provider/aws/config/sns.yaml
@@ -14,4 +14,4 @@ types:
         field: "Tags"
         pointer: true
         key: Key
-        value: Value
+        value: Value
diff --git a/pkg/provider/aws/provider.go b/pkg/provider/aws/provider.go
@@ -54,17 +54,19 @@ func (p *Provider) converterFor(resourceType string) resourceconverter.ResourceC
 	region := p.region.ID()
 	if mapping.UseMapConverter {
 		return &resourceconverter.MapConverter{
-			Region:       region,
-			ResourceType: resourceType,
-			TagField:     mapping.TagField,
-			IdField:      mapping.IdField,
+			Region:         region,
+			ResourceType:   resourceType,
+			TagField:       mapping.TagField,
+			IdField:        mapping.IdField,
+			DisplayIdField: mapping.DisplayIDField,
 		}
 	}
 	return &resourceconverter.ReflectionConverter{
-		Region:       region,
-		ResourceType: resourceType,
-		TagField:     mapping.TagField,
-		IdField:      mapping.IdField,
+		Region:         region,
+		ResourceType:   resourceType,
+		TagField:       mapping.TagField,
+		IdField:        mapping.IdField,
+		DisplayIdField: mapping.DisplayIDField,
 	}
 }
 

diff --git a/pkg/provider/aws/sqs.go b/pkg/provider/aws/sqs.go
@@ -16,6 +16,7 @@ func (p *Provider) register_sqs(mapping map[string]mapper) {
 	mapping["sqs.Queue"] = mapper{
 		FetchFunc:       p.fetch_sqs_Queue,
 		IdField:         "QueueUrl",
+		DisplayIDField:  "QueueArn",
 		IsGlobal:        false,
 		UseMapConverter: true,
 	}

diff --git a/pkg/provider/aws/types.go b/pkg/provider/aws/types.go
@@ -8,6 +8,7 @@ import (
 type mapper struct {
 	IdField         string
 	TagField        resourceconverter.TagField
+	DisplayIDField  string
 	FetchFunc       types.FetchFunc
 	IsGlobal        bool
 	UseMapConverter bool

diff --git a/pkg/provider/aws/zz_iam.go b/pkg/provider/aws/zz_iam.go
diff --git a/pkg/provider/aws/zz_route53.go b/pkg/provider/aws/zz_route53.go
diff --git a/pkg/resourceconverter/map_converter.go b/pkg/resourceconverter/map_converter.go
@@ -4,15 +4,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/run-x/cloudgrep/pkg/model"
 	"reflect"
+
+	"github.com/run-x/cloudgrep/pkg/model"
 )
 
 type MapConverter struct {
-	ResourceType string
-	TagField     TagField
-	IdField      string
-	Region       string
+	ResourceType   string
+	TagField       TagField
+	IdField        string
+	DisplayIdField string
+	Region         string
 }
 
 func (mc *MapConverter) ToResource(ctx context.Context, x any, tags model.Tags) (model.Resource, error) {
@@ -37,12 +39,21 @@ func (mc *MapConverter) ToResource(ctx context.Context, x any, tags model.Tags)
 	}
 	idString := fmt.Sprintf("%v", id)
 
+	var displayIdString string
+	if mc.DisplayIdField != "" {
+		displayId, ok := xConverted[mc.DisplayIdField]
+		if !ok {
+			return model.Resource{}, fmt.Errorf("could not find display id field %v in map %v", mc.DisplayIdField, xConverted)
+		}
+		displayIdString = fmt.Sprintf("%v", displayId)
+	}
+
 	// generate tags field
-	if tags == nil {
+	if !mc.TagField.IsZero() {
 		//use field
 		tagsValue, ok := xConverted[mc.TagField.Name]
 		if !ok {
-			return model.Resource{}, fmt.Errorf("Could not find tag field '%v' for type '%v", mc.TagField.Name, mc.ResourceType)
+			return model.Resource{}, fmt.Errorf("could not find tag field '%v' for type '%v", mc.TagField.Name, mc.ResourceType)
 		}
 
 		tags = getTags(reflect.ValueOf(tagsValue), mc.TagField)
@@ -52,10 +63,11 @@ func (mc *MapConverter) ToResource(ctx context.Context, x any, tags model.Tags)
 		return model.Resource{}, err
 	}
 	return model.Resource{
-		Id:      idString,
-		Region:  mc.Region,
-		Type:    mc.ResourceType,
-		RawData: marshaledMap,
-		Tags:    tags,
+		Id:        idString,
+		DisplayId: displayIdString,
+		Region:    mc.Region,
+		Type:      mc.ResourceType,
+		RawData:   marshaledMap,
+		Tags:      tags,
 	}, nil
 }
diff --git a/pkg/resourceconverter/map_converter_test.go b/pkg/resourceconverter/map_converter_test.go
@@ -2,11 +2,12 @@ package resourceconverter
 
 import (
 	"context"
+	"testing"
+
 	"github.com/run-x/cloudgrep/pkg/model"
 	"github.com/run-x/cloudgrep/pkg/testingutil"
 	"github.com/stretchr/testify/require"
 	"gorm.io/datatypes"
-	"testing"
 )
 
 func TestMapConverter(t *testing.T) {
@@ -20,18 +21,20 @@ func TestMapConverter(t *testing.T) {
 			"WeirdTags": []WeirdTags{{WeirdKey: "key1", WeirdValue: "val1"}, {WeirdKey: "key2", WeirdValue: "val2"}},
 		}
 		rC := &MapConverter{
-			IdField:      "ID",
-			ResourceType: "DummyResource",
-			Region:       "dummyRegion",
+			IdField:        "ID",
+			DisplayIdField: "Attr2",
+			ResourceType:   "DummyResource",
+			Region:         "dummyRegion",
 		}
 		resource, err := rC.ToResource(ctx, entry, model.Tags{{Key: "key1", Value: "val3"}, {Key: "key2", Value: "val4"}})
 		require.NoError(t, err)
 		expectedResource := model.Resource{
-			Region:  "dummyRegion",
-			Id:      "id1",
-			Type:    "DummyResource",
-			Tags:    model.Tags{{Key: "key1", Value: "val3"}, {Key: "key2", Value: "val4"}},
-			RawData: datatypes.JSON([]byte(`{"ID":"id1","Attr1":1,"Attr2":"hi","Attr3":{"a":"b","c":2},"WeirdTags":[{"WeirdKey":"key1","WeirdValue":"val1"},{"WeirdKey":"key2","WeirdValue":"val2"}]}`)),
+			Region:    "dummyRegion",
+			Id:        "id1",
+			DisplayId: "hi",
+			Type:      "DummyResource",
+			Tags:      model.Tags{{Key: "key1", Value: "val3"}, {Key: "key2", Value: "val4"}},
+			RawData:   datatypes.JSON([]byte(`{"ID":"id1","Attr1":1,"Attr2":"hi","Attr3":{"a":"b","c":2},"WeirdTags":[{"WeirdKey":"key1","WeirdValue":"val1"},{"WeirdKey":"key2","WeirdValue":"val2"}]}`)),
 		}
 		testingutil.AssertEqualsResource(t, expectedResource, resource)
 	})

diff --git a/pkg/resourceconverter/resource_converter.go b/pkg/resourceconverter/resource_converter.go
@@ -4,8 +4,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/run-x/cloudgrep/pkg/model"
 	"reflect"
+
+	"github.com/run-x/cloudgrep/pkg/model"
 )
 
 type ResourceConverter interface {
@@ -22,53 +23,87 @@ type TagField struct {
 	Value string `yaml:"value"`
 }
 
+func (f TagField) IsZero() bool {
+	return f.Name == ""
+}
+
 type ReflectionConverter struct {
-	ResourceType string
-	TagField     TagField
-	IdField      string
-	Region       string
+	ResourceType   string
+	TagField       TagField
+	IdField        string
+	DisplayIdField string
+	Region         string
 }
 
 func (rc *ReflectionConverter) ToResource(ctx context.Context, x any, tags model.Tags) (model.Resource, error) {
-	t := reflect.TypeOf(x)
-
-	// get the id field
-	var id string
-	for i := 0; i < t.NumField(); i++ {
-		field := t.Field(i)
-		name := field.Name
-		if name == rc.IdField {
-			fieldPtrRef := reflect.ValueOf(x).FieldByName(name)
-			fieldRef := reflect.Indirect(fieldPtrRef)
-			if !fieldRef.IsZero() {
-				id = fmt.Sprintf("%v", fieldRef.Interface())
-			}
-			break
-		}
+	resource := model.Resource{
+		Region: rc.Region,
+		Type:   rc.ResourceType,
 	}
 
-	if id == "" {
-		return model.Resource{}, fmt.Errorf("could not find id field '%v' for type '%v", rc.IdField, rc.ResourceType)
+	resource.Id = rc.findId(rc.IdField, x)
+	if resource.Id == "" {
+		return model.Resource{}, fmt.Errorf("could not find id field '%v' for type '%v'", rc.IdField, rc.ResourceType)
+	}
+
+	if err := rc.loadDisplayId(x, &resource); err != nil {
+		return model.Resource{}, err
 	}
 
 	// generate tags field
-	if tags == nil {
+	if !rc.TagField.IsZero() {
 		//use field
 		tagsValue := reflect.ValueOf(x).FieldByName(rc.TagField.Name)
 		if !tagsValue.IsValid() {
-			return model.Resource{}, fmt.Errorf("Could not find tag field '%v' for type '%v", rc.TagField.Name, rc.ResourceType)
+			return model.Resource{}, fmt.Errorf("could not find tag field '%v' for type '%v'", rc.TagField.Name, rc.ResourceType)
 		}
-		tags = getTags(tagsValue, rc.TagField)
+		resource.Tags = getTags(tagsValue, rc.TagField)
 	}
+
+	resource.Tags = append(resource.Tags, tags...)
+
 	marshaledStruct, err := json.Marshal(x)
 	if err != nil {
 		return model.Resource{}, err
 	}
-	return model.Resource{
-		Id:      id,
-		Region:  rc.Region,
-		Type:    rc.ResourceType,
-		RawData: marshaledStruct,
-		Tags:    tags,
-	}, nil
+
+	resource.RawData = marshaledStruct
+
+	return resource, nil
+}
+
+func (rc *ReflectionConverter) loadDisplayId(x any, resource *model.Resource) error {
+	if rc.DisplayIdField == "" {
+		return nil
+	}
+
+	id := rc.findId(rc.DisplayIdField, x)
+
+	if id == "" {
+		return fmt.Errorf("could not find display id field '%v' for type '%v'", rc.DisplayIdField, rc.ResourceType)
+	}
+
+	resource.DisplayId = id
+
+	return nil
+}
+
+func (rc *ReflectionConverter) findId(fieldName string, x any) string {
+	v := reflect.ValueOf(x)
+	if v.IsZero() {
+		return ""
+	}
+
+	fieldPtrRef := v.FieldByName(fieldName)
+	if !fieldPtrRef.IsValid() || fieldPtrRef.IsZero() {
+		// Field not on x
+		return ""
+	}
+
+	fieldRef := reflect.Indirect(fieldPtrRef)
+	if !fieldRef.IsValid() || fieldRef.IsZero() {
+		return ""
+	}
+
+	return fmt.Sprint(fieldRef.Interface())
 }