Merge pull request #294 from slashdevops/feat-performance-improve

Feat performance improve

go.mod

@@ -17,13 +17,13 @@ require (
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/oauth2 v0.23.0
-	google.golang.org/api v0.203.0
+	google.golang.org/api v0.204.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	cloud.google.com/go/auth v0.9.9 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
+	cloud.google.com/go/auth v0.10.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.18 // indirect
@@ -75,7 +75,7 @@ require (
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
 	google.golang.org/grpc v1.67.1 // indirect
 	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

go.sum

@@ -1,8 +1,8 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go/auth v0.9.9 h1:BmtbpNQozo8ZwW2t7QJjnrQtdganSdmqeIBxHxNkEZQ=
-cloud.google.com/go/auth v0.9.9/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
-cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
-cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
+cloud.google.com/go/auth v0.10.0 h1:tWlkvFAh+wwTOzXIjrwM64karR1iTBZ/GRr0S/DULYo=
+cloud.google.com/go/auth v0.10.0/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
+cloud.google.com/go/auth/oauth2adapt v0.2.5 h1:2p29+dePqsCHPP1bqDJcKj4qxRyYCcbzKpFyKGt3MTk=
+cloud.google.com/go/auth/oauth2adapt v0.2.5/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
 cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
 cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -235,17 +235,17 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.203.0 h1:SrEeuwU3S11Wlscsn+LA1kb/Y5xT8uggJSkIhD08NAU=
-google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI=
+google.golang.org/api v0.204.0 h1:3PjmQQEDkR/ENVZZwIYB4W/KzYtN8OrqnNcHWpeR8E4=
+google.golang.org/api v0.204.0/go.mod h1:69y8QSoKIbL9F94bWgWAq6wGqGwyjBgi2y8rAK8zLag=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53 h1:fVoAXEKA4+yufmbdVYv+SE73+cPZbbbe8paLsHfkK+U=
 google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.mod h1:riSXTwQ4+nqmPGtobMFyW5FqVAmIs0St6VPp4Ug7CE4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1:zciRKQ4kBpFgpfC5QQCVtnnNAcLIqweL7plyZRQHVpI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=

internal/core/actions.go

@@ -37,7 +37,7 @@ func scimSync(
 
 	groupsCreate, groupsUpdate, groupsEqual, groupsDelete, err := model.GroupsOperations(idpGroupsResult, scimGroupsResult)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("error reconciling groups: %w", err)
+		return nil, nil, nil, fmt.Errorf("error operating with groups: %w", err)
 	}
 
 	groupsCreated, groupsUpdated, err := reconcilingGroups(ctx, scim, groupsCreate, groupsUpdate, groupsDelete)

internal/core/sync.go

@@ -81,7 +81,7 @@ func (ss *SyncService) SyncGroupsAndTheirMembers(ctx context.Context) error {
 
 	slog.Info("groups members retrieved from the identity provider for syncing that match the filter",
 		"group_filter", ss.provGroupsFilter,
-		"groups_members", idpGroupsMembersResult.Items,
+		"groups", idpGroupsResult.Items,
 	)
 
 	slog.Info("getting users (using groups members) from the identity provider",

internal/scim/operations.go

@@ -8,7 +8,7 @@ import (
 // patchGroupOperations assembles the operations for patch groups
 // bases in the limits of operations we can execute in a single request.
 func patchGroupOperations(op, path string, pvs []patchValue, gms *model.GroupMembers) []*aws.PatchGroupRequest {
-	patchOperations := []*aws.PatchGroupRequest{}
+	patchOperations := make([]*aws.PatchGroupRequest, 0)
 
 	if len(pvs) > MaxPatchGroupMembersPerRequest {
 		for i := 0; i < len(pvs); i += MaxPatchGroupMembersPerRequest {
@@ -33,9 +33,11 @@ func patchGroupOperations(op, path string, pvs []patchValue, gms *model.GroupMem
 					},
 				},
 			}
+
 			patchOperations = append(patchOperations, patchGroupRequest)
 		}
 	} else {
+
 		patchGroupRequest := &aws.PatchGroupRequest{
 			Group: aws.Group{
 				ID:          gms.Group.SCIMID,
@@ -52,6 +54,7 @@ func patchGroupOperations(op, path string, pvs []patchValue, gms *model.GroupMem
 				},
 			},
 		}
+
 		patchOperations = append(patchOperations, patchGroupRequest)
 	}
 

internal/scim/operations_test.go

@@ -134,3 +134,14 @@ func Test_patchGroupOperations(t *testing.T) {
 		})
 	}
 }
+
+func Benchmark_patchGroupOperations(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		patchGroupOperations("add", "members", patchValueGenerator(1, 350), &model.GroupMembers{
+			Group: &model.Group{
+				SCIMID: "016722b2be-ee23ed58-6e4e-4b2f-a94a-3ace8456a36e",
+				Name:   "group 1",
+			},
+		})
+	}
+}

internal/scim/scim.go

@@ -18,9 +18,6 @@ type AWSSCIMProvider interface {
 	// ListUsers lists users in SCIM Provider
 	ListUsers(ctx context.Context, filter string) (*aws.ListUsersResponse, error)
 
-	// CreateUser creates a user in SCIM Provider
-	CreateUser(ctx context.Context, u *aws.CreateUserRequest) (*aws.CreateUserResponse, error)
-
 	// CreateOrGetUser creates a user in SCIM Provider
 	CreateOrGetUser(ctx context.Context, u *aws.CreateUserRequest) (*aws.CreateUserResponse, error)
 
@@ -39,9 +36,6 @@ type AWSSCIMProvider interface {
 	// ListGroups lists groups in SCIM Provider
 	ListGroups(ctx context.Context, filter string) (*aws.ListGroupsResponse, error)
 
-	// CreateGroup creates a group in SCIM Provider
-	CreateGroup(ctx context.Context, g *aws.CreateGroupRequest) (*aws.CreateGroupResponse, error)
-
 	// CreateOrGetGroup creates a group in SCIM Provider
 	CreateOrGetGroup(ctx context.Context, g *aws.CreateGroupRequest) (*aws.CreateGroupResponse, error)
 
@@ -81,24 +75,28 @@ func (s *Provider) GetGroups(ctx context.Context) (*model.GroupsResult, error) {
 
 	groups := make([]*model.Group, len(groupsResponse.Resources))
 	for i, group := range groupsResponse.Resources {
-		e := model.GroupBuilder().
+		g := model.GroupBuilder().
 			WithSCIMID(group.ID).
 			WithName(group.DisplayName).
 			WithIPID(group.ExternalID).
 			Build()
 
-		groups[i] = e
+		groups[i] = g
+
 	}
 
 	groupsResult := model.GroupsResultBuilder().WithResources(groups).Build()
-
 	slog.Debug("scim: GetGroups()", "groups", len(groups))
 
 	return groupsResult, nil
 }
 
 // CreateGroups creates groups in SCIM Provider
 func (s *Provider) CreateGroups(ctx context.Context, gr *model.GroupsResult) (*model.GroupsResult, error) {
+	if gr == nil {
+		return nil, fmt.Errorf("scim: error creating groups, groups result is nil")
+	}
+
 	groups := make([]*model.Group, len(gr.Resources))
 
 	for i, group := range gr.Resources {
@@ -114,18 +112,17 @@ func (s *Provider) CreateGroups(ctx context.Context, gr *model.GroupsResult) (*m
 			return nil, fmt.Errorf("scim: error creating group: %w", err)
 		}
 
-		e := model.GroupBuilder().
+		g := model.GroupBuilder().
 			WithSCIMID(r.ID).
 			WithName(group.Name).
 			WithIPID(group.IPID).
 			WithEmail(group.Email).
 			Build()
 
-		groups[i] = e
+		groups[i] = g
 	}
 
 	groupsResult := model.GroupsResultBuilder().WithResources(groups).Build()
-
 	slog.Debug("scim: CreateGroups()", "groups", len(groups))
 
 	return groupsResult, nil
@@ -162,14 +159,14 @@ func (s *Provider) UpdateGroups(ctx context.Context, gr *model.GroupsResult) (*m
 		}
 
 		// return the same group
-		e := model.GroupBuilder().
+		g := model.GroupBuilder().
 			WithSCIMID(group.SCIMID).
 			WithName(group.Name).
 			WithIPID(group.IPID).
 			WithEmail(group.Email).
 			Build()
 
-		groups[i] = e
+		groups[i] = g
 	}
 
 	groupsResult := model.GroupsResultBuilder().WithResources(groups).Build()
@@ -200,8 +197,8 @@ func (s *Provider) GetUsers(ctx context.Context) (*model.UsersResult, error) {
 
 	users := make([]*model.User, len(usersResponse.Resources))
 	for i, user := range usersResponse.Resources {
-		e := buildUser(user)
-		users[i] = e
+		u := buildUser(user)
+		users[i] = u
 	}
 
 	usersResult := model.UsersResultBuilder().WithResources(users).Build()
@@ -286,13 +283,13 @@ type patchValue struct {
 
 // CreateGroupsMembers creates groups members in SCIM Provider given a list of groups members
 func (s *Provider) CreateGroupsMembers(ctx context.Context, gmr *model.GroupsMembersResult) (*model.GroupsMembersResult, error) {
-	groupsMembers := make([]*model.GroupMembers, 0)
+	groupsMembers := make([]*model.GroupMembers, len(gmr.Resources))
 
-	for _, groupMembers := range gmr.Resources {
-		members := make([]*model.Member, 0)
-		membersIDValue := []patchValue{}
+	for i, groupMembers := range gmr.Resources {
+		members := make([]*model.Member, len(groupMembers.Resources))
+		membersIDValue := make([]patchValue, len(groupMembers.Resources))
 
-		for _, member := range groupMembers.Resources {
+		for j, member := range groupMembers.Resources {
 			if member.SCIMID == "" {
 				u, err := s.scim.GetUserByUserName(ctx, member.Email)
 				if err != nil {
@@ -301,28 +298,27 @@ func (s *Provider) CreateGroupsMembers(ctx context.Context, gmr *model.GroupsMem
 				member.SCIMID = u.ID
 			}
 
-			membersIDValue = append(membersIDValue, patchValue{
+			membersIDValue[j] = patchValue{
 				Value: member.SCIMID,
-			})
+			}
 
-			e := model.MemberBuilder().
+			m := model.MemberBuilder().
 				WithIPID(member.IPID).
 				WithSCIMID(member.SCIMID).
 				WithEmail(member.Email).
 				WithStatus(member.Status).
 				Build()
 
 			slog.Warn("adding member to group", "group", groupMembers.Group.Name, "email", member.Email)
-			members = append(members, e)
-
+			members[j] = m
 		}
 
-		e := model.GroupMembersBuilder().
+		gm := model.GroupMembersBuilder().
 			WithGroup(groupMembers.Group).
 			WithResources(members).
 			Build()
 
-		groupsMembers = append(groupsMembers, e)
+		groupsMembers[i] = gm
 
 		patchOperations := patchGroupOperations("add", "members", membersIDValue, groupMembers)
 
@@ -397,9 +393,9 @@ func (s *Provider) GetGroupsMembers(ctx context.Context, gr *model.GroupsResult)
 		}
 
 		for _, gr := range lgr.Resources {
-			members := make([]*model.Member, 0)
+			members := make([]*model.Member, len(gr.Members))
 
-			for _, member := range gr.Members {
+			for j, member := range gr.Members {
 				u, err := s.scim.GetUser(ctx, member.Value)
 				if err != nil {
 					return nil, fmt.Errorf("scim: error getting user: %s, error %w", member.Value, err)
@@ -410,15 +406,15 @@ func (s *Provider) GetGroupsMembers(ctx context.Context, gr *model.GroupsResult)
 					WithEmail(u.Emails[0].Value).
 					Build()
 
-				members = append(members, m)
+				members[j] = m
 			}
 
-			e := model.GroupMembersBuilder().
+			gms := model.GroupMembersBuilder().
 				WithGroup(group).
 				WithResources(members).
 				Build()
 
-			groupMembers = append(groupMembers, e)
+			groupMembers = append(groupMembers, gms)
 		}
 	}
 
@@ -431,22 +427,23 @@ func (s *Provider) GetGroupsMembers(ctx context.Context, gr *model.GroupsResult)
 // GetGroupsMembersBruteForce returns a list of groups and their members from the SCIM Provider
 // NOTE: this is an bad alternative to the method GetGroupsMembers,  because read the note in the method.
 func (s *Provider) GetGroupsMembersBruteForce(ctx context.Context, gr *model.GroupsResult, ur *model.UsersResult) (*model.GroupsMembersResult, error) {
-	groupMembers := make([]*model.GroupMembers, 0)
+	groupMembers := make([]*model.GroupMembers, len(gr.Resources))
 
 	// brute force implemented here thanks to the fxxckin' aws sso scim api
-	for _, group := range gr.Resources {
+	for i, group := range gr.Resources {
 		members := make([]*model.Member, 0)
 
 		for _, user := range ur.Resources {
 
 			// https://docs.aws.amazon.com/singlesignon/latest/developerguide/listgroups.html
-			f := fmt.Sprintf("id eq %q and members eq %q", group.SCIMID, user.SCIMID)
-			lgr, err := s.scim.ListGroups(ctx, f)
+			filter := fmt.Sprintf("id eq %q and members eq %q", group.SCIMID, user.SCIMID)
+			lgr, err := s.scim.ListGroups(ctx, filter)
 			if err != nil {
 				return nil, fmt.Errorf("scim: error listing groups: %w", err)
 			}
 
-			if lgr.TotalResults > 0 { // crazy thing of the AWS SSO SCIM API, it doesn't return the member into the Resources array
+			// AWS SSO SCIM API, it doesn't return the member into the Resources array
+			if lgr.TotalResults > 0 {
 				m := model.MemberBuilder().
 					WithIPID(user.IPID).
 					WithSCIMID(user.SCIMID).
@@ -460,12 +457,13 @@ func (s *Provider) GetGroupsMembersBruteForce(ctx context.Context, gr *model.Gro
 				members = append(members, m)
 			}
 		}
-		e := model.GroupMembersBuilder().
+
+		gms := model.GroupMembersBuilder().
 			WithGroup(group).
 			WithResources(members).
 			Build()
 
-		groupMembers = append(groupMembers, e)
+		groupMembers[i] = gms
 	}
 
 	slog.Debug("scim: GetGroupsMembersBruteForce()", "groups_members", len(groupMembers))