Merge pull request #5 from aidenkeating/ec2-subnet-groups

complete refactor to resource manager terminology

cmd/cli/cleanup.go

@@ -102,7 +102,11 @@ func buildAWSClientFromTypes(awsSession *session.Session, types []string, logger
 		case "s3":
 			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultS3Engine(awsSession, logger))
 		case "elasticache:replicationgroup":
-			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultElastiCacheEngine(awsSession, logger))
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultElasticacheManager(awsSession, logger))
+		case "elasticache:snapshot":
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultElasticacheSnapshotManager(awsSession, logger))
+		case "ec2:subnet":
+			client.ResourceManagers = append(client.ResourceManagers, awsclusterservice.NewDefaultSubnetManager(awsSession, logger))
 		default:
 			logger.Debugf("could not find resource manager for specified type %s", t)
 		}

cmd/cli/root.go

@@ -20,7 +20,7 @@ var logger = logrus.WithField("service", "cluster_service")
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "cli",
+	Use:   "cluster-service",
 	Short: "A brief description of your application",
 	Long: `A longer description that spans multiple lines and likely contains
 examples and usage of using your application. For example:

hack/aws-elasticache-create-snapshot.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+REGION="us-east-1"
+RG_ID=${1:-cluster-service-deleteme}
+
+echo "creating snapshot replication $RG_ID group in region $REGION"
+aws elasticache create-snapshot --region "$REGION" \
+  --cache-cluster-id "$RG_ID-001" --snapshot-name "$RG_ID"

pkg/aws/client.go

@@ -18,13 +18,14 @@ type Client struct {
 
 func NewDefaultClient(awsSession *session.Session, logger *logrus.Entry) *Client {
 	log := logger.WithField("cluster_service_provider", "aws")
-	rdsEngine := NewDefaultRDSInstanceManager(awsSession, logger)
+	rdsManager := NewDefaultRDSInstanceManager(awsSession, logger)
 	rdsSnapshotManager := NewDefaultRDSSnapshotManager(awsSession, logger)
-	s3Engine := NewDefaultS3Engine(awsSession, logger)
-	elasticacheEngine := NewDefaultElastiCacheEngine(awsSession, logger)
-	elasticacheSnapshotEngine := newDefaultElasticacheSnapshotEngine(awsSession, logger)
+	s3Manager := NewDefaultS3Engine(awsSession, logger)
+	elasticacheManager := NewDefaultElasticacheManager(awsSession, logger)
+	elasticacheSnapshotManager := NewDefaultElasticacheSnapshotManager(awsSession, logger)
+	subnetManager := NewDefaultSubnetManager(awsSession, logger)
 	return &Client{
-		ResourceManagers: []ClusterResourceManager{rdsEngine, elasticacheEngine, s3Engine, rdsSnapshotManager, elasticacheSnapshotEngine},
+		ResourceManagers: []ClusterResourceManager{rdsManager, elasticacheManager, s3Manager, rdsSnapshotManager, elasticacheSnapshotManager, subnetManager},
 		Logger:           log,
 	}
 }
@@ -35,7 +36,7 @@ func (c *Client) DeleteResourcesForCluster(clusterId string, tags map[string]str
 	logger.Debugf("deleting resources for cluster")
 	report := &clusterservice.Report{}
 	for _, engine := range c.ResourceManagers {
-		engineLogger := logger.WithField(loggingKeyEngine, engine.GetName())
+		engineLogger := logger.WithField(loggingKeyManager, engine.GetName())
 		engineLogger.Debugf("found Logger")
 		reportItems, err := engine.DeleteResourcesForCluster(clusterId, tags, dryRun)
 		if err != nil {

pkg/aws/manager_ec2_subnet.go

@@ -0,0 +1,98 @@
+package aws
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
+	"github.com/integr8ly/cluster-service/pkg/clusterservice"
+	"github.com/integr8ly/cluster-service/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	loggingKeySubnet = "subnet-id"
+
+	resourceTypeSubnet = "ec2:subnet"
+)
+
+var _ ClusterResourceManager = &SubnetManager{}
+
+type SubnetManager struct {
+	ec2Client     ec2Client
+	taggingClient taggingClient
+	logger        *logrus.Entry
+}
+
+func NewDefaultSubnetManager(session *session.Session, logger *logrus.Entry) *SubnetManager {
+	return &SubnetManager{
+		ec2Client:     ec2.New(session),
+		taggingClient: resourcegroupstaggingapi.New(session),
+		logger:        logger.WithField(loggingKeyManager, managerSubnet),
+	}
+}
+
+func (r *SubnetManager) GetName() string {
+	return "AWS EC2 Subnet Manager"
+}
+
+func (s *SubnetManager) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) ([]*clusterservice.ReportItem, error) {
+	s.logger.Debug("delete subnet resources for cluster")
+	resourceInput := &resourcegroupstaggingapi.GetResourcesInput{
+		ResourceTypeFilters: aws.StringSlice([]string{resourceTypeSubnet}),
+		TagFilters:          convertClusterTagsToAWSTagFilter(clusterId, tags),
+	}
+	resourceOutput, err := s.taggingClient.GetResources(resourceInput)
+	if err != nil {
+		return nil, errors.WrapLog(err, "failed to filter subnets", s.logger)
+	}
+	var subnetsToDelete []*basicResource
+	for _, resourceTagMapping := range resourceOutput.ResourceTagMappingList {
+		arn := aws.StringValue(resourceTagMapping.ResourceARN)
+		arnElements := strings.Split(arn, "/")
+		subnetId := arnElements[len(arnElements)-1]
+		if subnetId == "" {
+			return nil, errors.WrapLog(err, fmt.Sprintf("invalid subnet name from arn, %s", subnetId), s.logger)
+		}
+		subnetsToDelete = append(subnetsToDelete, &basicResource{
+			Name: subnetId,
+			ARN:  arn,
+		})
+	}
+	s.logger.Debugf("found list of %d subnets to delete", len(subnetsToDelete))
+	//delete resources
+	var reportItems []*clusterservice.ReportItem
+	for _, subnet := range subnetsToDelete {
+		subnetLogger := s.logger.WithField(loggingKeySubnet, subnet.Name)
+		reportItem := &clusterservice.ReportItem{
+			ID:           subnet.ARN,
+			Name:         subnet.Name,
+			Action:       clusterservice.ActionDelete,
+			ActionStatus: clusterservice.ActionStatusInProgress,
+		}
+		reportItems = append(reportItems, reportItem)
+		if dryRun {
+			subnetLogger.Debugf("dry run is enabled, skipping deletion")
+			reportItem.ActionStatus = clusterservice.ActionStatusDryRun
+			continue
+		}
+		subnetLogger.Debugf("performing subnet deletion")
+		deleteSubnetInput := &ec2.DeleteSubnetInput{
+			SubnetId: aws.String(subnet.Name),
+		}
+		if _, err := s.ec2Client.DeleteSubnet(deleteSubnetInput); err != nil {
+			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "DependencyViolation" {
+				subnetLogger.Debug("subnet has existing dependencies which have not been deleted, skipping")
+				reportItem.ActionStatus = clusterservice.ActionStatusSkipped
+				continue
+			}
+			return nil, errors.WrapLog(err, "failed to delete subnet", s.logger)
+		}
+	}
+	return reportItems, nil
+}

pkg/aws/elasticache.go → pkg/aws/manager_elasticache.go

@@ -14,43 +14,36 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var _ ClusterResourceManager = &ElasticacheEngine{}
+var _ ClusterResourceManager = &ElasticacheManager{}
 
-type ElasticacheEngine struct {
+type ElasticacheManager struct {
 	elasticacheClient elasticacheiface.ElastiCacheAPI
 	taggingClient     resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
 	logger            *logrus.Entry
 }
 
-func NewDefaultElastiCacheEngine(session *session.Session, logger *logrus.Entry) *ElasticacheEngine {
-	return &ElasticacheEngine{
+func NewDefaultElasticacheManager(session *session.Session, logger *logrus.Entry) *ElasticacheManager {
+	return &ElasticacheManager{
 		elasticacheClient: elasticache.New(session),
 		taggingClient:     resourcegroupstaggingapi.New(session),
-		logger:            logger.WithField("engine", "aws_elasticache"),
+		logger:            logger.WithField(loggingKeyManager, managerElasticache),
 	}
 }
 
-func (r *ElasticacheEngine) GetName() string {
-	return "AWS elasticache Engine"
+func (r *ElasticacheManager) GetName() string {
+	return "AWS ElastiCache Manager"
 }
 
 //Delete all elasticache resources for a specified cluster
-func (r *ElasticacheEngine) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) ([]*clusterservice.ReportItem, error) {
+func (r *ElasticacheManager) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) ([]*clusterservice.ReportItem, error) {
 	logger := r.logger.WithFields(logrus.Fields{"clusterId": clusterId, "dryRun": dryRun})
 	logger.Debug("deleting resources for cluster")
 
 	var reportItems []*clusterservice.ReportItem
 	var replicationGroupsToDelete []string
 	resourceInput := &resourcegroupstaggingapi.GetResourcesInput{
 		ResourceTypeFilters: aws.StringSlice([]string{"elasticache:cluster"}),
-		TagFilters: []*resourcegroupstaggingapi.TagFilter{
-			{
-				Key: aws.String(tagKeyClusterId),
-				Values: aws.StringSlice([]string{
-					clusterId,
-				}),
-			},
-		},
+		TagFilters:          convertClusterTagsToAWSTagFilter(clusterId, tags),
 	}
 	resourceOutput, err := r.taggingClient.GetResources(resourceInput)
 	if err != nil {

pkg/aws/elasticache_snapshot.go → pkg/aws/manager_elasticache_snapshot.go

@@ -1,6 +1,8 @@
 package aws
 
 import (
+	"strings"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/elasticache"
@@ -10,30 +12,29 @@ import (
 	"github.com/integr8ly/cluster-service/pkg/clusterservice"
 	"github.com/integr8ly/cluster-service/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"strings"
 )
 
-var _ ClusterResourceManager = &ElasticacheSnapshotEngine{}
+var _ ClusterResourceManager = &ElasticacheSnapshotManager{}
 
-type ElasticacheSnapshotEngine struct {
+type ElasticacheSnapshotManager struct {
 	elasticacheClient elasticacheiface.ElastiCacheAPI
 	taggingClient     resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
 	logger            *logrus.Entry
 }
 
-func newDefaultElasticacheSnapshotEngine(session *session.Session, logger *logrus.Entry) *ElasticacheSnapshotEngine {
-	return &ElasticacheSnapshotEngine{
+func NewDefaultElasticacheSnapshotManager(session *session.Session, logger *logrus.Entry) *ElasticacheSnapshotManager {
+	return &ElasticacheSnapshotManager{
 		elasticacheClient: elasticache.New(session),
 		taggingClient:     resourcegroupstaggingapi.New(session),
-		logger:            logger.WithField("engine", "aws_elasticache_snapshot"),
+		logger:            logger.WithField(loggingKeyManager, managerElasticacheSnapshot),
 	}
 }
 
-func (r *ElasticacheSnapshotEngine) GetName() string {
-	return "AWS elasticache Snapshot Engine"
+func (r *ElasticacheSnapshotManager) GetName() string {
+	return "AWS ElastiCache Snapshot Manager"
 }
 
-func (r *ElasticacheSnapshotEngine) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) ([]*clusterservice.ReportItem, error) {
+func (r *ElasticacheSnapshotManager) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) ([]*clusterservice.ReportItem, error) {
 	logger := r.logger.WithFields(logrus.Fields{"clusterId": clusterId, "dryRun": dryRun})
 	logger.Debug("deleting resources for cluster")
 
@@ -43,14 +44,7 @@ func (r *ElasticacheSnapshotEngine) DeleteResourcesForCluster(clusterId string,
 
 	resourceInput := &resourcegroupstaggingapi.GetResourcesInput{
 		ResourceTypeFilters: aws.StringSlice([]string{"elasticache:cluster"}),
-		TagFilters: []*resourcegroupstaggingapi.TagFilter{
-			{
-				Key: aws.String(tagKeyClusterId),
-				Values: aws.StringSlice([]string{
-					clusterId,
-				}),
-			},
-		},
+		TagFilters:          convertClusterTagsToAWSTagFilter(clusterId, tags),
 	}
 	resourceOutput, err := r.taggingClient.GetResources(resourceInput)
 	if err != nil {

pkg/aws/elasticache_test.go → pkg/aws/manager_elasticache_test.go

@@ -52,7 +52,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						c.GetResourcesFunc = func(in1 *resourcegroupstaggingapi.GetResourcesInput) (output *resourcegroupstaggingapi.GetResourcesOutput, e error) {
 							return nil, errors.New("")
 						}
@@ -87,7 +87,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						return nil
 					})
 					if err != nil {
@@ -118,7 +118,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						return nil
 					})
 					if err != nil {
@@ -149,7 +149,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						return nil
 					})
 					if err != nil {
@@ -177,7 +177,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						c.GetResourcesFunc = func(in1 *resourcegroupstaggingapi.GetResourcesInput) (output *resourcegroupstaggingapi.GetResourcesOutput, err error) {
 							return nil, errors.New("")
 						}
@@ -209,7 +209,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						return nil
 					})
 					if err != nil {
@@ -249,7 +249,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						return nil
 					})
 					if err != nil {
@@ -292,7 +292,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 					return fakeClient
 				},
 				taggingClient: func() *taggingClientMock {
-					fakeTaggingClient, err := fakeResourcetaggingClient(func(c *taggingClientMock) error {
+					fakeTaggingClient, err := fakeTaggingClient(func(c *taggingClientMock) error {
 						return nil
 					})
 					if err != nil {
@@ -321,7 +321,7 @@ func TestElasticacheEngine_DeleteResourcesForCluster(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			fakeClient := tt.fields.elasticacheClient()
-			r := &ElasticacheEngine{
+			r := &ElasticacheManager{
 				elasticacheClient: fakeClient,
 				taggingClient:     tt.fields.taggingClient(),
 				logger:            tt.fields.logger,

pkg/aws/manager_rds_snapshot.go

@@ -38,34 +38,21 @@ func NewDefaultRDSSnapshotManager(session *session.Session, logger *logrus.Entry
 	return &RDSSnapshotManager{
 		rdsClient:     rds.New(session),
 		taggingClient: resourcegroupstaggingapi.New(session),
-		logger:        logger.WithField(loggingKeyEngine, managerRDSSnapshot),
+		logger:        logger.WithField(loggingKeyManager, managerRDSSnapshot),
 	}
 }
 
 func (r *RDSSnapshotManager) GetName() string {
-	return "AWS RDS Snapshot Engine"
+	return "AWS RDS Snapshot Manager"
 }
 
 func (r *RDSSnapshotManager) DeleteResourcesForCluster(clusterId string, tags map[string]string, dryRun bool) ([]*clusterservice.ReportItem, error) {
 	r.logger.Debug("delete snapshots for cluster")
-	//convert provided tags to aws filter format
-	tagFilters := []*resourcegroupstaggingapi.TagFilter{
-		{
-			Key:    aws.String(tagKeyClusterId),
-			Values: aws.StringSlice([]string{clusterId}),
-		},
-	}
-	for tagKey, tagVal := range tags {
-		tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{
-			Key:    aws.String(tagKey),
-			Values: aws.StringSlice([]string{tagVal}),
-		})
-	}
 	//filter with tags
 	r.logger.Debug("listing rds snapshots using provided tag filters")
 	getResourcesInput := &resourcegroupstaggingapi.GetResourcesInput{
 		ResourceTypeFilters: aws.StringSlice([]string{resourceTypeRDSSnapshot}),
-		TagFilters:          tagFilters,
+		TagFilters:          convertClusterTagsToAWSTagFilter(clusterId, tags),
 	}
 	getResourcesOutput, err := r.taggingClient.GetResources(getResourcesInput)
 	if err != nil {