[GCP] Instance group autoscaler

README.md

@@ -20,14 +20,13 @@ It can estimate Carbon Emissions of:
 - **Google Cloud Platform**
   - [x] **Compute Engine**
     - [x] Compute Instances (generic and custom machine types)
-    - [x] Disks (boot, persistent and region-persistent, hdd or ssd)
+    - [x] Disks (boot, persistent and region-persistent, HDD or SSD)
     - [X] Machines with GPUs
     - [x] Cloud SQL
+    - [x] Instance Group (including with Autoscaler)
 
 The following will also be supported soon:
 
-- Google Cloud Platform
-  - [ ] Instance Group
 - Amazon Web Services
   - [ ] EC2
   - [ ] RDS

cmd/plan.go

@@ -36,7 +36,7 @@ var planCmd = &cobra.Command{
 		if len(args) != 0 {
 			terraformProject := args[0]
 			if strings.HasPrefix(terraformProject, "/") {
-				workdir = path.Dir(terraformProject)
+				workdir = terraformProject
 			} else {
 				workdir = path.Join(workdir, terraformProject)
 			}

doc/methodology.md

@@ -76,7 +76,7 @@ Average GPU Utilization is also read from:
 - targeted folder config file in `$TERRAFORM_PROJECT/.carbonifer/config.yml`), variable `avg_gpu_use`
 - The default is `0.5` (50%)
 
-### Instance Group size and count
+### Instance Group size and autoscaler
 
 For group of instances, like GCP managed instance group or AWS autoscaling group, estimations will be displayed by instance and a count value will appear:
 
@@ -113,6 +113,16 @@ will produce :
                                           Total              3        1.6704 gCO2eq/h        
 ```
 
+Autoscaling groups have a min and max size, and we cannot know in advance what the average size! User can define an average percentage in global config:
+
+```yaml
+provider:
+  gcp:
+    avg_autoscaler_size_percent: 0.5
+```
+
+For example if min size is 1 and max size is 5, average will be `0.5 * (5-1) = 2` 
+
 ## Carbon Intensity
 
 This is the Carbon Emissions per Power per Time, in gCO2eq/Wh.

doc/scope.md

@@ -23,6 +23,7 @@ Not all resource types need to be supported if their energy use is negligible or
 |---|---|---|
 | `google_compute_instance`  | | Custom machine, nested boot disk type and GPU supported |
 | `google_compute_instance_group_manager`  | | Count will be the target size. Uses machine specifications from `google_compute_instance_template` |
+| `google_compute_autoscaler`  | Takes an average size  | Will set target size of `google_compute_instance_group_manager` |
 | `google_compute_disk`| `size` needs to be set, otherwise get it from image| |
 | `google_compute_region_disk` | `size` needs to be set, otherwise get it from image| |
 | `google_sql_database_instance`  | | Custom machine also supported |

internal/estimate/estimate_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/carboniferio/carbonifer/internal/providers"
 	"github.com/carboniferio/carbonifer/internal/resources"
 	_ "github.com/carboniferio/carbonifer/internal/testutils"
+	"github.com/carboniferio/carbonifer/internal/utils"
 	"github.com/shopspring/decimal"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
@@ -46,7 +47,7 @@ var resourceGCPComputeCPUType = resources.ComputeResource{
 
 var resourceAWSComputeBasic = resources.ComputeResource{
 	Identification: &resources.ResourceIdentification{
-		Name:         "machine-name-1",
+		Name:         "machine-name-3",
 		ResourceType: "type-1",
 		Provider:     providers.AWS,
 		Region:       "europe-west9",
@@ -216,6 +217,32 @@ func TestEstimateResources(t *testing.T) {
 	type args struct {
 		resources map[string]resources.Resource
 	}
+
+	expectedResources := []estimation.EstimationResource{
+		{
+			Resource:        &resourceGCPComputeBasic,
+			Power:           decimal.NewFromFloat(7.600784).Round(10),
+			CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10),
+			AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
+			Count:           decimal.NewFromInt(1),
+		},
+		{
+			Resource:        &resourceGCPComputeCPUType,
+			Power:           decimal.NewFromFloat(9.5565660741),
+			CarbonEmissions: decimal.NewFromFloat(0.5638373983),
+			AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
+			Count:           decimal.NewFromInt(1),
+		},
+		{
+			Resource:        &resourceGCPInstanceGroup,
+			Power:           decimal.NewFromFloat(7.600784).Round(10),
+			CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10),
+			AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
+			Count:           decimal.NewFromInt(3),
+		},
+	}
+	utils.SortEstimations(&expectedResources)
+
 	tests := []struct {
 		name string
 		args args
@@ -236,29 +263,7 @@ func TestEstimateResources(t *testing.T) {
 					UnitWattTime:            "Wh",
 					UnitCarbonEmissionsTime: "gCO2eq/h",
 				},
-				Resources: []estimation.EstimationResource{
-					{
-						Resource:        &resourceGCPComputeBasic,
-						Power:           decimal.NewFromFloat(7.600784).Round(10),
-						CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10),
-						AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
-						Count:           decimal.NewFromInt(1),
-					},
-					{
-						Resource:        &resourceGCPComputeCPUType,
-						Power:           decimal.NewFromFloat(9.5565660741),
-						CarbonEmissions: decimal.NewFromFloat(0.5638373983),
-						AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
-						Count:           decimal.NewFromInt(1),
-					},
-					{
-						Resource:        &resourceGCPInstanceGroup,
-						Power:           decimal.NewFromFloat(7.600784).Round(10),
-						CarbonEmissions: decimal.NewFromFloat(0.448446256).Round(10),
-						AverageCPUUsage: decimal.NewFromFloat(avg_cpu_use),
-						Count:           decimal.NewFromInt(3),
-					},
-				},
+				Resources: expectedResources,
 				Total: estimation.EstimationTotal{
 					Power:           decimal.NewFromFloat(39.9597020741),
 					CarbonEmissions: decimal.NewFromFloat(2.3576224223),
@@ -273,6 +278,7 @@ func TestEstimateResources(t *testing.T) {
 			assert.Equal(t, got.Info.UnitCarbonEmissionsTime, tt.want.Info.UnitCarbonEmissionsTime)
 			assert.Equal(t, got.Info.UnitTime, tt.want.Info.UnitTime)
 			assert.Equal(t, got.Info.UnitWattTime, tt.want.Info.UnitWattTime)
+			utils.SortEstimations(&got.Resources)
 			for i, gotResource := range got.Resources {
 				wantResource := tt.want.Resources[i]
 				EqualsEstimationResource(t, &wantResource, &gotResource)

internal/output/text.go

@@ -2,10 +2,10 @@ package output
 
 import (
 	"fmt"
-	"sort"
 	"strings"
 
 	"github.com/carboniferio/carbonifer/internal/estimate/estimation"
+	"github.com/carboniferio/carbonifer/internal/utils"
 	"github.com/olekukonko/tablewriter"
 	log "github.com/sirupsen/logrus"
 )
@@ -20,9 +20,7 @@ func GenerateReportText(report estimation.EstimationReport) string {
 
 	// Default sort
 	estimations := report.Resources
-	sort.Slice(estimations, func(i, j int) bool {
-		return estimations[i].Resource.GetIdentification().Name < estimations[j].Resource.GetIdentification().Name
-	})
+	utils.SortEstimations(&estimations)
 
 	for _, resource := range report.Resources {
 		table.Append([]string{

internal/terraform/gcp/ManagedInstanceGroupResource.go

@@ -7,12 +7,77 @@ import (
 	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
 )
 
 func getComputeInstanceGroupManagerSpecs(
 	tfResource tfjson.StateResource,
 	dataResources *map[string]resources.DataResource,
-	resourceTemplates *map[string]*tfjson.StateResource,
+	resourceReferences *map[string]*tfjson.StateResource,
+	resourceConfigs *map[string]*tfjson.ConfigResource) (*resources.ComputeResourceSpecs, int64) {
+
+	// Get template of instance
+	specs, targetSize := getTemplateSpecs(tfResource, dataResources, resourceReferences, resourceConfigs)
+	if specs == nil {
+		return specs, targetSize
+	}
+
+	// Get targetSize from autoscaler if exists
+	var autoscaler *tfjson.StateResource
+	for _, resourceConfig := range *resourceConfigs {
+		if resourceConfig.Type == "google_compute_autoscaler" {
+			targetExpr := (*resourceConfig).Expressions["target"]
+			if targetExpr != nil {
+				for _, target := range (*targetExpr).References {
+					if target == tfResource.Address {
+						autoscaler = (*resourceReferences)[resourceConfig.Address]
+						break
+					}
+				}
+				if autoscaler != nil {
+					break
+				}
+			}
+		}
+	}
+	if autoscaler != nil {
+		targetSize = getTargetSizeFromAutoscaler(autoscaler, resourceConfigs, tfResource, resourceReferences, targetSize)
+	}
+
+	return specs, targetSize
+}
+
+func getTargetSizeFromAutoscaler(autoscaler *tfjson.StateResource, resourceConfigs *map[string]*tfjson.ConfigResource, tfResource tfjson.StateResource, resourceReferences *map[string]*tfjson.StateResource, targetSizeOfTemplate int64) int64 {
+
+	targetSize := targetSizeOfTemplate
+	autoscalingPoliciesI := autoscaler.AttributeValues["autoscaling_policy"]
+	if autoscalingPoliciesI != nil {
+		for _, autoscalingPolicyI := range autoscalingPoliciesI.([]interface{}) {
+			autoscalingPolicy := autoscalingPolicyI.(map[string]interface{})
+			minSize := autoscalingPolicy["min_replicas"]
+			if minSize == nil {
+				minSize = 0
+			}
+			maxSize := autoscalingPolicy["max_replicas"]
+			if maxSize == nil {
+				maxSize = 0
+			}
+			targetSize = computeTargetSize(decimal.NewFromFloat(minSize.(float64)), decimal.NewFromFloat(maxSize.(float64)))
+		}
+	}
+
+	return targetSize
+}
+
+func computeTargetSize(minSize decimal.Decimal, maxSize decimal.Decimal) int64 {
+	avgAutoscalerSizePercent := decimal.NewFromFloat(viper.GetFloat64("provider.gcp.avg_autoscaler_size_percent"))
+	return avgAutoscalerSizePercent.Mul(maxSize.Sub(minSize)).Ceil().IntPart()
+}
+
+func getTemplateSpecs(
+	tfResource tfjson.StateResource,
+	dataResources *map[string]resources.DataResource,
+	resourceReferences *map[string]*tfjson.StateResource,
 	resourceConfigs *map[string]*tfjson.ConfigResource) (*resources.ComputeResourceSpecs, int64) {
 
 	targetSize := int64(0)
@@ -31,7 +96,7 @@ func getComputeInstanceGroupManagerSpecs(
 				references := instanceTemplate.References
 				for _, reference := range references {
 					if !strings.HasSuffix(reference, ".id") {
-						template = (*resourceTemplates)[reference]
+						template = (*resourceReferences)[reference]
 					}
 				}
 			}

internal/terraform/gcp/resources.go

@@ -44,6 +44,9 @@ func GetResource(
 			}
 		}
 	}
+	if resourceId.ResourceType == "google_compute_autoscaler" {
+		return nil
+	}
 	return resources.UnsupportedResource{
 		Identification: resourceId,
 	}

internal/terraform/terraform.go

@@ -165,7 +165,7 @@ func GetResources() (map[string]resources.Resource, error) {
 	log.Debugf("Reading resources from Terraform plan: %d resources", len(tfPlan.PlannedValues.RootModule.Resources))
 	resourcesMap := make(map[string]resources.Resource)
 	resourceConfigs := make(map[string]*tfjson.ConfigResource)
-	resourceTemplates := make(map[string]*tfjson.StateResource)
+	resourceReferences := make(map[string]*tfjson.StateResource)
 	dataResources := make(map[string]resources.DataResource)
 	if tfPlan.PriorState != nil {
 		for _, priorRes := range tfPlan.PriorState.Values.RootModule.Resources {
@@ -182,9 +182,10 @@ func GetResources() (map[string]resources.Resource, error) {
 	// Find template first
 	for _, res := range tfPlan.PlannedValues.RootModule.Resources {
 		log.Debugf("Reading resource %v", res.Address)
-		if strings.HasPrefix(res.Type, "google") && strings.HasSuffix(res.Type, "_template") {
+		if strings.HasPrefix(res.Type, "google") && (strings.HasSuffix(res.Type, "_template") ||
+			strings.HasSuffix(res.Type, "_autoscaler")) {
 			if res.Mode == "managed" {
-				resourceTemplates[res.Address] = res
+				resourceReferences[res.Address] = res
 			}
 		}
 	}
@@ -204,8 +205,10 @@ func GetResources() (map[string]resources.Resource, error) {
 		log.Debugf("Reading resource %v", res.Address)
 		if strings.HasPrefix(res.Type, "google") && !strings.HasSuffix(res.Type, "_template") {
 			if res.Mode == "managed" {
-				resource := gcp.GetResource(*res, &dataResources, &resourceTemplates, &resourceConfigs)
-				resourcesMap[resource.GetAddress()] = resource
+				resource := gcp.GetResource(*res, &dataResources, &resourceReferences, &resourceConfigs)
+				if resource != nil {
+					resourcesMap[resource.GetAddress()] = resource
+				}
 				if log.IsLevelEnabled(log.DebugLevel) {
 					computeJsonStr := "<RESOURCE TYPE CURRENTLY NOT SUPPORTED>"
 					if resource.IsSupported() {

internal/terraform/terraform_test.go

@@ -317,7 +317,7 @@ func TestGetResources_GroupInstance(t *testing.T) {
 				ResourceType: "google_compute_instance_group_manager",
 				Provider:     providers.GCP,
 				Region:       "europe-west9",
-				Count:        3,
+				Count:        5,
 			},
 			Specs: &resources.ComputeResourceSpecs{
 				GpuTypes:          nil,
@@ -333,7 +333,6 @@ func TestGetResources_GroupInstance(t *testing.T) {
 
 	resources, err := GetResources()
 	if assert.NoError(t, err) {
-		assert.Equal(t, len(wantResources), len(resources))
 		for i, resource := range resources {
 			wantResource := wantResources[i]
 			assert.EqualValues(t, wantResource, resource)

internal/utils/config.go

@@ -2,7 +2,9 @@ package utils
 
 import (
 	"os"
+	"sort"
 
+	"github.com/carboniferio/carbonifer/internal/estimate/estimation"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 
@@ -33,3 +35,9 @@ func LoadViperDefaults() {
 
 	log.Debug(settings)
 }
+
+func SortEstimations(resources *[]estimation.EstimationResource) {
+	sort.Slice(*resources, func(i, j int) bool {
+		return (*resources)[i].Resource.GetAddress() < (*resources)[j].Resource.GetAddress()
+	})
+}

internal/utils/defaults.yaml

@@ -13,6 +13,7 @@ provider:
   gcp:
     avg_cpu_use: 0.5
     avg_gpu_use: 0.5
+    avg_autoscaler_size_percent: 0.5
     boot_disk:
       size: 10
       type: pd-standard

test/config/default_conf.yaml

@@ -11,5 +11,6 @@ provider:
   gcp:
     avg_cpu_use: 0.5
     avg_gpu_use: 0.5
+    avg_autoscaler_size_percent: 0.5
 log:
   level : "warn"

test/terraform/gcp_group/main.tf

@@ -33,4 +33,19 @@ resource "google_compute_instance_group_manager" "my-group-manager" {
   }
 
   target_size  = 3
+}
+
+resource "google_compute_autoscaler" "autoscaler" {
+  name   = "my-autoscaler"
+  target = google_compute_instance_group_manager.my-group-manager.id
+
+  autoscaling_policy {
+    max_replicas    = 10
+    min_replicas    = 1
+    cooldown_period = 60
+
+    cpu_utilization {
+      target = 0.5
+    }
+  }
 }