Skip to content

Commit

Permalink
Update GKE client used in cloudprovider
Browse files Browse the repository at this point in the history
  • Loading branch information
@MaciekPytel
MaciekPytel committed Jun 15, 2018
1 parent cb9ee11 commit 9206448
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 44 deletions.
52 changes: 25 additions & 27 deletions cluster-autoscaler/cloudprovider/gce/gce_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ import (
"golang.org/x/oauth2/google"
gce "google.golang.org/api/compute/v1"
gke "google.golang.org/api/container/v1"
gke_alpha "google.golang.org/api/container/v1alpha1"
gke_beta "google.golang.org/api/container/v1beta1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -149,10 +148,9 @@ type gceManagerImpl struct {
migs []*migInformation
migCache map[GceRef]*Mig

gceService *gce.Service
gkeService *gke.Service
gkeAlphaService *gke_alpha.Service
gkeBetaService *gke_beta.Service
gceService *gce.Service
gkeService *gke.Service
gkeBetaService *gke_beta.Service

cacheMutex sync.Mutex
migsMutex sync.Mutex
Expand Down Expand Up @@ -274,14 +272,14 @@ func CreateGceManager(configReader io.Reader, mode GcpCloudProviderMode, cluster
manager.gkeBetaService = gkeBetaService
}
case ModeGKENAP:
gkeAlphaService, err := gke_alpha.New(client)
gkeBetaService, err := gke_beta.New(client)
if err != nil {
return nil, err
}
if *gkeAPIEndpoint != "" {
gkeAlphaService.BasePath = *gkeAPIEndpoint
gkeBetaService.BasePath = *gkeAPIEndpoint
}
manager.gkeAlphaService = gkeAlphaService
manager.gkeBetaService = gkeBetaService
glog.V(1).Info("Using GKE-NAP mode")
}

Expand Down Expand Up @@ -464,7 +462,7 @@ func (m *gceManagerImpl) fetchAllNodePoolsGkeRegionalImpl() error {
func (m *gceManagerImpl) fetchAllNodePoolsGkeNapImpl() error {
m.assertGKENAP()

nodePoolsResponse, err := m.gkeAlphaService.Projects.Zones.Clusters.NodePools.List(m.projectId, m.location, m.clusterName).Do()
nodePoolsResponse, err := m.gkeBetaService.Projects.Zones.Clusters.NodePools.List(m.projectId, m.location, m.clusterName).Do()
if err != nil {
return err
}
Expand Down Expand Up @@ -589,7 +587,7 @@ func (m *gceManagerImpl) deleteNodePool(toBeRemoved *Mig) error {
return fmt.Errorf("only autoprovisioned node pools can be deleted")
}
// TODO: handle multi-zonal node pools.
deleteOp, err := m.gkeAlphaService.Projects.Zones.Clusters.NodePools.Delete(m.projectId, m.location, m.clusterName,
deleteOp, err := m.gkeBetaService.Projects.Zones.Clusters.NodePools.Delete(m.projectId, m.location, m.clusterName,
toBeRemoved.nodePoolName).Do()
if err != nil {
return err
Expand All @@ -607,22 +605,22 @@ func (m *gceManagerImpl) createNodePool(mig *Mig) error {
// TODO: handle preemptible
// TODO: handle SSDs

accelerators := []*gke_alpha.AcceleratorConfig{}
accelerators := []*gke_beta.AcceleratorConfig{}

if gpuRequest, found := mig.spec.extraResources[gpu.ResourceNvidiaGPU]; found {
gpuType, found := mig.spec.labels[gpu.GPULabel]
if !found {
return fmt.Errorf("failed to create node pool %v with gpu request of unspecified type", mig.nodePoolName)
}
gpuConfig := &gke_alpha.AcceleratorConfig{
gpuConfig := &gke_beta.AcceleratorConfig{
AcceleratorType: gpuType,
AcceleratorCount: gpuRequest.Value(),
}
accelerators = append(accelerators, gpuConfig)

}

taints := []*gke_alpha.NodeTaint{}
taints := []*gke_beta.NodeTaint{}
for _, taint := range mig.spec.taints {
if taint.Key == gpu.ResourceNvidiaGPU {
continue
Expand All @@ -631,7 +629,7 @@ func (m *gceManagerImpl) createNodePool(mig *Mig) error {
if !found {
effect = "EFFECT_UNSPECIFIED"
}
taint := &gke_alpha.NodeTaint{
taint := &gke_beta.NodeTaint{
Effect: effect,
Key: taint.Key,
Value: taint.Value,
Expand All @@ -646,31 +644,31 @@ func (m *gceManagerImpl) createNodePool(mig *Mig) error {
}
}

config := gke_alpha.NodeConfig{
config := gke_beta.NodeConfig{
MachineType: mig.spec.machineType,
OauthScopes: defaultOAuthScopes,
Labels: labels,
Accelerators: accelerators,
Taints: taints,
}

autoscaling := gke_alpha.NodePoolAutoscaling{
autoscaling := gke_beta.NodePoolAutoscaling{
Enabled: true,
MinNodeCount: napMinNodes,
MaxNodeCount: napMaxNodes,
Autoprovisioned: true,
}

createRequest := gke_alpha.CreateNodePoolRequest{
NodePool: &gke_alpha.NodePool{
createRequest := gke_beta.CreateNodePoolRequest{
NodePool: &gke_beta.NodePool{
Name: mig.nodePoolName,
InitialNodeCount: 0,
Config: &config,
Autoscaling: &autoscaling,
},
}

createOp, err := m.gkeAlphaService.Projects.Zones.Clusters.NodePools.Create(m.projectId, m.location, m.clusterName,
createOp, err := m.gkeBetaService.Projects.Zones.Clusters.NodePools.Create(m.projectId, m.location, m.clusterName,
&createRequest).Do()
if err != nil {
return err
Expand Down Expand Up @@ -698,7 +696,7 @@ func (m *gceManagerImpl) fetchMachinesCache() error {
}
var locations []string
if m.mode == ModeGKENAP {
cluster, err := m.gkeAlphaService.Projects.Locations.Clusters.Get(fmt.Sprintf("projects/%s/locations/%s/clusters/%s", m.projectId, m.location, m.clusterName)).Do()
cluster, err := m.gkeBetaService.Projects.Locations.Clusters.Get(fmt.Sprintf("projects/%s/locations/%s/clusters/%s", m.projectId, m.location, m.clusterName)).Do()
if err != nil {
return err
}
Expand Down Expand Up @@ -775,10 +773,10 @@ func (m *gceManagerImpl) waitForOp(operation *gce.Operation, project string, zon
}

// GKE
func (m *gceManagerImpl) waitForGkeOp(operation *gke_alpha.Operation) error {
func (m *gceManagerImpl) waitForGkeOp(operation *gke_beta.Operation) error {
for start := time.Now(); time.Since(start) < gkeOperationWaitTimeout; time.Sleep(operationPollInterval) {
glog.V(4).Infof("Waiting for operation %s %s %s", m.projectId, m.location, operation.Name)
if op, err := m.gkeAlphaService.Projects.Zones.Operations.Get(m.projectId, m.location, operation.Name).Do(); err == nil {
if op, err := m.gkeBetaService.Projects.Zones.Operations.Get(m.projectId, m.location, operation.Name).Do(); err == nil {
glog.V(4).Infof("Operation %s %s %s status: %s", m.projectId, m.location, operation.Name, op.Status)
if op.Status == "DONE" {
return nil
Expand Down Expand Up @@ -1082,7 +1080,7 @@ func (m *gceManagerImpl) buildMigFromAutoCfg(link string, cfg cloudprovider.MIGA

func (m *gceManagerImpl) fetchResourceLimiter() error {
if m.mode == ModeGKENAP {
cluster, err := m.gkeAlphaService.Projects.Zones.Clusters.Get(m.projectId, m.location, m.clusterName).Do()
cluster, err := m.gkeBetaService.Projects.Zones.Clusters.Get(m.projectId, m.location, m.clusterName).Do()
if err != nil {
return err
}
Expand All @@ -1093,11 +1091,11 @@ func (m *gceManagerImpl) fetchResourceLimiter() error {
minLimits := make(map[string]int64)
maxLimits := make(map[string]int64)
for _, limit := range cluster.Autoscaling.ResourceLimits {
if _, found := supportedResources[limit.Name]; !found {
glog.Warning("Unsupported limit defined %s: %d - %d", limit.Name, limit.Minimum, limit.Maximum)
if _, found := supportedResources[limit.ResourceType]; !found {
glog.Warning("Unsupported limit defined %s: %d - %d", limit.ResourceType, limit.Minimum, limit.Maximum)
}
minLimits[limit.Name] = limit.Minimum
maxLimits[limit.Name] = limit.Maximum
minLimits[limit.ResourceType] = limit.Minimum
maxLimits[limit.ResourceType] = limit.Maximum
}

// GKE API provides memory in GB, but ResourceLimiter expects them in bytes
Expand Down
33 changes: 16 additions & 17 deletions cluster-autoscaler/cloudprovider/gce/gce_manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (
"github.com/stretchr/testify/mock"
gce "google.golang.org/api/compute/v1"
gke "google.golang.org/api/container/v1"
gke_alpha "google.golang.org/api/container/v1alpha1"
gke_beta "google.golang.org/api/container/v1beta1"
apiv1 "k8s.io/api/core/v1"
)
Expand Down Expand Up @@ -439,7 +438,7 @@ const getClusterResponse = `{
"maxNodeCount": 5
},
"management": {},
"selfLink": "https:///v1alpha1/projects/user-gke-dev/zones/us-central1-c/clusters/usertest/nodePools/default-pool",
"selfLink": "https:///v1beta1/projects/user-gke-dev/zones/us-central1-c/clusters/usertest/nodePools/default-pool",
"version": "1.8.0-gke.1",
"instanceGroupUrls": [
"https://www.googleapis.com/compute/v1/projects/user-gke-dev/zones/us-central1-c/instanceGroupManagers/gke-usertest-default-pool-fdsafds2d5-grp"
Expand All @@ -455,12 +454,12 @@ const getClusterResponse = `{
"autoscaling": {
"resourceLimits": [
{
"name": "cpu",
"resourceType": "cpu",
"minimum": "2",
"maximum": "3"
},
{
"name": "memory",
"resourceType": "memory",
"minimum": "2000000000",
"maximum": "3000000000"
}
Expand All @@ -469,7 +468,7 @@ const getClusterResponse = `{
"networkConfig": {
"network": "https://www.googleapis.com/compute/v1/projects/user-gke-dev/global/networks/default"
},
"selfLink": "https:///v1alpha1/projects/user-gke-dev/zones/us-central1-c/clusters/usertest",
"selfLink": "https:///v1beta1/projects/user-gke-dev/zones/us-central1-c/clusters/usertest",
"zone": "us-central1-c",
"endpoint": "xxx",
"initialClusterVersion": "1.sdafsa",
Expand Down Expand Up @@ -552,10 +551,10 @@ func newTestGceManager(t *testing.T, testServerURL string, mode GcpCloudProvider
}

if mode == ModeGKENAP {
gkeService, err := gke_alpha.New(client)
gkeService, err := gke_beta.New(client)
assert.NoError(t, err)
gkeService.BasePath = testServerURL
manager.gkeAlphaService = gkeService
manager.gkeBetaService = gkeService
}

return manager
Expand Down Expand Up @@ -676,9 +675,9 @@ func TestDeleteNodePool(t *testing.T) {
defer server.Close()
g := newTestGceManager(t, server.URL, ModeGKENAP, false)

server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools/nodeautoprovisioning-323233232").Return(deleteNodePoolResponse).Once()
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/operations/operation-1505732351373-819ed94e").Return(deleteNodePoolOperationResponse).Once()
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools").Return(allNodePools2).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools/nodeautoprovisioning-323233232").Return(deleteNodePoolResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/operations/operation-1505732351373-819ed94e").Return(deleteNodePoolOperationResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools").Return(allNodePools2).Once()
server.On("handle", "/project1/zones/us-central1-b/instanceGroupManagers/gke-cluster-1-default-pool").Return(getInstanceGroupManager(zoneB)).Once()
server.On("handle", "/project1/zones/us-central1-b/instanceGroupManagers/gke-cluster-1-nodeautoprovisioning-323233232").Return(getInstanceGroupManager(zoneB)).Once()
server.On("handle", "/project1/global/instanceTemplates/gke-cluster-1-default-pool").Return(instanceTemplate).Once()
Expand Down Expand Up @@ -733,9 +732,9 @@ func TestCreateNodePool(t *testing.T) {
defer server.Close()
g := newTestGceManager(t, server.URL, ModeGKENAP, false)

server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/operations/operation-1505728466148-d16f5197").Return(createNodePoolOperationResponse).Once()
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools").Return(createNodePoolResponse).Once()
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools").Return(allNodePools2).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/operations/operation-1505728466148-d16f5197").Return(createNodePoolOperationResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools").Return(createNodePoolResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/clusters/cluster1/nodePools").Return(allNodePools2).Once()
server.On("handle", "/project1/zones/us-central1-b/instanceGroupManagers/gke-cluster-1-default-pool").Return(getInstanceGroupManager(zoneB)).Once()
server.On("handle", "/project1/zones/us-central1-b/instanceGroupManagers/gke-cluster-1-nodeautoprovisioning-323233232").Return(getInstanceGroupManager(zoneB)).Once()
server.On("handle", "/project1/global/instanceTemplates/gke-cluster-1-default-pool").Return(instanceTemplate).Once()
Expand Down Expand Up @@ -820,10 +819,10 @@ func TestWaitForGkeOp(t *testing.T) {
server := NewHttpServerMock()
defer server.Close()
g := newTestGceManager(t, server.URL, ModeGKENAP, false)
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/operations/operation-1505728466148-d16f5197").Return(operationRunningResponse).Once()
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/operations/operation-1505728466148-d16f5197").Return(operationDoneResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/operations/operation-1505728466148-d16f5197").Return(operationRunningResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/operations/operation-1505728466148-d16f5197").Return(operationDoneResponse).Once()

operation := &gke_alpha.Operation{Name: "operation-1505728466148-d16f5197"}
operation := &gke_beta.Operation{Name: "operation-1505728466148-d16f5197"}

err := g.waitForGkeOp(operation)
assert.NoError(t, err)
Expand Down Expand Up @@ -1099,7 +1098,7 @@ func TestFetchResourceLimiter(t *testing.T) {
defer server.Close()

g := newTestGceManager(t, server.URL, ModeGKENAP, false)
server.On("handle", "/v1alpha1/projects/project1/zones/us-central1-b/clusters/cluster1").Return(getClusterResponse).Once()
server.On("handle", "/v1beta1/projects/project1/zones/us-central1-b/clusters/cluster1").Return(getClusterResponse).Once()

err := g.fetchResourceLimiter()
assert.NoError(t, err)
Expand Down

0 comments on commit 9206448

Please sign in to comment.