Skip to content

Commit

Permalink
[Platform] Add API for suggested Kubernetes provider configuration
Browse files Browse the repository at this point in the history 
- This API tries to fetch the details which can be used to pre-fill
  the data during Kubernetes provider creation (should work on OpenShift
  or Tanzu as well). The returned JSON is similar to what we use for the
  create call. Please refer this issue for more details:
  #7394
- Use blank string for KUBECONFIG if kubeconfig file is not
  provided. This allows us to in-cluster credentials (ServiceAccount)
  when running in the same cluster as of the target cluster. Can be used
  to simplify the current flow later, we won't need a separate
  kubeconfig in that case.
- Finds out the region and AZ based on node annotations.
- Takes the pull secret name and the storage class name from app config.

Scenarios tested:

- Ran platform inside Kubernetes, curl the API, it returns the
  expected JSON.
- Tested all the failure scenarios like missing config, no permissions
  to get secret, nodes etc.

This PR is backend part for the #7394

Signed-off-by: Bhavin Gandhi <[email protected]>
  • Loading branch information
@bhavin192
bhavin192 committed Jun 7, 2021
1 parent 3900a7a commit c0fcd57
Show file tree
Hide file tree
Showing 9 changed files with 1,672 additions and 1,248 deletions.
3 changes: 2 additions & 1 deletion managed/build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,8 @@ libraryDependencies ++= Seq(
"com.fasterxml.jackson.core" % "jackson-core" % "2.10.5",
"com.jayway.jsonpath" % "json-path" % "2.4.0",
"commons-io" % "commons-io" % "2.8.0",
"commons-codec" % "commons-codec" % "1.15"
"commons-codec" % "commons-codec" % "1.15",
"org.apache.commons" % "commons-collections4" % "4.4"
)
// Clear default resolvers.
appResolvers := None
Expand Down
2 changes: 2 additions & 0 deletions .../src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ public void run() {
String pullSecret = this.getPullSecret();
if (pullSecret != null) {
response = kubernetesManager.applySecret(config, taskParams().namespace, pullSecret);
} else {
LOG.debug("Pull secret is missing, skipping the pull secret creation.");
}
break;
case HELM_INSTALL:
Expand Down
46 changes: 46 additions & 0 deletions managed/src/main/java/com/yugabyte/yw/common/KubernetesManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@

package com.yugabyte.yw.common;

import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.yugabyte.yw.common.Util;
import com.yugabyte.yw.models.Provider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Singleton;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
Expand Down Expand Up @@ -136,6 +140,48 @@ public ShellResponse getServiceIPs(
return execCommand(config, commandList);
}

public JsonNode getNodeInfos(Map<String, String> config) {
ShellResponse response = runGetNodeInfos(config);
if (response.code != 0) {
String msg = "Unable to get node information";
if (!response.message.isEmpty()) {
msg = String.format("%s: %s", msg, response.message);
}
throw new RuntimeException(msg);
}
return Util.convertStringToJson(response.message);
}

public ShellResponse runGetNodeInfos(Map<String, String> config) {
List<String> commandList = ImmutableList.of("kubectl", "get", "nodes", "-o", "json");
return execCommand(config, commandList);
}

public JsonNode getSecret(Map<String, String> config, String secretName, String namespace) {
ShellResponse response = runGetSecret(config, secretName, namespace);
if (response.code != 0) {
String msg = "Unable to get secret";
if (!response.message.isEmpty()) {
msg = String.format("%s: %s", msg, response.message);
}
throw new RuntimeException(msg);
}
return Util.convertStringToJson(response.message);
}

// TODO: disable the logging of stdout of this command if possibile,
// as it just leaks the secret content in the logs at DEBUG level.
public ShellResponse runGetSecret(
Map<String, String> config, String secretName, String namespace) {
List<String> commandList = new ArrayList<String>();
commandList.addAll(ImmutableList.of("kubectl", "get", "secret", secretName, "-o", "json"));
if (namespace != null) {
commandList.add("--namespace");
commandList.add(namespace);
}
return execCommand(config, commandList);
}

public ShellResponse helmUpgrade(
Map<String, String> config, String universePrefix, String namespace, String overridesFile) {
String helmPackagePath = appConfig.getString("yb.helm.package");
Expand Down
134 changes: 132 additions & 2 deletions managed/src/main/java/com/yugabyte/yw/controllers/CloudProviderController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.inject.Inject;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableList;
import com.typesafe.config.Config;
import com.yugabyte.yw.cloud.AWSInitializer;
import com.yugabyte.yw.cloud.AZUInitializer;
Expand All @@ -26,6 +29,8 @@
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.collections4.MultiValuedMap;
import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import play.api.Play;
Expand Down Expand Up @@ -84,6 +89,10 @@ public CloudProviderController(Config config) {

@Inject CloudAPI.Factory cloudAPIFactory;

@Inject KubernetesManager kubernetesManager;

@Inject play.Configuration appConfig;

/**
* GET endpoint for listing providers
*
Expand Down Expand Up @@ -216,7 +225,8 @@ public Result createKubernetes(UUID customerUUID) throws IOException {
throw new YWServiceException(BAD_REQUEST, "Kubeconfig can't be at two levels");
}
} else if (!hasConfig) {
throw new YWServiceException(BAD_REQUEST, "No Kubeconfig found for zone(s)");
LOG.warn(
"No Kubeconfig found at any level, in-cluster service account credentials will be used.");
}
}
}
Expand All @@ -238,7 +248,10 @@ public Result createKubernetes(UUID customerUUID) throws IOException {
Map<String, String> zoneConfig = zd.config;
AvailabilityZone az = AvailabilityZone.create(region, zd.code, zd.name, null);
boolean isConfigInZone = updateKubeConfig(provider, region, az, zoneConfig, false);
if (isConfigInZone) {}
if (!(isConfigInProvider || isConfigInRegion || isConfigInZone)) {
// Use in-cluster ServiceAccount credentials
az.setConfig(ImmutableMap.of("KUBECONFIG", ""));
}
}
}
try {
Expand Down Expand Up @@ -378,6 +391,123 @@ private void createKubernetesInstanceTypes(Provider provider, UUID customerUUID)
}
}

// Performs discovery of region, zones, pull secret, storageClass
// when running inside a Kubernetes cluster. Returns the discovered
// information as a JSON, which is similar to the one which is
// passed to the createKubernetes method.
@ApiOperation(
value = "getSuggestedKubernetesConfigs",
response = KubernetesProviderFormData.class)
public Result getSuggestedKubernetesConfigs(UUID customerUUID) {
try {
MultiValuedMap<String, String> regionToAZ = getKubernetesRegionToZoneInfo();
if (regionToAZ.isEmpty()) {
LOG.info(
"No regions and zones found, check if the region and zone labels are present on the nodes. https://k8s.io/docs/reference/labels-annotations-taints/");
throw new YWServiceException(
INTERNAL_SERVER_ERROR, "No region and zone information found.");
}

String storageClass = appConfig.getString("yb.kubernetes.storageClass");
String pullSecretName = appConfig.getString("yb.kubernetes.pullSecretName");
if (storageClass == null || pullSecretName == null) {
LOG.error("Required configuration keys from yb.kubernetes.* are missing.");
throw new YWServiceException(INTERNAL_SERVER_ERROR, "Required configuration is missing.");
}
String pullSecretContent = getKubernetesPullSecretContent(pullSecretName);

KubernetesProviderFormData formData = new KubernetesProviderFormData();
formData.code = Common.CloudType.kubernetes;
if (pullSecretContent != null) {
formData.config =
ImmutableMap.of(
"KUBECONFIG_IMAGE_PULL_SECRET_NAME", pullSecretName,
"KUBECONFIG_PULL_SECRET_NAME", pullSecretName, // filename
"KUBECONFIG_PULL_SECRET_CONTENT", pullSecretContent);
}

for (String region : regionToAZ.keySet()) {
RegionData regionData = new RegionData();
regionData.code = region;
for (String az : regionToAZ.get(region)) {
ZoneData zoneData = new ZoneData();
zoneData.code = az;
zoneData.name = az;
zoneData.config = ImmutableMap.of("STORAGE_CLASS", storageClass);
regionData.zoneList.add(zoneData);
}
formData.regionList.add(regionData);
}

ObjectMapper mapper = new ObjectMapper();
return ApiResponse.success(mapper.valueToTree(formData));
} catch (RuntimeException e) {
throw new YWServiceException(INTERNAL_SERVER_ERROR, e.getMessage());
}
}

// Performs region and zone discovery based on
// topology/failure-domain labels from the Kubernetes nodes.
private MultiValuedMap<String, String> getKubernetesRegionToZoneInfo() {
JsonNode nodeInfos = kubernetesManager.getNodeInfos(null);
MultiValuedMap<String, String> regionToAZ = new HashSetValuedHashMap<>();
for (JsonNode nodeInfo : nodeInfos.path("items")) {
JsonNode nodeLabels = nodeInfo.path("metadata").path("labels");
// failure-domain.beta.k8s.io is deprecated as of 1.17
String region = nodeLabels.path("topology.kubernetes.io/region").asText();
region =
region.isEmpty()
? nodeLabels.path("failure-domain.beta.kubernetes.io/region").asText()
: region;
String zone = nodeLabels.path("topology.kubernetes.io/zone").asText();
zone =
zone.isEmpty()
? nodeLabels.path("failure-domain.beta.kubernetes.io/zone").asText()
: zone;
if (region.isEmpty() || zone.isEmpty()) {
LOG.debug(
"Value of the zone or region label is empty for "
+ nodeInfo.path("metadata").path("name").asText()
+ ", skipping.");
continue;
}
regionToAZ.put(region, zone);
}
return regionToAZ;
}

// Fetches the secret secretName from current namespace, removes
// extra metadata and returns the secret as JSON string. Returns
// null if the secret is not present.
private String getKubernetesPullSecretContent(String secretName) {
JsonNode pullSecretJson;
try {
pullSecretJson = kubernetesManager.getSecret(null, secretName, null);
} catch (RuntimeException e) {
if (e.getMessage().contains("Error from server (NotFound): secrets")) {
LOG.debug(
"The pull secret " + secretName + " is not present, provider won't have this field.");
return null;
}
throw new RuntimeException("Unable to fetch the pull secret.");
}
JsonNode secretMetadata = pullSecretJson.get("metadata");
if (secretMetadata == null) {
LOG.error(
"metadata of the pull secret " + secretName + " is missing. This should never happen.");
throw new RuntimeException("Error while fetching the pull secret.");
}
((ObjectNode) secretMetadata)
.remove(
ImmutableList.of(
"namespace", "uid", "selfLink", "creationTimestamp", "resourceVersion"));
JsonNode secretAnnotations = secretMetadata.get("annotations");
if (secretAnnotations != null) {
((ObjectNode) secretAnnotations).remove("kubectl.kubernetes.io/last-applied-configuration");
}
return pullSecretJson.toString();
}

// TODO: This is temporary endpoint, so we can setup docker, will move this
// to standard provider bootstrap route soon.
public Result setupDocker(UUID customerUUID) {
Expand Down
2 changes: 2 additions & 0 deletions managed/src/main/resources/application.test.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ yb {
# Keep more frequent gc runs in non-prod to catch any bugs:
taskGC.gc_check_interval = 1 hour
taskGC.task_retention_duration = 5 days
kubernetes.storageClass = "ssd-class"
kubernetes.pullSecretName = "pull-sec"
}

ebean {
Expand Down
Loading

0 comments on commit c0fcd57

Please sign in to comment.