From 71875f48ec3ca50ab5a04e7ad054b1c3c53e513f Mon Sep 17 00:00:00 2001 From: Justin Lin Date: Thu, 21 Dec 2023 17:30:15 -0800 Subject: [PATCH] Add host identifier to resource info (#2679) Adding hostname as an identifier to find the resource. --- .../java/com/github/ambry/rest/RestUtils.java | 5 ++ .../ambry/clustermap/HelixClusterManager.java | 54 ++++++++++++++----- .../frontend/GetResourceInfoHandler.java | 3 ++ 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java b/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java index 5d32293179..cf4723d1d1 100644 --- a/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java +++ b/ambry-api/src/main/java/com/github/ambry/rest/RestUtils.java @@ -328,6 +328,11 @@ public static final class Headers { * Request header to carry resource name; */ public final static String RESOURCE = "x-ambry-resource"; + + /** + * Request header to carry hostname (with port); + */ + public final static String HOSTNAME = "x-ambry-hostname"; } public static final class TrackingHeaders { diff --git a/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixClusterManager.java b/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixClusterManager.java index 827ea31ef9..77b54d1d30 100644 --- a/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixClusterManager.java +++ b/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixClusterManager.java @@ -77,6 +77,7 @@ */ public class HelixClusterManager implements ClusterMap { private static final Logger logger = LoggerFactory.getLogger(HelixClusterManager.class); + private static final int DEFAULT_NUM_REPLICAS = 3; private final String clusterName; private final String selfInstanceName; private final MetricRegistry metricRegistry; @@ -1793,10 +1794,14 @@ private void updatePartitionResourceMappingFromIdealStates(Collection(state.getPartitionSet())); @@ -2169,11 +2174,13 @@ ReplicaSealStatus resolveReplicaSealStatus(String partitionName, Collection identifyResources(HelixClusterManager helixClusterManager) { } } + /** + * An implementation of {@link ResourceIdentifier} to return the resource name the given hostname belongs to . + */ + public static class HostnameIdentifier implements ResourceIdentifier { + private final String hostname; + + /** + * Constructor to create a {@link HostnameIdentifier}. + * @param hostname + */ + public HostnameIdentifier(String hostname) { + this.hostname = hostname; + } + + @Override + public List identifyResources(HelixClusterManager helixClusterManager) { + String dcName = helixClusterManager.clusterMapConfig.clusterMapDatacenterName; + DataNodeId dataNodeId = helixClusterManager.instanceNameToAmbryDataNode.get(hostname); + if (dataNodeId == null || !dataNodeId.getDatacenterName().equals(dcName)) { + throw new IllegalArgumentException("Host " + hostname + " doesn't exist in this datacenter"); + } + List tags = helixClusterManager.instanceNameToInstanceConfig.get(hostname).getTags(); + return tags.stream() + .map(tag -> helixClusterManager.dcToTagToResourceProperty.get(dataNodeId.getDatacenterName()).get(tag).name) + .collect(Collectors.toList()); + } + } + /** * An implementation to return all the resource this cluster map has. */ @@ -2283,8 +2318,7 @@ private ResourceInfo getResourceInfo(String resourceName) { List liveInstances = allInstances.stream() .map(instanceNameToAmbryDataNode::get) .filter(dn -> dn.getState() == HardwareState.AVAILABLE) - .map(ClusterMapUtils::getInstanceName) - .collect(Collectors.toList()); + .map(ClusterMapUtils::getInstanceName).collect(Collectors.toList()); List unavailableInstances = allInstances.stream() .map(instanceNameToAmbryDataNode::get) .filter(dn -> dn.getState() == HardwareState.UNAVAILABLE) @@ -2294,11 +2328,8 @@ private ResourceInfo getResourceInfo(String resourceName) { long liveCapacity = getResourceAvailableRegisteredHostDiskCapacity(resourceName); long unavailableCapacity = totalCapacity - liveCapacity; int numPartitions = getNumberOfPartitionsInResource(resourceName); - int replicationFactor = 3; // by default it is 3; - String numReplicaStr = getResourceConfig(resourceName, dcName).getNumReplica(); - if (numReplicaStr != null) { - replicationFactor = Integer.parseInt(numReplicaStr); - } + String tag = dcToResourceNameToTag.get(dcName).get(resourceName); + int replicationFactor = dcToTagToResourceProperty.get(dcName).get(tag).replicationFactor; int numExpectedReplicas = numPartitions * replicationFactor; int numCurrentReplicas = getReplicaCountForStateInResource(null, resourceName); int expectedTotalReplicaWeight = getResourceExpectedTotalDiskCapacityUsage(resourceName, replicationFactor); @@ -2306,9 +2337,8 @@ private ResourceInfo getResourceInfo(String resourceName) { Map> failedDisks = new HashMap<>(); for (String instanceName : getAllInstancesForResource(resourceName)) { Set disks = ambryDataNodeToAmbryDisks.get(instanceNameToAmbryDataNode.get(instanceName)); - Set failedDiskMountPath = disks.stream() - .filter(disk -> disk.getState() == HardwareState.UNAVAILABLE) - .map(AmbryDisk::getMountPath) + Set failedDiskMountPath = + disks.stream().filter(disk -> disk.getState() == HardwareState.UNAVAILABLE).map(AmbryDisk::getMountPath) .collect(Collectors.toSet()); if (!failedDiskMountPath.isEmpty()) { failedDisks.put(instanceName, failedDiskMountPath); diff --git a/ambry-frontend/src/main/java/com/github/ambry/frontend/GetResourceInfoHandler.java b/ambry-frontend/src/main/java/com/github/ambry/frontend/GetResourceInfoHandler.java index 8e867929a9..3d1babc574 100644 --- a/ambry-frontend/src/main/java/com/github/ambry/frontend/GetResourceInfoHandler.java +++ b/ambry-frontend/src/main/java/com/github/ambry/frontend/GetResourceInfoHandler.java @@ -135,10 +135,13 @@ private Callback securityPostProcessRequestCallback() { private List queryResourceInfo() throws RestServiceException { String partitionName = RestUtils.getHeader(restRequest.getArgs(), RestUtils.Headers.PARTITION, false); + String hostname = RestUtils.getHeader(restRequest.getArgs(), RestUtils.Headers.HOSTNAME, false); String resourceName = RestUtils.getHeader(restRequest.getArgs(), RestUtils.Headers.RESOURCE, false); HelixClusterManager.ResourceIdentifier resourceIdentifier; if (partitionName != null) { resourceIdentifier = new HelixClusterManager.PartitionIdIdentifier(partitionName); + } else if (hostname != null) { + resourceIdentifier = new HelixClusterManager.HostnameIdentifier(hostname); } else if (resourceName != null) { resourceIdentifier = new HelixClusterManager.ResourceNameIdentifier(resourceName); } else {