From 638c1e411ed0511b12b5f0eb544cfa8e5c09fec8 Mon Sep 17 00:00:00 2001 From: Gul Jain Date: Wed, 10 Jul 2024 08:59:15 +0530 Subject: [PATCH] Cherry Pick : Adding total time statics, refactoring event bus and adding bifurcation of status metrics (#1211) * Adding total time metrics (#1182) * Refactoring Event Bus (#1185) * Adding Bifurcation of status code (#1186) * Removing GCS_API_SERVER_NOT_IMPLEMENTED_ERROR_COUNT * Addressing comments --- .../fs/gcs/GhfsGlobalStorageStatistics.java | 221 ++++++++---------- .../hadoop/fs/gcs/GhfsInstrumentation.java | 3 + .../cloud/hadoop/fs/gcs/GhfsStatistic.java | 12 +- .../GoogleCloudStorageEventSubscriber.java | 180 ++++++++++++++ .../hadoop/fs/gcs/GoogleHadoopFileSystem.java | 3 +- .../gcs/GoogleCloudStorageStatisticsTest.java | 158 +++++++++++-- ...GoogleHadoopFileSystemIntegrationTest.java | 18 -- .../fs/gcs/GoogleHadoopFileSystemTest.java | 14 ++ .../GoogleCloudStorageFileSystemImpl.java | 1 - .../gcsio/GoogleCloudStorageStatistics.java | 76 +++++- .../cloud/hadoop/gcsio/StatisticTypeEnum.java | 7 +- .../util/GoogleCloudStorageEventBus.java | 12 - 12 files changed, 503 insertions(+), 202 deletions(-) create mode 100644 gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java index 1e2338cade..55b58282d9 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java @@ -17,24 +17,31 @@ package com.google.cloud.hadoop.fs.gcs; import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.EXCEPTION_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_RATE_LIMIT_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_SIDE_ERROR_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_REQUEST_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_SERVER_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_BAD_REQUEST_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_GONE_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_NOT_FOUND_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_PRECONDITION_FAILED_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_RATE_LIMIT_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_REQUESTED_RANGE_NOT_SATISFIABLE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_REQUEST_TIMEOUT_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_UNAUTHORIZED_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_REQUEST_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_BAD_GATEWAY_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_INTERNAL_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_TIMEOUT_COUNT; import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_DURATION; import static com.google.common.base.Preconditions.checkArgument; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.client.http.HttpResponseException; import com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics; -import com.google.cloud.hadoop.util.GcsRequestExecutionEvent; -import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus.StatisticsType; +import com.google.cloud.hadoop.gcsio.StatisticTypeEnum; import com.google.cloud.hadoop.util.ITraceFactory; import com.google.cloud.hadoop.util.ITraceOperation; import com.google.common.base.Stopwatch; -import com.google.common.eventbus.Subscribe; import com.google.common.flogger.GoogleLogger; -import io.grpc.Status; +import com.google.common.util.concurrent.AtomicDouble; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -70,6 +77,7 @@ public class GhfsGlobalStorageStatistics extends StorageStatistics { private final Map minimums = new HashMap<>(); private final Map maximums = new HashMap<>(); private final Map means = new HashMap<>(); + private final Map total = new HashMap<>(); public GhfsGlobalStorageStatistics() { super(NAME); @@ -83,10 +91,14 @@ public GhfsGlobalStorageStatistics() { String symbol = opType.getSymbol(); opsCount.put(symbol, new AtomicLong(0)); - if (opType.getType() == TYPE_DURATION) { + if (opType.getType() == StatisticTypeEnum.TYPE_DURATION + || opType.getType() == StatisticTypeEnum.TYPE_DURATION_TOTAL) { minimums.put(getMinKey(symbol), null); maximums.put(getMaxKey(symbol), new AtomicLong(0)); means.put(getMeanKey(symbol), new MeanStatistic()); + if (opType.getType() == StatisticTypeEnum.TYPE_DURATION_TOTAL) { + total.put(getTimeKey(symbol), new AtomicDouble(0.0)); + } } } } @@ -141,8 +153,9 @@ void incrementCounter(GoogleCloudStorageStatistics op, long count) { @Override public void reset() { - resetMetrics(opsCount); - resetMetrics(maximums); + resetLongMetrics(opsCount); + resetLongMetrics(maximums); + resetDoubleMetrics(total); for (String ms : means.keySet()) { means.get(ms).reset(); @@ -153,12 +166,18 @@ public void reset() { } } - private void resetMetrics(Map metrics) { + private void resetLongMetrics(Map metrics) { for (AtomicLong value : metrics.values()) { value.set(0); } } + private void resetDoubleMetrics(Map metrics) { + for (AtomicDouble value : metrics.values()) { + value.set(0.0); + } + } + void updateStats(GhfsStatistic statistic, long durationMs, Object context) { checkArgument( statistic.getType() == TYPE_DURATION, @@ -175,6 +194,18 @@ private void addMeanStatistic(GhfsStatistic statistic, long totalDurationMs, int } } + protected void addTotalTimeStatistic(String statistic) { + assert (statistic.contains("_duration")); + String parentCounterKey = statistic.replace("_duration", ""); + String parentMeanKey = getMeanKey(parentCounterKey); + + assert (means.containsKey(parentMeanKey) && opsCount.containsKey(parentCounterKey)); + double meanValue = means.get(parentMeanKey).getValue(); + long operationValue = opsCount.get(parentCounterKey).get(); + + total.get(statistic).set(1.0 * meanValue * operationValue); + } + void updateStats( GhfsStatistic statistic, long minLatency, @@ -219,148 +250,68 @@ private void updateMinMaxStats( } } - /** - * Updating the required gcs specific statistics based on httpresponse. - * - * @param statusCode - */ - private void updateGcsIOSpecificStatistics(int statusCode) { - - if (statusCode >= 400 && statusCode < 500) { - incrementGcsClientSideCounter(); - - if (statusCode == 429) { - incrementRateLimitingCounter(); - } - } + void incrementGcsExceptionCount() { + increment(EXCEPTION_COUNT); + } - if (statusCode >= 500 && statusCode < 600) { - incrementGcsServerSideCounter(); - } + void incrementGcsTotalRequestCount() { + increment(GCS_API_REQUEST_COUNT); } - private int grpcToHttpStatusCodeMapping(Status grpcStatusCode) { - // using code.proto as reference - // https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto - switch (grpcStatusCode.getCode()) { - case OK: - return 200; - case CANCELLED: - return 499; - case INVALID_ARGUMENT: - case FAILED_PRECONDITION: - case OUT_OF_RANGE: - return 400; - case DEADLINE_EXCEEDED: - return 504; - case NOT_FOUND: - return 404; - case ALREADY_EXISTS: - case ABORTED: - return 409; - case PERMISSION_DENIED: - return 403; - case RESOURCE_EXHAUSTED: - return 429; - case UNIMPLEMENTED: - return 501; - case UNAVAILABLE: - return 503; - case UNAUTHENTICATED: - return 401; - case UNKNOWN: - case INTERNAL: - case DATA_LOSS: - default: - return 500; - } + void incrementRateLimitingCounter() { + increment(GCS_API_CLIENT_RATE_LIMIT_COUNT); } - /** - * Updating the required gcs specific statistics based on HttpResponseException. - * - * @param responseException contains statusCode based on which metrics are updated - */ - @Subscribe - private void subscriberOnHttpResponseException(@Nonnull HttpResponseException responseException) { - updateGcsIOSpecificStatistics(responseException.getStatusCode()); + void incrementGcsClientSideCounter() { + increment(GCS_API_CLIENT_SIDE_ERROR_COUNT); } - /** - * Updating the required gcs specific statistics based on GoogleJsonResponseException. - * - * @param responseException contains statusCode based on which metrics are updated - */ - @Subscribe - private void subscriberOnGoogleJsonResponseException( - @Nonnull GoogleJsonResponseException responseException) { - updateGcsIOSpecificStatistics(responseException.getStatusCode()); + void incrementGcsServerSideCounter() { + increment(GCS_API_SERVER_SIDE_ERROR_COUNT); } - /** - * Updating the required gcs specific statistics based on HttpResponse. - * - * @param responseStatus responseStatus status code from HTTP response - */ - @Subscribe - private void subscriberOnHttpResponseStatus(@Nonnull Integer responseStatus) { - updateGcsIOSpecificStatistics(responseStatus); + void incrementGcsClientBadRequestCount() { + increment(GCS_API_CLIENT_BAD_REQUEST_COUNT); } - @Subscribe - private void subscriberOnGcsRequest(@Nonnull GcsRequestExecutionEvent event) { - incrementGcsTotalRequestCount(); + void incrementGcsClientUnauthorizedResponseCount() { + increment(GCS_API_CLIENT_UNAUTHORIZED_RESPONSE_COUNT); } - @Subscribe - private void subscriberOnGrpcStatus(@Nonnull Status status) { - updateGcsIOSpecificStatistics(grpcToHttpStatusCodeMapping(status)); + void incrementGcsClientNotFoundResponseCount() { + increment(GCS_API_CLIENT_NOT_FOUND_RESPONSE_COUNT); } - /** - * Updating the EXCEPTION_COUNT - * - * @param exception - */ - @Subscribe - private void subscriberOnException(IOException exception) { - incrementGcsExceptionCount(); + void incrementGcsClientRequestTimeoutCount() { + increment(GCS_API_CLIENT_REQUEST_TIMEOUT_COUNT); } - /** - * Updating the corresponding statistics - * - * @param strType - */ - @Subscribe - private void subscriberOnStatisticsType(StatisticsType strType) { - if (strType == StatisticsType.DIRECTORIES_DELETED) { - incrementDirectoriesDeleted(); - } + void incrementGcsClientGoneResponseCount() { + increment(GCS_API_CLIENT_GONE_RESPONSE_COUNT); } - private void incrementDirectoriesDeleted() { - increment(GhfsStatistic.DIRECTORIES_DELETED); + void incrementGcsClientPreconditionFailedResponseCount() { + increment(GCS_API_CLIENT_PRECONDITION_FAILED_RESPONSE_COUNT); } - private void incrementGcsExceptionCount() { - increment(EXCEPTION_COUNT); + void incrementGcsClientRequestedRangeNotSatisfiableCount() { + increment(GCS_API_CLIENT_REQUESTED_RANGE_NOT_SATISFIABLE_COUNT); } - private void incrementGcsTotalRequestCount() { - increment(GCS_REQUEST_COUNT); + void incrementGcsServerInternalErrorCount() { + increment(GCS_API_SERVER_INTERNAL_ERROR_COUNT); } - private void incrementRateLimitingCounter() { - increment(GCS_CLIENT_RATE_LIMIT_COUNT); + void incrementGcsServerBadGatewayCount() { + increment(GCS_API_SERVER_BAD_GATEWAY_COUNT); } - private void incrementGcsClientSideCounter() { - increment(GCS_CLIENT_SIDE_ERROR_COUNT); + void incrementGcsServerServiceUnavailableCount() { + increment(GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT); } - private void incrementGcsServerSideCounter() { - increment(GCS_SERVER_SIDE_ERROR_COUNT); + void incrementGcsServerTimeoutCount() { + increment(GCS_API_SERVER_TIMEOUT_COUNT); } void streamReadBytes(int bytesRead) { @@ -401,6 +352,11 @@ private Iterator getMetricNames() { metrics.addAll(minimums.keySet()); metrics.addAll(maximums.keySet()); metrics.addAll(means.keySet()); + for (String statistic : total.keySet()) { + addTotalTimeStatistic(statistic); + } + + metrics.addAll(total.keySet()); return metrics.iterator(); } @@ -443,6 +399,10 @@ private long getValue(String key) { return Math.round(means.get(key).getValue()); } + if (total.containsKey(key)) { + return total.get(key).longValue(); + } + return 0L; } @@ -461,7 +421,8 @@ public boolean isTracked(String key) { return opsCount.containsKey(key) || maximums.containsKey(key) || minimums.containsKey(key) - || means.containsKey(key); + || means.containsKey(key) + || total.containsKey(key); } /** @@ -491,6 +452,10 @@ private String getMeanKey(String symbol) { return symbol + "_mean"; } + private String getTimeKey(String symbol) { + return symbol + "_duration"; + } + /** * To get the maximum value which is stored with MAXIMUM extension * diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java index 487cb3cc0f..07c9a36196 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java @@ -777,6 +777,9 @@ private IOStatisticsStoreBuilder createStoreBuilder() { } else if (stat.getType() == StatisticTypeEnum.TYPE_DURATION) { duration(stat); storeBuilder.withDurationTracking(stat.getSymbol()); + } else if (stat.getType() == StatisticTypeEnum.TYPE_DURATION_TOTAL) { + duration(stat); + storeBuilder.withDurationTracking(stat.getSymbol()); } }); diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java index 6093b3db10..506fb5adf8 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java @@ -18,6 +18,7 @@ import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_COUNTER; import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_DURATION; +import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_DURATION_TOTAL; import com.google.cloud.hadoop.gcsio.StatisticTypeEnum; import com.google.common.collect.ImmutableMap; @@ -55,11 +56,6 @@ public enum GhfsStatistic { "Total number of directories created through the object store.", TYPE_COUNTER), - DIRECTORIES_DELETED( - "directories_deleted", - "Total number of directories deleted through the object store.", - TYPE_COUNTER), - FILES_CREATED( "files_created", "Total number of files created through the object store.", TYPE_COUNTER), FILES_DELETED( @@ -106,9 +102,6 @@ public enum GhfsStatistic { "Calls of read stream close()", TYPE_DURATION), - STREAM_READ_OPERATIONS( - StreamStatisticNames.STREAM_READ_OPERATIONS, "Calls of read()", TYPE_DURATION), - STREAM_READ_VECTORED_OPERATIONS( StreamStatisticNames.STREAM_READ_VECTORED_OPERATIONS, "Calls of readVectored()", @@ -126,6 +119,7 @@ public enum GhfsStatistic { StreamStatisticNames.STREAM_READ_VECTORED_INCOMING_RANGES, "size of fileRanges requested in readVectoredRequest", TYPE_COUNTER), + STREAM_READ_OPERATIONS("stream_read_operations", "Calls of read()", TYPE_DURATION_TOTAL), STREAM_READ_VECTORED_READ_COMBINED_RANGES( StreamStatisticNames.STREAM_READ_VECTORED_COMBINED_RANGES, @@ -161,7 +155,7 @@ public enum GhfsStatistic { TYPE_COUNTER), STREAM_WRITE_CLOSE_OPERATIONS( "stream_write_close_operations", "Calls of write stream close()", TYPE_DURATION), - STREAM_WRITE_OPERATIONS("stream_write_operations", "Calls of write()", TYPE_DURATION), + STREAM_WRITE_OPERATIONS("stream_write_operations", "Calls of write()", TYPE_DURATION_TOTAL), /** The XAttr API statistics */ INVOCATION_XATTR_GET_MAP( diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java new file mode 100644 index 0000000000..55aafcf21e --- /dev/null +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java @@ -0,0 +1,180 @@ +/* + * Copyright 2024 Google LLC. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.fs.gcs; + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpResponseException; +import com.google.cloud.hadoop.util.GcsRequestExecutionEvent; +import com.google.common.eventbus.Subscribe; +import io.grpc.Status; +import java.io.IOException; +import javax.annotation.Nonnull; + +/* Stores the subscriber methods corresponding to GoogleCloudStorageEventBus */ +public class GoogleCloudStorageEventSubscriber { + private static GhfsGlobalStorageStatistics storageStatistics; + + public GoogleCloudStorageEventSubscriber(GhfsGlobalStorageStatistics storageStatistics) { + this.storageStatistics = storageStatistics; + } + + /** + * Updating the required gcs specific statistics based on HttpResponseException. + * + * @param responseException contains statusCode based on which metrics are updated + */ + @Subscribe + private void subscriberOnHttpResponseException(@Nonnull HttpResponseException responseException) { + updateGcsIOSpecificStatistics(responseException.getStatusCode()); + } + + /** + * Updating the required gcs specific statistics based on GoogleJsonResponseException. + * + * @param responseException contains statusCode based on which metrics are updated + */ + @Subscribe + private void subscriberOnGoogleJsonResponseException( + @Nonnull GoogleJsonResponseException responseException) { + updateGcsIOSpecificStatistics(responseException.getStatusCode()); + } + + /** + * Updating the required gcs specific statistics based on HttpResponse. + * + * @param responseStatus status code from HTTP response + */ + @Subscribe + private void subscriberOnHttpResponseStatus(@Nonnull Integer responseStatus) { + updateGcsIOSpecificStatistics(responseStatus); + incrementStatusCode(responseStatus); + } + + @Subscribe + private void subscriberOnGcsRequest(@Nonnull GcsRequestExecutionEvent event) { + storageStatistics.incrementGcsTotalRequestCount(); + } + + @Subscribe + private void subscriberOnGrpcStatus(@Nonnull Status status) { + updateGcsIOSpecificStatistics(grpcToHttpStatusCodeMapping(status)); + } + + /** + * Updating the EXCEPTION_COUNT + * + * @param exception + */ + @Subscribe + private void subscriberOnException(IOException exception) { + storageStatistics.incrementGcsExceptionCount(); + } + + /** + * Updating the required gcs specific statistics based on httpresponse. + * + * @param statusCode + */ + protected void updateGcsIOSpecificStatistics(int statusCode) { + + if (statusCode >= 400 && statusCode < 500) { + storageStatistics.incrementGcsClientSideCounter(); + + if (statusCode == 429) { + storageStatistics.incrementRateLimitingCounter(); + } + } + if (statusCode >= 500 && statusCode < 600) { + storageStatistics.incrementGcsServerSideCounter(); + } + } + + private int grpcToHttpStatusCodeMapping(Status grpcStatusCode) { + // using code.proto as reference + // https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto + switch (grpcStatusCode.getCode()) { + case OK: + return 200; + case CANCELLED: + return 499; + case INVALID_ARGUMENT: + case FAILED_PRECONDITION: + case OUT_OF_RANGE: + return 400; + case DEADLINE_EXCEEDED: + return 504; + case NOT_FOUND: + return 404; + case ALREADY_EXISTS: + case ABORTED: + return 409; + case PERMISSION_DENIED: + return 403; + case RESOURCE_EXHAUSTED: + return 429; + case UNIMPLEMENTED: + return 501; + case UNAVAILABLE: + return 503; + case UNAUTHENTICATED: + return 401; + case UNKNOWN: + case INTERNAL: + case DATA_LOSS: + default: + return 500; + } + } + + private void incrementStatusCode(int statusCode) { + switch (statusCode) { + case 400: + storageStatistics.incrementGcsClientBadRequestCount(); + break; + case 401: + storageStatistics.incrementGcsClientUnauthorizedResponseCount(); + break; + case 404: + storageStatistics.incrementGcsClientNotFoundResponseCount(); + break; + case 408: + storageStatistics.incrementGcsClientRequestTimeoutCount(); + break; + case 410: + storageStatistics.incrementGcsClientGoneResponseCount(); + break; + case 412: + storageStatistics.incrementGcsClientPreconditionFailedResponseCount(); + break; + case 416: + storageStatistics.incrementGcsClientRequestedRangeNotSatisfiableCount(); + break; + case 500: + storageStatistics.incrementGcsServerInternalErrorCount(); + break; + case 502: + storageStatistics.incrementGcsServerBadGatewayCount(); + break; + case 503: + storageStatistics.incrementGcsServerServiceUnavailableCount(); + break; + case 504: + storageStatistics.incrementGcsServerTimeoutCount(); + break; + } + } +} diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java index fd5d7504cf..cfe2200d62 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java @@ -249,7 +249,8 @@ public GoogleHadoopFileSystem() { globalStorageStatistics = GhfsGlobalStorageStatistics.DUMMY_INSTANCE; } - GoogleCloudStorageEventBus.register(globalStorageStatistics); + GoogleCloudStorageEventBus.register( + new GoogleCloudStorageEventSubscriber(globalStorageStatistics)); } /** diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java index 7d01b01a0a..f5e02e59b9 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java @@ -15,10 +15,21 @@ package com.google.cloud.hadoop.fs.gcs; import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.EXCEPTION_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_RATE_LIMIT_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_SIDE_ERROR_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_REQUEST_COUNT; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_SERVER_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_BAD_REQUEST_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_GONE_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_NOT_FOUND_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_PRECONDITION_FAILED_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_RATE_LIMIT_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_REQUESTED_RANGE_NOT_SATISFIABLE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_REQUEST_TIMEOUT_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_CLIENT_UNAUTHORIZED_RESPONSE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_REQUEST_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_BAD_GATEWAY_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_INTERNAL_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_TIMEOUT_COUNT; import static com.google.common.truth.Truth.assertThat; import com.google.cloud.hadoop.util.GcsRequestExecutionEvent; @@ -36,17 +47,17 @@ @RunWith(JUnit4.class) public class GoogleCloudStorageStatisticsTest { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); - private GhfsGlobalStorageStatistics subscriber = new GhfsGlobalStorageStatistics(); + private GhfsGlobalStorageStatistics storageStatistics = new GhfsGlobalStorageStatistics(); + protected GoogleCloudStorageEventSubscriber subscriber = + new GoogleCloudStorageEventSubscriber(storageStatistics); @Before public void setUp() throws Exception { - GoogleCloudStorageEventBus.register(subscriber); } @After public void cleanup() throws Exception { - GoogleCloudStorageEventBus.unregister(subscriber); } @@ -55,7 +66,7 @@ private void verifyStatistics(GhfsGlobalStorageStatistics expectedStats) { boolean metricsVerified = true; while (statsIterator.hasNext()) { LongStatistic stats = statsIterator.next(); - Long value = subscriber.getLong(stats.getName()); + Long value = storageStatistics.getLong(stats.getName()); if (stats.getValue() != value) { logger.atWarning().log( "Metric values not matching. for: %s, expected: %d, got: %d", @@ -71,7 +82,7 @@ private void verifyStatistics(GhfsGlobalStorageStatistics expectedStats) { public void gcs_requestCounter() throws Exception { GoogleCloudStorageEventBus.onGcsRequest(new GcsRequestExecutionEvent()); GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); - verifyCounterStats.incrementCounter(GCS_REQUEST_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_REQUEST_COUNT, 1); verifyStatistics(verifyCounterStats); } @@ -80,11 +91,11 @@ public void gcs_rateLimitCounter() { // verify for http event i.e. via Apiary GoogleCloudStorageEventBus.postOnHttpResponseStatus(429); GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); - verifyCounterStats.incrementCounter(GCS_CLIENT_RATE_LIMIT_COUNT, 1); - verifyCounterStats.incrementCounter(GCS_CLIENT_SIDE_ERROR_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_RATE_LIMIT_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); verifyStatistics(verifyCounterStats); - subscriber.reset(); + storageStatistics.reset(); // verify for gRPC event i.e. via java-storage GoogleCloudStorageEventBus.onGrpcStatus(Status.RESOURCE_EXHAUSTED); @@ -95,13 +106,17 @@ public void gcs_rateLimitCounter() { public void gcs_clientSideErrorCounter() { GoogleCloudStorageEventBus.postOnHttpResponseStatus(404); GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); - verifyCounterStats.incrementCounter(GCS_CLIENT_SIDE_ERROR_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_NOT_FOUND_RESPONSE_COUNT, 1); verifyStatistics(verifyCounterStats); + } - subscriber.reset(); - - // verify for gRPC event i.e. via java-storage + @Test + public void gcs_grpcCancelledStatusCounter() { GoogleCloudStorageEventBus.onGrpcStatus(Status.CANCELLED); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + // verify for gRPC event i.e. via java-storage verifyStatistics(verifyCounterStats); } @@ -109,21 +124,124 @@ public void gcs_clientSideErrorCounter() { public void gcs_serverSideErrorCounter() { GoogleCloudStorageEventBus.postOnHttpResponseStatus(503); GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); - verifyCounterStats.incrementCounter(GCS_SERVER_SIDE_ERROR_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SIDE_ERROR_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT, 1); verifyStatistics(verifyCounterStats); + } - subscriber.reset(); - + @Test + public void gcs_grpcInternalStatusCounter() { // verify for gRPC event i.e. via java-storage GoogleCloudStorageEventBus.onGrpcStatus(Status.INTERNAL); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SIDE_ERROR_COUNT, 1); verifyStatistics(verifyCounterStats); } @Test - public void gcs_ExceptionCounter() { + public void gcs_exceptionCounter() { GoogleCloudStorageEventBus.postOnException(); GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); verifyCounterStats.incrementCounter(EXCEPTION_COUNT, 1); verifyStatistics(verifyCounterStats); } + + @Test + public void gcs_clientBadRequestCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(400); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_BAD_REQUEST_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_clientUnauthorizedResponseCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(401); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_UNAUTHORIZED_RESPONSE_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_clientNotFoundResponseCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(404); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_NOT_FOUND_RESPONSE_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_clientRequestTimeoutCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(408); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_REQUEST_TIMEOUT_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_clientGoneResponseCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(410); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_GONE_RESPONSE_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_clientPreconditionFailedResponseCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(412); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_PRECONDITION_FAILED_RESPONSE_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_clientRequestedRangeNotSatisfiableCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(416); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_REQUESTED_RANGE_NOT_SATISFIABLE_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_CLIENT_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_serverInternalErrorCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(500); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_SERVER_INTERNAL_ERROR_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_serverBadGatewayCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(502); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_SERVER_BAD_GATEWAY_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_serverServiceUnavailableCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(503); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + + @Test + public void gcs_serverTimeoutCount() { + GoogleCloudStorageEventBus.postOnHttpResponseStatus(504); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(GCS_API_SERVER_TIMEOUT_COUNT, 1); + verifyCounterStats.incrementCounter(GCS_API_SERVER_SIDE_ERROR_COUNT, 1); + verifyStatistics(verifyCounterStats); + } } diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java index 6664f4bbb2..0140664a88 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java @@ -20,7 +20,6 @@ import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_GET_REQUEST; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_PATCH_REQUEST; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_PUT_REQUEST; -import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.DIRECTORIES_DELETED; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_CREATED; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_COPY_FROM_LOCAL_FILE; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_CREATE; @@ -539,23 +538,6 @@ public void delete_IOstatistics() throws IOException { (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 1); } - @Test - public void statistics_check_directories_deleted() throws IOException { - - GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); - - StorageStatistics GlobalStorageStats = TestUtils.getStorageStatistics(); - Path testRoot = new Path("/directory1/"); - myGhfs.mkdirs(testRoot); - FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); - fout.writeBytes("Test Content"); - fout.close(); - - assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); - TestUtils.verifyCounter( - (GhfsGlobalStorageStatistics) GlobalStorageStats, DIRECTORIES_DELETED, 1); - } - @Test public void statistics_check_get_list_status_result_size() throws IOException { diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTest.java index c4b1671848..75b5e86131 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTest.java @@ -16,6 +16,8 @@ package com.google.cloud.hadoop.fs.gcs; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_OPERATIONS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_CLIENT_TYPE; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemTestHelper.createInMemoryGoogleHadoopFileSystem; import static com.google.cloud.hadoop.gcsio.testing.InMemoryGoogleCloudStorage.getInMemoryGoogleCloudStorageOptions; @@ -308,6 +310,18 @@ public void testFileOpenWithStatusInvalidType() throws Exception { } } + @Test + public void testTotalTimeStatistics() throws IOException { + GhfsGlobalStorageStatistics stats = new GhfsGlobalStorageStatistics(); + stats.updateStats(STREAM_READ_OPERATIONS, 10, 100, 200, 10, new Object()); + stats.addTotalTimeStatistic(STREAM_READ_OPERATIONS.getSymbol() + "_duration"); + assertThat(stats.getLong(STREAM_READ_OPERATIONS.getSymbol() + "_duration")).isEqualTo(200); + + stats.updateStats(STREAM_WRITE_OPERATIONS, 10, 100, 200, 10, new Object()); + stats.addTotalTimeStatistic(STREAM_WRITE_OPERATIONS.getSymbol() + "_duration"); + assertThat(stats.getLong(STREAM_WRITE_OPERATIONS.getSymbol() + "_duration")).isEqualTo(200); + } + // ----------------------------------------------------------------- // Inherited tests that we suppress because their behavior differs // from the base class. diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java index aca7cdb6e7..c98b48e85e 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java @@ -342,7 +342,6 @@ public void delete(URI path, boolean recursive) throws IOException { GoogleCloudStorageEventBus.postOnException(); throw new DirectoryNotEmptyException("Cannot delete a non-empty directory."); } - GoogleCloudStorageEventBus.postOnStatisticsType(); } else { itemsToDelete = new ArrayList<>(); } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java index 48d74d9fb1..9e234bd7bb 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java @@ -23,27 +23,79 @@ import com.google.common.collect.Maps; import java.util.EnumSet; -/** Statistics which are collected in GCS Connector */ +/** Statistics which are collected in GCS Connector. */ public enum GoogleCloudStorageStatistics { - - /** GCS connector specific statistics */ - GCS_REQUEST_COUNT( - "gcs_total_request_count", "Counts the total number of gcs requests made", TYPE_COUNTER), - EXCEPTION_COUNT("exception_count", "Counts the number of exceptions encountered", TYPE_COUNTER), - GCS_CLIENT_SIDE_ERROR_COUNT( - "gcs_client_side_error_count", + /** Status Code Counters for JSON Path */ + GCS_API_REQUEST_COUNT( + "gcs_api_total_request_count", "Counts the total number of gcs requests made", TYPE_COUNTER), + + GCS_API_CLIENT_SIDE_ERROR_COUNT( + "gcs_api_client_side_error_count", "Counts the occurrence of client side error status code", TYPE_COUNTER), - GCS_SERVER_SIDE_ERROR_COUNT( - "gcs_server_side_error_count", + GCS_API_SERVER_SIDE_ERROR_COUNT( + "gcs_api_server_side_error_count", "Counts the occurrence of server side error status code", TYPE_COUNTER), - GCS_CLIENT_RATE_LIMIT_COUNT( - "gcs_client_rate_limit_error_count", "Counts the occurence of 429 status code", TYPE_COUNTER); + GCS_API_CLIENT_RATE_LIMIT_COUNT( + "gcs_api_client_rate_limit_error_count", "Counts the occurence of rate limit", TYPE_COUNTER), + + GCS_API_CLIENT_BAD_REQUEST_COUNT( + "gcs_api_client_bad_request_count", "Counts the occurence of 400 status code", TYPE_COUNTER), + + GCS_API_CLIENT_UNAUTHORIZED_RESPONSE_COUNT( + "gcs_api_client_unauthorized_response_count", + "Counts the occurence of 401 status code", + TYPE_COUNTER), + + GCS_API_CLIENT_NOT_FOUND_RESPONSE_COUNT( + "gcs_api_client_non_found_response_count", + "Counts the occurence of 404 status code", + TYPE_COUNTER), + + GCS_API_CLIENT_REQUEST_TIMEOUT_COUNT( + "gcs_api_client_request_timeout_count", + "Counts the occurence of 408 status code", + TYPE_COUNTER), + + GCS_API_CLIENT_GONE_RESPONSE_COUNT( + "gcs_api_client_gone_response_count", + "Counts the occurence of 410 status code", + TYPE_COUNTER), + + GCS_API_CLIENT_PRECONDITION_FAILED_RESPONSE_COUNT( + "gcs_api_client_precondition_failed_response_count", + "Counts the occurence of 412 status code", + TYPE_COUNTER), + + GCS_API_CLIENT_REQUESTED_RANGE_NOT_SATISFIABLE_COUNT( + "gcs_api_client_requested_range_not_statisfiable_count", + "Counts the occurence of 416 status code", + TYPE_COUNTER), + + GCS_API_SERVER_INTERNAL_ERROR_COUNT( + "gcs_api_server_internal_error_count", + "Counts the occurrence of server side 500 error status code", + TYPE_COUNTER), + + GCS_API_SERVER_BAD_GATEWAY_COUNT( + "gcs_api_server_bad_gateway_count", + "Counts the occurrence of server side 502 error status code", + TYPE_COUNTER), + + GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT( + "gcs_api_server_unavailable_count", + "Counts the occurrence of server side 503 error status code", + TYPE_COUNTER), + + GCS_API_SERVER_TIMEOUT_COUNT( + "gcs_api_server_timeout_count", + "Counts the occurrence of server side 504 error status code", + TYPE_COUNTER); public static final ImmutableSet VALUES = ImmutableSet.copyOf(EnumSet.allOf(GoogleCloudStorageStatistics.class)); diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/StatisticTypeEnum.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/StatisticTypeEnum.java index 411125caa3..a2fe8aba6c 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/StatisticTypeEnum.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/StatisticTypeEnum.java @@ -26,5 +26,10 @@ public enum StatisticTypeEnum { TYPE_DURATION, /** Gauge. */ - TYPE_GAUGE + TYPE_GAUGE, + + /* Duration. Stores everything stored by TYPE_DURATION and total time taken. + * This is to avoid storing and computing total duration of an operation repeatedly. Instread this can be done at the time of querying metric by multipying mean and count + * */ + TYPE_DURATION_TOTAL } diff --git a/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java b/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java index 59c3ac7590..973b0a7561 100644 --- a/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java +++ b/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java @@ -25,11 +25,6 @@ /** Event Bus class */ public class GoogleCloudStorageEventBus { - /** Translates increment of statistics from API calls into StatisticsType */ - public enum StatisticsType { - DIRECTORIES_DELETED - } - public static void postGcsJsonApiEvent(GcsJsonApiEvent gcsJsonApiEvent) { eventBus.post(gcsJsonApiEvent); } @@ -102,13 +97,6 @@ public static void postOnException() { eventBus.post(exception); } - /** - * Posting StatisticsType to invoke corresponding Subscriber method. Passing an Object as EventBus - * has @ElementTypesAreNonnullByDefault annotation. - */ - public static void postOnStatisticsType() { - eventBus.post(StatisticsType.DIRECTORIES_DELETED); - } /** * Posting grpc Status to invoke the corresponding Subscriber method. *