TEZ-4347: Add some diagnostic endpoints to TezAM's WebUIService

tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java

@@ -1980,6 +1980,18 @@ public TezConfiguration(boolean loadDefaults) {
       + "tez-ui.webservice.enable";
   public static final boolean TEZ_AM_WEBSERVICE_ENABLE_DEFAULT = true;
 
+  /**
+   * String value. Range of ports that the AM can use for the WebUIService. Leave blank
+   * to use all possible ports. Expert level setting. It's hadoop standard range configuration.
+   * For example 50000-50050,50100-50200
+   */
+  @ConfigurationScope(Scope.AM)
+  @ConfigurationProperty(type="boolean")
+  public static final String TEZ_AM_WEBSERVICE_PORT_RANGE = TEZ_AM_PREFIX
+      + "tez-ui.webservice.port-range";
+
+  public static final String TEZ_AM_WEBSERVICE_PORT_RANGE_DEFAULT = "50000-50050";
+
   // TODO only validate property here, value can also be validated if necessary
   public static void validateProperty(String property, Scope usedScope) {
     Scope validScope = PropertyScope.get(property);

tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClient.java

@@ -140,4 +140,13 @@ public DAGStatus waitForCompletion(long timeMs) throws IOException, TezException
   public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set<StatusGetOpts> statusGetOpts)
       throws IOException, TezException, InterruptedException;
 
+  /**
+   * Returns the Tez AM's web ui address if any.
+   *
+   * @return The http web UI address
+   * @throws IOException
+   * @throws TezException
+   */
+  public abstract String getWebUIAddress() throws IOException, TezException;
+
 }

tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientImpl.java

@@ -30,10 +30,10 @@
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.tez.common.CachedEntity;
 import org.apache.tez.common.Preconditions;
-
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -670,6 +670,11 @@ public DAGClientInternal getRealClient() {
     return realClient;
   }
 
+  @Override
+  public String getWebUIAddress() throws IOException, TezException {
+    return realClient.getWebUIAddress();
+  }
+
   private double getProgress(Progress progress) {
     return (progress.getTotalTaskCount() == 0 ? 0.0 : (double) (progress.getSucceededTaskCount())
         / progress.getTotalTaskCount());

tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientInternal.java

@@ -125,4 +125,6 @@ public abstract VertexStatus getVertexStatus(String vertexName,
    */
   public abstract DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set<StatusGetOpts> statusGetOpts)
       throws IOException, TezException, InterruptedException;
+
+  public abstract String getWebUIAddress() throws IOException, TezException;
 }

tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java

@@ -523,4 +523,9 @@ public DAGStatus getDAGStatus(@Nullable Set<StatusGetOpts> statusOptions,
     return getDAGStatus(statusOptions);
   }
 
+  @Override
+  public String getWebUIAddress() throws IOException, TezException {
+    throw new TezException("DAGClientTimelineImpl.getWebUIAddress is not supported");
+  }
+
 }

tez-api/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientRPCImpl.java

@@ -48,6 +48,7 @@
 import org.apache.tez.dag.api.client.VertexStatus;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetWebUIAddressRequestProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TryKillDAGRequestProto;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -303,4 +304,15 @@ public DAGStatus waitForCompletionWithStatusUpdates(@Nullable Set<StatusGetOpts>
     throw new TezException("not supported");
   }
 
+  @Override
+  public String getWebUIAddress() throws IOException, TezException {
+    LOG.debug("getWebUIAddress via AM for app: {} dag: {}", appId, dagId);
+    GetWebUIAddressRequestProto.Builder requestProtoBuilder = GetWebUIAddressRequestProto.newBuilder();
+    try {
+      return proxy.getWebUIAddress(null, requestProtoBuilder.build()).getWebUiAddress();
+    } catch (ServiceException e) {
+      RPCUtil.unwrapAndThrowException(e);
+      throw new TezException(e);
+    }
+  }
 }

tez-api/src/main/proto/DAGClientAMProtocol.proto

@@ -90,6 +90,13 @@ message GetAMStatusResponseProto {
   required TezAppMasterStatusProto status = 1;
 }
 
+message GetWebUIAddressRequestProto {
+}
+
+message GetWebUIAddressResponseProto {
+  required string web_ui_address = 1;
+}
+
 service DAGClientAMProtocol {
   rpc getAllDAGs (GetAllDAGsRequestProto) returns (GetAllDAGsResponseProto);
   rpc getDAGStatus (GetDAGStatusRequestProto) returns (GetDAGStatusResponseProto);
@@ -98,4 +105,5 @@ service DAGClientAMProtocol {
   rpc submitDAG (SubmitDAGRequestProto) returns (SubmitDAGResponseProto);
   rpc shutdownSession (ShutdownSessionRequestProto) returns (ShutdownSessionResponseProto);
   rpc getAMStatus (GetAMStatusRequestProto) returns (GetAMStatusResponseProto);
+  rpc getWebUIAddress (GetWebUIAddressRequestProto) returns (GetWebUIAddressResponseProto);
 }

tez-common/src/main/java/org/apache/tez/common/web/AbstractServletToControllerAdapter.java

@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.web;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Enumeration;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.yarn.webapp.Controller;
+
+/**
+ * AbstractServletToControllerAdapter is a common ancestor for classes
+ * that wish to adapt servlets to yarn webapp controllers.
+ * The adapter is responsible for:
+ * 1. creating a servlet instance
+ * 2. creating a dummy ServletConfig
+ * 3. delegating calls to the servlet instance's doGet method
+ */
+public abstract class AbstractServletToControllerAdapter extends Controller {
+  private AtomicBoolean initialized = new AtomicBoolean(false);
+  protected HttpServlet servlet;
+
+  @Override
+  public void index() {
+    if (initialized.compareAndSet(false, true)) {
+      initServlet();
+    }
+    try {
+      /*
+       * This reflection workaround is needed because HttpServlet.doGet is protected
+       * (even if subclasses have it public).
+       */
+      Method doGetMethod =
+          this.servlet.getClass().getMethod("doGet", HttpServletRequest.class, HttpServletResponse.class);
+      doGetMethod.setAccessible(true);
+      doGetMethod.invoke(this.servlet, request(), response());
+    } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
+        | SecurityException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Creates a dummy servlet config which is suitable for initializing a servlet instance.
+   * @param servletName
+   * @return a ServletConfig instance initialized with a ServletContext
+   */
+  private ServletConfig getDummyServletConfig(String servletName) {
+    return new ServletConfig() {
+
+      @Override
+      public String getServletName() {
+        return servletName;
+      }
+
+      @Override
+      public ServletContext getServletContext() {
+        return request().getServletContext();
+      }
+
+      @Override
+      public Enumeration<String> getInitParameterNames() {
+        return null;
+      }
+
+      @Override
+      public String getInitParameter(String name) {
+        return null;
+      }
+    };
+  }
+
+  private void initServlet() {
+    try {
+      servlet.init(getDummyServletConfig(this.servlet.getClass().getSimpleName()));
+    } catch (ServletException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

tez-common/src/main/java/org/apache/tez/common/web/ServletToControllerAdapters.java

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.common.web;
+
+import javax.servlet.ServletException;
+
+import org.apache.hadoop.conf.ConfServlet;
+import org.apache.hadoop.http.HttpServer2.StackServlet;
+import org.apache.hadoop.jmx.JMXJsonServlet;
+
+public class ServletToControllerAdapters {
+  public static class JMXJsonServletController extends AbstractServletToControllerAdapter {
+    public JMXJsonServletController() throws ServletException {
+      this.servlet = new JMXJsonServlet();
+    }
+  }
+
+  public static class ConfServletController extends AbstractServletToControllerAdapter {
+    public ConfServletController() throws ServletException {
+      this.servlet = new ConfServlet();
+    }
+  }
+
+  public static class StackServletController extends AbstractServletToControllerAdapter {
+    public StackServletController() throws ServletException {
+      this.servlet = new StackServlet();
+    }
+  }
+}

tez-common/src/main/java/org/apache/tez/common/web/package-info.java

@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@Private
+package org.apache.tez.common.web;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;

tez-dag/src/main/java/org/apache/tez/dag/api/client/DAGClientHandler.java

@@ -186,4 +186,7 @@ public long getLastHeartbeatTime() {
     return lastHeartbeatTime.get();
   }
 
+  public String getWebUIAddress() {
+    return dagAppMaster.getWebUIAddress();
+  }
 }

tez-dag/src/main/java/org/apache/tez/dag/api/client/rpc/DAGClientAMProtocolBlockingPBServerImpl.java

@@ -45,6 +45,8 @@
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusResponseProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusRequestProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetVertexStatusResponseProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetWebUIAddressRequestProto;
+import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetWebUIAddressResponseProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.ShutdownSessionRequestProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.ShutdownSessionResponseProto;
 import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto;
@@ -226,4 +228,10 @@ public GetAMStatusResponseProto getAMStatus(RpcController controller,
     }
   }
 
+  @Override
+  public GetWebUIAddressResponseProto getWebUIAddress(RpcController controller, GetWebUIAddressRequestProto request)
+      throws ServiceException {
+    String address = real.getWebUIAddress();
+    return GetWebUIAddressResponseProto.newBuilder().setWebUiAddress(address).build();
+  }
 }

tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java

@@ -2618,6 +2618,10 @@ private boolean enableWebUIService() {
         TezConfiguration.TEZ_AM_WEBSERVICE_ENABLE_DEFAULT);
   }
 
+  public String getWebUIAddress() {
+    return webUIService == null ? null : webUIService.getBaseUrl();
+  }
+
   @VisibleForTesting
   static void parseAllPlugins(
       List<NamedEntityDescriptor> taskSchedulerDescriptors, BiMap<String, Integer> taskSchedulerPluginMap,