From 227670b212cc4d9a71e46548466b35b0d9887432 Mon Sep 17 00:00:00 2001 From: Vaibhav Sethi Date: Mon, 6 Jan 2025 13:36:48 +0530 Subject: [PATCH] Add cdap system service for appfabric processor --- .../guice/AppFabricServiceRuntimeModule.java | 6 ++++- .../cdap/app/guice/MonitorHandlerModule.java | 2 +- .../AppFabricProcessorManager.java | 23 +------------------ .../NonHadoopAppFabricProcessorManager.java | 2 +- .../services/AppFabricProcessorService.java | 7 +++++- .../app/services/AppFabricServer.java | 2 +- .../AppFabricProcessorServiceTest.java | 2 +- .../http/handlers/MonitorHandlerTest.java | 4 ++-- .../io/cdap/cdap/common/conf/Constants.java | 3 ++- .../source/http-restful-api/monitor.rst | 3 +++ .../k8s/AppFabricProcessorServiceMain.java | 2 +- .../environment/k8s/AppFabricServiceMain.java | 1 + .../tasks/SupportBundleSystemLogTaskTest.java | 2 +- 13 files changed, 26 insertions(+), 33 deletions(-) diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index 1d5a3b2bfdad..09157604131f 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -451,7 +451,7 @@ protected void configure() { bind(MetricsProvider.class).to(SparkProgramStatusMetricsProvider.class); Multibinder handlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(Constants.AppFabric.HANDLERS_BINDING)); + binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); CommonHandlers.add(handlerBinder); handlerBinder.addBinding().to(ConfigHandler.class); @@ -499,6 +499,10 @@ protected void configure() { for (Class handlerClass : handlerClasses) { handlerBinder.addBinding().to(handlerClass); } + + Multibinder procesorHandlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(AppFabric.PROCESSOR_HANDLERS_BINDING)); + CommonHandlers.add(procesorHandlerBinder); } @Provides diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java index 09aea826a811..b487faa37e15 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/MonitorHandlerModule.java @@ -94,7 +94,7 @@ protected void configure() { }); Multibinder handlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(Constants.AppFabric.HANDLERS_BINDING)); + binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); handlerBinder.addBinding().to(MonitorHandler.class); } diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AppFabricProcessorManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AppFabricProcessorManager.java index d92c403ebf71..e516e63fa0c0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AppFabricProcessorManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/distributed/AppFabricProcessorManager.java @@ -1,5 +1,5 @@ /* - * Copyright © 2015 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -16,34 +16,13 @@ package io.cdap.cdap.internal.app.runtime.distributed; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; import com.google.inject.Inject; -import com.google.inject.name.Named; import io.cdap.cdap.common.conf.CConfiguration; import io.cdap.cdap.common.conf.Constants; import io.cdap.cdap.common.conf.Constants.AppFabric; -import io.cdap.cdap.common.conf.Constants.Service; import io.cdap.cdap.common.twill.AbstractMasterServiceManager; -import io.cdap.cdap.common.twill.MasterServiceManager; -import io.cdap.cdap.common.zookeeper.election.LeaderElectionInfoService; -import io.cdap.cdap.proto.Containers; -import io.cdap.cdap.proto.SystemServiceLiveInfo; -import java.net.InetAddress; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; import org.apache.twill.api.TwillRunner; -import org.apache.twill.api.logging.LogEntry; import org.apache.twill.discovery.DiscoveryServiceClient; -import org.slf4j.ILoggerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * App Fabric Processor Service Management in Distributed Mode. diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/NonHadoopAppFabricProcessorManager.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/NonHadoopAppFabricProcessorManager.java index b8d9de20db59..90f980b837b4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/NonHadoopAppFabricProcessorManager.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/runtime/monitor/NonHadoopAppFabricProcessorManager.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java index 91409353ad7c..978a6424dea9 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricProcessorService.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2024 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -46,6 +46,7 @@ import io.cdap.cdap.security.auth.AuditLogSubscriberService; import io.cdap.cdap.sourcecontrol.RepositoryCleanupService; import io.cdap.cdap.sourcecontrol.operationrunner.SourceControlOperationRunner; +import io.cdap.http.HttpHandler; import io.cdap.http.NettyHttpService; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -89,6 +90,7 @@ public class AppFabricProcessorService extends AbstractIdleService { private final boolean sslEnabled; private CommonNettyHttpServiceFactory commonNettyHttpServiceFactory; private Cancellable cancelHttpService; + private Set handlers; /** * Construct the AppFabricProcessorService with service factory and cConf coming from guice injection. @@ -98,6 +100,7 @@ public AppFabricProcessorService(CConfiguration cConf, SConfiguration sConf, DiscoveryService discoveryService, @Named(Constants.Service.MASTER_SERVICES_BIND_ADDRESS) InetAddress hostname, + @Named(Constants.AppFabric.SERVER_HANDLERS_BINDING) Set handlers, ProgramRuntimeService programRuntimeService, RunRecordCorrectorService runRecordCorrectorService, ProgramRunStatusMonitorService programRunStatusMonitorService, @@ -119,6 +122,7 @@ public AppFabricProcessorService(CConfiguration cConf, ScheduleNotificationSubscriberService scheduleNotificationSubscriberService) { this.hostname = hostname; this.discoveryService = discoveryService; + this.handlers = handlers; this.cConf = cConf; this.sConf = sConf; this.sslEnabled = cConf.getBoolean(Constants.Security.SSL.INTERNAL_ENABLED); @@ -188,6 +192,7 @@ protected void startUp() throws Exception { NettyHttpService.Builder httpServiceBuilder = commonNettyHttpServiceFactory .builder(Constants.Service.APP_FABRIC_HTTP) .setHost(hostname.getCanonicalHostName()) + .setHttpHandlers(handlers) .setConnectionBacklog(cConf.getInt(Constants.AppFabric.BACKLOG_CONNECTIONS, Constants.AppFabric.DEFAULT_BACKLOG)) .setExecThreadPoolSize(cConf.getInt(Constants.AppFabric.EXEC_THREADS, diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java index cf5ea2b49228..f256320708d0 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/services/AppFabricServer.java @@ -109,7 +109,7 @@ public class AppFabricServer extends AbstractIdleService { public AppFabricServer(CConfiguration cConf, SConfiguration sConf, DiscoveryService discoveryService, @Named(Constants.Service.MASTER_SERVICES_BIND_ADDRESS) InetAddress hostname, - @Named(Constants.AppFabric.HANDLERS_BINDING) Set handlers, + @Named(Constants.AppFabric.SERVER_HANDLERS_BINDING) Set handlers, @Nullable MetricsCollectionService metricsCollectionService, ProgramRuntimeService programRuntimeService, ProgramRunStatusMonitorService programRunStatusMonitorService, diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java index 070542a4288e..a147f710fff3 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/AppFabricProcessorServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright © 2014-2018 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java index 9e875adb30ef..f3587dc7360a 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/app/services/http/handlers/MonitorHandlerTest.java @@ -66,7 +66,7 @@ public void testSystemServices() throws Exception { List actual = GSON.fromJson(new String(ByteStreams.toByteArray(urlConn.getInputStream()), Charsets.UTF_8), token); - Assert.assertEquals(9, actual.size()); + Assert.assertEquals(10, actual.size()); urlConn.disconnect(); } @@ -81,7 +81,7 @@ public void testSystemServicesStatus() throws Exception { Map result = GSON.fromJson(new String(ByteStreams.toByteArray(urlConn.getInputStream()), Charsets.UTF_8), token); - Assert.assertEquals(9, result.size()); + Assert.assertEquals(10, result.size()); urlConn.disconnect(); Assert.assertEquals("OK", result.get(Constants.Service.APP_FABRIC_HTTP)); } diff --git a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java index 53dd5b042631..cd5a49601754 100644 --- a/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java +++ b/cdap-common/src/main/java/io/cdap/cdap/common/conf/Constants.java @@ -303,7 +303,8 @@ public static final class AppFabric { /** * Guice named bindings. */ - public static final String HANDLERS_BINDING = "appfabric.http.handler"; + public static final String SERVER_HANDLERS_BINDING = "appfabric.http.handler"; + public static final String PROCESSOR_HANDLERS_BINDING = "appfabric.processor.http.handler"; /** * Defaults. diff --git a/cdap-docs/reference-manual/source/http-restful-api/monitor.rst b/cdap-docs/reference-manual/source/http-restful-api/monitor.rst index 69453af276e8..cd6ac0c40a60 100644 --- a/cdap-docs/reference-manual/source/http-restful-api/monitor.rst +++ b/cdap-docs/reference-manual/source/http-restful-api/monitor.rst @@ -105,6 +105,9 @@ The status of these CDAP system services can be checked: * - ``App Fabric`` - ``appfabric`` - Service that handles application fabric requests + * - ``App Fabric Processor`` + - ``appfabric.processor`` + - Service that handles application lifecycle * - ``Log Saver`` - ``log.saver`` - Service that aggregates all system and application logs diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java index bdca170101d5..feb37b85eb83 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java @@ -1,5 +1,5 @@ /* - * Copyright © 2024 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java index 331f378c8ca2..1e5fb4eef6a8 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java @@ -181,4 +181,5 @@ protected LoggingContext getLoggingContext(EnvironmentOptions options) { Constants.Logging.COMPONENT_NAME, Constants.Service.APP_FABRIC_HTTP); } + } diff --git a/cdap-support-bundle/src/test/java/io/cdap/cdap/support/tasks/SupportBundleSystemLogTaskTest.java b/cdap-support-bundle/src/test/java/io/cdap/cdap/support/tasks/SupportBundleSystemLogTaskTest.java index a020080cdcb6..d03e9a9986be 100644 --- a/cdap-support-bundle/src/test/java/io/cdap/cdap/support/tasks/SupportBundleSystemLogTaskTest.java +++ b/cdap-support-bundle/src/test/java/io/cdap/cdap/support/tasks/SupportBundleSystemLogTaskTest.java @@ -85,7 +85,7 @@ public void testSupportBundleSystemLogTask() throws Exception { List systemLogFiles = DirUtils.listFiles(systemLogFolder, file -> !file.isHidden() && !file.getParentFile().isHidden()); - Assert.assertEquals(9, systemLogFiles.size()); + Assert.assertEquals(10, systemLogFiles.size()); } @Before