From 101dc465cf2b7cd3a69cfe5ebb1c1ddf7c38472c Mon Sep 17 00:00:00 2001 From: Vaibhav Sethi Date: Sun, 12 Jan 2025 18:54:01 +0530 Subject: [PATCH] [CDAP-21096] Parametrize the AppFabricServiceRuntimeModule --- .../DistributedWorkflowProgramRunnerTest.java | 5 +- .../guice/AppFabricServiceRuntimeModule.java | 313 ++++++++++-------- .../system/SystemWorkerTwillRunnable.java | 5 +- .../internal/guice/AppFabricTestModule.java | 3 +- .../data/runtime/main/MasterServiceMain.java | 3 +- .../cdap/data/tools/JobQueueDebugger.java | 5 +- .../k8s/AppFabricProcessorServiceMain.java | 5 +- .../environment/k8s/AppFabricServiceMain.java | 4 +- .../environment/k8s/PreviewServiceMain.java | 4 +- .../java/io/cdap/cdap/StandaloneMain.java | 5 +- .../io/cdap/cdap/SupportBundleTestModule.java | 3 +- .../main/java/io/cdap/cdap/test/TestBase.java | 3 +- 12 files changed, 208 insertions(+), 150 deletions(-) diff --git a/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java b/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java index 9d1e4724c985..b5313cd3406d 100644 --- a/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java +++ b/cdap-app-fabric-tests/src/test/java/io/cdap/cdap/internal/app/runtime/distributed/DistributedWorkflowProgramRunnerTest.java @@ -25,6 +25,7 @@ import io.cdap.cdap.app.DefaultAppConfigurer; import io.cdap.cdap.app.DefaultApplicationContext; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule; import io.cdap.cdap.app.guice.TwillModule; @@ -69,6 +70,7 @@ import io.cdap.cdap.security.guice.SecureStoreServerModule; import java.io.IOException; import java.util.Collections; +import java.util.EnumSet; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.twill.api.Configs; @@ -289,7 +291,8 @@ private static ProgramRunnerFactory createProgramRunnerFactory(CConfiguration cC new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new TwillModule(), - new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), + new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getDistributedModules(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SecureStoreServerModule(), new OperationalStatsModule(), diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java index b82a1e58be18..35cde21565a4 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/app/guice/AppFabricServiceRuntimeModule.java @@ -19,6 +19,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.AbstractModule; import com.google.inject.Inject; import com.google.inject.Injector; @@ -166,7 +167,10 @@ import io.cdap.http.HttpHandler; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.quartz.SchedulerException; import org.quartz.core.JobRunShellFactory; import org.quartz.core.QuartzScheduler; @@ -184,18 +188,36 @@ */ public final class AppFabricServiceRuntimeModule extends RuntimeModule { + public enum ServiceType { + SERVER, + PROCESSOR, + } + public static final String NOAUTH_ARTIFACT_REPO = "noAuthArtifactRepo"; + public static final Set ALL_SERVICE_TYPES = ImmutableSet.copyOf(ServiceType.values()); + private final CConfiguration cConf; + private final Set serviceTypes; + + /** + * Create AppFabricServiceRuntimeModule for the provided service types. + * + * @param cConf CConfiguration + * @param serviceTypes Service types for which modules should be generated. + */ + public AppFabricServiceRuntimeModule(CConfiguration cConf, ServiceType... serviceTypes) { + this(cConf, new HashSet<>(Arrays.asList(serviceTypes))); + } - @Inject - public AppFabricServiceRuntimeModule(CConfiguration cConf) { + public AppFabricServiceRuntimeModule(CConfiguration cConf, Set serviceTypes) { this.cConf = cConf; + this.serviceTypes = serviceTypes; } @Override public Module getInMemoryModules() { - return Modules.combine(new AppFabricServiceModule(cConf), + return Modules.combine(new AppFabricServiceModule(cConf, serviceTypes), new CapabilityModule(), new NamespaceAdminModule().getInMemoryModules(), new ConfigStoreModule(), @@ -216,38 +238,40 @@ protected void configure() { bind(StorageProviderNamespaceAdmin.class).to(LocalStorageProviderNamespaceAdmin.class); bind(UGIProvider.class).toProvider(UgiProviderProvider.class); - Multibinder servicesNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.services.names")); - servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - Multibinder processorNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.processor.services.names")); - processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); - - // TODO: Uncomment after CDAP-7688 is resolved - // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); - - Multibinder handlerHookNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.handler.hooks")); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - // TODO: Uncomment after CDAP-7688 is resolved - // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); - - // TODO (CDAP-21112): Move HTTP handler from Appfabric processor to server after fixing - // ProgramRuntimeService and RunRecordMonitorService. - // Remove additional handlers added to server for in-memory module. - Multibinder handlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); - handlerBinder.addBinding().to(BootstrapHttpHandler.class); - handlerBinder.addBinding().to(AppLifecycleHttpHandler.class); - handlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); - handlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); - handlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); - handlerBinder.addBinding().to(WorkflowHttpHandler.class); + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder servicesNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.services.names")); + servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + Multibinder handlerHookNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.handler.hooks")); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + // TODO (CDAP-21112): Remove the addtional handler binding for in-memory and use the binding from + // AppFabricServiceModule, after fixing in-memory cache issue in ProgramRuntimeService and + // RunRecordMonitorService. + Multibinder handlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); + handlerBinder.addBinding().to(BootstrapHttpHandler.class); + handlerBinder.addBinding().to(AppLifecycleHttpHandler.class); + handlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); + handlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); + handlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); + handlerBinder.addBinding().to(WorkflowHttpHandler.class); + + // TODO: Uncomment after CDAP-7688 is resolved + // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + } + + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.processor.services.names")); + processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + } } }); } @@ -255,7 +279,7 @@ protected void configure() { @Override public Module getStandaloneModules() { - return Modules.combine(new AppFabricServiceModule(cConf), + return Modules.combine(new AppFabricServiceModule(cConf, serviceTypes), new CapabilityModule(), new NamespaceAdminModule().getStandaloneModules(), new ConfigStoreModule(), @@ -277,38 +301,40 @@ protected void configure() { bind(StorageProviderNamespaceAdmin.class).to(LocalStorageProviderNamespaceAdmin.class); bind(UGIProvider.class).toProvider(UgiProviderProvider.class); - Multibinder servicesNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.services.names")); - servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - // for PingHandler - servicesNamesBinder.addBinding().toInstance(Constants.Service.METRICS_PROCESSOR); - servicesNamesBinder.addBinding().toInstance(Constants.Service.LOGSAVER); - servicesNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); - servicesNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); - - Multibinder processorNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.processor.services.names")); - processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); - - // TODO: Uncomment after CDAP-7688 is resolved - // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); - - Multibinder handlerHookNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.handler.hooks")); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - - // for PingHandler - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.METRICS_PROCESSOR); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.LOGSAVER); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); - - // TODO: Uncomment after CDAP-7688 is resolved - // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder servicesNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.services.names")); + servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + // for PingHandler + servicesNamesBinder.addBinding().toInstance(Constants.Service.METRICS_PROCESSOR); + servicesNamesBinder.addBinding().toInstance(Constants.Service.LOGSAVER); + servicesNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); + servicesNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); + + Multibinder handlerHookNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.handler.hooks")); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + + // for PingHandler + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.METRICS_PROCESSOR); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.LOGSAVER); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.TRANSACTION_HTTP); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.RUNTIME); + + // TODO: Uncomment after CDAP-7688 is resolved + // servicesNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + // handlerHookNamesBinder.addBinding().toInstance(Constants.Service.MESSAGING_SERVICE); + } + + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.processor.services.names")); + processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + } } }); } @@ -316,7 +342,7 @@ protected void configure() { @Override public Module getDistributedModules() { - return Modules.combine(new AppFabricServiceModule(cConf, ImpersonationHandler.class), + return Modules.combine(new AppFabricServiceModule(cConf, serviceTypes, ImpersonationHandler.class), new CapabilityModule(), new NamespaceAdminModule().getDistributedModules(), new ConfigStoreModule(), @@ -342,22 +368,26 @@ protected void configure() { bind(ProgramRunDispatcher.class).to(RemoteProgramRunDispatcher.class) .in(Scopes.SINGLETON); - Multibinder servicesNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.services.names")); - servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); - - Multibinder handlerHookNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.handler.hooks")); - handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); - servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); - - Multibinder processorNamesBinder = - Multibinder.newSetBinder(binder(), String.class, - Names.named("appfabric.processor.services.names")); - processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder servicesNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.services.names")); + servicesNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); + + Multibinder handlerHookNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.handler.hooks")); + handlerHookNamesBinder.addBinding().toInstance(Constants.Service.APP_FABRIC_HTTP); + servicesNamesBinder.addBinding().toInstance(Constants.Service.SECURE_STORE_SERVICE); + } + + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorNamesBinder = + Multibinder.newSetBinder(binder(), String.class, + Names.named("appfabric.processor.services.names")); + processorNamesBinder.addBinding().toInstance(Service.APP_FABRIC_PROCESSOR); + } } }); } @@ -369,11 +399,14 @@ private static final class AppFabricServiceModule extends AbstractModule { private final List> handlerClasses; private final CConfiguration cConf; + private final Set serviceTypes; private AppFabricServiceModule(CConfiguration cConf, + Set serviceTypes, Class... handlerClasses) { - this.handlerClasses = ImmutableList.copyOf(handlerClasses); this.cConf = cConf; + this.serviceTypes = serviceTypes; + this.handlerClasses = ImmutableList.copyOf(handlerClasses); } @Override @@ -462,61 +495,65 @@ protected void configure() { bind(EventWriterProvider.class).to(EventWriterExtensionProvider.class); bind(MetricsProvider.class).to(SparkProgramStatusMetricsProvider.class); - Multibinder handlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); - - CommonHandlers.add(handlerBinder); - handlerBinder.addBinding().to(ConfigHandler.class); - handlerBinder.addBinding().to(VersionHandler.class); - handlerBinder.addBinding().to(UsageHandler.class); - handlerBinder.addBinding().to(InstanceOperationHttpHandler.class); - handlerBinder.addBinding().to(NamespaceHttpHandler.class); - handlerBinder.addBinding().to(SourceControlManagementHttpHandler.class); - // TODO: [CDAP-13355] Move OperationsDashboardHttpHandler into report generation app - handlerBinder.addBinding().to(OperationsDashboardHttpHandler.class); - handlerBinder.addBinding().to(PreferencesHttpHandler.class); - handlerBinder.addBinding().to(PreferencesHttpHandlerInternal.class); - handlerBinder.addBinding().to(ConsoleSettingsHttpHandler.class); - handlerBinder.addBinding().to(TransactionHttpHandler.class); - handlerBinder.addBinding().to(ArtifactHttpHandler.class); - handlerBinder.addBinding().to(ArtifactHttpHandlerInternal.class); - handlerBinder.addBinding().to(WorkflowStatsSLAHttpHandler.class); - handlerBinder.addBinding().to(AuthorizationHandler.class); - handlerBinder.addBinding().to(SecureStoreHandler.class); - handlerBinder.addBinding().to(RemotePrivilegesHandler.class); - handlerBinder.addBinding().to(OperationalStatsHttpHandler.class); - handlerBinder.addBinding().to(ProfileHttpHandler.class); - handlerBinder.addBinding().to(ProvisionerHttpHandler.class); - handlerBinder.addBinding().to(FileFetcherHttpHandlerInternal.class); - handlerBinder.addBinding().to(TetheringHandler.class); - handlerBinder.addBinding().to(TetheringServerHandler.class); - handlerBinder.addBinding().to(TetheringClientHandler.class); - handlerBinder.addBinding().to(AppStateHandler.class); - handlerBinder.addBinding().to(CredentialProviderHttpHandler.class); - handlerBinder.addBinding().to(CredentialProviderHttpHandlerInternal.class); - handlerBinder.addBinding().to(OperationHttpHandler.class); - - FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); - if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { - handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandler.class); - handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandlerInternal.class); - } + if (serviceTypes.contains(ServiceType.SERVER)) { + Multibinder handlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(Constants.AppFabric.SERVER_HANDLERS_BINDING)); + + CommonHandlers.add(handlerBinder); + handlerBinder.addBinding().to(ConfigHandler.class); + handlerBinder.addBinding().to(VersionHandler.class); + handlerBinder.addBinding().to(UsageHandler.class); + handlerBinder.addBinding().to(InstanceOperationHttpHandler.class); + handlerBinder.addBinding().to(NamespaceHttpHandler.class); + handlerBinder.addBinding().to(SourceControlManagementHttpHandler.class); + // TODO: [CDAP-13355] Move OperationsDashboardHttpHandler into report generation app + handlerBinder.addBinding().to(OperationsDashboardHttpHandler.class); + handlerBinder.addBinding().to(PreferencesHttpHandler.class); + handlerBinder.addBinding().to(PreferencesHttpHandlerInternal.class); + handlerBinder.addBinding().to(ConsoleSettingsHttpHandler.class); + handlerBinder.addBinding().to(TransactionHttpHandler.class); + handlerBinder.addBinding().to(ArtifactHttpHandler.class); + handlerBinder.addBinding().to(ArtifactHttpHandlerInternal.class); + handlerBinder.addBinding().to(WorkflowStatsSLAHttpHandler.class); + handlerBinder.addBinding().to(AuthorizationHandler.class); + handlerBinder.addBinding().to(SecureStoreHandler.class); + handlerBinder.addBinding().to(RemotePrivilegesHandler.class); + handlerBinder.addBinding().to(OperationalStatsHttpHandler.class); + handlerBinder.addBinding().to(ProfileHttpHandler.class); + handlerBinder.addBinding().to(ProvisionerHttpHandler.class); + handlerBinder.addBinding().to(FileFetcherHttpHandlerInternal.class); + handlerBinder.addBinding().to(TetheringHandler.class); + handlerBinder.addBinding().to(TetheringServerHandler.class); + handlerBinder.addBinding().to(TetheringClientHandler.class); + handlerBinder.addBinding().to(AppStateHandler.class); + handlerBinder.addBinding().to(CredentialProviderHttpHandler.class); + handlerBinder.addBinding().to(CredentialProviderHttpHandlerInternal.class); + handlerBinder.addBinding().to(OperationHttpHandler.class); + + FeatureFlagsProvider featureFlagsProvider = new DefaultFeatureFlagsProvider(cConf); + if (Feature.NAMESPACED_SERVICE_ACCOUNTS.isEnabled(featureFlagsProvider)) { + handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandler.class); + handlerBinder.addBinding().to(GcpWorkloadIdentityHttpHandlerInternal.class); + } - for (Class handlerClass : handlerClasses) { - handlerBinder.addBinding().to(handlerClass); + for (Class handlerClass : handlerClasses) { + handlerBinder.addBinding().to(handlerClass); + } } - Multibinder processorHandlerBinder = Multibinder.newSetBinder( - binder(), HttpHandler.class, Names.named(AppFabric.PROCESSOR_HANDLERS_BINDING)); - CommonHandlers.add(processorHandlerBinder); - // TODO (CDAP-21112): Move HTTP handler from Appfabric processor to server after fixing - // ProgramRuntimeService and RunRecordMonitorService. - processorHandlerBinder.addBinding().to(BootstrapHttpHandler.class); - processorHandlerBinder.addBinding().to(AppLifecycleHttpHandler.class); - processorHandlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); - processorHandlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); - processorHandlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); - processorHandlerBinder.addBinding().to(WorkflowHttpHandler.class); + if (serviceTypes.contains(ServiceType.PROCESSOR)) { + Multibinder processorHandlerBinder = Multibinder.newSetBinder( + binder(), HttpHandler.class, Names.named(AppFabric.PROCESSOR_HANDLERS_BINDING)); + CommonHandlers.add(processorHandlerBinder); + // TODO (CDAP-21112): Move HTTP handler from Appfabric processor to server after fixing + // ProgramRuntimeService and RunRecordMonitorService. + processorHandlerBinder.addBinding().to(BootstrapHttpHandler.class); + processorHandlerBinder.addBinding().to(AppLifecycleHttpHandler.class); + processorHandlerBinder.addBinding().to(AppLifecycleHttpHandlerInternal.class); + processorHandlerBinder.addBinding().to(ProgramLifecycleHttpHandler.class); + processorHandlerBinder.addBinding().to(ProgramLifecycleHttpHandlerInternal.class); + processorHandlerBinder.addBinding().to(WorkflowHttpHandler.class); + } } @Provides diff --git a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java index b8d0db7543e8..b45767ded47c 100644 --- a/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java +++ b/cdap-app-fabric/src/main/java/io/cdap/cdap/internal/app/worker/system/SystemWorkerTwillRunnable.java @@ -33,6 +33,7 @@ import io.cdap.cdap.api.artifact.ArtifactManager; import io.cdap.cdap.api.metrics.MetricsCollectionService; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuditLogWriterModule; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.DistributedArtifactManagerModule; @@ -91,6 +92,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -156,7 +158,8 @@ protected void bindKeyManager(Binder binder) { new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new AuditLogWriterModule(cConf).getDistributedModules(), - Modules.override(new AppFabricServiceRuntimeModule(cConf).getDistributedModules()) + Modules.override(new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getDistributedModules()) .with(new AbstractModule() { // To enable localisation of artifacts @Override diff --git a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java index 6f9a2990e081..05231a57bb8d 100644 --- a/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java +++ b/cdap-app-fabric/src/test/java/io/cdap/cdap/internal/guice/AppFabricTestModule.java @@ -91,7 +91,8 @@ protected void configure() { install(RemoteAuthenticatorModules.getNoOpModule()); install(new IOModule()); install(new InMemoryDiscoveryModule()); - install(new AppFabricServiceRuntimeModule(cConf).getInMemoryModules()); + install(new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getInMemoryModules()); install(new MonitorHandlerModule(false, cConf)); install(new ProgramRunnerRuntimeModule().getInMemoryModules()); install(new NonCustomLocationUnitTestModule()); diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java index 2493b4dc7705..b86819141e80 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/runtime/main/MasterServiceMain.java @@ -546,7 +546,8 @@ protected void configure() { new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new TwillModule(), - new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), + new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getDistributedModules(), new MonitorHandlerModule(true, cConf), new ProgramRunnerRuntimeModule().getDistributedModules(), new SecureStoreServerModule(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java b/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java index e75affead96b..b4de03eacfc3 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java +++ b/cdap-master/src/main/java/io/cdap/cdap/data/tools/JobQueueDebugger.java @@ -33,6 +33,7 @@ import io.cdap.cdap.api.dataset.lib.CloseableIterator; import io.cdap.cdap.api.schedule.Trigger; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule; import io.cdap.cdap.app.guice.TwillModule; @@ -73,6 +74,7 @@ import io.cdap.cdap.spi.data.transaction.TransactionRunner; import io.cdap.cdap.spi.data.transaction.TransactionRunners; import java.io.IOException; +import java.util.EnumSet; import java.util.List; import javax.annotation.Nullable; import org.apache.commons.cli.BasicParser; @@ -346,7 +348,8 @@ private static Injector createInjector() throws Exception { new TwillModule(), new DataFabricModules().getDistributedModules(), new DataSetsModules().getDistributedModules(), - new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), + new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getDistributedModules(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SystemDatasetRuntimeModule().getDistributedModules(), new KafkaLogAppenderModule(), diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java index 0bc90092a8c6..0669e3d3d34d 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricProcessorServiceMain.java @@ -25,6 +25,7 @@ import com.google.inject.Scopes; import com.google.inject.util.Modules; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.MonitorHandlerModule; import io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule; @@ -65,6 +66,7 @@ import io.cdap.cdap.security.guice.SecureStoreServerModule; import io.cdap.cdap.security.store.SecureStoreService; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -98,7 +100,8 @@ protected List getServiceModules(MasterEnvironment masterEnv, new AuditModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), - Modules.override(new AppFabricServiceRuntimeModule(cConf).getDistributedModules()) + Modules.override(new AppFabricServiceRuntimeModule(cConf, ServiceType.PROCESSOR) + .getDistributedModules()) .with(new AbstractModule() { @Override protected void configure() { diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java index 1b722dee0261..9c141b35463d 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/AppFabricServiceMain.java @@ -25,6 +25,7 @@ import com.google.inject.Scopes; import com.google.inject.util.Modules; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.MonitorHandlerModule; import io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule; @@ -65,6 +66,7 @@ import io.cdap.cdap.security.guice.SecureStoreServerModule; import io.cdap.cdap.security.store.SecureStoreService; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -98,7 +100,7 @@ protected List getServiceModules(MasterEnvironment masterEnv, new AuditModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), - Modules.override(new AppFabricServiceRuntimeModule(cConf).getDistributedModules()) + Modules.override(new AppFabricServiceRuntimeModule(cConf, ServiceType.SERVER).getDistributedModules()) .with(new AbstractModule() { @Override protected void configure() { diff --git a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMain.java b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMain.java index 5f78793eafc1..2f319f2b1c66 100644 --- a/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMain.java +++ b/cdap-master/src/main/java/io/cdap/cdap/master/environment/k8s/PreviewServiceMain.java @@ -25,6 +25,7 @@ import com.google.inject.Module; import com.google.inject.Scopes; import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule; +import io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule.ServiceType; import io.cdap.cdap.app.guice.AuthorizationModule; import io.cdap.cdap.app.guice.ProgramRunnerRuntimeModule; import io.cdap.cdap.app.preview.PreviewHttpServer; @@ -51,6 +52,7 @@ import io.cdap.cdap.security.guice.SecureStoreClientModule; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -98,7 +100,7 @@ protected List getServiceModules(MasterEnvironment masterEnv, List modules = new ArrayList<>(Arrays.asList( new DataSetServiceModules().getStandaloneModules(), new DataSetsModules().getStandaloneModules(), - new AppFabricServiceRuntimeModule(cConf).getStandaloneModules(), + new AppFabricServiceRuntimeModule(cConf, ServiceType.SERVER).getStandaloneModules(), new ProgramRunnerRuntimeModule().getStandaloneModules(), new MetricsStoreModule(), new MessagingServiceModule(cConf), diff --git a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java index c59b9aac0121..3f52499587bb 100644 --- a/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java +++ b/cdap-standalone/src/main/java/io/cdap/cdap/StandaloneMain.java @@ -80,9 +80,9 @@ import io.cdap.cdap.logging.guice.LogQueryRuntimeModule; import io.cdap.cdap.logging.guice.LogReaderRuntimeModules; import io.cdap.cdap.logging.service.LogQueryService; -import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule; import io.cdap.cdap.messaging.server.MessagingHttpService; +import io.cdap.cdap.messaging.spi.MessagingService; import io.cdap.cdap.metadata.MetadataReaderWriterModules; import io.cdap.cdap.metadata.MetadataService; import io.cdap.cdap.metadata.MetadataServiceModule; @@ -564,7 +564,8 @@ private static List createPersistentModules(CConfiguration cConf, Config new PreviewManagerModule(false), new PreviewRunnerManagerModule().getStandaloneModules(), new MessagingServerRuntimeModule().getStandaloneModules(), - new AppFabricServiceRuntimeModule(cConf).getStandaloneModules(), + new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getStandaloneModules(), new MonitorHandlerModule(false, cConf), new RuntimeServerModule(), new OperationalStatsModule(), diff --git a/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestModule.java b/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestModule.java index 50b7996ca5ab..d060947f6534 100644 --- a/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestModule.java +++ b/cdap-support-bundle/src/test/java/io/cdap/cdap/SupportBundleTestModule.java @@ -85,7 +85,8 @@ protected void configure() { install(RemoteAuthenticatorModules.getNoOpModule()); install(new IOModule()); install(new InMemoryDiscoveryModule()); - install(new AppFabricServiceRuntimeModule(cConf).getInMemoryModules()); + install(new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getInMemoryModules()); install(new MonitorHandlerModule(false, cConf)); install(new ProgramRunnerRuntimeModule().getInMemoryModules()); install(new NonCustomLocationUnitTestModule()); diff --git a/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java b/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java index b7d0a6815a24..12dd6555a760 100644 --- a/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java +++ b/cdap-unit-test/src/main/java/io/cdap/cdap/test/TestBase.java @@ -301,7 +301,8 @@ public static void initialize() throws Exception { new IOModule(), new LocalLocationModule(), new InMemoryDiscoveryModule(), - new AppFabricServiceRuntimeModule(cConf).getInMemoryModules(), + new AppFabricServiceRuntimeModule(cConf, AppFabricServiceRuntimeModule.ALL_SERVICE_TYPES) + .getInMemoryModules(), new MonitorHandlerModule(false, cConf), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(),