diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index 075da20cdd11cc..9e9a0ed9884b46 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -990,7 +990,7 @@ jobs: DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }} DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }} ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }} - ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5" + ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions" ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml" run: | ./smoke-test/run-quickstart.sh diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile index 92b17620998823..08cf2efdcb6a19 100644 --- a/docker/datahub-ingestion-base/Dockerfile +++ b/docker/datahub-ingestion-base/Dockerfile @@ -43,7 +43,9 @@ RUN apt-get update && apt-get upgrade -y \ krb5-user \ krb5-config \ libkrb5-dev \ + librdkafka-dev \ wget \ + curl \ zip \ unzip \ ldap-utils \ diff --git a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java index 269b9a41a89a9b..1e16f7d42c6e81 100644 --- a/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java +++ b/metadata-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java @@ -2,6 +2,7 @@ import com.datahub.auth.authentication.filter.AuthenticationFilter; import com.linkedin.gms.factory.auth.SystemAuthenticationFactory; +import com.linkedin.r2.transport.http.server.RAPJakartaServlet; import com.linkedin.restli.server.RestliHandlerServlet; import java.util.Collections; import org.springframework.beans.factory.annotation.Qualifier; @@ -36,8 +37,8 @@ public ServletRegistrationBean restliServletRegistration( } @Bean("restliHandlerServlet") - public RestliHandlerServlet restliHandlerServlet() { - return new RestliHandlerServlet(); + public RestliHandlerServlet restliHandlerServlet(final RAPJakartaServlet r2Servlet) { + return new RestliHandlerServlet(r2Servlet); } @Bean diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java index ee2efd2ae95365..0a54677eb6149b 100644 --- a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java +++ b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java @@ -77,6 +77,7 @@ public class AuthenticationFilter implements Filter { public void init(FilterConfig filterConfig) throws ServletException { SpringBeanAutowiringSupport.processInjectionBasedOnCurrentContext(this); buildAuthenticatorChain(); + log.info("AuthenticationFilter initialized."); } @Override diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java index ced315ba6e65fd..921246fa98f7a2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/OnBootApplicationListener.java @@ -73,6 +73,17 @@ public void onApplicationEvent(@Nonnull ContextRefreshedEvent event) { event); String schemaRegistryType = provider.getKafka().getSchemaRegistry().getType(); if (ROOT_WEB_APPLICATION_CONTEXT_ID.equals(event.getApplicationContext().getId())) { + + // Handle race condition, if ebean code is executed while waiting/bootstrapping (i.e. + // AuthenticationFilter) + try { + Class.forName("io.ebean.XServiceProvider"); + } catch (ClassNotFoundException e) { + log.error( + "Failure to initialize required class `io.ebean.XServiceProvider` during initialization."); + throw new RuntimeException(e); + } + if (InternalSchemaRegistryFactory.TYPE.equals(schemaRegistryType)) { executorService.submit(isSchemaRegistryAPIServletReady()); } else { diff --git a/metadata-service/factories/src/main/java/com/linkedin/restli/server/RAPServletFactory.java b/metadata-service/factories/src/main/java/com/linkedin/restli/server/RAPServletFactory.java index e07a25914a039e..be060477aeb1f6 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/restli/server/RAPServletFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/restli/server/RAPServletFactory.java @@ -16,6 +16,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -29,8 +30,10 @@ public class RAPServletFactory { private int maxSerializedStringLength; @Bean(name = "restliSpringInjectResourceFactory") - public SpringInjectResourceFactory springInjectResourceFactory() { - return new SpringInjectResourceFactory(); + public SpringInjectResourceFactory springInjectResourceFactory(final ApplicationContext ctx) { + SpringInjectResourceFactory springInjectResourceFactory = new SpringInjectResourceFactory(); + springInjectResourceFactory.setApplicationContext(ctx); + return springInjectResourceFactory; } @Bean("parseqEngineThreads") diff --git a/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java b/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java index 7702e8bd6de891..bfc25b7ddaef50 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java +++ b/metadata-service/factories/src/main/java/com/linkedin/restli/server/RestliHandlerServlet.java @@ -1,23 +1,29 @@ package com.linkedin.restli.server; import com.linkedin.r2.transport.http.server.RAPJakartaServlet; +import jakarta.servlet.ServletConfig; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import lombok.AllArgsConstructor; -import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.web.HttpRequestHandler; import org.springframework.web.context.support.HttpRequestHandlerServlet; +@Slf4j @AllArgsConstructor -@NoArgsConstructor -@Component public class RestliHandlerServlet extends HttpRequestHandlerServlet implements HttpRequestHandler { @Autowired private RAPJakartaServlet _r2Servlet; + @Override + public void init(ServletConfig config) throws ServletException { + log.info("Initializing RestliHandlerServlet"); + this._r2Servlet.init(config); + log.info("Initialized RestliHandlerServlet"); + } + @Override public void service(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java b/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java index fd32610be35321..4ed84e48a5049f 100644 --- a/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java +++ b/metadata-service/war/src/main/java/com/linkedin/gms/WebApplicationInitializer.java @@ -42,18 +42,13 @@ public void onStartup(ServletContext container) { // Independent dispatcher schemaRegistryServlet(container); - // Non-Spring servlets - healthCheckServlet(container); - configServlet(container); - - // Restli non-Dispatcher - servletNames.add(restliServlet(rootContext, container)); - // Spring Dispatcher servlets DispatcherServlet dispatcherServlet = new DispatcherServlet(rootContext); servletNames.add(authServlet(rootContext, dispatcherServlet, container)); servletNames.add(graphQLServlet(rootContext, dispatcherServlet, container)); servletNames.add(openAPIServlet(rootContext, dispatcherServlet, container)); + // Restli non-Dispatcher default + servletNames.add(restliServlet(rootContext, container)); FilterRegistration.Dynamic filterRegistration = container.addFilter("authenticationFilter", AuthenticationFilter.class); @@ -62,6 +57,10 @@ public void onStartup(ServletContext container) { EnumSet.of(DispatcherType.ASYNC, DispatcherType.REQUEST), false, servletNames.toArray(String[]::new)); + + // Non-Spring servlets + healthCheckServlet(container); + configServlet(container); } /* @@ -133,10 +132,13 @@ private String restliServlet( rootContext.register(RestliServletConfig.class); ServletRegistration.Dynamic registration = - container.addServlet(servletName, new HttpRequestHandlerServlet()); + container.addServlet(servletName, HttpRequestHandlerServlet.class); registration.addMapping("/*"); registration.setLoadOnStartup(10); registration.setAsyncSupported(true); + registration.setInitParameter( + "org.springframework.web.servlet.FrameworkServlet.ORDER", + String.valueOf(Integer.MAX_VALUE - 1)); return servletName; } diff --git a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java index 28e2aabe139da2..222e2356dfb1ca 100644 --- a/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java +++ b/metadata-service/war/src/main/java/com/linkedin/gms/servlet/RestliServletConfig.java @@ -7,14 +7,14 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; -import org.springframework.web.HttpRequestHandler; +import org.springframework.web.context.support.HttpRequestHandlerServlet; @ComponentScan(basePackages = {"com.linkedin.restli.server"}) @PropertySource(value = "classpath:/application.yaml", factory = YamlPropertySourceFactory.class) @Configuration public class RestliServletConfig { @Bean("restliRequestHandler") - public HttpRequestHandler restliHandlerServlet(final RAPJakartaServlet r2Servlet) { + public HttpRequestHandlerServlet restliHandlerServlet(final RAPJakartaServlet r2Servlet) { return new RestliHandlerServlet(r2Servlet); } }