merge master changes

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -724,7 +724,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
    * Returns a {@link Supplier} responsible for creating a new {@link DataLoader} from a {@link
    * LoadableType}.
    */
-  public Map<String, Function<QueryContext, DataLoader<?, ?>>> loaderSuppliers(
+  public static Map<String, Function<QueryContext, DataLoader<?, ?>>> loaderSuppliers(
       final Collection<? extends LoadableType<?, ?>> loadableTypes) {
     return loadableTypes.stream()
         .collect(
@@ -1135,16 +1135,16 @@ private DataFetcher getEntityResolver() {
         });
   }
 
-  private DataFetcher getResolver(LoadableType<?, String> loadableType) {
-    return getResolver(loadableType, this::getUrnField);
+  private static DataFetcher getResolver(LoadableType<?, String> loadableType) {
+    return getResolver(loadableType, GmsGraphQLEngine::getUrnField);
   }
 
-  private <T, K> DataFetcher getResolver(
+  private static <T, K> DataFetcher getResolver(
       LoadableType<T, K> loadableType, Function<DataFetchingEnvironment, K> keyProvider) {
     return new LoadableTypeResolver<>(loadableType, keyProvider);
   }
 
-  private String getUrnField(DataFetchingEnvironment env) {
+  private static String getUrnField(DataFetchingEnvironment env) {
     return env.getArgument(URN_FIELD_NAME);
   }
 
@@ -3025,7 +3025,7 @@ private void configureTestResultResolvers(final RuntimeWiring.Builder builder) {
                     })));
   }
 
-  private <T, K> DataLoader<K, DataFetcherResult<T>> createDataLoader(
+  private static <T, K> DataLoader<K, DataFetcherResult<T>> createDataLoader(
       final LoadableType<T, K> graphType, final QueryContext queryContext) {
     BatchLoaderContextProvider contextProvider = () -> queryContext;
     DataLoaderOptions loaderOptions =

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GraphQLEngine.java

@@ -28,7 +28,6 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.dataloader.DataLoader;
-import org.dataloader.DataLoaderRegistry;
 
 /**
  * Simple wrapper around a {@link GraphQL} instance providing APIs for building an engine and
@@ -100,7 +99,7 @@ public ExecutionResult execute(
     /*
      * Init DataLoaderRegistry - should be created for each request.
      */
-    DataLoaderRegistry register = createDataLoaderRegistry(_dataLoaderSuppliers, context);
+    LazyDataLoaderRegistry register = new LazyDataLoaderRegistry(context, _dataLoaderSuppliers);
 
     /*
      * Construct execution input
@@ -218,14 +217,4 @@ public GraphQLEngine build() {
           graphQLQueryIntrospectionEnabled);
     }
   }
-
-  private DataLoaderRegistry createDataLoaderRegistry(
-      final Map<String, Function<QueryContext, DataLoader<?, ?>>> dataLoaderSuppliers,
-      final QueryContext context) {
-    final DataLoaderRegistry registry = new DataLoaderRegistry();
-    for (String key : dataLoaderSuppliers.keySet()) {
-      registry.register(key, dataLoaderSuppliers.get(key).apply(context));
-    }
-    return registry;
-  }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/LazyDataLoaderRegistry.java

@@ -0,0 +1,53 @@
+package com.linkedin.datahub.graphql;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import lombok.extern.slf4j.Slf4j;
+import org.dataloader.DataLoader;
+import org.dataloader.DataLoaderRegistry;
+
+/**
+ * The purpose of this class is to avoid loading 42+ dataLoaders when many of the graphql queries do
+ * not use all of them.
+ */
+@Slf4j
+public class LazyDataLoaderRegistry extends DataLoaderRegistry {
+  private final QueryContext queryContext;
+  private final Map<String, Function<QueryContext, DataLoader<?, ?>>> dataLoaderSuppliers;
+
+  public LazyDataLoaderRegistry(
+      QueryContext queryContext,
+      Map<String, Function<QueryContext, DataLoader<?, ?>>> dataLoaderSuppliers) {
+    super();
+    this.queryContext = queryContext;
+    this.dataLoaderSuppliers = new ConcurrentHashMap<>(dataLoaderSuppliers);
+  }
+
+  @Override
+  public <K, V> DataLoader<K, V> getDataLoader(String key) {
+    return super.computeIfAbsent(
+        key,
+        k -> {
+          Function<QueryContext, DataLoader<?, ?>> supplier = dataLoaderSuppliers.get(key);
+          if (supplier == null) {
+            throw new IllegalArgumentException("No DataLoader registered for key: " + key);
+          }
+          return supplier.apply(queryContext);
+        });
+  }
+
+  @Override
+  public Set<String> getKeys() {
+    return Stream.concat(dataLoaders.keySet().stream(), dataLoaderSuppliers.keySet().stream())
+        .collect(Collectors.toSet());
+  }
+
+  @Override
+  public DataLoaderRegistry combine(DataLoaderRegistry registry) {
+    throw new UnsupportedOperationException();
+  }
+}

docker/datahub-ingestion-base/Dockerfile

@@ -49,7 +49,7 @@ RUN apt-get update && apt-get upgrade -y \
     ldap-utils \
     unixodbc \
     libodbc2 \
-    && python -m pip install --no-cache --upgrade pip uv>=0.1.10 wheel setuptools \
+    && python -m pip install --no-cache --upgrade pip 'uv>=0.1.10' wheel setuptools \
     && apt-get clean \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
 

docs/how/updating-datahub.md

@@ -19,6 +19,17 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 ## Next
 
 ### Breaking Changes
+- #9857 (#10773) `lower` method was removed from `get_db_name` of `SQLAlchemySource` class. This change will affect the urns of all related to `SQLAlchemySource` entities.
+
+  Old `urn`, where `data_base_name` is `Some_Database`:
+  ```
+  - urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar)
+  ```
+  New `urn`, where `data_base_name` is `Some_Database`:
+  ```
+  - urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)
+  ```
+  Re-running with stateful ingestion should automatically clear up the entities with old URNS and add entities with new URNs, therefore not duplicating the containers or jobs. 
 
 ### Potential Downtime
 

metadata-ingestion/scripts/install_deps.sh

@@ -19,7 +19,13 @@ else
             xz-devel \
             libxml2-devel \
             libxslt-devel \
-	    krb5-devel
+            krb5-devel
+    elif command -v apk; then
+        $sudo_cmd apk add \
+            build-base \
+            openldap-dev \
+            xz-dev \
+            krb5-dev
     else
         $sudo_cmd apt-get update && $sudo_cmd apt-get install -y \
             python3-ldap \

metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py

@@ -434,7 +434,7 @@ def get_db_name(self, inspector: Inspector) -> str:
         if engine and hasattr(engine, "url") and hasattr(engine.url, "database"):
             if engine.url.database is None:
                 return ""
-            return str(engine.url.database).strip('"').lower()
+            return str(engine.url.database).strip('"')
         else:
             raise Exception("Unable to get database name from Sqlalchemy inspector")