Merge branch 'master' into unity-catalog-connector-fix

build.gradle

@@ -13,7 +13,7 @@ buildscript {
   dependencies {
     classpath 'com.linkedin.pegasus:gradle-plugins:' + pegasusVersion
     classpath 'com.github.node-gradle:gradle-node-plugin:2.2.4'
-    classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.8.1'
+    classpath 'io.acryl.gradle.plugin:gradle-avro-plugin:0.8.1'
     classpath 'org.springframework.boot:spring-boot-gradle-plugin:' + springBootVersion
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"

metadata-events/mxe-avro-1.7/build.gradle

@@ -2,7 +2,7 @@ configurations {
   avsc
 }
 
-apply plugin: 'com.commercehub.gradle.plugin.avro'
+apply plugin: 'io.acryl.gradle.plugin.avro'
 apply plugin: 'java'
 
 dependencies {

metadata-ingestion/setup.cfg

@@ -26,6 +26,7 @@ plugins =
     pydantic.mypy
 exclude = ^(venv|build|dist)/
 ignore_missing_imports = yes
+namespace_packages = no
 strict_optional = yes
 check_untyped_defs = yes
 disallow_incomplete_defs = yes

metadata-ingestion/setup.py

@@ -373,7 +373,10 @@ def get_long_description():
     "flake8>=3.8.3",
     "flake8-tidy-imports>=4.3.0",
     "isort>=5.7.0",
-    "mypy>=0.981",
+    # mypy 0.990 enables namespace packages by default and sets
+    # no implicit optional to True.
+    # FIXME: Enable mypy 0.990 when our codebase is fixed.
+    "mypy>=0.981,<0.990",
     # pydantic 1.8.2 is incompatible with mypy 0.910.
     # See https://github.com/samuelcolvin/pydantic/pull/3175#issuecomment-995382910.
     # Restricting top version to <1.10 until we can fix our types.

metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py

@@ -30,6 +30,7 @@
 
 from datahub.configuration.common import AllowDenyPattern
 from datahub.emitter.mce_builder import (
+    make_container_urn,
     make_data_platform_urn,
     make_dataplatform_instance_urn,
     make_dataset_urn_with_platform_instance,
@@ -589,6 +590,12 @@ def gen_database_containers(self, database: str) -> Iterable[MetadataWorkUnit]:
             domain_urn=domain_urn,
         )
 
+        # Add container to the checkpoint state
+        container_urn = make_container_urn(database_container_key.guid())
+        self.stale_entity_removal_handler.add_entity_to_state(
+            "container", container_urn
+        )
+
         for wu in container_workunits:
             self.report.report_workunit(wu)
             yield wu
@@ -610,6 +617,12 @@ def gen_schema_containers(
             database_container_key,
         )
 
+        # Add container to the checkpoint state
+        container_urn = make_container_urn(schema_container_key.guid())
+        self.stale_entity_removal_handler.add_entity_to_state(
+            "container", container_urn
+        )
+
         for wu in container_workunits:
             self.report.report_workunit(wu)
             yield wu